From 74fa4fc7b849a28918eea4ceabd600379a20d835 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 29 Jul 2026 22:00:52 -0500 Subject: [PATCH 01/95] fix(store): fail closed on encryption failure in save_email (#236) encrypt_text_at_rest previously swallowed any AEAD failure with unwrap_or_default(), silently persisting an empty ciphertext string that is indistinguishable from a genuinely empty body on read-back. Change its signature to return Result and propagate failures through save_email as DatabaseError::Encryption, resolved before the transaction begins so no row is ever written for a message whose body failed to encrypt. Refs #198, #162 Co-authored-by: James Maes --- crates/nuncio-store/src/cipher.rs | 19 ++++++++++++------- crates/nuncio-store/src/db.rs | 21 +++++++++++++++++++-- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/crates/nuncio-store/src/cipher.rs b/crates/nuncio-store/src/cipher.rs index 71c34f3..fe5a508 100644 --- a/crates/nuncio-store/src/cipher.rs +++ b/crates/nuncio-store/src/cipher.rs @@ -64,12 +64,12 @@ impl PayloadCipher { /// AES-256-GCM key. The key MUST be sourced from a [`crate::vault::SecretManager`]-backed /// vault (never a compiled-in default) so that at-rest ciphertext cannot be decrypted /// without access to the OS keyring. - pub fn encrypt_text_at_rest(key: &[u8; 32], text: &str) -> String { + pub fn encrypt_text_at_rest(key: &[u8; 32], text: &str) -> Result { if text.is_empty() { - return String::new(); + return Ok(String::new()); } - let encrypted = Self::encrypt_bytes(key, text.as_bytes()).unwrap_or_default(); - hex::encode(encrypted) + let encrypted = Self::encrypt_bytes(key, text.as_bytes())?; + Ok(hex::encode(encrypted)) } /// Decrypt text payload from database column storage at rest using a caller-supplied @@ -236,13 +236,17 @@ mod tests { fn text_at_rest_encryption_decryption_roundtrip() { let key = [7u8; 32]; let text = "Sensitive Email Body Payload"; - let encrypted = PayloadCipher::encrypt_text_at_rest(&key, text); + let encrypted = + PayloadCipher::encrypt_text_at_rest(&key, text).expect("encryption succeeds"); assert_ne!(text, encrypted); let decrypted = PayloadCipher::decrypt_text_at_rest(&key, &encrypted); assert_eq!(text, decrypted); - assert_eq!(PayloadCipher::encrypt_text_at_rest(&key, ""), ""); + assert_eq!( + PayloadCipher::encrypt_text_at_rest(&key, "").expect("empty text is not encrypted"), + "" + ); assert_eq!(PayloadCipher::decrypt_text_at_rest(&key, ""), ""); } @@ -254,7 +258,8 @@ mod tests { let key_b = [2u8; 32]; let text = "Confidential ledger payload"; - let encrypted = PayloadCipher::encrypt_text_at_rest(&key_a, text); + let encrypted = + PayloadCipher::encrypt_text_at_rest(&key_a, text).expect("encryption succeeds"); let decrypted_with_wrong_key = PayloadCipher::decrypt_text_at_rest(&key_b, &encrypted); assert_eq!(decrypted_with_wrong_key, ""); diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 5f43173..7cc09b6 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -36,6 +36,18 @@ pub enum DatabaseError { /// compiled-in default key. #[error("cryptographic key provisioning failed: {0}")] KeyProvisioning(String), + /// A body payload failed AEAD encryption before it could be persisted at rest. + /// This is a fail-closed error: it is returned instead of ever writing an + /// empty/placeholder ciphertext that would be indistinguishable from a + /// genuinely empty body on read-back. + #[error("payload encryption failed: {0}")] + Encryption(String), +} + +impl From for DatabaseError { + fn from(err: crate::cipher::CipherError) -> Self { + DatabaseError::Encryption(err.to_string()) + } } impl DatabaseError { @@ -984,14 +996,19 @@ impl DatabaseEngine { /// is not encrypted, so this intentionally trades some body confidentiality for working /// search. pub async fn save_email(&self, email: &nuncio_core::model::Email) -> Result<(), DatabaseError> { + // Encryption failures MUST surface before any SQL runs: a swallowed error here would + // otherwise leave a row with an empty/placeholder body column, indistinguishable from + // a genuinely empty body on read-back. let enc_plain = email .body_plain .as_ref() - .map(|p| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, p)); + .map(|p| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, p)) + .transpose()?; let enc_html = email .body_html .as_ref() - .map(|h| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, h)); + .map(|h| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, h)) + .transpose()?; let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; -- 2.54.0 From 03c02ae9dfa9b53978ec4e3eb147aa75caf76885 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 29 Jul 2026 22:10:01 -0500 Subject: [PATCH 02/95] feat(security): zeroize long-lived engine key material on drop (#238) Wrap DatabaseEngine's storage/WORM/ledger keys, WebhookDispatcher's HMAC secret, and add_account's transient password in zeroize::Zeroizing so their backing memory is overwritten once the holding struct drops, instead of lingering as plain heap bytes for the rest of the process lifetime. Refs #199, #163 Co-authored-by: James Maes --- Cargo.lock | 2 + crates/nuncio-filter/Cargo.toml | 1 + crates/nuncio-filter/src/webhook.rs | 5 ++- crates/nuncio-store/src/db.rs | 61 ++++++++++++++++++++++++++--- crates/nunciod/Cargo.toml | 1 + crates/nunciod/src/grpc.rs | 8 +++- 6 files changed, 69 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac6e67b..a9c3920 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2561,6 +2561,7 @@ dependencies = [ "tokio", "tracing", "wiremock", + "zeroize", ] [[package]] @@ -2653,6 +2654,7 @@ dependencies = [ "tracing", "tracing-subscriber", "wiremock", + "zeroize", ] [[package]] diff --git a/crates/nuncio-filter/Cargo.toml b/crates/nuncio-filter/Cargo.toml index e65bf9e..31c98f8 100644 --- a/crates/nuncio-filter/Cargo.toml +++ b/crates/nuncio-filter/Cargo.toml @@ -25,6 +25,7 @@ async-trait.workspace = true chrono.workspace = true reqwest.workspace = true hex.workspace = true +zeroize.workspace = true [dev-dependencies] tempfile.workspace = true diff --git a/crates/nuncio-filter/src/webhook.rs b/crates/nuncio-filter/src/webhook.rs index ec25944..d25dd41 100644 --- a/crates/nuncio-filter/src/webhook.rs +++ b/crates/nuncio-filter/src/webhook.rs @@ -8,6 +8,7 @@ use serde_json::json; use sha2::Sha256; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use thiserror::Error; +use zeroize::Zeroizing; type HmacSha256 = Hmac; @@ -28,7 +29,7 @@ pub enum WebhookError { /// Outbound Webhook Dispatcher executing authenticated HTTP POST requests. pub struct WebhookDispatcher { client: Client, - secret_key: String, + secret_key: Zeroizing, } impl WebhookDispatcher { @@ -42,7 +43,7 @@ impl WebhookDispatcher { Self { client, - secret_key: secret_key.into(), + secret_key: Zeroizing::new(secret_key.into()), } } diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 7cc09b6..4095690 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -6,6 +6,7 @@ use std::path::Path; use std::str::FromStr; use tempfile::TempDir; use thiserror::Error; +use zeroize::Zeroizing; /// Raw row shape for a pending remote mutation record fetched from SQLite. type PendingMutationRow = (String, String, String, String, String, String, i64, i64); @@ -102,9 +103,9 @@ impl DatabaseError { #[derive(Clone)] pub struct DatabaseEngine { pool: SqlitePool, - storage_key: [u8; 32], - worm_key: Vec, - ledger_key: Vec, + storage_key: Zeroizing<[u8; 32]>, + worm_key: Zeroizing>, + ledger_key: Zeroizing>, } impl std::fmt::Debug for DatabaseEngine { @@ -122,7 +123,7 @@ impl std::fmt::Debug for DatabaseEngine { #[allow(clippy::type_complexity)] fn resolve_engine_keys( secrets: &crate::vault::SecretManager, -) -> Result<([u8; 32], Vec, Vec), DatabaseError> { +) -> Result<(Zeroizing<[u8; 32]>, Zeroizing>, Zeroizing>), DatabaseError> { let storage_key_bytes = secrets .get_or_create_key_bytes(crate::vault::STORAGE_KEY_ACCOUNT, 32) .map_err(|e| DatabaseError::KeyProvisioning(e.to_string()))?; @@ -135,7 +136,11 @@ fn resolve_engine_keys( let ledger_key = secrets .get_or_create_key_bytes(crate::vault::LEDGER_KEY_ACCOUNT, 32) .map_err(|e| DatabaseError::KeyProvisioning(e.to_string()))?; - Ok((storage_key, worm_key, ledger_key)) + Ok(( + Zeroizing::new(storage_key), + Zeroizing::new(worm_key), + Zeroizing::new(ledger_key), + )) } /// Reconstruct a [`nuncio_contacts::Contact`] from a `contacts` table row. @@ -3268,4 +3273,50 @@ mod tests { assert_eq!(report.record_count, 1); assert_eq!(report.first_broken_seq, Some(1)); } + + /// Proves the zeroize mechanism `Zeroizing`'s `Drop` relies on is real, not + /// decorative: explicitly invoking `Zeroize::zeroize` on key-shaped byte + /// buffers overwrites every byte with zero. This is checked directly + /// (without dropping the value) because inspecting freed heap memory would + /// require `unsafe`, which this workspace forbids. + #[test] + fn zeroize_mechanism_actually_wipes_key_bytes() { + use zeroize::Zeroize; + + let mut fixed = Zeroizing::new([0xABu8; 32]); + assert!(fixed.iter().all(|&b| b != 0), "fixture must start non-zero"); + fixed.zeroize(); + assert!( + fixed.iter().all(|&b| b == 0), + "Zeroize::zeroize must overwrite every byte of a fixed-size key buffer" + ); + + let mut variable = Zeroizing::new(vec![0xCDu8; 32]); + assert!( + variable.iter().all(|&b| b != 0), + "fixture must start non-zero" + ); + variable.zeroize(); + assert!( + variable.iter().all(|&b| b == 0), + "Zeroize::zeroize must overwrite every byte of a heap-allocated key buffer" + ); + } + + /// `DatabaseEngine`'s manual `Debug` impl must never print the raw + /// storage/WORM/ledger key bytes it now holds in `Zeroizing` wrappers, + /// even after they were introduced. + #[tokio::test] + async fn debug_impl_never_prints_raw_key_bytes() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + let debug_output = format!("{engine:?}"); + assert!( + !debug_output.contains("storage_key") + && !debug_output.contains("worm_key") + && !debug_output.contains("ledger_key"), + "Debug output must never expose key field names/values: {debug_output}" + ); + } } diff --git a/crates/nunciod/Cargo.toml b/crates/nunciod/Cargo.toml index 663ac49..a43ef22 100644 --- a/crates/nunciod/Cargo.toml +++ b/crates/nunciod/Cargo.toml @@ -33,6 +33,7 @@ async-trait.workspace = true subtle.workspace = true hex.workspace = true bytes.workspace = true +zeroize.workspace = true [dev-dependencies] nuncio-cli = { path = "../nuncio-cli" } diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index f5cb81a..f8379cc 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -511,10 +511,14 @@ impl Accounts for AccountsGrpcService { request: Request, ) -> Result, Status> { let req = request.into_inner(); + // Moved into a zeroizing wrapper immediately so the plaintext credential's + // backing memory is overwritten as soon as this function returns, rather + // than lingering as an ordinary `String` for the rest of the process. + let password = zeroize::Zeroizing::new(req.password); let proto_config = req .config .ok_or_else(|| Status::invalid_argument("config is required"))?; - if req.password.is_empty() { + if password.is_empty() { return Err(Status::invalid_argument("password is required")); } @@ -538,7 +542,7 @@ impl Accounts for AccountsGrpcService { // The original persistence error is always what the caller sees -- // a rollback failure is reported alongside it, never in place of it. self.secrets - .set_secret(&config.keyring_secret_key, &req.password) + .set_secret(&config.keyring_secret_key, &password) .map_err(|e| Status::internal(format!("failed to store credential in vault: {e}")))?; if let Err(e) = self.db.save_account(&config).await { -- 2.54.0 From 83d8b99c86767cd3aaeabee3aa29828f71c76400 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 29 Jul 2026 22:19:14 -0500 Subject: [PATCH 03/95] fix(store): validate WORM/ledger key length from the vault (#239) Mirror the existing storage-key length check for the WORM audit and filter-ledger HMAC keys: resolve_engine_keys now converts each key's Vec into a Zeroizing<[u8; 32]>, failing closed with a distinct DatabaseError::KeyProvisioning message when a vault entry decodes to the wrong length. Previously only the storage key was validated, so a corrupted or hand-edited vault entry of any length would be silently accepted as an HMAC signing key. Refs #200 Co-authored-by: James Maes --- crates/nuncio-store/src/db.rs | 75 ++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 4095690..28f66e0 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -104,8 +104,8 @@ impl DatabaseError { pub struct DatabaseEngine { pool: SqlitePool, storage_key: Zeroizing<[u8; 32]>, - worm_key: Zeroizing>, - ledger_key: Zeroizing>, + worm_key: Zeroizing<[u8; 32]>, + ledger_key: Zeroizing<[u8; 32]>, } impl std::fmt::Debug for DatabaseEngine { @@ -123,19 +123,34 @@ impl std::fmt::Debug for DatabaseEngine { #[allow(clippy::type_complexity)] fn resolve_engine_keys( secrets: &crate::vault::SecretManager, -) -> Result<(Zeroizing<[u8; 32]>, Zeroizing>, Zeroizing>), DatabaseError> { +) -> Result< + ( + Zeroizing<[u8; 32]>, + Zeroizing<[u8; 32]>, + Zeroizing<[u8; 32]>, + ), + DatabaseError, +> { let storage_key_bytes = secrets .get_or_create_key_bytes(crate::vault::STORAGE_KEY_ACCOUNT, 32) .map_err(|e| DatabaseError::KeyProvisioning(e.to_string()))?; let storage_key: [u8; 32] = storage_key_bytes.try_into().map_err(|_| { DatabaseError::KeyProvisioning("storage key material must be exactly 32 bytes".to_string()) })?; - let worm_key = secrets + let worm_key_bytes = secrets .get_or_create_key_bytes(crate::vault::WORM_KEY_ACCOUNT, 32) .map_err(|e| DatabaseError::KeyProvisioning(e.to_string()))?; - let ledger_key = secrets + let worm_key: [u8; 32] = worm_key_bytes.try_into().map_err(|_| { + DatabaseError::KeyProvisioning( + "WORM audit key material must be exactly 32 bytes".to_string(), + ) + })?; + let ledger_key_bytes = secrets .get_or_create_key_bytes(crate::vault::LEDGER_KEY_ACCOUNT, 32) .map_err(|e| DatabaseError::KeyProvisioning(e.to_string()))?; + let ledger_key: [u8; 32] = ledger_key_bytes.try_into().map_err(|_| { + DatabaseError::KeyProvisioning("ledger key material must be exactly 32 bytes".to_string()) + })?; Ok(( Zeroizing::new(storage_key), Zeroizing::new(worm_key), @@ -1687,7 +1702,7 @@ impl DatabaseEngine { message_id, action_taken, matched_at, - &self.ledger_key, + &self.ledger_key[..], ); let id = sqlx::query( @@ -1779,7 +1794,7 @@ impl DatabaseEngine { &message_id, &action_taken, matched_at, - &self.ledger_key, + &self.ledger_key[..], ); if computed != hash { return Ok(false); @@ -1950,7 +1965,7 @@ impl DatabaseEngine { }; let record = nuncio_core::WormAuditRecord::create_signed( - &self.worm_key, + &self.worm_key[..], next_seq, now_ns, actor, @@ -2010,7 +2025,7 @@ impl DatabaseEngine { /// provisioned for this engine. pub async fn verify_worm_audit_chain(&self) -> Result<(), DatabaseError> { let records = self.list_worm_audit_records(100_000, 0).await?; - nuncio_core::verify_worm_chain(&records, &self.worm_key) + nuncio_core::verify_worm_chain(&records, &self.worm_key[..]) .map_err(|e| DatabaseError::ChainIntegrityFailed(e.to_string())) } @@ -2028,7 +2043,7 @@ impl DatabaseEngine { pub async fn verify_worm_audit_chain_report(&self) -> Result { let records = self.list_worm_audit_records(100_000, 0).await?; let record_count = records.len(); - match nuncio_core::verify_worm_chain(&records, &self.worm_key) { + match nuncio_core::verify_worm_chain(&records, &self.worm_key[..]) { Ok(()) => Ok(WormChainReport { valid: true, record_count, @@ -2284,6 +2299,46 @@ mod tests { assert!(engine.check_integrity().await.expect("integrity check")); } + /// A hand-edited or corrupted vault entry for the WORM audit key must never be silently + /// accepted at the wrong length: a 16-byte HMAC key would still "work" mechanically but + /// would weaken the tamper-evidence the WORM log exists to provide. Construction must fail + /// closed instead. + #[tokio::test] + async fn connect_file_fails_closed_on_wrong_length_worm_key() { + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("bad_worm_key.db"); + let secrets = crate::vault::SecretManager::mock(); + secrets + .set_secret(crate::vault::WORM_KEY_ACCOUNT, &hex::encode([0u8; 16])) + .expect("seed undersized WORM key material"); + + let result = DatabaseEngine::connect_file(&db_path, &secrets).await; + assert!( + matches!(result, Err(DatabaseError::KeyProvisioning(_))), + "a WORM key of the wrong length must fail closed, never be silently accepted as an \ + HMAC signing key: {result:?}" + ); + } + + /// Mirrors `connect_file_fails_closed_on_wrong_length_worm_key` for the filter execution + /// ledger's HMAC key. + #[tokio::test] + async fn connect_file_fails_closed_on_wrong_length_ledger_key() { + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("bad_ledger_key.db"); + let secrets = crate::vault::SecretManager::mock(); + secrets + .set_secret(crate::vault::LEDGER_KEY_ACCOUNT, &hex::encode([0u8; 16])) + .expect("seed undersized ledger key material"); + + let result = DatabaseEngine::connect_file(&db_path, &secrets).await; + assert!( + matches!(result, Err(DatabaseError::KeyProvisioning(_))), + "a ledger key of the wrong length must fail closed, never be silently accepted as an \ + HMAC signing key: {result:?}" + ); + } + #[tokio::test] async fn insert_and_query_message() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); -- 2.54.0 From ec2b7b3ea6186c35144115a9104b7f9a6cca4b36 Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 30 Jul 2026 11:25:32 -0500 Subject: [PATCH 04/95] feat(security): reject non-loopback gRPC bind addresses (#240) serve() now parses the requested address and rejects it with a new GrpcServeError::NonLoopbackAddress before ever calling TcpListener::bind, so a misconfigured NUNCIO_GRPC_ADDR can't expose the bearer-token-only API beyond the local host. Also adds one consolidated canary test that dials all eight mounted services with no token and asserts Unauthenticated, giving reviewers a single checkpoint to update whenever a service is added to serve_on_listener_with_overrides. Refs #201 Co-authored-by: James Maes --- crates/nunciod/src/grpc.rs | 231 ++++++++++++++++++++++++++++++++++++- 1 file changed, 227 insertions(+), 4 deletions(-) diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index f8379cc..3d11220 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -84,6 +84,12 @@ pub enum GrpcServeError { /// The tonic transport server failed while serving requests. #[error("gRPC transport server failed: {0}")] Transport(#[from] tonic::transport::Error), + /// The requested bind address is not a loopback address (or isn't a + /// valid socket address at all). The gRPC API carries a bearer token + /// but no transport encryption, so it must never be reachable from + /// anywhere but the local host. + #[error("refusing to bind gRPC listener on non-loopback address {0}: must be a loopback address (e.g. 127.0.0.1 or ::1)")] + NonLoopbackAddress(String), } /// Maps a `nuncio_core::CoreEvent` domain event onto its wire-format @@ -2119,6 +2125,17 @@ pub async fn serve( secrets: Arc, token: impl Into>, ) -> Result<(), GrpcServeError> { + // Fail closed before ever touching the network: a misconfigured + // `NUNCIO_GRPC_ADDR` (or a future caller that forgets the loopback + // requirement) must not silently expose the unencrypted, + // bearer-token-only gRPC API beyond the local host. + let socket_addr = addr + .parse::() + .map_err(|_| GrpcServeError::NonLoopbackAddress(addr.to_string()))?; + if !socket_addr.ip().is_loopback() { + return Err(GrpcServeError::NonLoopbackAddress(addr.to_string())); + } + let listener = TcpListener::bind(addr) .await .map_err(|source| GrpcServeError::Bind { @@ -2140,7 +2157,11 @@ pub async fn serve( /// `System` and `Accounts` below -- never `add_service(SomeServer::new(...))` /// unwrapped. This is a hard, non-negotiable invariant: an un-intercepted /// service mounted here would be reachable by any local process without -/// authentication. If a future service is added, mount it the same way. +/// authentication. If a future service is added, mount it the same way, and +/// add it to the `tests::all_mounted_services_reject_unauthenticated_calls` +/// canary test's list of dialed services -- that test's list of services +/// must always match this function's `add_service` chain exactly, so a +/// service added to one but not the other is an obvious review gap. /// /// Exposed separately from [`serve`] so tests can bind an ephemeral loopback /// port (`127.0.0.1:0`), read back the OS-assigned port via @@ -2583,7 +2604,10 @@ mod tests { } #[tokio::test] - async fn serve_fails_closed_when_bind_address_is_invalid() { + async fn serve_fails_closed_when_bind_address_is_malformed() { + // A string that isn't a valid socket address at all can't be proven + // loopback, so it is rejected by the same address-validation check + // as a routable address -- never reaches `TcpListener::bind`. let event_bus = Arc::new(EventBus::new()); let (db, _dir) = DatabaseEngine::connect_ephemeral() .await @@ -2599,8 +2623,92 @@ mod tests { "token", ) .await - .expect_err("invalid bind address must fail"); - assert!(matches!(err, GrpcServeError::Bind { .. })); + .expect_err("malformed bind address must fail"); + assert!(matches!(err, GrpcServeError::NonLoopbackAddress(_))); + } + + #[tokio::test] + async fn serve_rejects_a_non_loopback_bind_address_without_binding() { + // A syntactically valid but routable address must be rejected by + // address validation before any socket is ever bound, so a + // misconfigured `NUNCIO_GRPC_ADDR` can't expose the API to the + // network even transiently. + let event_bus = Arc::new(EventBus::new()); + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + let secrets = Arc::new(SecretManager::mock()); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + let err = serve( + "0.0.0.0:0", + event_bus, + Arc::new(db), + filter_engine, + secrets, + "token", + ) + .await + .expect_err("non-loopback bind address must be rejected"); + assert!(matches!(err, GrpcServeError::NonLoopbackAddress(_))); + + let event_bus = Arc::new(EventBus::new()); + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + let secrets = Arc::new(SecretManager::mock()); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + let err = serve( + "10.0.0.5:9420", + event_bus, + Arc::new(db), + filter_engine, + secrets, + "token", + ) + .await + .expect_err("routable bind address must be rejected"); + assert!(matches!(err, GrpcServeError::NonLoopbackAddress(_))); + } + + #[tokio::test] + async fn serve_accepts_ipv6_loopback_bind_address() { + // The loopback check must not be IPv4-only: `is_loopback()` covers + // `::1` too, so `serve` should pass address validation for it and + // move on to actually binding and serving. `serve` never returns on + // a successful bind (it runs the transport server until it errors), + // so a short timeout distinguishes "still serving" (validation and + // bind both succeeded) from a real, immediate failure. A bind-time + // failure is acceptable on an environment without IPv6 loopback, but + // it must surface as `Bind`, never `NonLoopbackAddress`. + let event_bus = Arc::new(EventBus::new()); + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + let secrets = Arc::new(SecretManager::mock()); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + let result = tokio::time::timeout( + std::time::Duration::from_millis(200), + serve( + "[::1]:0", + event_bus, + Arc::new(db), + filter_engine, + secrets, + "token", + ), + ) + .await; + // `Err(_)` here is the timeout elapsing, i.e. `serve` is still + // running -- the success case. `Ok(Err(err))` is a real failure + // returned before the timeout, which must be a bind failure, not an + // address-validation rejection. + if let Ok(Err(err)) = result { + assert!( + matches!(err, GrpcServeError::Bind { .. }), + "IPv6 loopback must pass address validation even if the \ + environment can't actually bind it: got {err:?}" + ); + } } fn authed_bearer_request(payload: T) -> Request { @@ -5753,4 +5861,119 @@ mod tests { assert_eq!(fetched.display_name, "New Contact"); assert_eq!(fetched.organization.as_deref(), Some("Kof22")); } + + /// The single consolidated review checkpoint for the mounting invariant + /// documented on `serve_on_listener_with_overrides`: dials every one of + /// the services in that function's `add_service` chain with no + /// authorization metadata at all and asserts each rejects with + /// `Unauthenticated`. Whoever adds a ninth service must add it here too + /// -- if this test's list of dialed services ever stops matching that + /// `add_service` chain exactly, it has silently stopped doing its job. + #[tokio::test] + async fn all_mounted_services_reject_unauthenticated_calls() { + use nuncio_proto::v1::audit_client::AuditClient; + use nuncio_proto::v1::calendar_client::CalendarClient; + use nuncio_proto::v1::contacts_client::ContactsClient; + use nuncio_proto::v1::export_client::ExportClient; + use nuncio_proto::v1::filters_client::FiltersClient; + use nuncio_proto::v1::mail_client::MailClient; + + let event_bus = Arc::new(EventBus::new()); + let (addr, _handle, _dir) = spawn_test_server(event_bus, "correct-token").await; + let url = format!("http://{addr}"); + + // System + let mut client = SystemClient::connect(url.clone()) + .await + .expect("System client connects"); + let err = client + .get_status(GetStatusRequest {}) + .await + .expect_err("System must reject an unauthenticated call"); + assert_eq!(err.code(), Code::Unauthenticated); + + // Accounts + let mut client = AccountsClient::connect(url.clone()) + .await + .expect("Accounts client connects"); + let err = client + .list_accounts(ListAccountsRequest {}) + .await + .expect_err("Accounts must reject an unauthenticated call"); + assert_eq!(err.code(), Code::Unauthenticated); + + // Mail + let mut client = MailClient::connect(url.clone()) + .await + .expect("Mail client connects"); + let err = client + .list_folders(ListFoldersRequest {}) + .await + .expect_err("Mail must reject an unauthenticated call"); + assert_eq!(err.code(), Code::Unauthenticated); + + // Filters + let mut client = FiltersClient::connect(url.clone()) + .await + .expect("Filters client connects"); + let err = client + .list_rules(ListRulesRequest {}) + .await + .expect_err("Filters must reject an unauthenticated call"); + assert_eq!(err.code(), Code::Unauthenticated); + + // Export + let mut client = ExportClient::connect(url.clone()) + .await + .expect("Export client connects"); + let err = client + .export_mailbox(ExportRequest { + scope: None, + format: ExportFormatProto::Json.into(), + output_path: "irrelevant.json".to_string(), + }) + .await + .expect_err("Export must reject an unauthenticated call"); + assert_eq!(err.code(), Code::Unauthenticated); + + // Audit + let mut client = AuditClient::connect(url.clone()) + .await + .expect("Audit client connects"); + let err = client + .list_records(ListRecordsRequest { + limit: 10, + offset: 0, + }) + .await + .expect_err("Audit must reject an unauthenticated call"); + assert_eq!(err.code(), Code::Unauthenticated); + + // Calendar + let mut client = CalendarClient::connect(url.clone()) + .await + .expect("Calendar client connects"); + let err = client + .list_events(ListEventsRequest { + account_id: "acct-1".to_string(), + calendar_id: "cal-1".to_string(), + start_window: 0, + end_window: i64::MAX, + }) + .await + .expect_err("Calendar must reject an unauthenticated call"); + assert_eq!(err.code(), Code::Unauthenticated); + + // Contacts + let mut client = ContactsClient::connect(url) + .await + .expect("Contacts client connects"); + let err = client + .list_contacts(ListContactsRequest { + account_id: "acct-1".to_string(), + }) + .await + .expect_err("Contacts must reject an unauthenticated call"); + assert_eq!(err.code(), Code::Unauthenticated); + } } -- 2.54.0 From 3c11760a830799db4afd4b4c76f3a9bf39b26bb2 Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 30 Jul 2026 12:55:59 -0500 Subject: [PATCH 05/95] fix(update): fail closed on missing or unfetchable SHA256SUMS (#242) download_and_verify previously returned the downloaded archive unverified when a release had no SHA256SUMS.txt asset, or when fetching it returned a non-2xx status. Restructure so the only path to Ok is an exact SHA-256 match: absent asset, non-success fetch, filename-not-found, and hash mismatch all return Err. Adds a distinct ChecksumUnavailable variant and wiremock coverage for each failure mode plus the matching happy path. Refs #202 Co-authored-by: James Maes --- crates/nuncio-core/src/update.rs | 191 +++++++++++++++++++++++++++---- 1 file changed, 168 insertions(+), 23 deletions(-) diff --git a/crates/nuncio-core/src/update.rs b/crates/nuncio-core/src/update.rs index 540a5c9..8eef9da 100644 --- a/crates/nuncio-core/src/update.rs +++ b/crates/nuncio-core/src/update.rs @@ -40,6 +40,11 @@ pub enum UpdateError { #[error("checksum file error: {0}")] ChecksumFileError(String), + /// No checksum could be positively verified for the download, either because the release + /// has no `SHA256SUMS.txt` asset or because that asset could not be fetched. + #[error("checksum unavailable: {0}")] + ChecksumUnavailable(String), + /// Archive unpacking failure (tar.gz or zip). #[error("archive extraction failed: {0}")] ArchiveError(String), @@ -223,10 +228,41 @@ impl UpdateEngine { } /// Download release package bytes and verify SHA-256 checksum against `SHA256SUMS.txt`. + /// + /// This is the supply-chain trust boundary for the auto-updater: the only way to reach + /// `Ok` is an exact SHA-256 match against a successfully fetched `SHA256SUMS.txt`. Any + /// situation where the checksum cannot be positively verified — no checksum asset on the + /// release, a failed fetch of that asset, a missing filename entry, or a hash mismatch — + /// must fail closed rather than silently returning unverified bytes. pub async fn download_and_verify( &self, release_info: &ReleaseInfo, ) -> Result, UpdateError> { + // Refuse before spending bandwidth on the archive if there is no checksum asset to + // verify it against. + let checksum_url = release_info.checksum_url.as_ref().ok_or_else(|| { + UpdateError::ChecksumUnavailable( + "no SHA256SUMS.txt asset found for this release; refusing to apply an unverified update" + .to_string(), + ) + })?; + + let sums_resp = self.client.get(checksum_url).send().await?; + if !sums_resp.status().is_success() { + return Err(UpdateError::ChecksumUnavailable(format!( + "failed to fetch SHA256SUMS.txt (HTTP status {}); refusing to apply an unverified update", + sums_resp.status() + ))); + } + let sums_text = sums_resp.text().await?; + let expected_hash = parse_sha256sums(&sums_text, &release_info.archive_filename) + .ok_or_else(|| { + UpdateError::ChecksumFileError(format!( + "Filename '{}' not found in SHA256SUMS.txt", + release_info.archive_filename + )) + })?; + // Download main binary package archive let archive_resp = self.client.get(&release_info.download_url).send().await?; if !archive_resp.status().is_success() { @@ -237,29 +273,13 @@ impl UpdateEngine { } let archive_bytes = archive_resp.bytes().await?.to_vec(); - // Verify SHA-256 checksum if SHA256SUMS.txt asset is available - if let Some(checksum_url) = &release_info.checksum_url { - let sums_resp = self.client.get(checksum_url).send().await?; - if sums_resp.status().is_success() { - let sums_text = sums_resp.text().await?; - if let Some(expected_hash) = - parse_sha256sums(&sums_text, &release_info.archive_filename) - { - let actual_hash = compute_sha256(&archive_bytes); - if expected_hash.to_lowercase() != actual_hash.to_lowercase() { - return Err(UpdateError::ChecksumMismatch { - filename: release_info.archive_filename.clone(), - expected: expected_hash, - actual: actual_hash, - }); - } - } else { - return Err(UpdateError::ChecksumFileError(format!( - "Filename '{}' not found in SHA256SUMS.txt", - release_info.archive_filename - ))); - } - } + let actual_hash = compute_sha256(&archive_bytes); + if expected_hash.to_lowercase() != actual_hash.to_lowercase() { + return Err(UpdateError::ChecksumMismatch { + filename: release_info.archive_filename.clone(), + expected: expected_hash, + actual: actual_hash, + }); } Ok(archive_bytes) @@ -660,4 +680,129 @@ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 nuncio-x86_64- assert_eq!(info.release_notes, "Major security release"); assert!(info.download_url.contains(&archive_filename)); } + + fn release_info_for( + mock_server: &MockServer, + checksum_url: Option, + archive_filename: &str, + ) -> ReleaseInfo { + ReleaseInfo { + version: "99.0.0".to_string(), + tag_name: "v99.0.0".to_string(), + release_notes: String::new(), + download_url: format!("{}/download/{archive_filename}", mock_server.uri()), + checksum_url, + archive_filename: archive_filename.to_string(), + } + } + + #[tokio::test] + async fn download_and_verify_fails_closed_without_checksum_asset() { + let mock_server = MockServer::start().await; + let archive_filename = "nuncio-x86_64-unknown-linux-gnu.tar.gz"; + let archive_bytes = b"archive payload".to_vec(); + + Mock::given(method("GET")) + .and(path(format!("/download/{archive_filename}"))) + .respond_with(ResponseTemplate::new(200).set_body_bytes(archive_bytes)) + .mount(&mock_server) + .await; + + let updater = UpdateEngine::with_base_url(mock_server.uri()).unwrap(); + let info = release_info_for(&mock_server, None, archive_filename); + + let result = updater.download_and_verify(&info).await; + assert!(matches!(result, Err(UpdateError::ChecksumUnavailable(_)))); + } + + #[tokio::test] + async fn download_and_verify_fails_closed_on_sums_fetch_error() { + let mock_server = MockServer::start().await; + let archive_filename = "nuncio-x86_64-unknown-linux-gnu.tar.gz"; + let archive_bytes = b"archive payload".to_vec(); + + Mock::given(method("GET")) + .and(path(format!("/download/{archive_filename}"))) + .respond_with(ResponseTemplate::new(200).set_body_bytes(archive_bytes)) + .mount(&mock_server) + .await; + Mock::given(method("GET")) + .and(path("/download/SHA256SUMS.txt")) + .respond_with(ResponseTemplate::new(404)) + .mount(&mock_server) + .await; + + let updater = UpdateEngine::with_base_url(mock_server.uri()).unwrap(); + let info = release_info_for( + &mock_server, + Some(format!("{}/download/SHA256SUMS.txt", mock_server.uri())), + archive_filename, + ); + + let result = updater.download_and_verify(&info).await; + assert!(matches!(result, Err(UpdateError::ChecksumUnavailable(_)))); + } + + #[tokio::test] + async fn download_and_verify_fails_closed_on_hash_mismatch() { + let mock_server = MockServer::start().await; + let archive_filename = "nuncio-x86_64-unknown-linux-gnu.tar.gz"; + let archive_bytes = b"archive payload".to_vec(); + let wrong_hash = compute_sha256(b"not the archive bytes"); + let sums_text = format!("{wrong_hash} {archive_filename}\n"); + + Mock::given(method("GET")) + .and(path(format!("/download/{archive_filename}"))) + .respond_with(ResponseTemplate::new(200).set_body_bytes(archive_bytes)) + .mount(&mock_server) + .await; + Mock::given(method("GET")) + .and(path("/download/SHA256SUMS.txt")) + .respond_with(ResponseTemplate::new(200).set_body_string(sums_text)) + .mount(&mock_server) + .await; + + let updater = UpdateEngine::with_base_url(mock_server.uri()).unwrap(); + let info = release_info_for( + &mock_server, + Some(format!("{}/download/SHA256SUMS.txt", mock_server.uri())), + archive_filename, + ); + + let result = updater.download_and_verify(&info).await; + assert!(matches!(result, Err(UpdateError::ChecksumMismatch { .. }))); + } + + #[tokio::test] + async fn download_and_verify_succeeds_on_matching_checksum() { + let mock_server = MockServer::start().await; + let archive_filename = "nuncio-x86_64-unknown-linux-gnu.tar.gz"; + let archive_bytes = b"archive payload".to_vec(); + let correct_hash = compute_sha256(&archive_bytes); + let sums_text = format!("{correct_hash} {archive_filename}\n"); + + Mock::given(method("GET")) + .and(path(format!("/download/{archive_filename}"))) + .respond_with(ResponseTemplate::new(200).set_body_bytes(archive_bytes.clone())) + .mount(&mock_server) + .await; + Mock::given(method("GET")) + .and(path("/download/SHA256SUMS.txt")) + .respond_with(ResponseTemplate::new(200).set_body_string(sums_text)) + .mount(&mock_server) + .await; + + let updater = UpdateEngine::with_base_url(mock_server.uri()).unwrap(); + let info = release_info_for( + &mock_server, + Some(format!("{}/download/SHA256SUMS.txt", mock_server.uri())), + archive_filename, + ); + + let result = updater + .download_and_verify(&info) + .await + .expect("checksum matches, download succeeds"); + assert_eq!(result, archive_bytes); + } } -- 2.54.0 From fc5d6b337e63943d074cf4222a67730ec7a78075 Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 30 Jul 2026 13:10:06 -0500 Subject: [PATCH 06/95] fix(filter): enforce real IP/CIDR SSRF checks on webhook targets (#243) Replace the substring blocklist in pass 6 with parsed IP-literal range checks and add a dispatch-time DNS-resolution check so hostnames that resolve into private/metadata ranges are rejected before connecting. Refs #203 Co-authored-by: James Maes --- crates/nuncio-filter/src/validator.rs | 150 ++++++++++++++++++++++++-- crates/nuncio-filter/src/webhook.rs | 65 ++++++++++- 2 files changed, 206 insertions(+), 9 deletions(-) diff --git a/crates/nuncio-filter/src/validator.rs b/crates/nuncio-filter/src/validator.rs index ff67cbd..7975e55 100644 --- a/crates/nuncio-filter/src/validator.rs +++ b/crates/nuncio-filter/src/validator.rs @@ -33,7 +33,10 @@ pub struct ValidationOptions { pub available_folders: Option>, /// Optional domain whitelist for Pass 6 `FORWARD TO` actions. pub allowed_forward_domains: Option>, - /// Flag enabling Pass 6 pre-flight DNS IP blacklisting for webhooks. + /// When set, reject webhook targets that point at private, loopback, + /// link-local, or metadata address ranges. The validator checks literal-IP + /// hosts; the dispatcher additionally checks addresses a hostname resolves + /// to before connecting. pub block_private_webhooks: bool, } @@ -278,13 +281,25 @@ impl NsqlValidator { ))); } if opts.block_private_webhooks { - let url_lower = url.to_lowercase(); - if url_lower.contains("127.0.0.1") - || url_lower.contains("localhost") - || url_lower.contains("169.254.169.254") - || url_lower.contains("0.0.0.0") - { - return Err(ValidationError::ActionSecurityViolation(format!("CALL WEBHOOK URL '{url}' points to blocked private / metadata IP address"))); + // When the host is an IP literal, reject any that lands in a + // blocked range. A substring match on the raw URL is not a + // real defense: RFC 1918, alternate literal encodings, and + // IPv6 all slip past it. Hostnames cannot be resolved here + // (this pass is synchronous); the dispatcher performs the + // DNS-resolution check before it connects. + if let Ok(parsed) = reqwest::Url::parse(url) { + if let Some(host) = parsed.host_str() { + // `host_str` brackets IPv6 literals (`[::1]`); strip + // them so the address parses. + let host = host.trim_start_matches('[').trim_end_matches(']'); + if let Ok(ip) = host.parse::() { + if is_blocked_webhook_target(ip) { + return Err(ValidationError::ActionSecurityViolation(format!( + "CALL WEBHOOK URL '{url}' points to a blocked private / metadata address" + ))); + } + } + } } } } @@ -309,6 +324,51 @@ fn is_valid_email_format(email: &str) -> bool { && !parts[1].ends_with('.') } +/// Reports whether `ip` falls in a range that must never be a webhook target. +/// +/// This is the core SSRF egress control. It blocks loopback, RFC 1918 / ULA +/// private space, link-local (which subsumes the `169.254.169.254` cloud +/// metadata endpoint), the unspecified address, broadcast, and multicast. +/// It is shared by the synchronous validator (literal IPs in the URL) and the +/// dispatcher (addresses a hostname resolves to) so both apply identical rules. +pub(crate) fn is_blocked_webhook_target(ip: std::net::IpAddr) -> bool { + match ip { + std::net::IpAddr::V4(v4) => is_blocked_ipv4(v4), + std::net::IpAddr::V6(v6) => { + // Unwrap IPv4-mapped addresses (`::ffff:a.b.c.d`) first: an embedded + // private or loopback IPv4 must be caught by the v4 rules and not + // slip through as an ordinary v6 address. + match v6.to_ipv4_mapped() { + Some(mapped) => is_blocked_ipv4(mapped), + None => is_blocked_ipv6(v6), + } + } + } +} + +fn is_blocked_ipv4(ip: std::net::Ipv4Addr) -> bool { + // `is_private` implements the exact RFC 1918 blocks (10/8, 172.16/12, + // 192.168/16), so boundaries such as 172.32.0.0 stay allowed. The leading + // `0` octet covers 0.0.0.0/8 (this-network, wider than `is_unspecified`). + ip.octets()[0] == 0 + || ip.is_loopback() + || ip.is_private() + || ip.is_link_local() + || ip.is_multicast() + || ip.is_broadcast() +} + +fn is_blocked_ipv6(ip: std::net::Ipv6Addr) -> bool { + if ip.is_loopback() || ip.is_unspecified() || ip.is_multicast() { + return true; + } + // `is_unique_local` / `is_unicast_link_local` are not stable on the pinned + // toolchain, so match the prefixes by hand: fc00::/7 (unique-local) and + // fe80::/10 (link-local unicast). + let first = ip.segments()[0]; + (first & 0xfe00) == 0xfc00 || (first & 0xffc0) == 0xfe80 +} + #[cfg(test)] mod tests { use super::*; @@ -372,4 +432,78 @@ mod tests { let err = NsqlValidator::validate(&rule, &ValidationOptions::default()).unwrap_err(); assert!(matches!(err, ValidationError::ActionSecurityViolation(_))); } + + #[test] + fn test_pass6_rejects_literal_ip_bypasses() { + // Encodings the old case-insensitive substring blocklist let through. + let blocked_urls = [ + "http://10.1.2.3/hook", + "http://172.16.0.1/hook", + "http://192.168.1.1/hook", + "http://169.254.169.254/latest/meta-data/", + "http://[::1]/hook", + "http://[fc00::1]/hook", + "http://[fe80::1]/hook", + "http://[::ffff:127.0.0.1]/hook", + ]; + for url in blocked_urls { + let rule = crate::parser::NsqlParser::parse_rule( + "Webhook Bypass", + 1, + &format!("WHERE subject CONTAINS 'x' ACTION CALL WEBHOOK '{url}'"), + ) + .unwrap(); + let err = NsqlValidator::validate(&rule, &ValidationOptions::default()).unwrap_err(); + assert!( + matches!(err, ValidationError::ActionSecurityViolation(_)), + "expected {url} to be blocked" + ); + } + } + + #[test] + fn test_pass6_allows_public_literal_ip() { + let rule = crate::parser::NsqlParser::parse_rule( + "Webhook Public", + 1, + "WHERE subject CONTAINS 'x' ACTION CALL WEBHOOK 'http://93.184.216.34/hook'", + ) + .unwrap(); + assert!(NsqlValidator::validate(&rule, &ValidationOptions::default()).is_ok()); + } + + #[test] + fn test_is_blocked_webhook_target_blocks_private_ranges() { + let blocked: [std::net::IpAddr; 12] = [ + "127.0.0.1".parse().unwrap(), + "10.1.2.3".parse().unwrap(), + "172.16.0.1".parse().unwrap(), + "172.31.255.255".parse().unwrap(), + "192.168.1.1".parse().unwrap(), + "169.254.169.254".parse().unwrap(), + "169.254.0.1".parse().unwrap(), + "0.0.0.0".parse().unwrap(), + "::1".parse().unwrap(), + "fc00::1".parse().unwrap(), + "fe80::1".parse().unwrap(), + "::ffff:127.0.0.1".parse().unwrap(), + ]; + for ip in blocked { + assert!(is_blocked_webhook_target(ip), "{ip} should be blocked"); + } + } + + #[test] + fn test_is_blocked_webhook_target_allows_public() { + // 172.32.0.1 sits just outside 172.16.0.0/12 and proves the range math + // is exact rather than a naive `172.` prefix check. + let allowed: [std::net::IpAddr; 3] = [ + "172.32.0.1".parse().unwrap(), + "93.184.216.34".parse().unwrap(), + "2606:2800:220:1:248:1893:25c8:1946".parse().unwrap(), + ]; + for ip in allowed { + assert!(!is_blocked_webhook_target(ip), "{ip} should be allowed"); + } + } } diff --git a/crates/nuncio-filter/src/webhook.rs b/crates/nuncio-filter/src/webhook.rs index d25dd41..68e7422 100644 --- a/crates/nuncio-filter/src/webhook.rs +++ b/crates/nuncio-filter/src/webhook.rs @@ -1,17 +1,29 @@ //! Outbound HTTP Webhook Execution Engine with HMAC-SHA256 Signatures and SSRF Defense. use crate::ast::RuleAction; -use crate::validator::{NsqlValidator, ValidationOptions}; +use crate::validator::{is_blocked_webhook_target, NsqlValidator, ValidationOptions}; use hmac::{Hmac, Mac}; use reqwest::Client; use serde_json::json; use sha2::Sha256; +use std::net::IpAddr; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use thiserror::Error; +use tokio::net::lookup_host; use zeroize::Zeroizing; type HmacSha256 = Hmac; +/// Returns the first address in `addrs` that is a blocked webhook target, if +/// any. Split out from the DNS lookup so the egress policy can be exercised +/// against synthetic address lists without touching the network. +fn first_blocked_address(addrs: &[IpAddr]) -> Option { + addrs + .iter() + .copied() + .find(|ip| is_blocked_webhook_target(*ip)) +} + /// Errors emitted during HTTP webhook dispatch execution. #[derive(Error, Debug)] pub enum WebhookError { @@ -61,6 +73,34 @@ impl WebhookDispatcher { NsqlValidator::pass6_action_security(&[action], opts) .map_err(|e| WebhookError::SecurityViolation(e.to_string()))?; + if opts.block_private_webhooks { + // Pass 6 only sees literal-IP hosts. Resolve the target here and + // reject it if a hostname points into a blocked range, closing the + // DNS-rebind class of SSRF bypass. This is not airtight: reqwest + // re-resolves and connects on its own after this check, so a narrow + // TOCTOU window remains where DNS could flip between our lookup and + // the actual connection. Fully closing it needs a pinned + // resolver/connector, which is out of scope here. + if let Ok(parsed) = reqwest::Url::parse(url) { + if let Some(host) = parsed.host_str() { + // `host_str` brackets IPv6 literals (`[::1]`); strip them so + // the pair passes to the resolver cleanly. + let host = host.trim_start_matches('[').trim_end_matches(']'); + let port = parsed.port_or_known_default().unwrap_or(0); + let resolved: Vec = lookup_host((host, port)) + .await + .map_err(|e| WebhookError::NetworkError(e.to_string()))? + .map(|addr| addr.ip()) + .collect(); + if let Some(blocked) = first_blocked_address(&resolved) { + return Err(WebhookError::SecurityViolation(format!( + "webhook host '{host}' resolves to blocked address {blocked}" + ))); + } + } + } + } + let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() @@ -142,4 +182,27 @@ mod tests { WebhookError::SecurityViolation(_) )); } + + #[test] + fn test_first_blocked_address_flags_private_in_mixed_list() { + // Synthetic resolver output: a public address alongside a private one, + // as a rebinding host might return. The private one must be caught. + let addrs: Vec = vec![ + "93.184.216.34".parse().unwrap(), + "10.0.0.5".parse().unwrap(), + ]; + assert_eq!( + first_blocked_address(&addrs), + Some("10.0.0.5".parse().unwrap()) + ); + } + + #[test] + fn test_first_blocked_address_allows_all_public() { + let addrs: Vec = vec![ + "93.184.216.34".parse().unwrap(), + "172.32.0.1".parse().unwrap(), + ]; + assert_eq!(first_blocked_address(&addrs), None); + } } -- 2.54.0 From 17774acb16aba0b6dee06a36234224df5c2252a7 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 09:52:51 -0500 Subject: [PATCH 07/95] refactor(core): remove fabricated e2ee/ai/plugin modules (#261) An audit found e2ee.rs, ai.rs, and plugin.rs implemented no real functionality: signature verification did no cryptography, the AI engine returned canned strings with no model call, and the plugin runtime executed no code. These modules were pub-exported but wired nowhere in the workspace, and their tests asserted on the fabricated output rather than real behavior. E2EE, local AI, and plugins are deferred features; deferred features must be absent, not fake-present. A workspace-wide grep confirmed no code outside these three files and their lib.rs declarations referenced any of their exported symbols. Refs #244 #79 #80 #81 Co-authored-by: James Maes --- crates/nuncio-core/src/ai.rs | 145 ---------------------------- crates/nuncio-core/src/e2ee.rs | 156 ------------------------------- crates/nuncio-core/src/lib.rs | 6 -- crates/nuncio-core/src/plugin.rs | 135 -------------------------- 4 files changed, 442 deletions(-) delete mode 100644 crates/nuncio-core/src/ai.rs delete mode 100644 crates/nuncio-core/src/e2ee.rs delete mode 100644 crates/nuncio-core/src/plugin.rs diff --git a/crates/nuncio-core/src/ai.rs b/crates/nuncio-core/src/ai.rs deleted file mode 100644 index f8e26cd..0000000 --- a/crates/nuncio-core/src/ai.rs +++ /dev/null @@ -1,145 +0,0 @@ -//! Autonomous Local LLM Email Summarization & Action Item Extraction Engine. - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -/// Thread executive summary output containing bullet points and key topics. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ThreadSummary { - pub bullets: Vec, - pub sentiment: String, - pub primary_topic: String, -} - -/// Extracted actionable item from an email thread. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ActionItem { - pub task_description: String, - pub suggested_due_date: Option, - pub assignee_email: Option, -} - -/// Errors emitted by the Local LLM AI Engine. -#[derive(Error, Debug, PartialEq, Eq)] -pub enum AiEngineError { - #[error("ollama connection failed: {0}")] - ConnectionFailed(String), - - #[error("inference failed: {0}")] - InferenceFailed(String), - - #[error("empty email body provided")] - EmptyInput, -} - -/// Local LLM AI Connector (Ollama / Llama.cpp local HTTP API endpoint). -pub struct LocalAiEngine { - endpoint_url: String, - model_name: String, -} - -impl LocalAiEngine { - /// Create a new `LocalAiEngine` pointing to local Ollama / Llama.cpp HTTP service (e.g. `http://127.0.0.1:11434`). - pub fn new(endpoint_url: &str, model_name: &str) -> Self { - Self { - endpoint_url: endpoint_url.to_string(), - model_name: model_name.to_string(), - } - } - - /// Construct default instance targeting `http://127.0.0.1:11434` with model `llama3:8b`. - pub fn default_local() -> Self { - Self::new("http://127.0.0.1:11434", "llama3:8b") - } - - /// Retrieve configured Ollama endpoint URL. - pub fn endpoint_url(&self) -> &str { - &self.endpoint_url - } - - /// Retrieve configured LLM model name. - pub fn model_name(&self) -> &str { - &self.model_name - } - - /// Generate executive 3-bullet summary of an email thread. - pub fn summarize_thread(&self, email_body: &str) -> Result { - if email_body.trim().is_empty() { - return Err(AiEngineError::EmptyInput); - } - - // Local inference parser - let bullets = vec![ - "Discussion regarding upcoming Q3 roadmap and feature milestones.".to_string(), - "Requested confirmation for team meeting scheduled next Tuesday.".to_string(), - "Action required: Review security audit findings before deployment.".to_string(), - ]; - - Ok(ThreadSummary { - bullets, - sentiment: "Professional / Action-Oriented".to_string(), - primary_topic: "Project Synchronization".to_string(), - }) - } - - /// Extract actionable tasks and suggested calendar events from email content. - pub fn extract_action_items(&self, email_body: &str) -> Result, AiEngineError> { - if email_body.trim().is_empty() { - return Err(AiEngineError::EmptyInput); - } - - Ok(vec![ - ActionItem { - task_description: "Review security audit report".to_string(), - suggested_due_date: Some("2026-07-25".to_string()), - assignee_email: Some("james@kof22.com".to_string()), - }, - ActionItem { - task_description: "Confirm Q3 deployment schedule".to_string(), - suggested_due_date: Some("2026-07-28".to_string()), - assignee_email: None, - }, - ]) - } - - /// Generate 3 microsecond smart reply suggestions. - pub fn generate_smart_replies(&self, _email_body: &str) -> Vec { - vec![ - "Sounds good, let's proceed!".to_string(), - "Thanks for the update, I'll review this shortly.".to_string(), - "Could we reschedule for later this week?".to_string(), - ] - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn summarize_thread_returns_valid_structure() { - let engine = LocalAiEngine::default_local(); - let summary = engine - .summarize_thread("Meeting notes for Q3 project...") - .unwrap(); - assert_eq!(summary.bullets.len(), 3); - assert!(!summary.sentiment.is_empty()); - } - - #[test] - fn extract_action_items_returns_items() { - let engine = LocalAiEngine::default_local(); - let items = engine - .extract_action_items("Please review the audit by Friday.") - .unwrap(); - assert_eq!(items.len(), 2); - assert_eq!(items[0].task_description, "Review security audit report"); - } - - #[test] - fn generate_smart_replies_returns_three_options() { - let engine = LocalAiEngine::default_local(); - let replies = engine.generate_smart_replies("Can we talk?"); - assert_eq!(replies.len(), 3); - } -} diff --git a/crates/nuncio-core/src/e2ee.rs b/crates/nuncio-core/src/e2ee.rs deleted file mode 100644 index 284b4a1..0000000 --- a/crates/nuncio-core/src/e2ee.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! End-to-End Encryption (E2EE) module implementing OpenPGP (RFC 3156) and S/MIME (RFC 8551) validation. - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -/// Cryptographic signature type attached to an email payload. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum EncryptionType { - /// PGP/MIME or Inline OpenPGP. - OpenPGP, - /// S/MIME PKCS#7 / CMS signature. - Smime, - /// Unencrypted / Unsigned plain message. - None, -} - -/// Status verdict after cryptographic signature verification. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum SignatureStatus { - /// Valid cryptographic signature matching sender key. - Valid { - signer_fingerprint: String, - key_id: String, - }, - /// Invalid signature or tampered ciphertext payload. - Invalid { reason: String }, - /// Untrusted public key or unknown certificate authority. - UntrustedKey { fingerprint: String }, - /// No signature present. - Unsigned, -} - -/// Security badge representation for presentation layers (CLI, TUI, GUI, MCP). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct SecurityBadge { - pub encryption_type: EncryptionType, - pub signature_status: SignatureStatus, - pub is_encrypted: bool, - pub badge_label: String, -} - -/// Errors originating from E2EE operations. -#[derive(Error, Debug, PartialEq, Eq)] -pub enum E2eeError { - #[error("failed to verify signature: {0}")] - VerificationFailed(String), - - #[error("decryption failed: {0}")] - DecryptionFailed(String), - - #[error("key not found: {0}")] - KeyNotFound(String), -} - -/// Cryptographic E2EE Engine. -pub struct E2eeEngine; - -impl E2eeEngine { - /// Verify an OpenPGP signature string against a sender identity. - pub fn verify_openpgp_signature( - payload: &str, - signature: &str, - expected_fingerprint: &str, - ) -> Result { - if payload.is_empty() || signature.is_empty() { - return Ok(SecurityBadge { - encryption_type: EncryptionType::OpenPGP, - signature_status: SignatureStatus::Invalid { - reason: "empty payload or signature".to_string(), - }, - is_encrypted: false, - badge_label: "PGP INVALID".to_string(), - }); - } - - // Simulating Sequoia / OpenPGP verification verification logic - let valid = signature.contains("BEGIN PGP SIGNATURE") && expected_fingerprint.len() >= 8; - - if valid { - Ok(SecurityBadge { - encryption_type: EncryptionType::OpenPGP, - signature_status: SignatureStatus::Valid { - signer_fingerprint: expected_fingerprint.to_string(), - key_id: expected_fingerprint.chars().take(8).collect(), - }, - is_encrypted: true, - badge_label: "PGP SIGNED & ENCRYPTED".to_string(), - }) - } else { - Ok(SecurityBadge { - encryption_type: EncryptionType::OpenPGP, - signature_status: SignatureStatus::UntrustedKey { - fingerprint: expected_fingerprint.to_string(), - }, - is_encrypted: false, - badge_label: "PGP UNTRUSTED KEY".to_string(), - }) - } - } - - /// Verify an S/MIME PKCS#7 signature payload. - pub fn verify_smime_signature( - _payload: &[u8], - signature_pkcs7: &[u8], - signer_email: &str, - ) -> Result { - if signature_pkcs7.is_empty() { - return Ok(SecurityBadge { - encryption_type: EncryptionType::Smime, - signature_status: SignatureStatus::Unsigned, - is_encrypted: false, - badge_label: "S/MIME UNSIGNED".to_string(), - }); - } - - Ok(SecurityBadge { - encryption_type: EncryptionType::Smime, - signature_status: SignatureStatus::Valid { - signer_fingerprint: format!("SMIME-{signer_email}"), - key_id: signer_email.to_string(), - }, - is_encrypted: true, - badge_label: "S/MIME VERIFIED".to_string(), - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn verify_openpgp_valid_signature() { - let payload = "Hello world"; - let sig = "-----BEGIN PGP SIGNATURE-----\nVersion: Sequoia\n-----END PGP SIGNATURE-----"; - let fp = "4A8B9C0D1E2F3A4B5C6D7E8F9A0B1C2D3E4F5A6B"; - - let badge = E2eeEngine::verify_openpgp_signature(payload, sig, fp).unwrap(); - assert_eq!(badge.encryption_type, EncryptionType::OpenPGP); - assert!(badge.is_encrypted); - assert!(matches!( - badge.signature_status, - SignatureStatus::Valid { .. } - )); - } - - #[test] - fn verify_smime_valid_signature() { - let payload = b"Hello S/MIME"; - let sig = b"PKCS7_SIGNATURE_BYTES"; - - let badge = E2eeEngine::verify_smime_signature(payload, sig, "alice@kof22.com").unwrap(); - assert_eq!(badge.encryption_type, EncryptionType::Smime); - assert_eq!(badge.badge_label, "S/MIME VERIFIED"); - } -} diff --git a/crates/nuncio-core/src/lib.rs b/crates/nuncio-core/src/lib.rs index 5921251..80000df 100644 --- a/crates/nuncio-core/src/lib.rs +++ b/crates/nuncio-core/src/lib.rs @@ -3,23 +3,17 @@ #![forbid(unsafe_code)] #![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::panic))] -pub mod ai; pub mod config; -pub mod e2ee; pub mod export; pub mod mcp_policy; pub mod model; -pub mod plugin; pub mod update; pub mod worm_audit; -pub use ai::{ActionItem, AiEngineError, LocalAiEngine, ThreadSummary}; pub use config::{AccountConfig, AccountProtocol, ConfigError, TlsMode}; -pub use e2ee::{E2eeEngine, E2eeError, EncryptionType, SecurityBadge, SignatureStatus}; pub use export::{ExportEngine, ExportError, ExportFormat, ExportSummary}; pub use mcp_policy::{AgentPermissions, DataType, McpAgentPolicy}; pub use model::{Attachment, CalendarEvent, Contact, DaemonTelemetry, Email, Folder}; -pub use plugin::{PluginError, PluginHook, PluginManifest, PluginRuntime}; pub use update::{ReleaseInfo, UpdateCheckResult, UpdateEngine, UpdateError}; pub use worm_audit::{verify_worm_chain, WormAuditError, WormAuditRecord}; diff --git a/crates/nuncio-core/src/plugin.rs b/crates/nuncio-core/src/plugin.rs deleted file mode 100644 index b5d25f2..0000000 --- a/crates/nuncio-core/src/plugin.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! Sandboxed Plugin Runtime Engine for custom automation scripts. - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -/// Event triggers that plugins can subscribe to. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum PluginHook { - OnMailReceived, - OnMailSending, - OnCalendarRsvp, - OnFilterRuleExecuted, -} - -/// Metadata definition of an installed plugin package. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PluginManifest { - pub id: String, - pub name: String, - pub version: String, - pub author: String, - pub hooks: Vec, - pub permissions: Vec, -} - -/// Errors emitted by the plugin runtime. -#[derive(Error, Debug, PartialEq, Eq)] -pub enum PluginError { - #[error("plugin execution failed: {0}")] - ExecutionFailed(String), - - #[error("permission denied: {0}")] - PermissionDenied(String), - - #[error("plugin manifest invalid: {0}")] - InvalidManifest(String), -} - -/// Sandboxed Plugin Runtime Manager. -pub struct PluginRuntime { - installed_plugins: Vec, -} - -impl PluginRuntime { - /// Create a new plugin runtime instance. - pub fn new() -> Self { - Self { - installed_plugins: Vec::new(), - } - } - - /// Register a new sandboxed plugin manifest. - pub fn register_plugin(&mut self, manifest: PluginManifest) -> Result<(), PluginError> { - if manifest.id.is_empty() { - return Err(PluginError::InvalidManifest( - "missing plugin id".to_string(), - )); - } - self.installed_plugins.push(manifest); - Ok(()) - } - - /// List all currently installed plugin manifests. - pub fn list_plugins(&self) -> &[PluginManifest] { - &self.installed_plugins - } - - /// Execute subscribed plugins for a specific event hook. - pub fn trigger_hook( - &self, - hook: PluginHook, - event_payload_json: &str, - ) -> Vec> { - self.installed_plugins - .iter() - .filter(|p| p.hooks.contains(&hook)) - .map(|p| { - if event_payload_json.is_empty() { - Err(PluginError::ExecutionFailed("empty payload".to_string())) - } else { - Ok(format!( - "plugin {} executed successfully for hook {:?}", - p.id, hook - )) - } - }) - .collect() - } -} - -impl Default for PluginRuntime { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn register_and_list_plugins() { - let mut runtime = PluginRuntime::new(); - let manifest = PluginManifest { - id: "auto-tagger".to_string(), - name: "Auto Email Tagger".to_string(), - version: "1.0.0".to_string(), - author: "KofTwentyTwo".to_string(), - hooks: vec![PluginHook::OnMailReceived], - permissions: vec!["mail:read".to_string()], - }; - - runtime.register_plugin(manifest).expect("register"); - assert_eq!(runtime.list_plugins().len(), 1); - } - - #[test] - fn trigger_hook_executes_registered_plugins() { - let mut runtime = PluginRuntime::new(); - runtime - .register_plugin(PluginManifest { - id: "p1".to_string(), - name: "P1".to_string(), - version: "0.1.0".to_string(), - author: "Dev".to_string(), - hooks: vec![PluginHook::OnMailReceived], - permissions: vec![], - }) - .unwrap(); - - let results = runtime.trigger_hook(PluginHook::OnMailReceived, r#"{"msg":"hello"}"#); - assert_eq!(results.len(), 1); - assert!(results[0].is_ok()); - } -} -- 2.54.0 From aabda0806909abfc38e1a2551ee3ee0d88c7e946 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 10:05:49 -0500 Subject: [PATCH 08/95] refactor(mail): remove fake/dead protocol code paths (#262) Delete crates/nuncio-cal/src/carddav.rs, an orphaned duplicate CardDavClient/Contact that made no network call and was unused outside its own tests -- the real CardDAV client lives in nuncio-contacts. Make credential-less ImapEngine/JmapEngine sync_folders/sync_messages return an honest MailError::AuthError instead of returning hardcoded folders/messages. This also fixes a latent bug in ImapEngine::sync_folders: a genuinely empty real mailbox list previously fell through to fake canned folders instead of being returned as-is. Remove send_email from the MailBackend trait: production sends route through MessageSender/SmtpTransportEngine exclusively, and the IMAP/JMAP implementations were both no-op fabricated successes with no real transport behind them. Migrate the tests that exercised the removed fixture paths to real fixtures: a scripted loopback IMAP server drives ImapEngine::sync_folders through a real LOGIN/LIST/LOGOUT exchange, and credential-less calls are now asserted to fail honestly rather than fabricate data. Refs #253 Co-authored-by: James Maes --- crates/nuncio-cal/src/carddav.rs | 148 --------------------------- crates/nuncio-cal/src/lib.rs | 2 - crates/nuncio-mail/src/backend.rs | 3 - crates/nuncio-mail/src/imap.rs | 165 +++++++++++++++++------------- crates/nuncio-mail/src/jmap.rs | 126 ++++++++--------------- crates/nuncio-mail/src/mock.rs | 31 ------ 6 files changed, 137 insertions(+), 338 deletions(-) delete mode 100644 crates/nuncio-cal/src/carddav.rs diff --git a/crates/nuncio-cal/src/carddav.rs b/crates/nuncio-cal/src/carddav.rs deleted file mode 100644 index c83898b..0000000 --- a/crates/nuncio-cal/src/carddav.rs +++ /dev/null @@ -1,148 +0,0 @@ -//! CardDAV (vCard RFC 6350) contact sync and XML query parser. - -use crate::parser::CalendarError; -use serde::{Deserialize, Serialize}; - -/// Contact entity representation synced over CardDAV. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Contact { - pub id: String, - pub name: String, - pub email: String, - pub phone: Option, - pub organization: Option, -} - -/// CardDAV client protocol engine managing vCard address book queries. -pub struct CardDavClient { - account_id: String, -} - -impl CardDavClient { - /// Create a new `CardDavClient` bound to an account ID. - pub fn new(account_id: &str) -> Self { - Self { - account_id: account_id.to_string(), - } - } - - /// Construct a standard CardDAV `` XML payload. - pub fn build_addressbook_query() -> String { - r#" - - - - - -"# - .to_string() - } - - /// Parse raw vCard string into domain [`Contact`] entity. - pub fn parse_vcard(id: &str, raw_vcard: &str) -> Result { - let mut name = String::new(); - let mut email = String::new(); - let mut phone = None; - let mut organization = None; - - for line in raw_vcard.lines() { - let line = line.trim(); - if let Some(val) = line.strip_prefix("FN:") { - name = val.trim().to_string(); - } else if let Some(val) = line.strip_prefix("EMAIL:") { - email = val.trim().to_string(); - } else if line.starts_with("EMAIL;") { - if let Some((_, val)) = line.split_once(':') { - email = val.trim().to_string(); - } - } else if let Some(val) = line.strip_prefix("TEL:") { - phone = Some(val.trim().to_string()); - } else if line.starts_with("TEL;") { - if let Some((_, val)) = line.split_once(':') { - phone = Some(val.trim().to_string()); - } - } else if let Some(val) = line.strip_prefix("ORG:") { - organization = Some(val.trim().to_string()); - } - } - - if name.is_empty() && email.is_empty() { - return Err(CalendarError::ParseFailed( - "invalid or empty vCard data".to_string(), - )); - } - - if name.is_empty() { - name = email.clone(); - } - - Ok(Contact { - id: id.to_string(), - name, - email, - phone, - organization, - }) - } - - /// Parse CardDAV WebDAV XML `` response containing embedded vCard data. - pub fn parse_multistatus_response(&self, raw_xml: &str) -> Result, CalendarError> { - let mut contacts = Vec::new(); - - for block in raw_xml.split("") { - if let Some((vcard_data, _)) = block.split_once("") { - let clean_vcard = vcard_data.trim(); - if !clean_vcard.is_empty() { - let contact_id = format!("{}-contact-{}", self.account_id, contacts.len() + 1); - if let Ok(contact) = Self::parse_vcard(&contact_id, clean_vcard) { - contacts.push(contact); - } - } - } - } - - Ok(contacts) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_vcard_valid_structure() { - let vcard = r#"BEGIN:VCARD -VERSION:4.0 -FN:James Maes -EMAIL:james.maes@kof22.com -TEL:+1-555-0199 -ORG:Kof22 Labs -END:VCARD"#; - - let contact = CardDavClient::parse_vcard("c-1", vcard).expect("parse vcard"); - assert_eq!(contact.name, "James Maes"); - assert_eq!(contact.email, "james.maes@kof22.com"); - assert_eq!(contact.phone.as_deref(), Some("+1-555-0199")); - assert_eq!(contact.organization.as_deref(), Some("Kof22 Labs")); - } - - #[test] - fn parse_multistatus_carddav_xml() { - let client = CardDavClient::new("acct-1"); - let xml = r#" - - - BEGIN:VCARD -FN:Alice Dev -EMAIL:alice@nuncio.mx -END:VCARD - - "#; - - let contacts = client - .parse_multistatus_response(xml) - .expect("parse multistatus"); - assert_eq!(contacts.len(), 1); - assert_eq!(contacts[0].name, "Alice Dev"); - } -} diff --git a/crates/nuncio-cal/src/lib.rs b/crates/nuncio-cal/src/lib.rs index 58d3846..d9a72bf 100644 --- a/crates/nuncio-cal/src/lib.rs +++ b/crates/nuncio-cal/src/lib.rs @@ -5,7 +5,6 @@ pub mod backend; pub mod caldav; -pub mod carddav; pub mod mock; pub mod nlp; pub mod parser; @@ -14,7 +13,6 @@ pub mod scheduling; pub use backend::CalendarBackend; pub use caldav::{CalDavAccountConfig, CalDavClient}; -pub use carddav::{CardDavClient, Contact}; pub use mock::MockCalendarBackend; pub use nlp::{NaturalLanguageScheduler, NlpError, ParsedSchedulingIntent}; pub use parser::{CalendarError, IcalParserAdapter}; diff --git a/crates/nuncio-mail/src/backend.rs b/crates/nuncio-mail/src/backend.rs index 138c5ee..a766d4d 100644 --- a/crates/nuncio-mail/src/backend.rs +++ b/crates/nuncio-mail/src/backend.rs @@ -18,9 +18,6 @@ pub trait MailBackend: Send + Sync { folder_id: &str, since_state: Option<&str>, ) -> Result<(Vec, String), MailError>; - - /// Send an email message over the configured transport. - async fn send_email(&self, email: &Email) -> Result<(), MailError>; } /// A composed outbound email message ready to send over SMTP. Deliberately diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index 36e8a3c..2b2f50c 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -742,24 +742,9 @@ impl ImapEngine { let (username, password) = match (&self.username, &self.password) { (Some(u), Some(p)) => (u.as_str(), p.as_str()), _ => { - // Credential-less fixture path used by unit tests that never - // reach a real server. The checkpoint (one past the fixture's - // UID) is internally consistent with the fixture rather than - // derived from `emails.len()`. - let mock_emails = vec![Email { - id: "imap-uid-100".to_string(), - account_id: self.account_id.clone(), - folder_id: folder_id.to_string(), - subject: "IMAP Sync Message".to_string(), - sender: "sender@nuncio.mx".to_string(), - recipient: "me@nuncio.mx".to_string(), - received_at: 1700000000, - read: true, - body_plain: Some("IMAP message body content".to_string()), - body_html: None, - attachments: Vec::new(), - }]; - return Ok((mock_emails, "101".to_string())); + return Err(MailError::AuthError( + "IMAP message sync requires credentials".to_string(), + )) } }; @@ -875,47 +860,37 @@ impl ImapEngine { #[async_trait] impl MailBackend for ImapEngine { async fn sync_folders(&self) -> Result, MailError> { - if let (Some(u), Some(p)) = (&self.username, &self.password) { - let mut session = self.socket_manager.connect_session(u, p).await?; - let mut mailboxes = session - .list(None, Some("*")) - .await - .map_err(|e| MailError::ImapError(format!("failed to list mailboxes: {}", e)))?; - - let mut folders = Vec::new(); - while let Some(mb_res) = mailboxes.next().await { - let mb = mb_res.map_err(|e| { - MailError::ImapError(format!("failed reading mailbox item: {}", e)) - })?; - let folder_name = mb.name().to_string(); - folders.push(Folder { - id: folder_name.clone(), - name: folder_name, - total_messages: 0, - unread_messages: 0, - }); + let (u, p) = match (&self.username, &self.password) { + (Some(u), Some(p)) => (u.as_str(), p.as_str()), + _ => { + return Err(MailError::AuthError( + "IMAP folder sync requires credentials".to_string(), + )) } - drop(mailboxes); - let _ = session.logout().await; - if !folders.is_empty() { - return Ok(folders); - } - } + }; + let mut session = self.socket_manager.connect_session(u, p).await?; + let mut mailboxes = session + .list(None, Some("*")) + .await + .map_err(|e| MailError::ImapError(format!("failed to list mailboxes: {}", e)))?; - Ok(vec![ - Folder { - id: "INBOX".to_string(), - name: "Inbox".to_string(), - total_messages: 10, - unread_messages: 2, - }, - Folder { - id: "Sent".to_string(), - name: "Sent Messages".to_string(), - total_messages: 5, + let mut folders = Vec::new(); + while let Some(mb_res) = mailboxes.next().await { + let mb = mb_res + .map_err(|e| MailError::ImapError(format!("failed reading mailbox item: {}", e)))?; + let folder_name = mb.name().to_string(); + folders.push(Folder { + id: folder_name.clone(), + name: folder_name, + total_messages: 0, unread_messages: 0, - }, - ]) + }); + } + drop(mailboxes); + let _ = session.logout().await; + // An empty mailbox is a genuine result, not a signal to fall back to + // placeholder folders -- the caller gets exactly what the server reported. + Ok(folders) } async fn sync_messages( @@ -925,10 +900,6 @@ impl MailBackend for ImapEngine { ) -> Result<(Vec, String), MailError> { self.sync_folder_messages(folder_id, since_state).await } - - async fn send_email(&self, _email: &Email) -> Result<(), MailError> { - Ok(()) - } } #[cfg(test)] @@ -952,20 +923,72 @@ mod tests { } #[tokio::test] - async fn imap_engine_sync_folders_and_messages() -> Result<(), MailError> { + async fn sync_folders_and_sync_messages_require_credentials() { + // Without credentials there is no server to sync against; both + // methods must surface an honest auth error rather than fabricating + // folders or messages. let engine = ImapEngine::new("acct-1", "mail.kof22.com", 993); - let folders = engine.sync_folders().await?; + + let err = engine + .sync_folders() + .await + .expect_err("credential-less sync_folders must fail"); + assert!(matches!(err, MailError::AuthError(_))); + + let err = engine + .sync_messages("INBOX", None) + .await + .expect_err("credential-less sync_messages must fail"); + assert!(matches!(err, MailError::AuthError(_))); + } + + #[tokio::test] + async fn sync_folders_lists_real_mailboxes_over_a_scripted_server() { + // Drives `MailBackend::sync_folders` against a real, plain-TCP + // loopback IMAP server scripted to answer LOGIN/LIST/LOGOUT -- proving + // the returned folder list is genuinely read off the wire. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback listener"); + let addr = listener.local_addr().expect("listener addr"); + + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept connection"); + while let Some(line) = read_scripted_line(&mut socket).await { + let tag = line.split_whitespace().next().unwrap_or("").to_string(); + let upper = line.to_ascii_uppercase(); + if upper.contains("LOGIN") { + let _ = socket + .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) + .await; + } else if upper.contains("LIST") { + let resp = format!( + "* LIST () \"/\" INBOX\r\n* LIST () \"/\" Archive\r\n{tag} OK LIST completed\r\n" + ); + let _ = socket.write_all(resp.as_bytes()).await; + } else if upper.contains("LOGOUT") { + let _ = socket + .write_all(format!("* BYE\r\n{tag} OK LOGOUT\r\n").as_bytes()) + .await; + break; + } + } + }); + + let engine = ImapEngine::with_credentials( + "acct-1", + "127.0.0.1", + addr.port(), + TlsMode::Plain, + "user", + "pass", + ); + let folders = engine.sync_folders().await.expect("sync_folders succeeds"); + server.await.expect("scripted server task completes"); + assert_eq!(folders.len(), 2); assert_eq!(folders[0].id, "INBOX"); - - let (emails, checkpoint) = engine.sync_messages("INBOX", None).await?; - // Credential-less fixture: checkpoint is one past the fixture UID. - assert_eq!(checkpoint, "101"); - assert_eq!(emails.len(), 1); - assert_eq!(emails[0].id, "imap-uid-100"); - - engine.send_email(&emails[0]).await?; - Ok(()) + assert_eq!(folders[1].id, "Archive"); } #[test] diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index 5ff136f..76fad28 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -408,44 +408,26 @@ impl JmapEngine { #[async_trait] impl MailBackend for JmapEngine { async fn sync_folders(&self) -> Result, MailError> { - if self.has_credentials() { - let session = self.discover_session().await?; - let account_id = self.resolve_account_id(&session); - let request = Self::build_mailbox_get_request(&account_id); - let raw = self.post_jmap(&session.api_url, &request).await?; - let mailboxes = Self::parse_mailbox_get_response(&raw)?; - - return Ok(mailboxes - .into_iter() - .map(|mb| Folder { - id: mb.id, - name: mb.name, - total_messages: mb.total_emails as usize, - unread_messages: mb.unread_emails as usize, - }) - .collect()); + if !self.has_credentials() { + return Err(MailError::AuthError( + "JMAP folder sync requires credentials".to_string(), + )); } + let session = self.discover_session().await?; + let account_id = self.resolve_account_id(&session); + let request = Self::build_mailbox_get_request(&account_id); + let raw = self.post_jmap(&session.api_url, &request).await?; + let mailboxes = Self::parse_mailbox_get_response(&raw)?; - Ok(vec![ - Folder { - id: "inbox".to_string(), - name: "Inbox".to_string(), - total_messages: 1, - unread_messages: 1, - }, - Folder { - id: "sent".to_string(), - name: "Sent".to_string(), - total_messages: 0, - unread_messages: 0, - }, - Folder { - id: "archive".to_string(), - name: "Archive".to_string(), - total_messages: 0, - unread_messages: 0, - }, - ]) + Ok(mailboxes + .into_iter() + .map(|mb| Folder { + id: mb.id, + name: mb.name, + total_messages: mb.total_emails as usize, + unread_messages: mb.unread_emails as usize, + }) + .collect()) } async fn sync_messages( @@ -453,46 +435,21 @@ impl MailBackend for JmapEngine { folder_id: &str, _since_state: Option<&str>, ) -> Result<(Vec, String), MailError> { - if self.has_credentials() { - let session = self.discover_session().await?; - let account_id = self.resolve_account_id(&session); - - let query_request = Self::build_email_query_request(&account_id, folder_id); - let query_raw = self.post_jmap(&session.api_url, &query_request).await?; - let ids = Self::parse_email_query_response(&query_raw)?; - - let get_request = Self::build_email_get_request(&account_id, Some(ids)); - let get_raw = self.post_jmap(&session.api_url, &get_request).await?; - return self.parse_email_get_response(&get_raw); - } - - let sample_jmap = json!({ - "state": "s-100", - "list": [ - { - "id": "jmap-msg-1", - "subject": "Welcome to JMAP Sync", - "from": [{ "email": "support@nuncio.mx" }], - "to": [{ "email": "user@nuncio.mx" }], - "receivedAt": 1700000000i64, - "isUnread": true, - "bodySnippet": "RFC 8620 / 8621 JMAP Sync Engine active." - } - ] - }) - .to_string(); - - let (emails, state) = self.parse_email_get_response(&sample_jmap)?; - Ok((emails, state)) - } - - async fn send_email(&self, email: &Email) -> Result<(), MailError> { - if email.recipient.is_empty() { - return Err(MailError::TransportFailed( - "missing recipient address".to_string(), + if !self.has_credentials() { + return Err(MailError::AuthError( + "JMAP message sync requires credentials".to_string(), )); } - Ok(()) + let session = self.discover_session().await?; + let account_id = self.resolve_account_id(&session); + + let query_request = Self::build_email_query_request(&account_id, folder_id); + let query_raw = self.post_jmap(&session.api_url, &query_request).await?; + let ids = Self::parse_email_query_response(&query_raw)?; + + let get_request = Self::build_email_get_request(&account_id, Some(ids)); + let get_raw = self.post_jmap(&session.api_url, &get_request).await?; + self.parse_email_get_response(&get_raw) } } @@ -622,20 +579,23 @@ mod tests { } #[tokio::test] - async fn uncredentialed_engine_keeps_static_fallback_behaviour() { + async fn uncredentialed_engine_requires_credentials_to_sync() { + // Without credentials there is no server to discover a session from; + // both methods must surface an honest auth error rather than + // fabricating folders or messages. let engine = JmapEngine::new("acct-1"); assert!(!engine.has_credentials()); - let folders = engine.sync_folders().await.expect("sync folders"); - assert_eq!(folders.len(), 3); - assert_eq!(folders[0].id, "inbox"); + let err = engine + .sync_folders() + .await + .expect_err("credential-less sync_folders must fail"); + assert!(matches!(err, MailError::AuthError(_))); - let (emails, state) = engine + let err = engine .sync_messages("inbox", None) .await - .expect("sync messages"); - assert_eq!(state, "s-100"); - assert_eq!(emails.len(), 1); - assert_eq!(emails[0].id, "jmap-msg-1"); + .expect_err("credential-less sync_messages must fail"); + assert!(matches!(err, MailError::AuthError(_))); } } diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index d2352fe..ec2d4f3 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -12,7 +12,6 @@ use crate::parser::MailError; pub struct MockMailBackend { folders: Arc>>, messages: Arc>>, - sent_messages: Arc>>, should_fail: Arc>, /// Every `since_state` argument this mock's `sync_messages` was called /// with, in call order, so tests can prove a caller threads the returned @@ -71,14 +70,6 @@ impl MockMailBackend { guard.push(email); } } - - /// Retrieve sent messages recorded by the mock. - pub fn sent_messages(&self) -> Vec { - self.sent_messages - .lock() - .map(|guard| guard.clone()) - .unwrap_or_default() - } } #[async_trait] @@ -144,24 +135,6 @@ impl MailBackend for MockMailBackend { .collect(); Ok((matches, returned_state)) } - - async fn send_email(&self, email: &Email) -> Result<(), MailError> { - let should_fail = self - .should_fail - .lock() - .map_err(|e| MailError::ParseFailed(e.to_string()))?; - if *should_fail { - return Err(MailError::ParseFailed( - "simulated network failure".to_string(), - )); - } - let mut sent = self - .sent_messages - .lock() - .map_err(|e| MailError::ParseFailed(e.to_string()))?; - sent.push(email.clone()); - Ok(()) - } } /// Deterministic mock [`MessageSender`] for offline testing of the outbound @@ -257,14 +230,10 @@ mod tests { assert_eq!(msgs.len(), 1); assert_eq!(state, "mock-state-token-100"); - mock.send_email(&email).await.expect("send succeeds"); - assert_eq!(mock.sent_messages().len(), 1); - // Test error simulation mock.set_should_fail(true); assert!(mock.sync_folders().await.is_err()); assert!(mock.sync_messages("inbox", None).await.is_err()); - assert!(mock.send_email(&email).await.is_err()); } fn sample_outbound_message() -> OutboundMessage { -- 2.54.0 From 025e992c11a30156bd7d7dc59d5c9ac1e051fdd4 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 10:55:21 -0500 Subject: [PATCH 09/95] feat: build the outbox remote-mutation executor (filters made live) (#263) * feat(mail): add remote-mutation ops to MailBackend (IMAP/JMAP) Add apply_mutation to the MailBackend trait plus RemoteMutationKind/ RemoteMutationSpec covering move/copy/flag/delete. IMAP addresses by UID parsed from imap-uid-{uid}, guards on the folder checkpoint's UIDVALIDITY before acting, and uses UID MOVE with a UID COPY+STORE+EXPUNGE fallback. JMAP uses Email/set (keywords/mailboxIds patch, destroy) and confirms the id was genuinely updated/destroyed. MockMailBackend records applied mutations. Scripted-duplex (IMAP) and wiremock (JMAP) tests, offline only. Refs #252 * feat(nunciod): execute outbox remote mutations for real Replace the no-op outbox worker with a real executor that drains each pending remote mutation against the resolved account's backend: mailbox mutations (move/copy/flag/unflag/delete) via MailBackend::apply_mutation, FORWARD via the SMTP MessageSender, and CALL WEBHOOK via the SSRF-hardened dispatcher (signing key provisioned from the vault). A mutation is marked completed ONLY on genuine success; transient failures retry up to MAX_RETRIES then fail, permanent failures (gone message, unknown action) fail immediately. Offline E2E proves move/flag/delete/forward/webhook end to end and that a failed op is never marked completed. Refs #252 * fix(mail): scope EXPUNGE to target UID; fail closed without UIDPLUS A bare EXPUNGE removes every \Deleted message in the mailbox, so the DELETE path and the no-MOVE-capability move fallback could collaterally destroy other messages a different client had flagged \Deleted. Replace the mailbox-wide expunge with RFC 4315 UID EXPUNGE scoped to the target UID, gated on the UIDPLUS capability; without UIDPLUS the op fails closed with an honest error rather than falling back to a destructive bare EXPUNGE. Tests now assert the exact scoped 'UID EXPUNGE ' command (a bare EXPUNGE is rejected) and cover the non-UIDPLUS fail-closed case. Also provision the webhook signing key lazily so a vault issue with that key can no longer disable the mailbox-mutation and forward paths. Refs #252 --------- Co-authored-by: James Maes --- crates/nuncio-mail/src/backend.rs | 61 ++ crates/nuncio-mail/src/imap.rs | 589 +++++++++++++++++- crates/nuncio-mail/src/jmap.rs | 160 ++++- crates/nuncio-mail/src/lib.rs | 4 +- crates/nuncio-mail/src/mock.rs | 37 +- .../nuncio-mail/tests/wiremock_jmap_test.rs | 145 ++++- crates/nuncio-store/src/vault.rs | 8 + crates/nunciod/src/lib.rs | 1 + crates/nunciod/src/main.rs | 50 +- crates/nunciod/src/outbox.rs | 466 ++++++++++++++ crates/nunciod/src/sync.rs | 2 +- .../nunciod/tests/outbox_executor_e2e_test.rs | 329 ++++++++++ 12 files changed, 1817 insertions(+), 35 deletions(-) create mode 100644 crates/nunciod/src/outbox.rs create mode 100644 crates/nunciod/tests/outbox_executor_e2e_test.rs diff --git a/crates/nuncio-mail/src/backend.rs b/crates/nuncio-mail/src/backend.rs index a766d4d..5c9bb96 100644 --- a/crates/nuncio-mail/src/backend.rs +++ b/crates/nuncio-mail/src/backend.rs @@ -5,6 +5,60 @@ use nuncio_core::model::{Attachment, Email, Folder}; use crate::parser::MailError; +/// The remote change a [`RemoteMutationSpec`] applies to one addressed message. +/// +/// This is the closed vocabulary of genuinely-remote filter actions that the +/// outbox drains against a real server. `MARK READ`/`MARK UNREAD` are applied +/// locally to the store when a filter matches, so they are deliberately NOT +/// represented here -- only actions that mutate server-side state are. +/// `FORWARD` and `CALL WEBHOOK` are also excluded: they are not backend +/// mutations (one composes an outbound message, the other calls an HTTP +/// endpoint), so they are executed through the send/webhook seams instead. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RemoteMutationKind { + /// Add (`value == true`) or remove (`value == false`) the "flagged"/ + /// starred keyword: IMAP `\Flagged`, JMAP `$flagged`. Covers `FLAG`/`UNFLAG`. + SetFlagged { + /// Whether the message should end up flagged. + value: bool, + }, + /// Move the message from its source folder to `to_folder`. + Move { + /// Destination folder (IMAP mailbox name / JMAP mailbox id). + to_folder: String, + }, + /// Copy the message into `to_folder`, leaving the source copy in place. + Copy { + /// Destination folder (IMAP mailbox name / JMAP mailbox id). + to_folder: String, + }, + /// Delete the message (IMAP `\Deleted` + `EXPUNGE`; JMAP `destroy`). + Delete, +} + +/// A fully-addressed remote mutation for a single message: which message +/// (protocol-native id), the folder it currently lives in, that folder's +/// stored sync checkpoint (carrying IMAP UIDVALIDITY), and the change to +/// apply. +/// +/// `folder_checkpoint` is the value persisted by a prior sync +/// (`"{uidvalidity}:{uidnext}"` for IMAP). The IMAP backend uses it to refuse +/// to act when the mailbox's current UIDVALIDITY no longer matches -- a +/// renumbered mailbox reassigns UIDs, so the stored UID would otherwise +/// address the wrong message. Protocols that do not use UIDVALIDITY (JMAP) +/// ignore it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RemoteMutationSpec { + /// Protocol-native message id (IMAP `imap-uid-{uid}`, JMAP object id). + pub message_id: String, + /// Folder the message currently lives in. + pub folder_id: String, + /// The source folder's stored sync checkpoint, if any. + pub folder_checkpoint: Option, + /// The change to apply. + pub kind: RemoteMutationKind, +} + /// Protocol-agnostic mail backend engine trait implemented by JMAP and IMAP engines. #[async_trait] pub trait MailBackend: Send + Sync { @@ -18,6 +72,13 @@ pub trait MailBackend: Send + Sync { folder_id: &str, since_state: Option<&str>, ) -> Result<(Vec, String), MailError>; + + /// Apply a remote mutation (move/copy/flag/delete) to a single message + /// against the real server. MUST return `Ok(())` only when the server + /// genuinely applied the change -- implementations must never fabricate + /// success, and must refuse to act (returning an error) when the target + /// message cannot be safely identified (e.g. an IMAP UIDVALIDITY mismatch). + async fn apply_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError>; } /// A composed outbound email message ready to send over SMTP. Deliberately diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index 2b2f50c..adfbf63 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -31,9 +31,36 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); /// honest timeout rather than hanging the sync indefinitely. const FETCH_ITEM_TIMEOUT: Duration = Duration::from_secs(60); -use crate::backend::MailBackend; +use crate::backend::{MailBackend, RemoteMutationKind, RemoteMutationSpec}; use crate::parser::{MailError, MimeParserAdapter}; +/// Prefix an IMAP [`Email::id`] carries in front of its raw numeric UID (see +/// [`ImapEngine::sync_folder_messages_with_session`], which formats every id as +/// `"imap-uid-{uid}"`). A remote mutation must recover the UID from the id to +/// address the message, so the prefix is defined once here. +const IMAP_UID_ID_PREFIX: &str = "imap-uid-"; + +/// Recover the raw IMAP UID from an [`Email::id`] of the form +/// `"imap-uid-{uid}"`. Returns `None` for any id that does not carry the +/// expected prefix and a valid non-zero `u32` UID, so a mutation against a +/// malformed/foreign id fails honestly rather than addressing UID 0. +fn parse_imap_uid(message_id: &str) -> Option { + let raw = message_id.strip_prefix(IMAP_UID_ID_PREFIX)?; + let uid = raw.trim().parse::().ok()?; + (uid >= 1).then_some(uid) +} + +/// Extract the UIDVALIDITY component of a stored folder checkpoint +/// (`"{uidvalidity}:{uidnext}"`). Returns `None` for a missing, malformed, or +/// legacy bare-number checkpoint -- the caller treats an absent stored +/// UIDVALIDITY as "cannot prove the mailbox has not been renumbered" and +/// refuses to act. +fn checkpoint_uid_validity(checkpoint: Option<&str>) -> Option { + let raw = checkpoint?; + let (validity, _uid) = raw.split_once(CHECKPOINT_DELIM)?; + validity.trim().parse::().ok() +} + /// Delimiter between the two components of a folder sync checkpoint. The /// checkpoint state string is `"{uidvalidity}{DELIM}{uidnext}"` (e.g. /// `"42:105"`): the mailbox's UIDVALIDITY at the time it was written, then the @@ -759,6 +786,222 @@ impl ImapEngine { result } + /// Apply a remote mutation over an active IMAP session, selecting the + /// source folder and verifying its UIDVALIDITY against the stored + /// checkpoint BEFORE issuing any command that could touch the wrong + /// message. + /// + /// The UIDVALIDITY guard (RFC 3501 s2.3.1.1) is the safety-critical step: a + /// stored IMAP UID is only meaningful under the UIDVALIDITY it was captured + /// with. If the mailbox was recreated its UIDVALIDITY changes and UIDs are + /// reassigned, so acting on the old UID would mutate a different message. A + /// mismatch -- or a stored checkpoint that carries no UIDVALIDITY to + /// compare against, or a server that reports none -- is refused with an + /// honest [`MailError::ImapError`]; the mutation is never applied blind. + /// + /// `MOVE` uses `UID MOVE` (RFC 6851) when the server advertises the `MOVE` + /// capability, else falls back to the equivalent `UID COPY` + `UID STORE + /// +FLAGS (\Deleted)` + UID-scoped `UID EXPUNGE`. `DELETE` is `UID STORE + /// +FLAGS (\Deleted)` + `UID EXPUNGE`. The expunge is always scoped to the + /// target UID via RFC 4315 (never a mailbox-wide `EXPUNGE`, which would + /// destroy other `\Deleted` messages) and requires `UIDPLUS`, failing + /// closed otherwise. Each command's tagged result is checked, so only a + /// genuine server success returns `Ok(())`. + pub async fn apply_mutation_with_session( + &self, + spec: &RemoteMutationSpec, + session: &mut async_imap::Session, + ) -> Result<(), MailError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, + { + let uid = parse_imap_uid(&spec.message_id).ok_or_else(|| { + MailError::ImapError(format!( + "cannot parse an IMAP UID from message id '{}'", + spec.message_id + )) + })?; + + let mailbox = session.select(&spec.folder_id).await.map_err(|e| { + MailError::ImapError(format!( + "failed to select folder '{}': {}", + spec.folder_id, e + )) + })?; + + // Refuse to act unless the mailbox's current UIDVALIDITY provably + // matches the one the stored UID was captured under. + let stored_validity = checkpoint_uid_validity(spec.folder_checkpoint.as_deref()); + match (stored_validity, mailbox.uid_validity) { + (Some(stored), Some(current)) if stored == current => {} + (stored, current) => { + return Err(MailError::ImapError(format!( + "refusing to mutate message '{}' in folder '{}': UIDVALIDITY guard failed \ + (stored {stored:?}, current {current:?}); the stored UID may no longer \ + address the intended message", + spec.message_id, spec.folder_id + ))); + } + } + + let uid_set = uid.to_string(); + match &spec.kind { + RemoteMutationKind::SetFlagged { value } => { + let op = if *value { "+FLAGS" } else { "-FLAGS" }; + self.uid_store_flags(session, &uid_set, op, "\\Flagged") + .await + } + RemoteMutationKind::Copy { to_folder } => { + session.uid_copy(&uid_set, to_folder).await.map_err(|e| { + MailError::ImapError(format!( + "UID COPY of '{}' to '{}' failed: {}", + spec.message_id, to_folder, e + )) + }) + } + RemoteMutationKind::Move { to_folder } => { + self.uid_move(session, &uid_set, to_folder, &spec.message_id) + .await + } + RemoteMutationKind::Delete => { + self.uid_store_flags(session, &uid_set, "+FLAGS", "\\Deleted") + .await?; + self.uid_expunge_scoped(session, &uid_set).await + } + } + } + + /// Issue a `UID STORE {uid} {op} ({flag})` and drain its untagged `FETCH` + /// responses, surfacing any protocol error. `op` is `+FLAGS`/`-FLAGS`. + async fn uid_store_flags( + &self, + session: &mut async_imap::Session, + uid_set: &str, + op: &str, + flag: &str, + ) -> Result<(), MailError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, + { + let query = format!("{op} ({flag})"); + let updates = session.uid_store(uid_set, &query).await.map_err(|e| { + MailError::ImapError(format!("UID STORE {query} for uid {uid_set} failed: {e}")) + })?; + tokio::pin!(updates); + while let Some(item) = updates.next().await { + item.map_err(|e| { + MailError::ImapError(format!("reading UID STORE response failed: {e}")) + })?; + } + Ok(()) + } + + /// Permanently remove ONLY the target `uid_set` from the selected folder + /// via RFC 4315 `UID EXPUNGE`, draining its untagged `EXPUNGE` responses. + /// + /// A bare `EXPUNGE` (RFC 3501) removes EVERY `\Deleted` message in the + /// mailbox, so it would collaterally destroy any other message a different + /// client had already flagged `\Deleted`. `UID EXPUNGE` is scoped to + /// exactly the given UIDs, so it is the only safe expunge in a shared + /// mailbox. It requires the `UIDPLUS` capability; if the server does not + /// advertise it, this fails closed with an honest error rather than + /// falling back to the destructive mailbox-wide `EXPUNGE`. + async fn uid_expunge_scoped( + &self, + session: &mut async_imap::Session, + uid_set: &str, + ) -> Result<(), MailError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, + { + let caps = session + .capabilities() + .await + .map_err(|e| MailError::ImapError(format!("CAPABILITY query failed: {e}")))?; + let has_uidplus = caps.has_str("UIDPLUS"); + drop(caps); + if !has_uidplus { + return Err(MailError::ImapError(format!( + "refusing to expunge uid {uid_set}: server does not advertise UIDPLUS, so a \ + UID-scoped EXPUNGE (RFC 4315) is unavailable; a bare EXPUNGE could destroy \ + other \\Deleted messages in the folder" + ))); + } + + let expunged = session + .uid_expunge(uid_set) + .await + .map_err(|e| MailError::ImapError(format!("UID EXPUNGE {uid_set} failed: {e}")))?; + tokio::pin!(expunged); + while let Some(item) = expunged.next().await { + item.map_err(|e| { + MailError::ImapError(format!("reading UID EXPUNGE response failed: {e}")) + })?; + } + Ok(()) + } + + /// Move a message by UID, preferring RFC 6851 `UID MOVE` and falling back + /// to `UID COPY` + `UID STORE +FLAGS (\Deleted)` + UID-scoped `UID EXPUNGE` + /// when the server does not advertise the `MOVE` capability. The fallback's + /// expunge is scoped to the target UID (never a mailbox-wide `EXPUNGE`) and + /// requires `UIDPLUS`, failing closed otherwise. + async fn uid_move( + &self, + session: &mut async_imap::Session, + uid_set: &str, + to_folder: &str, + message_id: &str, + ) -> Result<(), MailError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, + { + let caps = session + .capabilities() + .await + .map_err(|e| MailError::ImapError(format!("CAPABILITY query failed: {e}")))?; + let has_move = caps.has_str("MOVE"); + drop(caps); + + if has_move { + return session.uid_mv(uid_set, to_folder).await.map_err(|e| { + MailError::ImapError(format!( + "UID MOVE of '{message_id}' to '{to_folder}' failed: {e}" + )) + }); + } + + session.uid_copy(uid_set, to_folder).await.map_err(|e| { + MailError::ImapError(format!( + "UID COPY (move fallback) of '{message_id}' to '{to_folder}' failed: {e}" + )) + })?; + self.uid_store_flags(session, uid_set, "+FLAGS", "\\Deleted") + .await?; + self.uid_expunge_scoped(session, uid_set).await + } + + /// Apply a remote mutation over a freshly authenticated session. See + /// [`Self::apply_mutation_with_session`] for the UIDVALIDITY guard and the + /// per-action protocol commands. + pub async fn apply_remote_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError> { + let (username, password) = match (&self.username, &self.password) { + (Some(u), Some(p)) => (u.as_str(), p.as_str()), + _ => { + return Err(MailError::AuthError( + "IMAP remote mutation requires credentials".to_string(), + )) + } + }; + let mut session = self + .socket_manager + .connect_session(username, password) + .await?; + let result = self.apply_mutation_with_session(spec, &mut session).await; + let _ = session.logout().await; + result + } + /// Listen for real-time IMAP IDLE notification events on an active IMAP session. pub async fn listen_idle_session( &self, @@ -900,6 +1143,10 @@ impl MailBackend for ImapEngine { ) -> Result<(Vec, String), MailError> { self.sync_folder_messages(folder_id, since_state).await } + + async fn apply_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError> { + self.apply_remote_mutation(spec).await + } } #[cfg(test)] @@ -1327,6 +1574,346 @@ mod tests { server.abort(); } + #[test] + fn parse_imap_uid_recovers_uid_and_rejects_malformed_ids() { + assert_eq!(parse_imap_uid("imap-uid-42"), Some(42)); + assert_eq!(parse_imap_uid("imap-uid-1"), Some(1)); + // UID 0 is not a valid IMAP UID; a bare/foreign id yields nothing. + assert_eq!(parse_imap_uid("imap-uid-0"), None); + assert_eq!(parse_imap_uid("42"), None); + assert_eq!(parse_imap_uid("jmap-object-id"), None); + } + + #[test] + fn checkpoint_uid_validity_extracts_only_a_well_formed_validity() { + assert_eq!(checkpoint_uid_validity(Some("7:105")), Some(7)); + assert_eq!(checkpoint_uid_validity(Some(" 7 : 105 ")), Some(7)); + assert_eq!(checkpoint_uid_validity(None), None); + // A legacy bare-number checkpoint carries no validity to compare. + assert_eq!(checkpoint_uid_validity(Some("105")), None); + assert_eq!(checkpoint_uid_validity(Some("x:105")), None); + } + + /// Drive a mutation over a scripted IMAP server (in-memory duplex), + /// capturing every command line the client issued so the test can assert + /// on the exact protocol commands. `advertise_move`/`advertise_uidplus` + /// control whether the scripted `CAPABILITY` response includes `MOVE`/ + /// `UIDPLUS`; `select_uid_validity` is the UIDVALIDITY the scripted + /// `SELECT` reports. + async fn run_scripted_mutation( + spec: &RemoteMutationSpec, + advertise_move: bool, + advertise_uidplus: bool, + select_uid_validity: u32, + ) -> (Result<(), MailError>, Vec) { + let (client_io, server_io) = tokio::io::duplex(8192); + let commands = Arc::new(std::sync::Mutex::new(Vec::::new())); + let server_commands = commands.clone(); + + let server = tokio::spawn(async move { + let mut io = server_io; + while let Some(line) = read_scripted_line(&mut io).await { + if let Ok(mut g) = server_commands.lock() { + g.push(line.clone()); + } + let tag = line.split_whitespace().next().unwrap_or("").to_string(); + let upper = line.to_ascii_uppercase(); + if upper.contains("LOGIN") { + let _ = io + .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) + .await; + } else if upper.contains("SELECT") { + let resp = format!( + "* FLAGS (\\Seen \\Flagged \\Deleted)\r\n* 3 EXISTS\r\n* 0 RECENT\r\n\ + * OK [UIDVALIDITY {select_uid_validity}] ok\r\n* OK [UIDNEXT 105] ok\r\n\ + {tag} OK [READ-WRITE] SELECT done\r\n" + ); + let _ = io.write_all(resp.as_bytes()).await; + } else if upper.contains("CAPABILITY") { + let mut caps = String::from("IMAP4rev1"); + if advertise_move { + caps.push_str(" MOVE"); + } + if advertise_uidplus { + caps.push_str(" UIDPLUS"); + } + let _ = io + .write_all( + format!("* CAPABILITY {caps}\r\n{tag} OK CAPABILITY done\r\n") + .as_bytes(), + ) + .await; + } else if upper.contains("UID EXPUNGE") { + let _ = io + .write_all(format!("{tag} OK UID EXPUNGE completed\r\n").as_bytes()) + .await; + } else if upper.contains("UID STORE") { + let _ = io + .write_all(format!("{tag} OK STORE completed\r\n").as_bytes()) + .await; + } else if upper.contains("UID COPY") { + let _ = io + .write_all(format!("{tag} OK COPY completed\r\n").as_bytes()) + .await; + } else if upper.contains("UID MOVE") { + let _ = io + .write_all(format!("{tag} OK MOVE completed\r\n").as_bytes()) + .await; + } else if upper.contains("EXPUNGE") { + let _ = io + .write_all(format!("{tag} OK EXPUNGE completed\r\n").as_bytes()) + .await; + } else if upper.contains("LOGOUT") { + let _ = io + .write_all(format!("* BYE\r\n{tag} OK LOGOUT\r\n").as_bytes()) + .await; + break; + } + } + }); + + let client = async_imap::Client::new(client_io); + let mut session = client + .login("user", "pass") + .await + .map_err(|(e, _)| e) + .expect("scripted login succeeds"); + let engine = ImapEngine::new("acct-1", "example.test", 993); + let result = engine.apply_mutation_with_session(spec, &mut session).await; + drop(session); + let _ = server.await; + + let captured = commands.lock().expect("lock captured commands").clone(); + (result, captured) + } + + /// True if `line` is a bare mailbox-wide `EXPUNGE` (RFC 3501), NOT a + /// UID-scoped `UID EXPUNGE` (RFC 4315). Lets a test prove the destructive + /// mailbox-wide form is never issued (a plain `contains("EXPUNGE")` would + /// be satisfied by the safe `UID EXPUNGE` too). + fn is_bare_expunge(line: &str) -> bool { + let u = line.to_ascii_uppercase(); + u.contains("EXPUNGE") && !u.contains("UID EXPUNGE") + } + + fn imap_spec(kind: RemoteMutationKind, checkpoint: Option<&str>) -> RemoteMutationSpec { + RemoteMutationSpec { + message_id: "imap-uid-42".to_string(), + folder_id: "INBOX".to_string(), + folder_checkpoint: checkpoint.map(str::to_string), + kind, + } + } + + #[tokio::test] + async fn imap_apply_mutation_flag_issues_uid_store_add_flags() { + let (result, commands) = run_scripted_mutation( + &imap_spec( + RemoteMutationKind::SetFlagged { value: true }, + Some("1:105"), + ), + false, + true, + 1, + ) + .await; + result.expect("flag mutation succeeds"); + assert!( + commands + .iter() + .any(|c| c.contains("UID STORE 42 +FLAGS (\\Flagged)")), + "expected a UID STORE +FLAGS (\\Flagged), got: {commands:?}" + ); + } + + #[tokio::test] + async fn imap_apply_mutation_move_uses_uid_move_when_server_advertises_move() { + let (result, commands) = run_scripted_mutation( + &imap_spec( + RemoteMutationKind::Move { + to_folder: "Archive".to_string(), + }, + Some("1:105"), + ), + true, + true, + 1, + ) + .await; + result.expect("move mutation succeeds"); + assert!( + commands.iter().any(|c| c.contains("UID MOVE 42")), + "expected a UID MOVE, got: {commands:?}" + ); + assert!( + !commands.iter().any(|c| c.contains("UID COPY")), + "must not fall back to COPY when MOVE is advertised: {commands:?}" + ); + } + + #[tokio::test] + async fn imap_apply_mutation_move_falls_back_to_copy_store_uid_expunge_without_move_capability() + { + let (result, commands) = run_scripted_mutation( + &imap_spec( + RemoteMutationKind::Move { + to_folder: "Archive".to_string(), + }, + Some("1:105"), + ), + false, + true, + 1, + ) + .await; + result.expect("move fallback succeeds"); + assert!(commands.iter().any(|c| c.contains("UID COPY 42"))); + assert!(commands + .iter() + .any(|c| c.contains("UID STORE 42 +FLAGS (\\Deleted)"))); + // The expunge MUST be scoped to the target UID, never a mailbox-wide + // bare EXPUNGE that could destroy other \Deleted messages. + assert!( + commands.iter().any(|c| c.contains("UID EXPUNGE 42")), + "fallback must issue a UID-scoped UID EXPUNGE 42, got: {commands:?}" + ); + assert!( + !commands.iter().any(|c| is_bare_expunge(c)), + "fallback must NOT issue a bare mailbox-wide EXPUNGE, got: {commands:?}" + ); + assert!( + !commands.iter().any(|c| c.contains("UID MOVE")), + "must not issue UID MOVE without the capability: {commands:?}" + ); + } + + #[tokio::test] + async fn imap_apply_mutation_delete_stores_deleted_and_uid_expunges_the_target_only() { + let (result, commands) = run_scripted_mutation( + &imap_spec(RemoteMutationKind::Delete, Some("1:105")), + false, + true, + 1, + ) + .await; + result.expect("delete mutation succeeds"); + assert!(commands + .iter() + .any(|c| c.contains("UID STORE 42 +FLAGS (\\Deleted)"))); + // The expunge MUST target exactly the deleted UID (RFC 4315), never a + // mailbox-wide bare EXPUNGE that would also destroy other client's + // \Deleted messages in the same folder. + assert!( + commands.iter().any(|c| c.contains("UID EXPUNGE 42")), + "delete must issue a UID-scoped UID EXPUNGE 42, got: {commands:?}" + ); + assert!( + !commands.iter().any(|c| is_bare_expunge(c)), + "delete must NOT issue a bare mailbox-wide EXPUNGE, got: {commands:?}" + ); + } + + #[tokio::test] + async fn imap_apply_mutation_delete_fails_closed_without_uidplus_and_issues_no_bare_expunge() { + // Without UIDPLUS there is no UID-scoped expunge; the op MUST fail + // closed rather than fall back to a destructive mailbox-wide EXPUNGE. + let (result, commands) = run_scripted_mutation( + &imap_spec(RemoteMutationKind::Delete, Some("1:105")), + false, + false, + 1, + ) + .await; + assert!( + matches!(result, Err(MailError::ImapError(_))), + "delete without UIDPLUS must fail closed, got: {result:?}" + ); + assert!( + !commands + .iter() + .any(|c| c.to_ascii_uppercase().contains("EXPUNGE")), + "no expunge of any kind may be issued when UIDPLUS is unavailable, got: {commands:?}" + ); + } + + #[tokio::test] + async fn imap_apply_mutation_move_fallback_fails_closed_without_uidplus() { + // The no-MOVE-capability move fallback also relies on a UID-scoped + // expunge; without UIDPLUS it must fail closed and issue no bare + // EXPUNGE (the preceding COPY/STORE are non-destructive to others). + let (result, commands) = run_scripted_mutation( + &imap_spec( + RemoteMutationKind::Move { + to_folder: "Archive".to_string(), + }, + Some("1:105"), + ), + false, + false, + 1, + ) + .await; + assert!( + matches!(result, Err(MailError::ImapError(_))), + "move fallback without UIDPLUS must fail closed, got: {result:?}" + ); + assert!( + !commands + .iter() + .any(|c| c.to_ascii_uppercase().contains("EXPUNGE")), + "no expunge of any kind may be issued when UIDPLUS is unavailable, got: {commands:?}" + ); + } + + #[tokio::test] + async fn imap_apply_mutation_refuses_to_act_on_a_uidvalidity_mismatch() { + // The stored checkpoint captured the UID under UIDVALIDITY 1, but the + // mailbox now reports 2 (renumbered). The mutation MUST be refused and + // NO mutating command (STORE/COPY/MOVE/EXPUNGE) may be issued. + let (result, commands) = run_scripted_mutation( + &imap_spec(RemoteMutationKind::Delete, Some("1:105")), + false, + true, + 2, + ) + .await; + let err = result.expect_err("a UIDVALIDITY mismatch must be refused"); + assert!(matches!(err, MailError::ImapError(_))); + assert!( + !commands.iter().any(|c| { + let u = c.to_ascii_uppercase(); + u.contains("UID STORE") + || u.contains("UID COPY") + || u.contains("UID MOVE") + || u.contains("EXPUNGE") + }), + "no mutating command may be issued on a UIDVALIDITY mismatch, got: {commands:?}" + ); + } + + #[tokio::test] + async fn imap_apply_mutation_refuses_when_no_checkpoint_proves_uidvalidity() { + // With no stored checkpoint there is no UIDVALIDITY to prove the UID + // still addresses the intended message, so the op is refused. + let (result, _commands) = + run_scripted_mutation(&imap_spec(RemoteMutationKind::Delete, None), false, true, 1) + .await; + assert!(matches!( + result.expect_err("missing checkpoint must be refused"), + MailError::ImapError(_) + )); + } + + #[tokio::test] + async fn imap_apply_remote_mutation_requires_credentials() { + let engine = ImapEngine::new("acct-1", "example.test", 993); + let err = engine + .apply_remote_mutation(&imap_spec(RemoteMutationKind::Delete, Some("1:105"))) + .await + .expect_err("credential-less mutation must fail"); + assert!(matches!(err, MailError::AuthError(_))); + } + #[tokio::test] async fn with_tls_mode_records_mode_and_resolves_default_ports() { // A `0` port resolves to the conventional default for each mode. diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index 76fad28..b7a1e0a 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -5,7 +5,7 @@ use nuncio_core::model::{Email, Folder}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use crate::backend::MailBackend; +use crate::backend::{MailBackend, RemoteMutationKind, RemoteMutationSpec}; use crate::parser::MailError; /// JMAP Session Object (RFC 8620 Section 2). @@ -203,6 +203,100 @@ impl JmapEngine { }) } + /// Build a JSON-RPC `Email/set` request applying `spec` to a single + /// message (RFC 8621 s4.6). A `Move`/`Copy`/`SetFlagged` becomes an + /// `update` patch keyed by the JMAP object id; a `Delete` becomes a + /// `destroy`. + /// + /// `Move` replaces the message's mailbox set with the destination (removing + /// the source), whereas `Copy` adds the destination via a `mailboxIds/{id}` + /// patch, leaving the existing membership intact. Flag toggles patch the + /// `$flagged` keyword (`true` to set, `null` to clear), the JMAP equivalent + /// of IMAP `\Flagged`. + pub fn build_email_set_request(account_id: &str, spec: &RemoteMutationSpec) -> Value { + let id = &spec.message_id; + let (update, destroy) = match &spec.kind { + RemoteMutationKind::SetFlagged { value } => { + let keyword = if *value { json!(true) } else { Value::Null }; + (json!({ id: { "keywords/$flagged": keyword } }), json!([])) + } + RemoteMutationKind::Move { to_folder } => ( + json!({ id: { "mailboxIds": { to_folder: true } } }), + json!([]), + ), + RemoteMutationKind::Copy { to_folder } => ( + json!({ id: { format!("mailboxIds/{to_folder}"): true } }), + json!([]), + ), + RemoteMutationKind::Delete => (json!({}), json!([id])), + }; + + json!({ + "using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], + "methodCalls": [ + [ + "Email/set", + { + "accountId": account_id, + "update": update, + "destroy": destroy + }, + "c1" + ] + ] + }) + } + + /// Confirm a JMAP `Email/set` response genuinely applied the mutation to + /// `message_id`: the id must appear in `updated`/`destroyed` and must NOT + /// appear in `notUpdated`/`notDestroyed`. Anything else -- a server-side + /// rejection, or an id the server silently ignored -- surfaces as an honest + /// error so the outbox never marks a rejected mutation completed. + pub fn confirm_email_set_applied( + raw_json: &str, + message_id: &str, + expect_destroy: bool, + ) -> Result<(), MailError> { + let val: Value = serde_json::from_str(raw_json) + .map_err(|e| MailError::ParseFailed(format!("invalid JMAP JSON response: {e}")))?; + let payload = Self::extract_method_response_payload(&val)?; + + // A rejection is reported in notUpdated/notDestroyed keyed by id; treat + // its presence as the authoritative failure signal. + let rejected = payload + .get(if expect_destroy { + "notDestroyed" + } else { + "notUpdated" + }) + .and_then(|v| v.as_object()) + .is_some_and(|m| m.contains_key(message_id)); + if rejected { + return Err(MailError::OperationFailed(format!( + "JMAP Email/set rejected mutation for message '{message_id}'" + ))); + } + + let applied = if expect_destroy { + payload + .get("destroyed") + .and_then(|v| v.as_array()) + .is_some_and(|ids| ids.iter().any(|v| v.as_str() == Some(message_id))) + } else { + payload + .get("updated") + .and_then(|v| v.as_object()) + .is_some_and(|m| m.contains_key(message_id)) + }; + if applied { + Ok(()) + } else { + Err(MailError::OperationFailed(format!( + "JMAP Email/set did not confirm the mutation for message '{message_id}'" + ))) + } + } + /// Build JSON-RPC request invocation for `Email/changes` (differential sync). pub fn build_email_changes_request(account_id: &str, since_state: &str) -> Value { json!({ @@ -451,6 +545,21 @@ impl MailBackend for JmapEngine { let get_raw = self.post_jmap(&session.api_url, &get_request).await?; self.parse_email_get_response(&get_raw) } + + async fn apply_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError> { + if !self.has_credentials() { + return Err(MailError::AuthError( + "JMAP remote mutation requires credentials".to_string(), + )); + } + let session = self.discover_session().await?; + let account_id = self.resolve_account_id(&session); + + let request = Self::build_email_set_request(&account_id, spec); + let raw = self.post_jmap(&session.api_url, &request).await?; + let expect_destroy = matches!(spec.kind, RemoteMutationKind::Delete); + Self::confirm_email_set_applied(&raw, &spec.message_id, expect_destroy) + } } #[cfg(test)] @@ -578,6 +687,49 @@ mod tests { assert_eq!(email.body_snippet, Some("hello world".to_string())); } + fn spec(kind: RemoteMutationKind) -> RemoteMutationSpec { + RemoteMutationSpec { + message_id: "m-1".to_string(), + folder_id: "mb-inbox".to_string(), + folder_checkpoint: None, + kind, + } + } + + #[test] + fn build_email_set_request_copy_adds_target_without_removing_source() { + let req = JmapEngine::build_email_set_request( + "acct-1", + &spec(RemoteMutationKind::Copy { + to_folder: "mb-archive".to_string(), + }), + ); + let call = &req["methodCalls"][0]; + assert_eq!(call[0], "Email/set"); + // A copy patches a single mailboxIds/{id} pointer to true, leaving the + // rest of the message's mailbox membership untouched. + assert_eq!(call[1]["update"]["m-1"]["mailboxIds/mb-archive"], true); + } + + #[test] + fn build_email_set_request_unflag_clears_keyword_with_null() { + let req = JmapEngine::build_email_set_request( + "acct-1", + &spec(RemoteMutationKind::SetFlagged { value: false }), + ); + assert!(req["methodCalls"][0][1]["update"]["m-1"]["keywords/$flagged"].is_null()); + } + + #[test] + fn confirm_email_set_applied_requires_the_id_in_the_updated_map() { + let ok = r#"{"methodResponses":[["Email/set",{"updated":{"m-1":null}},"c1"]]}"#; + assert!(JmapEngine::confirm_email_set_applied(ok, "m-1", false).is_ok()); + + // An empty updated map is NOT success -- the server silently ignored it. + let ignored = r#"{"methodResponses":[["Email/set",{"updated":{}},"c1"]]}"#; + assert!(JmapEngine::confirm_email_set_applied(ignored, "m-1", false).is_err()); + } + #[tokio::test] async fn uncredentialed_engine_requires_credentials_to_sync() { // Without credentials there is no server to discover a session from; @@ -597,5 +749,11 @@ mod tests { .await .expect_err("credential-less sync_messages must fail"); assert!(matches!(err, MailError::AuthError(_))); + + let err = engine + .apply_mutation(&spec(RemoteMutationKind::Delete)) + .await + .expect_err("credential-less apply_mutation must fail"); + assert!(matches!(err, MailError::AuthError(_))); } } diff --git a/crates/nuncio-mail/src/lib.rs b/crates/nuncio-mail/src/lib.rs index 3d2b6aa..cc8bfc9 100644 --- a/crates/nuncio-mail/src/lib.rs +++ b/crates/nuncio-mail/src/lib.rs @@ -10,7 +10,9 @@ pub mod mock; pub mod parser; pub mod smtp; -pub use backend::{MailBackend, MessageSender, OutboundMessage}; +pub use backend::{ + MailBackend, MessageSender, OutboundMessage, RemoteMutationKind, RemoteMutationSpec, +}; pub use imap::{IdleSocketState, ImapDualSocketManager, ImapEngine}; pub use jmap::JmapEngine; pub use mock::{MockMailBackend, MockMessageSender}; diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index ec2d4f3..4a581d5 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use nuncio_core::model::{Email, Folder}; use std::sync::{Arc, Mutex}; -use crate::backend::{MailBackend, MessageSender, OutboundMessage}; +use crate::backend::{MailBackend, MessageSender, OutboundMessage, RemoteMutationSpec}; use crate::parser::MailError; /// Thread-safe mock mail backend for offline testing. @@ -19,6 +19,10 @@ pub struct MockMailBackend { since_state_calls: Arc>>>, /// The checkpoint token this mock returns from `sync_messages`. returned_state: Arc>, + /// Every [`RemoteMutationSpec`] passed to `apply_mutation`, in call order, + /// so a daemon E2E can prove the outbox executor genuinely invoked the + /// backend op (not a fabricated completion). + applied_mutations: Arc>>, } impl MockMailBackend { @@ -70,6 +74,17 @@ impl MockMailBackend { guard.push(email); } } + + /// Every [`RemoteMutationSpec`] this mock's `apply_mutation` was called + /// with, in call order. A failing configuration (`set_should_fail(true)`) + /// records nothing, so a test can prove a failed op is never counted as + /// applied. + pub fn applied_mutations(&self) -> Vec { + self.applied_mutations + .lock() + .map(|guard| guard.clone()) + .unwrap_or_default() + } } #[async_trait] @@ -135,6 +150,26 @@ impl MailBackend for MockMailBackend { .collect(); Ok((matches, returned_state)) } + + async fn apply_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError> { + let should_fail = self + .should_fail + .lock() + .map_err(|e| MailError::ParseFailed(e.to_string()))?; + if *should_fail { + return Err(MailError::ImapError( + "simulated remote mutation failure".to_string(), + )); + } + // Record only genuinely-applied mutations, so a test can distinguish a + // real op from a fabricated completion. + let mut applied = self + .applied_mutations + .lock() + .map_err(|e| MailError::ParseFailed(e.to_string()))?; + applied.push(spec.clone()); + Ok(()) + } } /// Deterministic mock [`MessageSender`] for offline testing of the outbound diff --git a/crates/nuncio-mail/tests/wiremock_jmap_test.rs b/crates/nuncio-mail/tests/wiremock_jmap_test.rs index fabf20e..f7773af 100644 --- a/crates/nuncio-mail/tests/wiremock_jmap_test.rs +++ b/crates/nuncio-mail/tests/wiremock_jmap_test.rs @@ -2,9 +2,9 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use nuncio_mail::{JmapEngine, MailBackend}; +use nuncio_mail::{JmapEngine, MailBackend, RemoteMutationKind, RemoteMutationSpec}; use serde_json::json; -use wiremock::matchers::{method, path}; +use wiremock::matchers::{body_partial_json, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; #[tokio::test] @@ -237,3 +237,144 @@ async fn jmap_engine_sync_folders_surfaces_session_discovery_failure() { .expect_err("session discovery failure must not fall back to canned folders"); assert!(err.to_string().contains("network I/O error")); } + +/// Mount JMAP session discovery so `apply_mutation` can reach the mock `/jmap/api`. +async fn mount_session(mock_server: &MockServer) { + let session_body = json!({ + "username": "james.maes@kof22.com", + "primaryAccounts": { "urn:ietf:params:jmap:mail": "acct-100" }, + "apiUrl": format!("{}/jmap/api", mock_server.uri()), + "state": "session-state-1" + }); + Mock::given(method("GET")) + .and(path("/.well-known/jmap")) + .respond_with(ResponseTemplate::new(200).set_body_json(&session_body)) + .mount(mock_server) + .await; +} + +fn jmap_engine(mock_server: &MockServer) -> JmapEngine { + JmapEngine::with_credentials( + "acct-100", + &mock_server.uri(), + "james.maes@kof22.com", + "wiremock-password", + ) +} + +fn spec(kind: RemoteMutationKind) -> RemoteMutationSpec { + RemoteMutationSpec { + message_id: "jmap-msg-1".to_string(), + folder_id: "mb-inbox".to_string(), + folder_checkpoint: None, + kind, + } +} + +#[tokio::test] +async fn jmap_apply_mutation_flag_issues_keywords_patch_and_confirms_success() { + let mock_server = MockServer::start().await; + mount_session(&mock_server).await; + + // Assert the exact Email/set update patch reaches the server: a $flagged + // keyword set to true on the target id. + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(json!({ + "methodCalls": [[ + "Email/set", + { "update": { "jmap-msg-1": { "keywords/$flagged": true } } }, + "c1" + ]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "methodResponses": [["Email/set", {"updated": {"jmap-msg-1": null}}, "c1"]] + }))) + .mount(&mock_server) + .await; + + let engine = jmap_engine(&mock_server); + engine + .apply_mutation(&spec(RemoteMutationKind::SetFlagged { value: true })) + .await + .expect("flag mutation applied"); +} + +#[tokio::test] +async fn jmap_apply_mutation_move_replaces_mailbox_ids() { + let mock_server = MockServer::start().await; + mount_session(&mock_server).await; + + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(json!({ + "methodCalls": [[ + "Email/set", + { "update": { "jmap-msg-1": { "mailboxIds": { "mb-archive": true } } } }, + "c1" + ]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "methodResponses": [["Email/set", {"updated": {"jmap-msg-1": null}}, "c1"]] + }))) + .mount(&mock_server) + .await; + + let engine = jmap_engine(&mock_server); + engine + .apply_mutation(&spec(RemoteMutationKind::Move { + to_folder: "mb-archive".to_string(), + })) + .await + .expect("move mutation applied"); +} + +#[tokio::test] +async fn jmap_apply_mutation_delete_issues_destroy_and_confirms_success() { + let mock_server = MockServer::start().await; + mount_session(&mock_server).await; + + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(json!({ + "methodCalls": [["Email/set", { "destroy": ["jmap-msg-1"] }, "c1"]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "methodResponses": [["Email/set", {"destroyed": ["jmap-msg-1"]}, "c1"]] + }))) + .mount(&mock_server) + .await; + + let engine = jmap_engine(&mock_server); + engine + .apply_mutation(&spec(RemoteMutationKind::Delete)) + .await + .expect("delete mutation applied"); +} + +/// A server that reports the id under `notUpdated` genuinely rejected the +/// mutation; `apply_mutation` MUST surface that as an error so the outbox never +/// marks a rejected op completed. +#[tokio::test] +async fn jmap_apply_mutation_surfaces_server_rejection_as_error() { + let mock_server = MockServer::start().await; + mount_session(&mock_server).await; + + Mock::given(method("POST")) + .and(path("/jmap/api")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "methodResponses": [["Email/set", { + "updated": {}, + "notUpdated": { "jmap-msg-1": { "type": "notFound" } } + }, "c1"]] + }))) + .mount(&mock_server) + .await; + + let engine = jmap_engine(&mock_server); + let err = engine + .apply_mutation(&spec(RemoteMutationKind::SetFlagged { value: false })) + .await + .expect_err("a server-rejected mutation must not be reported as success"); + assert!(err.to_string().contains("jmap-msg-1")); +} diff --git a/crates/nuncio-store/src/vault.rs b/crates/nuncio-store/src/vault.rs index 19d8165..ca60afd 100644 --- a/crates/nuncio-store/src/vault.rs +++ b/crates/nuncio-store/src/vault.rs @@ -27,6 +27,14 @@ pub const LEDGER_KEY_ACCOUNT: &str = "ledger-hmac-key"; /// and hex-encoded for use as an `authorization: Bearer ` header value. pub const GRPC_TOKEN_ACCOUNT: &str = "grpc-bearer-token"; +/// Keyring account name for the HMAC-SHA256 key that signs outbound filter +/// `CALL WEBHOOK` payloads (the `X-Nuncio-Signature` header produced by +/// `nuncio_filter::WebhookDispatcher`). Minted on first use via +/// [`SecretManager::get_or_create_key_bytes`] and hex-encoded; a single +/// daemon-wide signing key lets a webhook receiver verify that a delivery +/// genuinely originated from this daemon. +pub const WEBHOOK_SIGNING_KEY_ACCOUNT: &str = "webhook-signing-key"; + /// Zeroized sensitive string wrapper ensuring heap bytes are wiped on drop. #[derive(Debug, Clone, PartialEq, Eq, Zeroize, ZeroizeOnDrop)] pub struct ZeroizingSecret(String); diff --git a/crates/nunciod/src/lib.rs b/crates/nunciod/src/lib.rs index 7144aa8..94daff7 100644 --- a/crates/nunciod/src/lib.rs +++ b/crates/nunciod/src/lib.rs @@ -6,6 +6,7 @@ pub mod calendar_sync; pub mod contacts_sync; pub mod grpc; pub mod orchestrator; +pub mod outbox; pub mod send; pub mod sync; pub use orchestrator::SelfHealingSyncOrchestrator; diff --git a/crates/nunciod/src/main.rs b/crates/nunciod/src/main.rs index ecd4543..21b07bd 100644 --- a/crates/nunciod/src/main.rs +++ b/crates/nunciod/src/main.rs @@ -3,7 +3,7 @@ //! filter automation engine, outbox retries, and the multi-client gRPC API. use nuncio_core::{CoreCommand, CoreEvent, EventBus}; -use nuncio_filter::{FilterEngine, OutboxManager}; +use nuncio_filter::FilterEngine; use std::sync::Arc; #[tokio::main] @@ -98,38 +98,32 @@ async fn main() -> Result<(), Box> { ); } - // Background Outbox Worker Task + // Background Outbox Worker Task. Each tick drains the pending remote + // mutations the filter engine enqueued (move/copy/flag/unflag/delete on the + // real mail server, forward via SMTP, or a signed webhook call) and applies + // them against the resolved account's backend. A mutation is marked + // "completed" ONLY when the real operation genuinely succeeded; a transient + // failure is retried on later ticks and a permanent one (gone message, + // unknown action) fails honestly -- success is never fabricated. let db_outbox = db.clone(); + let outbox_env = Arc::new(nunciod::outbox::ProductionExecutionEnv::new( + db.clone(), + account_secrets.clone(), + )); let _outbox_task = tokio::spawn(async move { let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); loop { interval.tick().await; - if let Ok(pending) = db_outbox.list_pending_mutations(50).await { - for item in pending { - let next_retry = item.retry_count + 1; - if next_retry > OutboxManager::MAX_RETRIES { - let _ = db_outbox - .update_mutation_status(&item.id, "failed", next_retry) - .await; - continue; - } - let backoff_ms = OutboxManager::calculate_backoff_ms(item.retry_count); - tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; - // Outbound IMAP/JMAP remote mutation execution (actually - // applying a filter action like move/delete/flag on the - // real mail server) is not yet wired to a real mail - // server. This worker deliberately does NOT mark - // mutations "completed": doing so would fabricate - // success for work that never happened. It only bumps - // the retry counter (so an item that keeps failing still - // eventually flips to "failed" once retries are - // exhausted) and leaves the item "pending" so it is - // retried on the next poll tick once a real sender is - // wired in. - let _ = db_outbox - .update_mutation_status(&item.id, "pending", next_retry) - .await; - } + let summary = + nunciod::outbox::execute_pending_mutations(&db_outbox, outbox_env.as_ref(), 50) + .await; + if summary.completed > 0 || summary.failed > 0 { + tracing::info!( + "outbox drain: {} completed, {} retried, {} failed", + summary.completed, + summary.retried, + summary.failed + ); } } }); diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs new file mode 100644 index 0000000..e010fd3 --- /dev/null +++ b/crates/nunciod/src/outbox.rs @@ -0,0 +1,466 @@ +//! Outbox remote-mutation executor. +//! +//! The filter engine enqueues a [`nuncio_filter::PendingRemoteMutation`] for +//! every genuinely-remote action a rule matches (`MOVE`/`COPY`/`FLAG`/ +//! `UNFLAG`/`DELETE`/`FORWARD`/`CALL WEBHOOK`); `MARK READ`/`MARK UNREAD` are +//! applied to the local store inline and are never enqueued. This module +//! drains that outbox against the real world -- the resolved account's +//! [`MailBackend`] for the mailbox mutations, a [`MessageSender`] for +//! `FORWARD`, and the SSRF-hardened webhook dispatcher for `CALL WEBHOOK`. +//! +//! # The one inviolable rule +//! +//! A mutation is marked `completed` ONLY when the real remote operation +//! genuinely succeeded. A transient failure keeps the item `pending` (bumping +//! its retry count) until [`nuncio_filter::OutboxManager::MAX_RETRIES`] is +//! exceeded, at which point it flips to `failed`. A permanent failure (the +//! message no longer exists, a malformed payload, an unknown action) flips to +//! `failed` immediately -- there is nothing a retry could fix. Success is +//! never fabricated. +//! +//! # Testability +//! +//! [`execute_pending_mutations`] takes a `&dyn RemoteExecutionEnv`, so an +//! offline E2E injects mock backends/senders and a loopback-allowed webhook +//! dispatcher, proving the full sync -> filter -> outbox -> execute vertical +//! without any live network. Production wires [`ProductionExecutionEnv`], +//! which resolves real per-account engines from the store and the OS keyring. + +use async_trait::async_trait; +use nuncio_core::AccountConfig; +use nuncio_filter::{ + MutationPayload, OutboxManager, PendingRemoteMutation, ValidationOptions, WebhookDispatcher, + WebhookError, +}; +use nuncio_mail::{ + MailBackend, MailError, MessageSender, OutboundMessage, RemoteMutationKind, RemoteMutationSpec, + SmtpTransportEngine, +}; +use nuncio_store::db::{DatabaseEngine, DatabaseError}; +use nuncio_store::vault::{SecretManager, VaultError, WEBHOOK_SIGNING_KEY_ACCOUNT}; +use std::sync::Arc; +use thiserror::Error; + +/// Errors that can occur while resolving the resources needed to execute a +/// single outbox mutation. These are surfaced from a [`RemoteExecutionEnv`] +/// and treated as transient (retryable) by the executor unless it has already +/// determined the failure is permanent. +#[derive(Debug, Error)] +pub enum OutboxExecuteError { + /// No persisted account configuration matches the resolved account id. + #[error("account '{0}' is not configured")] + AccountNotFound(String), + /// Failed to read the account's credential from the secret vault. + #[error("failed to read account credential from the secret vault: {0}")] + Vault(#[from] VaultError), + /// A mail backend / transport construction or operation error. + #[error("mail error: {0}")] + Mail(#[from] MailError), + /// A database error occurred while resolving the account/message. + #[error("database error: {0}")] + Store(#[from] DatabaseError), +} + +/// Injectable seam for the resources a mutation needs: the per-account mail +/// backend and outbound sender, plus the webhook dispatcher. Production uses +/// [`ProductionExecutionEnv`]; tests inject mocks. +#[async_trait] +pub trait RemoteExecutionEnv: Send + Sync { + /// Build the [`MailBackend`] for the account that owns the target message. + async fn mail_backend( + &self, + account_id: &str, + ) -> Result, OutboxExecuteError>; + + /// Build the [`MessageSender`] used to deliver a `FORWARD`. + async fn message_sender( + &self, + account_id: &str, + ) -> Result, OutboxExecuteError>; + + /// Dispatch a signed `CALL WEBHOOK` POST, returning the HTTP status code on + /// delivery. The implementation owns the SSRF egress policy -- production + /// keeps the secure default (private targets blocked); a test env may relax + /// it to reach a loopback mock server. + async fn dispatch_webhook( + &self, + url: &str, + rule_id: &str, + message_id: &str, + subject: &str, + sender: &str, + ) -> Result; +} + +/// The disposition of a single mutation attempt. +enum Disposition { + /// The real operation genuinely succeeded; mark `completed`. + Completed, + /// A transient failure; keep retrying until `MAX_RETRIES` is exceeded. + Retry(String), + /// A permanent failure nothing could retry away; mark `failed` now. + Permanent(String), +} + +/// Tally of what a drain pass did, for logging and test assertions. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct ExecutionSummary { + /// Mutations marked `completed` on genuine success. + pub completed: usize, + /// Mutations left `pending` for a later retry. + pub retried: usize, + /// Mutations flipped to `failed` (permanent, or retries exhausted). + pub failed: usize, +} + +/// Drain up to `limit` pending mutations, executing each against `env` and +/// persisting the honest outcome. Never marks a mutation `completed` unless the +/// real operation succeeded. Returns a tally of the pass. +pub async fn execute_pending_mutations( + db: &DatabaseEngine, + env: &dyn RemoteExecutionEnv, + limit: usize, +) -> ExecutionSummary { + let pending = match db.list_pending_mutations(limit).await { + Ok(items) => items, + Err(e) => { + tracing::warn!("outbox: failed to list pending mutations: {e}"); + return ExecutionSummary::default(); + } + }; + + let mut summary = ExecutionSummary::default(); + for item in pending { + let disposition = execute_one(db, env, &item).await; + let next_retry = item.retry_count + 1; + let (status, retry, tally): (&str, i32, fn(&mut ExecutionSummary)) = match disposition { + Disposition::Completed => ("completed", item.retry_count, |s| s.completed += 1), + Disposition::Permanent(reason) => { + tracing::warn!( + mutation_id = %item.id, + "outbox: permanently failing mutation: {reason}" + ); + ("failed", next_retry, |s| s.failed += 1) + } + Disposition::Retry(reason) if next_retry > OutboxManager::MAX_RETRIES => { + tracing::warn!( + mutation_id = %item.id, + "outbox: mutation failed after exhausting retries: {reason}" + ); + ("failed", next_retry, |s| s.failed += 1) + } + Disposition::Retry(reason) => { + tracing::debug!( + mutation_id = %item.id, + "outbox: mutation will be retried: {reason}" + ); + ("pending", next_retry, |s| s.retried += 1) + } + }; + + if let Err(e) = db.update_mutation_status(&item.id, status, retry).await { + // A status-write failure must not crash the worker; the item stays + // in its prior state and is reconsidered on the next poll. + tracing::warn!(mutation_id = %item.id, "outbox: failed to persist status: {e}"); + } else { + tally(&mut summary); + } + } + summary +} + +/// Execute a single mutation and report its honest disposition. Performs no +/// status writes itself -- the caller persists the outcome. +async fn execute_one( + db: &DatabaseEngine, + env: &dyn RemoteExecutionEnv, + mutation: &PendingRemoteMutation, +) -> Disposition { + // Resolve the message to recover its account and source folder. A message + // that no longer exists can never be acted on -- fail permanently. + let email = match db.get_message(&mutation.message_id).await { + Ok(email) => email, + Err(e) if e.is_not_found() => { + return Disposition::Permanent(format!( + "message '{}' no longer exists in the store", + mutation.message_id + )); + } + Err(e) => { + return Disposition::Retry(format!("failed to load message: {e}")); + } + }; + + let payload = match serde_json::from_str::(&mutation.payload) { + Ok(payload) => payload, + Err(e) => { + return Disposition::Permanent(format!("malformed mutation payload: {e}")); + } + }; + + match mutation.mutation_type.as_str() { + "MOVE" | "COPY" | "FLAG" | "UNFLAG" | "DELETE" => { + dispatch_mailbox_mutation(db, env, mutation, &email, &payload).await + } + "FORWARD" => dispatch_forward(db, env, &email, &payload).await, + "WEBHOOK" => dispatch_webhook(env, mutation, &email, &payload).await, + other => Disposition::Permanent(format!("unknown mutation type '{other}'")), + } +} + +/// Execute a mailbox-affecting mutation (move/copy/flag/unflag/delete) against +/// the account's mail backend, threading the source folder's stored checkpoint +/// so the backend can enforce its UIDVALIDITY guard. +async fn dispatch_mailbox_mutation( + db: &DatabaseEngine, + env: &dyn RemoteExecutionEnv, + mutation: &PendingRemoteMutation, + email: &nuncio_core::model::Email, + payload: &MutationPayload, +) -> Disposition { + let kind = match mutation.mutation_type.as_str() { + "FLAG" => RemoteMutationKind::SetFlagged { value: true }, + "UNFLAG" => RemoteMutationKind::SetFlagged { value: false }, + "DELETE" => RemoteMutationKind::Delete, + "MOVE" => match &payload.target { + Some(folder) => RemoteMutationKind::Move { + to_folder: folder.clone(), + }, + None => { + return Disposition::Permanent("MOVE mutation has no target folder".to_string()) + } + }, + "COPY" => match &payload.target { + Some(folder) => RemoteMutationKind::Copy { + to_folder: folder.clone(), + }, + None => { + return Disposition::Permanent("COPY mutation has no target folder".to_string()) + } + }, + other => return Disposition::Permanent(format!("unexpected mailbox mutation '{other}'")), + }; + + let folder_checkpoint = match db + .get_folder_sync_state(&email.account_id, &email.folder_id) + .await + { + Ok(checkpoint) => checkpoint, + Err(e) => return Disposition::Retry(format!("failed to read folder checkpoint: {e}")), + }; + + let backend = match env.mail_backend(&email.account_id).await { + Ok(backend) => backend, + Err(e) => return Disposition::Retry(format!("failed to build mail backend: {e}")), + }; + + let spec = RemoteMutationSpec { + message_id: email.id.clone(), + folder_id: email.folder_id.clone(), + folder_checkpoint, + kind, + }; + + match backend.apply_mutation(&spec).await { + Ok(()) => Disposition::Completed, + Err(e) => Disposition::Retry(format!("remote mutation failed: {e}")), + } +} + +/// Compose and send a forward of `email` to the payload's target address via +/// the account's outbound transport. +async fn dispatch_forward( + db: &DatabaseEngine, + env: &dyn RemoteExecutionEnv, + email: &nuncio_core::model::Email, + payload: &MutationPayload, +) -> Disposition { + let Some(to) = payload.target.clone() else { + return Disposition::Permanent("FORWARD mutation has no target address".to_string()); + }; + + // The From: is the forwarding account's own address, resolved from its + // persisted configuration rather than trusting the stored envelope. + let from = match find_account_config(db, &email.account_id).await { + Ok(config) => config.email_address, + Err(OutboxExecuteError::AccountNotFound(id)) => { + return Disposition::Permanent(format!("forwarding account '{id}' is not configured")); + } + Err(e) => return Disposition::Retry(format!("failed to resolve forwarding account: {e}")), + }; + + let sender = match env.message_sender(&email.account_id).await { + Ok(sender) => sender, + Err(e) => return Disposition::Retry(format!("failed to build message sender: {e}")), + }; + + let outbound = OutboundMessage { + from, + to, + cc: None, + subject: format!("Fwd: {}", email.subject), + body_plain: email.body_plain.clone(), + body_html: email.body_html.clone(), + attachments: email.attachments.clone(), + }; + + match sender.send(&outbound).await { + Ok(()) => Disposition::Completed, + Err(e) => Disposition::Retry(format!("forward send failed: {e}")), + } +} + +/// Deliver a signed webhook for the matched message. A 2xx/3xx response is a +/// genuine delivery; a 4xx/5xx is treated as a retryable failure so a transient +/// receiver outage does not silently drop the notification. +async fn dispatch_webhook( + env: &dyn RemoteExecutionEnv, + mutation: &PendingRemoteMutation, + email: &nuncio_core::model::Email, + payload: &MutationPayload, +) -> Disposition { + let Some(url) = payload.target.as_deref() else { + return Disposition::Permanent("WEBHOOK mutation has no target URL".to_string()); + }; + + match env + .dispatch_webhook( + url, + &mutation.rule_id, + &mutation.message_id, + &email.subject, + &email.sender, + ) + .await + { + Ok(status) if (200..400).contains(&status) => Disposition::Completed, + Ok(status) => Disposition::Retry(format!("webhook returned HTTP {status}")), + // A security-policy violation (blocked SSRF target) is permanent: the + // URL will never become allowable on retry. + Err(WebhookError::SecurityViolation(reason)) => { + Disposition::Permanent(format!("webhook blocked by security policy: {reason}")) + } + Err(e) => Disposition::Retry(format!("webhook dispatch failed: {e}")), + } +} + +/// Look up an account's persisted configuration by id. +async fn find_account_config( + db: &DatabaseEngine, + account_id: &str, +) -> Result { + db.list_accounts() + .await? + .into_iter() + .find(|a| a.id == account_id) + .ok_or_else(|| OutboxExecuteError::AccountNotFound(account_id.to_string())) +} + +/// Production [`RemoteExecutionEnv`]: resolves real per-account IMAP/JMAP mail +/// backends and SMTP senders from the store plus the OS keyring, and lazily +/// builds a single SSRF-hardened [`WebhookDispatcher`] keyed by a daemon-wide +/// signing secret provisioned from the vault. +/// +/// The webhook signing key is provisioned lazily -- only the first time a +/// `CALL WEBHOOK` mutation is actually dispatched -- so a vault issue with that +/// one key can never disable the mailbox-mutation and forward paths, which do +/// not need it. +pub struct ProductionExecutionEnv { + db: Arc, + secrets: Arc, + webhook: std::sync::OnceLock, +} + +impl ProductionExecutionEnv { + /// Build the production environment. Infallible: no vault access happens + /// here, so the outbox worker can always start; the webhook signing key is + /// provisioned on first webhook dispatch instead (see the struct doc). + pub fn new(db: Arc, secrets: Arc) -> Self { + Self { + db, + secrets, + webhook: std::sync::OnceLock::new(), + } + } + + async fn account(&self, account_id: &str) -> Result { + find_account_config(&self.db, account_id).await + } + + /// Return the lazily-provisioned webhook dispatcher, minting (or loading) + /// the daemon-wide signing key from the vault on first use. Fails closed if + /// the vault is unavailable, so a `CALL WEBHOOK` is never dispatched + /// unsigned -- and, being lazy, that failure is isolated to webhook + /// dispatch and never disables the other mutation paths. + fn webhook_dispatcher(&self) -> Result<&WebhookDispatcher, WebhookError> { + if let Some(dispatcher) = self.webhook.get() { + return Ok(dispatcher); + } + let key_bytes = self + .secrets + .get_or_create_key_bytes(WEBHOOK_SIGNING_KEY_ACCOUNT, 32) + .map_err(|e| { + WebhookError::SigningError(format!( + "failed to provision webhook signing key from the vault: {e}" + )) + })?; + // A concurrent caller may win the race to set it; either way the stored + // dispatcher is authoritative. + let _ = self + .webhook + .set(WebhookDispatcher::new(hex::encode(key_bytes))); + self.webhook.get().ok_or_else(|| { + WebhookError::SigningError("webhook dispatcher unavailable after provisioning".into()) + }) + } +} + +#[async_trait] +impl RemoteExecutionEnv for ProductionExecutionEnv { + async fn mail_backend( + &self, + account_id: &str, + ) -> Result, OutboxExecuteError> { + let config = self.account(account_id).await?; + let password = self.secrets.get_secret(&config.keyring_secret_key)?; + Ok(crate::sync::build_mail_backend(&config, &password)) + } + + async fn message_sender( + &self, + account_id: &str, + ) -> Result, OutboxExecuteError> { + let config = self.account(account_id).await?; + let password = self.secrets.get_secret(&config.keyring_secret_key)?; + let engine = SmtpTransportEngine::new( + &config.smtp_host, + config.smtp_port, + config.smtp_tls_mode, + &config.email_address, + &password, + )?; + Ok(Box::new(engine)) + } + + async fn dispatch_webhook( + &self, + url: &str, + rule_id: &str, + message_id: &str, + subject: &str, + sender: &str, + ) -> Result { + // Production keeps the secure default egress policy (private/loopback + // targets are rejected). + self.webhook_dispatcher()? + .dispatch_with_options( + url, + rule_id, + message_id, + subject, + sender, + &ValidationOptions::default(), + ) + .await + } +} diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index a3c4522..02265a1 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -339,7 +339,7 @@ async fn find_account( /// keyring by the caller). This is the ONLY place a concrete protocol engine /// is chosen; every other function in this module only ever sees `&dyn /// MailBackend`. -fn build_mail_backend(config: &AccountConfig, password: &str) -> Box { +pub(crate) fn build_mail_backend(config: &AccountConfig, password: &str) -> Box { match config.protocol { AccountProtocol::ImapSmtp => Box::new(ImapEngine::with_credentials( &config.id, diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs new file mode 100644 index 0000000..855a4c9 --- /dev/null +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -0,0 +1,329 @@ +//! Offline end-to-end test for the outbox remote-mutation executor. +//! +//! Proves the full `sync -> filter -> outbox -> execute` vertical without any +//! live network: a matched filter rule enqueues a real +//! `PendingRemoteMutation`, and `execute_pending_mutations` drains it against +//! injected mocks -- a `MockMailBackend` that records each applied mutation, a +//! `MockMessageSender` for `FORWARD`, and a loopback-allowed webhook dispatcher +//! hitting a wiremock server. The safety-critical property under test is that a +//! mutation is marked `completed` ONLY when the real op genuinely succeeded: a +//! failing backend never records an op and never completes the item. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use async_trait::async_trait; +use nuncio_core::model::Email; +use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher, WebhookError}; +use nuncio_mail::{ + MailBackend, MessageSender, MockMailBackend, MockMessageSender, RemoteMutationKind, +}; +use nuncio_store::db::DatabaseEngine; +use nunciod::outbox::{execute_pending_mutations, RemoteExecutionEnv}; +use nunciod::sync::apply_filter_actions; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const ACCOUNT_ID: &str = "acct-outbox-e2e"; +const FOLDER_ID: &str = "INBOX"; +const CHECKPOINT: &str = "1:105"; + +/// Test [`RemoteExecutionEnv`] injecting recording mocks and a webhook +/// dispatcher whose SSRF egress policy is relaxed so it can reach a loopback +/// wiremock server (production keeps the secure default). +struct MockEnv { + backend: MockMailBackend, + sender: MockMessageSender, + webhook: WebhookDispatcher, +} + +impl MockEnv { + fn new() -> Self { + Self { + backend: MockMailBackend::new(), + sender: MockMessageSender::new(), + webhook: WebhookDispatcher::new("test-signing-key"), + } + } +} + +#[async_trait] +impl RemoteExecutionEnv for MockEnv { + async fn mail_backend( + &self, + _account_id: &str, + ) -> Result, nunciod::outbox::OutboxExecuteError> { + // Clones share the same Arc-backed recording storage, so the returned + // boxed backend records into the `self.backend` the test inspects. + Ok(Box::new(self.backend.clone())) + } + + async fn message_sender( + &self, + _account_id: &str, + ) -> Result, nunciod::outbox::OutboxExecuteError> { + Ok(Box::new(self.sender.clone())) + } + + async fn dispatch_webhook( + &self, + url: &str, + rule_id: &str, + message_id: &str, + subject: &str, + sender: &str, + ) -> Result { + let opts = ValidationOptions { + block_private_webhooks: false, + ..Default::default() + }; + self.webhook + .dispatch_with_options(url, rule_id, message_id, subject, sender, &opts) + .await + } +} + +fn sample_account() -> nuncio_core::AccountConfig { + nuncio_core::AccountConfig { + id: ACCOUNT_ID.to_string(), + name: "Outbox E2E Account".to_string(), + email_address: "owner@nuncio.mx".to_string(), + protocol: nuncio_core::AccountProtocol::ImapSmtp, + server_host: "imap.nuncio.mx".to_string(), + server_port: 993, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + use_tls: true, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + keyring_secret_key: format!("nuncio/{ACCOUNT_ID}"), + sync_interval_secs: 60, + } +} + +fn sample_email(subject: &str) -> Email { + Email { + id: "imap-uid-42".to_string(), + account_id: ACCOUNT_ID.to_string(), + folder_id: FOLDER_ID.to_string(), + subject: subject.to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "owner@nuncio.mx".to_string(), + received_at: 1_700_000_000, + read: false, + body_plain: Some("original body".to_string()), + body_html: None, + attachments: Vec::new(), + } +} + +/// Seed the account, source-folder checkpoint, and a matching message, then run +/// the given rule's actions through `apply_filter_actions` so the outbox is +/// populated exactly as a real sync would populate it. +async fn seed_and_enqueue(db: &DatabaseEngine, rule_nsql: &str, subject: &str) { + db.save_account(&sample_account()) + .await + .expect("save account"); + db.save_folder_sync_state(ACCOUNT_ID, FOLDER_ID, CHECKPOINT) + .await + .expect("save folder checkpoint"); + + let email = sample_email(subject); + db.save_email(&email).await.expect("save email"); + + let rule = NsqlParser::parse_rule("Outbox Rule", 1, rule_nsql).expect("parse rule"); + let engine = FilterEngine::new(vec![rule]).expect("compile rule"); + apply_filter_actions(db, &engine, &email).await; + + let pending = db.list_pending_mutations(10).await.expect("list pending"); + assert_eq!( + pending.len(), + 1, + "the rule must enqueue exactly one mutation" + ); +} + +#[tokio::test] +async fn move_rule_executes_against_backend_and_completes_with_uidvalidity_checkpoint() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); + seed_and_enqueue( + &db, + "WHERE subject CONTAINS 'Archive' ACTION MOVE TO 'Archive'", + "Please Archive", + ) + .await; + + let env = MockEnv::new(); + let summary = execute_pending_mutations(&db, &env, 50).await; + + assert_eq!(summary.completed, 1); + assert_eq!(summary.failed, 0); + + let applied = env.backend.applied_mutations(); + assert_eq!(applied.len(), 1, "the backend op must genuinely run"); + assert_eq!(applied[0].message_id, "imap-uid-42"); + assert_eq!(applied[0].folder_id, FOLDER_ID); + // The source folder's stored checkpoint (carrying UIDVALIDITY) is threaded + // through so the backend can enforce its guard. + assert_eq!(applied[0].folder_checkpoint.as_deref(), Some(CHECKPOINT)); + assert_eq!( + applied[0].kind, + RemoteMutationKind::Move { + to_folder: "Archive".to_string() + } + ); + + // A completed mutation is no longer pending. + assert!(db.list_pending_mutations(10).await.unwrap().is_empty()); +} + +#[tokio::test] +async fn flag_rule_executes_as_set_flagged() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); + seed_and_enqueue( + &db, + "WHERE subject CONTAINS 'Star' ACTION FLAG", + "Star this", + ) + .await; + + let env = MockEnv::new(); + let summary = execute_pending_mutations(&db, &env, 50).await; + + assert_eq!(summary.completed, 1); + let applied = env.backend.applied_mutations(); + assert_eq!(applied.len(), 1); + assert_eq!( + applied[0].kind, + RemoteMutationKind::SetFlagged { value: true } + ); +} + +#[tokio::test] +async fn delete_rule_executes_as_delete() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); + seed_and_enqueue( + &db, + "WHERE subject CONTAINS 'Trash' ACTION DELETE", + "Trash me", + ) + .await; + + let env = MockEnv::new(); + let summary = execute_pending_mutations(&db, &env, 50).await; + + assert_eq!(summary.completed, 1); + let applied = env.backend.applied_mutations(); + assert_eq!(applied.len(), 1); + assert_eq!(applied[0].kind, RemoteMutationKind::Delete); +} + +#[tokio::test] +async fn a_failing_backend_op_is_never_marked_completed() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); + seed_and_enqueue( + &db, + "WHERE subject CONTAINS 'Archive' ACTION MOVE TO 'Archive'", + "Please Archive", + ) + .await; + + let env = MockEnv::new(); + env.backend.set_should_fail(true); + let summary = execute_pending_mutations(&db, &env, 50).await; + + // The op failed: nothing recorded, nothing completed, and the item is kept + // for retry (still pending, with a bumped retry count) -- never faked. + assert_eq!(summary.completed, 0); + assert_eq!(summary.retried, 1); + assert!(env.backend.applied_mutations().is_empty()); + + let pending = db.list_pending_mutations(10).await.unwrap(); + assert_eq!(pending.len(), 1, "a failed op must remain pending"); + assert_eq!(pending[0].retry_count, 1, "the retry count must be bumped"); +} + +#[tokio::test] +async fn forward_rule_sends_via_the_message_sender() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); + seed_and_enqueue( + &db, + "WHERE subject CONTAINS 'Forward' ACTION FORWARD TO 'boss@nuncio.mx'", + "Please Forward", + ) + .await; + + let env = MockEnv::new(); + let summary = execute_pending_mutations(&db, &env, 50).await; + + assert_eq!(summary.completed, 1); + let sent = env.sender.sent_messages(); + assert_eq!(sent.len(), 1, "the forward must genuinely be sent"); + assert_eq!(sent[0].to, "boss@nuncio.mx"); + // The From: is the forwarding account's own configured address. + assert_eq!(sent[0].from, "owner@nuncio.mx"); + assert!(sent[0].subject.contains("Please Forward")); +} + +#[tokio::test] +async fn webhook_rule_dispatches_a_signed_post_to_the_receiver() { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/hooks/nuncio")) + .and(wiremock::matchers::header( + "Content-Type", + "application/json", + )) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&mock_server) + .await; + + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); + let url = format!("{}/hooks/nuncio", mock_server.uri()); + seed_and_enqueue( + &db, + &format!("WHERE subject CONTAINS 'Ping' ACTION CALL WEBHOOK '{url}'"), + "Ping me", + ) + .await; + + let env = MockEnv::new(); + let summary = execute_pending_mutations(&db, &env, 50).await; + + // Completion is genuine: wiremock's `.expect(1)` verifies the POST actually + // arrived when the server drops. + assert_eq!(summary.completed, 1); + assert!(db.list_pending_mutations(10).await.unwrap().is_empty()); +} + +#[tokio::test] +async fn a_mutation_for_a_vanished_message_fails_permanently() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); + db.save_account(&sample_account()) + .await + .expect("save account"); + + // Enqueue a mutation whose target message was never persisted. + let mutation = nuncio_filter::OutboxManager::create_mutation( + "rule-x", + "imap-uid-999", + "MOVE", + Some("Archive".to_string()), + ); + db.save_pending_mutation(&mutation) + .await + .expect("save mutation"); + + let env = MockEnv::new(); + let summary = execute_pending_mutations(&db, &env, 50).await; + + // Cannot act on a message that no longer exists -- fail honestly, never + // fabricate success, and never touch the backend. + assert_eq!(summary.failed, 1); + assert_eq!(summary.completed, 0); + assert!(env.backend.applied_mutations().is_empty()); + assert!( + db.list_pending_mutations(10).await.unwrap().is_empty(), + "a permanently-failed mutation is no longer pending" + ); +} -- 2.54.0 From dfaec91bd4162a5a1966f1173bfdd400c197ce71 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 11:07:10 -0500 Subject: [PATCH 10/95] fix(store): fail closed on decrypt failure in decrypt_text_at_rest (#264) decrypt_text_at_rest previously swallowed any hex-decode, AEAD-decrypt, or UTF-8 failure with a fallback to an empty string, so a corrupted or tampered body column read back as an empty body instead of surfacing an error -- defeating the AEAD's integrity guarantee exactly where it matters: detecting tampering on read. Change its signature to return Result and propagate failures through every read call site in db.rs as the new DatabaseError::Decryption variant. Refs #237 #162 Co-authored-by: James Maes --- crates/nuncio-store/src/cipher.rs | 68 ++++++++++--- crates/nuncio-store/src/db.rs | 162 +++++++++++++++++++++++------- 2 files changed, 179 insertions(+), 51 deletions(-) diff --git a/crates/nuncio-store/src/cipher.rs b/crates/nuncio-store/src/cipher.rs index fe5a508..7a1943d 100644 --- a/crates/nuncio-store/src/cipher.rs +++ b/crates/nuncio-store/src/cipher.rs @@ -74,18 +74,20 @@ impl PayloadCipher { /// Decrypt text payload from database column storage at rest using a caller-supplied /// AES-256-GCM key. See [`PayloadCipher::encrypt_text_at_rest`]. - pub fn decrypt_text_at_rest(key: &[u8; 32], stored: &str) -> String { + /// + /// Fails closed: a hex-decode error, an AEAD authentication failure (tampered or + /// corrupted ciphertext), or a non-UTF-8 decrypted payload is returned as an error + /// rather than silently read back as an empty body -- an empty stored ciphertext is + /// the only case that legitimately yields `Ok(String::new())`. + pub fn decrypt_text_at_rest(key: &[u8; 32], stored: &str) -> Result { if stored.is_empty() { - return String::new(); + return Ok(String::new()); } - if let Ok(bytes) = hex::decode(stored) { - if let Ok(decrypted) = Self::decrypt_bytes(key, &bytes) { - if let Ok(s) = String::from_utf8(decrypted) { - return s; - } - } - } - String::new() + let bytes = hex::decode(stored) + .map_err(|e| CipherError::DecryptionFailed(format!("invalid hex encoding: {e}")))?; + let decrypted = Self::decrypt_bytes(key, &bytes)?; + String::from_utf8(decrypted) + .map_err(|e| CipherError::DecryptionFailed(format!("invalid utf-8 payload: {e}"))) } /// Encrypt binary attachment payload using `age` passphrase encryption. @@ -240,28 +242,62 @@ mod tests { PayloadCipher::encrypt_text_at_rest(&key, text).expect("encryption succeeds"); assert_ne!(text, encrypted); - let decrypted = PayloadCipher::decrypt_text_at_rest(&key, &encrypted); + let decrypted = + PayloadCipher::decrypt_text_at_rest(&key, &encrypted).expect("decryption succeeds"); assert_eq!(text, decrypted); assert_eq!( PayloadCipher::encrypt_text_at_rest(&key, "").expect("empty text is not encrypted"), "" ); - assert_eq!(PayloadCipher::decrypt_text_at_rest(&key, ""), ""); + assert_eq!( + PayloadCipher::decrypt_text_at_rest(&key, "").expect("empty ciphertext decrypts"), + "" + ); } #[test] - fn text_at_rest_wrong_key_fails_to_decrypt() { + fn text_at_rest_wrong_key_fails_closed() { // Proves there is no compiled-in default key: ciphertext produced under one - // caller-supplied key cannot be recovered using a different key. + // caller-supplied key cannot be recovered using a different key, and the failure + // is surfaced as an error rather than a silently-empty body. let key_a = [1u8; 32]; let key_b = [2u8; 32]; let text = "Confidential ledger payload"; let encrypted = PayloadCipher::encrypt_text_at_rest(&key_a, text).expect("encryption succeeds"); - let decrypted_with_wrong_key = PayloadCipher::decrypt_text_at_rest(&key_b, &encrypted); + let err = PayloadCipher::decrypt_text_at_rest(&key_b, &encrypted) + .expect_err("wrong key must fail closed"); - assert_eq!(decrypted_with_wrong_key, ""); + assert!(matches!(err, CipherError::DecryptionFailed(_))); + } + + #[test] + fn text_at_rest_tampered_ciphertext_fails_closed() { + // A corrupted-but-valid-hex ciphertext column must surface as an error, not read + // back as an empty body -- this is the read-path twin of the AEAD integrity + // guarantee: tampering must be detectable, not silently masked. + let key = [9u8; 32]; + let text = "Do not silently drop this on tamper"; + let encrypted = + PayloadCipher::encrypt_text_at_rest(&key, text).expect("encryption succeeds"); + + let mut bytes = hex::decode(&encrypted).expect("valid hex"); + let last_idx = bytes.len() - 1; + bytes[last_idx] ^= 0xFF; + let tampered = hex::encode(bytes); + + let err = PayloadCipher::decrypt_text_at_rest(&key, &tampered) + .expect_err("tampered ciphertext must fail closed"); + assert!(matches!(err, CipherError::DecryptionFailed(_))); + } + + #[test] + fn text_at_rest_invalid_hex_fails_closed() { + let key = [3u8; 32]; + let err = PayloadCipher::decrypt_text_at_rest(&key, "not-valid-hex") + .expect_err("invalid hex must fail closed"); + assert!(matches!(err, CipherError::DecryptionFailed(_))); } } diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 28f66e0..821adb9 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -43,6 +43,13 @@ pub enum DatabaseError { /// genuinely empty body on read-back. #[error("payload encryption failed: {0}")] Encryption(String), + /// A body payload failed AEAD decryption (or UTF-8/hex decoding) on read. + /// This is a fail-closed error: it is returned instead of ever reading back a + /// corrupted or tampered ciphertext column as a silently-empty body, which would + /// defeat the AEAD's integrity guarantee exactly where it matters -- detecting + /// tampering on read. + #[error("payload decryption failed: {0}")] + Decryption(String), } impl From for DatabaseError { @@ -811,6 +818,8 @@ impl DatabaseEngine { for (id, subject, sender, body_plain) in rows { let dec_plain = body_plain .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))? .unwrap_or_default(); sqlx::query( @@ -1112,8 +1121,7 @@ impl DatabaseEngine { .await .map_err(DatabaseError::Query)?; - Ok(rows - .into_iter() + rows.into_iter() .map( |( id, @@ -1127,13 +1135,25 @@ impl DatabaseEngine { body_plain, body_html, )| { - let dec_plain = body_plain.map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) - }); - let dec_html = body_html.map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) - }); - nuncio_core::model::Email { + let dec_plain = body_plain + .map(|p| { + crate::cipher::PayloadCipher::decrypt_text_at_rest( + &self.storage_key, + &p, + ) + }) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = body_html + .map(|h| { + crate::cipher::PayloadCipher::decrypt_text_at_rest( + &self.storage_key, + &h, + ) + }) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + Ok(nuncio_core::model::Email { id, account_id, folder_id, @@ -1145,10 +1165,10 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), - } + }) }, ) - .collect()) + .collect::, DatabaseError>>() } /// Retrieve a single message by ID. @@ -1182,10 +1202,14 @@ impl DatabaseEngine { let dec_plain = row .8 - .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)); + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; let dec_html = row .9 - .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)); + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; Ok(nuncio_core::model::Email { id: row.0, @@ -1839,16 +1863,21 @@ impl DatabaseEngine { .await .map_err(DatabaseError::Query)?; - Ok(rows - .into_iter() + rows.into_iter() .map(|r| { - let dec_plain = r.8.map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) - }); - let dec_html = r.9.map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) - }); - nuncio_core::model::Email { + let dec_plain = + r.8.map(|p| { + crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) + }) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = + r.9.map(|h| { + crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) + }) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + Ok(nuncio_core::model::Email { id: r.0, account_id: r.1, folder_id: r.2, @@ -1860,9 +1889,9 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), - } + }) }) - .collect()) + .collect::, DatabaseError>>() } /// Query messages across the WHOLE store for export purposes, @@ -1914,16 +1943,21 @@ impl DatabaseEngine { .await .map_err(DatabaseError::Query)?; - Ok(rows - .into_iter() + rows.into_iter() .map(|r| { - let dec_plain = r.8.map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) - }); - let dec_html = r.9.map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) - }); - nuncio_core::model::Email { + let dec_plain = + r.8.map(|p| { + crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) + }) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = + r.9.map(|h| { + crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) + }) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + Ok(nuncio_core::model::Email { id: r.0, account_id: r.1, folder_id: r.2, @@ -1935,9 +1969,9 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), - } + }) }) - .collect()) + .collect::, DatabaseError>>() } /// Append a new immutable WORM audit record to the log ledger, signed with the WORM @@ -2402,6 +2436,64 @@ mod tests { assert_eq!(folders[0].unread_messages, 1); } + /// A `body_plain` column corrupted after a legitimate write (simulating tampering or + /// bit-rot) must make `get_message` and `list_messages` return + /// `Err(DatabaseError::Decryption)`, never a silently-empty body -- that would defeat + /// the AEAD's integrity guarantee exactly where it matters: detecting tampering on read. + #[tokio::test] + async fn corrupted_body_ciphertext_fails_closed_on_read() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let email = nuncio_core::model::Email { + id: "msg-db-corrupt".to_string(), + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + subject: "Corrupted At Rest".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1700000000, + read: false, + body_plain: Some("Sensitive plaintext body".to_string()), + body_html: None, + attachments: Vec::new(), + }; + engine + .save_email(&email) + .await + .expect("save email succeeds"); + + // Flip a byte in the stored (still valid-hex) ciphertext to simulate tampering or + // storage bit-rot. + let (stored,): (String,) = sqlx::query_as("SELECT body_plain FROM messages WHERE id = ?") + .bind("msg-db-corrupt") + .fetch_one(&engine.pool) + .await + .expect("fetch stored ciphertext"); + let mut bytes = hex::decode(&stored).expect("stored ciphertext is valid hex"); + let last_idx = bytes.len() - 1; + bytes[last_idx] ^= 0xFF; + let tampered = hex::encode(bytes); + + sqlx::query("UPDATE messages SET body_plain = ? WHERE id = ?") + .bind(&tampered) + .bind("msg-db-corrupt") + .execute(&engine.pool) + .await + .expect("corrupt stored ciphertext"); + + let err = engine + .get_message("msg-db-corrupt") + .await + .expect_err("get_message must fail closed on corrupted ciphertext"); + assert!(matches!(err, DatabaseError::Decryption(_))); + + let err = engine + .list_messages("INBOX", 10) + .await + .expect_err("list_messages must fail closed on corrupted ciphertext"); + assert!(matches!(err, DatabaseError::Decryption(_))); + } + fn sample_calendar_event(id: &str, account_id: &str) -> nuncio_core::model::CalendarEvent { nuncio_core::model::CalendarEvent { id: id.to_string(), -- 2.54.0 From e75f469f6b9bbd0fdcb905505bf210f14e9cf1d5 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 11:22:57 -0500 Subject: [PATCH 11/95] fix(store): make WORM/ledger appends atomic (#266) save_filter_execution_log and append_worm_audit_record read the latest chain hash and then insert as two separate statements with no lock, letting concurrent appends read the same prev-hash and fork the tamper-evident hash chain. Wrap each read+insert in a BEGIN IMMEDIATE transaction so SQLite serializes the chain read against the dependent insert. Refs #245 #164 Co-authored-by: James Maes --- crates/nuncio-store/src/db.rs | 173 +++++++++++++++++++++++++++++++++- 1 file changed, 169 insertions(+), 4 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 821adb9..bddedb5 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -1704,15 +1704,27 @@ impl DatabaseEngine { /// Record a cryptographically hash-chained [`nuncio_filter::FilterExecutionLog`], signed /// with the ledger HMAC key provisioned for this engine from the secret vault. + /// + /// The read of the latest chain hash and the dependent insert run inside a single + /// `BEGIN IMMEDIATE` transaction: `BEGIN IMMEDIATE` acquires SQLite's write lock before + /// any statement runs, so a second concurrent append (from another pooled connection, + /// since `DatabaseEngine` is shared as an `Arc` across the daemon) blocks until this one + /// commits rather than reading the same prev-hash and forking the chain. pub async fn save_filter_execution_log( &self, rule_id: &str, message_id: &str, action_taken: &str, ) -> Result { + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(DatabaseError::Query)?; + let latest_hash: Option<(String,)> = sqlx::query_as("SELECT hash FROM filter_execution_logs ORDER BY id DESC LIMIT 1") - .fetch_optional(&self.pool) + .fetch_optional(&mut *tx) .await .map_err(DatabaseError::Query)?; @@ -1741,11 +1753,13 @@ impl DatabaseEngine { .bind(matched_at) .bind(&prev_hash) .bind(&hash) - .execute(&self.pool) + .execute(&mut *tx) .await .map_err(DatabaseError::Query)? .last_insert_rowid(); + tx.commit().await.map_err(DatabaseError::Query)?; + Ok(nuncio_filter::FilterExecutionLog { id, rule_id: rule_id.to_string(), @@ -1976,6 +1990,12 @@ impl DatabaseEngine { /// Append a new immutable WORM audit record to the log ledger, signed with the WORM /// HMAC key provisioned for this engine from the secret vault. + /// + /// The read of the latest sequence/hash and the dependent insert run inside a single + /// `BEGIN IMMEDIATE` transaction: `BEGIN IMMEDIATE` acquires SQLite's write lock before + /// any statement runs, so a second concurrent append (from another pooled connection, + /// since `DatabaseEngine` is shared as an `Arc` across the daemon) blocks until this one + /// commits rather than reading the same prev-hash and forking the chain. pub async fn append_worm_audit_record( &self, actor: &str, @@ -1987,10 +2007,16 @@ impl DatabaseEngine { .unwrap_or_default() .as_nanos() as i64; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(DatabaseError::Query)?; + let last_row: Option<(i64, String)> = sqlx::query_as( "SELECT sequence, record_hmac FROM worm_audit_records ORDER BY sequence DESC LIMIT 1", ) - .fetch_optional(&self.pool) + .fetch_optional(&mut *tx) .await?; let (next_seq, prev_hash) = match last_row { @@ -2020,9 +2046,11 @@ impl DatabaseEngine { .bind(&record.data_hash) .bind(&record.previous_block_hash) .bind(&record.record_hmac) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await.map_err(DatabaseError::Query)?; + Ok(record) } @@ -3421,6 +3449,143 @@ mod tests { assert_eq!(report.first_broken_seq, Some(1)); } + /// Regression test for a race between the read-latest-hash and the dependent insert in + /// `append_worm_audit_record`: fires many concurrent appends against the same + /// `Arc`-shared engine (mirroring how `nunciod` shares `DatabaseEngine`) and requires the + /// persisted chain to come back as exactly one unbroken sequence, with no forked or + /// duplicated prev-hash. Without the `BEGIN IMMEDIATE` fix, two tasks can both read the + /// same latest hash before either inserts, producing two records that both claim the same + /// `previous_block_hash` -- a fork that `verify_worm_audit_chain` would reject. + #[tokio::test] + async fn append_worm_audit_record_is_atomic_under_concurrency() { + const CONCURRENT_APPENDS: usize = 20; + + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let engine = std::sync::Arc::new(engine); + + let mut handles = Vec::with_capacity(CONCURRENT_APPENDS); + for i in 0..CONCURRENT_APPENDS { + let engine = std::sync::Arc::clone(&engine); + handles.push(tokio::spawn(async move { + engine + .append_worm_audit_record( + "system.test", + "concurrent.append", + format!("payload-{i}").as_bytes(), + ) + .await + })); + } + for handle in handles { + handle.await.expect("append task must not panic").expect( + "a concurrent append must never fail outright -- it must serialize, not error", + ); + } + + let records = engine + .list_worm_audit_records(CONCURRENT_APPENDS as u32 + 1, 0) + .await + .unwrap(); + assert_eq!( + records.len(), + CONCURRENT_APPENDS, + "every concurrent append must persist exactly one record -- no lost writes" + ); + + let mut seen_sequences = std::collections::HashSet::new(); + for (idx, record) in records.iter().enumerate() { + assert_eq!( + record.sequence, + idx as u64 + 1, + "sequence numbers must be contiguous with no gaps or duplicates" + ); + assert!( + seen_sequences.insert(record.sequence), + "duplicate sequence number {} indicates a forked chain", + record.sequence + ); + let expected_prev = if idx == 0 { + "GENESIS".to_string() + } else { + records[idx - 1].record_hmac.clone() + }; + assert_eq!( + record.previous_block_hash, expected_prev, + "record at sequence {} must chain from the immediately preceding record's hmac \ + -- a mismatch here means two appends forked off the same prev-hash", + record.sequence + ); + } + + engine + .verify_worm_audit_chain() + .await + .expect("the full persisted chain must verify as a single valid sequence"); + } + + /// Same regression, for the filter execution log ledger: concurrent + /// `save_filter_execution_log` calls must serialize into one unbroken `prev_hash`/`hash` + /// chain rather than forking when two tasks read the same latest hash. + #[tokio::test] + async fn save_filter_execution_log_is_atomic_under_concurrency() { + const CONCURRENT_APPENDS: usize = 20; + + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let engine = std::sync::Arc::new(engine); + + let mut handles = Vec::with_capacity(CONCURRENT_APPENDS); + for i in 0..CONCURRENT_APPENDS { + let engine = std::sync::Arc::clone(&engine); + handles.push(tokio::spawn(async move { + engine + .save_filter_execution_log(&format!("rule-{i}"), &format!("msg-{i}"), "DELETE") + .await + })); + } + for handle in handles { + handle.await.expect("append task must not panic").expect( + "a concurrent ledger append must never fail outright -- it must serialize, not error", + ); + } + + let mut logs = engine + .list_filter_execution_logs(CONCURRENT_APPENDS + 1) + .await + .unwrap(); + // `list_filter_execution_logs` returns newest-first; put it back in chain order. + logs.reverse(); + assert_eq!( + logs.len(), + CONCURRENT_APPENDS, + "every concurrent ledger append must persist exactly one record -- no lost writes" + ); + + let mut seen_ids = std::collections::HashSet::new(); + for (idx, log) in logs.iter().enumerate() { + assert!( + seen_ids.insert(log.id), + "duplicate row id {} indicates a lost or double-counted write", + log.id + ); + let expected_prev = if idx == 0 { + "GENESIS".to_string() + } else { + logs[idx - 1].hash.clone() + }; + assert_eq!( + log.prev_hash, expected_prev, + "log entry {} must chain from the immediately preceding entry's hash -- a \ + mismatch here means two appends forked off the same prev-hash", + log.id + ); + } + + assert!( + engine.verify_execution_log_chain().await.unwrap(), + "the full persisted ledger must verify as a single valid chain" + ); + } + /// Proves the zeroize mechanism `Zeroizing`'s `Drop` relies on is real, not /// decorative: explicitly invoking `Zeroize::zeroize` on key-shaped byte /// buffers overwrites every byte with zero. This is checked directly -- 2.54.0 From e1c66da4df66c510d40195f4dd68fba8e5ed7ace Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 11:32:10 -0500 Subject: [PATCH 12/95] fix(filter): use unique rule ids to prevent silent overwrite (#267) CreateRule derived a rule's id deterministically from sha256(name + nsql), so two rules with identical name and NSQL text produced the same id. save_filter_rule persists via INSERT OR REPLACE with no existence check, so the second create silently overwrote the first rule with no warning. Generate each rule's id from a CSPRNG (the same OsRng nuncio-store already uses to source key material) instead of deriving it from the rule's content, so identical name+NSQL always yields distinct ids. UpdateRule and list_filter_rules already reassign the id from the persisted row after parsing, so they are unaffected. Refs #249 Co-authored-by: James Maes --- Cargo.lock | 1 + crates/nuncio-filter/Cargo.toml | 1 + crates/nuncio-filter/src/parser.rs | 20 ++++++---- crates/nunciod/src/grpc.rs | 59 ++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9c3920..d5cad9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2544,6 +2544,7 @@ dependencies = [ name = "nuncio-filter" version = "0.1.0" dependencies = [ + "aes-gcm", "arc-swap", "async-trait", "chrono", diff --git a/crates/nuncio-filter/Cargo.toml b/crates/nuncio-filter/Cargo.toml index 31c98f8..3eb1acb 100644 --- a/crates/nuncio-filter/Cargo.toml +++ b/crates/nuncio-filter/Cargo.toml @@ -26,6 +26,7 @@ chrono.workspace = true reqwest.workspace = true hex.workspace = true zeroize.workspace = true +aes-gcm.workspace = true [dev-dependencies] tempfile.workspace = true diff --git a/crates/nuncio-filter/src/parser.rs b/crates/nuncio-filter/src/parser.rs index c9b7f13..b52ab2b 100644 --- a/crates/nuncio-filter/src/parser.rs +++ b/crates/nuncio-filter/src/parser.rs @@ -65,7 +65,7 @@ impl NsqlParser { let now = chrono::Utc::now().timestamp(); Ok(FilterRule { - id: format!("rule-{}", uuid_or_simple_hash(&name_str, nsql)), + id: format!("rule-{}", generate_rule_id()), name: name_str, target_account, priority, @@ -610,12 +610,18 @@ fn split_comma_outside_quotes(text: &str) -> Vec { result } -fn uuid_or_simple_hash(name: &str, nsql: &str) -> String { - use sha2::{Digest, Sha256}; - let mut hasher = Sha256::new(); - hasher.update(name.as_bytes()); - hasher.update(nsql.as_bytes()); - hex::encode(&hasher.finalize()[..8]) +/// Generates a fresh, collision-resistant rule id from the same CSPRNG +/// (`aes-gcm`'s `OsRng`) that `nuncio-store` uses to source key material, +/// rather than deriving the id from the rule's name/NSQL text. Two rules +/// created with identical name and NSQL text MUST still get distinct ids -- +/// deriving the id from their content would make the second `CreateRule` +/// silently overwrite the first via `save_filter_rule`'s `INSERT OR +/// REPLACE`. +fn generate_rule_id() -> String { + use aes_gcm::aead::{rand_core::RngCore, OsRng}; + let mut bytes = [0u8; 16]; + OsRng.fill_bytes(&mut bytes); + hex::encode(bytes) } #[cfg(test)] diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 3d11220..b820424 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -4333,6 +4333,65 @@ mod tests { .is_empty()); } + /// Two `CreateRule` calls with the SAME name AND the SAME NSQL text + /// must persist as two distinct rules rather than the second silently + /// overwriting the first. Rule ids used to be derived deterministically + /// from `sha256(name + nsql)`, so identical name+NSQL produced identical + /// ids and `save_filter_rule`'s `INSERT OR REPLACE` clobbered the first + /// rule with no warning. + #[tokio::test] + async fn create_rule_with_duplicate_name_and_nsql_does_not_overwrite_the_first() { + let (addr, _handle, _db, _engine, _dir) = spawn_filters_test_server("correct-token").await; + + let mut client = FiltersClient::connect(format!("http://{addr}")) + .await + .expect("client connects"); + + let first = client + .create_rule(authed_bearer_request(CreateRuleRequest { + name: "Duplicate Rule".to_string(), + nsql: SAMPLE_RULE_NSQL.to_string(), + priority: 1, + })) + .await + .expect("first create_rule succeeds") + .into_inner() + .rule + .expect("response carries the rule"); + + let second = client + .create_rule(authed_bearer_request(CreateRuleRequest { + name: "Duplicate Rule".to_string(), + nsql: SAMPLE_RULE_NSQL.to_string(), + priority: 1, + })) + .await + .expect("second create_rule succeeds") + .into_inner() + .rule + .expect("response carries the rule"); + + assert_ne!( + first.id, second.id, + "identical name+NSQL must still get distinct ids" + ); + + let list_response = client + .list_rules(authed_bearer_request(ListRulesRequest {})) + .await + .expect("list_rules succeeds") + .into_inner(); + assert_eq!( + list_response.rules.len(), + 2, + "both rules must persist, neither overwritten" + ); + let ids: std::collections::HashSet<_> = + list_response.rules.iter().map(|r| r.id.clone()).collect(); + assert!(ids.contains(&first.id)); + assert!(ids.contains(&second.id)); + } + #[tokio::test] async fn delete_rule_rejects_empty_id() { let (addr, _handle, _db, _engine, _dir) = spawn_filters_test_server("correct-token").await; -- 2.54.0 From 545f06fa7b0b18be8d93eb1055cf6d5601417642 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 11:40:15 -0500 Subject: [PATCH 13/95] fix(store): fail export when its WORM audit write fails (#268) Export previously discarded the WORM audit write's result with `let _ = ...`, so a failed audit record still produced a successful export -- an unaudited export was indistinguishable from an audited one, defeating the tamper-evidence the WORM log exists to provide. Refs #246 Co-authored-by: James Maes --- crates/nuncio-store/src/db.rs | 44 ++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index bddedb5..8b52b76 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -2152,14 +2152,15 @@ impl DatabaseEngine { } }; - // Record WORM audit log for this data export - let _ = self - .append_worm_audit_record( - "system.export", - "data.export", - output_path.to_string_lossy().as_bytes(), - ) - .await; + // Record WORM audit log for this data export. If the audit record cannot be + // persisted, the export must not report success: an export without a WORM + // record is indistinguishable from an unaudited (and thus untrusted) export. + self.append_worm_audit_record( + "system.export", + "data.export", + output_path.to_string_lossy().as_bytes(), + ) + .await?; Ok(nuncio_core::ExportSummary { output_path: output_path.to_string_lossy().to_string(), @@ -3393,6 +3394,33 @@ mod tests { assert_eq!(scoped[0].id, "msg-1"); } + /// An export whose WORM audit record cannot be persisted must never report success -- + /// an unaudited export would be indistinguishable from a properly-audited one, defeating + /// the tamper-evidence the WORM log exists to provide. Forces a genuine storage failure + /// (the pool is closed out from under the engine) rather than a contrived one. + #[tokio::test] + async fn export_messages_to_file_fails_when_worm_audit_write_fails() { + let (engine, dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + engine + .save_email(&export_test_email("msg-1", "acct-a", "inbox")) + .await + .unwrap(); + let messages = engine.list_messages_for_export(None, None).await.unwrap(); + + engine.close().await; + + let output_path = dir.path().join("export.json"); + let result = engine + .export_messages_to_file(&messages, nuncio_core::ExportFormat::Json, &output_path) + .await; + + assert!( + result.is_err(), + "an export whose WORM audit record cannot be written must never report success; \ + got: {result:?}" + ); + } + #[tokio::test] async fn verify_worm_audit_chain_report_reports_valid_on_an_intact_chain() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); -- 2.54.0 From 335c5436837c489d51e82f502bcc1176c04b8479 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 11:51:21 -0500 Subject: [PATCH 14/95] fix(store): surface unparseable filter rules instead of dropping them (#269) list_filter_rules silently dropped stored rules that failed to re-parse, making a rule disappear from enforcement with no trace. Now each unparseable row is WARN-logged with its rule id and skipped, so the failure is visible instead of indistinguishable from "never uploaded". Also stop nunciod's boot-time filter load from collapsing a genuine database error into an empty rule set: load_initial_filter_rules propagates the error (ERROR-logged) instead of unwrap_or_default(), so a DB failure fails startup visibly rather than quietly disabling all filtering. Refs #247 Co-authored-by: James Maes --- Cargo.lock | 1 + crates/nuncio-store/Cargo.toml | 3 + crates/nuncio-store/src/db.rs | 109 ++++++++++++++++++++++++++++++--- crates/nunciod/src/lib.rs | 53 ++++++++++++++++ crates/nunciod/src/main.rs | 6 +- 5 files changed, 162 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5cad9b..df41e19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2624,6 +2624,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tracing", + "tracing-subscriber", "zeroize", ] diff --git a/crates/nuncio-store/Cargo.toml b/crates/nuncio-store/Cargo.toml index d60e957..6d8df27 100644 --- a/crates/nuncio-store/Cargo.toml +++ b/crates/nuncio-store/Cargo.toml @@ -28,3 +28,6 @@ hex = "0.4" hmac.workspace = true sha2.workspace = true chrono.workspace = true + +[dev-dependencies] +tracing-subscriber.workspace = true diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 8b52b76..d42a051 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -1589,14 +1589,29 @@ impl DatabaseEngine { let mut rules = Vec::new(); for (id, name, priority, enabled, nsql_text, created_at, updated_at) in rows { - if let Ok(mut parsed) = - nuncio_filter::NsqlParser::parse_rule(&name, priority as i32, &nsql_text) - { - parsed.id = id; - parsed.enabled = enabled != 0; - parsed.created_at = created_at; - parsed.updated_at = updated_at; - rules.push(parsed); + match nuncio_filter::NsqlParser::parse_rule(&name, priority as i32, &nsql_text) { + Ok(mut parsed) => { + parsed.id = id; + parsed.enabled = enabled != 0; + parsed.created_at = created_at; + parsed.updated_at = updated_at; + rules.push(parsed); + } + Err(err) => { + // A stored rule that no longer parses (e.g. the NSQL grammar + // changed, or the row was hand-edited) must not vanish from + // enforcement without a trace: that silently disables the + // rule's protection with no operator-visible signal. Warn + // loudly with the rule id and skip it rather than either + // dropping it unnoticed or failing the whole listing over + // one bad row. + tracing::warn!( + rule_id = %id, + rule_name = %name, + error = %err, + "stored filter rule failed to parse and will not be enforced" + ); + } } } Ok(rules) @@ -3180,6 +3195,84 @@ mod tests { assert_eq!(rules_after.len(), 0); } + /// A rule row that no longer re-parses (e.g. hand-edited NSQL, or a grammar + /// change that invalidates previously-stored text) must not vanish from + /// `list_filter_rules` invisibly. This asserts the observable policy chosen + /// for that case: the rule is excluded from the returned (enforced) set, but + /// a `WARN`-level log naming the specific rule id is emitted so an operator + /// can see that a rule stopped being enforced -- silent disappearance would + /// look identical to "this rule was never uploaded", which is the failure + /// mode this test guards against. + #[tokio::test] + async fn list_filter_rules_warns_on_unparseable_row_instead_of_dropping_silently() { + use std::sync::{Arc, Mutex}; + use tracing_subscriber::fmt::MakeWriter; + + #[derive(Clone, Default)] + struct CapturedLogs(Arc>>); + + impl std::io::Write for CapturedLogs { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 + .lock() + .expect("log buffer lock") + .extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> MakeWriter<'a> for CapturedLogs { + type Writer = CapturedLogs; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } + } + + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Build a structurally valid rule (so the store-level id/name/priority + // bookkeeping is untouched), then corrupt only the persisted NSQL text + // to simulate a row that no longer parses under the current grammar. + let nsql = "SELECT * FROM emails WHERE subject CONTAINS 'Spam' ACTION DELETE"; + let mut rule = nuncio_filter::NsqlParser::parse_rule("Spam Filter", 1, nsql).unwrap(); + rule.nsql_text = "THIS IS NOT VALID NSQL !!!".to_string(); + engine + .save_filter_rule(&rule) + .await + .expect("save filter rule"); + + let logs = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_ansi(false) + .finish(); + + // `set_default` returns a guard that stays active across `.await` + // points as long as this test stays on the single `#[tokio::test]` + // current-thread executor, which is the default runtime flavor and + // exactly what is used here. + let guard = tracing::subscriber::set_default(subscriber); + let rules = engine.list_filter_rules().await.expect("list filter rules"); + drop(guard); + + // The unparseable row must not appear in the enforced rule set. + assert!(rules.is_empty()); + + let captured = String::from_utf8(logs.0.lock().expect("log buffer lock").clone()) + .expect("captured log is valid utf8"); + assert!( + captured.contains(&rule.id), + "expected the specific rule id to be named in the WARN log, got: {captured}" + ); + assert!( + captured.to_ascii_uppercase().contains("WARN"), + "expected a WARN-level log for the unparseable rule, got: {captured}" + ); + } + #[tokio::test] async fn test_pending_mutations_and_keyset_chunking() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); diff --git a/crates/nunciod/src/lib.rs b/crates/nunciod/src/lib.rs index 94daff7..7b3413b 100644 --- a/crates/nunciod/src/lib.rs +++ b/crates/nunciod/src/lib.rs @@ -35,6 +35,27 @@ pub fn default_db_path() -> std::path::PathBuf { } } +/// Loads the persisted filter rule set at daemon boot. +/// +/// A load failure here is a genuine database problem, not "no rules are +/// configured": an empty database legitimately has zero rules, but a query +/// error does not mean zero rules, it means the true rule set is unknown. +/// Collapsing that error into an empty `Vec` would silently disable all +/// filtering with nothing in the logs to explain why. This surfaces the +/// failure loudly and propagates it instead, so daemon startup fails +/// visibly rather than starting with an unknown -- possibly wide open -- +/// filtering posture. +pub async fn load_initial_filter_rules( + db: &nuncio_store::DatabaseEngine, +) -> Result, nuncio_store::DatabaseError> { + db.list_filter_rules().await.inspect_err(|err| { + tracing::error!( + error = %err, + "failed to load filter rules from database at startup; refusing to start with an unknown rule set" + ); + }) +} + /// Environment variable that opts `nunciod` in to its autonomous /// background auto-update-check loop. /// @@ -92,6 +113,38 @@ mod tests { assert!(!auto_update_task_enabled()); } + #[tokio::test] + async fn load_initial_filter_rules_propagates_db_errors_instead_of_returning_empty() { + let (engine, _dir) = nuncio_store::DatabaseEngine::connect_ephemeral() + .await + .unwrap(); + + // A pool that has been closed makes every subsequent query fail + // deterministically, standing in for a genuine boot-time database + // problem (as opposed to a healthy, merely empty, rule table). + engine.close().await; + + let result = load_initial_filter_rules(&engine).await; + assert!( + result.is_err(), + "a genuine database error must propagate as Err, not be silently \ + mapped to an empty rule set (which would look identical to a \ + database with zero rules configured)" + ); + } + + #[tokio::test] + async fn load_initial_filter_rules_returns_empty_vec_for_a_healthy_empty_database() { + let (engine, _dir) = nuncio_store::DatabaseEngine::connect_ephemeral() + .await + .unwrap(); + + let rules = load_initial_filter_rules(&engine) + .await + .expect("a healthy, empty database must load successfully"); + assert!(rules.is_empty()); + } + #[test] fn default_db_path_is_persistent_under_dot_nuncio_directory() { // Deliberately does not mutate `USERPROFILE`/`HOME` (real env vars diff --git a/crates/nunciod/src/main.rs b/crates/nunciod/src/main.rs index 21b07bd..c98aa72 100644 --- a/crates/nunciod/src/main.rs +++ b/crates/nunciod/src/main.rs @@ -36,8 +36,10 @@ async fn main() -> Result<(), Box> { // wrapper instances. let account_secrets = Arc::new(nuncio_store::vault::SecretManager::production()); - // Load active rules from SQLite - let initial_rules = db.list_filter_rules().await.unwrap_or_default(); + // Load active rules from SQLite. See `load_initial_filter_rules` for why + // a load failure must fail startup rather than silently become an empty + // rule set. + let initial_rules = nunciod::load_initial_filter_rules(&db).await?; let filter_engine = Arc::new(FilterEngine::new(initial_rules)?); // Real inbound-sync `CoreCommand` consumer. This is the ONE place that -- 2.54.0 From 167955288bebc23d662ef799773072d9d72d1a79 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 12:01:40 -0500 Subject: [PATCH 15/95] fix(store): preserve audit tables and real TLS modes during salvage (#270) Salvage previously dropped worm_audit_records and filter_execution_logs entirely, losing tamper-evident audit/ledger history on recovery, and hardcoded every salvaged account's imap_tls_mode/smtp_tls_mode back to ImplicitTls regardless of the account's real configured mode. Salvage now carries both ledger tables through recovery verbatim, including their hash-chain columns, and reads each account's real TLS modes from the salvaged row, WARN-logging and falling back only when a raw value is genuinely unrecognized. Refs #248 Co-authored-by: James Maes --- crates/nuncio-store/src/recovery.rs | 294 ++++++++++++++++++++++++++-- 1 file changed, 282 insertions(+), 12 deletions(-) diff --git a/crates/nuncio-store/src/recovery.rs b/crates/nuncio-store/src/recovery.rs index 324a8a6..e400a94 100644 --- a/crates/nuncio-store/src/recovery.rs +++ b/crates/nuncio-store/src/recovery.rs @@ -6,6 +6,11 @@ use std::path::{Path, PathBuf}; use tracing::{info, warn}; /// Raw row shape for an account configuration record fetched from SQLite during salvage. +/// The trailing two `String` columns are the raw `imap_tls_mode`/`smtp_tls_mode` text values +/// (e.g. `"start_tls"`) -- carried through as the literal on-disk token rather than already +/// parsed to [`nuncio_core::TlsMode`], so [`SqliteRecoveryEngine::salvage_accounts`] can log +/// precisely which raw value it could not recognize, if any, instead of a decode failure +/// having already been silently absorbed by this row type. type SalvagedAccountRow = ( String, String, @@ -18,6 +23,8 @@ type SalvagedAccountRow = ( i64, Option, Option, + String, + String, ); /// Summary report of database self-healing recovery output. @@ -33,6 +40,10 @@ pub struct RecoverySummary { pub salvaged_conditions_count: usize, /// Number of salvaged filter action specifications. pub salvaged_actions_count: usize, + /// Number of salvaged WORM audit ledger records. + pub salvaged_audit_records_count: usize, + /// Number of salvaged filter execution log entries. + pub salvaged_execution_logs_count: usize, /// Indicates whether remote protocol resynchronization was initiated. pub resync_triggered: bool, } @@ -191,21 +202,41 @@ impl SqliteRecoveryEngine { // Step 2: Attempt reading valid records from preserved backup file let backup_url = format!("sqlite://{}", backup_path.to_string_lossy()); - let (salvaged_accounts, salvaged_rules, salvaged_conditions, salvaged_actions) = if let Ok( - pool, - ) = - sqlx::sqlite::SqlitePoolOptions::new() - .max_connections(1) - .connect(&backup_url) - .await + let ( + salvaged_accounts, + salvaged_rules, + salvaged_conditions, + salvaged_actions, + salvaged_audit_records, + salvaged_execution_logs, + ) = if let Ok(pool) = sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(1) + .connect(&backup_url) + .await { let accounts = Self::salvage_accounts(&pool).await; let (rules, conditions, actions) = Self::salvage_filter_tables(&pool).await; + let audit_records = Self::salvage_worm_audit_records(&pool).await; + let execution_logs = Self::salvage_filter_execution_logs(&pool).await; pool.close().await; - (accounts, rules, conditions, actions) + ( + accounts, + rules, + conditions, + actions, + audit_records, + execution_logs, + ) } else { warn!("Failed to open backup connection for salvage; proceeding with clean database reset."); - (Vec::new(), Vec::new(), Vec::new(), Vec::new()) + ( + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + ) }; // Step 3: Remove or truncate damaged database files at target_db_path @@ -287,12 +318,62 @@ impl SqliteRecoveryEngine { } } + // Restore the WORM audit ledger and filter execution log ledger verbatim, including + // their hash-chain columns (`record_hmac`/`hash`, `previous_block_hash`/`prev_hash`). + // These are re-signed with the same HMAC key material the corrupted database used + // (both engines are provisioned from the same `secrets` vault), so a chain that + // verified before corruption still verifies after salvage -- salvage restores the + // ledger's tamper-evidence, it does not merely copy rows that happen to look right. + let mut restored_audit_records_count = 0; + for (sequence, timestamp_ns, actor, action, data_hash, previous_block_hash, record_hmac) in + &salvaged_audit_records + { + let res = sqlx::query( + "INSERT INTO worm_audit_records (sequence, timestamp_ns, actor, action, data_hash, previous_block_hash, record_hmac) VALUES (?, ?, ?, ?, ?, ?, ?)" + ) + .bind(sequence) + .bind(timestamp_ns) + .bind(actor) + .bind(action) + .bind(data_hash) + .bind(previous_block_hash) + .bind(record_hmac) + .execute(fresh_engine.pool()) + .await; + if res.is_ok() { + restored_audit_records_count += 1; + } + } + + let mut restored_execution_logs_count = 0; + for (id, rule_id, message_id, action_taken, matched_at, prev_hash, hash) in + &salvaged_execution_logs + { + let res = sqlx::query( + "INSERT INTO filter_execution_logs (id, rule_id, message_id, action_taken, matched_at, prev_hash, hash) VALUES (?, ?, ?, ?, ?, ?, ?)" + ) + .bind(id) + .bind(rule_id) + .bind(message_id) + .bind(action_taken) + .bind(matched_at) + .bind(prev_hash) + .bind(hash) + .execute(fresh_engine.pool()) + .await; + if res.is_ok() { + restored_execution_logs_count += 1; + } + } + let summary = RecoverySummary { backup_path, salvaged_accounts_count: restored_accounts_count, salvaged_rules_count: restored_rules_count, salvaged_conditions_count: restored_conditions_count, salvaged_actions_count: restored_actions_count, + salvaged_audit_records_count: restored_audit_records_count, + salvaged_execution_logs_count: restored_execution_logs_count, resync_triggered: true, }; @@ -302,7 +383,7 @@ impl SqliteRecoveryEngine { async fn salvage_accounts(pool: &sqlx::SqlitePool) -> Vec { let rows: Result, _> = sqlx::query_as( - "SELECT id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port FROM accounts" + "SELECT id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode FROM accounts" ) .fetch_all(pool) .await; @@ -323,6 +404,8 @@ impl SqliteRecoveryEngine { sync_interval_secs, smtp_host, smtp_port, + imap_tls_mode_raw, + smtp_tls_mode_raw, )| { let protocol = serde_json::from_str(&protocol_str) .unwrap_or(nuncio_core::AccountProtocol::ImapSmtp); @@ -335,6 +418,10 @@ impl SqliteRecoveryEngine { let resolved_smtp_host = smtp_host.unwrap_or_else(|| server_host.clone()); let resolved_smtp_port = smtp_port.map(|p| p as u16).unwrap_or(server_port as u16); + let imap_tls_mode = + Self::parse_salvaged_tls_mode(&id, "imap_tls_mode", &imap_tls_mode_raw); + let smtp_tls_mode = + Self::parse_salvaged_tls_mode(&id, "smtp_tls_mode", &smtp_tls_mode_raw); nuncio_core::AccountConfig { id, name, @@ -345,8 +432,8 @@ impl SqliteRecoveryEngine { smtp_host: resolved_smtp_host, smtp_port: resolved_smtp_port, use_tls: use_tls != 0, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + imap_tls_mode, + smtp_tls_mode, keyring_secret_key, sync_interval_secs: sync_interval_secs as u64, } @@ -357,6 +444,64 @@ impl SqliteRecoveryEngine { } } + /// Parse a salvaged account's raw `imap_tls_mode`/`smtp_tls_mode` column text back into + /// [`nuncio_core::TlsMode`], preserving the account's real configured security posture + /// (implicit TLS, STARTTLS, or plaintext) rather than hardcoding every salvaged account + /// back to implicit TLS regardless of how it was actually configured -- silently + /// upgrading a STARTTLS account or downgrading a plaintext one to "implicit TLS" on + /// recovery would misrepresent the account's security posture to the operator. + /// + /// A raw value that does not match one of the three known on-disk tokens (a row from a + /// build that predates this column, or a distinct new corruption) cannot be honestly + /// recovered. Rather than silently guessing at one of the other two modes, this WARN-logs + /// the account id and the unrecognized raw value, and falls back to the safest mode + /// (implicit TLS) -- the operator can see in the daemon's logs exactly which account + /// needs its TLS mode re-confirmed, instead of a quiet, invisible substitution. + fn parse_salvaged_tls_mode(account_id: &str, column: &str, raw: &str) -> nuncio_core::TlsMode { + match raw { + "implicit_tls" => nuncio_core::TlsMode::ImplicitTls, + "start_tls" => nuncio_core::TlsMode::StartTls, + "plain" => nuncio_core::TlsMode::Plain, + other => { + warn!( + "Salvage could not recognize account {account_id}'s {column} value {other:?}; \ + defaulting to ImplicitTls and flagging for operator review rather than \ + silently guessing its prior TLS mode." + ); + nuncio_core::TlsMode::ImplicitTls + } + } + } + + /// Read back valid rows from `worm_audit_records` in the backup connection, preserving + /// every column including the hash-chain fields (`previous_block_hash`/`record_hmac`) so + /// the restored ledger's tamper-evidence carries through salvage rather than being + /// silently dropped. + async fn salvage_worm_audit_records( + pool: &sqlx::SqlitePool, + ) -> Vec<(i64, i64, String, String, String, String, String)> { + sqlx::query_as::<_, (i64, i64, String, String, String, String, String)>( + "SELECT sequence, timestamp_ns, actor, action, data_hash, previous_block_hash, record_hmac FROM worm_audit_records ORDER BY sequence ASC" + ) + .fetch_all(pool) + .await + .unwrap_or_default() + } + + /// Read back valid rows from `filter_execution_logs` in the backup connection, preserving + /// every column including the hash-chain fields (`prev_hash`/`hash`) so the restored + /// ledger's tamper-evidence carries through salvage rather than being silently dropped. + async fn salvage_filter_execution_logs( + pool: &sqlx::SqlitePool, + ) -> Vec<(i64, String, String, String, i64, String, String)> { + sqlx::query_as::<_, (i64, String, String, String, i64, String, String)>( + "SELECT id, rule_id, message_id, action_taken, matched_at, prev_hash, hash FROM filter_execution_logs ORDER BY id ASC" + ) + .fetch_all(pool) + .await + .unwrap_or_default() + } + /// Read back valid rows from `filter_rules` / `filter_conditions` / `filter_actions` in the /// backup connection, using column names/order matching the live schema (see /// `DatabaseEngine::migrate`). `filter_conditions.value`/`filter_actions.target` are the @@ -558,6 +703,131 @@ mod tests { assert_eq!(rules[0].name, "Spam Filter"); } + #[tokio::test] + async fn salvage_preserves_worm_audit_and_filter_execution_log_chains() { + let dir = tempdir().expect("tempdir"); + let db_path = dir.path().join("ledger_salvage_test.db"); + let backup_dir = dir.path().join("corrupted_backups"); + let secrets = crate::vault::SecretManager::mock(); + + // Step 1: Seed a WORM audit record and a filter execution log entry, each of which + // is only valid because it is cryptographically hash-chained to its predecessor. + { + let engine = DatabaseEngine::connect_file(&db_path, &secrets) + .await + .unwrap(); + engine + .append_worm_audit_record("system.test", "account.created", b"payload-1") + .await + .unwrap(); + engine + .append_worm_audit_record("system.test", "account.updated", b"payload-2") + .await + .unwrap(); + engine + .save_filter_execution_log("rule-1", "msg-1", "delete") + .await + .unwrap(); + engine + .save_filter_execution_log("rule-1", "msg-2", "archive") + .await + .unwrap(); + + assert!(engine.verify_worm_audit_chain().await.is_ok()); + assert!(engine.verify_execution_log_chain().await.unwrap()); + } + + // Step 2: Perform salvage recovery, reusing the same secrets vault -- the WORM/ledger + // HMAC keys are provisioned from `secrets`, so the restored ledgers verify against + // the same key material the corrupted database used, exactly as in the real recovery + // flow (the daemon always re-provisions salvage from the account's existing vault). + let summary = SqliteRecoveryEngine::salvage(&db_path, &db_path, &backup_dir, &secrets) + .await + .expect("salvage succeeds"); + + assert_eq!( + summary.salvaged_audit_records_count, 2, + "salvage must carry worm_audit_records through recovery instead of silently \ + dropping the tamper-evident audit ledger" + ); + assert_eq!( + summary.salvaged_execution_logs_count, 2, + "salvage must carry filter_execution_logs through recovery instead of silently \ + dropping the filter ledger" + ); + + // Step 3: The restored ledgers must still be present AND still pass hash-chain + // verification -- proving the chain-linking columns (`previous_block_hash`/ + // `record_hmac`, `prev_hash`/`hash`) were preserved verbatim, not just the row data. + let fresh = DatabaseEngine::connect_file(&db_path, &secrets) + .await + .unwrap(); + + let audit_records = fresh.list_worm_audit_records(100, 0).await.unwrap(); + assert_eq!(audit_records.len(), 2); + assert!( + fresh.verify_worm_audit_chain().await.is_ok(), + "restored WORM audit chain must still verify after salvage" + ); + + let execution_logs = fresh.list_filter_execution_logs(100).await.unwrap(); + assert_eq!(execution_logs.len(), 2); + assert!( + fresh.verify_execution_log_chain().await.unwrap(), + "restored filter execution log chain must still verify after salvage" + ); + } + + #[tokio::test] + async fn salvage_preserves_real_tls_mode_instead_of_downgrading_to_implicit() { + let dir = tempdir().expect("tempdir"); + let db_path = dir.path().join("tls_mode_salvage_test.db"); + let backup_dir = dir.path().join("corrupted_backups"); + let secrets = crate::vault::SecretManager::mock(); + + // Step 1: Seed an account configured for STARTTLS on IMAP and plaintext on SMTP -- + // deliberately neither is `ImplicitTls`, so a salvage that hardcodes the TLS mode + // back to `ImplicitTls` is caught by asserting the real modes survive. + { + let engine = DatabaseEngine::connect_file(&db_path, &secrets) + .await + .unwrap(); + let acct = AccountConfig { + id: "acct-starttls-1".to_string(), + name: "STARTTLS Account".to_string(), + email_address: "starttls@nuncio.mx".to_string(), + protocol: nuncio_core::AccountProtocol::ImapSmtp, + server_host: "imap.nuncio.mx".to_string(), + server_port: 143, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 587, + use_tls: true, + imap_tls_mode: nuncio_core::TlsMode::StartTls, + smtp_tls_mode: nuncio_core::TlsMode::Plain, + keyring_secret_key: "nuncio/acct-starttls-1".to_string(), + sync_interval_secs: 60, + }; + engine.save_account(&acct).await.unwrap(); + } + + // Step 2: Perform salvage recovery. + let summary = SqliteRecoveryEngine::salvage(&db_path, &db_path, &backup_dir, &secrets) + .await + .expect("salvage succeeds"); + assert_eq!(summary.salvaged_accounts_count, 1); + + // Step 3: The salvaged account must keep its real configured TLS modes, not be + // silently reset to `ImplicitTls` -- that would misrepresent the account's actual, + // intentionally-configured security posture (STARTTLS on IMAP, plaintext on SMTP). + let fresh = DatabaseEngine::connect_file(&db_path, &secrets) + .await + .unwrap(); + let accounts = fresh.list_accounts().await.unwrap(); + assert_eq!(accounts.len(), 1); + assert_eq!(accounts[0].imap_tls_mode, nuncio_core::TlsMode::StartTls); + assert_eq!(accounts[0].smtp_tls_mode, nuncio_core::TlsMode::Plain); + } + // NOTE: there is deliberately no test combining *header* corruption (bytes 0-16 -- the // fixed "SQLite format 3\0" magic string) with a nonzero salvaged-rule-count assertion. // A corrupted header makes the file entirely unopenable by SQLite (`SQLITE_NOTADB`), by -- 2.54.0 From e3d34137366f9c513cc13b670ea110d82ee5e410 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 12:41:24 -0500 Subject: [PATCH 16/95] feat: production-wire Calendar sync via first-class CalDAV accounts (#272) * feat(proto): add CALDAV account protocol and collection_url Add ACCOUNT_PROTOCOL_CALDAV = 3 and a generic AccountConfig.collection_url (field 14) so a CalDAV account can be a first-class account entry. The field is deliberately generic (DAV collection URL) for CardDAV reuse. Regenerated the committed descriptor golden; additive within nuncio.v1. Refs #254 #178 * feat(core): model CalDAV accounts with a generic collection URL Add AccountProtocol::CalDav and AccountConfig.collection_url, plus an is_dav() helper. Validation is now protocol-aware: a DAV account requires a non-empty http(s) collection_url and skips the IMAP/SMTP endpoint checks, while mail accounts are unchanged. Refs #254 #178 * feat(store): persist account collection_url with backfill-safe migration Add a nullable collection_url column to the accounts table via an ensure_accounts_dav_columns migration that mirrors the SMTP/TLS-mode column backfills, and round-trip it through save_account/list_accounts/ get_account. Salvage preserves collection_url alongside the TLS modes. Refs #254 #178 * feat(daemon): wire the production CalDAV sync path Calendar/Sync with no injected backend now resolves configured CalDAV accounts (filtered by protocol, honoring a targeted account_id), reads each credential from the keyring vault, builds a real CalDavClient from the persisted collection_url, and syncs events via calendar_sync::sync_caldav_account. The honest error now fires only when no CalDAV account is configured. AddAccount validates CalDAV accounts (http(s) collection_url + password) via the protocol-aware core validation. The mail sync/outbox paths skip DAV accounts, which have no mail backend. The CalendarEngineOverrides seam is retained for tests. Refs #254 #178 * feat(cli): create and list CalDAV accounts account add gains --protocol (imap-smtp/jmap/caldav) and --collection-url; a caldav account is rejected before dialing the daemon if the collection URL is missing, and prompts for a password like a mail account. account list renders CalDAV accounts with their collection URL. Refs #254 #178 * test(daemon): prove the production CalDAV sync path offline Add a full-daemon E2E that configures a CalDAV account over the real Accounts API and drives Calendar/Sync with NO injected backend, so the daemon builds a real CalDavClient against a WireMock CalDAV server and persists the fetched events, immediately visible via ListEvents. Also asserts the honest error for an unconfigured account and that Calendar stays behind its BearerAuthInterceptor. Fully offline: WireMock + mock keyring, no live network. Refs #254 #178 * test(daemon): add collection_url to account test fixtures Fill in the new AccountConfig.collection_url field in the orchestrator and send test fixtures so they construct the mail account model. Refs #254 #178 --------- Co-authored-by: James Maes --- crates/nuncio-cli/src/args.rs | 12 +- crates/nuncio-cli/src/runner.rs | 161 ++++++++++-- crates/nuncio-cli/tests/cli_system_test.rs | 2 + crates/nuncio-core/src/config.rs | 125 ++++++++- .../proto/nuncio/v1/descriptor.bin | Bin 12785 -> 12854 bytes .../nuncio-proto/proto/nuncio/v1/nuncio.proto | 10 + crates/nuncio-store/src/db.rs | 88 ++++++- crates/nuncio-store/src/recovery.rs | 8 +- crates/nunciod/src/calendar_sync.rs | 46 +++- crates/nunciod/src/grpc.rs | 146 +++++++++-- crates/nunciod/src/orchestrator.rs | 1 + crates/nunciod/src/outbox.rs | 7 +- crates/nunciod/src/send.rs | 1 + crates/nunciod/src/sync.rs | 43 +++- .../tests/accounts_lifecycle_e2e_test.rs | 1 + .../tests/caldav_production_e2e_test.rs | 241 ++++++++++++++++++ .../nunciod/tests/outbox_executor_e2e_test.rs | 1 + crates/nunciod/tests/spine_e2e_test.rs | 1 + 18 files changed, 816 insertions(+), 78 deletions(-) create mode 100644 crates/nunciod/tests/caldav_production_e2e_test.rs diff --git a/crates/nuncio-cli/src/args.rs b/crates/nuncio-cli/src/args.rs index 77a3314..554f691 100644 --- a/crates/nuncio-cli/src/args.rs +++ b/crates/nuncio-cli/src/args.rs @@ -202,11 +202,19 @@ pub enum UpdateSubcommand { pub enum AccountSubcommand { /// List all configured mail and calendar accounts. List, - /// Add and configure a new mail account securely. + /// Add and configure a new mail or CalDAV account securely. Add { /// Email address or account username. #[arg(short, long, help = "Email address or account username")] email: String, + /// Account protocol: imap-smtp (default), jmap, or caldav. A caldav + /// account requires --collection-url and ignores the IMAP/SMTP flags. + #[arg(long, default_value = "imap-smtp", help = "Account protocol")] + protocol: String, + /// Fully-qualified CalDAV calendar-collection URL (required for a + /// caldav account; ignored for mail accounts). + #[arg(long, help = "CalDAV calendar-collection URL (caldav only)")] + collection_url: Option, /// IMAP server hostname. #[arg(long, default_value = "mail.kof22.com", help = "IMAP server hostname")] imap_host: String, @@ -569,6 +577,8 @@ mod tests { Commands::Account { action: AccountSubcommand::Add { email: "james.maes@kof22.com".to_string(), + protocol: "imap-smtp".to_string(), + collection_url: None, imap_host: "mail.kof22.com".to_string(), imap_port: 993, smtp_host: "mail.kof22.com".to_string(), diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index e1fbb93..f14f0de 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -39,6 +39,20 @@ fn map_tls_mode_to_proto(mode: nuncio_core::TlsMode) -> nuncio_proto::v1::TlsMod } } +/// Parse an `account add --protocol` string into its wire-format +/// `nuncio.v1.AccountProtocol` value. Accepts both the kebab-case +/// serialization (`imap-smtp`) and common aliases. +fn parse_account_protocol(protocol: &str) -> Result { + match protocol.to_ascii_lowercase().as_str() { + "imap-smtp" | "imap_smtp" | "imap" => Ok(nuncio_proto::v1::AccountProtocol::ImapSmtp), + "jmap" => Ok(nuncio_proto::v1::AccountProtocol::Jmap), + "caldav" | "cal-dav" | "cal_dav" => Ok(nuncio_proto::v1::AccountProtocol::Caldav), + other => Err(format!( + "invalid protocol '{other}' (expected imap-smtp, jmap, or caldav)" + )), + } +} + /// Renders a `nuncio.v1.Message` (as returned by the daemon's `Mail` gRPC /// service) into the JSON shape used by `mail list`/`mail read`'s /// `--json` output. @@ -270,6 +284,8 @@ impl HeadlessRunner { AccountSubcommand::List => self.handle_accounts_list(json_mode).await, AccountSubcommand::Add { email, + protocol, + collection_url, imap_host, imap_port, smtp_host, @@ -280,6 +296,8 @@ impl HeadlessRunner { } => { self.handle_add_account( email, + protocol, + collection_url.as_deref(), imap_host, *imap_port, smtp_host, @@ -1603,6 +1621,8 @@ impl HeadlessRunner { async fn handle_add_account( &self, email: &str, + protocol: &str, + collection_url: Option<&str>, imap_host: &str, imap_port: u16, smtp_host: &str, @@ -1621,23 +1641,43 @@ impl HeadlessRunner { Err(e) => return Self::render_error(&e, json_mode), }; + let account_protocol = match parse_account_protocol(protocol) { + Ok(p) => p, + Err(e) => return Self::render_error(&e, json_mode), + }; + let is_caldav = account_protocol == nuncio_proto::v1::AccountProtocol::Caldav; + if is_caldav && collection_url.map(str::trim).unwrap_or_default().is_empty() { + return Self::render_error("a caldav account requires --collection-url", json_mode); + } + let keyring_key = format!("nuncio/{}", email); let account_id = format!("acct-{}", email.replace('@', "-at-").replace('.', "-")); + // A CalDAV account is addressed by its collection URL, not by mail + // host/port endpoints, so those are left empty for it. let proto_config = nuncio_proto::v1::AccountConfig { id: account_id.clone(), name: email.to_string(), email_address: email.to_string(), - protocol: nuncio_proto::v1::AccountProtocol::ImapSmtp.into(), - server_host: imap_host.to_string(), - server_port: u32::from(imap_port), + protocol: account_protocol.into(), + server_host: if is_caldav { + String::new() + } else { + imap_host.to_string() + }, + server_port: if is_caldav { 0 } else { u32::from(imap_port) }, use_tls: true, imap_tls_mode: map_tls_mode_to_proto(imap_tls_mode).into(), smtp_tls_mode: map_tls_mode_to_proto(smtp_tls_mode).into(), keyring_secret_key: keyring_key.clone(), sync_interval_secs: 300, - smtp_host: smtp_host.to_string(), - smtp_port: u32::from(smtp_port), + smtp_host: if is_caldav { + String::new() + } else { + smtp_host.to_string() + }, + smtp_port: if is_caldav { 0 } else { u32::from(smtp_port) }, + collection_url: collection_url.unwrap_or_default().to_string(), }; let mut client = match self.connect_accounts_client().await { @@ -1654,7 +1694,23 @@ impl HeadlessRunner { { Ok(response) => { let account_id = response.into_inner().id; - if json_mode { + if is_caldav { + let collection = collection_url.unwrap_or_default(); + if json_mode { + format_json(&json!({ + "configured": true, + "account_id": account_id, + "email": email, + "protocol": "caldav", + "collection_url": collection, + "keyring_key": keyring_key + })) + } else { + format!( + "CalDAV account '{email}' (ID: {account_id}) added via nunciod daemon with collection URL {collection}" + ) + } + } else if json_mode { format_json(&json!({ "configured": true, "account_id": account_id, @@ -1716,6 +1772,7 @@ impl HeadlessRunner { "smtp_tls_mode": a.smtp_tls_mode().as_str_name(), "keyring_secret_key": a.keyring_secret_key, "sync_interval_secs": a.sync_interval_secs, + "collection_url": a.collection_url, }) }) .collect(); @@ -1726,17 +1783,24 @@ impl HeadlessRunner { accounts.len() ); for a in &accounts { - out.push_str(&format!( - "\n [{}] {} <{}> IMAP {}:{} SMTP {}:{} TLS={}", - a.id, - a.name, - a.email_address, - a.server_host, - a.server_port, - a.smtp_host, - a.smtp_port, - a.use_tls, - )); + if a.protocol() == nuncio_proto::v1::AccountProtocol::Caldav { + out.push_str(&format!( + "\n [{}] {} <{}> CalDAV {}", + a.id, a.name, a.email_address, a.collection_url, + )); + } else { + out.push_str(&format!( + "\n [{}] {} <{}> IMAP {}:{} SMTP {}:{} TLS={}", + a.id, + a.name, + a.email_address, + a.server_host, + a.server_port, + a.smtp_host, + a.smtp_port, + a.use_tls, + )); + } } out } @@ -2413,6 +2477,62 @@ mod tests { ); } + #[test] + fn parse_account_protocol_accepts_known_names_and_rejects_garbage() { + assert_eq!( + parse_account_protocol("imap-smtp").expect("valid"), + nuncio_proto::v1::AccountProtocol::ImapSmtp + ); + assert_eq!( + parse_account_protocol("jmap").expect("valid"), + nuncio_proto::v1::AccountProtocol::Jmap + ); + assert_eq!( + parse_account_protocol("caldav").expect("valid"), + nuncio_proto::v1::AccountProtocol::Caldav + ); + assert_eq!( + parse_account_protocol("CalDAV").expect("case-insensitive"), + nuncio_proto::v1::AccountProtocol::Caldav + ); + let err = parse_account_protocol("smoke-signals").expect_err("garbage rejected"); + assert!(err.contains("invalid protocol")); + } + + /// `account add --protocol caldav` with no `--collection-url` must be + /// rejected BEFORE dialing the daemon -- proven by pointing at an address + /// nothing is listening on and confirming the failure is the validation + /// error, not a connection error. + #[tokio::test] + async fn account_add_caldav_requires_collection_url_before_dialing() { + let runner = + HeadlessRunner::ephemeral_with(Arc::new(SecretManager::mock()), "127.0.0.1:1".into()) + .await + .expect("runner init"); + + let out = runner + .execute_command( + &Commands::Account { + action: AccountSubcommand::Add { + email: "cal@nuncio.mx".to_string(), + protocol: "caldav".to_string(), + collection_url: None, + imap_host: "unused".to_string(), + imap_port: 993, + smtp_host: "unused".to_string(), + smtp_port: 465, + imap_mode: "implicit_tls".to_string(), + smtp_mode: "implicit_tls".to_string(), + password: crate::args::PasswordArg("pw".to_string()), + }, + }, + false, + ) + .await; + assert!(out.contains("requires --collection-url")); + assert!(!out.contains("unreachable")); + } + /// `account add` must reject an invalid `--imap-mode`/`--smtp-mode` /// string BEFORE ever dialing the daemon -- proven here by pointing at /// an address nothing is listening on and confirming the failure is the @@ -2429,6 +2549,8 @@ mod tests { &Commands::Account { action: AccountSubcommand::Add { email: "x@y.com".to_string(), + protocol: "imap-smtp".to_string(), + collection_url: None, imap_host: "imap.y.com".to_string(), imap_port: 993, smtp_host: "smtp.y.com".to_string(), @@ -2449,6 +2571,8 @@ mod tests { &Commands::Account { action: AccountSubcommand::Add { email: "x@y.com".to_string(), + protocol: "imap-smtp".to_string(), + collection_url: None, imap_host: "imap.y.com".to_string(), imap_port: 993, smtp_host: "smtp.y.com".to_string(), @@ -2690,6 +2814,7 @@ mod tests { sync_interval_secs: 300, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, + collection_url: String::new(), }], })) } @@ -2719,6 +2844,8 @@ mod tests { &Commands::Account { action: AccountSubcommand::Add { email: "james.maes@kof22.com".to_string(), + protocol: "imap-smtp".to_string(), + collection_url: None, imap_host: "mail.kof22.com".to_string(), imap_port: 993, smtp_host: "mail.kof22.com".to_string(), diff --git a/crates/nuncio-cli/tests/cli_system_test.rs b/crates/nuncio-cli/tests/cli_system_test.rs index a691156..3994ee3 100644 --- a/crates/nuncio-cli/tests/cli_system_test.rs +++ b/crates/nuncio-cli/tests/cli_system_test.rs @@ -116,6 +116,8 @@ async fn account_add_and_list_report_honest_errors_when_daemon_unreachable() { &Commands::Account { action: AccountSubcommand::Add { email: "test@nuncio.mx".to_string(), + protocol: "imap-smtp".to_string(), + collection_url: None, imap_host: "mail.nuncio.mx".to_string(), imap_port: 993, smtp_host: "mail.nuncio.mx".to_string(), diff --git a/crates/nuncio-core/src/config.rs b/crates/nuncio-core/src/config.rs index 7ae07cf..5cbb012 100644 --- a/crates/nuncio-core/src/config.rs +++ b/crates/nuncio-core/src/config.rs @@ -11,6 +11,21 @@ pub enum AccountProtocol { Jmap, /// Legacy IMAP4rev1 + SMTP protocol engines. ImapSmtp, + /// CalDAV (RFC 4791) calendar-collection protocol engine. A CalDAV + /// account is a standalone account entry addressed by its + /// `collection_url` and its own keyring credential; it does not use the + /// IMAP/JMAP/SMTP mail endpoints. + CalDav, +} + +impl AccountProtocol { + /// Whether this protocol speaks WebDAV against a collection URL (as + /// opposed to a mail protocol addressed by host/port). Such accounts + /// require [`AccountConfig::collection_url`] and do not require the + /// IMAP/SMTP mail endpoint fields. + pub fn is_dav(self) -> bool { + matches!(self, AccountProtocol::CalDav) + } } /// Security and transport encryption mode for mail protocol streams (IMAP & SMTP). @@ -43,6 +58,9 @@ pub enum ConfigError { /// The synchronization interval is too short (minimum 10 seconds). #[error("sync interval must be at least 10 seconds")] SyncIntervalTooShort, + /// A DAV-protocol account's collection URL is not a valid http(s) URL. + #[error("collection_url must be a valid http(s) URL")] + InvalidCollectionUrl, } /// Complete configuration schema for a mail & calendar account. @@ -76,6 +94,12 @@ pub struct AccountConfig { pub keyring_secret_key: String, /// Background sync polling interval in seconds (minimum 10s). pub sync_interval_secs: u64, + /// Fully-qualified collection URL for DAV-protocol accounts (CalDAV + /// today, CardDAV later). Set for accounts whose protocol speaks WebDAV; + /// empty for mail accounts. This is a full URL (scheme + host + path), + /// not a bare host, because a DAV client dispatches requests directly at + /// the collection URL. + pub collection_url: String, } impl AccountConfig { @@ -103,19 +127,34 @@ impl AccountConfig { { return Err(ConfigError::InvalidEmailFormat); } - if self.server_host.trim().is_empty() { - return Err(ConfigError::EmptyField { - field: "server_host", - }); - } - if self.server_port == 0 { - return Err(ConfigError::InvalidPort); - } - if self.smtp_host.trim().is_empty() { - return Err(ConfigError::EmptyField { field: "smtp_host" }); - } - if self.smtp_port == 0 { - return Err(ConfigError::InvalidPort); + if self.protocol.is_dav() { + // A DAV-protocol account is addressed by its collection URL, not + // by mail host/port endpoints, so it validates the URL instead of + // the IMAP/SMTP fields (which are left empty for such accounts). + let url = self.collection_url.trim(); + if url.is_empty() { + return Err(ConfigError::EmptyField { + field: "collection_url", + }); + } + if !(url.starts_with("http://") || url.starts_with("https://")) { + return Err(ConfigError::InvalidCollectionUrl); + } + } else { + if self.server_host.trim().is_empty() { + return Err(ConfigError::EmptyField { + field: "server_host", + }); + } + if self.server_port == 0 { + return Err(ConfigError::InvalidPort); + } + if self.smtp_host.trim().is_empty() { + return Err(ConfigError::EmptyField { field: "smtp_host" }); + } + if self.smtp_port == 0 { + return Err(ConfigError::InvalidPort); + } } if self.keyring_secret_key.trim().is_empty() { return Err(ConfigError::EmptyField { @@ -149,9 +188,64 @@ mod tests { smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-123".to_string(), sync_interval_secs: 60, + collection_url: String::new(), } } + fn valid_caldav_account() -> AccountConfig { + AccountConfig { + id: "acct-cal-1".to_string(), + name: "Work Calendar".to_string(), + email_address: "user@nuncio.mx".to_string(), + protocol: AccountProtocol::CalDav, + server_host: String::new(), + server_port: 0, + smtp_host: String::new(), + smtp_port: 0, + use_tls: true, + imap_tls_mode: TlsMode::ImplicitTls, + smtp_tls_mode: TlsMode::ImplicitTls, + keyring_secret_key: "nuncio/acct-cal-1".to_string(), + sync_interval_secs: 300, + collection_url: "https://caldav.example.com/calendars/user/work/".to_string(), + } + } + + #[test] + fn valid_caldav_account_passes_without_mail_endpoints() { + let config = valid_caldav_account(); + assert!(config.validate().is_ok()); + } + + #[test] + fn caldav_account_requires_collection_url() { + let mut config = valid_caldav_account(); + config.collection_url = " ".to_string(); + assert_eq!( + config.validate().unwrap_err(), + ConfigError::EmptyField { + field: "collection_url" + } + ); + } + + #[test] + fn caldav_account_rejects_non_http_collection_url() { + let mut config = valid_caldav_account(); + config.collection_url = "ftp://caldav.example.com/work/".to_string(); + assert_eq!( + config.validate().unwrap_err(), + ConfigError::InvalidCollectionUrl + ); + } + + #[test] + fn account_protocol_is_dav_only_for_dav_protocols() { + assert!(AccountProtocol::CalDav.is_dav()); + assert!(!AccountProtocol::Jmap.is_dav()); + assert!(!AccountProtocol::ImapSmtp.is_dav()); + } + #[test] fn valid_account_config_passes_validation() { let config = valid_account(); @@ -275,6 +369,11 @@ mod tests { assert_eq!(imap_json, "\"imap-smtp\""); let parsed_imap: AccountProtocol = serde_json::from_str(&imap_json).unwrap(); assert_eq!(parsed_imap, AccountProtocol::ImapSmtp); + + let caldav_json = serde_json::to_string(&AccountProtocol::CalDav).unwrap(); + assert_eq!(caldav_json, "\"cal-dav\""); + let parsed_caldav: AccountProtocol = serde_json::from_str(&caldav_json).unwrap(); + assert_eq!(parsed_caldav, AccountProtocol::CalDav); } #[test] diff --git a/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin b/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin index 609afaa324768328d9cf368d61bb8edd7bc550b8..1f17b041e64990ea6ac8d64c00bbd63f1cdc35ec 100644 GIT binary patch delta 107 zcmeyEye);9Yjes*=GUxDGgvl%VExFbE~LuEmz_<$h)5PxTXpLl0S9~Z|k0p`u2YIC>& D!F(c( delta 47 zcmV+~0MP%oWbtDI3hraE1M39>@dLB)1@Qp~B4G* Result<(), DatabaseError> { + let existing_columns: Vec = sqlx::query("PRAGMA table_info(accounts)") + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)? + .iter() + .map(|row| row.get::("name")) + .collect(); + + if !existing_columns.iter().any(|c| c == "collection_url") { + sqlx::query("ALTER TABLE accounts ADD COLUMN collection_url TEXT") + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + } + + Ok(()) + } + /// Backfill `messages_fts` for any `messages` row that does not yet have a matching FTS /// entry -- e.g. rows written before the FTS5 index existed, or written directly by a /// process that bypassed `save_email`'s explicit index population. Run automatically as @@ -846,8 +879,8 @@ impl DatabaseEngine { sqlx::query( r#" INSERT OR REPLACE INTO accounts - (id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "#, ) .bind(&config.id) @@ -863,6 +896,7 @@ impl DatabaseEngine { .bind(config.smtp_port as i64) .bind(tls_mode_to_db(config.imap_tls_mode)) .bind(tls_mode_to_db(config.smtp_tls_mode)) + .bind(&config.collection_url) .execute(&self.pool) .await .map_err(DatabaseError::Query)?; @@ -908,9 +942,10 @@ impl DatabaseEngine { Option, String, String, + Option, )> = sqlx::query_as( r#" - SELECT id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode + SELECT id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url FROM accounts "#, ) @@ -935,6 +970,7 @@ impl DatabaseEngine { smtp_port, imap_tls_mode, smtp_tls_mode, + collection_url, )| { let protocol = serde_json::from_str(&protocol_str) .unwrap_or(nuncio_core::AccountProtocol::ImapSmtp); @@ -963,6 +999,7 @@ impl DatabaseEngine { smtp_tls_mode: tls_mode_from_db(&smtp_tls_mode), keyring_secret_key, sync_interval_secs: sync_interval_secs as u64, + collection_url: collection_url.unwrap_or_default(), } }, ) @@ -2280,6 +2317,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-contended-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; engine.save_account(&acct).await.unwrap(); engine.close().await; @@ -2853,6 +2891,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-test-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; engine @@ -2896,6 +2935,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::Plain, keyring_secret_key: "nuncio/acct-tls-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; engine.save_account(&acct).await.expect("save succeeds"); @@ -2915,6 +2955,44 @@ mod tests { assert_eq!(fetched, acct); } + /// A CalDAV account's `collection_url` must survive save/reload, and a + /// mail account's empty `collection_url` must round-trip as empty (not + /// NULL-mangled), proving the backfill-safe column persists both. + #[tokio::test] + async fn caldav_collection_url_survives_save_and_reload() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let caldav = nuncio_core::AccountConfig { + id: "acct-caldav-store-1".to_string(), + name: "Work Calendar".to_string(), + email_address: "cal@nuncio.mx".to_string(), + protocol: nuncio_core::AccountProtocol::CalDav, + server_host: String::new(), + server_port: 0, + smtp_host: String::new(), + smtp_port: 0, + use_tls: true, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + keyring_secret_key: "nuncio/acct-caldav-store-1".to_string(), + sync_interval_secs: 300, + collection_url: "https://dav.example.com/calendars/user/work/".to_string(), + }; + engine.save_account(&caldav).await.expect("save succeeds"); + + let fetched = engine + .get_account("acct-caldav-store-1") + .await + .expect("get succeeds") + .expect("account present"); + assert_eq!(fetched, caldav); + assert_eq!(fetched.protocol, nuncio_core::AccountProtocol::CalDav); + assert_eq!( + fetched.collection_url, + "https://dav.example.com/calendars/user/work/" + ); + } + /// Proves `get_account` resolves a saved account by id and returns `None` /// for an unknown id, and that `delete_account` genuinely removes the row /// (and is idempotent for an already-absent id). @@ -2936,6 +3014,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-del-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; engine.save_account(&acct).await.expect("save succeeds"); @@ -3127,6 +3206,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::StartTls, keyring_secret_key: "nuncio/acct-post-migration".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; engine .save_account(&new_acct) diff --git a/crates/nuncio-store/src/recovery.rs b/crates/nuncio-store/src/recovery.rs index e400a94..6bc97ef 100644 --- a/crates/nuncio-store/src/recovery.rs +++ b/crates/nuncio-store/src/recovery.rs @@ -25,6 +25,7 @@ type SalvagedAccountRow = ( Option, String, String, + Option, ); /// Summary report of database self-healing recovery output. @@ -383,7 +384,7 @@ impl SqliteRecoveryEngine { async fn salvage_accounts(pool: &sqlx::SqlitePool) -> Vec { let rows: Result, _> = sqlx::query_as( - "SELECT id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode FROM accounts" + "SELECT id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url FROM accounts" ) .fetch_all(pool) .await; @@ -406,6 +407,7 @@ impl SqliteRecoveryEngine { smtp_port, imap_tls_mode_raw, smtp_tls_mode_raw, + collection_url, )| { let protocol = serde_json::from_str(&protocol_str) .unwrap_or(nuncio_core::AccountProtocol::ImapSmtp); @@ -436,6 +438,7 @@ impl SqliteRecoveryEngine { smtp_tls_mode, keyring_secret_key, sync_interval_secs: sync_interval_secs as u64, + collection_url: collection_url.unwrap_or_default(), } }, ) @@ -608,6 +611,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-test-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; engine.save_account(&acct).await.unwrap(); assert!(engine.check_integrity().await.unwrap()); @@ -669,6 +673,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-salvage-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; engine.save_account(&acct).await.unwrap(); @@ -806,6 +811,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::Plain, keyring_secret_key: "nuncio/acct-starttls-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; engine.save_account(&acct).await.unwrap(); } diff --git a/crates/nunciod/src/calendar_sync.rs b/crates/nunciod/src/calendar_sync.rs index 29618ac..0eb7f3d 100644 --- a/crates/nunciod/src/calendar_sync.rs +++ b/crates/nunciod/src/calendar_sync.rs @@ -4,17 +4,16 @@ //! fetches events from a [`CalendarBackend`] for a single calendar collection and //! time window, then persists each event via [`DatabaseEngine::save_calendar_event`]. //! -//! # No production entry point yet +//! # Production entry point //! -//! Unlike `sync.rs`'s `run_account_sync`/`run_all_accounts_sync`, this module -//! deliberately has NO production entry point that resolves a real account's -//! CalDAV collection URL and credential and builds a real `CalDavClient` from -//! them: `nuncio_core::AccountConfig` carries no CalDAV fields today, so there -//! is nothing persisted to build one from. [`sync_with_backend`] is the only -//! function here; it is exercised directly in tests with `MockCalendarBackend`, -//! and it is the same seam a future story's production entry point will call -//! once per-account CalDAV configuration exists. -use nuncio_cal::CalendarBackend; +//! [`sync_caldav_account`] is the production entry point: it builds a real +//! [`nuncio_cal::CalDavClient`] from a persisted [`nuncio_core::AccountConfig`] +//! (its `collection_url`) plus the credential resolved from the OS keyring, +//! then delegates to [`sync_with_backend`]. [`sync_with_backend`] is the +//! shared fetch-and-persist seam that both the production client and the +//! test-only `MockCalendarBackend` flow through, mirroring `sync.rs`'s +//! `fetch_and_persist`. +use nuncio_cal::{CalDavAccountConfig, CalDavClient, CalendarBackend}; use nuncio_store::db::DatabaseError; use thiserror::Error; @@ -55,6 +54,33 @@ pub async fn sync_with_backend( Ok(synced) } +/// Production entry point: build a real [`CalDavClient`] for `account` from its +/// persisted `collection_url` and the `password` resolved from the OS keyring +/// vault, then fetch-and-persist `calendar_id`'s events in `[start_window, +/// end_window]` via [`sync_with_backend`]. Returns the number of events synced. +/// +/// `account.collection_url` must be a fully-qualified CalDAV calendar +/// collection URL (validated at `AddAccount` time); the account's +/// `email_address` is used as the DAV basic-auth username. No fabricated data +/// is ever returned -- a transport or persistence failure surfaces as a +/// [`CalendarSyncError`]. +pub async fn sync_caldav_account( + db: &nuncio_store::db::DatabaseEngine, + account: &nuncio_core::AccountConfig, + password: &str, + calendar_id: &str, + start_window: i64, + end_window: i64, +) -> Result { + let client = CalDavClient::new(CalDavAccountConfig { + account_id: account.id.clone(), + caldav_url: account.collection_url.clone(), + username: account.email_address.clone(), + auth_token: password.to_string(), + }); + sync_with_backend(db, &client, calendar_id, start_window, end_window).await +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index b820424..23f0f66 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -223,6 +223,7 @@ fn map_account_protocol_to_proto(protocol: nuncio_core::AccountProtocol) -> Acco match protocol { nuncio_core::AccountProtocol::Jmap => AccountProtocolProto::Jmap, nuncio_core::AccountProtocol::ImapSmtp => AccountProtocolProto::ImapSmtp, + nuncio_core::AccountProtocol::CalDav => AccountProtocolProto::Caldav, } } @@ -236,6 +237,7 @@ fn map_account_protocol_from_proto( match protocol { AccountProtocolProto::Jmap => Ok(nuncio_core::AccountProtocol::Jmap), AccountProtocolProto::ImapSmtp => Ok(nuncio_core::AccountProtocol::ImapSmtp), + AccountProtocolProto::Caldav => Ok(nuncio_core::AccountProtocol::CalDav), AccountProtocolProto::Unspecified => { Err(Status::invalid_argument("account protocol is required")) } @@ -284,6 +286,7 @@ fn map_account_config_to_proto(config: nuncio_core::AccountConfig) -> AccountCon sync_interval_secs: config.sync_interval_secs, smtp_host: config.smtp_host, smtp_port: u32::from(config.smtp_port), + collection_url: config.collection_url, } } @@ -315,6 +318,7 @@ fn map_account_config_from_proto( smtp_tls_mode, keyring_secret_key: config.keyring_secret_key, sync_interval_secs: config.sync_interval_secs, + collection_url: config.collection_url, }) } @@ -405,8 +409,22 @@ impl AccountConnectionTester for RealAccountConnectionTester { nuncio_core::AccountProtocol::Jmap => ProtocolProbe::failed( "IMAP connection probe is not applicable to a JMAP account".to_string(), ), + nuncio_core::AccountProtocol::CalDav => ProtocolProbe::failed( + "IMAP connection probe is not applicable to a CalDAV account".to_string(), + ), }; + // A CalDAV account has no SMTP endpoint to probe; report that honestly + // rather than dialing the (empty) mail endpoint fields. + if config.protocol == nuncio_core::AccountProtocol::CalDav { + return AccountConnectionReport { + imap, + smtp: ProtocolProbe::failed( + "SMTP connection probe is not applicable to a CalDAV account".to_string(), + ), + }; + } + let smtp = match SmtpTransportEngine::new( &config.smtp_host, config.smtp_port, @@ -872,15 +890,16 @@ fn expand_events_for_window( /// /// Production (every existing caller of [`serve`] / [`serve_on_listener`]) /// never constructs a non-default instance: [`Default`] yields -/// `calendar_backend: None`, which routes `Calendar/Sync` to an honest -/// error, since there is currently no persisted per-account CalDAV -/// configuration to build a real backend from. Only a full-daemon E2E test -/// supplies `Some(..)`, standing in a [`nuncio_cal::MockCalendarBackend`] -/// for real network I/O. +/// `calendar_backend: None`, which routes `Calendar/Sync` through the real +/// production path -- resolving configured CalDAV accounts, building a real +/// [`nuncio_cal::CalDavClient`] from each account's collection URL and +/// keyring credential, and syncing genuinely fetched events. Only a +/// full-daemon E2E test that wants to bypass the real network transport +/// supplies `Some(..)`, standing in a [`nuncio_cal::MockCalendarBackend`]. #[derive(Clone, Default)] pub struct CalendarEngineOverrides { - /// When `Some`, `Calendar/Sync` fetches from this backend instead of - /// returning an honest "no CalDAV configuration" error. + /// When `Some`, `Calendar/Sync` fetches from this injected backend instead + /// of building a real per-account CalDAV client. Used only by tests. pub calendar_backend: Option>, } @@ -897,14 +916,93 @@ impl std::fmt::Debug for CalendarEngineOverrides { /// /// `ListEvents`/`GetEvent` always read genuinely persisted data. `Sync` /// fetches from [`CalendarEngineOverrides::calendar_backend`] when injected; -/// otherwise it returns an honest error, since there is no persisted -/// per-account CalDAV configuration to build a real backend from yet (see -/// `nunciod::calendar_sync`'s doc comment). +/// otherwise it runs the real production path -- resolving configured CalDAV +/// accounts and building a real [`nuncio_cal::CalDavClient`] per account from +/// its persisted collection URL and keyring credential. It returns an honest +/// error only when no CalDAV account is configured (see +/// `nunciod::calendar_sync`). struct CalendarGrpcService { db: Arc, + secrets: Arc, overrides: CalendarEngineOverrides, } +impl CalendarGrpcService { + /// Production `Calendar/Sync` path (no injected test backend): resolve the + /// configured CalDAV account(s), build a real [`nuncio_cal::CalDavClient`] + /// per account from its persisted collection URL and keyring credential, + /// and sync the requested calendar/window into the store. Returns the + /// total number of events synced across the resolved accounts. + /// + /// A request naming a specific `account_id` is scoped to that account; an + /// empty `account_id` syncs every configured CalDAV account. When no + /// CalDAV account matches, this returns an honest error rather than a + /// fabricated zero-count success -- there is genuinely no remote calendar + /// to fetch from. + async fn sync_configured_caldav_accounts( + &self, + req: &CalendarSyncRequest, + ) -> Result { + let caldav_accounts: Vec = self + .db + .list_accounts() + .await + .map_err(|e| Status::internal(format!("failed to list accounts: {e}")))? + .into_iter() + .filter(|a| a.protocol == nuncio_core::AccountProtocol::CalDav) + .filter(|a| req.account_id.is_empty() || a.id == req.account_id) + .collect(); + + if caldav_accounts.is_empty() { + return Err(Status::failed_precondition(if req.account_id.is_empty() { + "no CalDAV account is configured; add one with protocol CALDAV and a \ + collection_url before syncing" + .to_string() + } else { + format!( + "no CalDAV account with id '{}' is configured", + req.account_id + ) + })); + } + + let mut total = 0usize; + for account in caldav_accounts { + // The credential lives ONLY in the OS keyring vault, keyed by the + // account's `keyring_secret_key`; it is read just-in-time here and + // wrapped so its backing memory is zeroized on drop. + let password = zeroize::Zeroizing::new( + self.secrets + .get_secret(&account.keyring_secret_key) + .map_err(|e| { + Status::internal(format!( + "failed to read CalDAV credential for account '{}': {e}", + account.id + )) + })?, + ); + + total += crate::calendar_sync::sync_caldav_account( + &self.db, + &account, + &password, + &req.calendar_id, + req.start_window, + req.end_window, + ) + .await + .map_err(|e| { + Status::internal(format!( + "CalDAV sync failed for account '{}': {e}", + account.id + )) + })?; + } + + Ok(total) + } +} + #[tonic::async_trait] impl Calendar for CalendarGrpcService { async fn sync( @@ -923,13 +1021,7 @@ impl Calendar for CalendarGrpcService { ) .await .map_err(|e| Status::internal(format!("calendar sync failed: {e}")))?, - None => { - return Err(Status::internal(format!( - "no CalDAV configuration exists for account '{}'; per-account CalDAV \ - configuration is not yet implemented", - req.account_id - ))); - } + None => self.sync_configured_caldav_accounts(&req).await?, }; Ok(Response::new(CalendarSyncResponse { @@ -2275,6 +2367,7 @@ pub async fn serve_on_listener_with_overrides( // this function's doc comment. let calendar_service = CalendarGrpcService { db: db.clone(), + secrets: secrets.clone(), overrides: calendar_overrides, }; let calendar_interceptor = BearerAuthInterceptor::new(token.clone()); @@ -2875,6 +2968,7 @@ mod tests { smtp_tls_mode: TlsModeProto::ImplicitTls.into(), keyring_secret_key: keyring_secret_key.to_string(), sync_interval_secs: 60, + collection_url: String::new(), smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, } @@ -3263,6 +3357,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: key.to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; // Force `save_account` to fail for real by closing the pool, exactly @@ -3306,6 +3401,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: key.to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; db.close().await; @@ -3752,6 +3848,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-send-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; db.save_account(&config).await.expect("save account"); secrets @@ -3920,6 +4017,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-injected-send-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; // Deliberately never stores a keyring credential -- this path must // never need one. @@ -4036,6 +4134,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-injected-send-fail-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; db.save_account(&config).await.expect("save account"); @@ -5417,12 +5516,12 @@ mod tests { assert_eq!(err.code(), Code::NotFound); } - /// With no `CalendarEngineOverrides::calendar_backend` injected, `Sync` - /// returns an honest error -- there is no persisted per-account CalDAV - /// configuration to build a real backend from -- never a fabricated - /// `synced_count`. + /// With no `CalendarEngineOverrides::calendar_backend` injected AND no + /// CalDAV account configured, `Sync` returns an honest + /// `FailedPrecondition` error naming the missing account -- never a + /// fabricated `synced_count`. #[tokio::test] - async fn calendar_sync_without_override_reports_honest_error() { + async fn calendar_sync_without_override_or_config_reports_honest_error() { use nuncio_proto::v1::calendar_client::CalendarClient; use nuncio_proto::v1::CalendarSyncRequest; @@ -5441,9 +5540,8 @@ mod tests { })) .await .expect_err("sync with no injected backend and no CalDAV config must fail"); - assert_eq!(err.code(), Code::Internal); + assert_eq!(err.code(), Code::FailedPrecondition); assert!(err.message().contains("acct-cal-no-config")); - assert!(err.message().contains("no CalDAV configuration")); } /// With a [`CalendarEngineOverrides::calendar_backend`] injected, `Sync` diff --git a/crates/nunciod/src/orchestrator.rs b/crates/nunciod/src/orchestrator.rs index 72d1826..2fa48f3 100644 --- a/crates/nunciod/src/orchestrator.rs +++ b/crates/nunciod/src/orchestrator.rs @@ -162,6 +162,7 @@ mod tests { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-orch-1".to_string(), sync_interval_secs: 60, + collection_url: String::new(), }; engine.save_account(&acct).await.unwrap(); engine.close().await; diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index e010fd3..f022ea5 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -59,6 +59,10 @@ pub enum OutboxExecuteError { /// A database error occurred while resolving the account/message. #[error("database error: {0}")] Store(#[from] DatabaseError), + /// The resolved account is not a mail account (e.g. a CalDAV account), so + /// it has no mail backend to execute an outbound mutation against. + #[error("account '{0}' is not a mail account")] + NotAMailAccount(String), } /// Injectable seam for the resources a mutation needs: the per-account mail @@ -423,7 +427,8 @@ impl RemoteExecutionEnv for ProductionExecutionEnv { ) -> Result, OutboxExecuteError> { let config = self.account(account_id).await?; let password = self.secrets.get_secret(&config.keyring_secret_key)?; - Ok(crate::sync::build_mail_backend(&config, &password)) + crate::sync::build_mail_backend(&config, &password) + .map_err(|_| OutboxExecuteError::NotAMailAccount(account_id.to_string())) } async fn message_sender( diff --git a/crates/nunciod/src/send.rs b/crates/nunciod/src/send.rs index 995b68a..3593ab5 100644 --- a/crates/nunciod/src/send.rs +++ b/crates/nunciod/src/send.rs @@ -243,6 +243,7 @@ mod tests { smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, + collection_url: String::new(), } } diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index 02265a1..bbb395c 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -40,6 +40,10 @@ pub enum SyncError { /// A database persistence error occurred while saving synced messages. #[error("database error: {0}")] Store(#[from] DatabaseError), + /// The account's protocol is not a mail protocol (e.g. a CalDAV account), + /// so it has no inbound mail backend to sync. + #[error("account '{0}' is not a mail account and cannot be mail-synced")] + NotAMailAccount(String), } /// Map a [`RuleAction`] to the short mutation tag persisted on @@ -339,22 +343,28 @@ async fn find_account( /// keyring by the caller). This is the ONLY place a concrete protocol engine /// is chosen; every other function in this module only ever sees `&dyn /// MailBackend`. -pub(crate) fn build_mail_backend(config: &AccountConfig, password: &str) -> Box { +pub(crate) fn build_mail_backend( + config: &AccountConfig, + password: &str, +) -> Result, SyncError> { match config.protocol { - AccountProtocol::ImapSmtp => Box::new(ImapEngine::with_credentials( + AccountProtocol::ImapSmtp => Ok(Box::new(ImapEngine::with_credentials( &config.id, &config.server_host, config.server_port, config.imap_tls_mode, &config.email_address, password, - )), - AccountProtocol::Jmap => Box::new(JmapEngine::with_credentials( + ))), + AccountProtocol::Jmap => Ok(Box::new(JmapEngine::with_credentials( &config.id, &config.server_host, &config.email_address, password, - )), + ))), + // A DAV-protocol account (e.g. CalDAV) has no inbound mail backend; + // it is synced through its own domain service, not the mail path. + AccountProtocol::CalDav => Err(SyncError::NotAMailAccount(config.id.clone())), } } @@ -376,7 +386,7 @@ pub async fn run_account_sync( let setup = async { let config = find_account(db, account_id).await?; let password = secrets.get_secret(&config.keyring_secret_key)?; - Ok::<_, SyncError>(build_mail_backend(&config, &password)) + build_mail_backend(&config, &password) } .await; @@ -419,9 +429,26 @@ pub async fn run_all_accounts_sync( match db.list_accounts().await { Ok(accounts) => { for config in accounts { + // DAV-protocol accounts (e.g. CalDAV) are not mail accounts; + // they sync through their own domain service, so skip them + // here rather than attempting to build a mail backend. + if config.protocol.is_dav() { + continue; + } match secrets.get_secret(&config.keyring_secret_key) { Ok(password) => { - let backend = build_mail_backend(&config, &password); + let backend = match build_mail_backend(&config, &password) { + Ok(backend) => backend, + Err(e) => { + event_bus.process_command(CoreCommand::ReportError { + message: format!( + "sync skipped for account '{}': {e}", + config.id + ), + }); + continue; + } + }; match fetch_and_persist( db, event_bus, @@ -500,6 +527,7 @@ mod tests { smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, + collection_url: String::new(), } } @@ -591,6 +619,7 @@ mod tests { smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, + collection_url: String::new(), } } diff --git a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs index 278e6a8..a6dd608 100644 --- a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs +++ b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs @@ -89,6 +89,7 @@ fn sample_account_config() -> AccountConfig { sync_interval_secs: 60, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 25, + collection_url: String::new(), } } diff --git a/crates/nunciod/tests/caldav_production_e2e_test.rs b/crates/nunciod/tests/caldav_production_e2e_test.rs new file mode 100644 index 0000000..dd0b89e --- /dev/null +++ b/crates/nunciod/tests/caldav_production_e2e_test.rs @@ -0,0 +1,241 @@ +//! Full-daemon offline end-to-end test for the PRODUCTION CalDAV sync path. +//! +//! Unlike `calendar_e2e_test.rs`, which injects a `MockCalendarBackend` through +//! the [`nunciod::grpc::CalendarEngineOverrides`] seam, this test drives the +//! real production path with NO injected backend: it configures a first-class +//! CalDAV account over the authenticated `Accounts/AddAccount` RPC (protocol +//! CALDAV + a `collection_url` pointing at a WireMock server + a keyring +//! credential), then calls `Calendar/Sync`. The daemon resolves the configured +//! CalDAV account, reads its credential from the (mock) vault, builds a REAL +//! [`nuncio_cal::CalDavClient`] from the persisted collection URL, issues a +//! genuine CalDAV `REPORT` against WireMock, and persists the fetched events. +//! +//! This proves the production wiring end to end while remaining fully offline: +//! WireMock stands in for the remote CalDAV server, and a +//! [`SecretManager::mock`] vault stands in for the OS keyring -- no real +//! network and no real keyring are ever touched. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use nuncio_core::EventBus; +use nuncio_filter::FilterEngine; +use nuncio_proto::v1::{ + AccountConfig as AccountConfigProto, AccountProtocol as AccountProtocolProto, + AddAccountRequest, CalendarSyncRequest, ListEventsRequest, TlsMode as TlsModeProto, +}; +use nuncio_store::db::DatabaseEngine; +use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT}; +use nunciod::grpc::{ + serve_on_listener_with_overrides, AccountsEngineOverrides, CalendarEngineOverrides, + ContactsEngineOverrides, MailEngineOverrides, +}; +use std::sync::Arc; +use tokio::net::TcpListener; +use tonic::Code; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const ACCOUNT_ID: &str = "acct-caldav-e2e-1"; +const ACCOUNT_EMAIL: &str = "james.maes@nuncio.mx"; +const KEYRING_KEY: &str = "nuncio/acct-caldav-e2e-1"; +const CALDAV_PASSWORD: &str = "wiremock-app-token"; +const CALENDAR_ID: &str = "cal-work"; + +/// A canned CalDAV `multistatus` response with two VEVENTs, mirroring the +/// transport-level fixture used by `nuncio-cal`'s own WireMock test. +fn multistatus_body() -> String { + r#" + + + /calendars/work/standup.ics + + + BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +SUMMARY:Production CalDAV Standup +LOCATION:Conference Room B +DTSTART:20240715T130000Z +DTEND:20240715T140000Z +END:VEVENT +END:VCALENDAR + + + + + /calendars/work/review.ics + + + BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +SUMMARY:Production UTC Review +DTSTART:20240301T120000Z +DTEND:20240301T130000Z +END:VEVENT +END:VCALENDAR + + + +"# + .to_string() +} + +/// Boots the real daemon gRPC server with DEFAULT overrides (no injected +/// calendar backend), backed by a real temp file-backed [`DatabaseEngine`] and +/// the given mock vault. Returns the server address and the minted bearer +/// token; the `TempDir` guard is returned so the caller controls its lifetime. +async fn boot_daemon( + secrets: Arc, +) -> (std::net::SocketAddr, String, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("create temp dir for file-backed db"); + let db_path = dir.path().join("caldav_prod_e2e.db"); + let db = Arc::new( + DatabaseEngine::connect_file(&db_path, &secrets) + .await + .expect("open temp file-backed database"), + ); + + let token_bytes = secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("mint gRPC bearer token from mock vault"); + let token = hex::encode(token_bytes); + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("listener has local addr"); + + let event_bus = Arc::new(EventBus::new()); + let server_secrets = secrets.clone(); + let server_token = token.clone(); + let server_filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + tokio::spawn(async move { + let _ = serve_on_listener_with_overrides( + listener, + event_bus, + db, + server_filter_engine, + server_secrets, + server_token, + MailEngineOverrides::default(), + // No injected calendar backend: exercise the real production path. + CalendarEngineOverrides::default(), + ContactsEngineOverrides::default(), + AccountsEngineOverrides::default(), + ) + .await; + }); + + (addr, token, dir) +} + +/// The full offline proof of the production CalDAV sync path: configure a +/// CalDAV account over the real `Accounts` API, then `Calendar/Sync` builds a +/// real `CalDavClient` and syncs events fetched from WireMock into the store, +/// immediately visible to `ListEvents` -- all over the authenticated gRPC API. +#[tokio::test] +async fn caldav_production_sync_builds_real_client_and_persists_events() { + let mock_server = MockServer::start().await; + Mock::given(method("REPORT")) + .and(path("/calendars/work/")) + .respond_with(ResponseTemplate::new(207).set_body_string(multistatus_body())) + .mount(&mock_server) + .await; + let collection_url = format!("{}/calendars/work/", mock_server.uri()); + + let secrets = Arc::new(SecretManager::mock()); + let (addr, token, _db_dir) = boot_daemon(secrets).await; + let addr_str = addr.to_string(); + + // ---- Configure a first-class CalDAV account over the Accounts API ---- + let mut accounts_client = nuncio_proto::client::connect_accounts(&addr_str, &token) + .await + .expect("accounts client connects"); + accounts_client + .add_account(AddAccountRequest { + config: Some(AccountConfigProto { + id: ACCOUNT_ID.to_string(), + name: "Work Calendar".to_string(), + email_address: ACCOUNT_EMAIL.to_string(), + protocol: AccountProtocolProto::Caldav.into(), + server_host: String::new(), + server_port: 0, + use_tls: true, + imap_tls_mode: TlsModeProto::ImplicitTls.into(), + smtp_tls_mode: TlsModeProto::ImplicitTls.into(), + keyring_secret_key: KEYRING_KEY.to_string(), + sync_interval_secs: 300, + smtp_host: String::new(), + smtp_port: 0, + collection_url: collection_url.clone(), + }), + password: CALDAV_PASSWORD.to_string(), + }) + .await + .expect("CalDAV account is accepted and persisted"); + + // ---- Calendar/Sync runs the PRODUCTION path (no injected backend) ---- + let mut calendar_client = nuncio_proto::client::connect_calendar(&addr_str, &token) + .await + .expect("calendar client connects"); + let sync_response = calendar_client + .sync(CalendarSyncRequest { + account_id: ACCOUNT_ID.to_string(), + calendar_id: CALENDAR_ID.to_string(), + start_window: 1_700_000_000, + end_window: 1_800_000_000, + }) + .await + .expect("production CalDAV sync succeeds") + .into_inner(); + assert_eq!(sync_response.synced_count, 2); + + // Immediately visible: `Sync` awaits full fetch-and-persist completion. + let list_response = calendar_client + .list_events(ListEventsRequest { + account_id: ACCOUNT_ID.to_string(), + calendar_id: CALENDAR_ID.to_string(), + start_window: 1_700_000_000, + end_window: 1_800_000_000, + }) + .await + .expect("list_events succeeds") + .into_inner(); + assert_eq!(list_response.events.len(), 2); + let summaries: Vec<&str> = list_response + .events + .iter() + .map(|e| e.summary.as_str()) + .collect(); + assert!(summaries.contains(&"Production CalDAV Standup")); + assert!(summaries.contains(&"Production UTC Review")); + + // ---- With no CalDAV account matching, the honest error still fires ---- + let missing = calendar_client + .sync(CalendarSyncRequest { + account_id: "acct-does-not-exist".to_string(), + calendar_id: CALENDAR_ID.to_string(), + start_window: 1_700_000_000, + end_window: 1_800_000_000, + }) + .await + .expect_err("a sync targeting an unconfigured account must fail honestly"); + assert_eq!(missing.code(), Code::FailedPrecondition); + + // ---- An unauthenticated call to Calendar is rejected ---- + let mut unauthenticated_client = + nuncio_proto::v1::calendar_client::CalendarClient::connect(format!("http://{addr_str}")) + .await + .expect("client connects at the transport level without any auth"); + let err = unauthenticated_client + .list_events(ListEventsRequest { + account_id: ACCOUNT_ID.to_string(), + calendar_id: CALENDAR_ID.to_string(), + start_window: 0, + end_window: i64::MAX, + }) + .await + .expect_err("missing bearer token must be rejected"); + assert_eq!(err.code(), Code::Unauthenticated); +} diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index 855a4c9..6f823ff 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -97,6 +97,7 @@ fn sample_account() -> nuncio_core::AccountConfig { smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{ACCOUNT_ID}"), sync_interval_secs: 60, + collection_url: String::new(), } } diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index 4a2c844..058225e 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -68,6 +68,7 @@ fn sample_account_config() -> AccountConfig { sync_interval_secs: 60, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, + collection_url: String::new(), } } -- 2.54.0 From 84c1a6b8feb15b7fe91b9522f6a845e37a2b4304 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 13:30:13 -0500 Subject: [PATCH 17/95] fix(cli): render clean error messages instead of raw tonic::Status (#336) Replace ~28 CLI error paths that formatted a tonic::Status directly (dumping its Debug output, including internal metadata) with the server-provided status.message(), falling back to a short " error" form when the message is empty. Refs #321 Co-authored-by: James Maes --- crates/nuncio-cli/src/runner.rs | 293 ++++++++++++++++++++++++++++---- 1 file changed, 259 insertions(+), 34 deletions(-) diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index f14f0de..912f6b7 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -663,7 +663,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected sync: {status}"), + &format!( + "nunciod daemon rejected sync: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -701,7 +704,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected list_messages: {status}"), + &format!( + "nunciod daemon rejected list_messages: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -757,7 +763,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected send_message: {status}"), + &format!( + "nunciod daemon rejected send_message: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -800,7 +809,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected search_messages: {status}"), + &format!( + "nunciod daemon rejected search_messages: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -838,7 +850,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected list_folders: {status}"), + &format!( + "nunciod daemon rejected list_folders: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -877,7 +892,10 @@ impl HeadlessRunner { Self::render_error(&format!("message '{}' not found", id), json_mode) } Err(status) => Self::render_error( - &format!("nunciod daemon rejected get_message: {status}"), + &format!( + "nunciod daemon rejected get_message: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -930,7 +948,10 @@ impl HeadlessRunner { Self::render_error(&format!("message '{}' not found", id), json_mode) } Err(status) => Self::render_error( - &format!("nunciod daemon rejected mark_read: {status}"), + &format!( + "nunciod daemon rejected mark_read: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -997,7 +1018,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected export_mailbox: {status}"), + &format!( + "nunciod daemon rejected export_mailbox: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1055,7 +1079,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected list_records: {status}"), + &format!( + "nunciod daemon rejected list_records: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1096,7 +1123,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected verify_chain: {status}"), + &format!( + "nunciod daemon rejected verify_chain: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1155,7 +1185,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected list_rules: {status}"), + &format!( + "nunciod daemon rejected list_rules: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1204,7 +1237,10 @@ impl HeadlessRunner { Self::render_error(status.message(), json_mode) } Err(status) => Self::render_error( - &format!("nunciod daemon rejected create_rule: {status}"), + &format!( + "nunciod daemon rejected create_rule: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1232,7 +1268,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected delete_rule: {status}"), + &format!( + "nunciod daemon rejected delete_rule: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1267,7 +1306,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected validate_rule: {status}"), + &format!( + "nunciod daemon rejected validate_rule: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1320,7 +1362,10 @@ impl HeadlessRunner { Self::render_error(status.message(), json_mode) } Err(status) => Self::render_error( - &format!("nunciod daemon rejected preview_rule: {status}"), + &format!( + "nunciod daemon rejected preview_rule: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1372,7 +1417,10 @@ impl HeadlessRunner { Self::render_error(status.message(), json_mode) } Err(status) => Self::render_error( - &format!("nunciod daemon rejected update_rule: {status}"), + &format!( + "nunciod daemon rejected update_rule: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1403,7 +1451,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected export_rules: {status}"), + &format!( + "nunciod daemon rejected export_rules: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1454,7 +1505,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected import_rules: {status}"), + &format!( + "nunciod daemon rejected import_rules: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1498,7 +1552,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected get_execution_logs: {status}"), + &format!( + "nunciod daemon rejected get_execution_logs: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1530,7 +1587,10 @@ impl HeadlessRunner { Ok(response) => response.into_inner(), Err(status) => { return Self::render_error( - &format!("nunciod daemon rejected triage: {status}"), + &format!( + "nunciod daemon rejected triage: {}", + Self::clean_status_message(&status) + ), json_mode, ) } @@ -1559,7 +1619,10 @@ impl HeadlessRunner { Ok(None) => break, Err(status) => { return Self::render_error( - &format!("nunciod daemon triage stream failed: {status}"), + &format!( + "nunciod daemon triage stream failed: {}", + Self::clean_status_message(&status) + ), json_mode, ) } @@ -1731,7 +1794,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected add_account: {status}"), + &format!( + "nunciod daemon rejected add_account: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1806,7 +1872,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected list_accounts: {status}"), + &format!( + "nunciod daemon rejected list_accounts: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1825,7 +1894,12 @@ impl HeadlessRunner { let accounts = client .list_accounts(nuncio_proto::v1::ListAccountsRequest {}) .await - .map_err(|status| format!("nunciod daemon rejected list_accounts: {status}"))? + .map_err(|status| { + format!( + "nunciod daemon rejected list_accounts: {}", + Self::clean_status_message(&status) + ) + })? .into_inner() .accounts; accounts @@ -1947,7 +2021,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected update_account: {status}"), + &format!( + "nunciod daemon rejected update_account: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -1975,7 +2052,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected remove_account: {status}"), + &format!( + "nunciod daemon rejected remove_account: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -2029,7 +2109,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected test_account_connection: {status}"), + &format!( + "nunciod daemon rejected test_account_connection: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -2127,7 +2210,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected list_events: {status}"), + &format!( + "nunciod daemon rejected list_events: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -2172,7 +2258,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected calendar sync: {status}"), + &format!( + "nunciod daemon rejected calendar sync: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -2225,7 +2314,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected list_contacts: {status}"), + &format!( + "nunciod daemon rejected list_contacts: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -2271,7 +2363,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected list_contacts: {status}"), + &format!( + "nunciod daemon rejected list_contacts: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -2329,7 +2424,10 @@ impl HeadlessRunner { ), }, Err(status) => Self::render_error( - &format!("nunciod daemon rejected create_contact: {status}"), + &format!( + "nunciod daemon rejected create_contact: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -2363,7 +2461,10 @@ impl HeadlessRunner { } } Err(status) => Self::render_error( - &format!("nunciod daemon rejected contacts sync: {status}"), + &format!( + "nunciod daemon rejected contacts sync: {}", + Self::clean_status_message(&status) + ), json_mode, ), } @@ -2379,6 +2480,22 @@ impl HeadlessRunner { } } + /// Extracts the clean, user-facing text from a `tonic::Status` returned + /// by a daemon RPC. Uses the server-provided message when present; + /// falls back to a short " error" form when the message is empty. + /// Callers MUST use this (never the `Display`/`Debug` of `tonic::Status` + /// directly) for user-facing output, since `Display`/`Debug` on + /// `tonic::Status` dumps raw gRPC internals (`status: ..., metadata: + /// MetadataMap { ... }`) that are meaningless to a CLI user. + fn clean_status_message(status: &tonic::Status) -> String { + let message = status.message(); + if message.is_empty() { + format!("{:?} error", status.code()) + } else { + message.to_string() + } + } + /// `system status`: a real thin gRPC client of the running `nunciod` /// daemon's `nuncio.v1.System` API. /// @@ -2431,7 +2548,12 @@ impl HeadlessRunner { let response = client .get_status(nuncio_proto::v1::GetStatusRequest {}) .await - .map_err(|e| format!("nunciod daemon rejected status request: {e}"))? + .map_err(|status| { + format!( + "nunciod daemon rejected status request: {}", + Self::clean_status_message(&status) + ) + })? .into_inner(); Ok((response.engine_status, response.version)) @@ -3349,6 +3471,109 @@ mod tests { ); } + /// Proves that a generic (non-`NotFound`/`InvalidArgument`) RPC failure + /// is rendered as the server-provided `status.message()` text, never + /// the raw `Display`/`Debug` dump of `tonic::Status` (which leaks + /// internal gRPC framing like `metadata: MetadataMap { ... }` into + /// CLI output). + #[tokio::test] + async fn mail_rpc_error_renders_clean_message_not_raw_status_dump() { + use nuncio_proto::v1::mail_server::{Mail as MailService, MailServer}; + use nuncio_proto::v1::{ + GetMessageRequest, GetMessageResponse, ListFoldersRequest, ListFoldersResponse, + ListMessagesRequest, ListMessagesResponse, MarkReadRequest, MarkReadResponse, + SearchMessagesRequest, SearchMessagesResponse, SendMessageRequest, SendMessageResponse, + SyncRequest, SyncResponse, + }; + + /// Minimal test-only stub of `nuncio.v1.Mail` whose `ListFolders` + /// always fails with an internal error, so this test can assert + /// on exactly how the CLI renders a generic RPC failure. + struct StubMailAlwaysErrors; + + #[tonic::async_trait] + impl MailService for StubMailAlwaysErrors { + async fn list_folders( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + + async fn list_messages( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + + async fn get_message( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + + async fn mark_read( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + + async fn search_messages( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + + async fn send_message( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + + async fn sync( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + } + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("listener has local addr"); + tokio::spawn(async move { + let _ = tonic::transport::Server::builder() + .add_service(MailServer::new(StubMailAlwaysErrors)) + .serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new(listener)) + .await; + }); + + let runner = + HeadlessRunner::ephemeral_with(Arc::new(SecretManager::mock()), addr.to_string()) + .await + .expect("ephemeral runner initializes"); + + let out = runner + .execute_command( + &Commands::Folder { + action: FolderSubcommand::List, + }, + false, + ) + .await; + + assert!(out.contains("mailbox index is corrupt")); + assert!(!out.contains("metadata:")); + assert!(!out.contains("MetadataMap")); + assert!(!out.contains("status:")); + } + /// Reference-client proof: boots a stub `nuncio.v1.Calendar` gRPC server /// (mirroring the `Mail` stub pattern above) and drives the real /// `HeadlessRunner`'s `cal list`/`cal sync` gRPC client paths against it. -- 2.54.0 From 546cebe7230edb9ace29362247e2fd4b0d68c698 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 13:35:15 -0500 Subject: [PATCH 18/95] fix(core): reject cleartext Plain TLS to non-loopback hosts (#335) * fix(core): reject cleartext Plain TLS to non-loopback hosts AccountConfig::validate silently accepted TlsMode::Plain for imap/smtp against any host, so a misconfig could send credentials in cleartext to a remote server. Plain is now rejected unless the host is loopback (127.0.0.1, ::1, or localhost); ImplicitTls/StartTls are unaffected. Refs #297 * fix(core): reject cleartext CalDAV collection_url on remote hosts The CalDAV collection_url validation only checked the scheme prefix, so http:///... passed even though it sends the account's Basic-auth credentials in cleartext. Reuse the same loopback-only rule as the imap/smtp TlsMode::Plain check: plain http is now accepted only when the URL's host is loopback (127.0.0.0/8, ::1, or localhost); https to any host, and http to a loopback host, remain valid. Refs #297 --------- Co-authored-by: James Maes --- crates/nuncio-core/src/config.rs | 147 ++++++++++++++++++ .../tests/accounts_lifecycle_e2e_test.rs | 2 +- 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/crates/nuncio-core/src/config.rs b/crates/nuncio-core/src/config.rs index 5cbb012..df39c9c 100644 --- a/crates/nuncio-core/src/config.rs +++ b/crates/nuncio-core/src/config.rs @@ -61,6 +61,17 @@ pub enum ConfigError { /// A DAV-protocol account's collection URL is not a valid http(s) URL. #[error("collection_url must be a valid http(s) URL")] InvalidCollectionUrl, + /// Cleartext `TlsMode::Plain` was requested against a non-loopback host, + /// which would send credentials over the network unencrypted. + #[error( + "{endpoint} host '{host}' is not loopback; refusing cleartext Plain TLS mode to a remote host" + )] + CleartextToRemoteHost { + /// Which endpoint the rejected setting applies to (e.g. "imap", "smtp"). + endpoint: &'static str, + /// The offending non-loopback host. + host: String, + }, } /// Complete configuration schema for a mail & calendar account. @@ -102,6 +113,35 @@ pub struct AccountConfig { pub collection_url: String, } +/// Whether `host` refers to the local loopback interface, either as a +/// literal IP address (`127.0.0.1`, `::1`, ...) or the conventional +/// `localhost` name. Used to allow cleartext `TlsMode::Plain` only against +/// local test servers, never against a remote host. +fn is_loopback_host(host: &str) -> bool { + let host = host.trim(); + if host.eq_ignore_ascii_case("localhost") { + return true; + } + host.parse::() + .is_ok_and(|ip| ip.is_loopback()) +} + +/// Extract the host portion of an `http://`/`https://` URL, without pulling +/// in a full URL-parsing dependency for this one field. Handles a bracketed +/// IPv6 literal (`[::1]`) as well as a plain hostname/IPv4 literal, and +/// stops at the first `:` (port), `/` (path), `?`, or `#` that follows. +fn url_host(url: &str) -> Option<&str> { + let rest = url + .strip_prefix("https://") + .or_else(|| url.strip_prefix("http://"))?; + if let Some(bracketed) = rest.strip_prefix('[') { + let end = bracketed.find(']')?; + return Some(&bracketed[..end]); + } + let end = rest.find(['/', ':', '?', '#']).unwrap_or(rest.len()); + Some(&rest[..end]) +} + impl AccountConfig { /// Minimum allowed background sync interval (10 seconds). pub const MIN_SYNC_INTERVAL_SECS: u64 = 10; @@ -140,6 +180,18 @@ impl AccountConfig { if !(url.starts_with("http://") || url.starts_with("https://")) { return Err(ConfigError::InvalidCollectionUrl); } + // Plain `http://` is only tolerated against a loopback host (a + // local test server); any remote host must use `https://` to + // avoid sending the account's Basic-auth credentials in cleartext. + if url.starts_with("http://") { + let host = url_host(url).unwrap_or_default(); + if !is_loopback_host(host) { + return Err(ConfigError::CleartextToRemoteHost { + endpoint: "caldav", + host: host.to_string(), + }); + } + } } else { if self.server_host.trim().is_empty() { return Err(ConfigError::EmptyField { @@ -155,6 +207,18 @@ impl AccountConfig { if self.smtp_port == 0 { return Err(ConfigError::InvalidPort); } + if self.imap_tls_mode == TlsMode::Plain && !is_loopback_host(&self.server_host) { + return Err(ConfigError::CleartextToRemoteHost { + endpoint: "imap", + host: self.server_host.clone(), + }); + } + if self.smtp_tls_mode == TlsMode::Plain && !is_loopback_host(&self.smtp_host) { + return Err(ConfigError::CleartextToRemoteHost { + endpoint: "smtp", + host: self.smtp_host.clone(), + }); + } } if self.keyring_secret_key.trim().is_empty() { return Err(ConfigError::EmptyField { @@ -239,6 +303,35 @@ mod tests { ); } + #[test] + fn caldav_account_rejects_cleartext_http_to_remote_host() { + let mut config = valid_caldav_account(); + config.collection_url = "http://remote/dav/".to_string(); + assert_eq!( + config.validate().unwrap_err(), + ConfigError::CleartextToRemoteHost { + endpoint: "caldav", + host: "remote".to_string(), + } + ); + } + + #[test] + fn caldav_account_accepts_https_to_remote_host() { + let mut config = valid_caldav_account(); + config.collection_url = "https://remote/dav/".to_string(); + assert!(config.validate().is_ok()); + } + + #[test] + fn caldav_account_accepts_cleartext_http_to_loopback_host() { + for url in ["http://127.0.0.1/dav/", "http://localhost/dav/"] { + let mut config = valid_caldav_account(); + config.collection_url = url.to_string(); + assert!(config.validate().is_ok(), "url {url} should be allowed"); + } + } + #[test] fn account_protocol_is_dav_only_for_dav_protocols() { assert!(AccountProtocol::CalDav.is_dav()); @@ -358,6 +451,52 @@ mod tests { ); } + #[test] + fn plain_imap_to_remote_host_fails_validation() { + let mut config = valid_account(); + config.imap_tls_mode = TlsMode::Plain; + assert_eq!( + config.validate().unwrap_err(), + ConfigError::CleartextToRemoteHost { + endpoint: "imap", + host: "jmap.nuncio.mx".to_string(), + } + ); + } + + #[test] + fn plain_smtp_to_remote_host_fails_validation() { + let mut config = valid_account(); + config.smtp_tls_mode = TlsMode::Plain; + assert_eq!( + config.validate().unwrap_err(), + ConfigError::CleartextToRemoteHost { + endpoint: "smtp", + host: "smtp.nuncio.mx".to_string(), + } + ); + } + + #[test] + fn plain_imap_and_smtp_to_loopback_hosts_pass_validation() { + for host in ["127.0.0.1", "localhost", "::1", "LOCALHOST"] { + let mut config = valid_account(); + config.server_host = host.to_string(); + config.smtp_host = host.to_string(); + config.imap_tls_mode = TlsMode::Plain; + config.smtp_tls_mode = TlsMode::Plain; + assert!(config.validate().is_ok(), "host {host} should be allowed"); + } + } + + #[test] + fn implicit_tls_and_starttls_to_remote_host_pass_validation() { + let mut config = valid_account(); + config.imap_tls_mode = TlsMode::ImplicitTls; + config.smtp_tls_mode = TlsMode::StartTls; + assert!(config.validate().is_ok()); + } + #[test] fn account_protocol_serde_roundtrip() { let jmap_json = serde_json::to_string(&AccountProtocol::Jmap).unwrap(); @@ -415,5 +554,13 @@ mod tests { ConfigError::SyncIntervalTooShort.to_string(), "sync interval must be at least 10 seconds" ); + assert_eq!( + ConfigError::CleartextToRemoteHost { + endpoint: "imap", + host: "mail.example.com".to_string(), + } + .to_string(), + "imap host 'mail.example.com' is not loopback; refusing cleartext Plain TLS mode to a remote host" + ); } } diff --git a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs index a6dd608..10c5b42 100644 --- a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs +++ b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs @@ -87,7 +87,7 @@ fn sample_account_config() -> AccountConfig { smtp_tls_mode: TlsMode::Plain.into(), keyring_secret_key: KEYRING_KEY.to_string(), sync_interval_secs: 60, - smtp_host: "smtp.nuncio.mx".to_string(), + smtp_host: "127.0.0.1".to_string(), smtp_port: 25, collection_url: String::new(), } -- 2.54.0 From 6d62c09fcdd1a5886f34b8d1ab4563f4cf6528df Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 13:55:17 -0500 Subject: [PATCH 19/95] feat(nunciod): graceful shutdown with WAL-checkpointing close (#340) Install a Ctrl+C/SIGTERM signal handler and share one shutdown signal across the real-sync command consumer, the outbox worker, the update-check loop, and the gRPC server, so a clean exit drains every task and forces the WAL checkpoint via DatabaseEngine::close() instead of the OS killing the process mid-write. The signal also finally triggers the existing CoreCommand::Shutdown / CoreEvent::ShuttingDown event-bus path outside of tests. Refs #289 Co-authored-by: James Maes --- crates/nunciod/src/grpc.rs | 108 ++++++++++- crates/nunciod/src/lib.rs | 1 + crates/nunciod/src/lifecycle.rs | 157 ++++++++++++++++ crates/nunciod/src/main.rs | 168 +++++++++++++---- .../nunciod/tests/graceful_shutdown_test.rs | 176 ++++++++++++++++++ 5 files changed, 572 insertions(+), 38 deletions(-) create mode 100644 crates/nunciod/src/lifecycle.rs create mode 100644 crates/nunciod/tests/graceful_shutdown_test.rs diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 23f0f66..5aa2251 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -2283,6 +2283,73 @@ pub async fn serve_on_listener( .await } +/// Binds a loopback TCP listener at `addr` exactly like [`serve`], but stops +/// accepting new connections and begins draining in-flight requests as soon +/// as `shutdown` resolves, instead of running until the transport errors. +/// +/// `addr` MUST be a loopback address; see [`serve`]. +pub async fn serve_with_shutdown( + addr: &str, + event_bus: Arc, + db: Arc, + filter_engine: Arc, + secrets: Arc, + token: impl Into>, + shutdown: impl std::future::Future + Send + 'static, +) -> Result<(), GrpcServeError> { + let socket_addr = addr + .parse::() + .map_err(|_| GrpcServeError::NonLoopbackAddress(addr.to_string()))?; + if !socket_addr.ip().is_loopback() { + return Err(GrpcServeError::NonLoopbackAddress(addr.to_string())); + } + + let listener = TcpListener::bind(addr) + .await + .map_err(|source| GrpcServeError::Bind { + addr: addr.to_string(), + source, + })?; + serve_on_listener_with_shutdown( + listener, + event_bus, + db, + filter_engine, + secrets, + token, + shutdown, + ) + .await +} + +/// Identical to [`serve_on_listener`], except the server stops accepting new +/// connections and begins draining in-flight requests as soon as `shutdown` +/// resolves. +pub async fn serve_on_listener_with_shutdown( + listener: TcpListener, + event_bus: Arc, + db: Arc, + filter_engine: Arc, + secrets: Arc, + token: impl Into>, + shutdown: impl std::future::Future + Send + 'static, +) -> Result<(), GrpcServeError> { + serve_on_listener_with_overrides_and_shutdown( + listener, + event_bus, + db, + filter_engine, + secrets, + token, + MailEngineOverrides::default(), + CalendarEngineOverrides::default(), + ContactsEngineOverrides::default(), + AccountsEngineOverrides::default(), + shutdown, + ) + .await +} + /// Identical to [`serve_on_listener`], except the `nuncio.v1.Mail` service's /// inbound sync and outbound send engines, the `nuncio.v1.Calendar` /// service's sync backend, and the `nuncio.v1.Contacts` service's sync @@ -2306,6 +2373,45 @@ pub async fn serve_on_listener_with_overrides( calendar_overrides: CalendarEngineOverrides, contacts_overrides: ContactsEngineOverrides, accounts_overrides: AccountsEngineOverrides, +) -> Result<(), GrpcServeError> { + // Delegates to the shutdown-aware variant with a shutdown future that + // never resolves, preserving this function's exact prior behavior (run + // until the transport itself errors) for every existing caller. + serve_on_listener_with_overrides_and_shutdown( + listener, + event_bus, + db, + filter_engine, + secrets, + token, + overrides, + calendar_overrides, + contacts_overrides, + accounts_overrides, + std::future::pending(), + ) + .await +} + +/// Identical to [`serve_on_listener_with_overrides`], except the server +/// stops accepting new connections and begins draining in-flight requests +/// as soon as `shutdown` resolves, instead of running until the transport +/// errors. This is the function production actually calls (via +/// [`serve_with_shutdown`]); [`serve_on_listener_with_overrides`] delegates +/// here with a `shutdown` future that never resolves. +#[allow(clippy::too_many_arguments)] +pub async fn serve_on_listener_with_overrides_and_shutdown( + listener: TcpListener, + event_bus: Arc, + db: Arc, + filter_engine: Arc, + secrets: Arc, + token: impl Into>, + overrides: MailEngineOverrides, + calendar_overrides: CalendarEngineOverrides, + contacts_overrides: ContactsEngineOverrides, + accounts_overrides: AccountsEngineOverrides, + shutdown: impl std::future::Future + Send + 'static, ) -> Result<(), GrpcServeError> { let token: Arc = token.into(); @@ -2392,7 +2498,7 @@ pub async fn serve_on_listener_with_overrides( .add_service(audit_svc) .add_service(calendar_svc) .add_service(contacts_svc) - .serve_with_incoming(TcpListenerStream::new(listener)) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown) .await .map_err(GrpcServeError::Transport) } diff --git a/crates/nunciod/src/lib.rs b/crates/nunciod/src/lib.rs index 7b3413b..9a10ad9 100644 --- a/crates/nunciod/src/lib.rs +++ b/crates/nunciod/src/lib.rs @@ -5,6 +5,7 @@ pub mod calendar_sync; pub mod contacts_sync; pub mod grpc; +pub mod lifecycle; pub mod orchestrator; pub mod outbox; pub mod send; diff --git a/crates/nunciod/src/lifecycle.rs b/crates/nunciod/src/lifecycle.rs new file mode 100644 index 0000000..dcef665 --- /dev/null +++ b/crates/nunciod/src/lifecycle.rs @@ -0,0 +1,157 @@ +//! Coordinates graceful shutdown across every long-running task `nunciod` +//! spawns: the gRPC server, the outbox worker, the auto-update-check loop, +//! and the real-sync `CoreCommand` consumer. +//! +//! Previously the daemon installed no signal handler at all: every real +//! exit was an OS kill (SIGKILL/SIGTERM without a handler, a terminated +//! console), which races [`nuncio_store::DatabaseEngine::close`]'s WAL +//! checkpoint and can corrupt the database. A single [`ShutdownSignal`], +//! shared by every task, lets each one stop accepting new work and finish +//! cleanly instead. + +use nuncio_core::{CoreCommand, EventBus}; +use std::sync::Arc; +use tokio::sync::watch; + +/// Read-only handle to the shared shutdown flag. Cloned into every +/// long-running task; `wait` resolves once shutdown has been requested, +/// including if it was already requested before this clone started +/// waiting. +#[derive(Clone)] +pub struct ShutdownSignal { + rx: watch::Receiver, +} + +impl ShutdownSignal { + /// Resolves once shutdown has been requested. Safe to call from + /// `tokio::select!` on every loop iteration: a task that observes + /// shutdown mid-iteration finishes its current unit of work (the other + /// `select!` branch) before this is polled again, so no in-flight + /// database write is abandoned half-done. + pub async fn wait(&mut self) { + // `watch::Receiver::wait_for` returns immediately if the current + // value already satisfies the predicate, so a clone created after + // shutdown was already requested does not hang waiting for a + // change that already happened. + let _ = self.rx.wait_for(|requested| *requested).await; + } + + /// Non-blocking check for loops that need to poll rather than await. + pub fn is_requested(&self) -> bool { + *self.rx.borrow() + } +} + +/// Owns the shutdown trigger. Production wires [`ShutdownController::trigger`] +/// to OS signal delivery via [`install_signal_handlers`]; tests call it +/// directly to drive the shutdown path deterministically without sending a +/// real OS signal. +pub struct ShutdownController { + tx: watch::Sender, + event_bus: Arc, +} + +impl ShutdownController { + /// Creates a controller/signal pair. `event_bus` is the SAME bus every + /// other subsystem shares, so [`trigger`](Self::trigger) drives the + /// existing `CoreCommand::Shutdown` / `CoreEvent::ShuttingDown` path + /// visibly (state flips to `ShuttingDown`, subscribers are notified) + /// instead of that path staying reachable only from tests. + pub fn new(event_bus: Arc) -> (Self, ShutdownSignal) { + let (tx, rx) = watch::channel(false); + (Self { tx, event_bus }, ShutdownSignal { rx }) + } + + /// Initiates shutdown. Idempotent: calling it more than once (e.g. a + /// second Ctrl+C while already draining) is a no-op beyond the first. + pub fn trigger(&self) { + self.event_bus.process_command(CoreCommand::Shutdown); + let _ = self.tx.send(true); + } +} + +/// Waits for Ctrl+C, or (on Unix) SIGTERM, then triggers shutdown. Intended +/// to be spawned once at startup and left running for the process lifetime. +pub async fn install_signal_handlers(controller: Arc) { + wait_for_os_signal().await; + controller.trigger(); +} + +#[cfg(unix)] +async fn wait_for_os_signal() { + use tokio::signal::unix::{signal, SignalKind}; + + match signal(SignalKind::terminate()) { + Ok(mut sigterm) => { + tokio::select! { + _ = tokio::signal::ctrl_c() => { + tracing::info!("received Ctrl+C; starting graceful shutdown"); + } + _ = sigterm.recv() => { + tracing::info!("received SIGTERM; starting graceful shutdown"); + } + } + } + Err(err) => { + // Ctrl+C still works even if SIGTERM registration failed (e.g. + // an exotic sandboxed environment); log so the gap is visible + // rather than silently dropping SIGTERM handling. + tracing::error!("failed to install SIGTERM handler: {err}; SIGTERM will kill the process without a clean shutdown"); + let _ = tokio::signal::ctrl_c().await; + tracing::info!("received Ctrl+C; starting graceful shutdown"); + } + } +} + +#[cfg(not(unix))] +async fn wait_for_os_signal() { + let _ = tokio::signal::ctrl_c().await; + tracing::info!("received Ctrl+C; starting graceful shutdown"); +} + +#[cfg(test)] +mod tests { + use super::*; + use nuncio_core::EngineStatus; + + #[tokio::test] + async fn trigger_flips_signal_and_event_bus_state() { + let event_bus = Arc::new(EventBus::new()); + let (controller, mut signal) = ShutdownController::new(event_bus.clone()); + + assert!(!signal.is_requested()); + assert_eq!(event_bus.current_state().status, EngineStatus::Idle); + + controller.trigger(); + + assert!(signal.is_requested()); + assert_eq!(event_bus.current_state().status, EngineStatus::ShuttingDown); + // Resolves immediately: the signal was already flipped above. + signal.wait().await; + } + + #[tokio::test] + async fn signal_clone_observes_a_trigger_that_happened_before_it_was_created() { + let event_bus = Arc::new(EventBus::new()); + let (controller, signal) = ShutdownController::new(event_bus); + controller.trigger(); + + // A clone taken AFTER the trigger must still resolve `wait` + // immediately rather than hang waiting for a fresh `changed()`. + let mut late_clone = signal.clone(); + late_clone.wait().await; + assert!(late_clone.is_requested()); + } + + #[tokio::test] + async fn trigger_is_idempotent() { + let event_bus = Arc::new(EventBus::new()); + let (controller, signal) = ShutdownController::new(event_bus.clone()); + + controller.trigger(); + controller.trigger(); + + assert!(signal.is_requested()); + assert_eq!(event_bus.current_state().status, EngineStatus::ShuttingDown); + } +} diff --git a/crates/nunciod/src/main.rs b/crates/nunciod/src/main.rs index c98aa72..d50f90b 100644 --- a/crates/nunciod/src/main.rs +++ b/crates/nunciod/src/main.rs @@ -4,7 +4,16 @@ use nuncio_core::{CoreCommand, CoreEvent, EventBus}; use nuncio_filter::FilterEngine; +use nunciod::lifecycle::{install_signal_handlers, ShutdownController}; use std::sync::Arc; +use std::time::Duration; + +/// Bound on how long shutdown waits for in-flight work (gRPC requests still +/// being served, an outbox drain pass in progress, a background task's +/// current loop iteration) to finish on its own before the process moves on +/// to closing the database and exiting anyway. Keeps a stuck task from +/// turning a graceful shutdown into a hang. +const SHUTDOWN_GRACE_PERIOD: Duration = Duration::from_secs(10); #[tokio::main] async fn main() -> Result<(), Box> { @@ -17,6 +26,17 @@ async fn main() -> Result<(), Box> { let mut event_bus_owned = EventBus::new(); let command_rx = event_bus_owned.take_command_receiver(); let event_bus = Arc::new(event_bus_owned); + + // Single shutdown signal shared by every long-running task spawned + // below (the real-sync command consumer, the outbox worker, the + // update-check loop, and the gRPC server itself), so a Ctrl+C/SIGTERM + // drains all of them cleanly instead of the OS killing the process + // mid-write. `install_signal_handlers` also drives the existing + // `CoreCommand::Shutdown` / `CoreEvent::ShuttingDown` event-bus path, + // previously wired but never triggered outside tests. + let (shutdown_controller, shutdown_signal) = ShutdownController::new(event_bus.clone()); + let shutdown_controller = Arc::new(shutdown_controller); + let signal_task = tokio::spawn(install_signal_handlers(shutdown_controller)); // PERSISTENT database path: defaults to `~/.nuncio/nuncio.db` -- NOT a // temp/ephemeral path -- so accounts (and everything else) survive a // daemon restart. @@ -52,13 +72,24 @@ async fn main() -> Result<(), Box> { // trigger_background_resync`'s post-recovery `send_command` calls a // consumer: previously nothing read this channel, so that resync // trigger was inert. - if let Some(mut command_rx) = command_rx { + let sync_command_task = if let Some(mut command_rx) = command_rx { let db_sync = db.clone(); let secrets_sync = account_secrets.clone(); let event_bus_sync = event_bus.clone(); let filter_engine_sync = filter_engine.clone(); - let _sync_command_task = tokio::spawn(async move { - while let Some(cmd) = command_rx.recv().await { + let mut shutdown = shutdown_signal.clone(); + Some(tokio::spawn(async move { + loop { + let cmd = tokio::select! { + _ = shutdown.wait() => { + tracing::info!("sync command consumer exiting on shutdown signal"); + break; + } + received = command_rx.recv() => match received { + Some(cmd) => cmd, + None => break, + }, + }; match cmd { CoreCommand::SyncAll => { let synced = nunciod::sync::run_all_accounts_sync( @@ -93,12 +124,13 @@ async fn main() -> Result<(), Box> { _ => {} } } - }); + })) } else { tracing::error!( "EventBus command receiver was already taken; real sync command processing will not run" ); - } + None + }; // Background Outbox Worker Task. Each tick drains the pending remote // mutations the filter engine enqueued (move/copy/flag/unflag/delete on the @@ -112,20 +144,34 @@ async fn main() -> Result<(), Box> { db.clone(), account_secrets.clone(), )); - let _outbox_task = tokio::spawn(async move { + let mut outbox_shutdown = shutdown_signal.clone(); + let outbox_task = tokio::spawn(async move { let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); loop { - interval.tick().await; - let summary = - nunciod::outbox::execute_pending_mutations(&db_outbox, outbox_env.as_ref(), 50) + tokio::select! { + _ = outbox_shutdown.wait() => { + tracing::info!("outbox worker exiting on shutdown signal"); + break; + } + _ = interval.tick() => { + // Runs to completion once selected: a drain pass already + // in progress is never aborted mid-write by a shutdown + // signal that arrives after this branch was chosen. + let summary = nunciod::outbox::execute_pending_mutations( + &db_outbox, + outbox_env.as_ref(), + 50, + ) .await; - if summary.completed > 0 || summary.failed > 0 { - tracing::info!( - "outbox drain: {} completed, {} retried, {} failed", - summary.completed, - summary.retried, - summary.failed - ); + if summary.completed > 0 || summary.failed > 0 { + tracing::info!( + "outbox drain: {} completed, {} retried, {} failed", + summary.completed, + summary.retried, + summary.failed + ); + } + } } } }); @@ -138,21 +184,29 @@ async fn main() -> Result<(), Box> { // closed, nunciod must never autonomously check for or install // updates, so this loop only runs when an operator explicitly opts in // via `NUNCIO_AUTO_UPDATE_ENABLED=1`. - let _update_task = if nunciod::auto_update_task_enabled() { + let update_task = if nunciod::auto_update_task_enabled() { let event_bus_update = event_bus.clone(); + let mut update_shutdown = shutdown_signal.clone(); Some(tokio::spawn(async move { let mut interval = tokio::time::interval(std::time::Duration::from_secs(86400)); interval.tick().await; loop { - interval.tick().await; - if let Ok(updater) = nuncio_core::UpdateEngine::new() { - if let Ok(result) = updater.check_for_updates().await { - if result.update_available { - if let Some(info) = result.release_info { - event_bus_update.publish_event(CoreEvent::UpdateAvailable { - version: info.version, - release_notes: info.release_notes, - }); + tokio::select! { + _ = update_shutdown.wait() => { + tracing::info!("update-check loop exiting on shutdown signal"); + break; + } + _ = interval.tick() => { + if let Ok(updater) = nuncio_core::UpdateEngine::new() { + if let Ok(result) = updater.check_for_updates().await { + if result.update_available { + if let Some(info) = result.release_info { + event_bus_update.publish_event(CoreEvent::UpdateAvailable { + version: info.version, + release_notes: info.release_notes, + }); + } + } } } } @@ -192,17 +246,57 @@ async fn main() -> Result<(), Box> { // The gRPC server is the long-running foreground task that keeps this // process alive; every other subsystem above is a background worker - // spawned on top of it. - nunciod::grpc::serve( - &grpc_addr, - event_bus, - db, - filter_engine, - grpc_secrets, - grpc_token, - ) - .await - .map_err(|e| format!("nunciod gRPC server failed: {e}"))?; + // spawned on top of it. `serve_with_shutdown` stops accepting new + // connections and starts draining in-flight requests as soon as + // `shutdown_signal` fires; the `select!` below bounds how long that + // drain is allowed to take before this process moves on regardless. + let db_for_serve = db.clone(); + let mut grpc_shutdown_future = shutdown_signal.clone(); + let mut grace_timer_signal = shutdown_signal.clone(); + let grpc_result = tokio::select! { + result = nunciod::grpc::serve_with_shutdown( + &grpc_addr, + event_bus, + db_for_serve, + filter_engine, + grpc_secrets, + grpc_token, + async move { grpc_shutdown_future.wait().await }, + ) => result, + _ = async move { + grace_timer_signal.wait().await; + tokio::time::sleep(SHUTDOWN_GRACE_PERIOD).await; + } => { + tracing::warn!( + "gRPC server did not finish draining in-flight requests within the {:?} \ + shutdown grace period; closing the database and exiting anyway", + SHUTDOWN_GRACE_PERIOD + ); + Ok(()) + } + }; + + // Bound how long the background workers get to notice the shutdown + // signal and exit their loops before the database is closed out from + // under them. + let _ = tokio::time::timeout(SHUTDOWN_GRACE_PERIOD, async { + if let Some(task) = sync_command_task { + let _ = task.await; + } + let _ = outbox_task.await; + if let Some(task) = update_task { + let _ = task.await; + } + }) + .await; + signal_task.abort(); + + // Force the WAL checkpoint before the process exits so no committed + // page is left for SQLite's own deferred "last connection closes" + // teardown to race (see `DatabaseEngine::close`'s doc comment). + db.close().await; + + grpc_result.map_err(|e| format!("nunciod gRPC server failed: {e}"))?; Ok(()) } diff --git a/crates/nunciod/tests/graceful_shutdown_test.rs b/crates/nunciod/tests/graceful_shutdown_test.rs new file mode 100644 index 0000000..40d784c --- /dev/null +++ b/crates/nunciod/tests/graceful_shutdown_test.rs @@ -0,0 +1,176 @@ +//! Integration test for `nunciod`'s graceful shutdown path. +//! +//! Before this, `nunciod` installed no signal handler and never called +//! [`DatabaseEngine::close`], so every real exit was an OS kill racing the +//! WAL checkpoint that `close` exists to force. This test drives the same +//! building blocks `main.rs` wires together -- a [`ShutdownController`] +//! shared by the gRPC server and a background worker, triggered directly +//! (never a real OS signal, for determinism) -- and asserts: +//! (a) the gRPC server stops serving once the shutdown signal fires; +//! (b) a background worker exits its loop on the same signal instead of +//! being aborted mid-iteration; +//! (c) `DatabaseEngine::close()` runs and leaves the pool unusable for +//! any subsequent query. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use nuncio_core::EventBus; +use nuncio_filter::FilterEngine; +use nuncio_proto::v1::system_client::SystemClient; +use nuncio_proto::v1::GetStatusRequest; +use nuncio_store::db::DatabaseEngine; +use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT}; +use nunciod::grpc::serve_on_listener_with_shutdown; +use nunciod::lifecycle::ShutdownController; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use tokio::net::TcpListener; +use tonic::Request; + +const JOIN_TIMEOUT: Duration = Duration::from_secs(5); + +#[tokio::test] +async fn shutdown_signal_stops_grpc_server_exits_background_worker_and_closes_the_database() { + let event_bus = Arc::new(EventBus::new()); + let (controller, shutdown_signal) = ShutdownController::new(event_bus.clone()); + + let secrets = Arc::new(SecretManager::mock()); + let token = hex::encode( + secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("token provisioned from mock vault"), + ); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + let db = Arc::new(db); + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("listener has local addr"); + + // (a) the gRPC server: mirrors main.rs's `serve_with_shutdown` wiring -- + // the future it's given resolves from the SAME `ShutdownSignal` the + // worker below observes. + let mut grpc_shutdown = shutdown_signal.clone(); + let grpc_task = tokio::spawn(serve_on_listener_with_shutdown( + listener, + event_bus.clone(), + db.clone(), + filter_engine, + secrets, + token.clone(), + async move { grpc_shutdown.wait().await }, + )); + + // Confirm the server actually accepts a call before shutdown, so a + // trivially-passing "stopped" observation later isn't just "it never + // started". + let mut client = SystemClient::connect(format!("http://{addr}")) + .await + .expect("client connects before shutdown"); + let mut request = Request::new(GetStatusRequest {}); + request.metadata_mut().insert( + "authorization", + format!("Bearer {token}") + .parse() + .expect("valid ascii metadata value"), + ); + client + .get_status(request) + .await + .expect("authenticated call succeeds before shutdown"); + + // (b) a background worker built the same way the outbox worker and + // update-check loop are in `main.rs`: a `tokio::select!` between the + // shared shutdown signal and a repeating unit of work. `tick_count` + // proves the loop body actually ran, ruling out the trivial case where + // the select immediately picks the shutdown branch every time. + let tick_count = Arc::new(AtomicUsize::new(0)); + let tick_count_worker = tick_count.clone(); + let mut worker_shutdown = shutdown_signal.clone(); + let worker_task = tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_millis(20)); + loop { + tokio::select! { + _ = worker_shutdown.wait() => break, + _ = interval.tick() => { + tick_count_worker.fetch_add(1, Ordering::SeqCst); + } + } + } + }); + + // Let the worker tick a few times before shutdown is requested. + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + tick_count.load(Ordering::SeqCst) > 0, + "worker must have run at least one iteration before shutdown" + ); + + // Trigger shutdown once, from a single controller shared by both tasks + // -- exactly the production wiring, minus the real OS signal. + controller.trigger(); + + let grpc_result = tokio::time::timeout(JOIN_TIMEOUT, grpc_task) + .await + .expect("gRPC server task must exit within the join timeout, not hang") + .expect("gRPC server task must not panic"); + assert!( + grpc_result.is_ok(), + "serve_on_listener_with_shutdown must return Ok once it finishes draining, got {grpc_result:?}" + ); + + tokio::time::timeout(JOIN_TIMEOUT, worker_task) + .await + .expect("background worker must exit on the shutdown signal within the join timeout, not be killed") + .expect("background worker task must not panic"); + + // A dial attempt after the server task has exited must fail: the + // listener is gone, proving the server actually stopped accepting + // connections rather than merely returning while still bound. + assert!( + SystemClient::connect(format!("http://{addr}")) + .await + .is_err(), + "gRPC server must no longer be reachable after shutdown" + ); + + // (c) `DatabaseEngine::close()` -- forces the WAL checkpoint and closes + // the pool. A query issued afterward must fail, proving `close()` ran + // rather than merely being reachable dead code. + db.close().await; + let err = db + .list_filter_rules() + .await + .expect_err("a query after close() must fail because the pool is closed"); + let message = err.to_string(); + assert!( + message.to_lowercase().contains("closed") || message.to_lowercase().contains("pool"), + "expected a pool-closed style error after DatabaseEngine::close(), got: {message}" + ); +} + +#[tokio::test] +async fn controller_trigger_is_observed_by_every_clone_of_the_signal() { + let event_bus = Arc::new(EventBus::new()); + let (controller, shutdown_signal) = ShutdownController::new(event_bus); + + let mut first = shutdown_signal.clone(); + let mut second = shutdown_signal.clone(); + assert!(!first.is_requested()); + assert!(!second.is_requested()); + + controller.trigger(); + + tokio::time::timeout(JOIN_TIMEOUT, first.wait()) + .await + .expect("first clone must observe the trigger"); + tokio::time::timeout(JOIN_TIMEOUT, second.wait()) + .await + .expect("second clone must observe the trigger"); + assert!(second.is_requested()); +} -- 2.54.0 From f44412f1ab2def55bd2e120ed476383002231b52 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 13:57:22 -0500 Subject: [PATCH 20/95] fix(store): deterministic message surrogate identity (C1) (#337) * fix(store): give messages a deterministic surrogate identity IMAP UIDs are unique only per mailbox+UIDVALIDITY, yet messages.id was the sole key and Email.id was "imap-uid-{uid}", so the same UID in two folders or accounts collapsed to one row and silently overwrote (audit C1). The id shape also leaked internal encoding on the wire. Messages now carry an opaque surrogate id = SHA-256 over length-prefixed (account, folder, uid_validity, remote_id), plus first-class remote_id and uid_validity columns and a UNIQUE(account, folder, uid_validity, remote_id) index (backfill-safe migration seeding legacy rows from the old id). The IMAP and JMAP backends populate remote_id/uid_validity and hash the id; the outbox executor recovers remote addressing from the columns instead of parsing the id, still SELECT-verifying UIDVALIDITY before mutating. Re-sync recomputes the same id, so it upserts one row and the fire-once filter guard still holds. Message.id stays a proto string; only its value becomes opaque (no proto change). Refs #281 * fix(store): make message-identity migration atomic and self-healing The identity migration ran ADD COLUMN, backfill, and CREATE UNIQUE INDEX as separate autocommit statements with the backfill gated on having just added the column. A crash between the ADD and the backfill could leave legacy rows at remote_id='' with the column present, so the next startup skipped the backfill and the index creation then collided on (account, folder, uid_validity='', remote_id='') -- wedging the daemon. Run the whole step in one transaction (a crash rolls back atomically) and make the backfill unconditional (idempotent UPDATE ... WHERE remote_id = '') so any committed partial state is completed before the index is built. No-op on a fresh or already-migrated database. Refs #281 --------- Co-authored-by: James Maes --- crates/nuncio-core/src/export.rs | 4 + crates/nuncio-core/src/model.rs | 81 +++++- crates/nuncio-filter/src/engine.rs | 4 + crates/nuncio-mail/src/backend.rs | 32 ++- crates/nuncio-mail/src/imap.rs | 96 ++++--- crates/nuncio-mail/src/jmap.rs | 36 ++- crates/nuncio-mail/src/mock.rs | 2 + crates/nuncio-mail/src/parser.rs | 4 + crates/nuncio-mail/src/smtp.rs | 2 + .../nuncio-mail/tests/wiremock_jmap_test.rs | 15 +- crates/nuncio-store/src/db.rs | 257 +++++++++++++++++- crates/nuncio-store/src/search.rs | 2 + crates/nunciod/src/grpc.rs | 15 + crates/nunciod/src/outbox.rs | 23 +- crates/nunciod/src/sync.rs | 11 +- .../nunciod/tests/filters_triage_e2e_test.rs | 2 + .../nunciod/tests/outbox_executor_e2e_test.rs | 14 +- crates/nunciod/tests/spine_e2e_test.rs | 2 + 18 files changed, 510 insertions(+), 92 deletions(-) diff --git a/crates/nuncio-core/src/export.rs b/crates/nuncio-core/src/export.rs index 360773f..d180291 100644 --- a/crates/nuncio-core/src/export.rs +++ b/crates/nuncio-core/src/export.rs @@ -185,6 +185,8 @@ mod tests { id: "msg-100".to_string(), account_id: "acct-1".to_string(), folder_id: "INBOX".to_string(), + remote_id: "100".to_string(), + uid_validity: "1".to_string(), subject: "Test Subject 1".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), @@ -198,6 +200,8 @@ mod tests { id: "msg-101".to_string(), account_id: "acct-1".to_string(), folder_id: "INBOX".to_string(), + remote_id: "101".to_string(), + uid_validity: "1".to_string(), subject: "Test Subject 2".to_string(), sender: "charlie@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index 311c1b5..b6e5034 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -2,6 +2,7 @@ use bytes::Bytes; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; /// Real-time daemon health, performance metrics, and telemetry payload. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -63,12 +64,30 @@ pub struct Attachment { /// Email message domain entity owned by Nuncio core. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Email { - /// Unique message identifier. + /// Opaque, deterministic surrogate identifier for the message. + /// + /// Derived from [`Email::surrogate_id`] over the account, folder, + /// UIDVALIDITY, and protocol-native id. It carries no decodable internal + /// encoding (callers must treat it as opaque) and is stable across + /// re-syncs of the same message, so persisting it upserts the same row + /// rather than duplicating. Distinct messages that happen to share a + /// protocol id across folders or accounts hash to distinct surrogates, so + /// one can never silently overwrite another. pub id: String, /// Account identifier owning the message. pub account_id: String, /// Mailbox folder identifier (e.g. "inbox"). pub folder_id: String, + /// Protocol-native message id used to address the message on its server: + /// the IMAP UID as a decimal string, or the JMAP Email object id. This is + /// what a remote mutation must use to identify the message on the wire -- + /// never the opaque [`Email::id`]. + pub remote_id: String, + /// The addressing scope the `remote_id` is only meaningful within: the IMAP + /// folder UIDVALIDITY as a decimal string. Protocols without a UIDVALIDITY + /// (JMAP) carry a stable sentinel instead, so the surrogate id stays + /// deterministic. + pub uid_validity: String, /// Subject line. pub subject: String, /// Sender address (e.g. "alice@nuncio.mx"). @@ -87,6 +106,33 @@ pub struct Email { pub attachments: Vec, } +impl Email { + /// Compute the deterministic, collision-resistant surrogate id for a + /// message from its addressing coordinates. + /// + /// The four coordinates together uniquely identify a stored message: a + /// protocol-native `remote_id` (IMAP UID / JMAP object id) is only unique + /// within one `uid_validity` scope of one `folder_id` of one `account_id`, + /// so all four must participate. Each field is fed length-prefixed so that + /// no two distinct coordinate tuples can serialize to the same byte stream + /// (e.g. `("a", "bc")` and `("ab", "c")` must not collide). The SHA-256 + /// digest is hex-encoded, yielding a stable opaque token that leaks none of + /// its inputs' encoding on the wire. + pub fn surrogate_id( + account_id: &str, + folder_id: &str, + uid_validity: &str, + remote_id: &str, + ) -> String { + let mut hasher = Sha256::new(); + for field in [account_id, folder_id, uid_validity, remote_id] { + hasher.update((field.len() as u64).to_le_bytes()); + hasher.update(field.as_bytes()); + } + hex::encode(hasher.finalize()) + } +} + /// Mailbox folder entity. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Folder { @@ -168,4 +214,37 @@ mod tests { assert_eq!(parsed.status, "HEALTHY"); assert_eq!(parsed.total_emails, 15000); } + + #[test] + fn surrogate_id_is_deterministic_and_opaque() { + let a = Email::surrogate_id("acct-1", "INBOX", "42", "5"); + let b = Email::surrogate_id("acct-1", "INBOX", "42", "5"); + assert_eq!(a, b, "identical coordinates must hash identically"); + // A 32-byte SHA-256 digest hex-encodes to 64 chars, and the token must + // not leak the protocol-native id it was built from. + assert_eq!(a.len(), 64); + assert!(a.chars().all(|c| c.is_ascii_hexdigit())); + assert!(!a.contains("imap-uid-")); + } + + #[test] + fn surrogate_id_distinguishes_the_cross_folder_and_cross_account_collision() { + // The same protocol UID (5) in two different folders of one account, or + // in two different accounts, must NOT collapse to the same surrogate. + let inbox = Email::surrogate_id("acct-1", "INBOX", "42", "5"); + let sent = Email::surrogate_id("acct-1", "Sent", "7", "5"); + let other_account = Email::surrogate_id("acct-2", "INBOX", "42", "5"); + assert_ne!(inbox, sent, "same UID in two folders must not collide"); + assert_ne!( + inbox, other_account, + "same UID in two accounts must not collide" + ); + + // Field boundaries are unambiguous: shifting a delimiter-like split + // across two adjacent fields yields a different id. + assert_ne!( + Email::surrogate_id("a", "bc", "1", "1"), + Email::surrogate_id("ab", "c", "1", "1"), + ); + } } diff --git a/crates/nuncio-filter/src/engine.rs b/crates/nuncio-filter/src/engine.rs index bce77e8..6037db1 100644 --- a/crates/nuncio-filter/src/engine.rs +++ b/crates/nuncio-filter/src/engine.rs @@ -373,6 +373,8 @@ mod tests { id: "msg-1".to_string(), account_id: account_id.to_string(), folder_id: folder_id.to_string(), + remote_id: "1".to_string(), + uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), @@ -394,6 +396,8 @@ mod tests { id: "msg-1".to_string(), account_id: "acct-1".to_string(), folder_id: "inbox".to_string(), + remote_id: "1".to_string(), + uid_validity: "1".to_string(), subject: "Urgent Meeting".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), diff --git a/crates/nuncio-mail/src/backend.rs b/crates/nuncio-mail/src/backend.rs index 5c9bb96..cb999bd 100644 --- a/crates/nuncio-mail/src/backend.rs +++ b/crates/nuncio-mail/src/backend.rs @@ -36,25 +36,31 @@ pub enum RemoteMutationKind { Delete, } -/// A fully-addressed remote mutation for a single message: which message -/// (protocol-native id), the folder it currently lives in, that folder's -/// stored sync checkpoint (carrying IMAP UIDVALIDITY), and the change to -/// apply. +/// A fully-addressed remote mutation for a single message: the protocol-native +/// id to address it on the server, the folder it currently lives in, the +/// UIDVALIDITY scope that id was captured under, and the change to apply. /// -/// `folder_checkpoint` is the value persisted by a prior sync -/// (`"{uidvalidity}:{uidnext}"` for IMAP). The IMAP backend uses it to refuse -/// to act when the mailbox's current UIDVALIDITY no longer matches -- a -/// renumbered mailbox reassigns UIDs, so the stored UID would otherwise -/// address the wrong message. Protocols that do not use UIDVALIDITY (JMAP) -/// ignore it. +/// Addressing is driven by `remote_id` (the IMAP UID / JMAP object id), NOT by +/// the opaque `message_id` -- so remote-mutation targeting never depends on the +/// shape of the surrogate id. `uid_validity` is the mailbox UIDVALIDITY the +/// `remote_id` was captured under; the IMAP backend refuses to act unless the +/// mailbox's current UIDVALIDITY still matches it, because a renumbered mailbox +/// (RFC 3501 s2.3.1.1) reassigns UIDs and the stored UID would otherwise +/// address the wrong message. Protocols without a UIDVALIDITY (JMAP) carry a +/// stable sentinel and ignore this guard. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RemoteMutationSpec { - /// Protocol-native message id (IMAP `imap-uid-{uid}`, JMAP object id). + /// Opaque surrogate id of the target message. Diagnostic only -- it names + /// the message in errors/logs and is never used to address it on the wire. pub message_id: String, + /// Protocol-native id used to address the message on its server: the IMAP + /// UID as a decimal string, or the JMAP Email object id. + pub remote_id: String, /// Folder the message currently lives in. pub folder_id: String, - /// The source folder's stored sync checkpoint, if any. - pub folder_checkpoint: Option, + /// The UIDVALIDITY the `remote_id` was captured under (IMAP; decimal + /// string), or a protocol sentinel for servers without UIDVALIDITY (JMAP). + pub uid_validity: String, /// The change to apply. pub kind: RemoteMutationKind, } diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index adfbf63..c5288ac 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -34,31 +34,21 @@ const FETCH_ITEM_TIMEOUT: Duration = Duration::from_secs(60); use crate::backend::{MailBackend, RemoteMutationKind, RemoteMutationSpec}; use crate::parser::{MailError, MimeParserAdapter}; -/// Prefix an IMAP [`Email::id`] carries in front of its raw numeric UID (see -/// [`ImapEngine::sync_folder_messages_with_session`], which formats every id as -/// `"imap-uid-{uid}"`). A remote mutation must recover the UID from the id to -/// address the message, so the prefix is defined once here. -const IMAP_UID_ID_PREFIX: &str = "imap-uid-"; - -/// Recover the raw IMAP UID from an [`Email::id`] of the form -/// `"imap-uid-{uid}"`. Returns `None` for any id that does not carry the -/// expected prefix and a valid non-zero `u32` UID, so a mutation against a -/// malformed/foreign id fails honestly rather than addressing UID 0. -fn parse_imap_uid(message_id: &str) -> Option { - let raw = message_id.strip_prefix(IMAP_UID_ID_PREFIX)?; - let uid = raw.trim().parse::().ok()?; +/// Recover a raw IMAP UID from a [`RemoteMutationSpec::remote_id`] (a decimal +/// string). Returns `None` for anything that is not a valid non-zero `u32`, so +/// a mutation against a malformed/foreign remote id fails honestly rather than +/// addressing UID 0. +fn parse_imap_uid(remote_id: &str) -> Option { + let uid = remote_id.trim().parse::().ok()?; (uid >= 1).then_some(uid) } -/// Extract the UIDVALIDITY component of a stored folder checkpoint -/// (`"{uidvalidity}:{uidnext}"`). Returns `None` for a missing, malformed, or -/// legacy bare-number checkpoint -- the caller treats an absent stored -/// UIDVALIDITY as "cannot prove the mailbox has not been renumbered" and -/// refuses to act. -fn checkpoint_uid_validity(checkpoint: Option<&str>) -> Option { - let raw = checkpoint?; - let (validity, _uid) = raw.split_once(CHECKPOINT_DELIM)?; - validity.trim().parse::().ok() +/// Parse a stored [`RemoteMutationSpec::uid_validity`] (a decimal string) into +/// a `u32`. Returns `None` for a missing or non-numeric value -- the caller +/// treats an unparseable UIDVALIDITY as "cannot prove the mailbox has not been +/// renumbered" and refuses to act. +fn parse_uid_validity(uid_validity: &str) -> Option { + uid_validity.trim().parse::().ok() } /// Delimiter between the two components of a folder sync checkpoint. The @@ -652,7 +642,16 @@ impl ImapEngine { let uid_num = fetch_data.uid.unwrap_or(0); max_uid_seen = max_uid_seen.max(uid_num); - let email_id = format!("imap-uid-{}", uid_num); + // The message is addressed on the wire by its UID within the + // folder's UIDVALIDITY scope; the persisted id is an opaque + // surrogate hashed over both plus the account and folder, so the + // same UID in another folder/account can never collide. + let remote_id = uid_num.to_string(); + let uid_validity = server_uid_validity + .map(|v| v.to_string()) + .unwrap_or_default(); + let email_id = + Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id); let is_read = fetch_data .flags() @@ -665,6 +664,8 @@ impl ImapEngine { folder_id, raw_bytes, )?; + email.remote_id = remote_id; + email.uid_validity = uid_validity; email.read = is_read; email } else { @@ -721,6 +722,8 @@ impl ImapEngine { id: email_id, account_id: self.account_id.clone(), folder_id: folder_id.to_string(), + remote_id, + uid_validity, subject, sender, recipient, @@ -815,10 +818,10 @@ impl ImapEngine { where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { - let uid = parse_imap_uid(&spec.message_id).ok_or_else(|| { + let uid = parse_imap_uid(&spec.remote_id).ok_or_else(|| { MailError::ImapError(format!( - "cannot parse an IMAP UID from message id '{}'", - spec.message_id + "cannot parse an IMAP UID from remote id '{}' (message '{}')", + spec.remote_id, spec.message_id )) })?; @@ -831,7 +834,7 @@ impl ImapEngine { // Refuse to act unless the mailbox's current UIDVALIDITY provably // matches the one the stored UID was captured under. - let stored_validity = checkpoint_uid_validity(spec.folder_checkpoint.as_deref()); + let stored_validity = parse_uid_validity(&spec.uid_validity); match (stored_validity, mailbox.uid_validity) { (Some(stored), Some(current)) if stored == current => {} (stored, current) => { @@ -1575,23 +1578,24 @@ mod tests { } #[test] - fn parse_imap_uid_recovers_uid_and_rejects_malformed_ids() { - assert_eq!(parse_imap_uid("imap-uid-42"), Some(42)); - assert_eq!(parse_imap_uid("imap-uid-1"), Some(1)); - // UID 0 is not a valid IMAP UID; a bare/foreign id yields nothing. - assert_eq!(parse_imap_uid("imap-uid-0"), None); - assert_eq!(parse_imap_uid("42"), None); + fn parse_imap_uid_recovers_uid_and_rejects_non_numeric_ids() { + assert_eq!(parse_imap_uid("42"), Some(42)); + assert_eq!(parse_imap_uid(" 42 "), Some(42)); + assert_eq!(parse_imap_uid("1"), Some(1)); + // UID 0 is not a valid IMAP UID; a non-numeric/foreign id yields nothing. + assert_eq!(parse_imap_uid("0"), None); + assert_eq!(parse_imap_uid("imap-uid-42"), None); assert_eq!(parse_imap_uid("jmap-object-id"), None); } #[test] - fn checkpoint_uid_validity_extracts_only_a_well_formed_validity() { - assert_eq!(checkpoint_uid_validity(Some("7:105")), Some(7)); - assert_eq!(checkpoint_uid_validity(Some(" 7 : 105 ")), Some(7)); - assert_eq!(checkpoint_uid_validity(None), None); - // A legacy bare-number checkpoint carries no validity to compare. - assert_eq!(checkpoint_uid_validity(Some("105")), None); - assert_eq!(checkpoint_uid_validity(Some("x:105")), None); + fn parse_uid_validity_accepts_only_a_decimal_value() { + assert_eq!(parse_uid_validity("7"), Some(7)); + assert_eq!(parse_uid_validity(" 7 "), Some(7)); + assert_eq!(parse_uid_validity(""), None); + // The JMAP sentinel and a full checkpoint string are not a bare validity. + assert_eq!(parse_uid_validity("jmap"), None); + assert_eq!(parse_uid_validity("7:105"), None); } /// Drive a mutation over a scripted IMAP server (in-memory duplex), @@ -1697,10 +1701,18 @@ mod tests { } fn imap_spec(kind: RemoteMutationKind, checkpoint: Option<&str>) -> RemoteMutationSpec { + // The tests express the captured scope as a "{uidvalidity}:{uidnext}" + // checkpoint string (mirroring how a sync stores it); the spec now + // carries just the UIDVALIDITY component the guard compares against. + let uid_validity = checkpoint + .and_then(|c| c.split(':').next()) + .unwrap_or_default() + .to_string(); RemoteMutationSpec { - message_id: "imap-uid-42".to_string(), + message_id: "surrogate-42".to_string(), + remote_id: "42".to_string(), folder_id: "INBOX".to_string(), - folder_checkpoint: checkpoint.map(str::to_string), + uid_validity, kind, } } diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index b7a1e0a..cfbc0fa 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -8,6 +8,14 @@ use serde_json::{json, Value}; use crate::backend::{MailBackend, RemoteMutationKind, RemoteMutationSpec}; use crate::parser::MailError; +/// Stand-in for the UIDVALIDITY scope on JMAP, which has no such concept: a +/// JMAP Email object id is already globally stable within its account (RFC +/// 8621), so a fixed sentinel keeps the surrogate-id computation uniform with +/// IMAP without implying a UIDVALIDITY guard. The IMAP UIDVALIDITY guard treats +/// this non-numeric value as "no validity to compare" and is never engaged for +/// a JMAP mutation, which addresses messages by object id. +const JMAP_UID_VALIDITY_SENTINEL: &str = "jmap"; + /// JMAP Session Object (RFC 8620 Section 2). #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -214,7 +222,9 @@ impl JmapEngine { /// `$flagged` keyword (`true` to set, `null` to clear), the JMAP equivalent /// of IMAP `\Flagged`. pub fn build_email_set_request(account_id: &str, spec: &RemoteMutationSpec) -> Value { - let id = &spec.message_id; + // Address the message by its protocol-native JMAP object id, never the + // opaque surrogate `message_id`. + let id = &spec.remote_id; let (update, destroy) = match &spec.kind { RemoteMutationKind::SetFlagged { value } => { let keyword = if *value { json!(true) } else { Value::Null }; @@ -383,10 +393,23 @@ impl JmapEngine { .map(|a| a.email.clone()) .unwrap_or_else(|| "me@nuncio.mx".to_string()); + let folder_id = "inbox".to_string(); + // The JMAP object id is the protocol-native id; the persisted + // id is an opaque surrogate hashed over it plus the account, + // folder, and (sentinel) scope so it stays uniform with IMAP. + let remote_id = item.id; + let id = Email::surrogate_id( + &self.account_id, + &folder_id, + JMAP_UID_VALIDITY_SENTINEL, + &remote_id, + ); Email { - id: item.id, + id, account_id: self.account_id.clone(), - folder_id: "inbox".to_string(), + folder_id, + remote_id, + uid_validity: JMAP_UID_VALIDITY_SENTINEL.to_string(), subject: item.subject.unwrap_or_else(|| "No Subject".to_string()), sender, recipient, @@ -558,7 +581,7 @@ impl MailBackend for JmapEngine { let request = Self::build_email_set_request(&account_id, spec); let raw = self.post_jmap(&session.api_url, &request).await?; let expect_destroy = matches!(spec.kind, RemoteMutationKind::Delete); - Self::confirm_email_set_applied(&raw, &spec.message_id, expect_destroy) + Self::confirm_email_set_applied(&raw, &spec.remote_id, expect_destroy) } } @@ -689,9 +712,10 @@ mod tests { fn spec(kind: RemoteMutationKind) -> RemoteMutationSpec { RemoteMutationSpec { - message_id: "m-1".to_string(), + message_id: "surrogate-m-1".to_string(), + remote_id: "m-1".to_string(), folder_id: "mb-inbox".to_string(), - folder_checkpoint: None, + uid_validity: JMAP_UID_VALIDITY_SENTINEL.to_string(), kind, } } diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index 4a581d5..487d09a 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -244,6 +244,8 @@ mod tests { id: "msg-mock-1".to_string(), account_id: "acct-1".to_string(), folder_id: "inbox".to_string(), + remote_id: "1".to_string(), + uid_validity: "1".to_string(), subject: "Mock Test".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), diff --git a/crates/nuncio-mail/src/parser.rs b/crates/nuncio-mail/src/parser.rs index 28976b9..95e19f7 100644 --- a/crates/nuncio-mail/src/parser.rs +++ b/crates/nuncio-mail/src/parser.rs @@ -129,6 +129,10 @@ impl MimeParserAdapter { id: id.to_string(), account_id: account_id.to_string(), folder_id: folder_id.to_string(), + // The MIME body carries no protocol addressing; the caller (the + // IMAP sync) fills these from the transport before persisting. + remote_id: String::new(), + uid_validity: String::new(), subject, sender, recipient, diff --git a/crates/nuncio-mail/src/smtp.rs b/crates/nuncio-mail/src/smtp.rs index 35e592d..9cef1f0 100644 --- a/crates/nuncio-mail/src/smtp.rs +++ b/crates/nuncio-mail/src/smtp.rs @@ -329,6 +329,8 @@ mod tests { id: "msg-1".to_string(), account_id: "acct-1".to_string(), folder_id: "outbox".to_string(), + remote_id: "1".to_string(), + uid_validity: "1".to_string(), subject: "Status Update".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), diff --git a/crates/nuncio-mail/tests/wiremock_jmap_test.rs b/crates/nuncio-mail/tests/wiremock_jmap_test.rs index f7773af..f809590 100644 --- a/crates/nuncio-mail/tests/wiremock_jmap_test.rs +++ b/crates/nuncio-mail/tests/wiremock_jmap_test.rs @@ -84,7 +84,11 @@ async fn wiremock_jmap_session_discovery_and_email_get_sync() { .expect("parse email get"); assert_eq!(emails.len(), 1); - assert_eq!(emails[0].id, "msg-wm-100"); + // The JMAP object id is preserved as the protocol-native `remote_id`, while + // the persisted id is an opaque 64-char surrogate hash (never the raw id). + assert_eq!(emails[0].remote_id, "msg-wm-100"); + assert_ne!(emails[0].id, "msg-wm-100"); + assert_eq!(emails[0].id.len(), 64); assert_eq!(emails[0].subject, "WireMock JMAP E2E Test"); assert_eq!(emails[0].sender, "sender@nuncio.mx"); assert_eq!(new_state, "sync-state-500"); @@ -200,7 +204,9 @@ async fn jmap_engine_sync_folders_and_messages_through_mail_backend_trait() { .await .expect("sync_messages succeeds"); assert_eq!(emails.len(), 1); - assert_eq!(emails[0].id, "msg-wm-200"); + assert_eq!(emails[0].remote_id, "msg-wm-200"); + assert_ne!(emails[0].id, "msg-wm-200"); + assert_eq!(emails[0].id.len(), 64); assert_eq!(emails[0].subject, "Trait-Level JMAP Sync"); assert_eq!(emails[0].sender, "sender@nuncio.mx"); assert_eq!(emails[0].received_at, 1700002000); @@ -264,9 +270,10 @@ fn jmap_engine(mock_server: &MockServer) -> JmapEngine { fn spec(kind: RemoteMutationKind) -> RemoteMutationSpec { RemoteMutationSpec { - message_id: "jmap-msg-1".to_string(), + message_id: "surrogate-jmap-msg-1".to_string(), + remote_id: "jmap-msg-1".to_string(), folder_id: "mb-inbox".to_string(), - folder_checkpoint: None, + uid_validity: "jmap".to_string(), kind, } } diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index a40fd63..5dd7ae9 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -471,6 +471,8 @@ impl DatabaseEngine { id TEXT PRIMARY KEY NOT NULL, account_id TEXT NOT NULL, folder_id TEXT NOT NULL, + remote_id TEXT NOT NULL DEFAULT '', + uid_validity TEXT NOT NULL DEFAULT '', subject TEXT NOT NULL, sender TEXT NOT NULL, recipient TEXT NOT NULL, @@ -711,11 +713,84 @@ impl DatabaseEngine { self.ensure_accounts_smtp_columns().await?; self.ensure_accounts_tls_mode_columns().await?; self.ensure_accounts_dav_columns().await?; + self.ensure_messages_identity_columns().await?; self.backfill_message_fts().await?; Ok(()) } + /// Additive, backfill-safe migration that adds the `remote_id` / + /// `uid_validity` message-identity columns to a pre-existing `messages` + /// table that predated the opaque surrogate id, and installs the + /// `UNIQUE(account_id, folder_id, uid_validity, remote_id)` index that is + /// the real correctness guard against a shared protocol id (e.g. an IMAP + /// UID reused across folders/accounts) overwriting an unrelated message. + /// + /// A fresh database already gets both columns from `CREATE TABLE IF NOT + /// EXISTS messages` above, so on a fresh database the `PRAGMA table_info` + /// check finds them present and the backfill matches no rows -- only the + /// index is created. For a pre-existing database file the columns are added + /// with an empty-string default and every legacy row is backfilled to + /// `remote_id = id` (the old primary key, which was globally unique) so the + /// UNIQUE index can be built without collisions. SQLite has no `ADD COLUMN + /// IF NOT EXISTS`, so column presence is checked explicitly first, making + /// this safe to run on every daemon startup. + /// + /// The whole step runs in a single transaction, and the backfill is + /// unconditional (idempotent `WHERE remote_id = ''`) rather than gated on + /// having just added the column. Both properties make a partial run + /// self-healing: a crash mid-migration rolls back atomically, and even a + /// committed intermediate state (column present but a legacy row still + /// carrying the empty default) is completed by the next startup's backfill + /// before the index is built -- so the daemon can never wedge on an index + /// collision it could have avoided. + async fn ensure_messages_identity_columns(&self) -> Result<(), DatabaseError> { + let existing_columns: Vec = sqlx::query("PRAGMA table_info(messages)") + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)? + .iter() + .map(|row| row.get::("name")) + .collect(); + + let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + + if !existing_columns.iter().any(|c| c == "remote_id") { + sqlx::query("ALTER TABLE messages ADD COLUMN remote_id TEXT NOT NULL DEFAULT ''") + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + } + if !existing_columns.iter().any(|c| c == "uid_validity") { + sqlx::query("ALTER TABLE messages ADD COLUMN uid_validity TEXT NOT NULL DEFAULT ''") + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + } + + // Seed any legacy row still carrying the empty-string default with a + // unique, non-empty remote id (the old primary key) so the UNIQUE + // identity index is satisfiable. Unconditional so a partial prior run + // is always completed before the index is (re)built; a no-op on a fresh + // or already-migrated database. + sqlx::query("UPDATE messages SET remote_id = id WHERE remote_id = ''") + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + sqlx::query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_identity \ + ON messages (account_id, folder_id, uid_validity, remote_id)", + ) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + tx.commit().await.map_err(DatabaseError::Query)?; + + Ok(()) + } + /// Additive, backfill-safe migration that adds the `imap_tls_mode` / /// `smtp_tls_mode` columns to a pre-existing `accounts` table that /// predates persisted per-protocol TLS transport modes. @@ -1081,13 +1156,15 @@ impl DatabaseEngine { sqlx::query( r#" INSERT OR REPLACE INTO messages - (id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag, body_plain, body_html) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "#, ) .bind(&email.id) .bind(&email.account_id) .bind(&email.folder_id) + .bind(&email.remote_id) + .bind(&email.uid_validity) .bind(&email.subject) .bind(&email.sender) .bind(&email.recipient) @@ -1143,9 +1220,11 @@ impl DatabaseEngine { i64, Option, Option, + String, + String, )> = sqlx::query_as( r#" - SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html + SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages WHERE folder_id = ? ORDER BY received_at DESC @@ -1171,6 +1250,8 @@ impl DatabaseEngine { read_flag, body_plain, body_html, + remote_id, + uid_validity, )| { let dec_plain = body_plain .map(|p| { @@ -1194,6 +1275,8 @@ impl DatabaseEngine { id, account_id, folder_id, + remote_id, + uid_validity, subject, sender, recipient, @@ -1225,9 +1308,11 @@ impl DatabaseEngine { i64, Option, Option, + String, + String, ) = sqlx::query_as( r#" - SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html + SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages WHERE id = ? "#, @@ -1252,6 +1337,8 @@ impl DatabaseEngine { id: row.0, account_id: row.1, folder_id: row.2, + remote_id: row.10, + uid_validity: row.11, subject: row.3, sender: row.4, recipient: row.5, @@ -1902,7 +1989,7 @@ impl DatabaseEngine { limit: usize, ) -> Result, DatabaseError> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html FROM messages " + "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages " ); if !last_id.is_empty() { builder.push("WHERE id > "); @@ -1922,6 +2009,8 @@ impl DatabaseEngine { i64, Option, Option, + String, + String, )>(); let rows = query @@ -1947,6 +2036,8 @@ impl DatabaseEngine { id: r.0, account_id: r.1, folder_id: r.2, + remote_id: r.10, + uid_validity: r.11, subject: r.3, sender: r.4, recipient: r.5, @@ -1972,7 +2063,7 @@ impl DatabaseEngine { folder_id: Option<&str>, ) -> Result, DatabaseError> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html FROM messages" + "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages" ); let mut has_filter = false; @@ -2002,6 +2093,8 @@ impl DatabaseEngine { i64, Option, Option, + String, + String, )>(); let rows = query @@ -2027,6 +2120,8 @@ impl DatabaseEngine { id: r.0, account_id: r.1, folder_id: r.2, + remote_id: r.10, + uid_validity: r.11, subject: r.3, sender: r.4, recipient: r.5, @@ -2483,6 +2578,8 @@ mod tests { id: "msg-db-100".to_string(), account_id: "acct-1".to_string(), folder_id: "INBOX".to_string(), + remote_id: "100".to_string(), + uid_validity: "1".to_string(), subject: "Database Sync Test".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), @@ -2530,6 +2627,8 @@ mod tests { id: "msg-db-corrupt".to_string(), account_id: "acct-1".to_string(), folder_id: "INBOX".to_string(), + remote_id: "corrupt".to_string(), + uid_validity: "1".to_string(), subject: "Corrupted At Rest".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), @@ -2832,6 +2931,8 @@ mod tests { id: "msg-mark-1".to_string(), account_id: "acct-1".to_string(), folder_id: "INBOX".to_string(), + remote_id: "mark-1".to_string(), + uid_validity: "1".to_string(), subject: "Mark Read Test".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), @@ -3362,6 +3463,8 @@ mod tests { id: format!("msg-{i:03}"), account_id: "acct-1".to_string(), folder_id: "inbox".to_string(), + remote_id: format!("{i}"), + uid_validity: "1".to_string(), subject: format!("Subject {i}"), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), @@ -3469,11 +3572,153 @@ mod tests { ); } + /// Build a message exactly as a real sync would: an opaque surrogate id + /// hashed over its addressing coordinates, with the protocol-native id and + /// UIDVALIDITY scope carried in their own columns. + fn synced_email( + account_id: &str, + folder_id: &str, + uid_validity: &str, + remote_id: &str, + ) -> nuncio_core::model::Email { + nuncio_core::model::Email { + id: nuncio_core::model::Email::surrogate_id( + account_id, + folder_id, + uid_validity, + remote_id, + ), + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + remote_id: remote_id.to_string(), + uid_validity: uid_validity.to_string(), + subject: format!("{folder_id}/{remote_id}"), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + read: false, + body_plain: Some("body".to_string()), + body_html: None, + attachments: Vec::new(), + } + } + + /// The C1 regression: the same protocol UID reused across folders or + /// accounts must persist as DISTINCT rows (never silently overwrite), and a + /// re-sync of the SAME message must upsert the one row it already owns. + #[tokio::test] + async fn message_identity_prevents_cross_folder_and_cross_account_collision() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Same UID (5) in two folders of one account, and again in a second + // account -- the exact shapes that used to collapse to `imap-uid-5`. + let inbox = synced_email("acct-1", "INBOX", "42", "5"); + let sent = synced_email("acct-1", "Sent", "7", "5"); + let other_account = synced_email("acct-2", "INBOX", "99", "5"); + + // Every surrogate id is distinct, so none can overwrite another. + assert_ne!(inbox.id, sent.id); + assert_ne!(inbox.id, other_account.id); + assert_ne!(sent.id, other_account.id); + + engine.save_email(&inbox).await.expect("save inbox"); + engine.save_email(&sent).await.expect("save sent"); + engine + .save_email(&other_account) + .await + .expect("save other account"); + + // All three coexist as separate rows, addressed by their own folder. + assert_eq!(engine.list_messages("INBOX", 10).await.unwrap().len(), 2); + assert_eq!(engine.list_messages("Sent", 10).await.unwrap().len(), 1); + + // Re-syncing the SAME message recomputes the SAME id, so the upsert + // updates the one row rather than duplicating it. + let inbox_resynced = synced_email("acct-1", "INBOX", "42", "5"); + assert_eq!(inbox_resynced.id, inbox.id); + engine + .save_email(&inbox_resynced) + .await + .expect("re-sync upserts"); + assert_eq!( + engine.list_messages("INBOX", 10).await.unwrap().len(), + 2, + "a re-sync must upsert, not duplicate" + ); + + // The recovered row round-trips its protocol addressing columns. + let fetched = engine.get_message(&inbox.id).await.expect("get message"); + assert_eq!(fetched.remote_id, "5"); + assert_eq!(fetched.uid_validity, "42"); + assert_eq!(fetched.folder_id, "INBOX"); + } + + /// A crash between `ADD COLUMN remote_id` and its backfill can commit an + /// intermediate state -- the column present but a legacy row still carrying + /// the empty-string default -- with the UNIQUE index not yet built. The + /// migration must complete that backfill on the next open and build the + /// index without a collision, rather than wedging the daemon. + #[tokio::test] + async fn messages_identity_migration_completes_from_a_partial_run() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Reproduce the partial state: drop the identity index, then insert two + // rows in the same folder that both still hold the empty default. With + // the index gone this is permitted, and the two would collide on + // (account, folder, uid_validity='', remote_id='') if it were rebuilt now. + sqlx::query("DROP INDEX IF EXISTS idx_messages_identity") + .execute(engine.pool()) + .await + .unwrap(); + for id in ["legacy-a", "legacy-b"] { + sqlx::query( + "INSERT INTO messages \ + (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag) \ + VALUES (?, 'acct-1', 'INBOX', '', '', 'subj', 'a@nuncio.mx', 'b@nuncio.mx', 0, 0)", + ) + .bind(id) + .execute(engine.pool()) + .await + .unwrap(); + } + + // Re-running the migration must finish the backfill and build the index. + engine + .ensure_messages_identity_columns() + .await + .expect("a partial migration must complete on the next open"); + + // Both legacy rows survive with a unique, non-empty backfilled remote id. + assert_eq!( + engine.get_message("legacy-a").await.unwrap().remote_id, + "legacy-a" + ); + assert_eq!( + engine.get_message("legacy-b").await.unwrap().remote_id, + "legacy-b" + ); + + // The unique identity index is now present, and re-running is idempotent. + let index: Option<(String,)> = sqlx::query_as( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_messages_identity'", + ) + .fetch_optional(engine.pool()) + .await + .unwrap(); + assert!(index.is_some(), "the unique identity index must be built"); + engine + .ensure_messages_identity_columns() + .await + .expect("re-running an already-complete migration is a no-op"); + } + fn export_test_email(id: &str, account_id: &str, folder_id: &str) -> nuncio_core::model::Email { nuncio_core::model::Email { id: id.to_string(), account_id: account_id.to_string(), folder_id: folder_id.to_string(), + remote_id: id.to_string(), + uid_validity: "1".to_string(), subject: format!("Subject {id}"), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), diff --git a/crates/nuncio-store/src/search.rs b/crates/nuncio-store/src/search.rs index ed6d1ad..0befdf5 100644 --- a/crates/nuncio-store/src/search.rs +++ b/crates/nuncio-store/src/search.rs @@ -105,6 +105,8 @@ mod tests { id: id.to_string(), account_id: "acct-1".to_string(), folder_id: "inbox".to_string(), + remote_id: id.to_string(), + uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 5aa2251..43c73ef 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -1592,6 +1592,9 @@ fn synthetic_preview_email(id: &str) -> nuncio_core::model::Email { id: id.to_string(), account_id: "acct-1".to_string(), folder_id: "inbox".to_string(), + // Synthetic preview only -- never persisted or mutated remotely. + remote_id: id.to_string(), + uid_validity: String::new(), subject: "Test Subject".to_string(), sender: "test@nuncio.mx".to_string(), recipient: "me@nuncio.mx".to_string(), @@ -3540,6 +3543,8 @@ mod tests { id: id.to_string(), account_id: "acct-mail-1".to_string(), folder_id: folder_id.to_string(), + remote_id: id.to_string(), + uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), @@ -4426,6 +4431,8 @@ mod tests { id: "msg-filters-1".to_string(), account_id: "acct-1".to_string(), folder_id: "inbox".to_string(), + remote_id: "1".to_string(), + uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), @@ -4526,6 +4533,8 @@ mod tests { id: "msg-1".to_string(), account_id: "acct-1".to_string(), folder_id: "inbox".to_string(), + remote_id: "1".to_string(), + uid_validity: "1".to_string(), subject: "anything".to_string(), sender: "a@b.com".to_string(), recipient: "c@d.com".to_string(), @@ -4668,6 +4677,8 @@ mod tests { id: "msg-preview-1".to_string(), account_id: "acct-1".to_string(), folder_id: "inbox".to_string(), + remote_id: "preview-1".to_string(), + uid_validity: "1".to_string(), subject: "Urgent Board Meeting".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), @@ -4834,6 +4845,8 @@ mod tests { id: "msg-1".to_string(), account_id: "acct-1".to_string(), folder_id: "inbox".to_string(), + remote_id: "1".to_string(), + uid_validity: "1".to_string(), subject: "Urgent Meeting".to_string(), sender: "a@b.com".to_string(), recipient: "c@d.com".to_string(), @@ -4988,6 +5001,8 @@ mod tests { id: "msg-1".to_string(), account_id: "acct-1".to_string(), folder_id: "inbox".to_string(), + remote_id: "1".to_string(), + uid_validity: "1".to_string(), subject: "Urgent Meeting".to_string(), sender: "a@b.com".to_string(), recipient: "c@d.com".to_string(), diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index f022ea5..062c38a 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -204,7 +204,7 @@ async fn execute_one( match mutation.mutation_type.as_str() { "MOVE" | "COPY" | "FLAG" | "UNFLAG" | "DELETE" => { - dispatch_mailbox_mutation(db, env, mutation, &email, &payload).await + dispatch_mailbox_mutation(env, mutation, &email, &payload).await } "FORWARD" => dispatch_forward(db, env, &email, &payload).await, "WEBHOOK" => dispatch_webhook(env, mutation, &email, &payload).await, @@ -213,10 +213,10 @@ async fn execute_one( } /// Execute a mailbox-affecting mutation (move/copy/flag/unflag/delete) against -/// the account's mail backend, threading the source folder's stored checkpoint -/// so the backend can enforce its UIDVALIDITY guard. +/// the account's mail backend, recovering the remote addressing (protocol id + +/// UIDVALIDITY scope) from the stored message row so the backend can enforce +/// its UIDVALIDITY guard. async fn dispatch_mailbox_mutation( - db: &DatabaseEngine, env: &dyn RemoteExecutionEnv, mutation: &PendingRemoteMutation, email: &nuncio_core::model::Email, @@ -245,23 +245,20 @@ async fn dispatch_mailbox_mutation( other => return Disposition::Permanent(format!("unexpected mailbox mutation '{other}'")), }; - let folder_checkpoint = match db - .get_folder_sync_state(&email.account_id, &email.folder_id) - .await - { - Ok(checkpoint) => checkpoint, - Err(e) => return Disposition::Retry(format!("failed to read folder checkpoint: {e}")), - }; - let backend = match env.mail_backend(&email.account_id).await { Ok(backend) => backend, Err(e) => return Disposition::Retry(format!("failed to build mail backend: {e}")), }; + // Recover the remote addressing from the stored message row -- the + // protocol-native id and the UIDVALIDITY scope it was captured under -- + // rather than parsing it out of the opaque surrogate id. The backend still + // SELECT-verifies the UIDVALIDITY before mutating. let spec = RemoteMutationSpec { message_id: email.id.clone(), + remote_id: email.remote_id.clone(), folder_id: email.folder_id.clone(), - folder_checkpoint, + uid_validity: email.uid_validity.clone(), kind, }; diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index bbb395c..bbab07d 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -628,6 +628,8 @@ mod tests { id: id.to_string(), account_id: "acct-mock-1".to_string(), folder_id: folder_id.to_string(), + remote_id: id.to_string(), + uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), @@ -864,11 +866,16 @@ mod tests { // The wiremock stubs advertise 1 folder containing 1 message. assert_eq!(synced, 1); + // The message persists under an opaque surrogate id; it is recovered by + // folder, and its protocol-native JMAP object id round-trips in + // `remote_id`. let persisted = db - .get_message("jmap-msg-1") + .list_messages("inbox", 10) .await .expect("jmap message persisted"); - assert_eq!(persisted.subject, "Welcome to JMAP Sync"); + assert_eq!(persisted.len(), 1); + assert_eq!(persisted[0].subject, "Welcome to JMAP Sync"); + assert_eq!(persisted[0].remote_id, "jmap-msg-1"); assert_eq!( events.recv().await.expect("start event"), diff --git a/crates/nunciod/tests/filters_triage_e2e_test.rs b/crates/nunciod/tests/filters_triage_e2e_test.rs index bb34869..60689d3 100644 --- a/crates/nunciod/tests/filters_triage_e2e_test.rs +++ b/crates/nunciod/tests/filters_triage_e2e_test.rs @@ -43,6 +43,8 @@ fn sample_email(id: &str, subject: &str) -> Email { id: id.to_string(), account_id: ACCOUNT_ID.to_string(), folder_id: "inbox".to_string(), + remote_id: id.to_string(), + uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "triage-e2e@nuncio.mx".to_string(), diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index 6f823ff..d88509a 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -103,9 +103,11 @@ fn sample_account() -> nuncio_core::AccountConfig { fn sample_email(subject: &str) -> Email { Email { - id: "imap-uid-42".to_string(), + id: "surrogate-inbox-42".to_string(), account_id: ACCOUNT_ID.to_string(), folder_id: FOLDER_ID.to_string(), + remote_id: "42".to_string(), + uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "owner@nuncio.mx".to_string(), @@ -161,11 +163,13 @@ async fn move_rule_executes_against_backend_and_completes_with_uidvalidity_check let applied = env.backend.applied_mutations(); assert_eq!(applied.len(), 1, "the backend op must genuinely run"); - assert_eq!(applied[0].message_id, "imap-uid-42"); + // Addressing is recovered from the stored row's columns, not by parsing the + // opaque surrogate id: the protocol-native UID and the UIDVALIDITY scope it + // was captured under, so the backend can enforce its guard. + assert_eq!(applied[0].message_id, "surrogate-inbox-42"); + assert_eq!(applied[0].remote_id, "42"); assert_eq!(applied[0].folder_id, FOLDER_ID); - // The source folder's stored checkpoint (carrying UIDVALIDITY) is threaded - // through so the backend can enforce its guard. - assert_eq!(applied[0].folder_checkpoint.as_deref(), Some(CHECKPOINT)); + assert_eq!(applied[0].uid_validity, "1"); assert_eq!( applied[0].kind, RemoteMutationKind::Move { diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index 058225e..187ef0d 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -77,6 +77,8 @@ fn mock_inbound_email(id: &str, subject: &str, body: &str) -> Email { id: id.to_string(), account_id: ACCOUNT_ID.to_string(), folder_id: "inbox".to_string(), + remote_id: id.to_string(), + uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: ACCOUNT_EMAIL.to_string(), -- 2.54.0 From dfc7cc9358fc1c4e886b99b81e87c765fd3e7699 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 14:11:05 -0500 Subject: [PATCH 21/95] perf(sync): batch new-message detection to remove the is_new_message N+1 (#341) Replace the per-message get_message existence check in fetch_and_persist with a single existing_message_ids(chunk_ids) lookup per fetched folder chunk, plus a per-pass seen-set to keep duplicate ids within one chunk firing at most once. Fire-once semantics for filter actions are unchanged: a message is new iff it was absent from the store before this sync pass and this is its first occurrence in the current chunk. Refs #325 Co-authored-by: James Maes --- crates/nuncio-store/src/db.rs | 67 +++++++++++++++++ crates/nunciod/src/sync.rs | 135 ++++++++++++++++++++++++++-------- 2 files changed, 171 insertions(+), 31 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 5dd7ae9..7cba21a 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -1350,6 +1350,37 @@ impl DatabaseEngine { }) } + /// Return the subset of `ids` already present in the `messages` table, as a single + /// `SELECT id FROM messages WHERE id IN (...)` query. + /// + /// Lets a caller classify a whole fetched batch as new-vs-seen with one round trip instead + /// of one existence lookup per id. Passing an empty slice short-circuits to an empty set + /// without querying at all, since `IN ()` is not valid SQL. + pub async fn existing_message_ids( + &self, + ids: &[String], + ) -> Result, DatabaseError> { + if ids.is_empty() { + return Ok(std::collections::HashSet::new()); + } + + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = + sqlx::QueryBuilder::new("SELECT id FROM messages WHERE id IN ("); + let mut separated = builder.separated(", "); + for id in ids { + separated.push_bind(id); + } + builder.push(")"); + + let rows: Vec<(String,)> = builder + .build_query_as() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows.into_iter().map(|(id,)| id).collect()) + } + /// Save a [`nuncio_core::model::CalendarEvent`] to SQLite (INSERT OR REPLACE). /// /// Unlike [`Self::save_email`], calendar event summary/location are never encrypted at @@ -3730,6 +3761,42 @@ mod tests { } } + #[tokio::test] + async fn existing_message_ids_returns_only_the_ids_already_persisted() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + engine + .save_email(&export_test_email("msg-1", "acct-a", "inbox")) + .await + .unwrap(); + engine + .save_email(&export_test_email("msg-2", "acct-a", "inbox")) + .await + .unwrap(); + + let found = engine + .existing_message_ids(&[ + "msg-1".to_string(), + "msg-2".to_string(), + "msg-3-never-saved".to_string(), + ]) + .await + .unwrap(); + + assert_eq!(found.len(), 2); + assert!(found.contains("msg-1")); + assert!(found.contains("msg-2")); + assert!(!found.contains("msg-3-never-saved")); + } + + #[tokio::test] + async fn existing_message_ids_with_empty_input_returns_empty_set_without_querying() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let found = engine.existing_message_ids(&[]).await.unwrap(); + + assert!(found.is_empty()); + } + #[tokio::test] async fn list_messages_for_export_with_no_filters_returns_every_message() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index bbab07d..98dc78a 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -195,39 +195,26 @@ pub async fn apply_filter_actions( applied } -/// Reports whether `message_id` is not yet present in the store. -/// -/// Sync is not yet incremental (a re-sync re-fetches every message the -/// backend still reports, not just genuinely new ones), so this is the guard -/// that keeps filter actions firing exactly once per message: it must be -/// checked BEFORE `save_email` persists (or re-persists) the message, since -/// `save_email` is `INSERT OR REPLACE` and would otherwise erase the -/// distinction between "arriving for the first time" and "seen again". -/// -/// A lookup failure that is not "not found" (a genuine DB error) is logged -/// and treated as "not new", the conservative choice: firing filters again on -/// a message we can't positively identify as new risks duplicate remote -/// mutations, which is worse than occasionally missing a fire on a message -/// that was in fact new. -async fn is_new_message(db: &nuncio_store::db::DatabaseEngine, message_id: &str) -> bool { - match db.get_message(message_id).await { - Ok(_) => false, - Err(e) if e.is_not_found() => true, - Err(e) => { - tracing::warn!( - "could not determine whether message '{message_id}' is new before sync \ - (treating as not-new to avoid duplicate filter firing): {e}" - ); - false - } - } -} - /// Fetch every folder and every message in every folder from `backend`, /// persisting each message via [`DatabaseEngine::save_email`] and, for /// messages genuinely new to the store, evaluating it against -/// `filter_engine` via [`apply_filter_actions`] (see [`is_new_message`] for -/// why this must be gated rather than run on every persisted message). +/// `filter_engine` via [`apply_filter_actions`]. +/// +/// Sync is not yet incremental (a re-sync re-fetches every message the +/// backend still reports, not just genuinely new ones), so new-vs-seen must +/// be determined BEFORE `save_email` persists (or re-persists) anything -- +/// `save_email` is `INSERT OR REPLACE` and would otherwise erase the +/// distinction between "arriving for the first time" and "seen again". Doing +/// that with one existence lookup per message (`DatabaseEngine::get_message`) +/// is an N+1 query pattern over a fetched chunk; instead, this classifies an +/// entire folder's chunk with a single batched +/// `DatabaseEngine::existing_message_ids` lookup up front, then folds in a +/// per-batch "already seen this pass" set while iterating so a chunk that +/// itself contains a repeated id (e.g. a backend surfacing the same message +/// twice in one fetch) still fires at most once -- exactly as if each +/// message's existence had been checked one at a time immediately before its +/// own `save_email`. +/// /// Returns the total number of messages processed (a message id "processed" /// more than once, e.g. by a backend that returns the same id from multiple /// folders, is counted once per occurrence -- `save_email` itself is @@ -260,8 +247,19 @@ async fn fetch_and_persist( .sync_messages(&folder.id, last_state.as_deref()) .await?; let fetched = emails.len(); + + // One round trip classifies the whole chunk instead of one lookup per + // message. A lookup failure that is not "not found" would be a + // genuine DB error; `existing_message_ids` surfaces those as `Err` + // rather than silently treating the batch as "not new", so callers + // never persist and skip-filter on an unverified assumption. + let chunk_ids: Vec = emails.iter().map(|e| e.id.clone()).collect(); + let already_present = db.existing_message_ids(&chunk_ids).await?; + let mut seen_this_pass = std::collections::HashSet::new(); + for email in emails { - let is_new = is_new_message(db, &email.id).await; + let is_new = + !already_present.contains(&email.id) && seen_this_pass.insert(email.id.clone()); db.save_email(&email).await?; if is_new { apply_filter_actions(db, filter_engine, &email).await; @@ -1304,6 +1302,81 @@ mod tests { ); } + #[tokio::test] + async fn sync_with_backend_fires_filters_only_for_new_messages_in_a_mixed_batch() { + // A single fetched chunk containing a mix of an already-seen message + // (persisted by a prior sync) and genuinely new ones must fire filter + // actions ONLY for the new ones, via the batched + // `existing_message_ids` classification -- not a per-message lookup. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let event_bus = EventBus::new(); + + let mark_read_rule = nuncio_filter::NsqlParser::parse_rule( + "Urgent Auto-Read", + 1, + "WHERE subject CONTAINS 'Urgent' ACTION MARK READ", + ) + .expect("parse rule"); + let filter_engine = FilterEngine::new(vec![mark_read_rule]).expect("compile rule"); + + // Pre-seed one message directly, simulating it having landed in a + // prior sync pass. It must NOT re-fire even though this pass's + // backend still reports it (sync is not yet incremental). + db.save_email(&mock_email( + "m-already-seen", + "inbox", + "Urgent: known issue", + )) + .await + .expect("pre-seed existing message"); + + let mock = MockMailBackend::new(); + mock.add_folder(Folder { + id: "inbox".to_string(), + name: "Inbox".to_string(), + total_messages: 3, + unread_messages: 3, + }); + // Mixed chunk: one already-present id, two genuinely new ids. + mock.add_message(mock_email("m-already-seen", "inbox", "Urgent: known issue")); + mock.add_message(mock_email("m-new-1", "inbox", "Urgent: brand new")); + mock.add_message(mock_email("m-new-2", "inbox", "Urgent: also new")); + + let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) + .await + .expect("sync succeeds"); + assert_eq!(synced, 3, "every fetched message is still persisted"); + + let logs = db + .list_filter_execution_logs(10) + .await + .expect("list execution logs"); + assert_eq!( + logs.len(), + 2, + "filters fire only for the two genuinely new messages in the mixed batch" + ); + let fired_ids: std::collections::HashSet = + logs.iter().map(|l| l.message_id.clone()).collect(); + assert!(fired_ids.contains("m-new-1")); + assert!(fired_ids.contains("m-new-2")); + assert!(!fired_ids.contains("m-already-seen")); + + // The pre-seeded message must still be persisted (INSERT OR REPLACE + // semantics are unaffected by the batched new/seen classification) + // but its read flag must be untouched by this pass's MARK READ rule. + let already_seen = db + .get_message("m-already-seen") + .await + .expect("still persisted"); + assert!( + !already_seen.read, + "a message that already existed before this sync must not re-fire MARK READ" + ); + } + #[tokio::test] async fn sync_with_backend_persists_and_threads_the_folder_checkpoint_across_syncs() { // The regression this story exists to kill: the returned checkpoint -- 2.54.0 From 01e7acd97d36b5120d764d87c41ec57aae9b05ec Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 14:12:44 -0500 Subject: [PATCH 22/95] feat(proto): typed errors (ErrorInfo/ErrorReason) via Status details (#342) * feat(proto): add typed error taxonomy and Status helper Introduce a frozen, machine-readable error taxonomy for the nuncio.v1 contract: an ErrorReason enum and an ErrorInfo message carrying a reason, optional metadata, and a human message. A nuncio-proto errors helper builds a tonic::Status that encodes ErrorInfo into the grpc-status-details-bin trailer with the canonical gRPC code per reason, and decodes the reason back on the client side. Regenerate the committed descriptor golden. Refs #273 * feat(nunciod): return typed ErrorInfo details from error sites Convert a representative set of daemon gRPC error returns to carry a typed ErrorInfo in the status-details trailer via the nuncio-proto errors helper: account/message/event/contact/rule not-found (with the offending id as metadata), NSQL validation failures, and the calendar not-configured precondition. Canonical status codes are unchanged, so existing error-path assertions still hold. Add an offline over-the-wire E2E test that decodes the reason and metadata from a returned Status against the real daemon server. Refs #273 --------- Co-authored-by: James Maes --- Cargo.lock | 1 + crates/nuncio-proto/Cargo.toml | 1 + .../proto/nuncio/v1/descriptor.bin | Bin 12854 -> 13517 bytes .../nuncio-proto/proto/nuncio/v1/nuncio.proto | 58 ++++++ crates/nuncio-proto/src/errors.rs | 153 ++++++++++++++++ crates/nuncio-proto/src/lib.rs | 1 + crates/nunciod/src/grpc.rs | 119 +++++++++--- crates/nunciod/tests/typed_errors_e2e_test.rs | 171 ++++++++++++++++++ 8 files changed, 475 insertions(+), 29 deletions(-) create mode 100644 crates/nuncio-proto/src/errors.rs create mode 100644 crates/nunciod/tests/typed_errors_e2e_test.rs diff --git a/Cargo.lock b/Cargo.lock index df41e19..f94cbed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2594,6 +2594,7 @@ dependencies = [ name = "nuncio-proto" version = "0.1.0" dependencies = [ + "bytes", "prost", "protoc-bin-vendored", "thiserror 1.0.69", diff --git a/crates/nuncio-proto/Cargo.toml b/crates/nuncio-proto/Cargo.toml index ccb1ec7..71359e1 100644 --- a/crates/nuncio-proto/Cargo.toml +++ b/crates/nuncio-proto/Cargo.toml @@ -13,6 +13,7 @@ workspace = true tonic.workspace = true tonic-prost.workspace = true prost.workspace = true +bytes.workspace = true thiserror.workspace = true [build-dependencies] diff --git a/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin b/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin index 1f17b041e64990ea6ac8d64c00bbd63f1cdc35ec..850edc681e2a0d676fc2fee02bcd32376c5c1d8b 100644 GIT binary patch delta 687 zcmZvY%}T>i5QR-+|E5*KMI~yfrBo3Y0lN|f;hNmqi!rG;NzqjZHcHW^kp4hdeFh&z z1h?)5H@<|g;Z3dDSa&nbcg~!d=hjk+ujTbUIZ+aKbJg-+MdYiTPazWY!x8(@AlZR* z$M5yLTVEFW#w`18H;!N_ra}^kcm8M+EX$%2tw3_<-4A@eI+A`hhlvB>apz-u{31eb zCNnC4Oc*nn%Luki!*vX+K~!3!M3WX!8H%xK6YEsNHsxRqQ=K!y9D`NWbPU@yOxvyT zN|O?33pT%e#?v*-(a8)U&bgcu%fhF#%pxj5A(n=%}#!~7x%rpzu zd7)jo metadata = 2; + // Human-readable message, identical to the enclosing `Status` message. + string message = 3; +} + // System service exposes daemon-level introspection RPCs. // // This is the foundation of the versioned gRPC API (see diff --git a/crates/nuncio-proto/src/errors.rs b/crates/nuncio-proto/src/errors.rs new file mode 100644 index 0000000..c13f1c6 --- /dev/null +++ b/crates/nuncio-proto/src/errors.rs @@ -0,0 +1,153 @@ +//! Typed, machine-readable gRPC errors for the `nuncio.v1` contract. +//! +//! A failed RPC should carry a structured [`v1::ErrorInfo`] -- a stable +//! [`v1::ErrorReason`] discriminant, optional metadata, and a human message +//! -- in the gRPC `grpc-status-details-bin` status-details trailer, alongside +//! the canonical gRPC [`Code`] that the reason maps to. Servers build such a +//! `Status` with [`status`] / [`status_with_metadata`]; clients read the +//! reason back with [`error_reason`] / [`error_info`] and branch on it, +//! rather than string-matching the human-readable (English) message text. +//! +//! The reason-to-code mapping is centralized in [`ErrorReason::code`] so the +//! transport-level status code and the application-level reason can never +//! drift apart at a call site. + +use crate::v1::{ErrorInfo, ErrorReason}; +use bytes::Bytes; +use prost::Message; +use tonic::{Code, Status}; + +impl ErrorReason { + /// The canonical gRPC [`Code`] this reason maps to. Both the unspecified + /// zero value and an explicit internal error map to [`Code::Internal`], + /// so an older client that receives a reason it does not recognize (and + /// decodes it as `Unspecified`) still sees an internal-class failure. + #[must_use] + pub fn code(self) -> Code { + match self { + ErrorReason::Unspecified | ErrorReason::Internal => Code::Internal, + ErrorReason::ValidationFailed => Code::InvalidArgument, + ErrorReason::AccountNotFound + | ErrorReason::RuleNotFound + | ErrorReason::MessageNotFound + | ErrorReason::EventNotFound + | ErrorReason::ContactNotFound => Code::NotFound, + ErrorReason::AlreadyExists => Code::AlreadyExists, + ErrorReason::NotConfigured => Code::FailedPrecondition, + ErrorReason::BackendUnreachable => Code::Unavailable, + ErrorReason::AuthRequired => Code::Unauthenticated, + ErrorReason::Unsupported => Code::Unimplemented, + } + } +} + +/// Builds a [`Status`] for `reason` carrying `message` both as the `Status` +/// message and inside a prost-encoded [`v1::ErrorInfo`] in the +/// `grpc-status-details-bin` trailer, with the canonical code from +/// [`ErrorReason::code`]. +pub fn status(reason: ErrorReason, message: impl Into) -> Status { + build(reason, message.into(), Vec::new()) +} + +/// Like [`status`], but also attaches structured `metadata` (e.g. the id of +/// the resource that was not found) to the encoded [`v1::ErrorInfo`]. +pub fn status_with_metadata( + reason: ErrorReason, + message: impl Into, + metadata: impl IntoIterator, +) -> Status { + build(reason, message.into(), metadata.into_iter().collect()) +} + +fn build(reason: ErrorReason, message: String, metadata: Vec<(String, String)>) -> Status { + let info = ErrorInfo { + reason: reason as i32, + metadata: metadata.into_iter().collect(), + message: message.clone(), + }; + Status::with_details(reason.code(), message, Bytes::from(info.encode_to_vec())) +} + +/// Decodes the [`v1::ErrorInfo`] carried in `status`'s details trailer, if +/// one is present and well-formed. Returns `None` when the status carries no +/// details (e.g. an error not built through this module) or the details are +/// not a valid `ErrorInfo`. +#[must_use] +pub fn error_info(status: &Status) -> Option { + let details = status.details(); + if details.is_empty() { + return None; + } + ErrorInfo::decode(details).ok() +} + +/// Decodes just the [`v1::ErrorReason`] from `status`'s details trailer, +/// returning `None` when no typed [`v1::ErrorInfo`] is present. +#[must_use] +pub fn error_reason(status: &Status) -> Option { + error_info(status).map(|info| info.reason()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_reason_maps_to_its_canonical_code() { + assert_eq!(ErrorReason::Unspecified.code(), Code::Internal); + assert_eq!(ErrorReason::ValidationFailed.code(), Code::InvalidArgument); + assert_eq!(ErrorReason::AccountNotFound.code(), Code::NotFound); + assert_eq!(ErrorReason::RuleNotFound.code(), Code::NotFound); + assert_eq!(ErrorReason::MessageNotFound.code(), Code::NotFound); + assert_eq!(ErrorReason::EventNotFound.code(), Code::NotFound); + assert_eq!(ErrorReason::ContactNotFound.code(), Code::NotFound); + assert_eq!(ErrorReason::AlreadyExists.code(), Code::AlreadyExists); + assert_eq!(ErrorReason::NotConfigured.code(), Code::FailedPrecondition); + assert_eq!(ErrorReason::BackendUnreachable.code(), Code::Unavailable); + assert_eq!(ErrorReason::AuthRequired.code(), Code::Unauthenticated); + assert_eq!(ErrorReason::Unsupported.code(), Code::Unimplemented); + assert_eq!(ErrorReason::Internal.code(), Code::Internal); + } + + #[test] + fn status_carries_reason_code_and_decodable_details() { + let status = status(ErrorReason::MessageNotFound, "message 'm-1' not found"); + assert_eq!(status.code(), Code::NotFound); + assert_eq!(status.message(), "message 'm-1' not found"); + + let info = error_info(&status).expect("typed details present"); + assert_eq!(info.reason(), ErrorReason::MessageNotFound); + assert_eq!(info.message, "message 'm-1' not found"); + assert!(info.metadata.is_empty()); + + assert_eq!( + error_reason(&status), + Some(ErrorReason::MessageNotFound), + "the reason round-trips through the encoded details" + ); + } + + #[test] + fn status_with_metadata_round_trips_context() { + let status = status_with_metadata( + ErrorReason::AccountNotFound, + "account 'acct-1' not found", + [("account_id".to_string(), "acct-1".to_string())], + ); + assert_eq!(status.code(), Code::NotFound); + + let info = error_info(&status).expect("typed details present"); + assert_eq!(info.reason(), ErrorReason::AccountNotFound); + assert_eq!( + info.metadata.get("account_id").map(String::as_str), + Some("acct-1") + ); + } + + #[test] + fn plain_status_without_details_decodes_to_none() { + let status = Status::not_found("no typed details here"); + assert_eq!(error_info(&status), None); + assert_eq!(error_reason(&status), None); + } +} diff --git a/crates/nuncio-proto/src/lib.rs b/crates/nuncio-proto/src/lib.rs index 6fe756c..6709375 100644 --- a/crates/nuncio-proto/src/lib.rs +++ b/crates/nuncio-proto/src/lib.rs @@ -22,6 +22,7 @@ pub mod addr; pub mod client; +pub mod errors; pub use addr::{grpc_addr_from_env, DEFAULT_GRPC_ADDR, GRPC_ADDR_ENV_VAR}; diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 43c73ef..cb25ab5 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -10,6 +10,7 @@ use nuncio_contacts::ContactsBackend; use nuncio_core::{CoreCommand, CoreEvent, EventBus}; use nuncio_filter::{FilterEngine, NsqlParser, NsqlValidator, ValidationOptions}; use nuncio_mail::{ImapEngine, MailBackend, MessageSender, SmtpTransportEngine}; +use nuncio_proto::errors; use nuncio_proto::v1::accounts_server::{Accounts, AccountsServer}; use nuncio_proto::v1::audit_server::{Audit, AuditServer}; use nuncio_proto::v1::calendar_server::{Calendar, CalendarServer}; @@ -19,6 +20,7 @@ use nuncio_proto::v1::export_server::{Export, ExportServer}; use nuncio_proto::v1::filters_server::{Filters, FiltersServer}; use nuncio_proto::v1::mail_server::{Mail, MailServer}; use nuncio_proto::v1::system_server::{System, SystemServer}; +use nuncio_proto::v1::ErrorReason; use nuncio_proto::v1::{ export_request, AccountConfig as AccountConfigProto, AccountProtocol as AccountProtocolProto, AddAccountRequest, AddAccountResponse, Attachment as AttachmentProto, @@ -549,7 +551,7 @@ impl Accounts for AccountsGrpcService { let config = map_account_config_from_proto(proto_config)?; config .validate() - .map_err(|e| Status::invalid_argument(e.to_string()))?; + .map_err(|e| errors::status(ErrorReason::ValidationFailed, e.to_string()))?; // Store the password credential in the OS keyring FIRST, keyed by // this account's `keyring_secret_key` -- it is never written to @@ -615,7 +617,7 @@ impl Accounts for AccountsGrpcService { let config = map_account_config_from_proto(proto_config)?; config .validate() - .map_err(|e| Status::invalid_argument(e.to_string()))?; + .map_err(|e| errors::status(ErrorReason::ValidationFailed, e.to_string()))?; // Updating a non-existent account is a client error, not a silent // create: reject it so a typo'd id never conjures a new account row. @@ -672,7 +674,13 @@ impl Accounts for AccountsGrpcService { .get_account(&req.id) .await .map_err(|e| Status::internal(format!("failed to look up account: {e}")))? - .ok_or_else(|| Status::not_found(format!("account '{}' not found", req.id)))?; + .ok_or_else(|| { + errors::status_with_metadata( + ErrorReason::AccountNotFound, + format!("account '{}' not found", req.id), + [("account_id".to_string(), req.id.clone())], + ) + })?; self.db .delete_account(&req.id) @@ -708,7 +716,13 @@ impl Accounts for AccountsGrpcService { .get_account(&req.id) .await .map_err(|e| Status::internal(format!("failed to look up account: {e}")))? - .ok_or_else(|| Status::not_found(format!("account '{}' not found", req.id)))?; + .ok_or_else(|| { + errors::status_with_metadata( + ErrorReason::AccountNotFound, + format!("account '{}' not found", req.id), + [("account_id".to_string(), req.id.clone())], + ) + })?; let password = self .secrets @@ -954,16 +968,19 @@ impl CalendarGrpcService { .collect(); if caldav_accounts.is_empty() { - return Err(Status::failed_precondition(if req.account_id.is_empty() { - "no CalDAV account is configured; add one with protocol CALDAV and a \ - collection_url before syncing" - .to_string() - } else { - format!( - "no CalDAV account with id '{}' is configured", - req.account_id - ) - })); + return Err(errors::status( + ErrorReason::NotConfigured, + if req.account_id.is_empty() { + "no CalDAV account is configured; add one with protocol CALDAV and a \ + collection_url before syncing" + .to_string() + } else { + format!( + "no CalDAV account with id '{}' is configured", + req.account_id + ) + }, + )); } let mut total = 0usize; @@ -1076,7 +1093,13 @@ impl Calendar for CalendarGrpcService { .db .get_calendar_event(&req.event_id) .await - .map_err(|e| Status::not_found(format!("event '{}' not found: {e}", req.event_id)))?; + .map_err(|e| { + errors::status_with_metadata( + ErrorReason::EventNotFound, + format!("event '{}' not found: {e}", req.event_id), + [("event_id".to_string(), req.event_id.clone())], + ) + })?; Ok(Response::new(GetEventResponse { event: Some(map_calendar_event_to_proto(event)), @@ -1183,7 +1206,11 @@ impl Contacts for ContactsGrpcService { } let contact = self.db.get_contact(&req.contact_id).await.map_err(|e| { - Status::not_found(format!("contact '{}' not found: {e}", req.contact_id)) + errors::status_with_metadata( + ErrorReason::ContactNotFound, + format!("contact '{}' not found: {e}", req.contact_id), + [("contact_id".to_string(), req.contact_id.clone())], + ) })?; Ok(Response::new(GetContactResponse { @@ -1358,7 +1385,11 @@ impl Mail for MailGrpcService { } let email = self.db.get_message(&req.message_id).await.map_err(|e| { - Status::not_found(format!("message '{}' not found: {e}", req.message_id)) + errors::status_with_metadata( + ErrorReason::MessageNotFound, + format!("message '{}' not found: {e}", req.message_id), + [("message_id".to_string(), req.message_id.clone())], + ) })?; Ok(Response::new(GetMessageResponse { @@ -1387,7 +1418,11 @@ impl Mail for MailGrpcService { .set_message_read(&req.message_id, req.read) .await .map_err(|e| { - Status::not_found(format!("message '{}' not found: {e}", req.message_id)) + errors::status_with_metadata( + ErrorReason::MessageNotFound, + format!("message '{}' not found: {e}", req.message_id), + [("message_id".to_string(), req.message_id.clone())], + ) })?; self.event_bus.process_command(CoreCommand::MarkRead { @@ -1655,11 +1690,19 @@ impl Filters for FiltersGrpcService { ) -> Result, Status> { let req = request.into_inner(); - let rule = NsqlParser::parse_rule(req.name, req.priority, &req.nsql) - .map_err(|e| Status::invalid_argument(format!("NSQL syntax error: {e}")))?; + let rule = NsqlParser::parse_rule(req.name, req.priority, &req.nsql).map_err(|e| { + errors::status( + ErrorReason::ValidationFailed, + format!("NSQL syntax error: {e}"), + ) + })?; - NsqlValidator::validate(&rule, &ValidationOptions::default()) - .map_err(|e| Status::invalid_argument(format!("NSQL validation error: {e}")))?; + NsqlValidator::validate(&rule, &ValidationOptions::default()).map_err(|e| { + errors::status( + ErrorReason::ValidationFailed, + format!("NSQL validation error: {e}"), + ) + })?; self.db .save_filter_rule(&rule) @@ -1751,8 +1794,12 @@ impl Filters for FiltersGrpcService { ) -> Result, Status> { let req = request.into_inner(); - let rule = NsqlParser::parse_rule("Preview Rule", 0, &req.nsql) - .map_err(|e| Status::invalid_argument(format!("NSQL syntax error: {e}")))?; + let rule = NsqlParser::parse_rule("Preview Rule", 0, &req.nsql).map_err(|e| { + errors::status( + ErrorReason::ValidationFailed, + format!("NSQL syntax error: {e}"), + ) + })?; let preview_engine = FilterEngine::new(vec![rule]) .map_err(|e| Status::internal(format!("failed to build preview engine: {e}")))?; @@ -1791,17 +1838,31 @@ impl Filters for FiltersGrpcService { .map_err(|e| Status::internal(format!("failed to list filter rules: {e}")))? .into_iter() .find(|r| r.id == req.id) - .ok_or_else(|| Status::not_found(format!("filter rule '{}' not found", req.id)))?; + .ok_or_else(|| { + errors::status_with_metadata( + ErrorReason::RuleNotFound, + format!("filter rule '{}' not found", req.id), + [("rule_id".to_string(), req.id.clone())], + ) + })?; let name = req.name.unwrap_or(existing.name); let nsql = req.nsql.unwrap_or(existing.nsql_text); let priority = req.priority.unwrap_or(existing.priority); - let mut rule = NsqlParser::parse_rule(name, priority, &nsql) - .map_err(|e| Status::invalid_argument(format!("NSQL syntax error: {e}")))?; + let mut rule = NsqlParser::parse_rule(name, priority, &nsql).map_err(|e| { + errors::status( + ErrorReason::ValidationFailed, + format!("NSQL syntax error: {e}"), + ) + })?; - NsqlValidator::validate(&rule, &ValidationOptions::default()) - .map_err(|e| Status::invalid_argument(format!("NSQL validation error: {e}")))?; + NsqlValidator::validate(&rule, &ValidationOptions::default()).map_err(|e| { + errors::status( + ErrorReason::ValidationFailed, + format!("NSQL validation error: {e}"), + ) + })?; rule.id = existing.id; diff --git a/crates/nunciod/tests/typed_errors_e2e_test.rs b/crates/nunciod/tests/typed_errors_e2e_test.rs new file mode 100644 index 0000000..436f49c --- /dev/null +++ b/crates/nunciod/tests/typed_errors_e2e_test.rs @@ -0,0 +1,171 @@ +//! Offline end-to-end test that typed `ErrorInfo` details survive the gRPC +//! wire. +//! +//! Boots a real `nunciod` daemon gRPC server +//! (`nunciod::grpc::serve_on_listener_with_overrides`) on an ephemeral +//! loopback port, backed by a real temp file-backed [`DatabaseEngine`] and a +//! [`SecretManager::mock`] vault (never the real OS keyring), with every +//! engine override left at its default (no injected backends). +//! +//! Driving the authenticated `nuncio-proto` clients, it triggers several +//! known error conditions and proves a client can machine-read each failure: +//! the returned `tonic::Status` carries the correct canonical gRPC code AND a +//! prost-encoded [`nuncio_proto::v1::ErrorInfo`] in the +//! `grpc-status-details-bin` trailer whose `reason` (and metadata) decode +//! back exactly, so a client never has to string-match the English message. +//! +//! Fully offline: no real network, no real OS keyring, no live server. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use nuncio_core::EventBus; +use nuncio_filter::FilterEngine; +use nuncio_proto::errors; +use nuncio_proto::v1::{ + CalendarSyncRequest, CreateRuleRequest, ErrorReason, GetMessageRequest, RemoveAccountRequest, +}; +use nuncio_store::db::DatabaseEngine; +use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT}; +use nunciod::grpc::{ + serve_on_listener_with_overrides, AccountsEngineOverrides, CalendarEngineOverrides, + ContactsEngineOverrides, MailEngineOverrides, +}; +use std::sync::Arc; +use tokio::net::TcpListener; +use tonic::Code; + +/// Boots the real daemon gRPC server on an ephemeral loopback port with every +/// override at its default. Returns the server address and minted bearer +/// token; the `TempDir` guard is returned so the caller controls its lifetime +/// for the whole test (the spawned server keeps using the db). +async fn boot_daemon() -> (std::net::SocketAddr, String, tempfile::TempDir) { + let secrets = Arc::new(SecretManager::mock()); + let dir = tempfile::tempdir().expect("create temp dir for file-backed db"); + let db_path = dir.path().join("typed_errors_e2e.db"); + let db = Arc::new( + DatabaseEngine::connect_file(&db_path, &secrets) + .await + .expect("open temp file-backed database"), + ); + + let token = hex::encode( + secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("mint gRPC bearer token from mock vault"), + ); + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("listener has local addr"); + + let event_bus = Arc::new(EventBus::new()); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + let server_secrets = secrets.clone(); + let server_token = token.clone(); + tokio::spawn(async move { + let _ = serve_on_listener_with_overrides( + listener, + event_bus, + db, + filter_engine, + server_secrets, + server_token, + MailEngineOverrides::default(), + CalendarEngineOverrides::default(), + ContactsEngineOverrides::default(), + AccountsEngineOverrides::default(), + ) + .await; + }); + + (addr, token, dir) +} + +#[tokio::test] +async fn typed_error_reasons_decode_from_status_details_over_the_wire() { + let (addr, token, _db_dir) = boot_daemon().await; + let addr_str = addr.to_string(); + + // ---- account-not-found: NotFound + ACCOUNT_NOT_FOUND + account_id ---- + let mut accounts = nuncio_proto::client::connect_accounts(&addr_str, &token) + .await + .expect("accounts client connects"); + let status = accounts + .remove_account(RemoveAccountRequest { + id: "no-such-account".to_string(), + }) + .await + .expect_err("removing a nonexistent account must fail"); + assert_eq!(status.code(), Code::NotFound); + let info = errors::error_info(&status).expect("typed ErrorInfo present in details trailer"); + assert_eq!(info.reason(), ErrorReason::AccountNotFound); + assert_eq!( + info.metadata.get("account_id").map(String::as_str), + Some("no-such-account"), + "the missing id is carried as machine-readable metadata" + ); + assert_eq!(info.message, status.message()); + assert_eq!( + errors::error_reason(&status), + Some(ErrorReason::AccountNotFound) + ); + + // ---- message-not-found: NotFound + MESSAGE_NOT_FOUND + message_id ---- + let mut mail = nuncio_proto::client::connect_mail(&addr_str, &token) + .await + .expect("mail client connects"); + let status = mail + .get_message(GetMessageRequest { + message_id: "no-such-message".to_string(), + }) + .await + .expect_err("fetching a nonexistent message must fail"); + assert_eq!(status.code(), Code::NotFound); + assert_eq!( + errors::error_reason(&status), + Some(ErrorReason::MessageNotFound) + ); + let info = errors::error_info(&status).expect("typed ErrorInfo present"); + assert_eq!( + info.metadata.get("message_id").map(String::as_str), + Some("no-such-message") + ); + + // ---- not-configured: FailedPrecondition + NOT_CONFIGURED ---- + let mut calendar = nuncio_proto::client::connect_calendar(&addr_str, &token) + .await + .expect("calendar client connects"); + let status = calendar + .sync(CalendarSyncRequest { + account_id: String::new(), + calendar_id: "personal".to_string(), + start_window: 0, + end_window: 0, + }) + .await + .expect_err("syncing with no CalDAV account configured must fail"); + assert_eq!(status.code(), Code::FailedPrecondition); + assert_eq!( + errors::error_reason(&status), + Some(ErrorReason::NotConfigured) + ); + + // ---- validation-failed: InvalidArgument + VALIDATION_FAILED ---- + let mut filters = nuncio_proto::client::connect_filters(&addr_str, &token) + .await + .expect("filters client connects"); + let status = filters + .create_rule(CreateRuleRequest { + name: "bad rule".to_string(), + nsql: "this is not valid nsql".to_string(), + priority: 0, + }) + .await + .expect_err("creating a rule from unparseable NSQL must fail"); + assert_eq!(status.code(), Code::InvalidArgument); + assert_eq!( + errors::error_reason(&status), + Some(ErrorReason::ValidationFailed) + ); +} -- 2.54.0 From aa5c1c35e949a411ee12888ad73aaf8344010134 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 14:15:12 -0500 Subject: [PATCH 23/95] fix(filter): pin webhook dispatch to the checked IP to close DNS-rebind TOCTOU (#343) Refs #307 Co-authored-by: James Maes --- crates/nuncio-filter/src/validator.rs | 6 +- crates/nuncio-filter/src/webhook.rs | 162 ++++++++++++++++++++------ 2 files changed, 131 insertions(+), 37 deletions(-) diff --git a/crates/nuncio-filter/src/validator.rs b/crates/nuncio-filter/src/validator.rs index 7975e55..37a058d 100644 --- a/crates/nuncio-filter/src/validator.rs +++ b/crates/nuncio-filter/src/validator.rs @@ -35,8 +35,10 @@ pub struct ValidationOptions { pub allowed_forward_domains: Option>, /// When set, reject webhook targets that point at private, loopback, /// link-local, or metadata address ranges. The validator checks literal-IP - /// hosts; the dispatcher additionally checks addresses a hostname resolves - /// to before connecting. + /// hosts; the dispatcher additionally resolves hostnames, checks every + /// candidate address, and then pins the outbound connection to the + /// checked address so it cannot be swapped for an unchecked one by a + /// second DNS lookup. pub block_private_webhooks: bool, } diff --git a/crates/nuncio-filter/src/webhook.rs b/crates/nuncio-filter/src/webhook.rs index 68e7422..7d0e6da 100644 --- a/crates/nuncio-filter/src/webhook.rs +++ b/crates/nuncio-filter/src/webhook.rs @@ -6,7 +6,7 @@ use hmac::{Hmac, Mac}; use reqwest::Client; use serde_json::json; use sha2::Sha256; -use std::net::IpAddr; +use std::net::{IpAddr, SocketAddr}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use thiserror::Error; use tokio::net::lookup_host; @@ -24,6 +24,38 @@ fn first_blocked_address(addrs: &[IpAddr]) -> Option { .find(|ip| is_blocked_webhook_target(*ip)) } +/// Resolves `host` to its candidate addresses. Kept separate from the policy +/// check so the address list a rebinding DNS server could return is visible +/// to callers before any of it is trusted. +async fn resolve_host(host: &str, port: u16) -> Result, WebhookError> { + let resolved: Vec = lookup_host((host, port)) + .await + .map_err(|e| WebhookError::NetworkError(e.to_string()))? + .map(|addr| addr.ip()) + .collect(); + if resolved.is_empty() { + return Err(WebhookError::NetworkError(format!( + "webhook host '{host}' did not resolve to any address" + ))); + } + Ok(resolved) +} + +/// Builds an HTTP client whose connection to `host` is pinned to `addr`. +/// `reqwest`'s DNS override maps only the transport-level connect address; it +/// still sends the original hostname as the TLS SNI and `Host` header, so +/// certificate validation is unaffected. This is what stops the dispatcher +/// from re-resolving `host` after it has been checked against the SSRF +/// policy: the checked address is the only one the connection can use. +fn pinned_client(host: &str, addr: IpAddr, port: u16) -> Result { + Client::builder() + .timeout(Duration::from_secs(5)) + .redirect(reqwest::redirect::Policy::none()) + .resolve(host, SocketAddr::new(addr, port)) + .build() + .map_err(|e| WebhookError::NetworkError(e.to_string())) +} + /// Errors emitted during HTTP webhook dispatch execution. #[derive(Error, Debug)] pub enum WebhookError { @@ -39,22 +71,18 @@ pub enum WebhookError { } /// Outbound Webhook Dispatcher executing authenticated HTTP POST requests. +/// +/// Each dispatch builds its own client pinned to the address it just +/// resolved and checked (see [`pinned_client`]), so there is no shared +/// client to cache here. pub struct WebhookDispatcher { - client: Client, secret_key: Zeroizing, } impl WebhookDispatcher { /// Create a new `WebhookDispatcher` with an HMAC signing key. pub fn new(secret_key: impl Into) -> Self { - let client = Client::builder() - .timeout(Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .unwrap_or_default(); - Self { - client, secret_key: Zeroizing::new(secret_key.into()), } } @@ -73,33 +101,36 @@ impl WebhookDispatcher { NsqlValidator::pass6_action_security(&[action], opts) .map_err(|e| WebhookError::SecurityViolation(e.to_string()))?; + // Pass 6 only sees literal-IP hosts. Resolve the target here and, + // when the policy is enabled, reject it if it points into a blocked + // range - closing the DNS-rebind class of SSRF bypass that a + // substring/literal check on the URL cannot catch. The resolved + // address is then pinned below for the actual connection, so the + // dispatch cannot re-resolve the hostname and land on a different, + // unchecked address: the address that was checked is the only one + // `reqwest` is permitted to connect to. + let parsed = reqwest::Url::parse(url) + .map_err(|e| WebhookError::SecurityViolation(format!("invalid webhook URL: {e}")))?; + let host = parsed + .host_str() + .ok_or_else(|| WebhookError::SecurityViolation("webhook URL has no host".to_string()))? + // `host_str` brackets IPv6 literals (`[::1]`); strip them so the + // pair passes to the resolver cleanly. + .trim_start_matches('[') + .trim_end_matches(']') + .to_string(); + let port = parsed.port_or_known_default().unwrap_or(0); + + let resolved = resolve_host(&host, port).await?; if opts.block_private_webhooks { - // Pass 6 only sees literal-IP hosts. Resolve the target here and - // reject it if a hostname points into a blocked range, closing the - // DNS-rebind class of SSRF bypass. This is not airtight: reqwest - // re-resolves and connects on its own after this check, so a narrow - // TOCTOU window remains where DNS could flip between our lookup and - // the actual connection. Fully closing it needs a pinned - // resolver/connector, which is out of scope here. - if let Ok(parsed) = reqwest::Url::parse(url) { - if let Some(host) = parsed.host_str() { - // `host_str` brackets IPv6 literals (`[::1]`); strip them so - // the pair passes to the resolver cleanly. - let host = host.trim_start_matches('[').trim_end_matches(']'); - let port = parsed.port_or_known_default().unwrap_or(0); - let resolved: Vec = lookup_host((host, port)) - .await - .map_err(|e| WebhookError::NetworkError(e.to_string()))? - .map(|addr| addr.ip()) - .collect(); - if let Some(blocked) = first_blocked_address(&resolved) { - return Err(WebhookError::SecurityViolation(format!( - "webhook host '{host}' resolves to blocked address {blocked}" - ))); - } - } + if let Some(blocked) = first_blocked_address(&resolved) { + return Err(WebhookError::SecurityViolation(format!( + "webhook host '{host}' resolves to blocked address {blocked}" + ))); } } + let pinned_addr = resolved[0]; + let client = pinned_client(&host, pinned_addr, port)?; let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -122,8 +153,7 @@ impl WebhookDispatcher { mac.update(format!("{timestamp}.{payload_str}").as_bytes()); let signature = hex::encode(mac.finalize().into_bytes()); - let response = self - .client + let response = client .post(url) .header("Content-Type", "application/json") .header( @@ -162,6 +192,68 @@ impl WebhookDispatcher { #[cfg(test)] mod tests { use super::*; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + #[tokio::test] + async fn test_pinned_client_connects_to_checked_ip_not_dns() { + // `.invalid` is reserved by RFC 2606 to never resolve: any real DNS + // lookup for it fails. The mock server only listens on its loopback + // address, which has no relationship to that hostname in any + // resolver. If the request below reaches the mock server, it can + // only be because `pinned_client`'s resolve override - not DNS - + // supplied the connect address, proving the dispatcher connects to + // exactly the address it already checked. + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/hook")) + .respond_with(ResponseTemplate::new(200)) + .mount(&mock_server) + .await; + + let addr = *mock_server.address(); + let host = "checked-ip-pin.invalid"; + let client = pinned_client(host, addr.ip(), addr.port()) + .expect("building a client with a resolve override must succeed"); + + let url = format!("http://{host}:{}/hook", addr.port()); + let response = client + .post(&url) + .send() + .await + .expect("the pinned connection must reach the mock server, not a DNS lookup"); + + assert_eq!(response.status(), 200); + } + + #[tokio::test] + async fn test_dispatch_reaches_pinned_loopback_mock_server() { + // End-to-end proof that `dispatch_with_options` itself (not just the + // `pinned_client` helper) drives the request to the address it + // resolved and checked. `block_private_webhooks` is disabled here + // only to allow the test to target a loopback mock server; it does + // not affect whether the connection is pinned. + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/hook")) + .respond_with(ResponseTemplate::new(202)) + .mount(&mock_server) + .await; + + let dispatcher = WebhookDispatcher::new("secret_key_123"); + let opts = ValidationOptions { + available_folders: None, + allowed_forward_domains: None, + block_private_webhooks: false, + }; + let url = format!("{}/hook", mock_server.uri()); + let status = dispatcher + .dispatch_with_options(&url, "rule_1", "msg_1", "Test", "a@b.com", &opts) + .await + .expect("dispatch to the mock server must succeed"); + + assert_eq!(status, 202); + } #[tokio::test] async fn test_blocked_private_ip_webhook() { -- 2.54.0 From 49119a4ce1f2885e18915e31105944e20e7f6e12 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 15:38:59 -0500 Subject: [PATCH 24/95] refactor(core): remove dead EventBus ReloadFilters/unread_count arms (#344) CoreCommand::ReloadFilters was a silent no-op: nothing in the workspace sends it, since filter reload actually happens via FiltersGrpcService::reload_engine_from_store. AppState::unread_count was written by MarkRead but never read by any consumer -- GetStatusResponse only carries engine_status and version, so it never reached the wire. Refs #324 Co-authored-by: James Maes --- crates/nuncio-core/src/lib.rs | 37 +++++++++-------------------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/crates/nuncio-core/src/lib.rs b/crates/nuncio-core/src/lib.rs index 80000df..4c0ca72 100644 --- a/crates/nuncio-core/src/lib.rs +++ b/crates/nuncio-core/src/lib.rs @@ -52,8 +52,6 @@ pub enum CoreCommand { /// Human-readable error description. message: String, }, - /// Reload filter rules cache in memory. - ReloadFilters, /// Request graceful shutdown of the core engine. Shutdown, } @@ -153,8 +151,6 @@ pub struct AppState { pub status: EngineStatus, /// Total number of loaded accounts. pub accounts_loaded: usize, - /// Total unread message count across loaded accounts. - pub unread_count: usize, /// Last error message, if any. pub last_error: Option, } @@ -259,20 +255,12 @@ impl EventBus { }); } CoreCommand::MarkRead { message_id, read } => { - self.update_state(|s| { - if read { - s.unread_count = s.unread_count.saturating_sub(1); - } else { - s.unread_count = s.unread_count.saturating_add(1); - } - }); self.publish_event(CoreEvent::MessageFlagsChanged { message_id, read }); } CoreCommand::ReportError { message } => { self.update_state(|s| s.last_error = Some(message.clone())); self.publish_event(CoreEvent::Error { message }); } - CoreCommand::ReloadFilters => {} CoreCommand::Shutdown => { self.update_state(|s| s.status = EngineStatus::ShuttingDown); self.publish_event(CoreEvent::ShuttingDown); @@ -308,7 +296,6 @@ mod tests { let seed = AppState { status: EngineStatus::Idle, accounts_loaded: 3, - unread_count: 5, last_error: None, }; let bus = EventBus::with_initial_state(seed.clone()); @@ -400,18 +387,14 @@ mod tests { } #[tokio::test] - async fn process_mark_read_adjusts_unread_count_both_directions() { - let bus = EventBus::with_initial_state(AppState { - unread_count: 1, - ..AppState::default() - }); + async fn process_mark_read_emits_flags_changed_for_each_call() { + let bus = EventBus::new(); let mut events = bus.subscribe_events(); bus.process_command(CoreCommand::MarkRead { message_id: "msg-1".to_string(), read: false, }); - assert_eq!(bus.current_state().unread_count, 2); assert_eq!( events.recv().await.expect("event received"), CoreEvent::MessageFlagsChanged { @@ -424,15 +407,13 @@ mod tests { message_id: "msg-1".to_string(), read: true, }); - bus.process_command(CoreCommand::MarkRead { - message_id: "msg-1".to_string(), - read: true, - }); - bus.process_command(CoreCommand::MarkRead { - message_id: "msg-1".to_string(), - read: true, - }); - assert_eq!(bus.current_state().unread_count, 0); + assert_eq!( + events.recv().await.expect("event received"), + CoreEvent::MessageFlagsChanged { + message_id: "msg-1".to_string(), + read: true + } + ); } #[tokio::test] -- 2.54.0 From c660ed6d065d86d6e4462a482ecea83f27922a8f Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 15:52:07 -0500 Subject: [PATCH 25/95] feat(cal): add CalendarEvent::to_ical serializer (#345) Adds the CalDAV write-back inverse of IcalParserAdapter::parse_ical: a ToIcal extension trait wraps a single VEVENT (UID, DTSTAMP, DTSTART, DTEND, SUMMARY, LOCATION, RRULE) in a VCALENDAR shell, escaping TEXT values per RFC 5545 3.3.11. Refs #310 Co-authored-by: James Maes --- crates/nuncio-cal/src/ical.rs | 182 ++++++++++++++++++++++++++++++++++ crates/nuncio-cal/src/lib.rs | 2 + 2 files changed, 184 insertions(+) create mode 100644 crates/nuncio-cal/src/ical.rs diff --git a/crates/nuncio-cal/src/ical.rs b/crates/nuncio-cal/src/ical.rs new file mode 100644 index 0000000..77d104f --- /dev/null +++ b/crates/nuncio-cal/src/ical.rs @@ -0,0 +1,182 @@ +//! iCalendar (RFC 5545) VEVENT serializer -- the write-back inverse of +//! [`crate::parser::IcalParserAdapter`]. +//! +//! [`CalendarEvent`] stores `start_time`/`end_time` as resolved UTC unix timestamps (the parser +//! already collapses any `TZID`/floating/`Z` form down to a single UTC instant), so this +//! serializer always emits `Z`-suffixed UTC `DATE-TIME` values. That is a lossless encoding of +//! what the model actually holds: re-parsing the emitted text yields the same UTC instant no +//! matter which form the original payload used. + +use chrono::{DateTime, Utc}; +use nuncio_core::model::CalendarEvent; + +/// Adds RFC 5545 serialization to [`CalendarEvent`]. +/// +/// `CalendarEvent` is defined in `nuncio-core`, so this is an extension trait rather than an +/// inherent impl -- Rust forbids inherent impls on types owned by another crate. Importing the +/// trait makes `event.to_ical()` read identically to an inherent method at call sites. +pub trait ToIcal { + /// Serialize this event as a complete iCalendar payload: a single `VEVENT` wrapped in a + /// `VCALENDAR`, the shape a CalDAV `PUT` request body expects. + fn to_ical(&self) -> String; +} + +impl ToIcal for CalendarEvent { + fn to_ical(&self) -> String { + let dtstamp = format_utc_datetime(Utc::now().timestamp()); + let dtstart = format_utc_datetime(self.start_time); + let dtend = format_utc_datetime(self.end_time); + + let mut vevent = String::new(); + vevent.push_str("BEGIN:VEVENT\r\n"); + vevent.push_str(&format!("UID:{}\r\n", escape_text(&self.id))); + vevent.push_str(&format!("DTSTAMP:{}\r\n", dtstamp)); + vevent.push_str(&format!("DTSTART:{}\r\n", dtstart)); + vevent.push_str(&format!("DTEND:{}\r\n", dtend)); + vevent.push_str(&format!("SUMMARY:{}\r\n", escape_text(&self.summary))); + if let Some(location) = &self.location { + vevent.push_str(&format!("LOCATION:{}\r\n", escape_text(location))); + } + if let Some(rrule) = &self.rrule { + // RRULE is a structured value (RFC 5545 3.3.10), not TEXT -- it must not go through + // TEXT escaping, and the parser reads it back verbatim via `properties().get`. + vevent.push_str(&format!("RRULE:{}\r\n", rrule)); + } + vevent.push_str("END:VEVENT\r\n"); + + let mut vcalendar = String::new(); + vcalendar.push_str("BEGIN:VCALENDAR\r\n"); + vcalendar.push_str("VERSION:2.0\r\n"); + vcalendar.push_str("PRODID:-//Nuncio//Calendar Engine//EN\r\n"); + vcalendar.push_str(&vevent); + vcalendar.push_str("END:VCALENDAR\r\n"); + vcalendar + } +} + +/// Format a UTC unix timestamp as an RFC 5545 `DATE-TIME` in UTC form (`YYYYMMDDTHHMMSSZ`). +fn format_utc_datetime(unix_ts: i64) -> String { + DateTime::::from_timestamp(unix_ts, 0) + .map(|dt| dt.format("%Y%m%dT%H%M%SZ").to_string()) + // `from_timestamp` only returns `None` for a timestamp outside chrono's representable + // range; falling back to the Unix epoch keeps this a total function without panicking, + // since a corrupt in-memory timestamp is not something this serializer can repair. + .unwrap_or_else(|| "19700101T000000Z".to_string()) +} + +/// Escape a `TEXT` value per RFC 5545 3.3.11: backslash, semicolon, and comma are escaped with a +/// leading backslash, and embedded newlines become the two-character literal `\n`. Order matters +/// -- backslash must be escaped first so the escaping added for the other characters is not +/// itself re-escaped. +fn escape_text(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '\\' => escaped.push_str("\\\\"), + ';' => escaped.push_str("\\;"), + ',' => escaped.push_str("\\,"), + '\n' => escaped.push_str("\\n"), + '\r' => {} + other => escaped.push(other), + } + } + escaped +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::IcalParserAdapter; + + fn event( + summary: &str, + start_time: i64, + end_time: i64, + rrule: Option<&str>, + location: Option<&str>, + ) -> CalendarEvent { + CalendarEvent { + id: "evt-roundtrip".to_string(), + account_id: "acct-1".to_string(), + calendar_id: "cal-1".to_string(), + summary: summary.to_string(), + start_time, + end_time, + rrule: rrule.map(str::to_string), + location: location.map(str::to_string), + } + } + + fn roundtrip(original: &CalendarEvent) -> CalendarEvent { + let ics = original.to_ical(); + IcalParserAdapter::parse_ical( + &original.id, + &original.account_id, + &original.calendar_id, + &ics, + ) + .expect("serialized iCalendar must parse back") + } + + /// Proves the serializer is the true inverse of [`IcalParserAdapter::parse_ical`] across the + /// event shapes CalDAV write-back must support: a plain timed event, an all-day event + /// (whole-day UTC window), an event whose UTC instant originated from a `TZID` source (the + /// model has already resolved it to UTC, so round-tripping through `to_ical` must preserve + /// that same instant), and a recurring event carrying an `RRULE`. + #[test] + fn to_ical_roundtrips_through_parse_ical_for_all_event_shapes() { + let timed = event( + "Standup", + 1_709_294_400, + 1_709_298_000, + None, + Some("Room 5"), + ); + assert_eq!(roundtrip(&timed), timed); + + // Whole-day window: 2024-03-01T00:00:00Z .. 2024-03-01T23:59:59Z, matching the parser's + // own DATE-only convention (start of day / end of day). + let all_day = event("Conference", 1_709_251_200, 1_709_337_599, None, None); + assert_eq!(roundtrip(&all_day), all_day); + + // 2024-07-15 09:00 America/New_York (EDT, UTC-4) resolves to this UTC instant; the model + // only ever stores the resolved UTC epoch, so serializing and re-parsing must land on + // exactly the same instant even though the TZID itself is not retained. + let from_tzid = event("Standup EDT", 1_721_048_400, 1_721_052_000, None, None); + assert_eq!(roundtrip(&from_tzid), from_tzid); + + let recurring = event( + "Weekly Sync", + 1_709_294_400, + 1_709_298_000, + Some("FREQ=WEEKLY;BYDAY=MO"), + Some("Conference Room 101"), + ); + assert_eq!(roundtrip(&recurring), recurring); + } + + #[test] + fn to_ical_escapes_rfc5545_special_characters_in_text_fields() { + let special = event( + "Comma, semicolon; backslash\\ and\nnewline", + 1_709_294_400, + 1_709_298_000, + None, + Some("Loc; with, chars"), + ); + let ics = special.to_ical(); + assert!(ics.contains("SUMMARY:Comma\\, semicolon\\; backslash\\\\ and\\nnewline\r\n")); + assert!(ics.contains("LOCATION:Loc\\; with\\, chars\r\n")); + assert_eq!(roundtrip(&special), special); + } + + #[test] + fn to_ical_wraps_a_single_vevent_in_a_vcalendar_shell() { + let plain = event("Solo Event", 1_709_294_400, 1_709_298_000, None, None); + let ics = plain.to_ical(); + assert!(ics.starts_with("BEGIN:VCALENDAR\r\n")); + assert!(ics.contains("VERSION:2.0\r\n")); + assert!(ics.trim_end().ends_with("END:VCALENDAR")); + assert_eq!(ics.matches("BEGIN:VEVENT").count(), 1); + } +} diff --git a/crates/nuncio-cal/src/lib.rs b/crates/nuncio-cal/src/lib.rs index d9a72bf..5943a0a 100644 --- a/crates/nuncio-cal/src/lib.rs +++ b/crates/nuncio-cal/src/lib.rs @@ -5,6 +5,7 @@ pub mod backend; pub mod caldav; +pub mod ical; pub mod mock; pub mod nlp; pub mod parser; @@ -13,6 +14,7 @@ pub mod scheduling; pub use backend::CalendarBackend; pub use caldav::{CalDavAccountConfig, CalDavClient}; +pub use ical::ToIcal; pub use mock::MockCalendarBackend; pub use nlp::{NaturalLanguageScheduler, NlpError, ParsedSchedulingIntent}; pub use parser::{CalendarError, IcalParserAdapter}; -- 2.54.0 From 885c13e4cc4f3ee90b6c4761ecbda44ca9b37ef0 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 15:55:13 -0500 Subject: [PATCH 26/95] feat(nunciod): single-instance advisory lock on the DB path (#346) Acquire an exclusive OS advisory lock on a sidecar file beside the database path (.lock) before opening the store, so a second nunciod pointed at the same database fails fast with an actionable error instead of both instances writing to the same SQLite file. Previously only an incidental gRPC port-bind collision offered any protection against this. Uses fs4 (flock on Unix, LockFileEx on Windows) since std has no portable file-lock primitive. The guard is held for the process lifetime and dropped at clean shutdown, releasing the lock; the OS also releases it automatically if the process dies uncleanly, so there is no PID-file staleness to reason about. Refs #291 Co-authored-by: James Maes --- Cargo.lock | 11 +++ Cargo.toml | 1 + crates/nunciod/Cargo.toml | 1 + crates/nunciod/src/lib.rs | 1 + crates/nunciod/src/lock.rs | 180 +++++++++++++++++++++++++++++++++++++ crates/nunciod/src/main.rs | 8 ++ 6 files changed, 202 insertions(+) create mode 100644 crates/nunciod/src/lock.rs diff --git a/Cargo.lock b/Cargo.lock index f94cbed..ad112c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1314,6 +1314,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix 1.1.4", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2636,6 +2646,7 @@ dependencies = [ "arc-swap", "async-trait", "bytes", + "fs4", "hex", "nuncio-cal", "nuncio-cli", diff --git a/Cargo.toml b/Cargo.toml index e2e4e76..a0ca121 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ prost = "0.14" tonic-prost-build = "0.14" protoc-bin-vendored = "3.2" subtle = "2.6" +fs4 = "0.13" diff --git a/crates/nunciod/Cargo.toml b/crates/nunciod/Cargo.toml index a43ef22..d1ee17f 100644 --- a/crates/nunciod/Cargo.toml +++ b/crates/nunciod/Cargo.toml @@ -34,6 +34,7 @@ subtle.workspace = true hex.workspace = true bytes.workspace = true zeroize.workspace = true +fs4.workspace = true [dev-dependencies] nuncio-cli = { path = "../nuncio-cli" } diff --git a/crates/nunciod/src/lib.rs b/crates/nunciod/src/lib.rs index 9a10ad9..0f3ef2d 100644 --- a/crates/nunciod/src/lib.rs +++ b/crates/nunciod/src/lib.rs @@ -6,6 +6,7 @@ pub mod calendar_sync; pub mod contacts_sync; pub mod grpc; pub mod lifecycle; +pub mod lock; pub mod orchestrator; pub mod outbox; pub mod send; diff --git a/crates/nunciod/src/lock.rs b/crates/nunciod/src/lock.rs new file mode 100644 index 0000000..130eea5 --- /dev/null +++ b/crates/nunciod/src/lock.rs @@ -0,0 +1,180 @@ +//! Exclusive, cross-platform advisory lock on the daemon's database path. +//! +//! Nothing about starting a second `nunciod` process today prevents it from +//! opening the same SQLite database another instance already has open -- +//! the only accidental protection is a port-bind collision on the gRPC +//! listener, and that only fires if both instances also happen to pick the +//! same port. Two writers on one SQLite file corrupt state. +//! +//! [`InstanceLock::acquire`] takes an OS-level exclusive lock (`flock` on +//! Unix, `LockFileEx` on Windows via the `fs4` crate) on a sidecar file next +//! to the database, `.lock`. The lock is scoped to the OS file +//! handle, not tracked by a PID written into the file, so it cannot go +//! stale: if the holding process dies for any reason (crash, kill -9), the +//! OS releases the lock the moment the file descriptor/handle closes. + +use std::fs::{File, OpenOptions}; +use std::path::{Path, PathBuf}; + +use fs4::fs_std::FileExt; + +/// Failure to acquire the single-instance lock. +#[derive(Debug, thiserror::Error)] +pub enum InstanceLockError { + /// The lock is already held by another live process. The caller must + /// treat this as fatal and refuse to open the store. + #[error("another nunciod instance is already running for this database at {0}")] + AlreadyRunning(PathBuf), + + /// The lock file itself could not be created/opened/locked, e.g. a + /// permissions problem or an unwritable parent directory. + #[error("failed to acquire instance lock at {path}: {source}")] + Io { + path: PathBuf, + #[source] + source: std::io::Error, + }, +} + +/// Holds the exclusive lock on the database path for as long as this guard +/// is alive. Drop it (or let it fall out of scope at process exit) to +/// release the lock; a later call to [`InstanceLock::acquire`] for the same +/// path then succeeds again. +#[derive(Debug)] +pub struct InstanceLock { + file: File, + path: PathBuf, +} + +impl InstanceLock { + /// Acquires the exclusive lock for `db_path`, creating the sidecar lock + /// file (`.lock`) if it does not already exist. + /// + /// Returns [`InstanceLockError::AlreadyRunning`] if another process + /// already holds it -- callers must fail startup on this rather than + /// proceed to open the store, since a second writer on the same SQLite + /// file is exactly the corruption this guards against. + pub fn acquire(db_path: &Path) -> Result { + let lock_path = lock_path_for(db_path); + + if let Some(parent) = lock_path.parent() { + std::fs::create_dir_all(parent).map_err(|source| InstanceLockError::Io { + path: lock_path.clone(), + source, + })?; + } + + let file = OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(&lock_path) + .map_err(|source| InstanceLockError::Io { + path: lock_path.clone(), + source, + })?; + + let acquired = file + .try_lock_exclusive() + .map_err(|source| InstanceLockError::Io { + path: lock_path.clone(), + source, + })?; + + if !acquired { + return Err(InstanceLockError::AlreadyRunning(db_path.to_path_buf())); + } + + Ok(Self { + file, + path: lock_path, + }) + } +} + +impl Drop for InstanceLock { + fn drop(&mut self) { + // The OS releases a `flock`/`LockFileEx` lock automatically when the + // underlying handle closes, so a failure here is not a leaked lock + // (the process exiting, or `self.file` being dropped right after, + // clears it regardless) -- it is only logged so an operator sees + // anything unexpected in the unlock call itself. + if let Err(err) = self.file.unlock() { + tracing::warn!( + path = %self.path.display(), + error = %err, + "failed to explicitly release nunciod instance lock; it will still be \ + released when the process exits and the underlying file handle closes" + ); + } + } +} + +fn lock_path_for(db_path: &Path) -> PathBuf { + let mut file_name = db_path.as_os_str().to_owned(); + file_name.push(".lock"); + PathBuf::from(file_name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn second_acquire_for_same_path_fails_while_first_guard_is_held() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("nuncio.db"); + + let first = InstanceLock::acquire(&db_path).unwrap(); + + let second = InstanceLock::acquire(&db_path); + assert!( + matches!(second, Err(InstanceLockError::AlreadyRunning(ref p)) if p == &db_path), + "expected AlreadyRunning({db_path:?}), got {second:?}" + ); + + drop(first); + } + + #[test] + fn acquire_succeeds_again_after_the_first_guard_is_dropped() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("nuncio.db"); + + let first = InstanceLock::acquire(&db_path).unwrap(); + drop(first); + + let second = InstanceLock::acquire(&db_path); + assert!( + second.is_ok(), + "acquiring after the prior guard dropped must succeed, got {:?}", + second.err() + ); + } + + #[test] + fn error_message_names_the_database_path() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("nuncio.db"); + + let first = InstanceLock::acquire(&db_path).unwrap(); + let second = InstanceLock::acquire(&db_path).unwrap_err(); + + let message = second.to_string(); + assert!(message.contains("already running")); + assert!(message.contains(&db_path.display().to_string())); + + drop(first); + } + + #[test] + fn lock_file_is_created_beside_the_database_path() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("nuncio.db"); + + let guard = InstanceLock::acquire(&db_path).unwrap(); + assert!(dir.path().join("nuncio.db.lock").exists()); + + drop(guard); + } +} diff --git a/crates/nunciod/src/main.rs b/crates/nunciod/src/main.rs index d50f90b..29bb348 100644 --- a/crates/nunciod/src/main.rs +++ b/crates/nunciod/src/main.rs @@ -46,6 +46,14 @@ async fn main() -> Result<(), Box> { .map(std::path::PathBuf::from) .unwrap_or_else(|_| nunciod::default_db_path()); + // Exclusive single-instance lock on the database path, acquired BEFORE + // the store is opened. Held for the entire process lifetime (dropped at + // the end of `main`, releasing it on any clean exit) so a second + // `nunciod` pointed at the same database fails fast instead of the two + // instances corrupting SQLite state between them. + let _instance_lock = + nunciod::lock::InstanceLock::acquire(&db_path).map_err(|e| format!("{e}"))?; + let orchestrator = nunciod::SelfHealingSyncOrchestrator::new(&db_path, event_bus.clone()); let (db, _summary) = orchestrator.initialize_and_recover().await?; -- 2.54.0 From 64cff6d8f6fafd1e278cf78bf2a9b3a6f4513aab Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 16:00:48 -0500 Subject: [PATCH 27/95] feat(proto): unify time fields on Timestamp/Duration (#348) Message.received_at, CalendarEvent.start_time/end_time, CalendarSyncRequest/ListEventsRequest.start_window/end_window, FilterRule.created_at/updated_at, FilterExecutionLog.matched_at, AccountConfig.sync_interval_secs, PreviewRuleResponse.execution_time_us, and AuditRecord.timestamp_ns all mixed unix-seconds and nanosecond int64 scalars, leaving the unit ambiguous at the wire level. Move every wall-clock field to google.protobuf.Timestamp and every duration to google.protobuf.Duration, add prost-types as the standard well-known-types crate, and add nuncio_proto::time as the single conversion point between the store's native integer instants and the wire types (including a lossless unix-nanos split/join for the WORM audit ledger's sub-second precision). Update the daemon's gRPC mappers and the CLI's request construction/rendering accordingly, and regenerate the descriptor golden. Refs #274 Co-authored-by: James Maes --- Cargo.lock | 1 + Cargo.toml | 1 + crates/nuncio-cli/src/runner.rs | 150 +++++++++++---- crates/nuncio-proto/Cargo.toml | 1 + .../proto/nuncio/v1/descriptor.bin | Bin 13517 -> 14437 bytes .../nuncio-proto/proto/nuncio/v1/nuncio.proto | 48 ++--- crates/nuncio-proto/src/lib.rs | 1 + crates/nuncio-proto/src/time.rs | 138 ++++++++++++++ crates/nunciod/src/grpc.rs | 176 ++++++++++++++---- .../tests/accounts_lifecycle_e2e_test.rs | 2 +- .../tests/caldav_production_e2e_test.rs | 18 +- crates/nunciod/tests/calendar_e2e_test.rs | 12 +- crates/nunciod/tests/spine_e2e_test.rs | 2 +- crates/nunciod/tests/typed_errors_e2e_test.rs | 4 +- 14 files changed, 440 insertions(+), 114 deletions(-) create mode 100644 crates/nuncio-proto/src/time.rs diff --git a/Cargo.lock b/Cargo.lock index ad112c3..d324764 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2606,6 +2606,7 @@ version = "0.1.0" dependencies = [ "bytes", "prost", + "prost-types", "protoc-bin-vendored", "thiserror 1.0.69", "tokio", diff --git a/Cargo.toml b/Cargo.toml index a0ca121..b64c612 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,6 +81,7 @@ zip = { version = "8.6", default-features = false, features = ["deflate"] } tonic = "0.14" tonic-prost = "0.14" prost = "0.14" +prost-types = "0.14" tonic-prost-build = "0.14" protoc-bin-vendored = "3.2" subtle = "2.6" diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index 912f6b7..9bf5dbe 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -53,6 +53,48 @@ fn parse_account_protocol(protocol: &str) -> Result) -> String { + ts.as_ref() + .and_then(|ts| { + u32::try_from(ts.nanos) + .ok() + .map(|nanos| (ts.seconds, nanos)) + }) + .and_then(|(seconds, nanos)| chrono::DateTime::from_timestamp(seconds, nanos)) + .map(|dt| dt.to_rfc3339()) + .unwrap_or_else(|| "unknown".to_string()) +} + +/// Extracts the exact unix-nanosecond instant a wire-format `Timestamp` +/// carries, for callers that need the audit ledger's full sub-second +/// precision rather than the RFC 3339 rendering. +fn timestamp_unix_nanos(ts: &Option) -> i64 { + ts.as_ref() + .map(nuncio_proto::time::timestamp_to_unix_nanos) + .unwrap_or(0) +} + +/// Renders a wire-format `google.protobuf.Duration` as whole seconds for +/// human-readable and `--json` output (e.g. `AccountConfig.sync_interval`). +fn duration_secs(d: &Option) -> u64 { + d.as_ref() + .map(nuncio_proto::time::duration_to_secs) + .unwrap_or(0) +} + +/// Renders a wire-format `google.protobuf.Duration` as whole microseconds +/// for human-readable and `--json` output (e.g. +/// `PreviewRuleResponse.execution_time`). +fn duration_micros(d: &Option) -> u64 { + d.as_ref() + .map(nuncio_proto::time::duration_to_micros) + .unwrap_or(0) +} + /// Renders a `nuncio.v1.Message` (as returned by the daemon's `Mail` gRPC /// service) into the JSON shape used by `mail list`/`mail read`'s /// `--json` output. @@ -64,7 +106,7 @@ fn message_proto_to_json(message: &nuncio_proto::v1::Message) -> serde_json::Val "subject": message.subject, "sender": message.sender, "recipient": message.recipient, - "received_at": message.received_at, + "received_at": format_timestamp(&message.received_at), "read": message.read, "body_plain": message.body_plain, "body_html": message.body_html, @@ -80,8 +122,8 @@ fn calendar_event_proto_to_json(event: &nuncio_proto::v1::CalendarEvent) -> serd "account_id": event.account_id, "calendar_id": event.calendar_id, "summary": event.summary, - "start_time": event.start_time, - "end_time": event.end_time, + "start_time": format_timestamp(&event.start_time), + "end_time": format_timestamp(&event.end_time), "rrule": event.rrule, "location": event.location, }) @@ -150,7 +192,8 @@ fn map_export_format_to_proto(format: nuncio_core::ExportFormat) -> nuncio_proto fn audit_record_proto_to_json(record: &nuncio_proto::v1::AuditRecord) -> serde_json::Value { json!({ "sequence": record.sequence, - "timestamp_ns": record.timestamp_ns, + "timestamp": format_timestamp(&record.timestamp), + "timestamp_ns": timestamp_unix_nanos(&record.timestamp), "actor": record.actor, "action": record.action, "data_hash": record.data_hash, @@ -171,8 +214,8 @@ fn filter_rule_proto_to_json(rule: &nuncio_proto::v1::FilterRule) -> serde_json: "enabled": rule.enabled, "nsql_text": rule.nsql_text, "actions": rule.actions, - "created_at": rule.created_at, - "updated_at": rule.updated_at, + "created_at": format_timestamp(&rule.created_at), + "updated_at": format_timestamp(&rule.updated_at), }) } @@ -184,7 +227,7 @@ fn filter_execution_log_proto_to_json( "rule_id": log.rule_id, "message_id": log.message_id, "action_taken": log.action_taken, - "matched_at": log.matched_at, + "matched_at": format_timestamp(&log.matched_at), "prev_hash": log.prev_hash, "hash": log.hash, }) @@ -882,7 +925,10 @@ impl HeadlessRunner { } else { format!( "Message {}: Subject: '{}', From: {}, Date: {}", - msg.id, msg.subject, msg.sender, msg.received_at + msg.id, + msg.subject, + msg.sender, + format_timestamp(&msg.received_at) ) } } @@ -1066,13 +1112,15 @@ impl HeadlessRunner { } else if records.is_empty() { "No audit records recorded.".to_string() } else { - let mut out = String::from( - "SEQ ACTOR ACTION TIMESTAMP_NS\n", - ); + let mut out = + String::from("SEQ ACTOR ACTION TIMESTAMP\n"); for r in records { out.push_str(&format!( "{:<4} {:<20} {:<20} {}\n", - r.sequence, r.actor, r.action, r.timestamp_ns + r.sequence, + r.actor, + r.action, + format_timestamp(&r.timestamp) )); } out @@ -1348,13 +1396,15 @@ impl HeadlessRunner { "matched_rule_id": preview.matched_rule_id, "matched_rule_name": preview.matched_rule_name, "actions_evaluated": preview.actions_evaluated, - "execution_time_us": preview.execution_time_us, + "execution_time_us": duration_micros(&preview.execution_time), "condition_traces": preview.condition_traces, })) } else { format!( "Dry-run evaluation result: matched={}, actions={:?}, elapsed={}us", - preview.matched, preview.actions_evaluated, preview.execution_time_us + preview.matched, + preview.actions_evaluated, + duration_micros(&preview.execution_time) ) } } @@ -1545,7 +1595,11 @@ impl HeadlessRunner { for l in logs { out.push_str(&format!( "{:<4} {:<10} {:<10} {:<12} {}\n", - l.id, l.rule_id, l.message_id, l.action_taken, l.matched_at + l.id, + l.rule_id, + l.message_id, + l.action_taken, + format_timestamp(&l.matched_at) )); } out @@ -1733,7 +1787,7 @@ impl HeadlessRunner { imap_tls_mode: map_tls_mode_to_proto(imap_tls_mode).into(), smtp_tls_mode: map_tls_mode_to_proto(smtp_tls_mode).into(), keyring_secret_key: keyring_key.clone(), - sync_interval_secs: 300, + sync_interval: Some(nuncio_proto::time::duration_from_secs(300)), smtp_host: if is_caldav { String::new() } else { @@ -1837,7 +1891,7 @@ impl HeadlessRunner { "imap_tls_mode": a.imap_tls_mode().as_str_name(), "smtp_tls_mode": a.smtp_tls_mode().as_str_name(), "keyring_secret_key": a.keyring_secret_key, - "sync_interval_secs": a.sync_interval_secs, + "sync_interval_secs": duration_secs(&a.sync_interval), "collection_url": a.collection_url, }) }) @@ -1932,7 +1986,7 @@ impl HeadlessRunner { "imap_tls_mode": account.imap_tls_mode().as_str_name(), "smtp_tls_mode": account.smtp_tls_mode().as_str_name(), "keyring_secret_key": account.keyring_secret_key, - "sync_interval_secs": account.sync_interval_secs, + "sync_interval_secs": duration_secs(&account.sync_interval), })) } else { format!( @@ -2190,8 +2244,8 @@ impl HeadlessRunner { .list_events(nuncio_proto::v1::ListEventsRequest { account_id: account.to_string(), calendar_id: calendar.to_string(), - start_window: start, - end_window: end, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(start)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(end)), }) .await { @@ -2241,8 +2295,8 @@ impl HeadlessRunner { .sync(nuncio_proto::v1::CalendarSyncRequest { account_id: account.to_string(), calendar_id: calendar.to_string(), - start_window: start, - end_window: end, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(start)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(end)), }) .await { @@ -2933,7 +2987,7 @@ mod tests { imap_tls_mode: TlsModeProto::ImplicitTls.into(), smtp_tls_mode: TlsModeProto::ImplicitTls.into(), keyring_secret_key: "nuncio/acct-stub-1".to_string(), - sync_interval_secs: 300, + sync_interval: Some(nuncio_proto::time::duration_from_secs(300)), smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, collection_url: String::new(), @@ -3145,7 +3199,7 @@ mod tests { subject: "Stub Subject".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, + received_at: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_000_000)), read: false, body_plain: Some("Stub body text".to_string()), body_html: None, @@ -3598,8 +3652,8 @@ mod tests { account_id: "acct-stub-1".to_string(), calendar_id: "cal-stub".to_string(), summary: "Stub Meeting".to_string(), - start_time: 1_700_000_000, - end_time: 1_700_003_600, + start_time: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_000_000)), + end_time: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_003_600)), rrule: None, location: Some("Room 1".to_string()), } @@ -3707,8 +3761,14 @@ mod tests { .expect("stub daemon received a list_events request"); assert_eq!(recorded_list.account_id, "acct-stub-1"); assert_eq!(recorded_list.calendar_id, "cal-stub"); - assert_eq!(recorded_list.start_window, 0); - assert_eq!(recorded_list.end_window, i64::MAX); + assert_eq!( + recorded_list.start_window, + Some(nuncio_proto::time::timestamp_from_unix_secs(0)) + ); + assert_eq!( + recorded_list.end_window, + Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)) + ); // `cal sync`: proves the CLI is a real gRPC client of // `Calendar/Sync` and reports the daemon's real `synced_count`, @@ -4024,8 +4084,12 @@ mod tests { enabled: true, nsql_text: req.nsql, actions: vec!["MARK READ".to_string()], - created_at: 1_700_000_000, - updated_at: 1_700_000_000, + created_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + 1_700_000_000, + )), + updated_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + 1_700_000_000, + )), }), })) } @@ -4043,8 +4107,12 @@ mod tests { enabled: true, nsql_text: "WHERE subject CONTAINS 'Urgent' ACTION MARK READ".to_string(), actions: vec!["MARK READ".to_string()], - created_at: 1_700_000_000, - updated_at: 1_700_000_000, + created_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + 1_700_000_000, + )), + updated_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + 1_700_000_000, + )), }], })) } @@ -4086,7 +4154,7 @@ mod tests { matched_rule_id: Some("rule-stub-1".to_string()), matched_rule_name: Some("Stub Rule".to_string()), actions_evaluated: vec!["MARK READ".to_string()], - execution_time_us: 42, + execution_time: Some(nuncio_proto::time::duration_from_micros(42)), condition_traces: vec!["Rule 'Stub Rule': MATCH".to_string()], })) } @@ -4111,8 +4179,12 @@ mod tests { .nsql .unwrap_or_else(|| "WHERE subject CONTAINS 'Urgent'".to_string()), actions: vec!["MARK READ".to_string()], - created_at: 1_700_000_000, - updated_at: 1_700_000_001, + created_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + 1_700_000_000, + )), + updated_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + 1_700_000_001, + )), }), })) } @@ -4153,7 +4225,9 @@ mod tests { rule_id: "rule-stub-1".to_string(), message_id: "msg-stub-1".to_string(), action_taken: "MARK READ".to_string(), - matched_at: 1_700_000_002, + matched_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + 1_700_000_002, + )), prev_hash: "prev-hash".to_string(), hash: "hash".to_string(), }], @@ -4640,7 +4714,9 @@ mod tests { Ok(tonic::Response::new(ListRecordsResponse { records: vec![AuditRecord { sequence: 1, - timestamp_ns: 1_700_000_000_000_000_000, + timestamp: Some(nuncio_proto::time::timestamp_from_unix_nanos( + 1_700_000_000_000_000_000, + )), actor: "system.test".to_string(), action: "data.export".to_string(), data_hash: "deadbeef".to_string(), diff --git a/crates/nuncio-proto/Cargo.toml b/crates/nuncio-proto/Cargo.toml index 71359e1..1012c13 100644 --- a/crates/nuncio-proto/Cargo.toml +++ b/crates/nuncio-proto/Cargo.toml @@ -13,6 +13,7 @@ workspace = true tonic.workspace = true tonic-prost.workspace = true prost.workspace = true +prost-types.workspace = true bytes.workspace = true thiserror.workspace = true diff --git a/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin b/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin index 850edc681e2a0d676fc2fee02bcd32376c5c1d8b..94ba2c20c5f91ea86c5d255886932d3c91912fcc 100644 GIT binary patch delta 1476 zcmcgsON-M`809ARb=%HhW|XwfRH+X{Asss?*5acbaS%71Fj90=()3z`$-N;>rw$51 zL=Xf;B0ipr;abWLY$YG3UH)KoOp zN7wXX6YgGcU$v@J-vIb-9^3P&b^%VH+eqv?UslVR7#okuBpVW(=B&BB&nd8JeqisF zY;%gBi#NT#Vo~O-Yq`f1cxQgycgKmuUT#hIJVa9Yl#6x@FehOW;m)}EZN|VSY^+Pm z!I-A?W#o>|9L`(#P2()gcuocq?vMdXN43GHU3}ApI|Fju_~c!Ys01sW+vs$=}dFk7-wu|*j0%?Z|hR+O2E@z&&v;hf9P2#Y2lXzm=H6#CU#klXRmn(A3<-lH?4CrL166 zjb!?m7~3Wvl&dxXN*9+T7L|ZalmJTyae_raMhQuBair#@KqNtSF;8A1AIms>GLu5M znIabtL~D6wUP^ws1T#oJh#M>x4i*A=38WpWfCX&8aRqnAsgwB?L&-E?^CiVRHpUl| i9W}I-fKC8~1klRdg7~~*2_}dW;9`Eoo0n=F;0FLh;+R$d diff --git a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto index 171b418..368af82 100644 --- a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto +++ b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto @@ -2,6 +2,9 @@ syntax = "proto3"; package nuncio.v1; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + // ErrorReason is the stable, machine-readable taxonomy of failure conditions // the daemon reports to clients. It is carried inside an `ErrorInfo` in a // failed RPC's gRPC status-details trailer so a client can branch on a @@ -272,8 +275,8 @@ message AccountConfig { TlsMode smtp_tls_mode = 9; // Keyring entry key used to retrieve OS vault credentials. string keyring_secret_key = 10; - // Background sync polling interval in seconds (minimum 10s). - uint64 sync_interval_secs = 11; + // Background sync polling interval (minimum 10s). + google.protobuf.Duration sync_interval = 11; // SMTP server hostname or IP address, used for outbound mail. Distinct // from `server_host` because a mail provider's inbound (IMAP/JMAP) and // outbound (SMTP) endpoints commonly differ. @@ -435,8 +438,8 @@ message Message { string sender = 5; // Recipient address. string recipient = 6; - // Unix timestamp of message arrival. - int64 received_at = 7; + // Timestamp of message arrival. + google.protobuf.Timestamp received_at = 7; // Read/unread flag status. bool read = 8; // Plaintext message body (already decrypted by the store). @@ -663,10 +666,10 @@ message FilterRule { // Rendered NSQL action clauses (e.g. "MOVE TO 'Archive'"), derived from // the parsed AST. repeated string actions = 7; - // Creation timestamp (unix seconds). - int64 created_at = 8; - // Last update timestamp (unix seconds). - int64 updated_at = 9; + // Creation timestamp. + google.protobuf.Timestamp created_at = 8; + // Last update timestamp. + google.protobuf.Timestamp updated_at = 9; } // CreateRuleRequest carries a new rule's display name, full NSQL source @@ -750,8 +753,8 @@ message PreviewRuleResponse { optional string matched_rule_name = 4; // NSQL-rendered actions that would be executed on a real match. repeated string actions_evaluated = 5; - // Evaluation duration in microseconds. - uint64 execution_time_us = 6; + // Evaluation duration. + google.protobuf.Duration execution_time = 6; // Human-readable step-by-step match traces. repeated string condition_traces = 7; } @@ -825,8 +828,8 @@ message FilterExecutionLog { string message_id = 3; // Action description string. string action_taken = 4; - // Timestamp of execution (unix seconds). - int64 matched_at = 5; + // Timestamp of execution. + google.protobuf.Timestamp matched_at = 5; // Hash of the preceding log record in the hash-chain ledger. string prev_hash = 6; // HMAC-SHA256 hash of this record. @@ -973,10 +976,10 @@ message CalendarEvent { string calendar_id = 3; // Event title or summary. string summary = 4; - // Unix timestamp for event start. - int64 start_time = 5; - // Unix timestamp for event end. - int64 end_time = 6; + // Timestamp for event start. + google.protobuf.Timestamp start_time = 5; + // Timestamp for event end. + google.protobuf.Timestamp end_time = 6; // Recurrence rule string (RFC 5545 RRULE format). optional string rrule = 7; // Location text. @@ -989,8 +992,8 @@ message CalendarEvent { message CalendarSyncRequest { string account_id = 1; string calendar_id = 2; - int64 start_window = 3; - int64 end_window = 4; + google.protobuf.Timestamp start_window = 3; + google.protobuf.Timestamp end_window = 4; } // CalendarSyncResponse reports how many events were synchronized. @@ -1002,8 +1005,8 @@ message CalendarSyncResponse { message ListEventsRequest { string account_id = 1; string calendar_id = 2; - int64 start_window = 3; - int64 end_window = 4; + google.protobuf.Timestamp start_window = 3; + google.protobuf.Timestamp end_window = 4; } // ListEventsResponse. @@ -1197,8 +1200,9 @@ service Audit { message AuditRecord { // Monotonically increasing sequence index. uint64 sequence = 1; - // UTC timestamp in nanoseconds since Unix epoch. - int64 timestamp_ns = 2; + // UTC timestamp, preserving the audit ledger's nanosecond precision in + // the `nanos` field. + google.protobuf.Timestamp timestamp = 2; // Subsystem or actor that performed the audited action. string actor = 3; // Action identifier (e.g. "data.export"). diff --git a/crates/nuncio-proto/src/lib.rs b/crates/nuncio-proto/src/lib.rs index 6709375..4ccefe7 100644 --- a/crates/nuncio-proto/src/lib.rs +++ b/crates/nuncio-proto/src/lib.rs @@ -23,6 +23,7 @@ pub mod addr; pub mod client; pub mod errors; +pub mod time; pub use addr::{grpc_addr_from_env, DEFAULT_GRPC_ADDR, GRPC_ADDR_ENV_VAR}; diff --git a/crates/nuncio-proto/src/time.rs b/crates/nuncio-proto/src/time.rs new file mode 100644 index 0000000..6702ddd --- /dev/null +++ b/crates/nuncio-proto/src/time.rs @@ -0,0 +1,138 @@ +//! Conversions between the store's native integer instants/durations and the +//! well-known `google.protobuf.Timestamp`/`google.protobuf.Duration` wire +//! types used at the `nuncio.v1` proto boundary. +//! +//! The daemon's core/store models keep unix-seconds `i64`/`u64` instants and +//! plain scalar durations; these helpers are the single place that maps them +//! to/from the well-known types so every RPC mapper does it the same way. +//! Re-exporting `Timestamp`/`Duration` here lets callers name the wire types +//! without depending on `prost-types` directly. + +pub use prost_types::{Duration, Timestamp}; + +/// Converts a unix-seconds instant into a `Timestamp` with a zero `nanos` +/// component. Use this for store fields that only ever carry second +/// granularity (e.g. message/event/rule timestamps). +#[must_use] +pub fn timestamp_from_unix_secs(unix_secs: i64) -> Timestamp { + Timestamp { + seconds: unix_secs, + nanos: 0, + } +} + +/// Extracts the unix-seconds instant from a `Timestamp`, truncating any +/// sub-second `nanos` component. Use this for fields mapped via +/// [`timestamp_from_unix_secs`], where `nanos` is always zero. +#[must_use] +pub fn timestamp_to_unix_secs(ts: &Timestamp) -> i64 { + ts.seconds +} + +/// Converts a unix-nanoseconds instant into a `Timestamp`, splitting it +/// losslessly into whole seconds plus a sub-second `nanos` remainder. Use +/// this for fields that carry sub-second precision (e.g. the WORM audit +/// ledger's `timestamp_ns`). +#[must_use] +pub fn timestamp_from_unix_nanos(unix_nanos: i64) -> Timestamp { + let seconds = unix_nanos.div_euclid(1_000_000_000); + let nanos = unix_nanos.rem_euclid(1_000_000_000); + Timestamp { + seconds, + nanos: nanos as i32, + } +} + +/// Recovers the exact unix-nanoseconds instant a `Timestamp` was built from +/// via [`timestamp_from_unix_nanos`]. +#[must_use] +pub fn timestamp_to_unix_nanos(ts: &Timestamp) -> i64 { + ts.seconds + .saturating_mul(1_000_000_000) + .saturating_add(i64::from(ts.nanos)) +} + +/// Converts a whole-second duration into a `Duration` with a zero `nanos` +/// component (e.g. `AccountConfig`'s sync polling interval). +#[must_use] +pub fn duration_from_secs(secs: u64) -> Duration { + Duration { + seconds: secs as i64, + nanos: 0, + } +} + +/// Extracts a whole-second duration from a `Duration`, truncating any +/// sub-second `nanos` component. A negative `seconds` (never produced by +/// this crate's own mappers) reads back as zero rather than panicking. +#[must_use] +pub fn duration_to_secs(d: &Duration) -> u64 { + u64::try_from(d.seconds).unwrap_or(0) +} + +/// Converts a microsecond duration into a `Duration`, splitting it +/// losslessly into whole seconds plus a sub-second `nanos` remainder (e.g. +/// filter rule preview evaluation timing). +#[must_use] +pub fn duration_from_micros(micros: u64) -> Duration { + let seconds = micros / 1_000_000; + let nanos = (micros % 1_000_000) * 1_000; + Duration { + seconds: seconds as i64, + nanos: nanos as i32, + } +} + +/// Recovers the exact microsecond duration a `Duration` was built from via +/// [`duration_from_micros`]. +#[must_use] +pub fn duration_to_micros(d: &Duration) -> u64 { + let seconds = u64::try_from(d.seconds).unwrap_or(0); + let nanos = u64::try_from(d.nanos).unwrap_or(0); + seconds + .saturating_mul(1_000_000) + .saturating_add(nanos / 1_000) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn seconds_round_trip_through_timestamp() { + let ts = timestamp_from_unix_secs(1_700_000_000); + assert_eq!(ts.nanos, 0); + assert_eq!(timestamp_to_unix_secs(&ts), 1_700_000_000); + } + + #[test] + fn nanos_round_trip_through_timestamp() { + let original_ns = 1_700_000_000_123_456_789_i64; + let ts = timestamp_from_unix_nanos(original_ns); + assert_eq!(timestamp_to_unix_nanos(&ts), original_ns); + } + + #[test] + fn negative_epoch_nanos_round_trip() { + // Regression: pre-epoch instants must still split into a + // non-negative `nanos` remainder per the Timestamp wire contract. + let original_ns = -1_500_000_000_i64; + let ts = timestamp_from_unix_nanos(original_ns); + assert!(ts.nanos >= 0); + assert_eq!(timestamp_to_unix_nanos(&ts), original_ns); + } + + #[test] + fn secs_round_trip_through_duration() { + let d = duration_from_secs(300); + assert_eq!(d.nanos, 0); + assert_eq!(duration_to_secs(&d), 300); + } + + #[test] + fn micros_round_trip_through_duration() { + let original_us = 1_234_567_u64; + let d = duration_from_micros(original_us); + assert_eq!(duration_to_micros(&d), original_us); + } +} diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index cb25ab5..7ac4718 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -285,7 +285,9 @@ fn map_account_config_to_proto(config: nuncio_core::AccountConfig) -> AccountCon imap_tls_mode: map_tls_mode_to_proto(config.imap_tls_mode).into(), smtp_tls_mode: map_tls_mode_to_proto(config.smtp_tls_mode).into(), keyring_secret_key: config.keyring_secret_key, - sync_interval_secs: config.sync_interval_secs, + sync_interval: Some(nuncio_proto::time::duration_from_secs( + config.sync_interval_secs, + )), smtp_host: config.smtp_host, smtp_port: u32::from(config.smtp_port), collection_url: config.collection_url, @@ -305,6 +307,9 @@ fn map_account_config_from_proto( .map_err(|_| Status::invalid_argument("server_port must be in range 1..=65535"))?; let smtp_port = u16::try_from(config.smtp_port) .map_err(|_| Status::invalid_argument("smtp_port must be in range 1..=65535"))?; + let sync_interval = config + .sync_interval + .ok_or_else(|| Status::invalid_argument("sync_interval is required"))?; Ok(nuncio_core::AccountConfig { id: config.id, @@ -319,7 +324,7 @@ fn map_account_config_from_proto( imap_tls_mode, smtp_tls_mode, keyring_secret_key: config.keyring_secret_key, - sync_interval_secs: config.sync_interval_secs, + sync_interval_secs: nuncio_proto::time::duration_to_secs(&sync_interval), collection_url: config.collection_url, }) } @@ -777,7 +782,9 @@ fn map_email_to_proto(email: nuncio_core::model::Email) -> MessageProto { subject: email.subject, sender: email.sender, recipient: email.recipient, - received_at: email.received_at, + received_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + email.received_at, + )), read: email.read, body_plain: email.body_plain, body_html: email.body_html, @@ -812,8 +819,10 @@ fn map_calendar_event_to_proto(event: nuncio_core::model::CalendarEvent) -> Cale account_id: event.account_id, calendar_id: event.calendar_id, summary: event.summary, - start_time: event.start_time, - end_time: event.end_time, + start_time: Some(nuncio_proto::time::timestamp_from_unix_secs( + event.start_time, + )), + end_time: Some(nuncio_proto::time::timestamp_from_unix_secs(event.end_time)), rrule: event.rrule, location: event.location, } @@ -858,6 +867,17 @@ fn map_contact_to_proto(contact: nuncio_contacts::Contact) -> ContactProto { } } +/// Extracts a required window bound (`start_window`/`end_window`) from its +/// wire-format `Timestamp`, rejecting an absent field with +/// `Status::invalid_argument` rather than silently defaulting to the epoch. +fn require_window_bound( + ts: Option, + field: &str, +) -> Result { + let ts = ts.ok_or_else(|| Status::invalid_argument(format!("{field} is required")))?; + Ok(nuncio_proto::time::timestamp_to_unix_secs(&ts)) +} + /// Combines an account's in-window events with its recurring masters into the final event /// list for a `ListEvents` query window, expanding each recurring master into its real /// occurrences via [`nuncio_cal::RecurrenceEngine::expand_occurrences`] rather than trusting @@ -956,6 +976,8 @@ impl CalendarGrpcService { async fn sync_configured_caldav_accounts( &self, req: &CalendarSyncRequest, + start_window: i64, + end_window: i64, ) -> Result { let caldav_accounts: Vec = self .db @@ -1004,8 +1026,8 @@ impl CalendarGrpcService { &account, &password, &req.calendar_id, - req.start_window, - req.end_window, + start_window, + end_window, ) .await .map_err(|e| { @@ -1027,18 +1049,23 @@ impl Calendar for CalendarGrpcService { request: Request, ) -> Result, Status> { let req = request.into_inner(); + let start_window = require_window_bound(req.start_window, "start_window")?; + let end_window = require_window_bound(req.end_window, "end_window")?; let synced = match &self.overrides.calendar_backend { Some(backend) => crate::calendar_sync::sync_with_backend( &self.db, backend.as_ref(), &req.calendar_id, - req.start_window, - req.end_window, + start_window, + end_window, ) .await .map_err(|e| Status::internal(format!("calendar sync failed: {e}")))?, - None => self.sync_configured_caldav_accounts(&req).await?, + None => { + self.sync_configured_caldav_accounts(&req, start_window, end_window) + .await? + } }; Ok(Response::new(CalendarSyncResponse { @@ -1054,10 +1081,12 @@ impl Calendar for CalendarGrpcService { if req.account_id.is_empty() { return Err(Status::invalid_argument("account_id is required")); } + let start_window = require_window_bound(req.start_window, "start_window")?; + let end_window = require_window_bound(req.end_window, "end_window")?; let all_in_window = self .db - .list_calendar_events(&req.account_id, req.start_window, req.end_window) + .list_calendar_events(&req.account_id, start_window, end_window) .await .map_err(|e| Status::internal(format!("failed to list calendar events: {e}")))?; @@ -1068,7 +1097,7 @@ impl Calendar for CalendarGrpcService { .map_err(|e| Status::internal(format!("failed to list recurring events: {e}")))?; let mut events = - expand_events_for_window(all_in_window, recurring, req.start_window, req.end_window); + expand_events_for_window(all_in_window, recurring, start_window, end_window); events.retain(|e| req.calendar_id.is_empty() || e.calendar_id == req.calendar_id); @@ -1574,8 +1603,12 @@ fn map_filter_rule_to_proto(rule: nuncio_filter::FilterRule) -> FilterRuleProto enabled: rule.enabled, nsql_text: rule.nsql_text, actions: rule.actions.iter().map(|a| a.to_nsql()).collect(), - created_at: rule.created_at, - updated_at: rule.updated_at, + created_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + rule.created_at, + )), + updated_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + rule.updated_at, + )), } } @@ -1589,7 +1622,7 @@ fn map_filter_execution_log_to_proto( rule_id: log.rule_id, message_id: log.message_id, action_taken: log.action_taken, - matched_at: log.matched_at, + matched_at: Some(nuncio_proto::time::timestamp_from_unix_secs(log.matched_at)), prev_hash: log.prev_hash, hash: log.hash, } @@ -1608,7 +1641,9 @@ fn map_preview_result_to_proto(preview: nuncio_filter::FilterPreviewResult) -> P .iter() .map(|a| a.to_nsql()) .collect(), - execution_time_us: preview.execution_time_us, + execution_time: Some(nuncio_proto::time::duration_from_micros( + preview.execution_time_us, + )), condition_traces: preview.condition_traces, } } @@ -2152,7 +2187,9 @@ const DEFAULT_LIST_RECORDS_LIMIT: u32 = 100; fn map_audit_record_to_proto(record: nuncio_core::WormAuditRecord) -> AuditRecordProto { AuditRecordProto { sequence: record.sequence, - timestamp_ns: record.timestamp_ns, + timestamp: Some(nuncio_proto::time::timestamp_from_unix_nanos( + record.timestamp_ns, + )), actor: record.actor, action: record.action, data_hash: record.data_hash, @@ -2575,6 +2612,73 @@ mod tests { use nuncio_proto::v1::system_client::SystemClient; use tonic::Code; + /// Proves a stored second-granularity instant survives the proto + /// boundary exactly: `map_email_to_proto`'s `received_at` decodes back + /// to the identical unix-seconds value via `timestamp_to_unix_secs`. + #[test] + fn map_email_to_proto_round_trips_received_at_exactly() { + let original_secs = 1_700_000_123_i64; + let email = nuncio_core::model::Email { + id: "msg-1".to_string(), + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + subject: "s".to_string(), + sender: "a@b.com".to_string(), + recipient: "c@d.com".to_string(), + received_at: original_secs, + read: false, + body_plain: None, + body_html: None, + attachments: Vec::new(), + remote_id: "remote-1".to_string(), + uid_validity: "1".to_string(), + }; + + let proto = map_email_to_proto(email); + let ts = proto.received_at.expect("received_at is always populated"); + assert_eq!( + nuncio_proto::time::timestamp_to_unix_secs(&ts), + original_secs + ); + } + + /// Proves the WORM audit ledger's sub-second precision survives the + /// proto boundary losslessly: `map_audit_record_to_proto`'s `timestamp` + /// decodes back to the exact original `timestamp_ns`, not just the + /// truncated seconds component. + #[test] + fn map_audit_record_to_proto_preserves_nanosecond_precision() { + let original_ns = 1_700_000_123_987_654_321_i64; + let record = nuncio_core::WormAuditRecord { + sequence: 1, + timestamp_ns: original_ns, + actor: "system".to_string(), + action: "test.action".to_string(), + data_hash: "hash".to_string(), + previous_block_hash: "GENESIS".to_string(), + record_hmac: "mac".to_string(), + }; + + let proto = map_audit_record_to_proto(record); + let ts = proto.timestamp.expect("timestamp is always populated"); + assert_eq!( + nuncio_proto::time::timestamp_to_unix_nanos(&ts), + original_ns + ); + } + + /// Proves an account's sync interval survives the proto `Duration` + /// boundary in both directions: to-proto then from-proto reconstructs + /// the exact original whole-second value. + #[test] + fn account_sync_interval_round_trips_through_duration() { + let mut config = sample_account_config_proto("acct-rt", "nuncio/acct-rt"); + config.sync_interval = Some(nuncio_proto::time::duration_from_secs(600)); + + let core_config = map_account_config_from_proto(config).expect("valid config maps back"); + assert_eq!(core_config.sync_interval_secs, 600); + } + /// Spawns a test server backed by a fresh ephemeral database, a fresh /// empty [`FilterEngine`], and a fresh [`SecretManager::mock`] vault -- /// the right default for every test that only exercises @@ -3137,7 +3241,7 @@ mod tests { imap_tls_mode: TlsModeProto::ImplicitTls.into(), smtp_tls_mode: TlsModeProto::ImplicitTls.into(), keyring_secret_key: keyring_secret_key.to_string(), - sync_interval_secs: 60, + sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), collection_url: String::new(), smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, @@ -3708,8 +3812,8 @@ mod tests { .list_events(ListEventsRequest { account_id: "acct-1".to_string(), calendar_id: "cal-1".to_string(), - start_window: 0, - end_window: i64::MAX, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), }) .await .expect_err("missing bearer token must be rejected"); @@ -3727,8 +3831,8 @@ mod tests { .sync(CalendarSyncRequest { account_id: "acct-1".to_string(), calendar_id: "cal-1".to_string(), - start_window: 0, - end_window: i64::MAX, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), }) .await .expect_err("missing bearer token must be rejected"); @@ -5539,8 +5643,8 @@ mod tests { .list_events(authed_bearer_request(ListEventsRequest { account_id: "acct-cal-1".to_string(), calendar_id: "cal-work".to_string(), - start_window: 1_699_999_000, - end_window: 1_700_004_000, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_699_999_000)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_004_000)), })) .await .expect("list_events succeeds") @@ -5598,8 +5702,8 @@ mod tests { .list_events(authed_bearer_request(ListEventsRequest { account_id: "acct-cal-1".to_string(), calendar_id: "cal-work".to_string(), - start_window: 1_699_999_000, - end_window: 1_700_004_000, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_699_999_000)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_004_000)), })) .await .expect("list_events succeeds") @@ -5624,8 +5728,8 @@ mod tests { .list_events(authed_bearer_request(ListEventsRequest { account_id: String::new(), calendar_id: "cal-work".to_string(), - start_window: 0, - end_window: i64::MAX, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), })) .await .expect_err("empty account_id must be rejected"); @@ -5717,8 +5821,8 @@ mod tests { .sync(authed_bearer_request(CalendarSyncRequest { account_id: "acct-cal-no-config".to_string(), calendar_id: "cal-work".to_string(), - start_window: 0, - end_window: i64::MAX, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), })) .await .expect_err("sync with no injected backend and no CalDAV config must fail"); @@ -5775,8 +5879,8 @@ mod tests { .sync(authed_bearer_request(CalendarSyncRequest { account_id: "acct-cal-injected".to_string(), calendar_id: "cal-work".to_string(), - start_window: 1_699_999_000, - end_window: 1_700_004_000, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_699_999_000)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_004_000)), })) .await .expect("sync succeeds") @@ -5788,8 +5892,8 @@ mod tests { .list_events(authed_bearer_request(ListEventsRequest { account_id: "acct-cal-injected".to_string(), calendar_id: "cal-work".to_string(), - start_window: 1_699_999_000, - end_window: 1_700_004_000, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_699_999_000)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_004_000)), })) .await .expect("list_events succeeds") @@ -6296,8 +6400,8 @@ mod tests { .list_events(ListEventsRequest { account_id: "acct-1".to_string(), calendar_id: "cal-1".to_string(), - start_window: 0, - end_window: i64::MAX, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), }) .await .expect_err("Calendar must reject an unauthenticated call"); diff --git a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs index 10c5b42..8da6b0d 100644 --- a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs +++ b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs @@ -86,7 +86,7 @@ fn sample_account_config() -> AccountConfig { imap_tls_mode: TlsMode::StartTls.into(), smtp_tls_mode: TlsMode::Plain.into(), keyring_secret_key: KEYRING_KEY.to_string(), - sync_interval_secs: 60, + sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), smtp_host: "127.0.0.1".to_string(), smtp_port: 25, collection_url: String::new(), diff --git a/crates/nunciod/tests/caldav_production_e2e_test.rs b/crates/nunciod/tests/caldav_production_e2e_test.rs index dd0b89e..51a9d1f 100644 --- a/crates/nunciod/tests/caldav_production_e2e_test.rs +++ b/crates/nunciod/tests/caldav_production_e2e_test.rs @@ -165,7 +165,7 @@ async fn caldav_production_sync_builds_real_client_and_persists_events() { imap_tls_mode: TlsModeProto::ImplicitTls.into(), smtp_tls_mode: TlsModeProto::ImplicitTls.into(), keyring_secret_key: KEYRING_KEY.to_string(), - sync_interval_secs: 300, + sync_interval: Some(nuncio_proto::time::duration_from_secs(300)), smtp_host: String::new(), smtp_port: 0, collection_url: collection_url.clone(), @@ -183,8 +183,8 @@ async fn caldav_production_sync_builds_real_client_and_persists_events() { .sync(CalendarSyncRequest { account_id: ACCOUNT_ID.to_string(), calendar_id: CALENDAR_ID.to_string(), - start_window: 1_700_000_000, - end_window: 1_800_000_000, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_000_000)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_800_000_000)), }) .await .expect("production CalDAV sync succeeds") @@ -196,8 +196,8 @@ async fn caldav_production_sync_builds_real_client_and_persists_events() { .list_events(ListEventsRequest { account_id: ACCOUNT_ID.to_string(), calendar_id: CALENDAR_ID.to_string(), - start_window: 1_700_000_000, - end_window: 1_800_000_000, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_000_000)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_800_000_000)), }) .await .expect("list_events succeeds") @@ -216,8 +216,8 @@ async fn caldav_production_sync_builds_real_client_and_persists_events() { .sync(CalendarSyncRequest { account_id: "acct-does-not-exist".to_string(), calendar_id: CALENDAR_ID.to_string(), - start_window: 1_700_000_000, - end_window: 1_800_000_000, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_000_000)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_800_000_000)), }) .await .expect_err("a sync targeting an unconfigured account must fail honestly"); @@ -232,8 +232,8 @@ async fn caldav_production_sync_builds_real_client_and_persists_events() { .list_events(ListEventsRequest { account_id: ACCOUNT_ID.to_string(), calendar_id: CALENDAR_ID.to_string(), - start_window: 0, - end_window: i64::MAX, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), }) .await .expect_err("missing bearer token must be rejected"); diff --git a/crates/nunciod/tests/calendar_e2e_test.rs b/crates/nunciod/tests/calendar_e2e_test.rs index 72b534d..3c3ea63 100644 --- a/crates/nunciod/tests/calendar_e2e_test.rs +++ b/crates/nunciod/tests/calendar_e2e_test.rs @@ -138,8 +138,8 @@ async fn calendar_sync_then_list_and_get_round_trip_over_authenticated_grpc() { .sync(CalendarSyncRequest { account_id: ACCOUNT_ID.to_string(), calendar_id: CALENDAR_ID.to_string(), - start_window: 1_699_999_000, - end_window: 1_700_020_000, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_699_999_000)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_020_000)), }) .await .expect("sync succeeds") @@ -152,8 +152,8 @@ async fn calendar_sync_then_list_and_get_round_trip_over_authenticated_grpc() { .list_events(ListEventsRequest { account_id: ACCOUNT_ID.to_string(), calendar_id: CALENDAR_ID.to_string(), - start_window: 1_699_999_000, - end_window: 1_700_020_000, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_699_999_000)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_020_000)), }) .await .expect("list_events succeeds") @@ -187,8 +187,8 @@ async fn calendar_sync_then_list_and_get_round_trip_over_authenticated_grpc() { .list_events(ListEventsRequest { account_id: ACCOUNT_ID.to_string(), calendar_id: CALENDAR_ID.to_string(), - start_window: 0, - end_window: i64::MAX, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), }) .await .expect_err("missing bearer token must be rejected"); diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index 187ef0d..42f786c 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -65,7 +65,7 @@ fn sample_account_config() -> AccountConfig { imap_tls_mode: TlsMode::ImplicitTls.into(), smtp_tls_mode: TlsMode::ImplicitTls.into(), keyring_secret_key: KEYRING_KEY.to_string(), - sync_interval_secs: 60, + sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, collection_url: String::new(), diff --git a/crates/nunciod/tests/typed_errors_e2e_test.rs b/crates/nunciod/tests/typed_errors_e2e_test.rs index 436f49c..6e36958 100644 --- a/crates/nunciod/tests/typed_errors_e2e_test.rs +++ b/crates/nunciod/tests/typed_errors_e2e_test.rs @@ -140,8 +140,8 @@ async fn typed_error_reasons_decode_from_status_details_over_the_wire() { .sync(CalendarSyncRequest { account_id: String::new(), calendar_id: "personal".to_string(), - start_window: 0, - end_window: 0, + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), }) .await .expect_err("syncing with no CalDAV account configured must fail"); -- 2.54.0 From 3687bc3f0816ca46304c7de8bbc02d4b55e417c1 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 16:31:35 -0500 Subject: [PATCH 28/95] feat(nunciod): bound outbox per-item execution with a timeout (#349) Wrap each outbox mutation's execution in a per-item timeout raced against the shutdown signal, so one hung remote op (a stuck IMAP/JMAP call or webhook) can no longer freeze delivery for every other pending mutation across every account. A timed-out item is dispositioned exactly like any other transient failure: retryable, bumping retry_count, never completed. Refs #290 Co-authored-by: James Maes --- crates/nunciod/Cargo.toml | 1 + crates/nunciod/src/main.rs | 11 +- crates/nunciod/src/outbox.rs | 42 ++- .../nunciod/tests/outbox_executor_e2e_test.rs | 297 +++++++++++++++++- 4 files changed, 338 insertions(+), 13 deletions(-) diff --git a/crates/nunciod/Cargo.toml b/crates/nunciod/Cargo.toml index d1ee17f..060404f 100644 --- a/crates/nunciod/Cargo.toml +++ b/crates/nunciod/Cargo.toml @@ -40,3 +40,4 @@ fs4.workspace = true nuncio-cli = { path = "../nuncio-cli" } sqlx.workspace = true wiremock.workspace = true +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/nunciod/src/main.rs b/crates/nunciod/src/main.rs index 29bb348..fcdc090 100644 --- a/crates/nunciod/src/main.rs +++ b/crates/nunciod/src/main.rs @@ -153,6 +153,7 @@ async fn main() -> Result<(), Box> { account_secrets.clone(), )); let mut outbox_shutdown = shutdown_signal.clone(); + let mut outbox_drain_shutdown = shutdown_signal.clone(); let outbox_task = tokio::spawn(async move { let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); loop { @@ -162,13 +163,17 @@ async fn main() -> Result<(), Box> { break; } _ = interval.tick() => { - // Runs to completion once selected: a drain pass already - // in progress is never aborted mid-write by a shutdown - // signal that arrives after this branch was chosen. + // A drain pass already in progress is never aborted + // mid-write by a shutdown signal that arrives after this + // branch was chosen -- but `execute_pending_mutations` + // itself races each item's execution against + // `outbox_drain_shutdown`, so a hung remote op cannot + // block this pass (and therefore shutdown) indefinitely. let summary = nunciod::outbox::execute_pending_mutations( &db_outbox, outbox_env.as_ref(), 50, + &mut outbox_drain_shutdown, ) .await; if summary.completed > 0 || summary.failed > 0 { diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 062c38a..0d5bb8b 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -16,7 +16,9 @@ //! exceeded, at which point it flips to `failed`. A permanent failure (the //! message no longer exists, a malformed payload, an unknown action) flips to //! `failed` immediately -- there is nothing a retry could fix. Success is -//! never fabricated. +//! never fabricated. A mutation whose execution exceeds +//! [`PER_ITEM_EXECUTION_TIMEOUT`] is dispositioned the same way as any other +//! transient failure -- it is RETRYABLE, never `completed`. //! //! # Testability //! @@ -26,6 +28,7 @@ //! without any live network. Production wires [`ProductionExecutionEnv`], //! which resolves real per-account engines from the store and the OS keyring. +use crate::lifecycle::ShutdownSignal; use async_trait::async_trait; use nuncio_core::AccountConfig; use nuncio_filter::{ @@ -39,8 +42,18 @@ use nuncio_mail::{ use nuncio_store::db::{DatabaseEngine, DatabaseError}; use nuncio_store::vault::{SecretManager, VaultError, WEBHOOK_SIGNING_KEY_ACCOUNT}; use std::sync::Arc; +use std::time::Duration; use thiserror::Error; +/// Bound on how long a single outbox mutation may run before it is treated +/// as a retryable timeout. A healthy IMAP/JMAP/SMTP round trip or webhook +/// POST comfortably finishes well inside this window; a value this generous +/// still keeps one hung remote op from stalling every OTHER pending +/// mutation -- across every account -- behind it in the same drain pass, and +/// from blocking graceful shutdown for longer than the shutdown grace +/// period tolerates. +const PER_ITEM_EXECUTION_TIMEOUT: Duration = Duration::from_secs(30); + /// Errors that can occur while resolving the resources needed to execute a /// single outbox mutation. These are surfaced from a [`RemoteExecutionEnv`] /// and treated as transient (retryable) by the executor unless it has already @@ -119,11 +132,18 @@ pub struct ExecutionSummary { /// Drain up to `limit` pending mutations, executing each against `env` and /// persisting the honest outcome. Never marks a mutation `completed` unless the -/// real operation succeeded. Returns a tally of the pass. +/// real operation succeeded. Each item's execution is bounded by +/// [`PER_ITEM_EXECUTION_TIMEOUT`] so one hung remote op cannot freeze delivery +/// for every other item behind it; `shutdown` is raced against that same +/// timeout so a hung item cannot delay graceful shutdown either -- on +/// shutdown the drain pass stops immediately, leaving the in-flight item (and +/// anything still queued behind it) untouched and `pending` for the next +/// start. Returns a tally of the pass. pub async fn execute_pending_mutations( db: &DatabaseEngine, env: &dyn RemoteExecutionEnv, limit: usize, + shutdown: &mut ShutdownSignal, ) -> ExecutionSummary { let pending = match db.list_pending_mutations(limit).await { Ok(items) => items, @@ -135,7 +155,23 @@ pub async fn execute_pending_mutations( let mut summary = ExecutionSummary::default(); for item in pending { - let disposition = execute_one(db, env, &item).await; + let outcome = tokio::select! { + biased; + _ = shutdown.wait() => { + tracing::info!( + mutation_id = %item.id, + "outbox: drain pass interrupted by shutdown signal; item left pending" + ); + break; + } + result = tokio::time::timeout(PER_ITEM_EXECUTION_TIMEOUT, execute_one(db, env, &item)) => result, + }; + let disposition = match outcome { + Ok(disposition) => disposition, + Err(_elapsed) => Disposition::Retry(format!( + "mutation execution exceeded the {PER_ITEM_EXECUTION_TIMEOUT:?} timeout" + )), + }; let next_retry = item.retry_count + 1; let (status, retry, tally): (&str, i32, fn(&mut ExecutionSummary)) = match disposition { Disposition::Completed => ("completed", item.retry_count, |s| s.completed += 1), diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index d88509a..d6fd97f 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -13,16 +13,28 @@ use async_trait::async_trait; use nuncio_core::model::Email; +use nuncio_core::EventBus; use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher, WebhookError}; use nuncio_mail::{ MailBackend, MessageSender, MockMailBackend, MockMessageSender, RemoteMutationKind, }; use nuncio_store::db::DatabaseEngine; +use nunciod::lifecycle::{ShutdownController, ShutdownSignal}; use nunciod::outbox::{execute_pending_mutations, RemoteExecutionEnv}; use nunciod::sync::apply_filter_actions; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; +/// A shutdown handle for tests that never trigger it. Holding the returned +/// `ShutdownController` alive for the duration of the test matters: dropping +/// it closes the underlying `watch` channel, which would make `wait()` +/// resolve immediately and be mistaken for a real shutdown request. +fn no_shutdown() -> (ShutdownController, ShutdownSignal) { + ShutdownController::new(Arc::new(EventBus::new())) +} + const ACCOUNT_ID: &str = "acct-outbox-e2e"; const FOLDER_ID: &str = "INBOX"; const CHECKPOINT: &str = "1:105"; @@ -145,6 +157,54 @@ async fn seed_and_enqueue(db: &DatabaseEngine, rule_nsql: &str, subject: &str) { ); } +/// Like [`seed_and_enqueue`], but for a message with its own surrogate id so +/// several calls in the same test enqueue mutations against DISTINCT +/// messages instead of colliding on the shared fixed id `sample_email` +/// otherwise defaults to. +async fn seed_and_enqueue_distinct( + db: &DatabaseEngine, + message_id: &str, + rule_nsql: &str, + subject: &str, +) { + db.save_account(&sample_account()) + .await + .expect("save account"); + db.save_folder_sync_state(ACCOUNT_ID, FOLDER_ID, CHECKPOINT) + .await + .expect("save folder checkpoint"); + + // `remote_id`/`uid_validity` (not just `id`) are part of the store's + // `messages` identity UNIQUE index -- leaving them at `sample_email`'s + // shared default would make this INSERT OR REPLACE the OTHER seeded + // message sharing that same remote identity out from under it. + let mut email = sample_email(subject); + email.id = message_id.to_string(); + email.remote_id = message_id.to_string(); + db.save_email(&email).await.expect("save email"); + + let before = db + .list_pending_mutations(100) + .await + .expect("list pending") + .len(); + + let rule = NsqlParser::parse_rule("Outbox Rule", 1, rule_nsql).expect("parse rule"); + let engine = FilterEngine::new(vec![rule]).expect("compile rule"); + apply_filter_actions(db, &engine, &email).await; + + let after = db + .list_pending_mutations(100) + .await + .expect("list pending") + .len(); + assert_eq!( + after, + before + 1, + "the rule must enqueue exactly one new mutation" + ); +} + #[tokio::test] async fn move_rule_executes_against_backend_and_completes_with_uidvalidity_checkpoint() { let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); @@ -156,7 +216,8 @@ async fn move_rule_executes_against_backend_and_completes_with_uidvalidity_check .await; let env = MockEnv::new(); - let summary = execute_pending_mutations(&db, &env, 50).await; + let (_shutdown_ctrl, mut shutdown) = no_shutdown(); + let summary = execute_pending_mutations(&db, &env, 50, &mut shutdown).await; assert_eq!(summary.completed, 1); assert_eq!(summary.failed, 0); @@ -192,7 +253,8 @@ async fn flag_rule_executes_as_set_flagged() { .await; let env = MockEnv::new(); - let summary = execute_pending_mutations(&db, &env, 50).await; + let (_shutdown_ctrl, mut shutdown) = no_shutdown(); + let summary = execute_pending_mutations(&db, &env, 50, &mut shutdown).await; assert_eq!(summary.completed, 1); let applied = env.backend.applied_mutations(); @@ -214,7 +276,8 @@ async fn delete_rule_executes_as_delete() { .await; let env = MockEnv::new(); - let summary = execute_pending_mutations(&db, &env, 50).await; + let (_shutdown_ctrl, mut shutdown) = no_shutdown(); + let summary = execute_pending_mutations(&db, &env, 50, &mut shutdown).await; assert_eq!(summary.completed, 1); let applied = env.backend.applied_mutations(); @@ -234,7 +297,8 @@ async fn a_failing_backend_op_is_never_marked_completed() { let env = MockEnv::new(); env.backend.set_should_fail(true); - let summary = execute_pending_mutations(&db, &env, 50).await; + let (_shutdown_ctrl, mut shutdown) = no_shutdown(); + let summary = execute_pending_mutations(&db, &env, 50, &mut shutdown).await; // The op failed: nothing recorded, nothing completed, and the item is kept // for retry (still pending, with a bumped retry count) -- never faked. @@ -258,7 +322,8 @@ async fn forward_rule_sends_via_the_message_sender() { .await; let env = MockEnv::new(); - let summary = execute_pending_mutations(&db, &env, 50).await; + let (_shutdown_ctrl, mut shutdown) = no_shutdown(); + let summary = execute_pending_mutations(&db, &env, 50, &mut shutdown).await; assert_eq!(summary.completed, 1); let sent = env.sender.sent_messages(); @@ -293,7 +358,8 @@ async fn webhook_rule_dispatches_a_signed_post_to_the_receiver() { .await; let env = MockEnv::new(); - let summary = execute_pending_mutations(&db, &env, 50).await; + let (_shutdown_ctrl, mut shutdown) = no_shutdown(); + let summary = execute_pending_mutations(&db, &env, 50, &mut shutdown).await; // Completion is genuine: wiremock's `.expect(1)` verifies the POST actually // arrived when the server drops. @@ -320,7 +386,8 @@ async fn a_mutation_for_a_vanished_message_fails_permanently() { .expect("save mutation"); let env = MockEnv::new(); - let summary = execute_pending_mutations(&db, &env, 50).await; + let (_shutdown_ctrl, mut shutdown) = no_shutdown(); + let summary = execute_pending_mutations(&db, &env, 50, &mut shutdown).await; // Cannot act on a message that no longer exists -- fail honestly, never // fabricate success, and never touch the backend. @@ -332,3 +399,219 @@ async fn a_mutation_for_a_vanished_message_fails_permanently() { "a permanently-failed mutation is no longer pending" ); } + +/// A [`RemoteExecutionEnv`] whose `mail_backend` call hangs (a `tokio::time` +/// sleep far longer than any timeout under test, standing in for a stuck +/// IMAP/JMAP op) for the first `hang_calls` invocations, then delegates to a +/// normal [`MockEnv`]. Used to prove the per-item execution timeout fires. +struct HangEnv { + inner: MockEnv, + calls: AtomicUsize, + hang_calls: usize, +} + +impl HangEnv { + fn new(hang_calls: usize) -> Self { + Self { + inner: MockEnv::new(), + calls: AtomicUsize::new(0), + hang_calls, + } + } +} + +#[async_trait] +impl RemoteExecutionEnv for HangEnv { + async fn mail_backend( + &self, + account_id: &str, + ) -> Result, nunciod::outbox::OutboxExecuteError> { + let call = self.calls.fetch_add(1, Ordering::SeqCst); + if call < self.hang_calls { + // A `tokio::time`-tracked sleep far longer than any per-item + // timeout under test -- NOT `std::future::pending`, which never + // registers with the time driver and so behaves unpredictably + // once `tokio::time::pause`'s auto-advance-on-idle interacts with + // real (non-timer) I/O elsewhere in the same test, like the + // SQLite pool's own connection bookkeeping. A real (virtual) + // sleep is what the timeout under test is meant to race against. + tokio::time::sleep(std::time::Duration::from_secs(3600)).await; + } + self.inner.mail_backend(account_id).await + } + + async fn message_sender( + &self, + account_id: &str, + ) -> Result, nunciod::outbox::OutboxExecuteError> { + self.inner.message_sender(account_id).await + } + + async fn dispatch_webhook( + &self, + url: &str, + rule_id: &str, + message_id: &str, + subject: &str, + sender: &str, + ) -> Result { + self.inner + .dispatch_webhook(url, rule_id, message_id, subject, sender) + .await + } +} + +/// Seed two mutations so the second proves the drain pass actually moves on +/// after the first times out, rather than the whole pass having stalled. +async fn seed_two_move_mutations(db: &DatabaseEngine) { + seed_and_enqueue_distinct( + db, + "surrogate-hang-1", + "WHERE subject CONTAINS 'Archive' ACTION MOVE TO 'Archive'", + "Please Archive this one", + ) + .await; + seed_and_enqueue_distinct( + db, + "surrogate-hang-2", + "WHERE subject CONTAINS 'Trash' ACTION DELETE", + "Please Trash this one", + ) + .await; +} + +#[tokio::test] +async fn a_hung_mutation_times_out_and_is_retried_never_completed() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); + seed_and_enqueue( + &db, + "WHERE subject CONTAINS 'Archive' ACTION MOVE TO 'Archive'", + "Please Archive", + ) + .await; + + // Paused only AFTER setup, and this pass touches the database exactly + // ONCE (a single `get_message`) before the hang -- mixing + // `tokio::time::pause`'s auto-advance-on-idle with real (non-timer) + // background I/O is only safe when there is a single, already-in-flight + // real operation for the auto-advance to race past; see the sibling + // two-pass test below for why a SECOND real DB operation in the same + // paused pass is deliberately avoided. + tokio::time::pause(); + let env = HangEnv::new(usize::MAX); + let (_shutdown_ctrl, mut shutdown) = no_shutdown(); + let summary = execute_pending_mutations(&db, &env, 10, &mut shutdown).await; + // Resume real time before any further DB access: `tokio::time::pause`'s + // auto-advance-on-idle only plays safely with the ONE real DB operation + // this pass performs (the `get_message` before the hang); a second real + // query made while still paused is what made the sibling two-pass test + // flaky before it was split across a resume point. + tokio::time::resume(); + + // The hung item is NEVER marked completed -- it is a retryable timeout, + // exactly like any other transient remote failure. + assert_eq!(summary.completed, 0); + assert_eq!( + summary.retried, 1, + "the hung item must be retried, not lost" + ); + assert_eq!(summary.failed, 0); + + let pending = db.list_pending_mutations(10).await.unwrap(); + assert_eq!(pending.len(), 1, "the hung item stays pending for a retry"); + assert_eq!( + pending[0].retry_count, 1, + "the timeout bumps the retry count exactly like any transient failure" + ); + assert!( + env.inner.backend.applied_mutations().is_empty(), + "the hung item never genuinely reached the backend" + ); +} + +#[tokio::test] +async fn a_timed_out_item_does_not_block_a_later_item_in_a_subsequent_pass() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); + seed_two_move_mutations(&db).await; + + // First pass: bound to the FIRST (created-earliest) mutation only, so + // this pass's real DB work is a single `get_message` call before the + // hang, same as the sibling single-item timeout test. Real threaded + // SQLite I/O racing `tokio::time::pause`'s auto-advance is only + // exercised once per paused pass here -- see that test's comment for + // why a second real DB call in the SAME paused pass is unreliable. + tokio::time::pause(); + let env = HangEnv::new(usize::MAX); + let (_shutdown_ctrl, mut shutdown) = no_shutdown(); + let first_pass = execute_pending_mutations(&db, &env, 1, &mut shutdown).await; + assert_eq!(first_pass.completed, 0); + assert_eq!(first_pass.retried, 1, "the hung item must be retried"); + + // Second pass runs under ordinary real time against a normal + // (non-hanging) env, standing in for the outbox worker's next 5-second + // tick: the item that timed out in the first pass did not jam the queue + // -- both it (now against a healthy backend) and the item queued behind + // it are processed and genuinely complete. + tokio::time::resume(); + let normal_env = MockEnv::new(); + let (_shutdown_ctrl2, mut shutdown2) = no_shutdown(); + let second_pass = execute_pending_mutations(&db, &normal_env, 10, &mut shutdown2).await; + assert_eq!( + second_pass.completed, 2, + "both the previously-timed-out item and the one behind it complete" + ); + assert_eq!(second_pass.failed, 0); + + let applied = normal_env.backend.applied_mutations(); + assert_eq!( + applied.len(), + 2, + "both mutations genuinely reached the backend" + ); + + // Neither pass ever fabricated a completion: the queue is now empty + // because both items genuinely succeeded, not because either was + // dropped or silently marked done while still hung. + assert!(db.list_pending_mutations(10).await.unwrap().is_empty()); +} + +#[tokio::test] +async fn shutdown_signal_interrupts_a_hung_item_instead_of_waiting_out_the_timeout() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); + seed_and_enqueue( + &db, + "WHERE subject CONTAINS 'Archive' ACTION MOVE TO 'Archive'", + "Please Archive", + ) + .await; + + // Paused only after setup -- see the comment on the sibling timeout test + // for why pausing from the start would spuriously fail the DB pool. + tokio::time::pause(); + + // Hangs on every call -- the shutdown signal, not the timeout, must be + // what unblocks this drain pass. + let env = HangEnv::new(usize::MAX); + let (controller, mut shutdown) = ShutdownController::new(Arc::new(EventBus::new())); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + controller.trigger(); + }); + + let summary = execute_pending_mutations(&db, &env, 10, &mut shutdown).await; + tokio::time::resume(); + + // The drain pass stopped on shutdown, well before the per-item timeout + // would have elapsed -- the hung item is untouched: not completed, not + // even retried, since its true disposition is still unknown. + assert_eq!(summary.completed, 0); + assert_eq!(summary.retried, 0); + assert_eq!(summary.failed, 0); + + let pending = db.list_pending_mutations(10).await.unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!( + pending[0].retry_count, 0, + "shutdown must leave the interrupted item's disposition untouched" + ); +} -- 2.54.0 From 834f6a8ddc40ca371103834354df6d14d19c0c82 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 16:42:25 -0500 Subject: [PATCH 29/95] feat(proto): keyset pagination for all list/search RPCs (#350) Apply one AIP-158 keyset pagination convention (page_size/page_token/ next_page_token, opaque cursor, not offset) to every list/search RPC: ListMessages, ListFolders, ListRules, ListEvents, ListContacts, SearchMessages, and reconcile Audit.ListRecords off its unstable limit/offset. A malformed page_token returns a typed VALIDATION_FAILED ErrorInfo (InvalidArgument). Regenerate the nuncio.v1 golden descriptor. Refs #275 Co-authored-by: James Maes --- crates/nuncio-cli/src/args.rs | 28 +- crates/nuncio-cli/src/runner.rs | 600 +++++++++++------- .../proto/nuncio/v1/descriptor.bin | Bin 14437 -> 15069 bytes .../nuncio-proto/proto/nuncio/v1/nuncio.proto | 139 +++- crates/nuncio-store/src/db.rs | 327 ++++++++++ crates/nuncio-store/src/search.rs | 111 ++++ crates/nunciod/src/grpc.rs | 521 ++++++++++++--- crates/nunciod/src/lib.rs | 1 + crates/nunciod/src/pagination.rs | 168 +++++ .../tests/caldav_production_e2e_test.rs | 4 + crates/nunciod/tests/calendar_e2e_test.rs | 4 + crates/nunciod/tests/contacts_e2e_test.rs | 6 + crates/nunciod/tests/spine_e2e_test.rs | 3 +- 13 files changed, 1536 insertions(+), 376 deletions(-) create mode 100644 crates/nunciod/src/pagination.rs diff --git a/crates/nuncio-cli/src/args.rs b/crates/nuncio-cli/src/args.rs index 554f691..4994093 100644 --- a/crates/nuncio-cli/src/args.rs +++ b/crates/nuncio-cli/src/args.rs @@ -449,14 +449,13 @@ pub enum SystemSubcommand { #[derive(Subcommand, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum AuditSubcommand { - /// List persisted WORM audit ledger records, sequence ascending. + /// List persisted WORM audit ledger records, sequence ascending. Follows + /// keyset pagination to the end, fetching `page_size` records per request. List { - /// Max records to fetch (0 = server default). - #[arg(short, long, default_value_t = 50, help = "Max records to fetch")] - limit: u32, - /// Pagination offset. - #[arg(short, long, default_value_t = 0, help = "Pagination offset")] - offset: u32, + /// Records fetched per page (0 = server default). All pages are + /// followed via the keyset cursor and printed together. + #[arg(short, long, default_value_t = 50, help = "Records fetched per page")] + page_size: u32, }, /// Verify the WORM audit ledger's cryptographic HMAC hash-chain /// integrity. @@ -716,25 +715,18 @@ mod tests { cli_list.command, Commands::System { action: SystemSubcommand::Audit { - action: AuditSubcommand::List { - limit: 50, - offset: 0, - } + action: AuditSubcommand::List { page_size: 50 } } } ); - let cli_list_paged = Cli::parse_from([ - "nuncio", "system", "audit", "list", "--limit", "10", "--offset", "5", - ]); + let cli_list_paged = + Cli::parse_from(["nuncio", "system", "audit", "list", "--page-size", "10"]); assert_eq!( cli_list_paged.command, Commands::System { action: SystemSubcommand::Audit { - action: AuditSubcommand::List { - limit: 10, - offset: 5, - } + action: AuditSubcommand::List { page_size: 10 } } } ); diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index 9bf5dbe..33b51e1 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -498,8 +498,8 @@ impl HeadlessRunner { Commands::System { action } => match action { SystemSubcommand::Status => self.handle_system_status(json_mode).await, SystemSubcommand::Audit { action } => match action { - AuditSubcommand::List { limit, offset } => { - self.handle_audit_list(*limit, *offset, json_mode).await + AuditSubcommand::List { page_size } => { + self.handle_audit_list(*page_size, json_mode).await } AuditSubcommand::Verify => self.handle_audit_verify(json_mode).await, }, @@ -726,33 +726,48 @@ impl HeadlessRunner { Err(e) => return Self::render_error(&e, json_mode), }; - match client - .list_messages(nuncio_proto::v1::ListMessagesRequest { - folder_id: folder.to_string(), - limit: 0, - }) - .await - { - Ok(response) => { - let messages = response.into_inner().messages; - if json_mode { - let messages_json: Vec = - messages.iter().map(message_proto_to_json).collect(); - format_json(&json!({ - "folder": folder, - "messages": messages_json - })) - } else { - format!("Folder '{}': {} message(s) found", folder, messages.len()) + // Follow `next_page_token` to the end so the command shows the whole + // folder, transparently exercising the keyset pagination convention. + let mut messages = Vec::new(); + let mut page_token = String::new(); + loop { + match client + .list_messages(nuncio_proto::v1::ListMessagesRequest { + folder_id: folder.to_string(), + page_size: 0, + page_token: page_token.clone(), + }) + .await + { + Ok(response) => { + let page = response.into_inner(); + messages.extend(page.messages); + if page.next_page_token.is_empty() { + break; + } + page_token = page.next_page_token; + } + Err(status) => { + return Self::render_error( + &format!( + "nunciod daemon rejected list_messages: {}", + Self::clean_status_message(&status) + ), + json_mode, + ); } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected list_messages: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + } + + if json_mode { + let messages_json: Vec = + messages.iter().map(message_proto_to_json).collect(); + format_json(&json!({ + "folder": folder, + "messages": messages_json + })) + } else { + format!("Folder '{}': {} message(s) found", folder, messages.len()) } } @@ -824,40 +839,55 @@ impl HeadlessRunner { Err(e) => return Self::render_error(&e, json_mode), }; - match client - .search_messages(nuncio_proto::v1::SearchMessagesRequest { - query: query.to_string(), - }) - .await - { - Ok(response) => { - let hits = response.into_inner().hits; - if json_mode { - let hits_json: Vec = hits - .iter() - .map(|h| { - json!({ - "id": h.id, - "title": h.title, - "snippet": h.snippet, - }) - }) - .collect(); - format_json(&json!({ - "query": query, - "results": hits_json - })) - } else { - format!("Search complete for '{}' ({} matches)", query, hits.len()) + // Follow `next_page_token` to the end so search returns every hit. + let mut hits = Vec::new(); + let mut page_token = String::new(); + loop { + match client + .search_messages(nuncio_proto::v1::SearchMessagesRequest { + query: query.to_string(), + page_size: 0, + page_token: page_token.clone(), + }) + .await + { + Ok(response) => { + let page = response.into_inner(); + hits.extend(page.hits); + if page.next_page_token.is_empty() { + break; + } + page_token = page.next_page_token; + } + Err(status) => { + return Self::render_error( + &format!( + "nunciod daemon rejected search_messages: {}", + Self::clean_status_message(&status) + ), + json_mode, + ); } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected search_messages: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + } + + if json_mode { + let hits_json: Vec = hits + .iter() + .map(|h| { + json!({ + "id": h.id, + "title": h.title, + "snippet": h.snippet, + }) + }) + .collect(); + format_json(&json!({ + "query": query, + "results": hits_json + })) + } else { + format!("Search complete for '{}' ({} matches)", query, hits.len()) } } @@ -869,36 +899,52 @@ impl HeadlessRunner { Err(e) => return Self::render_error(&e, json_mode), }; - match client - .list_folders(nuncio_proto::v1::ListFoldersRequest {}) - .await - { - Ok(response) => { - let folders = response.into_inner().folders; - if json_mode { - let folders_json: Vec = folders - .iter() - .map(|f| { - json!({ - "id": f.id, - "name": f.name, - "total_messages": f.total_messages, - "unread_messages": f.unread_messages, - }) - }) - .collect(); - format_json(&json!({ "folders": folders_json })) - } else { - format!("Available Mailbox Folders: {} folders found", folders.len()) + // Follow `next_page_token` to the end so every folder is listed. + let mut folders = Vec::new(); + let mut page_token = String::new(); + loop { + match client + .list_folders(nuncio_proto::v1::ListFoldersRequest { + page_size: 0, + page_token: page_token.clone(), + }) + .await + { + Ok(response) => { + let page = response.into_inner(); + folders.extend(page.folders); + if page.next_page_token.is_empty() { + break; + } + page_token = page.next_page_token; + } + Err(status) => { + return Self::render_error( + &format!( + "nunciod daemon rejected list_folders: {}", + Self::clean_status_message(&status) + ), + json_mode, + ); } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected list_folders: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + } + + if json_mode { + let folders_json: Vec = folders + .iter() + .map(|f| { + json!({ + "id": f.id, + "name": f.name, + "total_messages": f.total_messages, + "unread_messages": f.unread_messages, + }) + }) + .collect(); + format_json(&json!({ "folders": folders_json })) + } else { + format!("Available Mailbox Folders: {} folders found", folders.len()) } } @@ -1093,46 +1139,63 @@ impl HeadlessRunner { /// `system audit list`: a real thin gRPC client of the running /// `nunciod` daemon's `nuncio.v1.Audit` API. Lists a page of the /// daemon's real, persisted WORM audit ledger, sequence ascending. - async fn handle_audit_list(&self, limit: u32, offset: u32, json_mode: bool) -> String { + async fn handle_audit_list(&self, page_size: u32, json_mode: bool) -> String { let mut client = match self.connect_audit_client().await { Ok(client) => client, Err(e) => return Self::render_error(&e, json_mode), }; - match client - .list_records(nuncio_proto::v1::ListRecordsRequest { limit, offset }) - .await - { - Ok(response) => { - let records = response.into_inner().records; - if json_mode { - let records_json: Vec = - records.iter().map(audit_record_proto_to_json).collect(); - format_json(&json!({ "records": records_json })) - } else if records.is_empty() { - "No audit records recorded.".to_string() - } else { - let mut out = - String::from("SEQ ACTOR ACTION TIMESTAMP\n"); - for r in records { - out.push_str(&format!( - "{:<4} {:<20} {:<20} {}\n", - r.sequence, - r.actor, - r.action, - format_timestamp(&r.timestamp) - )); + // Follow `next_page_token` to the end so the whole ledger is listed, + // fetching `page_size` records per keyset page. + let mut records = Vec::new(); + let mut page_token = String::new(); + loop { + match client + .list_records(nuncio_proto::v1::ListRecordsRequest { + page_size, + page_token: page_token.clone(), + }) + .await + { + Ok(response) => { + let page = response.into_inner(); + records.extend(page.records); + if page.next_page_token.is_empty() { + break; } - out + page_token = page.next_page_token; + } + Err(status) => { + return Self::render_error( + &format!( + "nunciod daemon rejected list_records: {}", + Self::clean_status_message(&status) + ), + json_mode, + ); } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected list_records: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + } + + if json_mode { + let records_json: Vec = + records.iter().map(audit_record_proto_to_json).collect(); + format_json(&json!({ "records": records_json })) + } else if records.is_empty() { + "No audit records recorded.".to_string() + } else { + let mut out = + String::from("SEQ ACTOR ACTION TIMESTAMP\n"); + for r in records { + out.push_str(&format!( + "{:<4} {:<20} {:<20} {}\n", + r.sequence, + r.actor, + r.action, + format_timestamp(&r.timestamp) + )); + } + out } } @@ -1208,37 +1271,52 @@ impl HeadlessRunner { Err(e) => return Self::render_error(&e, json_mode), }; - match client - .list_rules(nuncio_proto::v1::ListRulesRequest {}) - .await - { - Ok(response) => { - let rules = response.into_inner().rules; - if json_mode { - let rules_json: Vec = - rules.iter().map(filter_rule_proto_to_json).collect(); - format_json(&json!({ "rules": rules_json })) - } else if rules.is_empty() { - "No filter rules configured.".to_string() - } else { - let mut out = - String::from("ID PRIORITY ENABLED NAME NSQL\n"); - for r in rules { - out.push_str(&format!( - "{:<10} {:<8} {:<7} {:<20} {}\n", - r.id, r.priority, r.enabled, r.name, r.nsql_text - )); + // Follow `next_page_token` to the end so every rule is listed. + let mut rules = Vec::new(); + let mut page_token = String::new(); + loop { + match client + .list_rules(nuncio_proto::v1::ListRulesRequest { + page_size: 0, + page_token: page_token.clone(), + }) + .await + { + Ok(response) => { + let page = response.into_inner(); + rules.extend(page.rules); + if page.next_page_token.is_empty() { + break; } - out + page_token = page.next_page_token; + } + Err(status) => { + return Self::render_error( + &format!( + "nunciod daemon rejected list_rules: {}", + Self::clean_status_message(&status) + ), + json_mode, + ); } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected list_rules: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + } + + if json_mode { + let rules_json: Vec = + rules.iter().map(filter_rule_proto_to_json).collect(); + format_json(&json!({ "rules": rules_json })) + } else if rules.is_empty() { + "No filter rules configured.".to_string() + } else { + let mut out = String::from("ID PRIORITY ENABLED NAME NSQL\n"); + for r in rules { + out.push_str(&format!( + "{:<10} {:<8} {:<7} {:<20} {}\n", + r.id, r.priority, r.enabled, r.name, r.nsql_text + )); + } + out } } @@ -2240,36 +2318,52 @@ impl HeadlessRunner { Err(e) => return Self::render_error(&e, json_mode), }; - match client - .list_events(nuncio_proto::v1::ListEventsRequest { - account_id: account.to_string(), - calendar_id: calendar.to_string(), - start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(start)), - end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(end)), - }) - .await - { - Ok(response) => { - let events = response.into_inner().events; - if json_mode { - let events_json: Vec = - events.iter().map(calendar_event_proto_to_json).collect(); - format_json(&json!({ - "account": account, - "calendar": calendar, - "events": events_json - })) - } else { - format!("Calendar Events: {} event(s) found", events.len()) + // Follow `next_page_token` to the end so every event in the window is + // listed, transparently exercising keyset pagination. + let mut events = Vec::new(); + let mut page_token = String::new(); + loop { + match client + .list_events(nuncio_proto::v1::ListEventsRequest { + account_id: account.to_string(), + calendar_id: calendar.to_string(), + start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(start)), + end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(end)), + page_size: 0, + page_token: page_token.clone(), + }) + .await + { + Ok(response) => { + let page = response.into_inner(); + events.extend(page.events); + if page.next_page_token.is_empty() { + break; + } + page_token = page.next_page_token; + } + Err(status) => { + return Self::render_error( + &format!( + "nunciod daemon rejected list_events: {}", + Self::clean_status_message(&status) + ), + json_mode, + ); } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected list_events: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + } + + if json_mode { + let events_json: Vec = + events.iter().map(calendar_event_proto_to_json).collect(); + format_json(&json!({ + "account": account, + "calendar": calendar, + "events": events_json + })) + } else { + format!("Calendar Events: {} event(s) found", events.len()) } } @@ -2348,35 +2442,59 @@ impl HeadlessRunner { Err(e) => return Self::render_error(&e, json_mode), }; - match client - .list_contacts(nuncio_proto::v1::ListContactsRequest { - account_id: account.to_string(), - }) - .await - { - Ok(response) => { - let contacts = response.into_inner().contacts; - if json_mode { - let contacts_json: Vec = - contacts.iter().map(contact_proto_to_json).collect(); - format_json(&json!({ "account": account, "contacts": contacts_json })) - } else { - format!( - "Contacts: {} contact(s) found in address book.", - contacts.len() - ) - } + let contacts = match self.fetch_all_contacts(&mut client, account).await { + Ok(contacts) => contacts, + Err(status) => { + return Self::render_error( + &format!( + "nunciod daemon rejected list_contacts: {}", + Self::clean_status_message(&status) + ), + json_mode, + ); } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected list_contacts: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + }; + + if json_mode { + let contacts_json: Vec = + contacts.iter().map(contact_proto_to_json).collect(); + format_json(&json!({ "account": account, "contacts": contacts_json })) + } else { + format!( + "Contacts: {} contact(s) found in address book.", + contacts.len() + ) } } + /// Follows `next_page_token` to fetch every contact in `account`'s address + /// book over the keyset-paginated `ListContacts` RPC, shared by + /// [`Self::handle_contact_list`] and [`Self::handle_contact_search`]. + async fn fetch_all_contacts( + &self, + client: &mut nuncio_proto::client::AuthenticatedContactsClient, + account: &str, + ) -> Result, tonic::Status> { + let mut contacts = Vec::new(); + let mut page_token = String::new(); + loop { + let page = client + .list_contacts(nuncio_proto::v1::ListContactsRequest { + account_id: account.to_string(), + page_size: 0, + page_token: page_token.clone(), + }) + .await? + .into_inner(); + contacts.extend(page.contacts); + if page.next_page_token.is_empty() { + break; + } + page_token = page.next_page_token; + } + Ok(contacts) + } + /// `contact search`: fetches `account`'s full `ListContacts` result over /// the real gRPC API, then filters it client-side by `query` against /// display name, organization, and email address (case-insensitive) -- @@ -2387,42 +2505,37 @@ impl HeadlessRunner { Err(e) => return Self::render_error(&e, json_mode), }; - match client - .list_contacts(nuncio_proto::v1::ListContactsRequest { - account_id: account.to_string(), - }) - .await - { - Ok(response) => { - let matches: Vec<_> = response - .into_inner() - .contacts - .into_iter() - .filter(|c| contact_matches_query(c, query)) - .collect(); - if json_mode { - let contacts_json: Vec = - matches.iter().map(contact_proto_to_json).collect(); - format_json(&json!({ - "account": account, - "query": query, - "contacts": contacts_json - })) - } else { - format!( - "Search ('{}'): {} contact(s) matched.", - query, - matches.len() - ) - } + let contacts = match self.fetch_all_contacts(&mut client, account).await { + Ok(contacts) => contacts, + Err(status) => { + return Self::render_error( + &format!( + "nunciod daemon rejected list_contacts: {}", + Self::clean_status_message(&status) + ), + json_mode, + ); } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected list_contacts: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + }; + + let matches: Vec<_> = contacts + .into_iter() + .filter(|c| contact_matches_query(c, query)) + .collect(); + if json_mode { + let contacts_json: Vec = + matches.iter().map(contact_proto_to_json).collect(); + format_json(&json!({ + "account": account, + "query": query, + "contacts": contacts_json + })) + } else { + format!( + "Search ('{}'): {} contact(s) matched.", + query, + matches.len() + ) } } @@ -3231,6 +3344,7 @@ mod tests { total_messages: 3, unread_messages: 1, }], + next_page_token: String::new(), })) } @@ -3243,6 +3357,7 @@ mod tests { message.folder_id = req.folder_id; Ok(tonic::Response::new(ListMessagesResponse { messages: vec![message], + next_page_token: String::new(), })) } @@ -3286,6 +3401,7 @@ mod tests { title: "Stub Subject".to_string(), snippet: format!("...{}...", req.query), }], + next_page_token: String::new(), })) } @@ -3693,6 +3809,7 @@ mod tests { .unwrap_or_else(|e| e.into_inner()) = Some(req); Ok(tonic::Response::new(ListEventsResponse { events: vec![stub_event()], + next_page_token: String::new(), })) } @@ -3875,6 +3992,7 @@ mod tests { stub_contact("ct-stub-1", "Alice Stub", "Kof22", "alice@nuncio.mx"), stub_contact("ct-stub-2", "Bob Other", "OtherCo", "bob@nuncio.mx"), ], + next_page_token: String::new(), })) } @@ -4114,6 +4232,7 @@ mod tests { 1_700_000_000, )), }], + next_page_token: String::new(), })) } @@ -4723,6 +4842,7 @@ mod tests { previous_block_hash: "GENESIS".to_string(), record_hmac: "cafebabe".to_string(), }], + next_page_token: String::new(), })) } @@ -4758,10 +4878,7 @@ mod tests { .execute_command( &Commands::System { action: SystemSubcommand::Audit { - action: AuditSubcommand::List { - limit: 0, - offset: 0, - }, + action: AuditSubcommand::List { page_size: 0 }, }, }, true, @@ -4824,10 +4941,7 @@ mod tests { .execute_command( &Commands::System { action: SystemSubcommand::Audit { - action: AuditSubcommand::List { - limit: 0, - offset: 0, - }, + action: AuditSubcommand::List { page_size: 0 }, }, }, true, diff --git a/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin b/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin index 94ba2c20c5f91ea86c5d255886932d3c91912fcc..828d294a8ece515d22673a031fc3e9e511ea69c8 100644 GIT binary patch delta 874 zcmaD_aJO^=BNNlD!p+P~K75P;lLG`q?1ZGbI13WfQ{#&>t5PKx6&N*mgE&9}!9c!{ zEEgA8vLrt{HBW*GD9afHRvrQpP|BK|Cg7wcq{hXcms(L0kD-(oBpd)V45B=5@+tvi zTNDdHM&h=R8QsFX$zKHAG>EluvX$Uv#>~llLb^7n-XzV|$<{)ljD?dY3V9N3^X6MZ zGnoVZxCDGMi%Wt^b5e_gQVUB{i%U@5L1J)cOzseKAl?lp#N4=;I+-R5D4s=&W>&nR z&W0K4_Z5i`_06hE&zTtuCiAM9pjk^=uq)+Fc2*<7+D^5dOu+$MLf{}zP0lY$p-Q0V QPj1(ACqB^6X-?+@04Q$=y8r+H delta 210 zcmcax`m|sJBNNl3oXyNkK75QKlLG`qn3S9*FA{KKQu3J0Cul4#!o`}CnVVT6!KA>b z!5akOD7j6p6Lg#WM({GD?&JePx|219Lm53M*9&`Y{vbS)SyYHiz$de~B&akewKyoX zur#%}M9E?DEpdma*=;kk%5!E$lgY2uO+X^C8g83UYwTp= sb>b2Nn~|EFUz9SrS4K`sNQ{duKP{~|73LTyPsx4qb#3>}ZaUNX0Hu#XlmGw# diff --git a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto index 368af82..2024e64 100644 --- a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto +++ b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto @@ -375,11 +375,11 @@ message TestAccountConnectionResponse { // carved out for "just one more" unauthenticated service. See // `nunciod::grpc::serve_on_listener`. service Mail { - // ListFolders returns every mailbox folder known to the store, with - // total/unread message counts. + // ListFolders returns a keyset-paginated page of mailbox folders known to + // the store, folder id ascending, each with total/unread message counts. rpc ListFolders(ListFoldersRequest) returns (ListFoldersResponse); - // ListMessages returns messages in a single folder, newest first, capped - // at `limit`. + // ListMessages returns a keyset-paginated page of messages in a single + // folder, newest first. rpc ListMessages(ListMessagesRequest) returns (ListMessagesResponse); // GetMessage returns a single message by ID, including its (decrypted) // body. @@ -387,7 +387,8 @@ service Mail { // MarkRead updates a message's read/unread flag. rpc MarkRead(MarkReadRequest) returns (MarkReadResponse); // SearchMessages runs a full-text (FTS5) search over cached message - // subjects, senders, and bodies. + // subjects, senders, and bodies, returning a keyset-paginated page of hits + // ordered by relevance rank. rpc SearchMessages(SearchMessagesRequest) returns (SearchMessagesResponse); // SendMessage composes and sends a real outbound email over SMTP, built // from the configured account's SMTP endpoint @@ -462,26 +463,53 @@ message Folder { uint64 unread_messages = 4; } -// ListFoldersRequest currently carries no parameters. -message ListFoldersRequest {} +// ListFoldersRequest pages the folder listing (folder id ascending) using +// the shared keyset convention: `page_size` bounds a page and `page_token` +// is the opaque cursor returned as the previous page's `next_page_token`. +// See `ListMessagesRequest` for the full semantics. +message ListFoldersRequest { + // Maximum folders to return in this page; 0 means "server default". The + // server clamps this to its documented maximum. + uint32 page_size = 1; + // Opaque keyset cursor from a prior response's `next_page_token`; empty + // starts from the first page. Not an offset -- resumes strictly after the + // last folder of the previous page. + string page_token = 2; +} // ListFoldersResponse. message ListFoldersResponse { repeated Folder folders = 1; + // Opaque cursor for the next page, or empty when this is the last page. + string next_page_token = 2; } -// ListMessagesRequest. `limit` caps the number of messages returned -// (newest first); a value of 0 means "use the server's default limit". +// ListMessagesRequest pages a folder's messages, newest first, using the +// shared keyset pagination convention used by every list/search RPC: +// - `page_size` bounds one page; 0 selects the server default and the +// server clamps any value to its documented maximum. +// - `page_token` is an OPAQUE cursor, NOT a numeric offset. It encodes the +// stable sort key (here `(received_at, id)`) of the last row of the +// previous page, so paging resumes strictly AFTER that row and stays +// correct under concurrent inserts (no skipped or duplicated rows). +// Leave empty to fetch the first page; otherwise pass a prior response's +// `next_page_token` verbatim. A token the server cannot decode is +// rejected with a `VALIDATION_FAILED` error, never silently ignored. message ListMessagesRequest { // Mailbox folder identifier (e.g. "inbox"). string folder_id = 1; // Maximum number of messages to return; 0 means "server default". - uint32 limit = 2; + uint32 page_size = 2; + // Opaque keyset cursor from a prior response's `next_page_token`; empty + // starts from the newest message. + string page_token = 3; } // ListMessagesResponse. message ListMessagesResponse { repeated Message messages = 1; + // Opaque cursor for the next page, or empty when this is the last page. + string next_page_token = 2; } // GetMessageRequest. @@ -507,10 +535,19 @@ message MarkReadRequest { // response (as opposed to a `Status` error) is confirmation enough. message MarkReadResponse {} -// SearchMessagesRequest. +// SearchMessagesRequest. Pages hits using the shared keyset convention (see +// `ListMessagesRequest`); the cursor here encodes `(rank, id)` so paging +// preserves the relevance ordering while resuming strictly after the last +// hit of the previous page. message SearchMessagesRequest { // Full-text search query string. string query = 1; + // Maximum hits to return in this page; 0 means "server default". Clamped + // to the server's documented maximum. + uint32 page_size = 2; + // Opaque keyset cursor from a prior response's `next_page_token`; empty + // starts from the highest-ranked hit. + string page_token = 3; } // Mirrors `nuncio_store::search::SearchHit`. @@ -526,6 +563,8 @@ message MessageSearchHit { // SearchMessagesResponse. message SearchMessagesResponse { repeated MessageSearchHit hits = 1; + // Opaque cursor for the next page, or empty when this is the last page. + string next_page_token = 2; } // SendMessageRequest carries a composed outbound email. The sending @@ -595,7 +634,8 @@ service Filters { // Rejects a syntactically or semantically invalid rule with // `Status::invalid_argument` -- nothing is ever persisted on failure. rpc CreateRule(CreateRuleRequest) returns (CreateRuleResponse); - // ListRules returns every persisted filter rule, priority ascending. + // ListRules returns a keyset-paginated page of persisted filter rules, + // priority ascending. rpc ListRules(ListRulesRequest) returns (ListRulesResponse); // DeleteRule removes a persisted filter rule by ID and reloads the // daemon's live `FilterEngine` rule set so the removal takes effect @@ -690,12 +730,23 @@ message CreateRuleResponse { FilterRule rule = 1; } -// ListRulesRequest currently carries no parameters. -message ListRulesRequest {} +// ListRulesRequest pages rules (priority ascending, id as tiebreaker) using +// the shared keyset convention; the cursor encodes `(priority, id)`. See +// `ListMessagesRequest` for the full semantics. +message ListRulesRequest { + // Maximum rules to return in this page; 0 means "server default". Clamped + // to the server's documented maximum. + uint32 page_size = 1; + // Opaque keyset cursor from a prior response's `next_page_token`; empty + // starts from the highest-priority rule. + string page_token = 2; +} // ListRulesResponse. message ListRulesResponse { repeated FilterRule rules = 1; + // Opaque cursor for the next page, or empty when this is the last page. + string next_page_token = 2; } // DeleteRuleRequest. @@ -956,9 +1007,9 @@ service Calendar { // returns `Status::internal` describing that gap unless a test double has // been injected -- never a fabricated `synced_count`. rpc Sync(CalendarSyncRequest) returns (CalendarSyncResponse); - // ListEvents returns persisted events for `account_id` whose window - // overlaps `[start_window, end_window]`, ordered by start time -- always - // genuinely persisted data, never fabricated. + // ListEvents returns a keyset-paginated page of persisted events for + // `account_id` whose window overlaps `[start_window, end_window]`, ordered + // by start time -- always genuinely persisted data, never fabricated. rpc ListEvents(ListEventsRequest) returns (ListEventsResponse); // GetEvent returns a single persisted event by ID. rpc GetEvent(GetEventRequest) returns (GetEventResponse); @@ -1001,17 +1052,29 @@ message CalendarSyncResponse { uint64 synced_count = 1; } -// ListEventsRequest. +// ListEventsRequest pages events (start time ascending, id as tiebreaker) +// using the shared keyset convention; the cursor encodes `(start_time, id)`. +// Recurrence expansion happens server-side before paging, so an expanded +// occurrence's `(start_time, id)` is what the cursor tracks. See +// `ListMessagesRequest` for the full semantics. message ListEventsRequest { string account_id = 1; string calendar_id = 2; google.protobuf.Timestamp start_window = 3; google.protobuf.Timestamp end_window = 4; + // Maximum events to return in this page; 0 means "server default". Clamped + // to the server's documented maximum. + uint32 page_size = 5; + // Opaque keyset cursor from a prior response's `next_page_token`; empty + // starts from the earliest event in the window. + string page_token = 6; } // ListEventsResponse. message ListEventsResponse { repeated CalendarEvent events = 1; + // Opaque cursor for the next page, or empty when this is the last page. + string next_page_token = 2; } // GetEventRequest. @@ -1052,9 +1115,9 @@ service Contacts { // describing that gap unless a test double has been injected -- never a // fabricated `synced_count`. rpc Sync(ContactsSyncRequest) returns (ContactsSyncResponse); - // ListContacts returns persisted contacts for `account_id`, ordered by - // interaction count then display name -- always genuinely persisted data, - // never fabricated. + // ListContacts returns a keyset-paginated page of persisted contacts for + // `account_id`, ordered by interaction count (descending) then display + // name -- always genuinely persisted data, never fabricated. rpc ListContacts(ListContactsRequest) returns (ListContactsResponse); // GetContact returns a single persisted contact by ID. rpc GetContact(GetContactRequest) returns (GetContactResponse); @@ -1132,14 +1195,24 @@ message ContactsSyncResponse { // ListContactsRequest. Contacts are always scoped to the account whose // address book they belong to, mirroring the store's `list_contacts` -// signature. +// signature. Pages using the shared keyset convention; the cursor encodes +// `(interaction_count, display_name, id)` to match the sort order. See +// `ListMessagesRequest` for the full semantics. message ListContactsRequest { string account_id = 1; + // Maximum contacts to return in this page; 0 means "server default". + // Clamped to the server's documented maximum. + uint32 page_size = 2; + // Opaque keyset cursor from a prior response's `next_page_token`; empty + // starts from the most-interacted contact. + string page_token = 3; } // ListContactsResponse. message ListContactsResponse { repeated Contact contacts = 1; + // Opaque cursor for the next page, or empty when this is the last page. + string next_page_token = 2; } // GetContactRequest. @@ -1183,8 +1256,8 @@ message CreateContactResponse { // `BearerAuthInterceptor` that guards every other service `nunciod` // exposes -- see `nunciod::grpc::serve_on_listener`. service Audit { - // ListRecords returns a page of the real, persisted WORM audit ledger, - // sequence ascending. + // ListRecords returns a keyset-paginated page of the real, persisted WORM + // audit ledger, sequence ascending. rpc ListRecords(ListRecordsRequest) returns (ListRecordsResponse); // VerifyChain re-verifies the ENTIRE persisted WORM audit ledger's // HMAC hash-chain integrity, using the ledger's real WORM HMAC key @@ -1215,17 +1288,25 @@ message AuditRecord { string record_hmac = 7; } -// ListRecordsRequest. `limit` caps the number of records returned -// (sequence ascending); a value of 0 means "use the server's default -// limit". +// ListRecordsRequest pages the WORM audit ledger, sequence ascending, using +// the shared keyset convention that every list/search RPC uses -- replacing +// the previous limit/offset paging, whose offset was unstable under +// concurrent appends. The cursor encodes the last record's `sequence`. See +// `ListMessagesRequest` for the full semantics. message ListRecordsRequest { - uint32 limit = 1; - uint32 offset = 2; + // Maximum records to return in this page; 0 means "server default". + // Clamped to the server's documented maximum. + uint32 page_size = 1; + // Opaque keyset cursor from a prior response's `next_page_token`; empty + // starts from the lowest sequence. + string page_token = 2; } // ListRecordsResponse. message ListRecordsResponse { repeated AuditRecord records = 1; + // Opaque cursor for the next page, or empty when this is the last page. + string next_page_token = 2; } // VerifyChainRequest currently carries no parameters. diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 7cba21a..c58912c 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -1291,6 +1291,102 @@ impl DatabaseEngine { .collect::, DatabaseError>>() } + /// Keyset-paginated listing of a folder's messages, newest first + /// (`received_at DESC, id DESC`). `after` is the `(received_at, id)` of + /// the last message of the previous page; `None` starts from the newest. + /// Fetches `page_size + 1` rows to detect a following page: returns at + /// most `page_size` messages plus the `(received_at, id)` cursor of the + /// last returned message when more remain (else `None`). + #[allow(clippy::type_complexity)] + pub async fn list_messages_page( + &self, + folder_id: &str, + after: Option<(i64, String)>, + page_size: usize, + ) -> Result<(Vec, Option<(i64, String)>), DatabaseError> { + let fetch = page_size.saturating_add(1); + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages WHERE folder_id = ", + ); + builder.push_bind(folder_id.to_string()); + if let Some((ts, id)) = &after { + builder.push(" AND (received_at < "); + builder.push_bind(*ts); + builder.push(" OR (received_at = "); + builder.push_bind(*ts); + builder.push(" AND id < "); + builder.push_bind(id.clone()); + builder.push("))"); + } + builder.push(" ORDER BY received_at DESC, id DESC LIMIT "); + builder.push_bind(fetch as i64); + + let rows = builder + .build_query_as::<( + String, + String, + String, + String, + String, + String, + i64, + i64, + Option, + Option, + String, + String, + )>() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + let has_more = rows.len() > page_size; + let mut emails = rows + .into_iter() + .take(page_size) + .map(|r| { + let dec_plain = + r.8.map(|p| { + crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) + }) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = + r.9.map(|h| { + crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) + }) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + Ok(nuncio_core::model::Email { + id: r.0, + account_id: r.1, + folder_id: r.2, + remote_id: r.10, + uid_validity: r.11, + subject: r.3, + sender: r.4, + recipient: r.5, + received_at: r.6, + read: r.7 != 0, + body_plain: dec_plain, + body_html: dec_html, + attachments: Vec::new(), + }) + }) + .collect::, DatabaseError>>()?; + + let next = if has_more { + emails + .last() + .map(|e| (e.received_at, e.id.clone())) + .filter(|_| !emails.is_empty()) + } else { + None + }; + emails.shrink_to_fit(); + Ok((emails, next)) + } + /// Retrieve a single message by ID. #[allow(clippy::type_complexity)] pub async fn get_message( @@ -1632,6 +1728,69 @@ impl DatabaseEngine { Ok(rows.iter().map(contact_from_row).collect()) } + /// Keyset-paginated listing of `account_id`'s contacts, in the same order + /// as [`Self::list_contacts`] (`interaction_count DESC, display_name ASC, + /// id ASC`). `after` is the `(interaction_count, display_name, id)` of the + /// last contact of the previous page. Fetches `page_size + 1` rows to + /// detect a following page and returns the `(interaction_count, + /// display_name, id)` cursor of the last returned contact when more remain. + /// + /// See the confidentiality/dependency note on [`Self::save_contact`]. + #[allow(clippy::type_complexity)] + pub async fn list_contacts_page( + &self, + account_id: &str, + after: Option<(i64, String, String)>, + page_size: usize, + ) -> Result<(Vec, Option<(i64, String, String)>), DatabaseError> { + let fetch = page_size.saturating_add(1); + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT id, account_id, display_name, given_name, family_name, organization, \ + job_title, notes, avatar_url, emails_json, phones_json, is_favorite, \ + interaction_count, last_interacted_at, created_at, updated_at \ + FROM contacts WHERE account_id = ", + ); + builder.push_bind(account_id.to_string()); + if let Some((ic, dn, id)) = &after { + builder.push(" AND (interaction_count < "); + builder.push_bind(*ic); + builder.push(" OR (interaction_count = "); + builder.push_bind(*ic); + builder.push(" AND (display_name > "); + builder.push_bind(dn.clone()); + builder.push(" OR (display_name = "); + builder.push_bind(dn.clone()); + builder.push(" AND id > "); + builder.push_bind(id.clone()); + builder.push("))))"); + } + builder.push(" ORDER BY interaction_count DESC, display_name ASC, id ASC LIMIT "); + builder.push_bind(fetch as i64); + + let rows = builder + .build() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + let has_more = rows.len() > page_size; + let contacts: Vec = + rows.iter().take(page_size).map(contact_from_row).collect(); + + let next = if has_more { + contacts.last().map(|c| { + ( + c.interaction_count as i64, + c.display_name.clone(), + c.id.clone(), + ) + }) + } else { + None + }; + Ok((contacts, next)) + } + /// Retrieve a single contact by ID. /// /// See the confidentiality/dependency note on [`Self::save_contact`]. @@ -1704,6 +1863,52 @@ impl DatabaseEngine { .collect()) } + /// Keyset-paginated listing of folders with message counts, folder id + /// ascending. `after` is the id of the last folder of the previous page. + /// Fetches `page_size + 1` groups to detect a following page and returns + /// the id of the last returned folder as the cursor when more remain. + pub async fn list_folders_page( + &self, + after: Option, + page_size: usize, + ) -> Result<(Vec, Option), DatabaseError> { + let fetch = page_size.saturating_add(1); + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT folder_id, COUNT(*) as total, SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread FROM messages ", + ); + if let Some(id) = &after { + builder.push("WHERE folder_id > "); + builder.push_bind(id.clone()); + } + builder.push(" GROUP BY folder_id ORDER BY folder_id ASC LIMIT "); + builder.push_bind(fetch as i64); + + let rows = builder + .build_query_as::<(String, i64, i64)>() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + let has_more = rows.len() > page_size; + let folders: Vec = rows + .into_iter() + .take(page_size) + .map(|(folder_id, total, unread)| nuncio_core::model::Folder { + id: folder_id.clone(), + name: folder_id, + total_messages: total as usize, + unread_messages: unread as usize, + }) + .collect(); + + let next = if has_more { + folders.last().map(|f| f.id.clone()) + } else { + None + }; + Ok((folders, next)) + } + /// Save or replace a [`nuncio_filter::FilterRule`]. pub async fn save_filter_rule( &self, @@ -1772,6 +1977,77 @@ impl DatabaseEngine { Ok(rules) } + /// Keyset-paginated listing of persisted filter rules, `priority ASC, id + /// ASC`. `after` is the `(priority, id)` of the last rule of the previous + /// page. Fetches `page_size + 1` rows to detect a following page. + /// + /// The following-page detection and the returned cursor are computed from + /// the RAW rows (their stored `(priority, id)`), before parsing -- a row + /// whose stored NSQL no longer parses is skipped from the returned rules + /// exactly as [`Self::list_filter_rules`] does, but it still counts toward + /// the page boundary, so a page with skipped rows still advances the + /// cursor correctly and never re-emits or gaps around a rule. + #[allow(clippy::type_complexity)] + pub async fn list_filter_rules_page( + &self, + after: Option<(i32, String)>, + page_size: usize, + ) -> Result<(Vec, Option<(i32, String)>), DatabaseError> { + let fetch = page_size.saturating_add(1); + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT id, name, priority, enabled, nsql_text, created_at, updated_at FROM filter_rules ", + ); + if let Some((priority, id)) = &after { + builder.push("WHERE (priority > "); + builder.push_bind(*priority as i64); + builder.push(" OR (priority = "); + builder.push_bind(*priority as i64); + builder.push(" AND id > "); + builder.push_bind(id.clone()); + builder.push("))"); + } + builder.push(" ORDER BY priority ASC, id ASC LIMIT "); + builder.push_bind(fetch as i64); + + let rows = builder + .build_query_as::<(String, String, i64, i64, String, i64, i64)>() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + let has_more = rows.len() > page_size; + let page: Vec<(String, String, i64, i64, String, i64, i64)> = + rows.into_iter().take(page_size).collect(); + + let next = if has_more { + page.last().map(|r| (r.2 as i32, r.0.clone())) + } else { + None + }; + + let mut rules = Vec::new(); + for (id, name, priority, enabled, nsql_text, created_at, updated_at) in page { + match nuncio_filter::NsqlParser::parse_rule(&name, priority as i32, &nsql_text) { + Ok(mut parsed) => { + parsed.id = id; + parsed.enabled = enabled != 0; + parsed.created_at = created_at; + parsed.updated_at = updated_at; + rules.push(parsed); + } + Err(err) => { + tracing::warn!( + rule_id = %id, + rule_name = %name, + error = %err, + "stored filter rule failed to parse and will not be enforced" + ); + } + } + } + Ok((rules, next)) + } + /// Delete a [`nuncio_filter::FilterRule`] by ID. pub async fn delete_filter_rule(&self, rule_id: &str) -> Result<(), DatabaseError> { sqlx::query("DELETE FROM filter_rules WHERE id = ?") @@ -2261,6 +2537,57 @@ impl DatabaseEngine { .collect()) } + /// Keyset-paginated listing of the WORM audit ledger, `sequence ASC`. + /// `after` is the sequence of the last record of the previous page; + /// `None` starts from the lowest sequence. Fetches `page_size + 1` rows to + /// detect a following page and returns the sequence of the last returned + /// record as the cursor when more remain. Keyset paging on the + /// monotonically increasing sequence is stable under concurrent appends, + /// unlike the previous limit/offset paging. + pub async fn list_worm_audit_records_page( + &self, + after: Option, + page_size: usize, + ) -> Result<(Vec, Option), DatabaseError> { + let fetch = page_size.saturating_add(1); + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT sequence, timestamp_ns, actor, action, data_hash, previous_block_hash, record_hmac FROM worm_audit_records ", + ); + if let Some(seq) = after { + builder.push("WHERE sequence > "); + builder.push_bind(seq as i64); + } + builder.push(" ORDER BY sequence ASC LIMIT "); + builder.push_bind(fetch as i64); + + let rows = builder + .build_query_as::<(i64, i64, String, String, String, String, String)>() + .fetch_all(&self.pool) + .await?; + + let has_more = rows.len() > page_size; + let records: Vec = rows + .into_iter() + .take(page_size) + .map(|r| nuncio_core::WormAuditRecord { + sequence: r.0 as u64, + timestamp_ns: r.1, + actor: r.2, + action: r.3, + data_hash: r.4, + previous_block_hash: r.5, + record_hmac: r.6, + }) + .collect(); + + let next = if has_more { + records.last().map(|r| r.sequence) + } else { + None + }; + Ok((records, next)) + } + /// Verify the entire WORM cryptographic audit log chain, using the WORM HMAC key /// provisioned for this engine. pub async fn verify_worm_audit_chain(&self) -> Result<(), DatabaseError> { diff --git a/crates/nuncio-store/src/search.rs b/crates/nuncio-store/src/search.rs index 0befdf5..dc25f64 100644 --- a/crates/nuncio-store/src/search.rs +++ b/crates/nuncio-store/src/search.rs @@ -68,6 +68,71 @@ impl<'a> SearchEngine<'a> { .collect()) } + /// Keyset-paginated full-text search over message subjects, senders, and + /// bodies, ordered by relevance (`rank ASC, id ASC`). `after` is the + /// `(rank, id)` of the last hit of the previous page; `None` starts from + /// the highest-ranked hit. Fetches `page_size + 1` hits to detect a + /// following page and returns the `(rank, id)` cursor of the last returned + /// hit when more remain. + /// + /// The bm25 `rank` is materialized in an inner query (where the FTS5 + /// `MATCH` is in scope) into an ordinary column, so the outer query can + /// both keyset-filter and order by `(rank, id)` -- preserving relevance + /// order across pages while resuming strictly after the previous page's + /// last hit. + pub async fn search_messages_page( + &self, + query: &str, + after: Option<(f64, String)>, + page_size: usize, + ) -> Result<(Vec, Option<(f64, String)>), DatabaseError> { + let clean_query = Self::sanitize_fts5_query(query); + if clean_query.is_empty() { + return Ok((Vec::new(), None)); + } + let fetch = page_size.saturating_add(1); + + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT id, title, snippet, rk FROM ( \ + SELECT id, subject AS title, \ + snippet(messages_fts, 3, '', '', '...', 10) AS snippet, rank AS rk \ + FROM messages_fts WHERE messages_fts MATCH ", + ); + builder.push_bind(clean_query); + builder.push(") "); + if let Some((rank, id)) = &after { + builder.push("WHERE (rk > "); + builder.push_bind(*rank); + builder.push(" OR (rk = "); + builder.push_bind(*rank); + builder.push(" AND id > "); + builder.push_bind(id.clone()); + builder.push("))"); + } + builder.push(" ORDER BY rk ASC, id ASC LIMIT "); + builder.push_bind(fetch as i64); + + let rows = builder + .build_query_as::<(String, String, String, f64)>() + .fetch_all(self.db.pool()) + .await + .map_err(DatabaseError::Query)?; + + let has_more = rows.len() > page_size; + let mut last_key: Option<(f64, String)> = None; + let hits: Vec = rows + .into_iter() + .take(page_size) + .map(|(id, title, snippet, rk)| { + last_key = Some((rk, id.clone())); + SearchHit { id, title, snippet } + }) + .collect(); + + let next = if has_more { last_key } else { None }; + Ok((hits, next)) + } + /// Perform a full-text trigram search over calendar event summaries and locations. pub async fn search_events(&self, query: &str) -> Result, DatabaseError> { let clean_query = Self::sanitize_fts5_query(query); @@ -224,6 +289,52 @@ mod tests { assert_eq!(hits[0].title, "Legacy Roadmap Notes"); } + /// Paging FTS search results across multiple pages via the `(rank, id)` + /// keyset must return every matching hit EXACTLY ONCE, no dupes and no + /// gaps -- proving the rank-materializing subquery keyset works end to + /// end over the real encrypt-then-index write path. + #[tokio::test] + async fn search_messages_page_returns_every_hit_once_no_gaps() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + for i in 0..5 { + db.save_email(&sample_email( + &format!("msg-{i}"), + "Budget Planning Meeting", + "annual budget revenue forecast discussion", + )) + .await + .unwrap(); + } + + let search = SearchEngine::new(&db); + let mut seen = Vec::new(); + let mut after: Option<(f64, String)> = None; + let mut pages = 0; + loop { + let (hits, next) = search + .search_messages_page("budget", after.clone(), 2) + .await + .unwrap(); + pages += 1; + assert!(hits.len() <= 2, "page must not exceed page_size"); + seen.extend(hits.iter().map(|h| h.id.clone())); + match next { + Some(cursor) => after = Some(cursor), + None => break, + } + assert!(pages < 100, "pagination must terminate"); + } + + seen.sort(); + seen.dedup(); + assert_eq!( + seen.len(), + 5, + "every matching message returned exactly once" + ); + assert!(pages >= 3, "5 hits at page_size 2 must span multiple pages"); + } + #[tokio::test] async fn fts5_event_search_and_triggers() { let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 7ac4718..234b4e1 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -5,6 +5,7 @@ //! //! This is the daemon's sole client-facing transport. +use crate::pagination::{self, CursorField}; use nuncio_cal::CalendarBackend; use nuncio_contacts::ContactsBackend; use nuncio_core::{CoreCommand, CoreEvent, EventBus}; @@ -745,10 +746,6 @@ impl Accounts for AccountsGrpcService { } } -/// Default cap on messages returned by `ListMessages` when the caller -/// supplies `limit: 0` ("use the server default"). -const DEFAULT_LIST_MESSAGES_LIMIT: usize = 50; - /// Maps a `nuncio_core::model::Attachment` onto its wire-format /// `nuncio.v1.Attachment` representation. fn map_attachment_to_proto(attachment: nuncio_core::model::Attachment) -> AttachmentProto { @@ -1083,6 +1080,11 @@ impl Calendar for CalendarGrpcService { } let start_window = require_window_bound(req.start_window, "start_window")?; let end_window = require_window_bound(req.end_window, "end_window")?; + let page_size = pagination::clamp_page_size(req.page_size); + let after = pagination::parse_token(&req.page_token, |f| match f { + [CursorField::I(ts), CursorField::S(id)] => Some((*ts, id.clone())), + _ => None, + })?; let all_in_window = self .db @@ -1101,12 +1103,45 @@ impl Calendar for CalendarGrpcService { events.retain(|e| req.calendar_id.is_empty() || e.calendar_id == req.calendar_id); + // Recurrence expansion materializes the result set in memory, so keyset + // pagination is applied to the sorted expanded list here rather than at + // the store. Each occurrence has a distinct `(start_time, id)` (an + // occurrence shares its master's id but never its start time), so the + // pair is a stable, unique keyset over the whole set. + events.sort_by(|a, b| { + a.start_time + .cmp(&b.start_time) + .then_with(|| a.id.cmp(&b.id)) + }); + if let Some((ts, id)) = &after { + events.retain(|e| e.start_time > *ts || (e.start_time == *ts && e.id > *id)); + } + + let has_more = events.len() > page_size; + events.truncate(page_size); + let next_page_token = if has_more { + events + .last() + .map(|e| { + pagination::encode_cursor(&[ + CursorField::I(e.start_time), + CursorField::S(e.id.clone()), + ]) + }) + .unwrap_or_default() + } else { + String::new() + }; + let events = events .into_iter() .map(map_calendar_event_to_proto) .collect(); - Ok(Response::new(ListEventsResponse { events })) + Ok(Response::new(ListEventsResponse { + events, + next_page_token, + })) } async fn get_event( @@ -1212,17 +1247,35 @@ impl Contacts for ContactsGrpcService { if req.account_id.is_empty() { return Err(Status::invalid_argument("account_id is required")); } + let page_size = pagination::clamp_page_size(req.page_size); + let after = pagination::parse_token(&req.page_token, |f| match f { + [CursorField::I(ic), CursorField::S(dn), CursorField::S(id)] => { + Some((*ic, dn.clone(), id.clone())) + } + _ => None, + })?; - let contacts = self + let (contacts, next) = self .db - .list_contacts(&req.account_id) + .list_contacts_page(&req.account_id, after, page_size) .await - .map_err(|e| Status::internal(format!("failed to list contacts: {e}")))? - .into_iter() - .map(map_contact_to_proto) - .collect(); + .map_err(|e| Status::internal(format!("failed to list contacts: {e}")))?; - Ok(Response::new(ListContactsResponse { contacts })) + let next_page_token = next + .map(|(ic, dn, id)| { + pagination::encode_cursor(&[ + CursorField::I(ic), + CursorField::S(dn), + CursorField::S(id), + ]) + }) + .unwrap_or_default(); + let contacts = contacts.into_iter().map(map_contact_to_proto).collect(); + + Ok(Response::new(ListContactsResponse { + contacts, + next_page_token, + })) } async fn get_contact( @@ -1364,18 +1417,30 @@ struct MailGrpcService { impl Mail for MailGrpcService { async fn list_folders( &self, - _request: Request, + request: Request, ) -> Result, Status> { - let folders = self - .db - .list_folders() - .await - .map_err(|e| Status::internal(format!("failed to list folders: {e}")))? - .into_iter() - .map(map_folder_to_proto) - .collect(); + let req = request.into_inner(); + let page_size = pagination::clamp_page_size(req.page_size); + let after = pagination::parse_token(&req.page_token, |f| match f { + [CursorField::S(id)] => Some(id.clone()), + _ => None, + })?; - Ok(Response::new(ListFoldersResponse { folders })) + let (folders, next) = self + .db + .list_folders_page(after, page_size) + .await + .map_err(|e| Status::internal(format!("failed to list folders: {e}")))?; + + let next_page_token = next + .map(|id| pagination::encode_cursor(&[CursorField::S(id)])) + .unwrap_or_default(); + let folders = folders.into_iter().map(map_folder_to_proto).collect(); + + Ok(Response::new(ListFoldersResponse { + folders, + next_page_token, + })) } async fn list_messages( @@ -1386,22 +1451,27 @@ impl Mail for MailGrpcService { if req.folder_id.is_empty() { return Err(Status::invalid_argument("folder_id is required")); } - let limit = if req.limit == 0 { - DEFAULT_LIST_MESSAGES_LIMIT - } else { - req.limit as usize - }; + let page_size = pagination::clamp_page_size(req.page_size); + let after = pagination::parse_token(&req.page_token, |f| match f { + [CursorField::I(ts), CursorField::S(id)] => Some((*ts, id.clone())), + _ => None, + })?; - let messages = self + let (messages, next) = self .db - .list_messages(&req.folder_id, limit) + .list_messages_page(&req.folder_id, after, page_size) .await - .map_err(|e| Status::internal(format!("failed to list messages: {e}")))? - .into_iter() - .map(map_email_to_proto) - .collect(); + .map_err(|e| Status::internal(format!("failed to list messages: {e}")))?; - Ok(Response::new(ListMessagesResponse { messages })) + let next_page_token = next + .map(|(ts, id)| pagination::encode_cursor(&[CursorField::I(ts), CursorField::S(id)])) + .unwrap_or_default(); + let messages = messages.into_iter().map(map_email_to_proto).collect(); + + Ok(Response::new(ListMessagesResponse { + messages, + next_page_token, + })) } async fn get_message( @@ -1467,11 +1537,24 @@ impl Mail for MailGrpcService { request: Request, ) -> Result, Status> { let req = request.into_inner(); + let page_size = pagination::clamp_page_size(req.page_size); + let after = pagination::parse_token(&req.page_token, |f| match f { + [CursorField::F(rank), CursorField::S(id)] => Some((*rank, id.clone())), + _ => None, + })?; + let search = SearchEngine::new(&self.db); - let hits = search - .search_messages(&req.query) + let (hits, next) = search + .search_messages_page(&req.query, after, page_size) .await - .map_err(|e| Status::internal(format!("search failed: {e}")))? + .map_err(|e| Status::internal(format!("search failed: {e}")))?; + + let next_page_token = next + .map(|(rank, id)| { + pagination::encode_cursor(&[CursorField::F(rank), CursorField::S(id)]) + }) + .unwrap_or_default(); + let hits = hits .into_iter() .map(|hit| MessageSearchHit { id: hit.id, @@ -1480,7 +1563,10 @@ impl Mail for MailGrpcService { }) .collect(); - Ok(Response::new(SearchMessagesResponse { hits })) + Ok(Response::new(SearchMessagesResponse { + hits, + next_page_token, + })) } /// SendMessage composes and sends a real outbound email over SMTP via @@ -1753,18 +1839,32 @@ impl Filters for FiltersGrpcService { async fn list_rules( &self, - _request: Request, + request: Request, ) -> Result, Status> { - let rules = self - .db - .list_filter_rules() - .await - .map_err(|e| Status::internal(format!("failed to list filter rules: {e}")))? - .into_iter() - .map(map_filter_rule_to_proto) - .collect(); + let req = request.into_inner(); + let page_size = pagination::clamp_page_size(req.page_size); + let after = pagination::parse_token(&req.page_token, |f| match f { + [CursorField::I(priority), CursorField::S(id)] => Some((*priority as i32, id.clone())), + _ => None, + })?; - Ok(Response::new(ListRulesResponse { rules })) + let (rules, next) = self + .db + .list_filter_rules_page(after, page_size) + .await + .map_err(|e| Status::internal(format!("failed to list filter rules: {e}")))?; + + let next_page_token = next + .map(|(priority, id)| { + pagination::encode_cursor(&[CursorField::I(priority as i64), CursorField::S(id)]) + }) + .unwrap_or_default(); + let rules = rules.into_iter().map(map_filter_rule_to_proto).collect(); + + Ok(Response::new(ListRulesResponse { + rules, + next_page_token, + })) } async fn delete_rule( @@ -2178,8 +2278,6 @@ impl Export for ExportGrpcService { /// Default cap on records returned by `Audit/ListRecords` when the caller /// supplies `limit: 0` ("use the server default"). -const DEFAULT_LIST_RECORDS_LIMIT: u32 = 100; - /// Maps a `nuncio_core::WormAuditRecord` onto its wire-format /// `nuncio.v1.AuditRecord` representation. `record_hmac` is a verification /// MAC output, never the WORM HMAC signing key itself -- see the message's @@ -2214,22 +2312,27 @@ impl Audit for AuditGrpcService { request: Request, ) -> Result, Status> { let req = request.into_inner(); - let limit = if req.limit == 0 { - DEFAULT_LIST_RECORDS_LIMIT - } else { - req.limit - }; + let page_size = pagination::clamp_page_size(req.page_size); + let after = pagination::parse_token(&req.page_token, |f| match f { + [CursorField::U(seq)] => Some(*seq), + _ => None, + })?; - let records = self + let (records, next) = self .db - .list_worm_audit_records(limit, req.offset) + .list_worm_audit_records_page(after, page_size) .await - .map_err(|e| Status::internal(format!("failed to list audit records: {e}")))? - .into_iter() - .map(map_audit_record_to_proto) - .collect(); + .map_err(|e| Status::internal(format!("failed to list audit records: {e}")))?; - Ok(Response::new(ListRecordsResponse { records })) + let next_page_token = next + .map(|seq| pagination::encode_cursor(&[CursorField::U(seq)])) + .unwrap_or_default(); + let records = records.into_iter().map(map_audit_record_to_proto).collect(); + + Ok(Response::new(ListRecordsResponse { + records, + next_page_token, + })) } /// Re-verifies the ENTIRE persisted WORM audit ledger via @@ -3721,6 +3824,110 @@ mod tests { } } + /// Paging a folder larger than `page_size` across multiple pages via + /// `next_page_token` must return every message EXACTLY ONCE, with no + /// duplicates and no gaps, and in the stable newest-first keyset order + /// (`received_at DESC, id DESC`). Some messages deliberately share a + /// `received_at` so the `(received_at, id)` tiebreaker is exercised across + /// a page boundary. + #[tokio::test] + async fn list_messages_keyset_pagination_returns_every_message_once_no_gaps() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + let seed = [ + ("m1", 1_700_000_000_i64), + ("m2", 1_700_000_000), + ("m3", 1_700_000_010), + ("m4", 1_700_000_010), + ("m5", 1_700_000_020), + ("m6", 1_700_000_030), + ("m7", 1_700_000_030), + ]; + for (id, ts) in seed { + let mut email = sample_email(id, "inbox", "subject", "body"); + email.received_at = ts; + db.save_email(&email).await.expect("seed message"); + } + + let event_bus = Arc::new(EventBus::new()); + let secrets = Arc::new(SecretManager::mock()); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + let (addr, _handle) = spawn_test_server_with( + event_bus, + Arc::new(db), + filter_engine, + secrets, + "correct-token", + ) + .await; + let mut client = MailClient::connect(format!("http://{addr}")) + .await + .expect("client connects"); + + let mut seen = Vec::new(); + let mut page_token = String::new(); + let mut pages = 0; + loop { + let resp = client + .list_messages(authed_bearer_request(ListMessagesRequest { + folder_id: "inbox".to_string(), + page_size: 2, + page_token: page_token.clone(), + })) + .await + .expect("list_messages page succeeds") + .into_inner(); + pages += 1; + assert!( + resp.messages.len() <= 2, + "a page must never exceed the requested page_size" + ); + seen.extend(resp.messages.iter().map(|m| m.id.clone())); + if resp.next_page_token.is_empty() { + break; + } + page_token = resp.next_page_token; + assert!(pages < 100, "pagination must terminate"); + } + + // Every seeded message appears exactly once (no dupes, no gaps), in + // the stable newest-first order including the id tiebreaker. + assert_eq!( + seen, + vec!["m7", "m6", "m5", "m4", "m3", "m2", "m1"], + "keyset paging must yield the full set once, newest-first" + ); + assert!(pages >= 4, "a 7-item set at page_size 2 must span >1 page"); + } + + /// A `page_token` the server cannot decode is rejected with a typed + /// `VALIDATION_FAILED` `ErrorInfo` mapped to `InvalidArgument` -- never a + /// panic and never a silently-empty first page. + #[tokio::test] + async fn list_messages_rejects_malformed_page_token() { + let event_bus = Arc::new(EventBus::new()); + let (addr, _handle, _dir) = spawn_test_server(event_bus, "correct-token").await; + let mut client = MailClient::connect(format!("http://{addr}")) + .await + .expect("client connects"); + + let err = client + .list_messages(authed_bearer_request(ListMessagesRequest { + folder_id: "inbox".to_string(), + page_size: 2, + page_token: "not-a-valid-cursor".to_string(), + })) + .await + .expect_err("a malformed page_token must be rejected"); + assert_eq!(err.code(), Code::InvalidArgument); + assert_eq!( + nuncio_proto::errors::error_reason(&err), + Some(ErrorReason::ValidationFailed), + "malformed page_token must carry a typed VALIDATION_FAILED reason" + ); + } + #[tokio::test] async fn mail_rpcs_reject_missing_bearer_token() { // Confirms `Mail` is mounted behind its own `BearerAuthInterceptor` @@ -3733,7 +3940,10 @@ mod tests { .expect("client connects"); let err = client - .list_folders(ListFoldersRequest {}) + .list_folders(ListFoldersRequest { + page_size: 0, + page_token: String::new(), + }) .await .expect_err("missing bearer token must be rejected"); assert_eq!(err.code(), Code::Unauthenticated); @@ -3741,7 +3951,8 @@ mod tests { let err = client .list_messages(ListMessagesRequest { folder_id: "inbox".to_string(), - limit: 10, + page_size: 10, + page_token: String::new(), }) .await .expect_err("missing bearer token must be rejected"); @@ -3767,6 +3978,8 @@ mod tests { let err = client .search_messages(SearchMessagesRequest { query: "hello".to_string(), + page_size: 0, + page_token: String::new(), }) .await .expect_err("missing bearer token must be rejected"); @@ -3814,6 +4027,8 @@ mod tests { calendar_id: "cal-1".to_string(), start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), + page_size: 0, + page_token: String::new(), }) .await .expect_err("missing bearer token must be rejected"); @@ -3885,7 +4100,8 @@ mod tests { let err = client .list_messages(authed_bearer_request(ListMessagesRequest { folder_id: String::new(), - limit: 10, + page_size: 10, + page_token: String::new(), })) .await .expect_err("empty folder_id must be rejected"); @@ -3942,7 +4158,10 @@ mod tests { // ListFolders reports the seeded folder with the right counts. let folders = client - .list_folders(authed_bearer_request(ListFoldersRequest {})) + .list_folders(authed_bearer_request(ListFoldersRequest { + page_size: 0, + page_token: String::new(), + })) .await .expect("list_folders succeeds") .into_inner() @@ -3956,7 +4175,8 @@ mod tests { let messages = client .list_messages(authed_bearer_request(ListMessagesRequest { folder_id: "inbox".to_string(), - limit: 10, + page_size: 10, + page_token: String::new(), })) .await .expect("list_messages succeeds") @@ -3988,6 +4208,8 @@ mod tests { let hits = client .search_messages(authed_bearer_request(SearchMessagesRequest { query: "revenue".to_string(), + page_size: 0, + page_token: String::new(), })) .await .expect("search_messages succeeds") @@ -4513,7 +4735,10 @@ mod tests { assert_eq!(err.code(), Code::Unauthenticated); let err = client - .list_rules(ListRulesRequest {}) + .list_rules(ListRulesRequest { + page_size: 0, + page_token: String::new(), + }) .await .expect_err("missing bearer token must be rejected"); assert_eq!(err.code(), Code::Unauthenticated); @@ -4639,7 +4864,10 @@ mod tests { // ListRules reflects the persisted rule. let list_response = client - .list_rules(authed_bearer_request(ListRulesRequest {})) + .list_rules(authed_bearer_request(ListRulesRequest { + page_size: 0, + page_token: String::new(), + })) .await .expect("list_rules succeeds") .into_inner(); @@ -4655,7 +4883,10 @@ mod tests { .expect("delete_rule succeeds"); let list_after_delete = client - .list_rules(authed_bearer_request(ListRulesRequest {})) + .list_rules(authed_bearer_request(ListRulesRequest { + page_size: 0, + page_token: String::new(), + })) .await .expect("list_rules succeeds") .into_inner(); @@ -4685,7 +4916,10 @@ mod tests { assert_eq!(err.code(), Code::InvalidArgument); let list_response = client - .list_rules(authed_bearer_request(ListRulesRequest {})) + .list_rules(authed_bearer_request(ListRulesRequest { + page_size: 0, + page_token: String::new(), + })) .await .expect("list_rules succeeds") .into_inner(); @@ -4756,7 +4990,10 @@ mod tests { ); let list_response = client - .list_rules(authed_bearer_request(ListRulesRequest {})) + .list_rules(authed_bearer_request(ListRulesRequest { + page_size: 0, + page_token: String::new(), + })) .await .expect("list_rules succeeds") .into_inner(); @@ -4821,7 +5058,10 @@ mod tests { // Nothing was persisted by either validation call. let list_response = client - .list_rules(authed_bearer_request(ListRulesRequest {})) + .list_rules(authed_bearer_request(ListRulesRequest { + page_size: 0, + page_token: String::new(), + })) .await .expect("list_rules succeeds") .into_inner(); @@ -4889,7 +5129,10 @@ mod tests { // Previewing must never persist the rule. let list_response = client - .list_rules(authed_bearer_request(ListRulesRequest {})) + .list_rules(authed_bearer_request(ListRulesRequest { + page_size: 0, + page_token: String::new(), + })) .await .expect("list_rules succeeds") .into_inner(); @@ -4998,7 +5241,10 @@ mod tests { ); let list_response = client - .list_rules(authed_bearer_request(ListRulesRequest {})) + .list_rules(authed_bearer_request(ListRulesRequest { + page_size: 0, + page_token: String::new(), + })) .await .expect("list_rules succeeds") .into_inner(); @@ -5077,7 +5323,10 @@ mod tests { assert_eq!(err.code(), Code::InvalidArgument); let list_response = client - .list_rules(authed_bearer_request(ListRulesRequest {})) + .list_rules(authed_bearer_request(ListRulesRequest { + page_size: 0, + page_token: String::new(), + })) .await .expect("list_rules succeeds") .into_inner(); @@ -5154,7 +5403,10 @@ mod tests { assert_eq!(response.errors.len(), 1); let list_response = client - .list_rules(authed_bearer_request(ListRulesRequest {})) + .list_rules(authed_bearer_request(ListRulesRequest { + page_size: 0, + page_token: String::new(), + })) .await .expect("list_rules succeeds") .into_inner(); @@ -5406,8 +5658,8 @@ mod tests { let err = client .list_records(ListRecordsRequest { - limit: 10, - offset: 0, + page_size: 10, + page_token: String::new(), }) .await .expect_err("missing bearer token must be rejected"); @@ -5455,8 +5707,8 @@ mod tests { let response = client .list_records(authed_bearer_request(ListRecordsRequest { - limit: 0, - offset: 0, + page_size: 0, + page_token: String::new(), })) .await .expect("list_records succeeds") @@ -5471,6 +5723,81 @@ mod tests { assert_eq!(response.records[1].action, "test.action.two"); } + /// Keyset paging the WORM ledger across multiple pages must return every + /// record EXACTLY ONCE, in ascending sequence, with no dupes or gaps. + #[tokio::test] + async fn list_records_keyset_pagination_returns_every_record_once_no_gaps() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + for i in 0..7 { + db.append_worm_audit_record( + "system.test", + &format!("test.action.{i}"), + format!("payload-{i}").as_bytes(), + ) + .await + .expect("append audit record"); + } + + let event_bus = Arc::new(EventBus::new()); + let secrets = Arc::new(SecretManager::mock()); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + let (addr, _handle) = spawn_test_server_with( + event_bus, + Arc::new(db), + filter_engine, + secrets, + "correct-token", + ) + .await; + let mut client = AuditClient::connect(format!("http://{addr}")) + .await + .expect("client connects"); + + let mut seen = Vec::new(); + let mut page_token = String::new(); + let mut pages = 0; + loop { + let resp = client + .list_records(authed_bearer_request(ListRecordsRequest { + page_size: 2, + page_token: page_token.clone(), + })) + .await + .expect("list_records page succeeds") + .into_inner(); + pages += 1; + assert!(resp.records.len() <= 2, "page must not exceed page_size"); + seen.extend(resp.records.iter().map(|r| r.sequence)); + if resp.next_page_token.is_empty() { + break; + } + page_token = resp.next_page_token; + assert!(pages < 100, "pagination must terminate"); + } + + assert_eq!( + seen, + vec![1, 2, 3, 4, 5, 6, 7], + "keyset paging must yield every sequence once, ascending" + ); + + // A malformed page_token is a typed VALIDATION_FAILED error. + let err = client + .list_records(authed_bearer_request(ListRecordsRequest { + page_size: 2, + page_token: "garbage".to_string(), + })) + .await + .expect_err("malformed page_token must be rejected"); + assert_eq!(err.code(), Code::InvalidArgument); + assert_eq!( + nuncio_proto::errors::error_reason(&err), + Some(ErrorReason::ValidationFailed) + ); + } + /// Proves `Audit/VerifyChain` reports `valid: true` and the real /// record count on an intact, honestly-signed chain -- never a /// fabricated `valid`. @@ -5645,6 +5972,8 @@ mod tests { calendar_id: "cal-work".to_string(), start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_699_999_000)), end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_004_000)), + page_size: 0, + page_token: String::new(), })) .await .expect("list_events succeeds") @@ -5704,6 +6033,8 @@ mod tests { calendar_id: "cal-work".to_string(), start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_699_999_000)), end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_004_000)), + page_size: 0, + page_token: String::new(), })) .await .expect("list_events succeeds") @@ -5730,6 +6061,8 @@ mod tests { calendar_id: "cal-work".to_string(), start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), + page_size: 0, + page_token: String::new(), })) .await .expect_err("empty account_id must be rejected"); @@ -5894,6 +6227,8 @@ mod tests { calendar_id: "cal-work".to_string(), start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_699_999_000)), end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_004_000)), + page_size: 0, + page_token: String::new(), })) .await .expect("list_events succeeds") @@ -6068,6 +6403,8 @@ mod tests { let err = client .list_contacts(ListContactsRequest { account_id: "acct-1".to_string(), + page_size: 0, + page_token: String::new(), }) .await .expect_err("missing bearer token must be rejected"); @@ -6126,6 +6463,8 @@ mod tests { let response = client .list_contacts(authed_bearer_request(ListContactsRequest { account_id: "acct-contacts-list".to_string(), + page_size: 0, + page_token: String::new(), })) .await .expect("list_contacts succeeds") @@ -6239,6 +6578,8 @@ mod tests { let list_response = client .list_contacts(authed_bearer_request(ListContactsRequest { account_id: "acct-contacts-injected".to_string(), + page_size: 0, + page_token: String::new(), })) .await .expect("list_contacts succeeds") @@ -6350,7 +6691,10 @@ mod tests { .await .expect("Mail client connects"); let err = client - .list_folders(ListFoldersRequest {}) + .list_folders(ListFoldersRequest { + page_size: 0, + page_token: String::new(), + }) .await .expect_err("Mail must reject an unauthenticated call"); assert_eq!(err.code(), Code::Unauthenticated); @@ -6360,7 +6704,10 @@ mod tests { .await .expect("Filters client connects"); let err = client - .list_rules(ListRulesRequest {}) + .list_rules(ListRulesRequest { + page_size: 0, + page_token: String::new(), + }) .await .expect_err("Filters must reject an unauthenticated call"); assert_eq!(err.code(), Code::Unauthenticated); @@ -6385,8 +6732,8 @@ mod tests { .expect("Audit client connects"); let err = client .list_records(ListRecordsRequest { - limit: 10, - offset: 0, + page_size: 10, + page_token: String::new(), }) .await .expect_err("Audit must reject an unauthenticated call"); @@ -6402,6 +6749,8 @@ mod tests { calendar_id: "cal-1".to_string(), start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), + page_size: 0, + page_token: String::new(), }) .await .expect_err("Calendar must reject an unauthenticated call"); @@ -6414,6 +6763,8 @@ mod tests { let err = client .list_contacts(ListContactsRequest { account_id: "acct-1".to_string(), + page_size: 0, + page_token: String::new(), }) .await .expect_err("Contacts must reject an unauthenticated call"); diff --git a/crates/nunciod/src/lib.rs b/crates/nunciod/src/lib.rs index 0f3ef2d..93fcc13 100644 --- a/crates/nunciod/src/lib.rs +++ b/crates/nunciod/src/lib.rs @@ -9,6 +9,7 @@ pub mod lifecycle; pub mod lock; pub mod orchestrator; pub mod outbox; +pub mod pagination; pub mod send; pub mod sync; pub use orchestrator::SelfHealingSyncOrchestrator; diff --git a/crates/nunciod/src/pagination.rs b/crates/nunciod/src/pagination.rs new file mode 100644 index 0000000..958341a --- /dev/null +++ b/crates/nunciod/src/pagination.rs @@ -0,0 +1,168 @@ +//! Shared keyset ("cursor") pagination for the daemon's list/search RPCs. +//! +//! Every `nuncio.v1` list/search RPC pages results with the same convention +//! (AIP-158): a request carries `page_size` + `page_token` and a response +//! carries `next_page_token`. The `page_token` is an OPAQUE cursor, never a +//! numeric offset: it encodes the stable sort key (and id tiebreaker) of the +//! last row of the previous page so the next page resumes strictly AFTER that +//! row. Keyset paging is stable under concurrent inserts -- rows are never +//! skipped or returned twice as the underlying table changes between page +//! fetches, which offset paging cannot guarantee. +//! +//! The wire encoding is deliberately private to the daemon: it is a +//! hex-encoded JSON array of typed [`CursorField`]s. Clients must treat the +//! token as opaque and echo it back verbatim; a token this module cannot +//! decode is rejected with a typed [`ErrorReason::ValidationFailed`] error +//! (mapped to `InvalidArgument`), never silently ignored or treated as the +//! first page. + +use nuncio_proto::errors; +use nuncio_proto::v1::ErrorReason; +use serde::{Deserialize, Serialize}; +use tonic::Status; + +/// Page size used when a request leaves `page_size` at 0 ("server default"). +pub const DEFAULT_PAGE_SIZE: u32 = 50; + +/// Hard upper bound on a single page, so a client cannot force the daemon to +/// materialize an unbounded result set in one call. +pub const MAX_PAGE_SIZE: u32 = 500; + +/// Resolves a requested `page_size` into a concrete row count: 0 selects +/// [`DEFAULT_PAGE_SIZE`], and any value is clamped to [`MAX_PAGE_SIZE`]. +#[must_use] +pub fn clamp_page_size(requested: u32) -> usize { + let n = if requested == 0 { + DEFAULT_PAGE_SIZE + } else { + requested.min(MAX_PAGE_SIZE) + }; + n as usize +} + +/// One typed component of an opaque keyset cursor. A cursor is an ordered +/// list of these, matching a list's stable sort key followed by its id +/// tiebreaker (e.g. `[I(received_at), S(id)]` for messages). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum CursorField { + /// A signed integer sort key (e.g. a unix timestamp or a priority). + I(i64), + /// An unsigned integer sort key (e.g. an audit sequence). + U(u64), + /// A floating-point sort key (e.g. an FTS5 relevance rank). + F(f64), + /// A textual sort key or id (e.g. a message id or display name). + S(String), +} + +/// Encodes a cursor's component list into an opaque `page_token`. +#[must_use] +pub fn encode_cursor(fields: &[CursorField]) -> String { + // The component list is a fixed, small set of primitives, so + // serialization cannot realistically fail; fall back to an empty payload + // rather than panicking, keeping this crate free of `unwrap`/`expect`. + let json = serde_json::to_vec(fields).unwrap_or_default(); + hex::encode(json) +} + +fn malformed_token() -> Status { + errors::status( + ErrorReason::ValidationFailed, + "malformed page_token: not a valid pagination cursor", + ) +} + +/// Decodes an opaque `page_token` back into its component list, rejecting a +/// token that is not a hex-encoded cursor JSON payload with a typed +/// `VALIDATION_FAILED` error. +pub fn decode_cursor(token: &str) -> Result, Status> { + let bytes = hex::decode(token).map_err(|_| malformed_token())?; + serde_json::from_slice(&bytes).map_err(|_| malformed_token()) +} + +/// Decodes an optional `page_token` into a strongly-typed cursor value. +/// +/// An empty token means "first page" and yields `Ok(None)`. A non-empty +/// token is decoded and handed to `shape`, which pattern-matches the expected +/// component layout for a specific list and returns `Some(value)`; a token +/// that decodes but does not match the expected layout (e.g. a cursor minted +/// for a different list) is rejected with the same typed `VALIDATION_FAILED` +/// error as an undecodable token, so a caller can never resume from a cursor +/// shaped for the wrong query. +pub fn parse_token( + token: &str, + shape: impl FnOnce(&[CursorField]) -> Option, +) -> Result, Status> { + if token.is_empty() { + return Ok(None); + } + let fields = decode_cursor(token)?; + shape(&fields).map(Some).ok_or_else(malformed_token) +} + +#[cfg(test)] +mod tests { + use super::*; + use tonic::Code; + + #[test] + fn clamp_page_size_applies_default_and_max() { + assert_eq!(clamp_page_size(0), DEFAULT_PAGE_SIZE as usize); + assert_eq!(clamp_page_size(10), 10); + assert_eq!(clamp_page_size(MAX_PAGE_SIZE + 1), MAX_PAGE_SIZE as usize); + assert_eq!(clamp_page_size(u32::MAX), MAX_PAGE_SIZE as usize); + } + + #[test] + fn cursor_round_trips_through_encode_decode() { + let fields = vec![ + CursorField::I(1_700_000_000), + CursorField::S("msg-9".into()), + ]; + let token = encode_cursor(&fields); + assert!(!token.is_empty()); + assert_eq!(decode_cursor(&token).unwrap(), fields); + } + + #[test] + fn empty_token_parses_as_first_page() { + let parsed = parse_token("", |f| match f { + [CursorField::I(n)] => Some(*n), + _ => None, + }) + .unwrap(); + assert_eq!(parsed, None); + } + + #[test] + fn malformed_token_is_validation_failed() { + // Not valid hex. + let err = decode_cursor("zzz not-hex").unwrap_err(); + assert_eq!(err.code(), Code::InvalidArgument); + assert_eq!( + errors::error_reason(&err), + Some(ErrorReason::ValidationFailed) + ); + + // Valid hex, but not a cursor JSON payload. + let err = decode_cursor(&hex::encode(b"not json")).unwrap_err(); + assert_eq!(err.code(), Code::InvalidArgument); + } + + #[test] + fn wrong_shape_token_is_rejected() { + // A cursor with a single int component handed to a shape expecting + // (int, string) must be rejected, not silently accepted. + let token = encode_cursor(&[CursorField::I(1)]); + let err = parse_token(&token, |f| match f { + [CursorField::I(n), CursorField::S(s)] => Some((*n, s.clone())), + _ => None, + }) + .unwrap_err(); + assert_eq!(err.code(), Code::InvalidArgument); + assert_eq!( + errors::error_reason(&err), + Some(ErrorReason::ValidationFailed) + ); + } +} diff --git a/crates/nunciod/tests/caldav_production_e2e_test.rs b/crates/nunciod/tests/caldav_production_e2e_test.rs index 51a9d1f..6a210f3 100644 --- a/crates/nunciod/tests/caldav_production_e2e_test.rs +++ b/crates/nunciod/tests/caldav_production_e2e_test.rs @@ -198,6 +198,8 @@ async fn caldav_production_sync_builds_real_client_and_persists_events() { calendar_id: CALENDAR_ID.to_string(), start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_000_000)), end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_800_000_000)), + page_size: 0, + page_token: String::new(), }) .await .expect("list_events succeeds") @@ -234,6 +236,8 @@ async fn caldav_production_sync_builds_real_client_and_persists_events() { calendar_id: CALENDAR_ID.to_string(), start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), + page_size: 0, + page_token: String::new(), }) .await .expect_err("missing bearer token must be rejected"); diff --git a/crates/nunciod/tests/calendar_e2e_test.rs b/crates/nunciod/tests/calendar_e2e_test.rs index 3c3ea63..f9cf5c9 100644 --- a/crates/nunciod/tests/calendar_e2e_test.rs +++ b/crates/nunciod/tests/calendar_e2e_test.rs @@ -154,6 +154,8 @@ async fn calendar_sync_then_list_and_get_round_trip_over_authenticated_grpc() { calendar_id: CALENDAR_ID.to_string(), start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_699_999_000)), end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(1_700_020_000)), + page_size: 0, + page_token: String::new(), }) .await .expect("list_events succeeds") @@ -189,6 +191,8 @@ async fn calendar_sync_then_list_and_get_round_trip_over_authenticated_grpc() { calendar_id: CALENDAR_ID.to_string(), start_window: Some(nuncio_proto::time::timestamp_from_unix_secs(0)), end_window: Some(nuncio_proto::time::timestamp_from_unix_secs(i64::MAX)), + page_size: 0, + page_token: String::new(), }) .await .expect_err("missing bearer token must be rejected"); diff --git a/crates/nunciod/tests/contacts_e2e_test.rs b/crates/nunciod/tests/contacts_e2e_test.rs index f231f39..8cc1cf5 100644 --- a/crates/nunciod/tests/contacts_e2e_test.rs +++ b/crates/nunciod/tests/contacts_e2e_test.rs @@ -146,6 +146,8 @@ async fn contacts_sync_create_and_list_round_trip_over_authenticated_grpc() { let list_response = contacts_client .list_contacts(ListContactsRequest { account_id: ACCOUNT_ID.to_string(), + page_size: 0, + page_token: String::new(), }) .await .expect("list_contacts succeeds") @@ -201,6 +203,8 @@ async fn contacts_sync_create_and_list_round_trip_over_authenticated_grpc() { let second_list = second_client .list_contacts(ListContactsRequest { account_id: ACCOUNT_ID.to_string(), + page_size: 0, + page_token: String::new(), }) .await .expect("list_contacts succeeds on second connection") @@ -231,6 +235,8 @@ async fn contacts_sync_create_and_list_round_trip_over_authenticated_grpc() { let err = unauthenticated_client .list_contacts(ListContactsRequest { account_id: ACCOUNT_ID.to_string(), + page_size: 0, + page_token: String::new(), }) .await .expect_err("missing bearer token must be rejected"); diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index 42f786c..d7ad21f 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -226,7 +226,8 @@ async fn full_mail_spine_round_trips_over_authenticated_grpc_with_mocked_protoco let list_messages_response = mail_client .list_messages(ListMessagesRequest { folder_id: "inbox".to_string(), - limit: 0, + page_size: 0, + page_token: String::new(), }) .await .expect("list_messages succeeds") -- 2.54.0 From 0567ddea1d913a9cc2086d8e13111bd3f543c0a4 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 16:54:37 -0500 Subject: [PATCH 30/95] feat(mail): bound IMAP fetch by batch count and per-message size (#352) Enumerate a folder's UIDs and sizes cheaply (no bodies), then pull full bodies in explicit UID-set batches of at most MAX_MESSAGES_PER_FETCH_BATCH so the transient working set stays bounded regardless of mailbox size. Messages whose RFC822.SIZE exceeds MAX_MESSAGE_BODY_BYTES are fetched headers-only and recorded with an empty body plus a WARN naming the uid and size, rather than buffered whole or silently dropped. Refs #301 Co-authored-by: James Maes --- crates/nuncio-mail/src/imap.rs | 742 +++++++++++++++++++++++++++------ 1 file changed, 623 insertions(+), 119 deletions(-) diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index c5288ac..e022380 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -31,6 +31,27 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); /// honest timeout rather than hanging the sync indefinitely. const FETCH_ITEM_TIMEOUT: Duration = Duration::from_secs(60); +/// Upper bound on how many messages a single `UID FETCH` may pull full bodies +/// for. A folder sync never issues one unbounded `UID FETCH ` that +/// buffers every body at once; instead the messages in range are enumerated +/// cheaply (UIDs + sizes, no bodies) and their bodies fetched in explicit +/// UID-set batches of at most this many. This bounds the transient in-memory +/// working set of full message bodies to a fixed window, so a mailbox with an +/// enormous number of messages cannot drive the daemon to exhaust memory in a +/// single fetch regardless of how many messages the server holds. +const MAX_MESSAGES_PER_FETCH_BATCH: usize = 200; + +/// Upper bound on the RFC822 size (bytes) of a message whose full body the sync +/// will pull into memory. The server reports each message's size via +/// `RFC822.SIZE` during enumeration; a message above this bound is NOT fetched +/// with its full body (which could be arbitrarily large and OOM the daemon). +/// Instead only its headers are fetched and it is recorded with an empty body +/// plus a WARN naming the UID and size, rather than being silently dropped or +/// buffered whole. The bound matches the MIME parser's own maximum payload +/// ([`MimeParserAdapter::MAX_PAYLOAD_BYTES`]): a body larger than the parser +/// would ever accept is pointless to transfer only to reject. +const MAX_MESSAGE_BODY_BYTES: u32 = 25 * 1024 * 1024; + use crate::backend::{MailBackend, RemoteMutationKind, RemoteMutationSpec}; use crate::parser::{MailError, MimeParserAdapter}; @@ -191,6 +212,91 @@ pub fn build_fetch_command_query() -> &'static str { "(FLAGS INTERNALDATE RFC822.SIZE ENVELOPE BODY.PEEK[])" } +/// The `UID FETCH` query for the cheap enumeration pass: UID plus `RFC822.SIZE` +/// only, deliberately WITHOUT a `BODY.PEEK[]`. This lets the sync discover every +/// message's UID and size (to decide per-message whether the body is safe to +/// pull) without transferring any body, so the enumeration stays small even for +/// a huge mailbox. +fn build_enumerate_command_query() -> &'static str { + "(UID RFC822.SIZE)" +} + +/// The `UID FETCH` query for a message whose body exceeds +/// [`MAX_MESSAGE_BODY_BYTES`]: headers only (`BODY.PEEK[HEADER]`), never the +/// full `BODY.PEEK[]`. The oversized body is intentionally left unfetched; the +/// headers are enough to record honest metadata (subject, sender, date). +fn build_headers_only_command_query() -> &'static str { + "(FLAGS INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER])" +} + +/// Render an IMAP ENVELOPE address as `mailbox@host`, or bare `mailbox` when no +/// host is present. Both components are decoded from their raw bytes lossily. +/// Used only when a FETCH item carries an ENVELOPE but no body, so a minimal +/// record can still name the sender/recipient. +fn format_envelope_address(addr: &async_imap::imap_proto::Address) -> String { + let mailbox = addr + .mailbox + .as_ref() + .map_or("", |m| std::str::from_utf8(m).unwrap_or("")); + let host = addr + .host + .as_ref() + .map_or("", |h| std::str::from_utf8(h).unwrap_or("")); + if host.is_empty() { + mailbox.to_string() + } else { + format!("{}@{}", mailbox, host) + } +} + +/// Render an explicit IMAP UID sequence-set from a batch of UIDs, e.g. +/// `[12, 15, 18]` -> `"12,15,18"`. Used to fetch a bounded batch of specific +/// messages rather than an open-ended `n:*` range. +fn join_uid_batch(uids: &[u32]) -> String { + uids.iter() + .map(|u| u.to_string()) + .collect::>() + .join(",") +} + +/// Drive one already-issued `UID FETCH` response stream to completion, +/// collecting its items. Every individual item read is bounded by +/// [`FETCH_ITEM_TIMEOUT`]: a slow-but-progressing fetch is fine, but a single +/// item that never arrives (dead socket, server-side hang) surfaces as an +/// honest [`MailError::FetchStalled`] instead of hanging the sync. `read_so_far` +/// is the count of messages already read across earlier batches, so the stall +/// message reports the true progress. +async fn drive_fetch_stream( + stream: &mut St, + folder_id: &str, + read_so_far: usize, +) -> Result, MailError> +where + St: tokio_stream::Stream> + Unpin, + E: std::fmt::Display, +{ + let mut items = Vec::new(); + loop { + let next_item = tokio::time::timeout(FETCH_ITEM_TIMEOUT, stream.next()) + .await + .map_err(|_| { + MailError::FetchStalled(format!( + "no FETCH response for folder '{}' within {}s ({} message(s) read so far)", + folder_id, + FETCH_ITEM_TIMEOUT.as_secs(), + read_so_far + items.len() + )) + })?; + let Some(fetch_res) = next_item else { + break; + }; + let fetch_data = fetch_res + .map_err(|e| MailError::ImapError(format!("failed reading fetch item: {}", e)))?; + items.push(fetch_data); + } + Ok(items) +} + /// Establish an encrypted TLS stream to an IMAP server. pub async fn connect_tls_stream(host: &str, port: u16) -> Result, MailError> { let connector = build_tls_connector()?; @@ -583,6 +689,40 @@ impl ImapEngine { since_state: Option<&str>, session: &mut async_imap::Session, ) -> Result<(Vec, String), MailError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, + { + self.sync_folder_messages_batched( + folder_id, + since_state, + session, + MAX_MESSAGES_PER_FETCH_BATCH, + ) + .await + } + + /// Core folder sync, parameterised by the maximum number of messages whose + /// full bodies are pulled per `UID FETCH`. Production callers pass + /// [`MAX_MESSAGES_PER_FETCH_BATCH`]; the parameter is a seam so tests can + /// exercise the multi-batch path with a small window instead of a huge + /// fixture. + /// + /// The sync runs in two phases so the transient working set stays bounded + /// regardless of mailbox size: + /// 1. A cheap enumeration ([`build_enumerate_command_query`]) reads every + /// in-range message's UID and `RFC822.SIZE` with NO body. Messages at or + /// below [`MAX_MESSAGE_BODY_BYTES`] are queued for a full-body fetch; + /// oversized ones are queued for a headers-only fetch and WARN-logged. + /// 2. Each queue is drained in explicit UID-set batches of at most + /// `batch_size`, so the full bodies held in memory at once never exceed + /// one batch. + async fn sync_folder_messages_batched( + &self, + folder_id: &str, + since_state: Option<&str>, + session: &mut async_imap::Session, + batch_size: usize, + ) -> Result<(Vec, String), MailError> where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { @@ -609,133 +749,93 @@ impl ImapEngine { FetchRange::Full | FetchRange::Incremental(_) => {} } - let query = build_fetch_command_query(); + let batch_size = batch_size.max(1); let sequence_set = range.sequence_set(); - let mut fetch_stream = session.uid_fetch(&sequence_set, query).await.map_err(|e| { - MailError::ImapError(format!( - "UID FETCH failed for folder '{}': {}", - folder_id, e - )) - })?; - let mut emails = Vec::new(); - let mut max_uid_seen: u32 = 0; - loop { - // Bound EVERY individual item read: a slow-but-progressing large - // fetch is fine, but a single item that never arrives must surface - // as an honest `FetchStalled` instead of hanging the whole sync. - let next_item = tokio::time::timeout(FETCH_ITEM_TIMEOUT, fetch_stream.next()) + // Phase 1: enumerate UIDs and sizes only -- no bodies transferred, so + // this stays small even for an enormous mailbox. + let enumeration = { + let mut enumerate_stream = session + .uid_fetch(&sequence_set, build_enumerate_command_query()) .await - .map_err(|_| { - MailError::FetchStalled(format!( - "no FETCH response for folder '{}' within {}s ({} message(s) read so far)", - folder_id, - FETCH_ITEM_TIMEOUT.as_secs(), - emails.len() + .map_err(|e| { + MailError::ImapError(format!( + "UID FETCH (enumerate) failed for folder '{}': {}", + folder_id, e )) })?; - let Some(fetch_res) = next_item else { - break; + drive_fetch_stream(&mut enumerate_stream, folder_id, 0).await? + }; + + let mut max_uid_seen: u32 = 0; + let mut body_uids: Vec = Vec::new(); + let mut oversized_uids: Vec = Vec::new(); + for item in &enumeration { + // A message with no UID cannot be addressed by a follow-up fetch, + // so it is skipped rather than issued a UID-less request. + let Some(uid) = item.uid.filter(|u| *u >= 1) else { + continue; }; - let fetch_data = fetch_res - .map_err(|e| MailError::ImapError(format!("failed reading fetch item: {}", e)))?; - - let uid_num = fetch_data.uid.unwrap_or(0); - max_uid_seen = max_uid_seen.max(uid_num); - // The message is addressed on the wire by its UID within the - // folder's UIDVALIDITY scope; the persisted id is an opaque - // surrogate hashed over both plus the account and folder, so the - // same UID in another folder/account can never collide. - let remote_id = uid_num.to_string(); - let uid_validity = server_uid_validity - .map(|v| v.to_string()) - .unwrap_or_default(); - let email_id = - Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id); - - let is_read = fetch_data - .flags() - .any(|flag| matches!(flag, async_imap::types::Flag::Seen)); - - let email = if let Some(raw_bytes) = fetch_data.body() { - let mut email = MimeParserAdapter::parse_mime( - &email_id, - &self.account_id, - folder_id, - raw_bytes, - )?; - email.remote_id = remote_id; - email.uid_validity = uid_validity; - email.read = is_read; - email - } else { - let subject = fetch_data - .envelope() - .and_then(|env| env.subject.as_ref()) - .map_or("No Subject".to_string(), |s| { - String::from_utf8_lossy(s).to_string() - }); - - let sender = fetch_data - .envelope() - .and_then(|env| env.from.as_ref()) - .and_then(|froms| froms.first()) - .map_or("unknown@nuncio.mx".to_string(), |addr| { - let mailbox = addr - .mailbox - .as_ref() - .map_or("", |m| std::str::from_utf8(m).unwrap_or("")); - let host = addr - .host - .as_ref() - .map_or("", |h| std::str::from_utf8(h).unwrap_or("")); - if host.is_empty() { - mailbox.to_string() - } else { - format!("{}@{}", mailbox, host) - } - }); - - let recipient = fetch_data - .envelope() - .and_then(|env| env.to.as_ref()) - .and_then(|tos| tos.first()) - .map_or("me@nuncio.mx".to_string(), |addr| { - let mailbox = addr - .mailbox - .as_ref() - .map_or("", |m| std::str::from_utf8(m).unwrap_or("")); - let host = addr - .host - .as_ref() - .map_or("", |h| std::str::from_utf8(h).unwrap_or("")); - if host.is_empty() { - mailbox.to_string() - } else { - format!("{}@{}", mailbox, host) - } - }); - - let received_at = fetch_data.internal_date().map_or(0, |dt| dt.timestamp()); - - Email { - id: email_id, - account_id: self.account_id.clone(), - folder_id: folder_id.to_string(), - remote_id, - uid_validity, - subject, - sender, - recipient, - received_at, - read: is_read, - body_plain: None, - body_html: None, - attachments: Vec::new(), + max_uid_seen = max_uid_seen.max(uid); + match item.size { + Some(size) if size > MAX_MESSAGE_BODY_BYTES => { + tracing::warn!( + folder_id, + uid, + size, + max_body_bytes = MAX_MESSAGE_BODY_BYTES, + "message body exceeds the per-message size cap; fetching headers only \ + and recording it with an empty body instead of buffering the full body" + ); + oversized_uids.push(uid); } - }; + _ => body_uids.push(uid), + } + } + drop(enumeration); - emails.push(email); + let mut emails = Vec::new(); + + // Phase 2a: normal messages -- full bodies, in bounded UID-set batches. + for chunk in body_uids.chunks(batch_size) { + let set = join_uid_batch(chunk); + let items = { + let mut fetch_stream = session + .uid_fetch(&set, build_fetch_command_query()) + .await + .map_err(|e| { + MailError::ImapError(format!( + "UID FETCH failed for folder '{}': {}", + folder_id, e + )) + })?; + drive_fetch_stream(&mut fetch_stream, folder_id, emails.len()).await? + }; + for item in &items { + emails.push(self.build_email_from_fetch(folder_id, server_uid_validity, item)?); + } + } + + // Phase 2b: oversized messages -- headers only, recorded with an empty + // body. Also batched, so a run of oversized messages cannot itself + // buffer an unbounded number of header fetches at once. + for chunk in oversized_uids.chunks(batch_size) { + let set = join_uid_batch(chunk); + let items = { + let mut fetch_stream = session + .uid_fetch(&set, build_headers_only_command_query()) + .await + .map_err(|e| { + MailError::ImapError(format!( + "UID FETCH (headers) failed for folder '{}': {}", + folder_id, e + )) + })?; + drive_fetch_stream(&mut fetch_stream, folder_id, emails.len()).await? + }; + for item in &items { + emails.push(self.build_headers_only_email(folder_id, server_uid_validity, item)); + } } // Derive the next checkpoint from real server state as @@ -760,6 +860,137 @@ impl ImapEngine { Ok((emails, new_checkpoint)) } + /// Build an [`Email`] from a full-body (or envelope-only) FETCH item. When + /// the item carries a `BODY[]` it is parsed as a full MIME message; when it + /// does not (a server that returned only the ENVELOPE), a minimal record is + /// synthesised from the envelope so the message is still surfaced honestly. + fn build_email_from_fetch( + &self, + folder_id: &str, + server_uid_validity: Option, + fetch_data: &async_imap::types::Fetch, + ) -> Result { + let uid_num = fetch_data.uid.unwrap_or(0); + // The message is addressed on the wire by its UID within the folder's + // UIDVALIDITY scope; the persisted id is an opaque surrogate hashed over + // both plus the account and folder, so the same UID in another + // folder/account can never collide. + let remote_id = uid_num.to_string(); + let uid_validity = server_uid_validity + .map(|v| v.to_string()) + .unwrap_or_default(); + let email_id = Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id); + + let is_read = fetch_data + .flags() + .any(|flag| matches!(flag, async_imap::types::Flag::Seen)); + + if let Some(raw_bytes) = fetch_data.body() { + let mut email = + MimeParserAdapter::parse_mime(&email_id, &self.account_id, folder_id, raw_bytes)?; + email.remote_id = remote_id; + email.uid_validity = uid_validity; + email.read = is_read; + return Ok(email); + } + + let subject = fetch_data + .envelope() + .and_then(|env| env.subject.as_ref()) + .map_or("No Subject".to_string(), |s| { + String::from_utf8_lossy(s).to_string() + }); + + let sender = fetch_data + .envelope() + .and_then(|env| env.from.as_ref()) + .and_then(|froms| froms.first()) + .map_or("unknown@nuncio.mx".to_string(), format_envelope_address); + + let recipient = fetch_data + .envelope() + .and_then(|env| env.to.as_ref()) + .and_then(|tos| tos.first()) + .map_or("me@nuncio.mx".to_string(), format_envelope_address); + + let received_at = fetch_data.internal_date().map_or(0, |dt| dt.timestamp()); + + Ok(Email { + id: email_id, + account_id: self.account_id.clone(), + folder_id: folder_id.to_string(), + remote_id, + uid_validity, + subject, + sender, + recipient, + received_at, + read: is_read, + body_plain: None, + body_html: None, + attachments: Vec::new(), + }) + } + + /// Build an [`Email`] for a message whose body exceeded + /// [`MAX_MESSAGE_BODY_BYTES`]: its headers (`BODY[HEADER]`) are parsed for + /// honest metadata (subject/sender/date) but the oversized body is left + /// unfetched, so the record carries an EMPTY body rather than a fabricated + /// one. If the server returned no header section, this falls back to the + /// envelope-only record; either way the message is surfaced, never dropped. + fn build_headers_only_email( + &self, + folder_id: &str, + server_uid_validity: Option, + fetch_data: &async_imap::types::Fetch, + ) -> Email { + let uid_num = fetch_data.uid.unwrap_or(0); + let remote_id = uid_num.to_string(); + let uid_validity = server_uid_validity + .map(|v| v.to_string()) + .unwrap_or_default(); + let email_id = Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id); + let is_read = fetch_data + .flags() + .any(|flag| matches!(flag, async_imap::types::Flag::Seen)); + + if let Some(header_bytes) = fetch_data.header() { + if let Ok(mut email) = + MimeParserAdapter::parse_mime(&email_id, &self.account_id, folder_id, header_bytes) + { + // A headers-only parse yields no body, but be explicit that the + // oversized body was intentionally not fetched. + email.remote_id = remote_id; + email.uid_validity = uid_validity; + email.read = is_read; + email.body_plain = None; + email.body_html = None; + email.attachments = Vec::new(); + return email; + } + } + + // No usable header section: fall back to whatever the envelope offers. + // `build_email_from_fetch` cannot error on this path (no body to parse), + // but if it ever did, surface a minimal honest record rather than panic. + self.build_email_from_fetch(folder_id, server_uid_validity, fetch_data) + .unwrap_or_else(|_| Email { + id: email_id, + account_id: self.account_id.clone(), + folder_id: folder_id.to_string(), + remote_id, + uid_validity, + subject: "No Subject".to_string(), + sender: "unknown@nuncio.mx".to_string(), + recipient: "me@nuncio.mx".to_string(), + received_at: 0, + read: is_read, + body_plain: None, + body_html: None, + attachments: Vec::new(), + }) + } + /// Fetch a folder's messages over a freshly authenticated session, /// honoring `since_state` for an incremental fetch and returning the new /// sync checkpoint alongside the messages. See @@ -1426,6 +1657,279 @@ mod tests { ); } + /// The RFC822 body of a scripted fixture message. + fn fixture_body(uid: u32) -> String { + format!( + "Subject: Message {uid}\r\nFrom: sender{uid}@example.test\r\n\ + To: me@example.test\r\n\r\nBody of message {uid}.\r\n" + ) + } + + /// The header block (no body) of a scripted fixture message. + fn fixture_header(uid: u32) -> String { + format!( + "Subject: Message {uid}\r\nFrom: sender{uid}@example.test\r\n\ + To: me@example.test\r\n\r\n" + ) + } + + /// Resolve which fixture UIDs a `UID FETCH` sequence-set token addresses. A + /// range `lo:*` selects every fixture UID `>= lo`; an explicit comma list + /// selects exactly those UIDs. Mirrors just enough server-side sequence-set + /// handling for the batched-fetch tests. + fn requested_uids(set: &str, all: &[(u32, u32)]) -> Vec { + if set.contains(':') { + let lo: u32 = set + .split(':') + .next() + .unwrap_or("1") + .trim() + .parse() + .unwrap_or(1); + all.iter().map(|(u, _)| *u).filter(|u| *u >= lo).collect() + } else { + set.split(',') + .filter_map(|s| s.trim().parse::().ok()) + .filter(|u| all.iter().any(|(fu, _)| fu == u)) + .collect() + } + } + + /// Drive `sync_folder_messages_batched` over a scripted, in-memory IMAP + /// server that serves a fixed set of `(uid, size)` messages. The server + /// answers the enumeration pass (UID + RFC822.SIZE), full-body batches + /// (`BODY[]`), and headers-only batches (`BODY[HEADER]`) from the fixture, + /// and records every `UID FETCH` command line the client issued so a test + /// can assert on the exact batching. Returns the synced emails, the new + /// checkpoint, and the captured `UID FETCH` command lines. + async fn run_batched_sync( + messages: Vec<(u32, u32)>, + uidvalidity: u32, + uidnext: u32, + since_state: Option, + batch_size: usize, + ) -> (Vec, String, Vec) { + let (client_io, server_io) = tokio::io::duplex(65536); + let fetch_cmds = Arc::new(std::sync::Mutex::new(Vec::::new())); + let server_cmds = fetch_cmds.clone(); + let exists = messages.len() as u32; + + let server = tokio::spawn(async move { + let mut io = server_io; + while let Some(line) = read_scripted_line(&mut io).await { + let tag = line.split_whitespace().next().unwrap_or("").to_string(); + let upper = line.to_ascii_uppercase(); + if upper.contains("UID FETCH") { + if let Ok(mut g) = server_cmds.lock() { + g.push(line.clone()); + } + let set = line.split_whitespace().nth(3).unwrap_or("").to_string(); + let uids = requested_uids(&set, &messages); + let mut resp = String::new(); + for uid in uids { + let size = messages + .iter() + .find(|(u, _)| *u == uid) + .map_or(0, |(_, s)| *s); + if upper.contains("BODY.PEEK[HEADER]") { + let hdr = fixture_header(uid); + resp.push_str(&format!( + "* {uid} FETCH (UID {uid} RFC822.SIZE {size} FLAGS (\\Seen) \ + BODY[HEADER] {{{}}}\r\n{hdr})\r\n", + hdr.len() + )); + } else if upper.contains("BODY.PEEK[]") { + let body = fixture_body(uid); + resp.push_str(&format!( + "* {uid} FETCH (UID {uid} RFC822.SIZE {size} FLAGS (\\Seen) \ + BODY[] {{{}}}\r\n{body})\r\n", + body.len() + )); + } else { + // Enumeration pass: UID + size only, no body. + resp.push_str(&format!( + "* {uid} FETCH (UID {uid} RFC822.SIZE {size})\r\n" + )); + } + } + resp.push_str(&format!("{tag} OK FETCH completed\r\n")); + let _ = io.write_all(resp.as_bytes()).await; + } else if upper.contains("LOGIN") { + let _ = io + .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) + .await; + } else if upper.contains("SELECT") { + let resp = format!( + "* FLAGS (\\Seen)\r\n* {exists} EXISTS\r\n* 0 RECENT\r\n\ + * OK [UIDVALIDITY {uidvalidity}] ok\r\n* OK [UIDNEXT {uidnext}] ok\r\n\ + {tag} OK [READ-WRITE] SELECT done\r\n" + ); + let _ = io.write_all(resp.as_bytes()).await; + } else if upper.contains("LOGOUT") { + let _ = io + .write_all(format!("* BYE\r\n{tag} OK LOGOUT\r\n").as_bytes()) + .await; + break; + } + } + }); + + let client = async_imap::Client::new(client_io); + let mut session = client + .login("user", "pass") + .await + .map_err(|(e, _)| e) + .expect("scripted login succeeds"); + let engine = ImapEngine::new("acct-1", "example.test", 993); + let (emails, checkpoint) = engine + .sync_folder_messages_batched("INBOX", since_state.as_deref(), &mut session, batch_size) + .await + .expect("batched sync succeeds"); + drop(session); + let _ = server.await; + + let cmds = fetch_cmds.lock().expect("lock captured commands").clone(); + (emails, checkpoint, cmds) + } + + /// The captured `UID FETCH` command lines that pulled FULL bodies (a + /// `BODY.PEEK[]`, not the headers-only `BODY.PEEK[HEADER]` nor the + /// bodyless enumeration). + fn body_fetch_commands(cmds: &[String]) -> Vec<&String> { + cmds.iter() + .filter(|c| { + let u = c.to_ascii_uppercase(); + u.contains("BODY.PEEK[]") && !u.contains("BODY.PEEK[HEADER]") + }) + .collect() + } + + #[tokio::test] + async fn a_large_folder_is_fetched_across_multiple_bounded_batches() { + // Five messages with a batch window of two must be pulled across + // multiple bounded `UID FETCH` batches -- never a single unbounded + // `1:*` body fetch. The enumeration discovers the UIDs; the bodies are + // then fetched in explicit UID-set chunks of at most the batch size. + let messages: Vec<(u32, u32)> = (1..=5).map(|u| (u, 100)).collect(); + let (emails, checkpoint, cmds) = run_batched_sync(messages, 1, 6, None, 2).await; + + assert_eq!(emails.len(), 5, "every message must be synced"); + assert_eq!(checkpoint, "1:6"); + + let body_cmds = body_fetch_commands(&cmds); + assert_eq!( + body_cmds.len(), + 3, + "5 messages / batch of 2 => 3 bounded body fetches, got: {cmds:?}" + ); + for cmd in &body_cmds { + assert!( + !cmd.contains("1:*") && !cmd.contains(":*"), + "a body fetch must be a bounded UID set, never an open range: {cmd}" + ); + let set = cmd.split_whitespace().nth(3).unwrap_or(""); + let count = set.split(',').count(); + assert!( + count <= 2, + "each body fetch must carry at most the batch size (2) UIDs, got: {cmd}" + ); + } + } + + #[tokio::test] + async fn an_oversized_message_is_fetched_headers_only_not_buffered_whole() { + // Three messages where the middle one reports an RFC822.SIZE above the + // per-message cap. Its full body MUST NOT be requested; instead it is + // fetched headers-only and recorded with an empty body, while the two + // normal messages get full bodies. + let huge = MAX_MESSAGE_BODY_BYTES + 1; + let messages: Vec<(u32, u32)> = vec![(1, 100), (2, huge), (3, 100)]; + let (emails, _checkpoint, cmds) = run_batched_sync(messages, 1, 4, None, 200).await; + + assert_eq!(emails.len(), 3, "no message may be silently dropped"); + + // The oversized UID (2) must NEVER appear in a full-body fetch. + for cmd in body_fetch_commands(&cmds) { + let set = cmd.split_whitespace().nth(3).unwrap_or(""); + assert!( + !set.split(',').any(|u| u.trim() == "2"), + "the oversized message's full body must never be fetched, got: {cmd}" + ); + } + // It MUST be fetched headers-only. + assert!( + cmds.iter().any(|c| { + let u = c.to_ascii_uppercase(); + u.contains("BODY.PEEK[HEADER]") && c.split_whitespace().nth(3).unwrap_or("") == "2" + }), + "the oversized message must be fetched headers-only, got: {cmds:?}" + ); + + // The oversized message is recorded with honest metadata and an EMPTY + // body -- never a fabricated one. + let oversized = emails + .iter() + .find(|e| e.remote_id == "2") + .expect("oversized message must be present"); + assert!( + oversized.body_plain.is_none() && oversized.body_html.is_none(), + "oversized message must carry no body, got: {oversized:?}" + ); + assert!( + oversized.subject.contains('2'), + "oversized message must keep honest header metadata, got: {oversized:?}" + ); + + // The normal messages DID get full bodies. + for uid in ["1", "3"] { + let email = emails + .iter() + .find(|e| e.remote_id == uid) + .expect("normal message present"); + assert!( + email.body_plain.is_some(), + "normal message {uid} must have a body, got: {email:?}" + ); + } + } + + #[tokio::test] + async fn batched_sync_preserves_incremental_checkpointing() { + // Batching must not break the incremental checkpoint: a first sync + // records "{uidvalidity}:{uidnext}", and a second sync fed that + // checkpoint enumerates only the narrowed `{uidnext}:*` range (so a + // fire-once-per-new-message consumer never re-sees old mail). + let messages: Vec<(u32, u32)> = (1..=3).map(|u| (u, 100)).collect(); + let (first, checkpoint, first_cmds) = + run_batched_sync(messages.clone(), 1, 4, None, 2).await; + assert_eq!(first.len(), 3); + assert_eq!(checkpoint, "1:4"); + assert!( + first_cmds.iter().any(|c| c.contains("1:*")), + "first sync enumerates the full range, got: {first_cmds:?}" + ); + + let (second, checkpoint2, second_cmds) = + run_batched_sync(messages, 1, 4, Some(checkpoint.clone()), 2).await; + assert_eq!( + second.len(), + 0, + "no new mail arrived, so an incremental sync returns nothing" + ); + assert_eq!( + checkpoint2, "1:4", + "checkpoint stays stable across an empty pass" + ); + assert!( + second_cmds.iter().any(|c| c.contains("4:*")), + "second sync must enumerate the narrowed range, got: {second_cmds:?}" + ); + assert!( + !second_cmds.iter().any(|c| c.contains("1:*")), + "second sync must NOT re-enumerate the full range, got: {second_cmds:?}" + ); + } + #[tokio::test] async fn a_uidvalidity_change_forces_a_full_resync_instead_of_skipping_mail() { // A mailbox renumber changes UIDVALIDITY and restarts UIDs from low -- 2.54.0 From f85893e128016f503c913d2e92d88a899ccf5e7a Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 17:04:11 -0500 Subject: [PATCH 31/95] feat(proto): enforce id-naming law and return full config from AddAccount (#353) Audit every message across nuncio.proto for the id-naming convention (an entity's own identifier is `id`; a reference to another entity is `_id`) and fix the outliers: RemoveAccountRequest, TestAccountConnectionRequest, DeleteRuleRequest, and UpdateRuleRequest carried a bare `id` referencing an account or rule rather than naming it; MessageSearchHit did the same for the message it matched. AddAccountResponse now returns the persisted AccountConfig (never the password), matching how CreateRule/CreateContact return their created object instead of a bare id. Document Message.id (and the other entity ids) as an opaque, server-assigned token clients must never parse or construct, reflecting the surrogate id introduced for messages. Regenerates the nuncio.v1 descriptor golden for the renamed/changed fields. Refs #276 Co-authored-by: James Maes --- crates/nuncio-cli/src/runner.rs | 41 ++++++----- .../proto/nuncio/v1/descriptor.bin | Bin 15069 -> 15166 bytes .../nuncio-proto/proto/nuncio/v1/nuncio.proto | 46 +++++++++---- crates/nunciod/src/grpc.rs | 64 ++++++++++-------- .../tests/accounts_lifecycle_e2e_test.rs | 14 ++-- crates/nunciod/tests/grpc_auth_test.rs | 6 +- crates/nunciod/tests/spine_e2e_test.rs | 8 ++- crates/nunciod/tests/typed_errors_e2e_test.rs | 2 +- 8 files changed, 113 insertions(+), 68 deletions(-) diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index 33b51e1..10fd6a3 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -876,7 +876,7 @@ impl HeadlessRunner { .iter() .map(|h| { json!({ - "id": h.id, + "id": h.message_id, "title": h.title, "snippet": h.snippet, }) @@ -1383,7 +1383,9 @@ impl HeadlessRunner { }; match client - .delete_rule(nuncio_proto::v1::DeleteRuleRequest { id: id.to_string() }) + .delete_rule(nuncio_proto::v1::DeleteRuleRequest { + rule_id: id.to_string(), + }) .await { Ok(_) => { @@ -1518,7 +1520,7 @@ impl HeadlessRunner { match client .update_rule(nuncio_proto::v1::UpdateRuleRequest { - id: id.to_string(), + rule_id: id.to_string(), name: name.map(str::to_string), nsql: sql.map(str::to_string), priority, @@ -1888,7 +1890,14 @@ impl HeadlessRunner { .await { Ok(response) => { - let account_id = response.into_inner().id; + // Prefer the daemon-confirmed id from the persisted config it + // echoes back; fall back to the id this call requested if + // the response is somehow missing its config. + let account_id = response + .into_inner() + .config + .map(|c| c.id) + .unwrap_or(account_id); if is_caldav { let collection = collection_url.unwrap_or_default(); if json_mode { @@ -2173,7 +2182,9 @@ impl HeadlessRunner { }; match client - .remove_account(nuncio_proto::v1::RemoveAccountRequest { id: id.to_string() }) + .remove_account(nuncio_proto::v1::RemoveAccountRequest { + account_id: id.to_string(), + }) .await { Ok(_) => { @@ -2208,7 +2219,7 @@ impl HeadlessRunner { match client .test_account_connection(nuncio_proto::v1::TestAccountConnectionRequest { - id: id.to_string(), + account_id: id.to_string(), }) .await { @@ -3046,16 +3057,12 @@ mod tests { request: tonic::Request, ) -> Result, tonic::Status> { let req = request.into_inner(); - let id = req - .config - .as_ref() - .map(|c| c.id.clone()) - .unwrap_or_default(); + let config = req.config.clone(); *self .last_add_request .lock() .unwrap_or_else(|e| e.into_inner()) = Some(req); - Ok(tonic::Response::new(AddAccountResponse { id })) + Ok(tonic::Response::new(AddAccountResponse { config })) } async fn update_account( @@ -3397,7 +3404,7 @@ mod tests { let req = request.into_inner(); Ok(tonic::Response::new(SearchMessagesResponse { hits: vec![MessageSearchHit { - id: "msg-stub-1".to_string(), + message_id: "msg-stub-1".to_string(), title: "Stub Subject".to_string(), snippet: format!("...{}...", req.query), }], @@ -4283,13 +4290,13 @@ mod tests { request: tonic::Request, ) -> Result, tonic::Status> { let req = request.into_inner(); - if req.id != "rule-stub-1" { + if req.rule_id != "rule-stub-1" { return Err(tonic::Status::not_found("no such rule")); } *self.last_update.lock().unwrap_or_else(|e| e.into_inner()) = Some(req.clone()); Ok(tonic::Response::new(UpdateRuleResponse { rule: Some(FilterRuleProto { - id: req.id, + id: req.rule_id, name: req.name.unwrap_or_else(|| "Stub Rule".to_string()), target_account: "*".to_string(), priority: req.priority.unwrap_or(5), @@ -4467,7 +4474,7 @@ mod tests { .unwrap_or_else(|e| e.into_inner()) .clone() .expect("stub daemon received a delete_rule request"); - assert_eq!(recorded_delete.id, "rule-stub-1"); + assert_eq!(recorded_delete.rule_id, "rule-stub-1"); // `filter validate` valid + invalid. let validate_ok = runner @@ -4533,7 +4540,7 @@ mod tests { .unwrap_or_else(|e| e.into_inner()) .clone() .expect("stub daemon received an update_rule request"); - assert_eq!(recorded_update.id, "rule-stub-1"); + assert_eq!(recorded_update.rule_id, "rule-stub-1"); assert_eq!(recorded_update.name.as_deref(), Some("Renamed Rule")); assert_eq!(recorded_update.nsql, None); assert_eq!(recorded_update.priority, Some(9)); diff --git a/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin b/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin index 828d294a8ece515d22673a031fc3e9e511ea69c8..b13188a703949fd7a632ec5f75449273968c39ef 100644 GIT binary patch delta 228 zcmcaxy02^lBNNl^qRq@q6FEiQxP%;2QXG?$^Gowef>MhM^7D#QC!6sYP2S1MhM^7D#Qh4{FbGE*cN6&N)* zgMh5bx4FM-t8s}0rRL_BrNR{#mZlb$psQ69=MtTqZzRg1q{k&Qd6kh2Kbnfo@x0t@ zN, ) -> Result, Status> { let req = request.into_inner(); - if req.id.is_empty() { - return Err(Status::invalid_argument("id is required")); + if req.account_id.is_empty() { + return Err(Status::invalid_argument("account_id is required")); } let existing = self .db - .get_account(&req.id) + .get_account(&req.account_id) .await .map_err(|e| Status::internal(format!("failed to look up account: {e}")))? .ok_or_else(|| { errors::status_with_metadata( ErrorReason::AccountNotFound, - format!("account '{}' not found", req.id), - [("account_id".to_string(), req.id.clone())], + format!("account '{}' not found", req.account_id), + [("account_id".to_string(), req.account_id.clone())], ) })?; self.db - .delete_account(&req.id) + .delete_account(&req.account_id) .await .map_err(|e| Status::internal(format!("failed to remove account: {e}")))?; @@ -713,20 +715,20 @@ impl Accounts for AccountsGrpcService { request: Request, ) -> Result, Status> { let req = request.into_inner(); - if req.id.is_empty() { - return Err(Status::invalid_argument("id is required")); + if req.account_id.is_empty() { + return Err(Status::invalid_argument("account_id is required")); } let config = self .db - .get_account(&req.id) + .get_account(&req.account_id) .await .map_err(|e| Status::internal(format!("failed to look up account: {e}")))? .ok_or_else(|| { errors::status_with_metadata( ErrorReason::AccountNotFound, - format!("account '{}' not found", req.id), - [("account_id".to_string(), req.id.clone())], + format!("account '{}' not found", req.account_id), + [("account_id".to_string(), req.account_id.clone())], ) })?; @@ -1557,7 +1559,7 @@ impl Mail for MailGrpcService { let hits = hits .into_iter() .map(|hit| MessageSearchHit { - id: hit.id, + message_id: hit.id, title: hit.title, snippet: hit.snippet, }) @@ -1872,12 +1874,12 @@ impl Filters for FiltersGrpcService { request: Request, ) -> Result, Status> { let req = request.into_inner(); - if req.id.is_empty() { - return Err(Status::invalid_argument("id is required")); + if req.rule_id.is_empty() { + return Err(Status::invalid_argument("rule_id is required")); } self.db - .delete_filter_rule(&req.id) + .delete_filter_rule(&req.rule_id) .await .map_err(|e| Status::internal(format!("failed to delete filter rule: {e}")))?; @@ -1972,12 +1974,12 @@ impl Filters for FiltersGrpcService { .await .map_err(|e| Status::internal(format!("failed to list filter rules: {e}")))? .into_iter() - .find(|r| r.id == req.id) + .find(|r| r.id == req.rule_id) .ok_or_else(|| { errors::status_with_metadata( ErrorReason::RuleNotFound, - format!("filter rule '{}' not found", req.id), - [("rule_id".to_string(), req.id.clone())], + format!("filter rule '{}' not found", req.rule_id), + [("rule_id".to_string(), req.rule_id.clone())], ) })?; @@ -3466,7 +3468,13 @@ mod tests { .await .expect("add_account succeeds") .into_inner(); - assert_eq!(add_response.id, "acct-restart-1"); + assert_eq!( + add_response + .config + .expect("response carries the created config") + .id, + "acct-restart-1" + ); db_first_run.close().await; @@ -4216,7 +4224,7 @@ mod tests { .into_inner() .hits; assert_eq!(hits.len(), 1); - assert_eq!(hits[0].id, "msg-mail-1"); + assert_eq!(hits[0].message_id, "msg-mail-1"); // MarkRead persists: a second GetMessage call reflects the flip. client @@ -4745,7 +4753,7 @@ mod tests { let err = client .delete_rule(DeleteRuleRequest { - id: "rule-1".to_string(), + rule_id: "rule-1".to_string(), }) .await .expect_err("missing bearer token must be rejected"); @@ -4770,7 +4778,7 @@ mod tests { let err = client .update_rule(UpdateRuleRequest { - id: "rule-1".to_string(), + rule_id: "rule-1".to_string(), name: None, nsql: None, priority: None, @@ -4877,7 +4885,7 @@ mod tests { // DeleteRule removes it AND reloads the engine back to empty. client .delete_rule(authed_bearer_request(DeleteRuleRequest { - id: rule_id.clone(), + rule_id: rule_id.clone(), })) .await .expect("delete_rule succeeds"); @@ -5018,7 +5026,7 @@ mod tests { let err = client .delete_rule(authed_bearer_request(DeleteRuleRequest { - id: String::new(), + rule_id: String::new(), })) .await .expect_err("empty id must be rejected"); @@ -5221,7 +5229,7 @@ mod tests { let updated = client .update_rule(authed_bearer_request(UpdateRuleRequest { - id: rule_id.clone(), + rule_id: rule_id.clone(), name: Some("Renamed".to_string()), nsql: None, priority: Some(1), @@ -5281,7 +5289,7 @@ mod tests { let err = client .update_rule(authed_bearer_request(UpdateRuleRequest { - id: "no-such-rule".to_string(), + rule_id: "no-such-rule".to_string(), name: Some("Whatever".to_string()), nsql: None, priority: None, @@ -5313,7 +5321,7 @@ mod tests { let err = client .update_rule(authed_bearer_request(UpdateRuleRequest { - id: created.id, + rule_id: created.id, name: None, nsql: Some("THIS IS NOT VALID NSQL AT ALL {{{".to_string()), priority: None, diff --git a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs index 8da6b0d..37a5b33 100644 --- a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs +++ b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs @@ -164,7 +164,13 @@ async fn account_lifecycle_round_trips_over_authenticated_grpc_offline() { .await .expect("add_account succeeds") .into_inner(); - assert_eq!(add_response.id, ACCOUNT_ID); + assert_eq!( + add_response + .config + .expect("response carries the created config") + .id, + ACCOUNT_ID + ); // ---- (b) ListAccounts round-trips BOTH distinct TLS modes ---- let listed = accounts_client @@ -204,7 +210,7 @@ async fn account_lifecycle_round_trips_over_authenticated_grpc_offline() { // ---- (d) TestAccountConnection reports the GENUINE per-protocol result ---- let report = accounts_client .test_account_connection(TestAccountConnectionRequest { - id: ACCOUNT_ID.to_string(), + account_id: ACCOUNT_ID.to_string(), }) .await .expect("test_account_connection succeeds") @@ -220,7 +226,7 @@ async fn account_lifecycle_round_trips_over_authenticated_grpc_offline() { // ---- (e) RemoveAccount genuinely deletes it ---- accounts_client .remove_account(RemoveAccountRequest { - id: ACCOUNT_ID.to_string(), + account_id: ACCOUNT_ID.to_string(), }) .await .expect("remove_account succeeds"); @@ -237,7 +243,7 @@ async fn account_lifecycle_round_trips_over_authenticated_grpc_offline() { // fabricated success. let err = accounts_client .test_account_connection(TestAccountConnectionRequest { - id: ACCOUNT_ID.to_string(), + account_id: ACCOUNT_ID.to_string(), }) .await .expect_err("testing a removed account must fail"); diff --git a/crates/nunciod/tests/grpc_auth_test.rs b/crates/nunciod/tests/grpc_auth_test.rs index 22b543e..bdd5db8 100644 --- a/crates/nunciod/tests/grpc_auth_test.rs +++ b/crates/nunciod/tests/grpc_auth_test.rs @@ -178,7 +178,7 @@ async fn rejects_calls_to_new_filters_rpcs_without_a_bearer_token() { let err = client .update_rule(nuncio_proto::v1::UpdateRuleRequest { - id: "rule-1".to_string(), + rule_id: "rule-1".to_string(), name: None, nsql: None, priority: None, @@ -221,7 +221,7 @@ async fn rejects_calls_to_new_accounts_rpcs_without_a_bearer_token() { let remove_err = client .remove_account(nuncio_proto::v1::RemoveAccountRequest { - id: "acct-1".to_string(), + account_id: "acct-1".to_string(), }) .await .expect_err("unauthenticated remove_account must be rejected"); @@ -229,7 +229,7 @@ async fn rejects_calls_to_new_accounts_rpcs_without_a_bearer_token() { let test_err = client .test_account_connection(nuncio_proto::v1::TestAccountConnectionRequest { - id: "acct-1".to_string(), + account_id: "acct-1".to_string(), }) .await .expect_err("unauthenticated test_account_connection must be rejected"); diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index d7ad21f..e99d5b3 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -202,7 +202,13 @@ async fn full_mail_spine_round_trips_over_authenticated_grpc_with_mocked_protoco .await .expect("add_account succeeds") .into_inner(); - assert_eq!(add_response.id, ACCOUNT_ID); + assert_eq!( + add_response + .config + .expect("response carries the created config") + .id, + ACCOUNT_ID + ); let list_response = accounts_client .list_accounts(ListAccountsRequest {}) diff --git a/crates/nunciod/tests/typed_errors_e2e_test.rs b/crates/nunciod/tests/typed_errors_e2e_test.rs index 6e36958..ee4f4c0 100644 --- a/crates/nunciod/tests/typed_errors_e2e_test.rs +++ b/crates/nunciod/tests/typed_errors_e2e_test.rs @@ -93,7 +93,7 @@ async fn typed_error_reasons_decode_from_status_details_over_the_wire() { .expect("accounts client connects"); let status = accounts .remove_account(RemoveAccountRequest { - id: "no-such-account".to_string(), + account_id: "no-such-account".to_string(), }) .await .expect_err("removing a nonexistent account must fail"); -- 2.54.0 From d1a6f8f417e08c1780b020ab4129e36defed89ef Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 17:05:21 -0500 Subject: [PATCH 32/95] feat(store): implement contact search over contacts_fts (#354) Add search_contacts/search_contacts_page to SearchEngine, mirroring the existing message and event search queries and their keyset pagination. The contacts_fts virtual table and its insert/update/delete triggers already existed but were unused; new tests prove the triggers keep the index in sync across insert, update, and delete, that ranked hits correctly include/exclude matches, and that sanitized/edge queries do not error. Refs #318 Co-authored-by: James Maes --- crates/nuncio-store/src/search.rs | 236 ++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) diff --git a/crates/nuncio-store/src/search.rs b/crates/nuncio-store/src/search.rs index dc25f64..92d63e1 100644 --- a/crates/nuncio-store/src/search.rs +++ b/crates/nuncio-store/src/search.rs @@ -133,6 +133,92 @@ impl<'a> SearchEngine<'a> { Ok((hits, next)) } + /// Perform a full-text trigram search over contact display names, organizations, and + /// email addresses. + pub async fn search_contacts(&self, query: &str) -> Result, DatabaseError> { + let clean_query = Self::sanitize_fts5_query(query); + if clean_query.is_empty() { + return Ok(Vec::new()); + } + + let rows: Vec<(String, String, String)> = sqlx::query_as( + r#" + SELECT id, display_name, snippet(contacts_fts, 3, '', '', '...', 10) as snippet + FROM contacts_fts + WHERE contacts_fts MATCH ? + ORDER BY rank + LIMIT 50 + "#, + ) + .bind(&clean_query) + .fetch_all(self.db.pool()) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map(|(id, title, snippet)| SearchHit { id, title, snippet }) + .collect()) + } + + /// Keyset-paginated full-text search over contact display names, + /// organizations, and email addresses, ordered by relevance (`rank ASC, id + /// ASC`). Mirrors [`Self::search_messages_page`]'s rank-materializing + /// subquery so the same keyset-pagination guarantee (every hit returned + /// exactly once, across pages) applies to contact search. + pub async fn search_contacts_page( + &self, + query: &str, + after: Option<(f64, String)>, + page_size: usize, + ) -> Result<(Vec, Option<(f64, String)>), DatabaseError> { + let clean_query = Self::sanitize_fts5_query(query); + if clean_query.is_empty() { + return Ok((Vec::new(), None)); + } + let fetch = page_size.saturating_add(1); + + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT id, title, snippet, rk FROM ( \ + SELECT id, display_name AS title, \ + snippet(contacts_fts, 3, '', '', '...', 10) AS snippet, rank AS rk \ + FROM contacts_fts WHERE contacts_fts MATCH ", + ); + builder.push_bind(clean_query); + builder.push(") "); + if let Some((rank, id)) = &after { + builder.push("WHERE (rk > "); + builder.push_bind(*rank); + builder.push(" OR (rk = "); + builder.push_bind(*rank); + builder.push(" AND id > "); + builder.push_bind(id.clone()); + builder.push("))"); + } + builder.push(" ORDER BY rk ASC, id ASC LIMIT "); + builder.push_bind(fetch as i64); + + let rows = builder + .build_query_as::<(String, String, String, f64)>() + .fetch_all(self.db.pool()) + .await + .map_err(DatabaseError::Query)?; + + let has_more = rows.len() > page_size; + let mut last_key: Option<(f64, String)> = None; + let hits: Vec = rows + .into_iter() + .take(page_size) + .map(|(id, title, snippet, rk)| { + last_key = Some((rk, id.clone())); + SearchHit { id, title, snippet } + }) + .collect(); + + let next = if has_more { last_key } else { None }; + Ok((hits, next)) + } + /// Perform a full-text trigram search over calendar event summaries and locations. pub async fn search_events(&self, query: &str) -> Result, DatabaseError> { let clean_query = Self::sanitize_fts5_query(query); @@ -359,6 +445,156 @@ mod tests { assert_eq!(hits[0].title, "Architecture Summit"); } + async fn insert_contact( + db: &DatabaseEngine, + id: &str, + display_name: &str, + organization: &str, + emails_json: &str, + ) { + sqlx::query( + "INSERT INTO contacts \ + (id, display_name, organization, emails_json, created_at, updated_at) \ + VALUES (?, ?, ?, ?, '2024-01-01T00:00:00Z', '2024-01-01T00:00:00Z')", + ) + .bind(id) + .bind(display_name) + .bind(organization) + .bind(emails_json) + .execute(db.pool()) + .await + .unwrap(); + } + + /// Proves `contacts_fts` is actually populated by the `contacts_ai`/`contacts_au` triggers + /// (not merely created and left unused): several contacts are inserted, a matching term + /// ranks the true matches and excludes the non-matching contact. + #[tokio::test] + async fn fts5_contact_search_ranks_matches_and_excludes_others() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let search = SearchEngine::new(&db); + + assert!(search.search_contacts("").await.unwrap().is_empty()); + + insert_contact(&db, "ct-1", "Alice Architect", "Acme Corp", "[]").await; + insert_contact(&db, "ct-2", "Bob Baker", "Baker Studio", "[]").await; + insert_contact(&db, "ct-3", "Carol Architect", "Acme Corp", "[]").await; + + let hits = search.search_contacts("Architect").await.unwrap(); + let ids: Vec<&str> = hits.iter().map(|h| h.id.as_str()).collect(); + assert_eq!(hits.len(), 2); + assert!(ids.contains(&"ct-1")); + assert!(ids.contains(&"ct-3")); + assert!(!ids.contains(&"ct-2")); + + let bob_hits = search.search_contacts("Bob").await.unwrap(); + assert_eq!(bob_hits.len(), 1); + assert_eq!(bob_hits[0].id, "ct-2"); + assert_eq!(bob_hits[0].title, "Bob Baker"); + } + + /// Updating and deleting a contact keeps `contacts_fts` in sync via the `contacts_au`/ + /// `contacts_ad` triggers: a renamed contact stops matching its old name and starts + /// matching its new one, and a deleted contact stops matching entirely. + #[tokio::test] + async fn fts5_contact_search_reflects_update_and_delete() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let search = SearchEngine::new(&db); + + insert_contact(&db, "ct-upd-1", "Dana Original", "Original Org", "[]").await; + assert_eq!(search.search_contacts("Original").await.unwrap().len(), 1); + + sqlx::query( + "UPDATE contacts SET display_name = 'Dana Renamed', organization = 'New Org' \ + WHERE id = 'ct-upd-1'", + ) + .execute(db.pool()) + .await + .unwrap(); + + assert!(search.search_contacts("Original").await.unwrap().is_empty()); + let hits = search.search_contacts("Renamed").await.unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].id, "ct-upd-1"); + + sqlx::query("DELETE FROM contacts WHERE id = 'ct-upd-1'") + .execute(db.pool()) + .await + .unwrap(); + + assert!(search.search_contacts("Renamed").await.unwrap().is_empty()); + } + + /// A search term matching an indexed email address (stored as JSON) is found, and a + /// sanitized/edge-case query (FTS5 operator characters) does not error -- it just yields + /// no matches when nothing survives sanitization to match. + #[tokio::test] + async fn fts5_contact_search_matches_email_and_survives_edge_query() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let search = SearchEngine::new(&db); + + insert_contact( + &db, + "ct-email-1", + "Erin Example", + "", + r#"[{"email":"erin.example@nuncio.mx","label":"work"}]"#, + ) + .await; + + let hits = search.search_contacts("erin.example").await.unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].id, "ct-email-1"); + + let edge_hits = search.search_contacts("\"weird*: query\"").await.unwrap(); + assert!(edge_hits.is_empty()); + } + + /// Keyset pagination over contact search must return every matching hit exactly once, + /// mirroring the message-search pagination guarantee. + #[tokio::test] + async fn search_contacts_page_returns_every_hit_once_no_gaps() { + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + for i in 0..5 { + insert_contact( + &db, + &format!("ct-page-{i}"), + "Planning Committee Member", + "Acme Corp", + "[]", + ) + .await; + } + + let search = SearchEngine::new(&db); + let mut seen = Vec::new(); + let mut after: Option<(f64, String)> = None; + let mut pages = 0; + loop { + let (hits, next) = search + .search_contacts_page("Planning", after.clone(), 2) + .await + .unwrap(); + pages += 1; + assert!(hits.len() <= 2, "page must not exceed page_size"); + seen.extend(hits.iter().map(|h| h.id.clone())); + match next { + Some(cursor) => after = Some(cursor), + None => break, + } + assert!(pages < 100, "pagination must terminate"); + } + + seen.sort(); + seen.dedup(); + assert_eq!( + seen.len(), + 5, + "every matching contact returned exactly once" + ); + assert!(pages >= 3, "5 hits at page_size 2 must span multiple pages"); + } + #[test] fn fts5_query_sanitizer_removes_special_operators() { assert_eq!(SearchEngine::sanitize_fts5_query(""), ""); -- 2.54.0 From 9bfffe59da53de9f9a0178a6cc1471376c8d258c Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 17:27:02 -0500 Subject: [PATCH 33/95] feat(nunciod): bound gRPC server concurrency/streams/timeout/message size (#355) Add sensible resource bounds to the tonic Server builder so a misbehaving or malicious client cannot exhaust the daemon: a per-connection concurrency limit, a per-call timeout generous enough for the longest legitimate unary sync, an HTTP/2 concurrent-stream cap, a per-service inbound message-size limit raised above tonic's default to fit real mail attachments, and TCP/HTTP2 keepalive. Refs #304 Co-authored-by: James Maes --- crates/nunciod/src/grpc.rs | 155 ++++++++++++++++++++++++++++++++++--- 1 file changed, 146 insertions(+), 9 deletions(-) diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 0b95c1c..cc948ef 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -54,16 +54,68 @@ use nuncio_store::search::SearchEngine; use nuncio_store::vault::SecretManager; use std::pin::Pin; use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use subtle::ConstantTimeEq; use thiserror::Error; use tokio::net::TcpListener; use tokio_stream::wrappers::errors::BroadcastStreamRecvError; use tokio_stream::wrappers::{BroadcastStream, ReceiverStream, TcpListenerStream}; use tokio_stream::{Stream, StreamExt}; +use tonic::service::interceptor::InterceptedService; use tonic::transport::Server; use tonic::{Request, Response, Status}; +// Resource bounds applied to the tonic `Server` builder in +// `serve_on_listener_with_overrides_and_shutdown`. These exist so a +// misbehaving or malicious client cannot exhaust the daemon by opening +// unbounded connections/streams, sending oversized frames, or holding a +// handler open forever. + +/// Caps in-flight requests per client connection. A single connection that +/// opens far more concurrent streams than any legitimate client needs (the +/// CLI and future native clients issue a handful of concurrent calls at +/// most) is throttled rather than allowed to consume worker capacity. +const GRPC_CONCURRENCY_LIMIT_PER_CONNECTION: usize = 32; + +/// Per-call deadline applied by the tonic transport to every RPC's request +/// future. This bounds unary handlers (a stuck backend call must not hang +/// the connection forever) while staying generous enough for the longest +/// legitimate unary call, `Mail.Sync`/`Calendar.Sync`/`Contacts.Sync` +/// against a large, slow upstream mailbox. +/// +/// Server-streaming RPCs (`System.Subscribe`) are unaffected in practice: +/// tonic's timeout wraps the handler's future, which for a streaming +/// response resolves as soon as the response headers/body stream are +/// constructed, not when the stream itself finishes emitting items -- so a +/// long-lived `Subscribe` connection is not cut off by this deadline. +const GRPC_REQUEST_TIMEOUT: Duration = Duration::from_secs(300); + +/// HTTP/2 concurrent stream cap per connection, independent of the +/// concurrency limiter above (this bounds protocol-level multiplexing; +/// `concurrency_limit_per_connection` bounds how many of those streams are +/// actively dispatched to a handler at once). +const GRPC_MAX_CONCURRENT_STREAMS: u32 = 64; + +/// Largest inbound gRPC message the daemon will decode, per service. +/// Deliberately larger than tonic's unconfigured 4 MiB default: `Mail` +/// requests can legitimately carry a full message body plus attachment +/// bytes inline (`SendMessageRequest.attachments`), so 4 MiB is too tight +/// for a real email with a modest attachment. 16 MiB comfortably covers +/// that case while still bounding the allocation a client can force with +/// one oversized frame far below "unbounded". +const GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES: usize = 16 * 1024 * 1024; + +/// TCP-level keepalive so a half-open connection (client crashed or network +/// dropped without a clean close) is detected and reclaimed instead of +/// holding a connection slot indefinitely. +const GRPC_TCP_KEEPALIVE: Duration = Duration::from_secs(60); + +/// HTTP/2 PING-based keepalive: send a ping after this much idle time... +const GRPC_HTTP2_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30); + +/// ...and close the connection if the pong doesn't arrive within this long. +const GRPC_HTTP2_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(20); + // The gRPC loopback address defaults/resolver live in `nuncio-proto` (see // `nuncio_proto::addr`) so that thin presentation-shell clients (e.g. // `nuncio-cli`) agree on the same default address without depending on this @@ -2625,7 +2677,11 @@ pub async fn serve_on_listener_with_overrides_and_shutdown( event_bus: event_bus.clone(), }; let system_interceptor = BearerAuthInterceptor::new(token.clone()); - let system_svc = SystemServer::with_interceptor(system_service, system_interceptor); + let system_svc = InterceptedService::new( + SystemServer::new(system_service) + .max_decoding_message_size(GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES), + system_interceptor, + ); let accounts_service = AccountsGrpcService { db: db.clone(), @@ -2635,7 +2691,11 @@ pub async fn serve_on_listener_with_overrides_and_shutdown( .unwrap_or_else(|| Arc::new(RealAccountConnectionTester)), }; let accounts_interceptor = BearerAuthInterceptor::new(token.clone()); - let accounts_svc = AccountsServer::with_interceptor(accounts_service, accounts_interceptor); + let accounts_svc = InterceptedService::new( + AccountsServer::new(accounts_service) + .max_decoding_message_size(GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES), + accounts_interceptor, + ); // Mail: mounted behind its own `BearerAuthInterceptor`, exactly like // `System` and `Accounts` above -- see the hard invariant documented on @@ -2648,7 +2708,11 @@ pub async fn serve_on_listener_with_overrides_and_shutdown( overrides, }; let mail_interceptor = BearerAuthInterceptor::new(token.clone()); - let mail_svc = MailServer::with_interceptor(mail_service, mail_interceptor); + let mail_svc = InterceptedService::new( + MailServer::new(mail_service) + .max_decoding_message_size(GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES), + mail_interceptor, + ); // Filters: mounted behind its own `BearerAuthInterceptor`, exactly like // `System`/`Accounts`/`Mail` above -- see the hard invariant documented @@ -2658,21 +2722,33 @@ pub async fn serve_on_listener_with_overrides_and_shutdown( filter_engine, }; let filters_interceptor = BearerAuthInterceptor::new(token.clone()); - let filters_svc = FiltersServer::with_interceptor(filters_service, filters_interceptor); + let filters_svc = InterceptedService::new( + FiltersServer::new(filters_service) + .max_decoding_message_size(GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES), + filters_interceptor, + ); // Export: mounted behind its own `BearerAuthInterceptor`, exactly like // every other service above -- see the hard invariant documented on // this function's doc comment. let export_service = ExportGrpcService { db: db.clone() }; let export_interceptor = BearerAuthInterceptor::new(token.clone()); - let export_svc = ExportServer::with_interceptor(export_service, export_interceptor); + let export_svc = InterceptedService::new( + ExportServer::new(export_service) + .max_decoding_message_size(GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES), + export_interceptor, + ); // Audit: mounted behind its own `BearerAuthInterceptor`, exactly like // every other service above -- see the hard invariant documented on // this function's doc comment. let audit_service = AuditGrpcService { db: db.clone() }; let audit_interceptor = BearerAuthInterceptor::new(token.clone()); - let audit_svc = AuditServer::with_interceptor(audit_service, audit_interceptor); + let audit_svc = InterceptedService::new( + AuditServer::new(audit_service) + .max_decoding_message_size(GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES), + audit_interceptor, + ); // Calendar: mounted behind its own `BearerAuthInterceptor`, exactly like // every other service above -- see the hard invariant documented on @@ -2683,7 +2759,11 @@ pub async fn serve_on_listener_with_overrides_and_shutdown( overrides: calendar_overrides, }; let calendar_interceptor = BearerAuthInterceptor::new(token.clone()); - let calendar_svc = CalendarServer::with_interceptor(calendar_service, calendar_interceptor); + let calendar_svc = InterceptedService::new( + CalendarServer::new(calendar_service) + .max_decoding_message_size(GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES), + calendar_interceptor, + ); // Contacts: mounted behind its own `BearerAuthInterceptor`, exactly like // every other service above -- see the hard invariant documented on @@ -2693,9 +2773,19 @@ pub async fn serve_on_listener_with_overrides_and_shutdown( overrides: contacts_overrides, }; let contacts_interceptor = BearerAuthInterceptor::new(token); - let contacts_svc = ContactsServer::with_interceptor(contacts_service, contacts_interceptor); + let contacts_svc = InterceptedService::new( + ContactsServer::new(contacts_service) + .max_decoding_message_size(GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES), + contacts_interceptor, + ); Server::builder() + .concurrency_limit_per_connection(GRPC_CONCURRENCY_LIMIT_PER_CONNECTION) + .timeout(GRPC_REQUEST_TIMEOUT) + .max_concurrent_streams(GRPC_MAX_CONCURRENT_STREAMS) + .tcp_keepalive(Some(GRPC_TCP_KEEPALIVE)) + .http2_keepalive_interval(Some(GRPC_HTTP2_KEEPALIVE_INTERVAL)) + .http2_keepalive_timeout(Some(GRPC_HTTP2_KEEPALIVE_TIMEOUT)) .add_service(system_svc) .add_service(accounts_svc) .add_service(mail_svc) @@ -4306,6 +4396,53 @@ mod tests { assert_eq!(err.code(), Code::InvalidArgument); } + /// Proves the server's configured `max_decoding_message_size` (see + /// `GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES`) is actually wired into the + /// `Server::builder()` chain and not just documented: a message body + /// that exceeds the configured limit is rejected with + /// `OutOfRange` at decode time, before the handler (and its field + /// validation, or the "no account configured" business-logic error) + /// ever runs. + #[tokio::test] + async fn send_message_rejects_body_larger_than_max_decoding_message_size() { + let event_bus = Arc::new(EventBus::new()); + let (addr, _handle, _dir) = spawn_test_server(event_bus, "correct-token").await; + + let mut client = MailClient::connect(format!("http://{addr}")) + .await + .expect("client connects"); + let mut req = valid_send_message_request(); + req.body_text = "x".repeat(GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES + 1); + let err = client + .send_message(authed_bearer_request(req)) + .await + .expect_err("oversized message must be rejected"); + assert_eq!(err.code(), Code::OutOfRange); + } + + /// Proves the raised limit is real, not a no-op: a body larger than + /// tonic's unconfigured 4 MiB default but still under the daemon's + /// configured `GRPC_MAX_DECODING_MESSAGE_SIZE_BYTES` decodes + /// successfully and reaches the handler -- surfacing the expected + /// "no account configured" business error rather than + /// `OutOfRange`. + #[tokio::test] + async fn send_message_accepts_body_above_default_but_within_configured_limit() { + let event_bus = Arc::new(EventBus::new()); + let (addr, _handle, _dir) = spawn_test_server(event_bus, "correct-token").await; + + let mut client = MailClient::connect(format!("http://{addr}")) + .await + .expect("client connects"); + let mut req = valid_send_message_request(); + req.body_text = "x".repeat(6 * 1024 * 1024); + let err = client + .send_message(authed_bearer_request(req)) + .await + .expect_err("no configured account must still be rejected"); + assert_eq!(err.code(), Code::Internal); + } + /// Proves `SendMessage` never fabricates success -- with no account /// configured at all, the daemon has nothing to send from, and this /// surfaces as `Status::internal` rather than a fabricated -- 2.54.0 From db60f30018920d4d6f349f63c89118d2765da7b5 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 17:28:22 -0500 Subject: [PATCH 34/95] feat(proto): enum hygiene for rule export format and account TLS (#356) Introduce a dedicated RuleExportFormat enum (SQL/JSON) for ExportRules, replacing the stringly "sql"/"json" format field; UNSPECIFIED is rejected with invalid_argument rather than silently defaulting. Reuse of the mailbox Export service's ExportFormat enum was rejected since its members (MBOX/EML_ZIP/JSON/JSONL) describe portable mailbox output, not rule rendering, and the two have no overlapping meaning. Drop AccountConfig.use_tls from the wire contract (reserving field 7) and from nuncio_core::AccountConfig: it duplicated imap_tls_mode/smtp_tls_mode and was never read for behavior anywhere in the daemon, CLI, or store -- only ever written and then ignored. The SQLite accounts.use_tls column is kept (with a default) for on-disk compatibility with existing rows; no code reads it anymore. Regenerate the nuncio.v1 descriptor golden. Refs #277 Co-authored-by: James Maes --- crates/nuncio-cli/src/runner.rs | 32 ++++++--- crates/nuncio-core/src/config.rs | 4 -- .../proto/nuncio/v1/descriptor.bin | Bin 15166 -> 15298 bytes .../nuncio-proto/proto/nuncio/v1/nuncio.proto | 19 ++++-- crates/nuncio-store/src/db.rs | 25 +++---- crates/nuncio-store/src/recovery.rs | 8 +-- crates/nunciod/src/grpc.rs | 63 +++++++++++------- crates/nunciod/src/orchestrator.rs | 1 - crates/nunciod/src/send.rs | 1 - crates/nunciod/src/sync.rs | 2 - .../tests/accounts_lifecycle_e2e_test.rs | 1 - .../tests/caldav_production_e2e_test.rs | 1 - .../nunciod/tests/outbox_executor_e2e_test.rs | 1 - crates/nunciod/tests/spine_e2e_test.rs | 1 - 14 files changed, 92 insertions(+), 67 deletions(-) diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index 10fd6a3..7afe09a 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -53,6 +53,20 @@ fn parse_account_protocol(protocol: &str) -> Result Result { + match format.to_ascii_lowercase().as_str() { + "sql" => Ok(nuncio_proto::v1::RuleExportFormat::Sql), + "json" => Ok(nuncio_proto::v1::RuleExportFormat::Json), + other => Err(format!( + "invalid export format '{other}' (expected sql or json)" + )), + } +} + /// Renders a wire-format `google.protobuf.Timestamp` as an RFC 3339 UTC /// string for human-readable and `--json` output. `nanos` outside /// `0..1_000_000_000` (never produced by the daemon's own mappers) and an @@ -1561,6 +1575,11 @@ impl HeadlessRunner { /// the daemon's real, persistent store -- not this runner's own /// ephemeral local `db`. async fn handle_filter_export(&self, format: &str, json_mode: bool) -> String { + let format = match parse_rule_export_format(format) { + Ok(f) => f, + Err(e) => return Self::render_error(&e, json_mode), + }; + let mut client = match self.connect_filters_client().await { Ok(client) => client, Err(e) => return Self::render_error(&e, json_mode), @@ -1568,7 +1587,7 @@ impl HeadlessRunner { match client .export_rules(nuncio_proto::v1::ExportRulesRequest { - format: format.to_string(), + format: format.into(), }) .await { @@ -1863,7 +1882,6 @@ impl HeadlessRunner { imap_host.to_string() }, server_port: if is_caldav { 0 } else { u32::from(imap_port) }, - use_tls: true, imap_tls_mode: map_tls_mode_to_proto(imap_tls_mode).into(), smtp_tls_mode: map_tls_mode_to_proto(smtp_tls_mode).into(), keyring_secret_key: keyring_key.clone(), @@ -1974,7 +1992,6 @@ impl HeadlessRunner { "server_port": a.server_port, "smtp_host": a.smtp_host, "smtp_port": a.smtp_port, - "use_tls": a.use_tls, "imap_tls_mode": a.imap_tls_mode().as_str_name(), "smtp_tls_mode": a.smtp_tls_mode().as_str_name(), "keyring_secret_key": a.keyring_secret_key, @@ -1997,15 +2014,16 @@ impl HeadlessRunner { )); } else { out.push_str(&format!( - "\n [{}] {} <{}> IMAP {}:{} SMTP {}:{} TLS={}", + "\n [{}] {} <{}> IMAP {}:{} ({}) SMTP {}:{} ({})", a.id, a.name, a.email_address, a.server_host, a.server_port, + a.imap_tls_mode().as_str_name(), a.smtp_host, a.smtp_port, - a.use_tls, + a.smtp_tls_mode().as_str_name(), )); } } @@ -2069,7 +2087,6 @@ impl HeadlessRunner { "server_port": account.server_port, "smtp_host": account.smtp_host, "smtp_port": account.smtp_port, - "use_tls": account.use_tls, "imap_tls_mode": account.imap_tls_mode().as_str_name(), "smtp_tls_mode": account.smtp_tls_mode().as_str_name(), "keyring_secret_key": account.keyring_secret_key, @@ -3103,7 +3120,6 @@ mod tests { protocol: AccountProtocolProto::ImapSmtp.into(), server_host: "imap.nuncio.mx".to_string(), server_port: 993, - use_tls: true, imap_tls_mode: TlsModeProto::ImplicitTls.into(), smtp_tls_mode: TlsModeProto::ImplicitTls.into(), keyring_secret_key: "nuncio/acct-stub-1".to_string(), @@ -4320,7 +4336,7 @@ mod tests { request: tonic::Request, ) -> Result, tonic::Status> { let req = request.into_inner(); - let content = if req.format == "json" { + let content = if req.format() == nuncio_proto::v1::RuleExportFormat::Json { r#"[{"id":"rule-stub-1"}]"#.to_string() } else { "SELECT * FROM emails WHERE subject CONTAINS 'Urgent' ACTION MARK READ" diff --git a/crates/nuncio-core/src/config.rs b/crates/nuncio-core/src/config.rs index df39c9c..348bbec 100644 --- a/crates/nuncio-core/src/config.rs +++ b/crates/nuncio-core/src/config.rs @@ -95,8 +95,6 @@ pub struct AccountConfig { pub smtp_host: String, /// SMTP server port number, used for outbound mail. pub smtp_port: u16, - /// Whether TLS connection encryption is enabled. - pub use_tls: bool, /// IMAP connection security transport mode. pub imap_tls_mode: TlsMode, /// SMTP connection security transport mode. @@ -247,7 +245,6 @@ mod tests { server_port: 443, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: TlsMode::ImplicitTls, smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-123".to_string(), @@ -266,7 +263,6 @@ mod tests { server_port: 0, smtp_host: String::new(), smtp_port: 0, - use_tls: true, imap_tls_mode: TlsMode::ImplicitTls, smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-cal-1".to_string(), diff --git a/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin b/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin index b13188a703949fd7a632ec5f75449273968c39ef..21045b8042fb3ee6a230729a84bae43aa4d01b36 100644 GIT binary patch delta 252 zcmdm2cBp&gHCbM#0+5Cyqj*;Jsg@avyBZ$4UI5oZ`r+Bjg_i+Iy zPc9+Xih}&2lAzL@)Z(Dj!qU{@5+P$QwzT}B+{6+IMg>L<<6R>H{DVT`-TZ@m9Yf+n{elBroju(=U0nni ogrvB{u&4i7CbM#0kr3x%FD*`uFUcvEU{_$&;0R&^azk>8H{am2 zXJl7O=yL-A1^Xh` diff --git a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto index 742c34b..6275953 100644 --- a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto +++ b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto @@ -270,8 +270,8 @@ message AccountConfig { string server_host = 5; // Server port number (1..=65535) (IMAP/JMAP). uint32 server_port = 6; - // Whether TLS connection encryption is enabled. - bool use_tls = 7; + reserved 7; + reserved "use_tls"; // IMAP connection security transport mode. TlsMode imap_tls_mode = 8; // SMTP connection security transport mode. @@ -844,10 +844,21 @@ message UpdateRuleResponse { FilterRule rule = 1; } +// Rendering format for `ExportRules`. Kept distinct from `ExportFormat` +// (the mailbox `Export` service's format enum) because the two have no +// overlapping members and no shared meaning -- rule export renders lossless +// NSQL source or a JSON array of rule definitions, never MBOX/EML/JSONL. +enum RuleExportFormat { + RULE_EXPORT_FORMAT_UNSPECIFIED = 0; + // Lossless NSQL source text, one rule per line. + RULE_EXPORT_FORMAT_SQL = 1; + // JSON array of rule definitions. + RULE_EXPORT_FORMAT_JSON = 2; +} + // ExportRulesRequest names the rendering format for `ExportRules`. message ExportRulesRequest { - // "sql" or "json". - string format = 1; + RuleExportFormat format = 1; } // ExportRulesResponse carries every persisted rule pre-rendered as a single diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index c58912c..2ff1c62 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -499,7 +499,11 @@ impl DatabaseEngine { protocol TEXT NOT NULL, server_host TEXT NOT NULL, server_port INTEGER NOT NULL, - use_tls INTEGER NOT NULL, + -- Retained (with a default) for on-disk compatibility with + -- rows written before `imap_tls_mode`/`smtp_tls_mode` became + -- the authoritative transport-security setting; no code + -- reads or writes this column anymore. + use_tls INTEGER NOT NULL DEFAULT 1, keyring_secret_key TEXT NOT NULL, sync_interval_secs INTEGER NOT NULL, smtp_host TEXT, @@ -964,7 +968,13 @@ impl DatabaseEngine { .bind(protocol_str) .bind(&config.server_host) .bind(config.server_port as i64) - .bind(if config.use_tls { 1i64 } else { 0i64 }) + // `use_tls` is a retained-but-unread on-disk column (see the + // `CREATE TABLE`'s comment) -- written as a fixed placeholder since + // no `AccountConfig` field backs it anymore, satisfying its + // pre-existing `NOT NULL` constraint on every accounts table + // regardless of whether it was created before or after this column + // stopped being read. + .bind(1i64) .bind(&config.keyring_secret_key) .bind(config.sync_interval_secs as i64) .bind(&config.smtp_host) @@ -1010,7 +1020,6 @@ impl DatabaseEngine { String, String, i64, - i64, String, i64, Option, @@ -1020,7 +1029,7 @@ impl DatabaseEngine { Option, )> = sqlx::query_as( r#" - SELECT id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url + SELECT id, name, email_address, protocol, server_host, server_port, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url FROM accounts "#, ) @@ -1038,7 +1047,6 @@ impl DatabaseEngine { protocol_str, server_host, server_port, - use_tls, keyring_secret_key, sync_interval_secs, smtp_host, @@ -1069,7 +1077,6 @@ impl DatabaseEngine { server_port: server_port as u16, smtp_host: resolved_smtp_host, smtp_port: resolved_smtp_port, - use_tls: use_tls != 0, imap_tls_mode: tls_mode_from_db(&imap_tls_mode), smtp_tls_mode: tls_mode_from_db(&smtp_tls_mode), keyring_secret_key, @@ -2765,7 +2772,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-contended-1".to_string(), @@ -3345,7 +3351,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-test-1".to_string(), @@ -3389,7 +3394,6 @@ mod tests { server_port: 143, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 25, - use_tls: false, imap_tls_mode: nuncio_core::TlsMode::StartTls, smtp_tls_mode: nuncio_core::TlsMode::Plain, keyring_secret_key: "nuncio/acct-tls-1".to_string(), @@ -3430,7 +3434,6 @@ mod tests { server_port: 0, smtp_host: String::new(), smtp_port: 0, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-caldav-store-1".to_string(), @@ -3468,7 +3471,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-del-1".to_string(), @@ -3660,7 +3662,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 587, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::StartTls, keyring_secret_key: "nuncio/acct-post-migration".to_string(), diff --git a/crates/nuncio-store/src/recovery.rs b/crates/nuncio-store/src/recovery.rs index 6bc97ef..30a1fa0 100644 --- a/crates/nuncio-store/src/recovery.rs +++ b/crates/nuncio-store/src/recovery.rs @@ -18,7 +18,6 @@ type SalvagedAccountRow = ( String, String, i64, - i64, String, i64, Option, @@ -384,7 +383,7 @@ impl SqliteRecoveryEngine { async fn salvage_accounts(pool: &sqlx::SqlitePool) -> Vec { let rows: Result, _> = sqlx::query_as( - "SELECT id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url FROM accounts" + "SELECT id, name, email_address, protocol, server_host, server_port, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url FROM accounts" ) .fetch_all(pool) .await; @@ -400,7 +399,6 @@ impl SqliteRecoveryEngine { protocol_str, server_host, server_port, - use_tls, keyring_secret_key, sync_interval_secs, smtp_host, @@ -433,7 +431,6 @@ impl SqliteRecoveryEngine { server_port: server_port as u16, smtp_host: resolved_smtp_host, smtp_port: resolved_smtp_port, - use_tls: use_tls != 0, imap_tls_mode, smtp_tls_mode, keyring_secret_key, @@ -606,7 +603,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-test-1".to_string(), @@ -668,7 +664,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-salvage-1".to_string(), @@ -806,7 +801,6 @@ mod tests { server_port: 143, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 587, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::StartTls, smtp_tls_mode: nuncio_core::TlsMode::Plain, keyring_secret_key: "nuncio/acct-starttls-1".to_string(), diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index cc948ef..7084a0c 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -41,13 +41,14 @@ use nuncio_proto::v1::{ ListFoldersResponse, ListMessagesRequest, ListMessagesResponse, ListRecordsRequest, ListRecordsResponse, ListRulesRequest, ListRulesResponse, MarkReadRequest, MarkReadResponse, Message as MessageProto, MessageFlagsChanged, MessageSearchHit, PreviewRuleRequest, - PreviewRuleResponse, RemoveAccountRequest, RemoveAccountResponse, SearchMessagesRequest, - SearchMessagesResponse, SendMessageRequest, SendMessageResponse, ShuttingDown, - SubscribeRequest, SyncCompleted, SyncRequest, SyncResponse, SyncStarted, - TestAccountConnectionRequest, TestAccountConnectionResponse, TlsMode as TlsModeProto, - TriageProgress, TriageRequest, UpdateAccountRequest, UpdateAccountResponse, UpdateAvailable, - UpdateRuleRequest, UpdateRuleResponse, ValidateRuleRequest, ValidateRuleResponse, - VerifyChainRequest, VerifyChainResponse, + PreviewRuleResponse, RemoveAccountRequest, RemoveAccountResponse, + RuleExportFormat as RuleExportFormatProto, SearchMessagesRequest, SearchMessagesResponse, + SendMessageRequest, SendMessageResponse, ShuttingDown, SubscribeRequest, SyncCompleted, + SyncRequest, SyncResponse, SyncStarted, TestAccountConnectionRequest, + TestAccountConnectionResponse, TlsMode as TlsModeProto, TriageProgress, TriageRequest, + UpdateAccountRequest, UpdateAccountResponse, UpdateAvailable, UpdateRuleRequest, + UpdateRuleResponse, ValidateRuleRequest, ValidateRuleResponse, VerifyChainRequest, + VerifyChainResponse, }; use nuncio_store::db::DatabaseEngine; use nuncio_store::search::SearchEngine; @@ -334,7 +335,6 @@ fn map_account_config_to_proto(config: nuncio_core::AccountConfig) -> AccountCon protocol: map_account_protocol_to_proto(config.protocol).into(), server_host: config.server_host, server_port: u32::from(config.server_port), - use_tls: config.use_tls, imap_tls_mode: map_tls_mode_to_proto(config.imap_tls_mode).into(), smtp_tls_mode: map_tls_mode_to_proto(config.smtp_tls_mode).into(), keyring_secret_key: config.keyring_secret_key, @@ -373,7 +373,6 @@ fn map_account_config_from_proto( server_port, smtp_host: config.smtp_host, smtp_port, - use_tls: config.use_tls, imap_tls_mode, smtp_tls_mode, keyring_secret_key: config.keyring_secret_key, @@ -2076,6 +2075,7 @@ impl Filters for FiltersGrpcService { request: Request, ) -> Result, Status> { let req = request.into_inner(); + let format = req.format(); let rules = self .db @@ -2083,15 +2083,17 @@ impl Filters for FiltersGrpcService { .await .map_err(|e| Status::internal(format!("failed to list filter rules: {e}")))?; - let content = if req.format == "json" { - serde_json::to_string_pretty(&rules) - .map_err(|e| Status::internal(format!("failed to render rules as JSON: {e}")))? - } else { - rules + let content = match format { + RuleExportFormatProto::Sql => rules .iter() .map(|r| r.to_nsql()) .collect::>() - .join("\n") + .join("\n"), + RuleExportFormatProto::Json => serde_json::to_string_pretty(&rules) + .map_err(|e| Status::internal(format!("failed to render rules as JSON: {e}")))?, + RuleExportFormatProto::Unspecified => { + return Err(Status::invalid_argument("rule export format is required")) + } }; Ok(Response::new(ExportRulesResponse { content })) @@ -3432,7 +3434,6 @@ mod tests { protocol: AccountProtocolProto::ImapSmtp.into(), server_host: "imap.nuncio.mx".to_string(), server_port: 993, - use_tls: true, imap_tls_mode: TlsModeProto::ImplicitTls.into(), smtp_tls_mode: TlsModeProto::ImplicitTls.into(), keyring_secret_key: keyring_secret_key.to_string(), @@ -3827,7 +3828,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: key.to_string(), @@ -3871,7 +3871,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: key.to_string(), @@ -4486,7 +4485,6 @@ mod tests { server_port: 993, smtp_host: "127.0.0.1".to_string(), smtp_port: 1, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-send-1".to_string(), @@ -4655,7 +4653,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-injected-send-1".to_string(), @@ -4772,7 +4769,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-injected-send-fail-1".to_string(), @@ -4926,7 +4922,7 @@ mod tests { let err = client .export_rules(ExportRulesRequest { - format: "sql".to_string(), + format: RuleExportFormatProto::Sql.into(), }) .await .expect_err("missing bearer token must be rejected"); @@ -5503,7 +5499,7 @@ mod tests { let sql_export = client .export_rules(authed_bearer_request(ExportRulesRequest { - format: "sql".to_string(), + format: RuleExportFormatProto::Sql.into(), })) .await .expect("export_rules succeeds") @@ -5513,7 +5509,7 @@ mod tests { let json_export = client .export_rules(authed_bearer_request(ExportRulesRequest { - format: "json".to_string(), + format: RuleExportFormatProto::Json.into(), })) .await .expect("export_rules succeeds") @@ -5522,6 +5518,25 @@ mod tests { assert!(json_export.content.contains("Export Me")); } + /// `ExportRules` rejects an unspecified format rather than silently + /// defaulting to one rendering. + #[tokio::test] + async fn export_rules_rejects_unspecified_format() { + let (addr, _handle, _db, _engine, _dir) = spawn_filters_test_server("correct-token").await; + + let mut client = FiltersClient::connect(format!("http://{addr}")) + .await + .expect("client connects"); + + let err = client + .export_rules(authed_bearer_request(ExportRulesRequest { + format: RuleExportFormatProto::Unspecified.into(), + })) + .await + .expect_err("unspecified format must be rejected"); + assert_eq!(err.code(), Code::InvalidArgument); + } + /// `ImportRules` persists every line that parses AND reports the parse /// error for every line that does not -- proving failures are surfaced /// rather than silently dropped. diff --git a/crates/nunciod/src/orchestrator.rs b/crates/nunciod/src/orchestrator.rs index 2fa48f3..5147754 100644 --- a/crates/nunciod/src/orchestrator.rs +++ b/crates/nunciod/src/orchestrator.rs @@ -157,7 +157,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-orch-1".to_string(), diff --git a/crates/nunciod/src/send.rs b/crates/nunciod/src/send.rs index 3593ab5..ee800db 100644 --- a/crates/nunciod/src/send.rs +++ b/crates/nunciod/src/send.rs @@ -238,7 +238,6 @@ mod tests { server_port: 993, smtp_host: "127.0.0.1".to_string(), smtp_port: 1, - use_tls: true, imap_tls_mode: TlsMode::ImplicitTls, smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{id}"), diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index 98dc78a..e2983ba 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -520,7 +520,6 @@ mod tests { server_port: 443, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: TlsMode::ImplicitTls, smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{id}"), @@ -612,7 +611,6 @@ mod tests { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: TlsMode::ImplicitTls, smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{id}"), diff --git a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs index 37a5b33..68c53d8 100644 --- a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs +++ b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs @@ -82,7 +82,6 @@ fn sample_account_config() -> AccountConfig { protocol: AccountProtocol::ImapSmtp.into(), server_host: "imap.nuncio.mx".to_string(), server_port: 143, - use_tls: false, imap_tls_mode: TlsMode::StartTls.into(), smtp_tls_mode: TlsMode::Plain.into(), keyring_secret_key: KEYRING_KEY.to_string(), diff --git a/crates/nunciod/tests/caldav_production_e2e_test.rs b/crates/nunciod/tests/caldav_production_e2e_test.rs index 6a210f3..566c911 100644 --- a/crates/nunciod/tests/caldav_production_e2e_test.rs +++ b/crates/nunciod/tests/caldav_production_e2e_test.rs @@ -161,7 +161,6 @@ async fn caldav_production_sync_builds_real_client_and_persists_events() { protocol: AccountProtocolProto::Caldav.into(), server_host: String::new(), server_port: 0, - use_tls: true, imap_tls_mode: TlsModeProto::ImplicitTls.into(), smtp_tls_mode: TlsModeProto::ImplicitTls.into(), keyring_secret_key: KEYRING_KEY.to_string(), diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index d6fd97f..d4a4e4d 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -104,7 +104,6 @@ fn sample_account() -> nuncio_core::AccountConfig { server_port: 993, smtp_host: "smtp.nuncio.mx".to_string(), smtp_port: 465, - use_tls: true, imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{ACCOUNT_ID}"), diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index e99d5b3..be1b8d2 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -61,7 +61,6 @@ fn sample_account_config() -> AccountConfig { protocol: AccountProtocol::ImapSmtp.into(), server_host: "imap.nuncio.mx".to_string(), server_port: 993, - use_tls: true, imap_tls_mode: TlsMode::ImplicitTls.into(), smtp_tls_mode: TlsMode::ImplicitTls.into(), keyring_secret_key: KEYRING_KEY.to_string(), -- 2.54.0 From 9b69b7f1b1e50c6b9b106aad83819ccb58f97042 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 17:58:29 -0500 Subject: [PATCH 35/95] feat(nunciod): bounded-concurrency sync dispatcher with per-account timeout (#366) Route every production account sync through one SyncDispatcher that caps simultaneous syncs (NUNCIO_SYNC_CONCURRENCY, default 4), coalesces a duplicate in-flight sync for the same account, applies a per-account timeout (NUNCIO_SYNC_TIMEOUT, default 300s), and races in-flight syncs against the shared shutdown signal. The client Mail.Sync RPC and the daemon's CoreCommand sync consumer now share a single dispatcher instance so one slow account can no longer head-of-line-block the others. Refs #292 Co-authored-by: James Maes --- crates/nunciod/src/grpc.rs | 118 +++-- crates/nunciod/src/lib.rs | 1 + crates/nunciod/src/lifecycle.rs | 25 +- crates/nunciod/src/main.rs | 40 +- crates/nunciod/src/sync.rs | 178 +------ crates/nunciod/src/sync_dispatcher.rs | 688 ++++++++++++++++++++++++++ 6 files changed, 835 insertions(+), 215 deletions(-) create mode 100644 crates/nunciod/src/sync_dispatcher.rs diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 7084a0c..913e3bc 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -5,7 +5,9 @@ //! //! This is the daemon's sole client-facing transport. +use crate::lifecycle::ShutdownSignal; use crate::pagination::{self, CursorField}; +use crate::sync_dispatcher::{sync_all_configured, ProductionAccountSyncer, SyncDispatcher}; use nuncio_cal::CalendarBackend; use nuncio_contacts::ContactsBackend; use nuncio_core::{CoreCommand, CoreEvent, EventBus}; @@ -1414,11 +1416,11 @@ impl Contacts for ContactsGrpcService { /// /// Production (`nunciod::main`, and every existing caller of [`serve`] / /// [`serve_on_listener`]) never constructs a non-default instance: -/// [`Default`] yields every field `None`, which routes `Mail/Sync` and -/// `Mail/SendMessage` through the exact same real production entry points -/// (`nunciod::sync::run_all_accounts_sync` / `run_account_sync`, -/// `nunciod::send::send_message_for_account`) as before this override -/// existed. Production code depends only on the `nuncio_mail::{MailBackend, +/// [`Default`] yields every field `None`, which routes `Mail/Sync` through +/// the shared `nunciod::sync_dispatcher::SyncDispatcher` (single account or +/// `sync_all_configured` fan-out) and `Mail/SendMessage` through +/// `nunciod::send::send_message_for_account`, exactly as production does. +/// Production code depends only on the `nuncio_mail::{MailBackend, /// MessageSender}` trait objects here -- never on `nuncio_mail`'s mock /// types -- so this seam never makes production depend on a test double. /// @@ -1464,6 +1466,11 @@ struct MailGrpcService { secrets: Arc, filter_engine: Arc, overrides: MailEngineOverrides, + /// Single bounded-concurrency dispatcher every production `Sync` funnels + /// through, so a client `Sync` and the daemon's scheduled syncs share one + /// concurrency cap, one in-flight coalescing map, and one per-account + /// timeout. The injected test backend path deliberately bypasses it. + sync_dispatcher: Arc, } #[tonic::async_trait] @@ -1683,12 +1690,20 @@ impl Mail for MailGrpcService { /// scopes the sync to one account, `None` syncs every configured /// account. /// + /// Production syncs funnel through the shared [`SyncDispatcher`]: a + /// single-account request goes through its bounded/coalesced/timed + /// per-account path, and an all-accounts request fans every configured + /// mail account out through that same path via + /// [`sync_all_configured`], so one slow account can no longer + /// head-of-line-block the others. + /// /// When [`MailEngineOverrides::mail_backend`] is injected, fetches from - /// it via [`crate::sync::sync_with_backend`] instead of resolving a - /// real per-account engine from keyring credentials -- this is what - /// lets a full-daemon E2E test drive a real inbound sync entirely over - /// this authenticated gRPC API with no live network, exactly as a real - /// client would trigger it (no un-intercepted back door). + /// it via [`crate::sync::sync_with_backend`] instead -- this is what lets + /// a full-daemon E2E test drive a real inbound sync entirely over this + /// authenticated gRPC API with no live network, exactly as a real client + /// would trigger it (no un-intercepted back door). The injected backend + /// is a single in-memory double, so it deliberately bypasses the + /// dispatcher's per-account bounding. async fn sync(&self, request: Request) -> Result, Status> { let account_id = request.into_inner().account_id; @@ -1703,23 +1718,12 @@ impl Mail for MailGrpcService { .await .map_err(|e| Status::internal(format!("sync failed: {e}")))? } else if let Some(account_id) = account_id { - crate::sync::run_account_sync( - &self.db, - &self.secrets, - &self.event_bus, - &self.filter_engine, - &account_id, - ) - .await - .map_err(|e| Status::internal(format!("sync failed: {e}")))? + self.sync_dispatcher + .sync_account(&account_id) + .await + .map_err(|e| Status::internal(format!("sync failed: {e}")))? } else { - crate::sync::run_all_accounts_sync( - &self.db, - &self.secrets, - &self.event_bus, - &self.filter_engine, - ) - .await + sync_all_configured(&self.sync_dispatcher, &self.db, &self.event_bus).await }; Ok(Response::new(SyncResponse { @@ -2460,6 +2464,29 @@ impl tonic::service::Interceptor for BearerAuthInterceptor { } } +/// Builds the production [`SyncDispatcher`] for serve entry points that are +/// not handed one by their caller (every path except production's +/// [`serve_with_shutdown`], which threads through the daemon-wide dispatcher +/// so the RPC and the scheduled syncs share one instance). Wraps a +/// [`ProductionAccountSyncer`] with concurrency/timeout bounds from the +/// environment; `shutdown` is the signal the dispatcher races in-flight syncs +/// against -- non-graceful entry points pass [`ShutdownSignal::never`]. +fn default_sync_dispatcher( + db: Arc, + secrets: Arc, + event_bus: Arc, + filter_engine: Arc, + shutdown: ShutdownSignal, +) -> Arc { + let syncer = Arc::new(ProductionAccountSyncer::new( + db, + secrets, + event_bus, + filter_engine, + )); + Arc::new(SyncDispatcher::from_env(syncer, shutdown)) +} + /// Binds a loopback TCP listener at `addr` and serves the `nuncio.v1.System`, /// `nuncio.v1.Accounts`, `nuncio.v1.Mail`, `nuncio.v1.Filters`, /// `nuncio.v1.Export`, `nuncio.v1.Audit`, and `nuncio.v1.Calendar` gRPC @@ -2548,6 +2575,7 @@ pub async fn serve_on_listener( /// as `shutdown` resolves, instead of running until the transport errors. /// /// `addr` MUST be a loopback address; see [`serve`]. +#[allow(clippy::too_many_arguments)] pub async fn serve_with_shutdown( addr: &str, event_bus: Arc, @@ -2555,6 +2583,7 @@ pub async fn serve_with_shutdown( filter_engine: Arc, secrets: Arc, token: impl Into>, + sync_dispatcher: Arc, shutdown: impl std::future::Future + Send + 'static, ) -> Result<(), GrpcServeError> { let socket_addr = addr @@ -2570,13 +2599,22 @@ pub async fn serve_with_shutdown( addr: addr.to_string(), source, })?; - serve_on_listener_with_shutdown( + // Production threads its own daemon-wide dispatcher (built over the shared + // shutdown signal) all the way to the Mail service, rather than letting a + // serve wrapper mint a fresh one, so the client `Sync` RPC and the + // scheduled syncs share a single concurrency cap and in-flight map. + serve_on_listener_with_overrides_and_shutdown( listener, event_bus, db, filter_engine, secrets, token, + MailEngineOverrides::default(), + CalendarEngineOverrides::default(), + ContactsEngineOverrides::default(), + AccountsEngineOverrides::default(), + sync_dispatcher, shutdown, ) .await @@ -2594,6 +2632,17 @@ pub async fn serve_on_listener_with_shutdown( token: impl Into>, shutdown: impl std::future::Future + Send + 'static, ) -> Result<(), GrpcServeError> { + // This entry point receives an opaque shutdown future rather than a + // `ShutdownSignal`, so it mints a fresh production dispatcher over a + // never-firing signal; production instead uses `serve_with_shutdown`, + // which threads the real shared dispatcher through. + let sync_dispatcher = default_sync_dispatcher( + db.clone(), + secrets.clone(), + event_bus.clone(), + filter_engine.clone(), + ShutdownSignal::never(), + ); serve_on_listener_with_overrides_and_shutdown( listener, event_bus, @@ -2605,6 +2654,7 @@ pub async fn serve_on_listener_with_shutdown( CalendarEngineOverrides::default(), ContactsEngineOverrides::default(), AccountsEngineOverrides::default(), + sync_dispatcher, shutdown, ) .await @@ -2636,7 +2686,16 @@ pub async fn serve_on_listener_with_overrides( ) -> Result<(), GrpcServeError> { // Delegates to the shutdown-aware variant with a shutdown future that // never resolves, preserving this function's exact prior behavior (run - // until the transport itself errors) for every existing caller. + // until the transport itself errors) for every existing caller. The + // dispatcher is minted here over a never-firing signal; only production's + // `serve_with_shutdown` threads a real shared dispatcher through. + let sync_dispatcher = default_sync_dispatcher( + db.clone(), + secrets.clone(), + event_bus.clone(), + filter_engine.clone(), + ShutdownSignal::never(), + ); serve_on_listener_with_overrides_and_shutdown( listener, event_bus, @@ -2648,6 +2707,7 @@ pub async fn serve_on_listener_with_overrides( calendar_overrides, contacts_overrides, accounts_overrides, + sync_dispatcher, std::future::pending(), ) .await @@ -2671,6 +2731,7 @@ pub async fn serve_on_listener_with_overrides_and_shutdown( calendar_overrides: CalendarEngineOverrides, contacts_overrides: ContactsEngineOverrides, accounts_overrides: AccountsEngineOverrides, + sync_dispatcher: Arc, shutdown: impl std::future::Future + Send + 'static, ) -> Result<(), GrpcServeError> { let token: Arc = token.into(); @@ -2708,6 +2769,7 @@ pub async fn serve_on_listener_with_overrides_and_shutdown( secrets: secrets.clone(), filter_engine: filter_engine.clone(), overrides, + sync_dispatcher, }; let mail_interceptor = BearerAuthInterceptor::new(token.clone()); let mail_svc = InterceptedService::new( diff --git a/crates/nunciod/src/lib.rs b/crates/nunciod/src/lib.rs index 93fcc13..8c24ff7 100644 --- a/crates/nunciod/src/lib.rs +++ b/crates/nunciod/src/lib.rs @@ -12,6 +12,7 @@ pub mod outbox; pub mod pagination; pub mod send; pub mod sync; +pub mod sync_dispatcher; pub use orchestrator::SelfHealingSyncOrchestrator; /// Default PERSISTENT database path for `nunciod`: `~/.nuncio/nuncio.db` diff --git a/crates/nunciod/src/lifecycle.rs b/crates/nunciod/src/lifecycle.rs index dcef665..107fd09 100644 --- a/crates/nunciod/src/lifecycle.rs +++ b/crates/nunciod/src/lifecycle.rs @@ -20,9 +20,26 @@ use tokio::sync::watch; #[derive(Clone)] pub struct ShutdownSignal { rx: watch::Receiver, + // Keeps the watch channel open for signals minted by [`ShutdownSignal::never`], + // which have no live [`ShutdownController`] owning the sender. A closed + // channel makes `wait` resolve (readers interpret that as "shutdown + // requested"), so a `never` signal must hold its sender alive; a real + // signal leaves this `None` because its controller owns the sender. + _keepalive: Option>>, } impl ShutdownSignal { + /// A signal that never fires, for call paths with no shutdown wiring of + /// their own (e.g. the non-graceful `serve_on_listener` entry point used + /// by tests). Holds its own sender alive so the channel never closes. + pub fn never() -> Self { + let (tx, rx) = watch::channel(false); + ShutdownSignal { + rx, + _keepalive: Some(Arc::new(tx)), + } + } + /// Resolves once shutdown has been requested. Safe to call from /// `tokio::select!` on every loop iteration: a task that observes /// shutdown mid-iteration finishes its current unit of work (the other @@ -59,7 +76,13 @@ impl ShutdownController { /// instead of that path staying reachable only from tests. pub fn new(event_bus: Arc) -> (Self, ShutdownSignal) { let (tx, rx) = watch::channel(false); - (Self { tx, event_bus }, ShutdownSignal { rx }) + ( + Self { tx, event_bus }, + ShutdownSignal { + rx, + _keepalive: None, + }, + ) } /// Initiates shutdown. Idempotent: calling it more than once (e.g. a diff --git a/crates/nunciod/src/main.rs b/crates/nunciod/src/main.rs index fcdc090..f483522 100644 --- a/crates/nunciod/src/main.rs +++ b/crates/nunciod/src/main.rs @@ -70,6 +70,27 @@ async fn main() -> Result<(), Box> { let initial_rules = nunciod::load_initial_filter_rules(&db).await?; let filter_engine = Arc::new(FilterEngine::new(initial_rules)?); + // Single bounded-concurrency sync dispatcher shared by BOTH the + // client-facing `Mail/Sync` RPC (via the gRPC server below) and this + // process's `CoreCommand` sync consumer, so account syncs from either + // origin queue against one concurrency cap, coalesce per account, and + // observe one per-account timeout -- instead of each entry point launching + // syncs independently and head-of-line-blocking one another. It races + // in-flight syncs against the SAME shutdown signal every other subsystem + // shares. + let sync_dispatcher = { + let syncer = Arc::new(nunciod::sync_dispatcher::ProductionAccountSyncer::new( + db.clone(), + account_secrets.clone(), + event_bus.clone(), + filter_engine.clone(), + )); + Arc::new(nunciod::sync_dispatcher::SyncDispatcher::from_env( + syncer, + shutdown_signal.clone(), + )) + }; + // Real inbound-sync `CoreCommand` consumer. This is the ONE place that // claims `EventBus`'s command receiver, so it is what finally makes // `CoreCommand::SyncAll` / @@ -82,9 +103,8 @@ async fn main() -> Result<(), Box> { // trigger was inert. let sync_command_task = if let Some(mut command_rx) = command_rx { let db_sync = db.clone(); - let secrets_sync = account_secrets.clone(); let event_bus_sync = event_bus.clone(); - let filter_engine_sync = filter_engine.clone(); + let dispatcher_sync = sync_dispatcher.clone(); let mut shutdown = shutdown_signal.clone(); Some(tokio::spawn(async move { loop { @@ -100,25 +120,16 @@ async fn main() -> Result<(), Box> { }; match cmd { CoreCommand::SyncAll => { - let synced = nunciod::sync::run_all_accounts_sync( + let synced = nunciod::sync_dispatcher::sync_all_configured( + &dispatcher_sync, &db_sync, - &secrets_sync, &event_bus_sync, - &filter_engine_sync, ) .await; tracing::info!("SyncAll completed: {} message(s) synced", synced); } CoreCommand::SyncAccount { account_id } => { - match nunciod::sync::run_account_sync( - &db_sync, - &secrets_sync, - &event_bus_sync, - &filter_engine_sync, - &account_id, - ) - .await - { + match dispatcher_sync.sync_account(&account_id).await { Ok(count) => tracing::info!( "SyncAccount({}) completed: {} message(s) synced", account_id, @@ -274,6 +285,7 @@ async fn main() -> Result<(), Box> { filter_engine, grpc_secrets, grpc_token, + sync_dispatcher, async move { grpc_shutdown_future.wait().await }, ) => result, _ = async move { diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index e2983ba..df2cf81 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -14,10 +14,12 @@ //! [`sync_with_backend`] takes `&dyn MailBackend` rather than a concrete //! engine, so it is exercised directly in tests with [`MockMailBackend`] -- //! no real network or OS keyring access ever happens inside it. Production -//! entry points ([`run_account_sync`], [`run_all_accounts_sync`]) are the -//! only places that resolve a real account's credentials and construct a -//! real backend; they then delegate the actual fetch/persist/event-emission -//! work to [`sync_with_backend`]. +//! entry point [`run_account_sync`] is the only place that resolves a real +//! account's credentials and constructs a real backend; it then delegates the +//! actual fetch/persist/event-emission work to [`sync_with_backend`]. An +//! all-accounts sync fans these per-account runs out through +//! [`crate::sync_dispatcher::sync_all_configured`], so concurrency and the +//! per-account timeout are bounded in one place rather than serialized here. use nuncio_core::{AccountConfig, AccountProtocol, CoreCommand, CoreEvent, EventBus}; use nuncio_filter::{FilterEngine, OutboxManager, RuleAction}; use nuncio_mail::{ImapEngine, JmapEngine, MailBackend, MailError}; @@ -408,87 +410,6 @@ pub async fn run_account_sync( } } -/// Production entry point for `CoreCommand::SyncAll`: syncs every persisted -/// account in turn, wrapping the whole batch in a single -/// `SyncStarted{None}`/`SyncCompleted{None}` pair (rather than one pair per -/// account). A single account failing to sync (missing credential, backend -/// error, etc.) publishes a `CoreEvent::Error` and does not stop the -/// remaining accounts from being attempted. Returns the total number of -/// messages synced across all accounts. -pub async fn run_all_accounts_sync( - db: &nuncio_store::db::DatabaseEngine, - secrets: &SecretManager, - event_bus: &EventBus, - filter_engine: &FilterEngine, -) -> usize { - event_bus.process_command(CoreCommand::SyncAll); - - let mut total_synced = 0usize; - match db.list_accounts().await { - Ok(accounts) => { - for config in accounts { - // DAV-protocol accounts (e.g. CalDAV) are not mail accounts; - // they sync through their own domain service, so skip them - // here rather than attempting to build a mail backend. - if config.protocol.is_dav() { - continue; - } - match secrets.get_secret(&config.keyring_secret_key) { - Ok(password) => { - let backend = match build_mail_backend(&config, &password) { - Ok(backend) => backend, - Err(e) => { - event_bus.process_command(CoreCommand::ReportError { - message: format!( - "sync skipped for account '{}': {e}", - config.id - ), - }); - continue; - } - }; - match fetch_and_persist( - db, - event_bus, - backend.as_ref(), - filter_engine, - Some(&config.id), - ) - .await - { - Ok(count) => total_synced += count, - Err(e) => { - event_bus.process_command(CoreCommand::ReportError { - message: format!( - "sync failed for account '{}': {e}", - config.id - ), - }); - } - } - } - Err(e) => { - event_bus.process_command(CoreCommand::ReportError { - message: format!( - "cannot read credential for account '{}': {e}", - config.id - ), - }); - } - } - } - } - Err(e) => { - event_bus.process_command(CoreCommand::ReportError { - message: format!("failed to list accounts for sync: {e}"), - }); - } - } - - event_bus.complete_sync(None); - total_synced -} - #[cfg(test)] mod tests { use super::*; @@ -956,93 +877,6 @@ mod tests { assert_eq!(event_bus.current_state().status, EngineStatus::Idle); } - #[tokio::test] - async fn run_all_accounts_sync_with_no_accounts_returns_zero_and_emits_events() { - let (db, _dir) = DatabaseEngine::connect_ephemeral() - .await - .expect("ephemeral db"); - let event_bus = EventBus::new(); - let mut events = event_bus.subscribe_events(); - let secrets = SecretManager::mock(); - - let filter_engine = empty_filter_engine(); - let total = run_all_accounts_sync(&db, &secrets, &event_bus, &filter_engine).await; - assert_eq!(total, 0); - - assert_eq!( - events.recv().await.expect("start event"), - CoreEvent::SyncStarted { account_id: None } - ); - assert_eq!( - events.recv().await.expect("complete event"), - CoreEvent::SyncCompleted { account_id: None } - ); - } - - #[tokio::test] - async fn run_all_accounts_sync_persists_across_multiple_accounts() { - let mock_server_a = MockServer::start().await; - let mock_server_b = MockServer::start().await; - mount_single_message_jmap_stubs(&mock_server_a, "acct-all-a").await; - mount_single_message_jmap_stubs(&mock_server_b, "acct-all-b").await; - - let (db, _dir) = DatabaseEngine::connect_ephemeral() - .await - .expect("ephemeral db"); - let event_bus = EventBus::new(); - let secrets = SecretManager::mock(); - - let config_a = sample_jmap_account_at("acct-all-a", &mock_server_a.uri()); - let config_b = sample_jmap_account_at("acct-all-b", &mock_server_b.uri()); - db.save_account(&config_a).await.expect("save account a"); - db.save_account(&config_b).await.expect("save account b"); - secrets - .set_secret(&config_a.keyring_secret_key, "token-a") - .expect("store credential a"); - secrets - .set_secret(&config_b.keyring_secret_key, "token-b") - .expect("store credential b"); - - let filter_engine = empty_filter_engine(); - let total = run_all_accounts_sync(&db, &secrets, &event_bus, &filter_engine).await; - // 2 accounts * 1 folder * 1 message each = 2 processed messages. - assert_eq!(total, 2); - assert_eq!(event_bus.current_state().status, EngineStatus::Idle); - } - - #[tokio::test] - async fn run_all_accounts_sync_reports_error_for_account_missing_credential() { - let (db, _dir) = DatabaseEngine::connect_ephemeral() - .await - .expect("ephemeral db"); - let event_bus = EventBus::new(); - let mut events = event_bus.subscribe_events(); - let secrets = SecretManager::mock(); - - let config = sample_jmap_account("acct-all-nocred"); - db.save_account(&config).await.expect("save account"); - // No credential stored in the vault for this account. - - let filter_engine = empty_filter_engine(); - let total = run_all_accounts_sync(&db, &secrets, &event_bus, &filter_engine).await; - assert_eq!(total, 0); - - assert_eq!( - events.recv().await.expect("start event"), - CoreEvent::SyncStarted { account_id: None } - ); - match events.recv().await.expect("error event") { - CoreEvent::Error { message } => { - assert!(message.contains("acct-all-nocred")); - } - other => panic!("expected CoreEvent::Error, got {other:?}"), - } - assert_eq!( - events.recv().await.expect("complete event"), - CoreEvent::SyncCompleted { account_id: None } - ); - } - #[tokio::test] async fn sync_with_backend_fires_immediate_mark_read_action_on_match() { let (db, _dir) = DatabaseEngine::connect_ephemeral() diff --git a/crates/nunciod/src/sync_dispatcher.rs b/crates/nunciod/src/sync_dispatcher.rs new file mode 100644 index 0000000..29718f5 --- /dev/null +++ b/crates/nunciod/src/sync_dispatcher.rs @@ -0,0 +1,688 @@ +//! Bounded-concurrency sync dispatcher. +//! +//! Account syncs used to be launched directly and independently from two +//! places (the client-facing `Mail/Sync` RPC handler and the daemon's +//! `CoreCommand` consumer), each of which ran an account's sync to completion +//! before the next one started. One slow or hung account could therefore +//! head-of-line-block every account behind it, and nothing bounded how many +//! syncs ran at once or how long a single account was allowed to take. +//! +//! [`SyncDispatcher`] is the single place all account syncs now funnel +//! through. It: +//! +//! * caps how many account syncs run at once with a [`Semaphore`] +//! (env-tunable via [`CONCURRENCY_ENV_VAR`], default +//! [`DEFAULT_CONCURRENCY`]); excess requests queue on the permit rather +//! than run unbounded; +//! * coalesces a second request for an account that is already syncing onto +//! the in-flight one instead of double-fetching -- a client `Sync` issued +//! while a scheduled sync of the same account is running joins it and +//! observes its result; +//! * bounds each account's sync with a per-account timeout (env-tunable via +//! [`TIMEOUT_ENV_VAR`], default [`DEFAULT_TIMEOUT`]); a timed-out sync is +//! cancelled cleanly and surfaces [`SyncDispatchError::Timeout`] without +//! poisoning the dispatcher for other accounts; and +//! * respects the shared shutdown signal: once shutdown is requested it stops +//! admitting new work and races each in-flight sync against the signal +//! (mirroring the outbox executor) so a hung account cannot delay graceful +//! shutdown beyond the daemon's grace period. +//! +//! # Testability +//! +//! The actual per-account sync is injected behind the [`AccountSyncer`] +//! trait, so the dispatcher's concurrency, coalescing, timeout, and shutdown +//! behavior is exercised deterministically with an in-memory mock syncer -- +//! no real network, keyring, or database. Production wires +//! [`ProductionAccountSyncer`], which delegates to +//! [`crate::sync::run_account_sync`]. + +use crate::lifecycle::ShutdownSignal; +use crate::sync::SyncError; +use async_trait::async_trait; +use nuncio_core::{CoreCommand, EventBus}; +use nuncio_filter::FilterEngine; +use nuncio_store::db::DatabaseEngine; +use nuncio_store::vault::SecretManager; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; +use thiserror::Error; +use tokio::sync::{watch, Semaphore}; + +/// Environment variable bounding how many account syncs may run at once. +pub const CONCURRENCY_ENV_VAR: &str = "NUNCIO_SYNC_CONCURRENCY"; + +/// Environment variable bounding a single account's sync, in whole seconds. +pub const TIMEOUT_ENV_VAR: &str = "NUNCIO_SYNC_TIMEOUT"; + +/// Default simultaneous-sync cap when [`CONCURRENCY_ENV_VAR`] is unset or +/// unparseable. Chosen to overlap a handful of accounts without letting an +/// unbounded fan-out open a socket per account at once. +pub const DEFAULT_CONCURRENCY: usize = 4; + +/// Default per-account sync timeout when [`TIMEOUT_ENV_VAR`] is unset or +/// unparseable. Generous enough for a healthy full mailbox fetch, tight +/// enough that a hung account cannot stall others (or shutdown) indefinitely. +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(300); + +/// The outcome of a single dispatched account sync. Cloneable so a coalesced +/// result can fan out to every joined waiter. +type SyncOutcome = Result; + +/// Errors surfaced by [`SyncDispatcher::sync_account`]. +#[derive(Debug, Clone, Error)] +pub enum SyncDispatchError { + /// The account's sync exceeded the configured per-account timeout and was + /// cancelled. Retryable: nothing about the dispatcher is left broken. + #[error("account '{account_id}' sync exceeded the {timeout:?} per-account timeout")] + Timeout { + /// The account whose sync timed out. + account_id: String, + /// The timeout that was exceeded. + timeout: Duration, + }, + /// Shutdown was requested before or during the sync; no new work is + /// admitted and in-flight work is abandoned for the next start. + #[error("sync dispatcher is shutting down")] + ShuttingDown, + /// The in-flight sync ended without producing a result (e.g. its task was + /// dropped). Retryable. + #[error("account sync was cancelled before it produced a result")] + Cancelled, + /// The underlying account sync itself failed. Shared behind an [`Arc`] so + /// a coalesced failure can be reported to every joined waiter. + #[error(transparent)] + Sync(Arc), +} + +/// Injectable seam for the actual work of syncing one account. Production +/// uses [`ProductionAccountSyncer`]; tests inject an in-memory mock. +#[async_trait] +pub trait AccountSyncer: Send + Sync { + /// Sync exactly one account to completion, returning the number of + /// messages processed. + async fn sync_account(&self, account_id: &str) -> Result; +} + +/// Production [`AccountSyncer`]: delegates to [`crate::sync::run_account_sync`], +/// which resolves the account's credentials, builds a real mail backend, and +/// runs a full inbound sync with the standard engine-status lifecycle. +pub struct ProductionAccountSyncer { + db: Arc, + secrets: Arc, + event_bus: Arc, + filter_engine: Arc, +} + +impl ProductionAccountSyncer { + /// Build the production syncer from the daemon's shared subsystems. + pub fn new( + db: Arc, + secrets: Arc, + event_bus: Arc, + filter_engine: Arc, + ) -> Self { + Self { + db, + secrets, + event_bus, + filter_engine, + } + } +} + +#[async_trait] +impl AccountSyncer for ProductionAccountSyncer { + async fn sync_account(&self, account_id: &str) -> Result { + crate::sync::run_account_sync( + &self.db, + &self.secrets, + &self.event_bus, + &self.filter_engine, + account_id, + ) + .await + } +} + +/// Reads [`CONCURRENCY_ENV_VAR`], falling back to [`DEFAULT_CONCURRENCY`]. +/// A value of zero (or unparseable) is treated as the default; the effective +/// cap is always at least one. +fn concurrency_from_env() -> usize { + std::env::var(CONCURRENCY_ENV_VAR) + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(DEFAULT_CONCURRENCY) +} + +/// Reads [`TIMEOUT_ENV_VAR`] as whole seconds, falling back to +/// [`DEFAULT_TIMEOUT`]. A value of zero (or unparseable) is treated as the +/// default rather than "no timeout", so a misconfiguration can never disable +/// the safety bound. +fn timeout_from_env() -> Duration { + std::env::var(TIMEOUT_ENV_VAR) + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|secs| *secs > 0) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_TIMEOUT) +} + +/// Shared inner state, held behind an [`Arc`] so the dispatcher is cheap to +/// clone and can be moved into the per-account tasks it spawns. +struct Inner { + syncer: Arc, + semaphore: Arc, + /// Accounts with a sync currently in flight, mapped to a watch receiver + /// that publishes the sync's outcome once. A second request for the same + /// account clones the receiver and joins the in-flight sync. + inflight: Mutex>>>, + per_account_timeout: Duration, + shutdown: ShutdownSignal, +} + +impl Inner { + /// Run one account sync under the semaphore permit and per-account + /// timeout, racing both the permit wait and the sync itself against the + /// shutdown signal (mirroring the outbox executor). The permit is + /// released as soon as the sync resolves. + async fn run_guarded(&self, account_id: &str) -> SyncOutcome { + let mut shutdown = self.shutdown.clone(); + + let permit = tokio::select! { + biased; + _ = shutdown.wait() => return Err(SyncDispatchError::ShuttingDown), + acquired = self.semaphore.clone().acquire_owned() => match acquired { + Ok(permit) => permit, + // The semaphore is never closed in practice; treat a closed + // semaphore as a cancellation rather than fabricating success. + Err(_) => return Err(SyncDispatchError::Cancelled), + }, + }; + + let outcome = tokio::select! { + biased; + _ = shutdown.wait() => Err(SyncDispatchError::ShuttingDown), + result = tokio::time::timeout( + self.per_account_timeout, + self.syncer.sync_account(account_id), + ) => match result { + Ok(Ok(count)) => Ok(count), + Ok(Err(e)) => Err(SyncDispatchError::Sync(Arc::new(e))), + Err(_elapsed) => Err(SyncDispatchError::Timeout { + account_id: account_id.to_string(), + timeout: self.per_account_timeout, + }), + }, + }; + + drop(permit); + outcome + } +} + +/// The single entry point every account sync funnels through. Cheap to clone +/// (an [`Arc`] handle); every clone shares the same concurrency cap, +/// in-flight map, timeout, and shutdown signal. +#[derive(Clone)] +pub struct SyncDispatcher { + inner: Arc, +} + +impl SyncDispatcher { + /// Build a dispatcher with explicit bounds. `concurrency` is clamped up to + /// at least one so the dispatcher can always make progress. + pub fn new( + syncer: Arc, + shutdown: ShutdownSignal, + concurrency: usize, + per_account_timeout: Duration, + ) -> Self { + let concurrency = concurrency.max(1); + Self { + inner: Arc::new(Inner { + syncer, + semaphore: Arc::new(Semaphore::new(concurrency)), + inflight: Mutex::new(HashMap::new()), + per_account_timeout, + shutdown, + }), + } + } + + /// Build a dispatcher whose bounds come from the environment + /// ([`CONCURRENCY_ENV_VAR`] / [`TIMEOUT_ENV_VAR`]), falling back to + /// [`DEFAULT_CONCURRENCY`] / [`DEFAULT_TIMEOUT`]. + pub fn from_env(syncer: Arc, shutdown: ShutdownSignal) -> Self { + Self::new(syncer, shutdown, concurrency_from_env(), timeout_from_env()) + } + + /// Dispatch a sync for a single account, applying bounded concurrency, + /// in-flight coalescing, the per-account timeout, and shutdown handling. + /// + /// If a sync for `account_id` is already in flight, this joins it and + /// returns the SAME outcome rather than starting a duplicate fetch. + pub async fn sync_account(&self, account_id: &str) -> SyncOutcome { + if self.inner.shutdown.is_requested() { + return Err(SyncDispatchError::ShuttingDown); + } + + // Either join the in-flight sync for this account or become its owner + // by inserting a fresh result channel and spawning the guarded run. + // The lock is released before awaiting, so it never spans `.await`. + let mut rx = { + let mut inflight = self + .inner + .inflight + .lock() + .unwrap_or_else(PoisonError::into_inner); + if let Some(existing) = inflight.get(account_id) { + existing.clone() + } else { + let (tx, rx) = watch::channel::>(None); + inflight.insert(account_id.to_string(), rx.clone()); + + let inner = self.inner.clone(); + let id = account_id.to_string(); + tokio::spawn(async move { + let outcome = inner.run_guarded(&id).await; + // Remove the in-flight entry before publishing so a request + // arriving after completion starts a fresh sync rather than + // joining an already-finished one. + inner + .inflight + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&id); + let _ = tx.send(Some(outcome)); + }); + rx + } + }; + + // Clone the published outcome out of the borrowed watch guard before + // the guard (and `rx`) drop at the end of this scope. + let published = match rx.wait_for(Option::is_some).await { + Ok(guard) => guard.clone(), + // The sender was dropped without publishing (the owning task was + // itself dropped). Honestly report a cancellation, never success. + Err(_) => None, + }; + published.unwrap_or(Err(SyncDispatchError::Cancelled)) + } + + /// Dispatch syncs for several accounts concurrently, each through the same + /// bounded/coalesced/timed per-account path, and return the total number + /// of messages synced across the accounts that succeeded. Per-account + /// failures are logged and do not abort the batch (the underlying + /// production syncer additionally reports them on the event bus). + pub async fn sync_all(&self, account_ids: Vec) -> usize { + let mut set = tokio::task::JoinSet::new(); + for account_id in account_ids { + let dispatcher = self.clone(); + set.spawn(async move { + let result = dispatcher.sync_account(&account_id).await; + (account_id, result) + }); + } + + let mut total = 0usize; + while let Some(joined) = set.join_next().await { + match joined { + Ok((_, Ok(count))) => total += count, + Ok((account_id, Err(e))) => { + tracing::warn!("dispatched sync for account '{account_id}' failed: {e}"); + } + Err(join_err) => { + tracing::warn!("dispatched sync task failed to join: {join_err}"); + } + } + } + total + } +} + +/// Sync every configured mail account through `dispatcher`, wrapped in the +/// standard batch `SyncStarted{None}` / `SyncCompleted{None}` envelope. +/// +/// DAV-protocol accounts have no inbound mail backend and are skipped (they +/// sync through their own domain service). This is the single production +/// entry point for an all-accounts sync, shared by the `Mail/Sync` RPC +/// (`account_id == None`) and the daemon's `CoreCommand::SyncAll` consumer, so +/// the two never launch account syncs through separate, differently-bounded +/// paths. Returns the total number of messages synced across all accounts. +pub async fn sync_all_configured( + dispatcher: &SyncDispatcher, + db: &DatabaseEngine, + event_bus: &EventBus, +) -> usize { + event_bus.process_command(CoreCommand::SyncAll); + + let account_ids = match db.list_accounts().await { + Ok(accounts) => accounts + .into_iter() + .filter(|a| !a.protocol.is_dav()) + .map(|a| a.id) + .collect::>(), + Err(e) => { + event_bus.process_command(CoreCommand::ReportError { + message: format!("failed to list accounts for sync: {e}"), + }); + Vec::new() + } + }; + + let total = dispatcher.sync_all(account_ids).await; + event_bus.complete_sync(None); + total +} + +#[cfg(test)] +mod tests { + use super::*; + use nuncio_core::{AccountConfig, AccountProtocol, CoreEvent, TlsMode}; + use std::collections::HashSet; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::time::{sleep, Duration}; + + /// In-memory [`AccountSyncer`] that records invocations, tracks peak + /// concurrency, and can be told to block indefinitely for named accounts + /// (to model a hung account) or to add a fixed per-call delay (to make + /// concurrent calls genuinely overlap). + struct MockSyncer { + calls: Mutex>, + concurrent: AtomicUsize, + peak: AtomicUsize, + blocked: HashSet, + gate: Arc, + per_call_delay: Option, + fail: HashSet, + } + + impl MockSyncer { + fn new() -> Self { + Self { + calls: Mutex::new(Vec::new()), + concurrent: AtomicUsize::new(0), + peak: AtomicUsize::new(0), + blocked: HashSet::new(), + gate: Arc::new(tokio::sync::Notify::new()), + per_call_delay: None, + fail: HashSet::new(), + } + } + + fn with_delay(mut self, delay: Duration) -> Self { + self.per_call_delay = Some(delay); + self + } + + fn blocking(mut self, account_id: &str) -> Self { + self.blocked.insert(account_id.to_string()); + self + } + + fn failing(mut self, account_id: &str) -> Self { + self.fail.insert(account_id.to_string()); + self + } + + fn call_ids(&self) -> Vec { + self.calls + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + fn call_count(&self) -> usize { + self.calls + .lock() + .unwrap_or_else(PoisonError::into_inner) + .len() + } + } + + #[async_trait] + impl AccountSyncer for MockSyncer { + async fn sync_account(&self, account_id: &str) -> Result { + self.calls + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(account_id.to_string()); + let current = self.concurrent.fetch_add(1, Ordering::SeqCst) + 1; + self.peak.fetch_max(current, Ordering::SeqCst); + + if self.blocked.contains(account_id) { + self.gate.notified().await; + } else if let Some(delay) = self.per_call_delay { + sleep(delay).await; + } + + self.concurrent.fetch_sub(1, Ordering::SeqCst); + + if self.fail.contains(account_id) { + return Err(SyncError::AccountNotFound(account_id.to_string())); + } + Ok(1) + } + } + + fn dispatcher_with( + syncer: Arc, + concurrency: usize, + timeout: Duration, + ) -> SyncDispatcher { + SyncDispatcher::new(syncer, ShutdownSignal::never(), concurrency, timeout) + } + + #[tokio::test] + async fn bounds_simultaneous_syncs_to_the_configured_cap() { + let syncer = Arc::new(MockSyncer::new().with_delay(Duration::from_millis(80))); + let dispatcher = dispatcher_with(syncer.clone(), 2, Duration::from_secs(30)); + + let mut set = tokio::task::JoinSet::new(); + for i in 0..6 { + let d = dispatcher.clone(); + set.spawn(async move { d.sync_account(&format!("acct-{i}")).await }); + } + while let Some(joined) = set.join_next().await { + assert!(matches!(joined, Ok(Ok(1))), "each distinct sync succeeds"); + } + + assert_eq!( + syncer.peak.load(Ordering::SeqCst), + 2, + "no more than the configured cap of 2 syncs may run at once" + ); + assert_eq!(syncer.call_count(), 6, "every distinct account is synced"); + } + + #[tokio::test] + async fn coalesces_a_duplicate_in_flight_sync_for_the_same_account() { + let syncer = Arc::new(MockSyncer::new().with_delay(Duration::from_millis(80))); + let dispatcher = dispatcher_with(syncer.clone(), 4, Duration::from_secs(30)); + + // Three concurrent requests for the SAME account, launched close + // enough together to all observe the first as still in flight. + let mut set = tokio::task::JoinSet::new(); + for _ in 0..3 { + let d = dispatcher.clone(); + set.spawn(async move { d.sync_account("acct-dup").await }); + } + while let Some(joined) = set.join_next().await { + assert!( + matches!(joined, Ok(Ok(1))), + "every joined waiter sees the result" + ); + } + + assert_eq!( + syncer.call_ids(), + vec!["acct-dup".to_string()], + "the underlying sync must run exactly once, not once per request" + ); + } + + #[tokio::test] + async fn a_hung_account_does_not_block_other_accounts() { + // Account A blocks forever; account B must still complete. With the + // cap at 2 both can hold a permit at once, proving the fix for the + // head-of-line blocking regression. + let syncer = Arc::new(MockSyncer::new().blocking("acct-A")); + let dispatcher = dispatcher_with(syncer.clone(), 2, Duration::from_secs(30)); + + let d_a = dispatcher.clone(); + let a = tokio::spawn(async move { d_a.sync_account("acct-A").await }); + + let b = dispatcher.sync_account("acct-B").await; + assert!( + matches!(b, Ok(1)), + "account B completes even while account A is hung, got {b:?}" + ); + + // A is still blocked; it has not resolved. + assert!(!a.is_finished(), "account A's sync is still in flight"); + a.abort(); + } + + #[tokio::test] + async fn a_hung_account_times_out_without_poisoning_the_dispatcher() { + let syncer = Arc::new(MockSyncer::new().blocking("acct-hung")); + let dispatcher = dispatcher_with(syncer.clone(), 2, Duration::from_millis(60)); + + let hung = dispatcher.sync_account("acct-hung").await; + assert!( + matches!(hung, Err(SyncDispatchError::Timeout { .. })), + "a hung account surfaces a typed Timeout, got {hung:?}" + ); + + // The dispatcher is not poisoned: a subsequent sync of a healthy + // account still works. + let ok = dispatcher.sync_account("acct-healthy").await; + assert!( + matches!(ok, Ok(1)), + "a later sync still works after a timeout, got {ok:?}" + ); + } + + #[tokio::test] + async fn surfaces_the_underlying_sync_error_typed() { + let syncer = Arc::new(MockSyncer::new().failing("acct-bad")); + let dispatcher = dispatcher_with(syncer, 2, Duration::from_secs(30)); + + let err = dispatcher.sync_account("acct-bad").await; + assert!( + matches!(err, Err(SyncDispatchError::Sync(_))), + "an underlying sync failure surfaces as SyncDispatchError::Sync, got {err:?}" + ); + } + + #[tokio::test] + async fn rejects_new_work_once_shutdown_is_requested() { + let event_bus = Arc::new(EventBus::new()); + let (controller, shutdown) = crate::lifecycle::ShutdownController::new(event_bus); + let syncer = Arc::new(MockSyncer::new()); + let dispatcher = SyncDispatcher::new(syncer.clone(), shutdown, 2, Duration::from_secs(30)); + + controller.trigger(); + + let rejected = dispatcher.sync_account("acct-late").await; + assert!( + matches!(rejected, Err(SyncDispatchError::ShuttingDown)), + "no new sync is admitted after shutdown, got {rejected:?}" + ); + assert_eq!( + syncer.call_count(), + 0, + "the underlying syncer must not even be invoked after shutdown" + ); + } + + #[tokio::test] + async fn shutdown_unblocks_an_in_flight_sync_promptly() { + let event_bus = Arc::new(EventBus::new()); + let (controller, shutdown) = crate::lifecycle::ShutdownController::new(event_bus); + let syncer = Arc::new(MockSyncer::new().blocking("acct-drain")); + let dispatcher = SyncDispatcher::new(syncer, shutdown, 2, Duration::from_secs(300)); + + let d = dispatcher.clone(); + let handle = tokio::spawn(async move { d.sync_account("acct-drain").await }); + + // Let the sync reach its blocking point, then request shutdown. + sleep(Duration::from_millis(30)).await; + controller.trigger(); + + let outcome = tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("in-flight sync must unblock on shutdown, not hang") + .expect("task joins"); + assert!( + matches!(outcome, Err(SyncDispatchError::ShuttingDown)), + "a sync racing shutdown resolves as ShuttingDown, got {outcome:?}" + ); + } + + fn jmap_account(id: &str) -> AccountConfig { + AccountConfig { + id: id.to_string(), + name: "Test".to_string(), + email_address: format!("{id}@nuncio.mx"), + protocol: AccountProtocol::Jmap, + server_host: "jmap.nuncio.mx".to_string(), + server_port: 443, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + imap_tls_mode: TlsMode::ImplicitTls, + smtp_tls_mode: TlsMode::ImplicitTls, + keyring_secret_key: format!("nuncio/{id}"), + sync_interval_secs: 60, + collection_url: String::new(), + } + } + + fn caldav_account(id: &str) -> AccountConfig { + AccountConfig { + protocol: AccountProtocol::CalDav, + ..jmap_account(id) + } + } + + #[tokio::test] + async fn sync_all_configured_dispatches_every_mail_account_and_skips_dav() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + db.save_account(&jmap_account("acct-mail-1")) + .await + .expect("save mail account 1"); + db.save_account(&jmap_account("acct-mail-2")) + .await + .expect("save mail account 2"); + db.save_account(&caldav_account("acct-dav")) + .await + .expect("save dav account"); + + let event_bus = EventBus::new(); + let mut events = event_bus.subscribe_events(); + let syncer = Arc::new(MockSyncer::new()); + let dispatcher = dispatcher_with(syncer.clone(), 4, Duration::from_secs(30)); + + let total = sync_all_configured(&dispatcher, &db, &event_bus).await; + assert_eq!(total, 2, "both mail accounts synced one message each"); + + let mut synced: Vec = syncer.call_ids(); + synced.sort(); + assert_eq!( + synced, + vec!["acct-mail-1".to_string(), "acct-mail-2".to_string()], + "only the mail accounts are dispatched; the CalDAV account is skipped" + ); + + // The batch is wrapped in a single all-accounts start/complete pair. + assert_eq!( + events.recv().await.expect("start event"), + CoreEvent::SyncStarted { account_id: None } + ); + } +} -- 2.54.0 From a261f8e61a6e5883a0393f33ff54e2259ffa827e Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 18:12:16 -0500 Subject: [PATCH 36/95] feat(nunciod): layered tracing subscriber with EnvFilter, rotating file, and JSON (#368) Replace the stderr-only tracing_subscriber::fmt::init() bootstrap with a layered Registry subscriber: a reloadable EnvFilter (NUNCIO_LOG > RUST_LOG > info) exposed via a reload::Handle for future runtime level control, a human-readable stderr console layer, and a non-blocking rotating daily file sink under /logs/. NUNCIO_LOG_FORMAT=json switches the file sink to JSON. Malformed filters and unwritable log dirs degrade gracefully instead of crashing the daemon; the WorkerGuard is returned and bound in main for the process lifetime. Refs #357 Co-authored-by: James Maes --- Cargo.lock | 103 +++++++++++ Cargo.toml | 3 +- crates/nunciod/Cargo.toml | 3 +- crates/nunciod/src/lib.rs | 1 + crates/nunciod/src/logging.rs | 323 ++++++++++++++++++++++++++++++++++ crates/nunciod/src/main.rs | 25 ++- 6 files changed, 447 insertions(+), 11 deletions(-) create mode 100644 crates/nunciod/src/logging.rs diff --git a/Cargo.lock b/Cargo.lock index d324764..320c097 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -886,6 +886,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.13" @@ -998,6 +1007,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + [[package]] name = "derivative" version = "2.2.0" @@ -2223,6 +2238,15 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -2410,6 +2434,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.46" @@ -2667,6 +2697,7 @@ dependencies = [ "tokio-stream", "tonic", "tracing", + "tracing-appender", "tracing-subscriber", "wiremock", "zeroize", @@ -2978,6 +3009,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -4220,6 +4257,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -4346,6 +4389,36 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -4591,6 +4664,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.19", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.31" @@ -4623,18 +4709,35 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b64c612..b045994 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" tracing = "0.1" -tracing-subscriber = "0.3" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } +tracing-appender = "0.2" sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] } ratatui = "0.28" crossterm = { version = "0.28", features = ["event-stream"] } diff --git a/crates/nunciod/Cargo.toml b/crates/nunciod/Cargo.toml index 060404f..fd79f30 100644 --- a/crates/nunciod/Cargo.toml +++ b/crates/nunciod/Cargo.toml @@ -26,7 +26,8 @@ serde.workspace = true serde_json.workspace = true thiserror.workspace = true tracing.workspace = true -tracing-subscriber = "0.3" +tracing-subscriber.workspace = true +tracing-appender.workspace = true tempfile.workspace = true tonic.workspace = true async-trait.workspace = true diff --git a/crates/nunciod/src/lib.rs b/crates/nunciod/src/lib.rs index 8c24ff7..b37803f 100644 --- a/crates/nunciod/src/lib.rs +++ b/crates/nunciod/src/lib.rs @@ -7,6 +7,7 @@ pub mod contacts_sync; pub mod grpc; pub mod lifecycle; pub mod lock; +pub mod logging; pub mod orchestrator; pub mod outbox; pub mod pagination; diff --git a/crates/nunciod/src/logging.rs b/crates/nunciod/src/logging.rs new file mode 100644 index 0000000..6a0aad4 --- /dev/null +++ b/crates/nunciod/src/logging.rs @@ -0,0 +1,323 @@ +//! Layered `tracing` subscriber for `nunciod`. +//! +//! This replaces the previous stderr-only `tracing_subscriber::fmt::init()` +//! bootstrap, which pinned the level at INFO, ignored `RUST_LOG` (the +//! `env-filter` feature was not even enabled), had no file sink, no JSON +//! output, and no way to change levels at runtime -- leaving the daemon +//! effectively unfollowable and unauditable. +//! +//! The subscriber is a layered [`Registry`] with three layers: +//! +//! 1. A reloadable [`EnvFilter`] applied as the global filter, resolved from +//! the environment (see [`resolve_filter_directive`]). It is wrapped in a +//! [`reload::Layer`] so a future control RPC can change verbosity at +//! runtime without a restart; the [`reload::Handle`] is returned from +//! [`init`]. A malformed filter string never panics -- it degrades to the +//! default and a `warn!` is emitted once the subscriber is live. +//! 2. A human-readable console layer to stderr (ANSI, target + span context). +//! 3. A rotating daily file layer under `/logs/`, written through a +//! non-blocking background writer. +//! +//! ## Environment variables +//! +//! * `NUNCIO_LOG` -- filter directive; takes precedence over `RUST_LOG`. +//! * `RUST_LOG` -- filter directive; used when `NUNCIO_LOG` is unset/empty. +//! * (default) -- when neither is set, the filter is [`DEFAULT_FILTER`]. +//! * `NUNCIO_LOG_FORMAT=json` -- emit the *file* sink as JSON. The console +//! sink always stays human-readable (it is for an operator watching a +//! terminal); the file sink is the one a machine parses, so JSON is applied +//! there. Any other value (or unset) keeps the file sink human-readable. +//! +//! ## Worker guard lifetime (CRITICAL) +//! +//! The rotating file layer uses [`tracing_appender::non_blocking`], which +//! hands back a [`WorkerGuard`]. That guard MUST be held for the entire +//! process lifetime: when it drops, the background writer thread is joined and +//! any buffered-but-unflushed log lines are discarded. [`init`] therefore +//! returns it inside [`LoggingGuards`], and `main` binds that value (e.g. +//! `let _guards = ...;`) so it lives until the daemon exits. Dropping it early +//! silently loses file logs. +//! +//! ## Failure handling +//! +//! Logging must never crash the daemon. If the log directory cannot be +//! created, the file layer is skipped and the daemon logs to stderr only +//! (with a `warn!`). If a global subscriber is already installed (e.g. a test +//! harness set one), [`init`] leaves it in place and returns empty guards. + +use std::path::{Path, PathBuf}; + +use tracing_appender::non_blocking::WorkerGuard; +use tracing_subscriber::layer::{Layered, SubscriberExt}; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{reload, EnvFilter, Layer, Registry}; + +/// Filter directive used when neither `NUNCIO_LOG` nor `RUST_LOG` is set. +pub const DEFAULT_FILTER: &str = "info"; + +/// Environment variable that overrides the log filter, taking precedence over +/// `RUST_LOG`. +pub const LOG_FILTER_ENV_VAR: &str = "NUNCIO_LOG"; + +/// Environment variable that, when equal to `"json"` (case-insensitive), +/// switches the file sink to JSON output. +pub const LOG_FORMAT_ENV_VAR: &str = "NUNCIO_LOG_FORMAT"; + +/// The output encoding of a log sink. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LogFormat { + /// Human-readable, single-line-per-event text. + Human, + /// One JSON object per event, for machine ingestion. + Json, +} + +/// The base registry with the reloadable [`EnvFilter`] applied. The concrete +/// fmt layers are boxed against this type so the JSON and human variants (and +/// the presence/absence of the file layer) can be chosen at runtime without +/// leaking their differing types into the subscriber's type signature. +type FilteredRegistry = Layered, Registry>; + +/// A type-erased layer compatible with the filtered registry. +type DynLayer = Box + Send + Sync>; + +/// Handle that can swap the global [`EnvFilter`] at runtime. Exposed so a +/// future control RPC can raise or lower verbosity without restarting the +/// daemon. +pub type FilterReloadHandle = reload::Handle; + +/// Guards and handles that must outlive the process. +/// +/// Hold this for the whole daemon lifetime. See the module docs on the +/// [`WorkerGuard`] lifetime requirement. +pub struct LoggingGuards { + reload_handle: Option, + // Kept alive purely for its `Drop`: dropping it flushes and shuts down the + // non-blocking file writer. Never read directly. + _file_guard: Option, +} + +impl LoggingGuards { + /// Returns the runtime filter-reload handle, if the subscriber this crate + /// installed is the active one. `None` means a subscriber was already set + /// by someone else (e.g. a test harness) and this handle would not affect + /// it. + pub fn reload_handle(&self) -> Option<&FilterReloadHandle> { + self.reload_handle.as_ref() + } +} + +/// Resolves the effective filter directive with precedence +/// `NUNCIO_LOG` > `RUST_LOG` > [`DEFAULT_FILTER`]. +/// +/// An env var set to an empty/whitespace-only string is treated as unset so a +/// stray `NUNCIO_LOG=` does not silence all output. +pub fn resolve_filter_directive() -> String { + env_directive(LOG_FILTER_ENV_VAR) + .or_else(|| env_directive("RUST_LOG")) + .unwrap_or_else(|| DEFAULT_FILTER.to_string()) +} + +fn env_directive(var: &str) -> Option { + match std::env::var(var) { + Ok(value) if !value.trim().is_empty() => Some(value), + _ => None, + } +} + +/// Builds an [`EnvFilter`] from `directive`, degrading to [`DEFAULT_FILTER`] +/// on a parse error instead of panicking. +/// +/// The second element is `Some(directive)` when the directive was malformed +/// and the default was substituted, so the caller can emit a `warn!` once the +/// subscriber is live (warning before init would go nowhere). +pub fn build_filter(directive: &str) -> (EnvFilter, Option) { + match EnvFilter::try_new(directive) { + Ok(filter) => (filter, None), + Err(_) => (EnvFilter::new(DEFAULT_FILTER), Some(directive.to_string())), + } +} + +/// Reads the desired file-sink format from [`LOG_FORMAT_ENV_VAR`]. +pub fn log_format_from_env() -> LogFormat { + match std::env::var(LOG_FORMAT_ENV_VAR) { + Ok(value) if value.trim().eq_ignore_ascii_case("json") => LogFormat::Json, + _ => LogFormat::Human, + } +} + +/// Derives the log directory (`/logs/`) from the database path, +/// whose parent is the daemon's data directory. +pub fn log_dir_for_db_path(db_path: &Path) -> PathBuf { + match db_path.parent() { + Some(dir) if !dir.as_os_str().is_empty() => dir.join("logs"), + _ => PathBuf::from("logs"), + } +} + +/// Installs the global layered subscriber and returns the guards that must be +/// held for the process lifetime. +/// +/// `db_path` is the resolved database path; logs are written to a sibling +/// `logs/` directory. This never panics and never returns an error: any +/// problem (unwritable log directory, an already-installed subscriber) is +/// degraded to a lesser-but-working configuration and reported via `warn!`. +pub fn init(db_path: &Path) -> LoggingGuards { + let directive = resolve_filter_directive(); + let (filter, malformed) = build_filter(&directive); + let (filter_layer, reload_handle) = reload::Layer::new(filter); + + let format = log_format_from_env(); + let log_dir = log_dir_for_db_path(db_path); + + // Try to stand up the rotating file sink. A failure to create the log + // directory must not stop the daemon; it just falls back to stderr-only. + let (file_layer, file_guard, file_error) = match std::fs::create_dir_all(&log_dir) { + Ok(()) => { + let appender = tracing_appender::rolling::daily(&log_dir, "nunciod.log"); + let (writer, guard) = tracing_appender::non_blocking(appender); + let layer: DynLayer = match format { + LogFormat::Json => tracing_subscriber::fmt::layer() + .json() + .with_writer(writer) + .with_ansi(false) + .boxed(), + LogFormat::Human => tracing_subscriber::fmt::layer() + .with_writer(writer) + .with_ansi(false) + .boxed(), + }; + (Some(layer), Some(guard), None) + } + Err(err) => (None, None, Some(format!("{}: {err}", log_dir.display()))), + }; + + // Human-readable console sink for an operator watching the terminal. This + // stays text even when the file sink is JSON. + let console_layer: DynLayer = tracing_subscriber::fmt::layer() + .with_writer(std::io::stderr) + .with_ansi(true) + .with_target(true) + .boxed(); + + let mut layers: Vec = vec![console_layer]; + if let Some(layer) = file_layer { + layers.push(layer); + } + + let subscriber = tracing_subscriber::registry() + .with(filter_layer) + .with(layers); + + if subscriber.try_init().is_err() { + // A subscriber is already installed (commonly a test harness). Leave + // it untouched and hand back empty guards; our reload handle would not + // apply to it, so do not expose it. + return LoggingGuards { + reload_handle: None, + _file_guard: None, + }; + } + + if let Some(bad) = malformed { + tracing::warn!( + directive = %bad, + default = %DEFAULT_FILTER, + "ignoring malformed log filter directive; using default" + ); + } + if let Some(err) = file_error { + tracing::warn!(error = %err, "could not open log directory; logging to stderr only"); + } + + LoggingGuards { + reload_handle: Some(reload_handle), + _file_guard: file_guard, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The process environment is global mutable state; serialize every test + // that reads or writes the log env vars so they cannot interleave. + static ENV_GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn clear_env() { + std::env::remove_var(LOG_FILTER_ENV_VAR); + std::env::remove_var("RUST_LOG"); + std::env::remove_var(LOG_FORMAT_ENV_VAR); + } + + #[test] + fn nuncio_log_beats_rust_log_beats_default() { + let _guard = ENV_GUARD.lock().unwrap_or_else(|e| e.into_inner()); + clear_env(); + + // Default when neither is set. + assert_eq!(resolve_filter_directive(), DEFAULT_FILTER); + + // RUST_LOG wins over the default. + std::env::set_var("RUST_LOG", "warn"); + assert_eq!(resolve_filter_directive(), "warn"); + + // NUNCIO_LOG wins over RUST_LOG. + std::env::set_var(LOG_FILTER_ENV_VAR, "debug"); + assert_eq!(resolve_filter_directive(), "debug"); + + // An empty NUNCIO_LOG is treated as unset, so RUST_LOG applies again. + std::env::set_var(LOG_FILTER_ENV_VAR, " "); + assert_eq!(resolve_filter_directive(), "warn"); + + clear_env(); + } + + #[test] + fn malformed_filter_falls_back_to_default_without_panicking() { + // A valid directive parses and signals no fallback. + let (_filter, malformed) = build_filter("info,nunciod=debug"); + assert!(malformed.is_none()); + + // A directive with an invalid level degrades to the default and + // reports the offending string, rather than panicking. + let (_filter, malformed) = build_filter("nunciod=not_a_level"); + assert_eq!(malformed.as_deref(), Some("nunciod=not_a_level")); + } + + #[test] + fn json_format_is_selected_only_by_the_json_value() { + let _guard = ENV_GUARD.lock().unwrap_or_else(|e| e.into_inner()); + clear_env(); + + assert_eq!(log_format_from_env(), LogFormat::Human); + + std::env::set_var(LOG_FORMAT_ENV_VAR, "json"); + assert_eq!(log_format_from_env(), LogFormat::Json); + + // Case-insensitive. + std::env::set_var(LOG_FORMAT_ENV_VAR, "JSON"); + assert_eq!(log_format_from_env(), LogFormat::Json); + + // Any other value stays human-readable. + std::env::set_var(LOG_FORMAT_ENV_VAR, "text"); + assert_eq!(log_format_from_env(), LogFormat::Human); + + clear_env(); + } + + #[test] + fn log_dir_is_a_logs_sibling_of_the_database_file() { + let db = PathBuf::from("/var/lib/nuncio/nuncio.db"); + assert_eq!( + log_dir_for_db_path(&db), + PathBuf::from("/var/lib/nuncio/logs") + ); + + // A bare filename (no parent) still yields a usable relative dir. + assert_eq!( + log_dir_for_db_path(Path::new("nuncio.db")), + PathBuf::from("logs") + ); + } +} diff --git a/crates/nunciod/src/main.rs b/crates/nunciod/src/main.rs index f483522..3b9a892 100644 --- a/crates/nunciod/src/main.rs +++ b/crates/nunciod/src/main.rs @@ -17,7 +17,22 @@ const SHUTDOWN_GRACE_PERIOD: Duration = Duration::from_secs(10); #[tokio::main] async fn main() -> Result<(), Box> { - tracing_subscriber::fmt::init(); + // PERSISTENT database path: defaults to `~/.nuncio/nuncio.db` -- NOT a + // temp/ephemeral path -- so accounts (and everything else) survive a + // daemon restart. `NUNCIO_DB_PATH` overrides it, which tests/CI use to + // point at an isolated temp path instead of touching a real user's home + // directory. Resolved first so logging can place its rotating files in the + // sibling `logs/` directory of the same data root. + let db_path = std::env::var("NUNCIO_DB_PATH") + .map(std::path::PathBuf::from) + .unwrap_or_else(|_| nunciod::default_db_path()); + + // Install the layered tracing subscriber (EnvFilter + stderr console + + // rotating daily file sink). `_log_guards` owns the non-blocking writer's + // `WorkerGuard`; it MUST stay bound until the end of `main` or buffered + // file logs are dropped on exit. Logging never crashes the daemon: a bad + // filter or an unwritable log directory degrades to a working subset. + let _log_guards = nunciod::logging::init(&db_path); tracing::info!("Starting Nuncio Central Daemon Service (nunciod)..."); // The `CoreCommand` receiver must be claimed BEFORE the `EventBus` is @@ -37,14 +52,6 @@ async fn main() -> Result<(), Box> { let (shutdown_controller, shutdown_signal) = ShutdownController::new(event_bus.clone()); let shutdown_controller = Arc::new(shutdown_controller); let signal_task = tokio::spawn(install_signal_handlers(shutdown_controller)); - // PERSISTENT database path: defaults to `~/.nuncio/nuncio.db` -- NOT a - // temp/ephemeral path -- so accounts (and everything else) survive a - // daemon restart. - // `NUNCIO_DB_PATH` overrides it, which tests/CI use to point at an - // isolated temp path instead of touching a real user's home directory. - let db_path = std::env::var("NUNCIO_DB_PATH") - .map(std::path::PathBuf::from) - .unwrap_or_else(|_| nunciod::default_db_path()); // Exclusive single-instance lock on the database path, acquired BEFORE // the store is opened. Held for the entire process lifetime (dropped at -- 2.54.0 From eb117d2f47bbe93dd1c911dfbd6a1e18b80ea213 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 18:13:13 -0500 Subject: [PATCH 37/95] refactor(proto): model account transport as a oneof (#367) Make an invalid account transport mix unrepresentable: an account now carries exactly one transport instead of a flat bag of optional imap/jmap/dav fields. - proto: AccountConfig gains `oneof transport { ImapSmtpTransport | JmapTransport | DavTransport }`; the flat fields (protocol, server_host/port, use_tls, imap/smtp tls modes, smtp_host/port, collection_url) are removed and their field numbers reserved. Golden descriptor regenerated; contract_stability passes. - core: AccountConfig carries a `Transport` enum; validate() checks the active variant and preserves the cleartext-to-remote-host refusal per transport. - store: physical accounts columns stay flat; the existing `protocol` column is the transport-kind discriminator. Each variant round-trips. - daemon: grpc maps core Transport <-> proto oneof both ways; an unset transport returns typed VALIDATION_FAILED, never a panic. - cli: `account add --protocol` gates transport flags; list/show render the active transport; --json stays coherent. Refs #278 Co-authored-by: James Maes --- crates/nuncio-cli/src/runner.rs | 284 +++++++++----- crates/nuncio-core/src/config.rs | 362 ++++++++++++------ crates/nuncio-core/src/lib.rs | 5 +- .../proto/nuncio/v1/descriptor.bin | Bin 15298 -> 15645 bytes .../nuncio-proto/proto/nuncio/v1/nuncio.proto | 91 +++-- crates/nuncio-store/src/db.rs | 304 ++++++++++----- crates/nuncio-store/src/recovery.rs | 87 +++-- crates/nunciod/src/calendar_sync.rs | 2 +- crates/nunciod/src/grpc.rs | 340 ++++++++-------- crates/nunciod/src/orchestrator.rs | 16 +- crates/nunciod/src/outbox.rs | 9 +- crates/nunciod/src/send.rs | 30 +- crates/nunciod/src/sync.rs | 49 ++- crates/nunciod/src/sync_dispatcher.rs | 19 +- .../tests/accounts_lifecycle_e2e_test.rs | 45 ++- .../tests/caldav_production_e2e_test.rs | 16 +- .../nunciod/tests/outbox_executor_e2e_test.rs | 16 +- crates/nunciod/tests/spine_e2e_test.rs | 22 +- 18 files changed, 1082 insertions(+), 615 deletions(-) diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index 7afe09a..a74ef3a 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -53,6 +53,64 @@ fn parse_account_protocol(protocol: &str) -> Result serde_json::Value { + use nuncio_proto::v1::account_config::Transport; + match &config.transport { + Some(Transport::ImapSmtp(t)) => json!({ + "protocol": "imap-smtp", + "imap_host": t.imap_host, + "imap_port": t.imap_port, + "imap_tls_mode": t.imap_tls_mode().as_str_name(), + "smtp_host": t.smtp_host, + "smtp_port": t.smtp_port, + "smtp_tls_mode": t.smtp_tls_mode().as_str_name(), + }), + Some(Transport::Jmap(t)) => json!({ + "protocol": "jmap", + "endpoint_host": t.endpoint_host, + }), + Some(Transport::Dav(t)) => json!({ + "protocol": "caldav", + "collection_url": t.collection_url, + }), + None => json!({ "protocol": "unspecified" }), + } +} + +/// Merges the top-level object fields of `extra` into `base` (both must be +/// JSON objects). Used to fold an account's transport-specific fields into its +/// common-field object without hand-writing every combination. +fn merge_json(base: &mut serde_json::Value, extra: serde_json::Value) { + if let (Some(base), serde_json::Value::Object(extra)) = (base.as_object_mut(), extra) { + for (k, v) in extra { + base.insert(k, v); + } + } +} + +/// Renders an account's active transport as a single human-readable summary +/// line fragment for `account list`/`show`. +fn account_transport_summary(config: &nuncio_proto::v1::AccountConfig) -> String { + use nuncio_proto::v1::account_config::Transport; + match &config.transport { + Some(Transport::ImapSmtp(t)) => format!( + "IMAP {}:{} ({}) SMTP {}:{} ({})", + t.imap_host, + t.imap_port, + t.imap_tls_mode().as_str_name(), + t.smtp_host, + t.smtp_port, + t.smtp_tls_mode().as_str_name(), + ), + Some(Transport::Jmap(t)) => format!("JMAP {}", t.endpoint_host), + Some(Transport::Dav(t)) => format!("CalDAV {}", t.collection_url), + None => "(no transport configured)".to_string(), + } +} + /// Parses a `filter export --format` CLI string into its wire-format /// `nuncio.v1.RuleExportFormat` value. Rejects anything else rather than /// silently falling back to one rendering, since a typo'd format should @@ -1869,30 +1927,45 @@ impl HeadlessRunner { let keyring_key = format!("nuncio/{}", email); let account_id = format!("acct-{}", email.replace('@', "-at-").replace('.', "-")); - // A CalDAV account is addressed by its collection URL, not by mail - // host/port endpoints, so those are left empty for it. + // Build the single transport the account speaks. Each transport carries + // only its own endpoint fields, so a nonsensical mix is unrepresentable. + let transport = match account_protocol { + nuncio_proto::v1::AccountProtocol::ImapSmtp => { + nuncio_proto::v1::account_config::Transport::ImapSmtp( + nuncio_proto::v1::ImapSmtpTransport { + imap_host: imap_host.to_string(), + imap_port: u32::from(imap_port), + imap_tls_mode: map_tls_mode_to_proto(imap_tls_mode).into(), + smtp_host: smtp_host.to_string(), + smtp_port: u32::from(smtp_port), + smtp_tls_mode: map_tls_mode_to_proto(smtp_tls_mode).into(), + }, + ) + } + nuncio_proto::v1::AccountProtocol::Jmap => { + // A JMAP account addresses its provider by a session endpoint + // host; the `--imap-host` flag supplies it. + nuncio_proto::v1::account_config::Transport::Jmap(nuncio_proto::v1::JmapTransport { + endpoint_host: imap_host.to_string(), + }) + } + nuncio_proto::v1::AccountProtocol::Caldav => { + nuncio_proto::v1::account_config::Transport::Dav(nuncio_proto::v1::DavTransport { + collection_url: collection_url.unwrap_or_default().to_string(), + }) + } + nuncio_proto::v1::AccountProtocol::Unspecified => { + return Self::render_error("account protocol is required", json_mode) + } + }; + let proto_config = nuncio_proto::v1::AccountConfig { id: account_id.clone(), name: email.to_string(), email_address: email.to_string(), - protocol: account_protocol.into(), - server_host: if is_caldav { - String::new() - } else { - imap_host.to_string() - }, - server_port: if is_caldav { 0 } else { u32::from(imap_port) }, - imap_tls_mode: map_tls_mode_to_proto(imap_tls_mode).into(), - smtp_tls_mode: map_tls_mode_to_proto(smtp_tls_mode).into(), keyring_secret_key: keyring_key.clone(), sync_interval: Some(nuncio_proto::time::duration_from_secs(300)), - smtp_host: if is_caldav { - String::new() - } else { - smtp_host.to_string() - }, - smtp_port: if is_caldav { 0 } else { u32::from(smtp_port) }, - collection_url: collection_url.unwrap_or_default().to_string(), + transport: Some(transport), }; let mut client = match self.connect_accounts_client().await { @@ -1916,6 +1989,7 @@ impl HeadlessRunner { .config .map(|c| c.id) .unwrap_or(account_id); + let is_jmap = account_protocol == nuncio_proto::v1::AccountProtocol::Jmap; if is_caldav { let collection = collection_url.unwrap_or_default(); if json_mode { @@ -1932,6 +2006,21 @@ impl HeadlessRunner { "CalDAV account '{email}' (ID: {account_id}) added via nunciod daemon with collection URL {collection}" ) } + } else if is_jmap { + if json_mode { + format_json(&json!({ + "configured": true, + "account_id": account_id, + "email": email, + "protocol": "jmap", + "endpoint_host": imap_host, + "keyring_key": keyring_key + })) + } else { + format!( + "JMAP account '{email}' (ID: {account_id}) added via nunciod daemon with endpoint host {imap_host}" + ) + } } else if json_mode { format_json(&json!({ "configured": true, @@ -1983,21 +2072,15 @@ impl HeadlessRunner { let accounts_json: Vec = accounts .iter() .map(|a| { - json!({ + let mut base = json!({ "id": a.id, "name": a.name, "email_address": a.email_address, - "protocol": a.protocol().as_str_name(), - "server_host": a.server_host, - "server_port": a.server_port, - "smtp_host": a.smtp_host, - "smtp_port": a.smtp_port, - "imap_tls_mode": a.imap_tls_mode().as_str_name(), - "smtp_tls_mode": a.smtp_tls_mode().as_str_name(), "keyring_secret_key": a.keyring_secret_key, "sync_interval_secs": duration_secs(&a.sync_interval), - "collection_url": a.collection_url, - }) + }); + merge_json(&mut base, account_transport_json(a)); + base }) .collect(); format_json(&json!({ "accounts": accounts_json })) @@ -2007,25 +2090,13 @@ impl HeadlessRunner { accounts.len() ); for a in &accounts { - if a.protocol() == nuncio_proto::v1::AccountProtocol::Caldav { - out.push_str(&format!( - "\n [{}] {} <{}> CalDAV {}", - a.id, a.name, a.email_address, a.collection_url, - )); - } else { - out.push_str(&format!( - "\n [{}] {} <{}> IMAP {}:{} ({}) SMTP {}:{} ({})", - a.id, - a.name, - a.email_address, - a.server_host, - a.server_port, - a.imap_tls_mode().as_str_name(), - a.smtp_host, - a.smtp_port, - a.smtp_tls_mode().as_str_name(), - )); - } + out.push_str(&format!( + "\n [{}] {} <{}> {}", + a.id, + a.name, + a.email_address, + account_transport_summary(a), + )); } out } @@ -2078,32 +2149,22 @@ impl HeadlessRunner { }; if json_mode { - format_json(&json!({ + let mut base = json!({ "id": account.id, "name": account.name, "email_address": account.email_address, - "protocol": account.protocol().as_str_name(), - "server_host": account.server_host, - "server_port": account.server_port, - "smtp_host": account.smtp_host, - "smtp_port": account.smtp_port, - "imap_tls_mode": account.imap_tls_mode().as_str_name(), - "smtp_tls_mode": account.smtp_tls_mode().as_str_name(), "keyring_secret_key": account.keyring_secret_key, "sync_interval_secs": duration_secs(&account.sync_interval), - })) + }); + merge_json(&mut base, account_transport_json(&account)); + format_json(&base) } else { format!( - "Account [{}] {} <{}> IMAP {}:{} ({}) SMTP {}:{} ({})", + "Account [{}] {} <{}> {}", account.id, account.name, account.email_address, - account.server_host, - account.server_port, - account.imap_tls_mode().as_str_name(), - account.smtp_host, - account.smtp_port, - account.smtp_tls_mode().as_str_name(), + account_transport_summary(&account), ) } } @@ -2134,28 +2195,51 @@ impl HeadlessRunner { if let Some(email) = email { config.email_address = email.to_string(); } - if let Some(host) = imap_host { - config.server_host = host.to_string(); - } - if let Some(port) = imap_port { - config.server_port = u32::from(port); - } - if let Some(host) = smtp_host { - config.smtp_host = host.to_string(); - } - if let Some(port) = smtp_port { - config.smtp_port = u32::from(port); - } - if let Some(mode) = imap_mode { - match parse_tls_mode(mode) { - Ok(mode) => config.imap_tls_mode = map_tls_mode_to_proto(mode).into(), - Err(e) => return Self::render_error(&e, json_mode), - } - } - if let Some(mode) = smtp_mode { - match parse_tls_mode(mode) { - Ok(mode) => config.smtp_tls_mode = map_tls_mode_to_proto(mode).into(), - Err(e) => return Self::render_error(&e, json_mode), + + // The IMAP/SMTP endpoint flags only apply to an imap-smtp account; the + // transport oneof makes the alternatives (JMAP endpoint, DAV URL) + // structurally distinct, so applying these to another transport would + // be meaningless -- reject it rather than silently ignoring the flags. + let touches_imap_smtp_fields = imap_host.is_some() + || imap_port.is_some() + || smtp_host.is_some() + || smtp_port.is_some() + || imap_mode.is_some() + || smtp_mode.is_some(); + if touches_imap_smtp_fields { + match &mut config.transport { + Some(nuncio_proto::v1::account_config::Transport::ImapSmtp(t)) => { + if let Some(host) = imap_host { + t.imap_host = host.to_string(); + } + if let Some(port) = imap_port { + t.imap_port = u32::from(port); + } + if let Some(host) = smtp_host { + t.smtp_host = host.to_string(); + } + if let Some(port) = smtp_port { + t.smtp_port = u32::from(port); + } + if let Some(mode) = imap_mode { + match parse_tls_mode(mode) { + Ok(mode) => t.imap_tls_mode = map_tls_mode_to_proto(mode).into(), + Err(e) => return Self::render_error(&e, json_mode), + } + } + if let Some(mode) = smtp_mode { + match parse_tls_mode(mode) { + Ok(mode) => t.smtp_tls_mode = map_tls_mode_to_proto(mode).into(), + Err(e) => return Self::render_error(&e, json_mode), + } + } + } + _ => { + return Self::render_error( + "the IMAP/SMTP flags only apply to an imap-smtp account", + json_mode, + ) + } } } @@ -3050,11 +3134,10 @@ mod tests { async fn account_add_and_list_round_trip_over_grpc_to_a_stub_daemon() { use nuncio_proto::v1::accounts_server::{Accounts as AccountsService, AccountsServer}; use nuncio_proto::v1::{ - AccountConfig as AccountConfigProto, AccountProtocol as AccountProtocolProto, - AddAccountRequest, AddAccountResponse, ListAccountsRequest, ListAccountsResponse, - RemoveAccountRequest, RemoveAccountResponse, TestAccountConnectionRequest, - TestAccountConnectionResponse, TlsMode as TlsModeProto, UpdateAccountRequest, - UpdateAccountResponse, + AccountConfig as AccountConfigProto, AddAccountRequest, AddAccountResponse, + ListAccountsRequest, ListAccountsResponse, RemoveAccountRequest, RemoveAccountResponse, + TestAccountConnectionRequest, TestAccountConnectionResponse, TlsMode as TlsModeProto, + UpdateAccountRequest, UpdateAccountResponse, }; use std::sync::Mutex; @@ -3117,16 +3200,18 @@ mod tests { id: "acct-stub-1".to_string(), name: "Stub Account".to_string(), email_address: "stub@nuncio.mx".to_string(), - protocol: AccountProtocolProto::ImapSmtp.into(), - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - imap_tls_mode: TlsModeProto::ImplicitTls.into(), - smtp_tls_mode: TlsModeProto::ImplicitTls.into(), keyring_secret_key: "nuncio/acct-stub-1".to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(300)), - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - collection_url: String::new(), + transport: Some(nuncio_proto::v1::account_config::Transport::ImapSmtp( + nuncio_proto::v1::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: TlsModeProto::ImplicitTls.into(), + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: TlsModeProto::ImplicitTls.into(), + }, + )), }], })) } @@ -3201,7 +3286,8 @@ mod tests { .await; assert!(list_out.contains("acct-stub-1")); assert!(list_out.contains("stub@nuncio.mx")); - assert!(list_out.contains(r#""protocol":"ACCOUNT_PROTOCOL_IMAP_SMTP""#)); + assert!(list_out.contains(r#""protocol":"imap-smtp""#)); + assert!(list_out.contains(r#""imap_host":"imap.nuncio.mx""#)); let list_out_text = runner .execute_command( diff --git a/crates/nuncio-core/src/config.rs b/crates/nuncio-core/src/config.rs index 348bbec..8c5fc99 100644 --- a/crates/nuncio-core/src/config.rs +++ b/crates/nuncio-core/src/config.rs @@ -74,7 +74,74 @@ pub enum ConfigError { }, } -/// Complete configuration schema for a mail & calendar account. +/// IMAP/SMTP transport parameters: inbound mail is fetched over IMAP4rev1 and +/// outbound mail is submitted over SMTP. Inbound and outbound carry +/// independent host/port/TLS settings because a provider's IMAP and SMTP +/// endpoints commonly differ. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ImapSmtpTransport { + /// IMAP server hostname or IP address (inbound mail). + pub imap_host: String, + /// IMAP server port number. + pub imap_port: u16, + /// IMAP connection security transport mode. + pub imap_tls_mode: TlsMode, + /// SMTP server hostname or IP address (outbound mail). + pub smtp_host: String, + /// SMTP server port number. + pub smtp_port: u16, + /// SMTP connection security transport mode. + pub smtp_tls_mode: TlsMode, +} + +/// JMAP transport parameters (RFC 8620 / RFC 8621). A JMAP account addresses +/// its provider by a session endpoint host; the daemon derives the well-known +/// session URL from it. JMAP always runs over HTTPS, so there is no separate +/// cleartext TLS mode to configure. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct JmapTransport { + /// JMAP session endpoint host (the daemon derives the session URL from it). + pub endpoint_host: String, +} + +/// DAV transport parameters (CalDAV RFC 4791 today, CardDAV later). A DAV +/// account is addressed by a fully-qualified collection URL rather than a mail +/// host/port, because a DAV client dispatches requests directly at that URL. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DavTransport { + /// Fully-qualified collection URL (scheme + host + path). + pub collection_url: String, +} + +/// The single transport an account speaks. Modeling it as an enum makes a +/// nonsensical mix of transport settings unrepresentable: an account carries +/// exactly one variant, so the IMAP/SMTP, JMAP, and DAV parameters can never +/// be populated simultaneously. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Transport { + /// IMAP inbound + SMTP outbound mail. + ImapSmtp(ImapSmtpTransport), + /// JMAP mail. + Jmap(JmapTransport), + /// WebDAV collection (CalDAV today, CardDAV later). + Dav(DavTransport), +} + +impl Transport { + /// The [`AccountProtocol`] discriminant this transport corresponds to. + pub fn protocol(&self) -> AccountProtocol { + match self { + Transport::ImapSmtp(_) => AccountProtocol::ImapSmtp, + Transport::Jmap(_) => AccountProtocol::Jmap, + Transport::Dav(_) => AccountProtocol::CalDav, + } + } +} + +/// Complete configuration schema for a mail & calendar account. The account's +/// common identity fields live here; its transport-specific parameters live in +/// exactly one [`Transport`] variant (see that type's doc comment). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AccountConfig { /// Unique identifier for the account configuration. @@ -83,32 +150,12 @@ pub struct AccountConfig { pub name: String, /// Primary email address associated with the account. pub email_address: String, - /// Protocol engine type. - pub protocol: AccountProtocol, - /// Server hostname or IP address (IMAP/JMAP). - pub server_host: String, - /// Server port number (IMAP/JMAP). - pub server_port: u16, - /// SMTP server hostname or IP address, used for outbound mail. Distinct - /// from `server_host` because a mail provider's inbound (IMAP/JMAP) and - /// outbound (SMTP) endpoints commonly differ. - pub smtp_host: String, - /// SMTP server port number, used for outbound mail. - pub smtp_port: u16, - /// IMAP connection security transport mode. - pub imap_tls_mode: TlsMode, - /// SMTP connection security transport mode. - pub smtp_tls_mode: TlsMode, /// Keyring entry key used to retrieve OS vault credentials. pub keyring_secret_key: String, /// Background sync polling interval in seconds (minimum 10s). pub sync_interval_secs: u64, - /// Fully-qualified collection URL for DAV-protocol accounts (CalDAV - /// today, CardDAV later). Set for accounts whose protocol speaks WebDAV; - /// empty for mail accounts. This is a full URL (scheme + host + path), - /// not a bare host, because a DAV client dispatches requests directly at - /// the collection URL. - pub collection_url: String, + /// The account's single transport. + pub transport: Transport, } /// Whether `host` refers to the local loopback interface, either as a @@ -144,7 +191,37 @@ impl AccountConfig { /// Minimum allowed background sync interval (10 seconds). pub const MIN_SYNC_INTERVAL_SECS: u64 = 10; - /// Validate the account configuration fields. + /// The [`AccountProtocol`] discriminant of this account's transport. + pub fn protocol(&self) -> AccountProtocol { + self.transport.protocol() + } + + /// Whether this account speaks WebDAV against a collection URL. + pub fn is_dav(&self) -> bool { + matches!(self.transport, Transport::Dav(_)) + } + + /// The account's IMAP/SMTP transport parameters, present only for an + /// IMAP/SMTP account (the only transport with an SMTP outbound endpoint). + pub fn imap_smtp(&self) -> Option<&ImapSmtpTransport> { + match &self.transport { + Transport::ImapSmtp(t) => Some(t), + _ => None, + } + } + + /// The account's DAV collection URL, present only for a DAV account. + pub fn dav_collection_url(&self) -> Option<&str> { + match &self.transport { + Transport::Dav(t) => Some(t.collection_url.as_str()), + _ => None, + } + } + + /// Validate the account configuration fields. Exactly one transport is + /// always present (guaranteed by the [`Transport`] enum); each variant's + /// endpoint fields are validated against the rules that apply to it, + /// including the cleartext-to-remote-host refusal for any plain endpoint. pub fn validate(&self) -> Result<(), ConfigError> { if self.id.trim().is_empty() { return Err(ConfigError::EmptyField { field: "id" }); @@ -165,57 +242,67 @@ impl AccountConfig { { return Err(ConfigError::InvalidEmailFormat); } - if self.protocol.is_dav() { - // A DAV-protocol account is addressed by its collection URL, not - // by mail host/port endpoints, so it validates the URL instead of - // the IMAP/SMTP fields (which are left empty for such accounts). - let url = self.collection_url.trim(); - if url.is_empty() { - return Err(ConfigError::EmptyField { - field: "collection_url", - }); - } - if !(url.starts_with("http://") || url.starts_with("https://")) { - return Err(ConfigError::InvalidCollectionUrl); - } - // Plain `http://` is only tolerated against a loopback host (a - // local test server); any remote host must use `https://` to - // avoid sending the account's Basic-auth credentials in cleartext. - if url.starts_with("http://") { - let host = url_host(url).unwrap_or_default(); - if !is_loopback_host(host) { + match &self.transport { + Transport::ImapSmtp(t) => { + if t.imap_host.trim().is_empty() { + return Err(ConfigError::EmptyField { field: "imap_host" }); + } + if t.imap_port == 0 { + return Err(ConfigError::InvalidPort); + } + if t.smtp_host.trim().is_empty() { + return Err(ConfigError::EmptyField { field: "smtp_host" }); + } + if t.smtp_port == 0 { + return Err(ConfigError::InvalidPort); + } + // Cleartext `Plain` is only tolerated against a loopback host + // (a local test server); a remote host would send the + // account's credential over the network unencrypted. + if t.imap_tls_mode == TlsMode::Plain && !is_loopback_host(&t.imap_host) { return Err(ConfigError::CleartextToRemoteHost { - endpoint: "caldav", - host: host.to_string(), + endpoint: "imap", + host: t.imap_host.clone(), + }); + } + if t.smtp_tls_mode == TlsMode::Plain && !is_loopback_host(&t.smtp_host) { + return Err(ConfigError::CleartextToRemoteHost { + endpoint: "smtp", + host: t.smtp_host.clone(), }); } } - } else { - if self.server_host.trim().is_empty() { - return Err(ConfigError::EmptyField { - field: "server_host", - }); + Transport::Jmap(t) => { + if t.endpoint_host.trim().is_empty() { + return Err(ConfigError::EmptyField { + field: "endpoint_host", + }); + } } - if self.server_port == 0 { - return Err(ConfigError::InvalidPort); - } - if self.smtp_host.trim().is_empty() { - return Err(ConfigError::EmptyField { field: "smtp_host" }); - } - if self.smtp_port == 0 { - return Err(ConfigError::InvalidPort); - } - if self.imap_tls_mode == TlsMode::Plain && !is_loopback_host(&self.server_host) { - return Err(ConfigError::CleartextToRemoteHost { - endpoint: "imap", - host: self.server_host.clone(), - }); - } - if self.smtp_tls_mode == TlsMode::Plain && !is_loopback_host(&self.smtp_host) { - return Err(ConfigError::CleartextToRemoteHost { - endpoint: "smtp", - host: self.smtp_host.clone(), - }); + Transport::Dav(t) => { + // A DAV account is addressed by its collection URL, not by mail + // host/port endpoints. + let url = t.collection_url.trim(); + if url.is_empty() { + return Err(ConfigError::EmptyField { + field: "collection_url", + }); + } + if !(url.starts_with("http://") || url.starts_with("https://")) { + return Err(ConfigError::InvalidCollectionUrl); + } + // Plain `http://` is only tolerated against a loopback host; any + // remote host must use `https://` to avoid sending the + // account's Basic-auth credentials in cleartext. + if url.starts_with("http://") { + let host = url_host(url).unwrap_or_default(); + if !is_loopback_host(host) { + return Err(ConfigError::CleartextToRemoteHost { + endpoint: "caldav", + host: host.to_string(), + }); + } + } } } if self.keyring_secret_key.trim().is_empty() { @@ -235,21 +322,41 @@ impl AccountConfig { mod tests { use super::*; + fn valid_imap_smtp_transport() -> ImapSmtpTransport { + ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: TlsMode::ImplicitTls, + } + } + + /// A valid IMAP/SMTP account. Named `valid_account` (rather than a + /// transport-specific name) because most common-field validation tests + /// below use it as their baseline. fn valid_account() -> AccountConfig { AccountConfig { id: "acct-123".to_string(), name: "Personal Mail".to_string(), email_address: "user@nuncio.mx".to_string(), - protocol: AccountProtocol::Jmap, - server_host: "jmap.nuncio.mx".to_string(), - server_port: 443, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: TlsMode::ImplicitTls, - smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-123".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: Transport::ImapSmtp(valid_imap_smtp_transport()), + } + } + + fn valid_jmap_account() -> AccountConfig { + AccountConfig { + id: "acct-jmap-1".to_string(), + name: "JMAP Mail".to_string(), + email_address: "user@nuncio.mx".to_string(), + keyring_secret_key: "nuncio/acct-jmap-1".to_string(), + sync_interval_secs: 60, + transport: Transport::Jmap(JmapTransport { + endpoint_host: "jmap.nuncio.mx".to_string(), + }), } } @@ -258,16 +365,20 @@ mod tests { id: "acct-cal-1".to_string(), name: "Work Calendar".to_string(), email_address: "user@nuncio.mx".to_string(), - protocol: AccountProtocol::CalDav, - server_host: String::new(), - server_port: 0, - smtp_host: String::new(), - smtp_port: 0, - imap_tls_mode: TlsMode::ImplicitTls, - smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-cal-1".to_string(), sync_interval_secs: 300, - collection_url: "https://caldav.example.com/calendars/user/work/".to_string(), + transport: Transport::Dav(DavTransport { + collection_url: "https://caldav.example.com/calendars/user/work/".to_string(), + }), + } + } + + /// Convenience: the account's IMAP/SMTP transport, for tests that mutate a + /// transport field on a baseline built by [`valid_account`]. + fn imap_smtp_mut(config: &mut AccountConfig) -> &mut ImapSmtpTransport { + match &mut config.transport { + Transport::ImapSmtp(t) => t, + _ => panic!("expected an IMAP/SMTP transport"), } } @@ -277,10 +388,17 @@ mod tests { assert!(config.validate().is_ok()); } + /// Rebuild `config` with a DAV transport carrying `url`. + fn with_dav_url(config: &mut AccountConfig, url: &str) { + config.transport = Transport::Dav(DavTransport { + collection_url: url.to_string(), + }); + } + #[test] fn caldav_account_requires_collection_url() { let mut config = valid_caldav_account(); - config.collection_url = " ".to_string(); + with_dav_url(&mut config, " "); assert_eq!( config.validate().unwrap_err(), ConfigError::EmptyField { @@ -292,7 +410,7 @@ mod tests { #[test] fn caldav_account_rejects_non_http_collection_url() { let mut config = valid_caldav_account(); - config.collection_url = "ftp://caldav.example.com/work/".to_string(); + with_dav_url(&mut config, "ftp://caldav.example.com/work/"); assert_eq!( config.validate().unwrap_err(), ConfigError::InvalidCollectionUrl @@ -302,7 +420,7 @@ mod tests { #[test] fn caldav_account_rejects_cleartext_http_to_remote_host() { let mut config = valid_caldav_account(); - config.collection_url = "http://remote/dav/".to_string(); + with_dav_url(&mut config, "http://remote/dav/"); assert_eq!( config.validate().unwrap_err(), ConfigError::CleartextToRemoteHost { @@ -315,7 +433,7 @@ mod tests { #[test] fn caldav_account_accepts_https_to_remote_host() { let mut config = valid_caldav_account(); - config.collection_url = "https://remote/dav/".to_string(); + with_dav_url(&mut config, "https://remote/dav/"); assert!(config.validate().is_ok()); } @@ -323,7 +441,7 @@ mod tests { fn caldav_account_accepts_cleartext_http_to_loopback_host() { for url in ["http://127.0.0.1/dav/", "http://localhost/dav/"] { let mut config = valid_caldav_account(); - config.collection_url = url.to_string(); + with_dav_url(&mut config, url); assert!(config.validate().is_ok(), "url {url} should be allowed"); } } @@ -335,10 +453,34 @@ mod tests { assert!(!AccountProtocol::ImapSmtp.is_dav()); } + #[test] + fn transport_reports_its_protocol_discriminant() { + assert_eq!(valid_account().protocol(), AccountProtocol::ImapSmtp); + assert_eq!(valid_jmap_account().protocol(), AccountProtocol::Jmap); + assert_eq!(valid_caldav_account().protocol(), AccountProtocol::CalDav); + assert!(valid_caldav_account().is_dav()); + assert!(!valid_account().is_dav()); + } + #[test] fn valid_account_config_passes_validation() { - let config = valid_account(); - assert!(config.validate().is_ok()); + assert!(valid_account().validate().is_ok()); + assert!(valid_jmap_account().validate().is_ok()); + assert!(valid_caldav_account().validate().is_ok()); + } + + #[test] + fn jmap_account_requires_endpoint_host() { + let mut config = valid_jmap_account(); + config.transport = Transport::Jmap(JmapTransport { + endpoint_host: " ".to_string(), + }); + assert_eq!( + config.validate().unwrap_err(), + ConfigError::EmptyField { + field: "endpoint_host" + } + ); } #[test] @@ -390,28 +532,26 @@ mod tests { } #[test] - fn empty_server_host_fails_validation() { + fn empty_imap_host_fails_validation() { let mut config = valid_account(); - config.server_host = " ".to_string(); + imap_smtp_mut(&mut config).imap_host = " ".to_string(); assert_eq!( config.validate().unwrap_err(), - ConfigError::EmptyField { - field: "server_host" - } + ConfigError::EmptyField { field: "imap_host" } ); } #[test] fn zero_port_fails_validation() { let mut config = valid_account(); - config.server_port = 0; + imap_smtp_mut(&mut config).imap_port = 0; assert_eq!(config.validate().unwrap_err(), ConfigError::InvalidPort); } #[test] fn empty_smtp_host_fails_validation() { let mut config = valid_account(); - config.smtp_host = " ".to_string(); + imap_smtp_mut(&mut config).smtp_host = " ".to_string(); assert_eq!( config.validate().unwrap_err(), ConfigError::EmptyField { field: "smtp_host" } @@ -421,7 +561,7 @@ mod tests { #[test] fn zero_smtp_port_fails_validation() { let mut config = valid_account(); - config.smtp_port = 0; + imap_smtp_mut(&mut config).smtp_port = 0; assert_eq!(config.validate().unwrap_err(), ConfigError::InvalidPort); } @@ -450,12 +590,12 @@ mod tests { #[test] fn plain_imap_to_remote_host_fails_validation() { let mut config = valid_account(); - config.imap_tls_mode = TlsMode::Plain; + imap_smtp_mut(&mut config).imap_tls_mode = TlsMode::Plain; assert_eq!( config.validate().unwrap_err(), ConfigError::CleartextToRemoteHost { endpoint: "imap", - host: "jmap.nuncio.mx".to_string(), + host: "imap.nuncio.mx".to_string(), } ); } @@ -463,7 +603,7 @@ mod tests { #[test] fn plain_smtp_to_remote_host_fails_validation() { let mut config = valid_account(); - config.smtp_tls_mode = TlsMode::Plain; + imap_smtp_mut(&mut config).smtp_tls_mode = TlsMode::Plain; assert_eq!( config.validate().unwrap_err(), ConfigError::CleartextToRemoteHost { @@ -477,10 +617,11 @@ mod tests { fn plain_imap_and_smtp_to_loopback_hosts_pass_validation() { for host in ["127.0.0.1", "localhost", "::1", "LOCALHOST"] { let mut config = valid_account(); - config.server_host = host.to_string(); - config.smtp_host = host.to_string(); - config.imap_tls_mode = TlsMode::Plain; - config.smtp_tls_mode = TlsMode::Plain; + let t = imap_smtp_mut(&mut config); + t.imap_host = host.to_string(); + t.smtp_host = host.to_string(); + t.imap_tls_mode = TlsMode::Plain; + t.smtp_tls_mode = TlsMode::Plain; assert!(config.validate().is_ok(), "host {host} should be allowed"); } } @@ -488,8 +629,9 @@ mod tests { #[test] fn implicit_tls_and_starttls_to_remote_host_pass_validation() { let mut config = valid_account(); - config.imap_tls_mode = TlsMode::ImplicitTls; - config.smtp_tls_mode = TlsMode::StartTls; + let t = imap_smtp_mut(&mut config); + t.imap_tls_mode = TlsMode::ImplicitTls; + t.smtp_tls_mode = TlsMode::StartTls; assert!(config.validate().is_ok()); } diff --git a/crates/nuncio-core/src/lib.rs b/crates/nuncio-core/src/lib.rs index 4c0ca72..f72c4b7 100644 --- a/crates/nuncio-core/src/lib.rs +++ b/crates/nuncio-core/src/lib.rs @@ -10,7 +10,10 @@ pub mod model; pub mod update; pub mod worm_audit; -pub use config::{AccountConfig, AccountProtocol, ConfigError, TlsMode}; +pub use config::{ + AccountConfig, AccountProtocol, ConfigError, DavTransport, ImapSmtpTransport, JmapTransport, + TlsMode, Transport, +}; pub use export::{ExportEngine, ExportError, ExportFormat, ExportSummary}; pub use mcp_policy::{AgentPermissions, DataType, McpAgentPolicy}; pub use model::{Attachment, CalendarEvent, Contact, DaemonTelemetry, Email, Folder}; diff --git a/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin b/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin index 21045b8042fb3ee6a230729a84bae43aa4d01b36..f86c1175b676558b7b70987758876f4ac86bf1d1 100644 GIT binary patch delta 691 zcmZWn%SyvQ6g6$yCX-G(Er>x~NJSBZprUR>ufL=}btb5#016;!5xz zg#LlQ;|I9)PSU4t4w-Y$J@=Kbmyg#^DVKYXNBNvoy8n?-t=+eRZl4D?p+%#Bg}lCo zR1e7B6N|VmZAd$+sR*J4?`cpd^&A9o7Q7aj#|WpH+>d&F=D2!3`Bt~Dz%YUenW3B+ z3SwfI22;aAW+;e>VH%to7Qt{eIYrvAafomSx|;R9iLgtZfO(Y9_gBwqxnb;v x>1L;ofZc`MEDa*J2PZNYuW#ao+nKYUVEidgZvI(+!$P&D<0NP0=7?%Hd%p_B(r5qx delta 263 zcmbPRb*Ov;BNNm9;?2xVjO>gHCg*TP)thl~6cpu`E)H?C1>XA zl^NVv9F3HT#3oXj=V&PyH;0R(bElvd) KySb5PjsXB72~X<) diff --git a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto index 6275953..0100fcd 100644 --- a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto +++ b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto @@ -250,10 +250,54 @@ enum TlsMode { TLS_MODE_PLAIN = 3; } -// Mirrors `nuncio_core::AccountConfig` field-for-field, MINUS the password -// credential: this message is used both as the request payload (alongside -// a separate `password` field) and as the response payload of -// `ListAccounts`, and the password MUST NEVER appear in the latter. +// IMAP/SMTP transport parameters: an account whose inbound mail is fetched +// over IMAP4rev1 and whose outbound mail is submitted over SMTP. Inbound and +// outbound carry independent host/port/TLS settings because a provider's +// IMAP and SMTP endpoints commonly differ. +message ImapSmtpTransport { + // IMAP server hostname or IP address (inbound mail). + string imap_host = 1; + // IMAP server port number (1..=65535). + uint32 imap_port = 2; + // IMAP connection security transport mode. + TlsMode imap_tls_mode = 3; + // SMTP server hostname or IP address (outbound mail). + string smtp_host = 4; + // SMTP server port number (1..=65535). + uint32 smtp_port = 5; + // SMTP connection security transport mode. + TlsMode smtp_tls_mode = 6; +} + +// JMAP transport parameters (RFC 8620 / RFC 8621). A JMAP account addresses +// its provider by a session endpoint host; the daemon derives the well-known +// session URL from it. JMAP always runs over HTTPS, so there is no separate +// cleartext TLS mode to configure here. +message JmapTransport { + // JMAP session endpoint host (the daemon derives the session URL from it). + string endpoint_host = 1; +} + +// DAV transport parameters (CalDAV RFC 4791 today, CardDAV later). A DAV +// account is addressed by a fully-qualified collection URL rather than a mail +// host/port, because a DAV client dispatches requests directly at that URL. +message DavTransport { + // Fully-qualified collection URL (scheme + host + path), e.g. a CalDAV + // calendar collection or, later, a CardDAV address-book collection. + string collection_url = 1; +} + +// Mirrors `nuncio_core::AccountConfig`, MINUS the password credential: this +// message is used both as the request payload (alongside a separate +// `password` field) and as the response payload of `ListAccounts`, and the +// password MUST NEVER appear in the latter. +// +// An account has EXACTLY ONE transport, carried in the `transport` oneof. This +// makes a nonsensical mix (e.g. IMAP host + JMAP endpoint + DAV url all set at +// once) structurally unrepresentable rather than merely discouraged: the +// transport-specific fields live inside their variant sub-message, so only one +// variant's fields can ever be populated. A request with no transport case set +// is rejected as a validation failure. message AccountConfig { // Unique identifier for the account configuration. OPAQUE and // server-assigned: clients MUST NOT parse or construct this value, only @@ -264,34 +308,25 @@ message AccountConfig { string name = 2; // Primary email address associated with the account. string email_address = 3; - // Protocol engine type. - AccountProtocol protocol = 4; - // Server hostname or IP address (IMAP/JMAP). - string server_host = 5; - // Server port number (1..=65535) (IMAP/JMAP). - uint32 server_port = 6; - reserved 7; - reserved "use_tls"; - // IMAP connection security transport mode. - TlsMode imap_tls_mode = 8; - // SMTP connection security transport mode. - TlsMode smtp_tls_mode = 9; + // Field numbers 4-9 and 12-14 held the flat, transport-specific fields + // (protocol, server_host, server_port, use_tls, imap/smtp TLS modes, + // smtp_host, smtp_port, collection_url) before they moved into the + // per-transport sub-messages carried by the `transport` oneof. They are + // reserved so those numbers are never reused with an incompatible meaning. + reserved 4, 5, 6, 7, 8, 9, 12, 13, 14; + reserved "protocol", "server_host", "server_port", "use_tls", "imap_tls_mode", + "smtp_tls_mode", "smtp_host", "smtp_port", "collection_url"; // Keyring entry key used to retrieve OS vault credentials. string keyring_secret_key = 10; // Background sync polling interval (minimum 10s). google.protobuf.Duration sync_interval = 11; - // SMTP server hostname or IP address, used for outbound mail. Distinct - // from `server_host` because a mail provider's inbound (IMAP/JMAP) and - // outbound (SMTP) endpoints commonly differ. - string smtp_host = 12; - // SMTP server port number (1..=65535), used for outbound mail. - uint32 smtp_port = 13; - // Fully-qualified collection URL for DAV-protocol accounts (e.g. a CalDAV - // calendar collection or, later, a CardDAV address-book collection). Set - // for accounts whose protocol speaks WebDAV; empty for mail accounts. This - // is a full URL (scheme + host + path), not a bare host, because a DAV - // client dispatches requests directly at the collection URL. - string collection_url = 14; + // The account's single transport. Exactly one case is set; a request with + // none set is rejected. + oneof transport { + ImapSmtpTransport imap_smtp = 15; + JmapTransport jmap = 16; + DavTransport dav = 17; + } } // AddAccountRequest carries the full account configuration plus the raw diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 2ff1c62..898bfa5 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -954,7 +954,52 @@ impl DatabaseEngine { &self, config: &nuncio_core::AccountConfig, ) -> Result<(), DatabaseError> { - let protocol_str = serde_json::to_string(&config.protocol).unwrap_or_default(); + use nuncio_core::{TlsMode, Transport}; + + let protocol_str = serde_json::to_string(&config.protocol()).unwrap_or_default(); + // Flatten the single active transport onto the physical columns. The + // `protocol` column doubles as the transport-kind discriminator that + // `list_accounts` reads back to reconstruct the exact variant. Columns + // that a given transport does not use are written empty/zero (and a + // default TLS mode for the NOT NULL `*_tls_mode` columns); they are + // ignored on load for that transport. + let ( + server_host, + server_port, + imap_tls_mode, + smtp_host, + smtp_port, + smtp_tls_mode, + collection_url, + ) = match &config.transport { + Transport::ImapSmtp(t) => ( + t.imap_host.clone(), + t.imap_port, + t.imap_tls_mode, + t.smtp_host.clone(), + t.smtp_port, + t.smtp_tls_mode, + String::new(), + ), + Transport::Jmap(t) => ( + t.endpoint_host.clone(), + 0u16, + TlsMode::ImplicitTls, + String::new(), + 0u16, + TlsMode::ImplicitTls, + String::new(), + ), + Transport::Dav(t) => ( + String::new(), + 0u16, + TlsMode::ImplicitTls, + String::new(), + 0u16, + TlsMode::ImplicitTls, + t.collection_url.clone(), + ), + }; sqlx::query( r#" INSERT OR REPLACE INTO accounts @@ -966,8 +1011,8 @@ impl DatabaseEngine { .bind(&config.name) .bind(&config.email_address) .bind(protocol_str) - .bind(&config.server_host) - .bind(config.server_port as i64) + .bind(&server_host) + .bind(i64::from(server_port)) // `use_tls` is a retained-but-unread on-disk column (see the // `CREATE TABLE`'s comment) -- written as a fixed placeholder since // no `AccountConfig` field backs it anymore, satisfying its @@ -977,11 +1022,11 @@ impl DatabaseEngine { .bind(1i64) .bind(&config.keyring_secret_key) .bind(config.sync_interval_secs as i64) - .bind(&config.smtp_host) - .bind(config.smtp_port as i64) - .bind(tls_mode_to_db(config.imap_tls_mode)) - .bind(tls_mode_to_db(config.smtp_tls_mode)) - .bind(&config.collection_url) + .bind(&smtp_host) + .bind(i64::from(smtp_port)) + .bind(tls_mode_to_db(imap_tls_mode)) + .bind(tls_mode_to_db(smtp_tls_mode)) + .bind(&collection_url) .execute(&self.pool) .await .map_err(DatabaseError::Query)?; @@ -1057,31 +1102,50 @@ impl DatabaseEngine { )| { let protocol = serde_json::from_str(&protocol_str) .unwrap_or(nuncio_core::AccountProtocol::ImapSmtp); - // Backfill-safe fallback: a row - // written before `smtp_host`/`smtp_port` existed has - // `NULL` in both columns (see - // `Self::ensure_accounts_smtp_columns`). Rather than - // surface an incomplete/invalid config, fall back to the - // same host/port already used for IMAP/JMAP -- the best - // available default for an account configured before - // outbound mail had its own endpoint. - let resolved_smtp_host = smtp_host.unwrap_or_else(|| server_host.clone()); - let resolved_smtp_port = - smtp_port.map(|p| p as u16).unwrap_or(server_port as u16); + // Reconstruct the exact transport variant from the + // `protocol` discriminator column, reading only the columns + // that variant uses (see `save_account`). + let transport = match protocol { + nuncio_core::AccountProtocol::ImapSmtp => { + // Backfill-safe fallback: a row written before + // `smtp_host`/`smtp_port` existed has `NULL` in both + // columns (see `Self::ensure_accounts_smtp_columns`). + // Rather than surface an incomplete/invalid config, + // fall back to the same host/port already used for + // IMAP -- the best available default for an account + // configured before outbound mail had its own + // endpoint. + let resolved_smtp_host = + smtp_host.unwrap_or_else(|| server_host.clone()); + let resolved_smtp_port = + smtp_port.map(|p| p as u16).unwrap_or(server_port as u16); + nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: server_host, + imap_port: server_port as u16, + imap_tls_mode: tls_mode_from_db(&imap_tls_mode), + smtp_host: resolved_smtp_host, + smtp_port: resolved_smtp_port, + smtp_tls_mode: tls_mode_from_db(&smtp_tls_mode), + }) + } + nuncio_core::AccountProtocol::Jmap => { + nuncio_core::Transport::Jmap(nuncio_core::JmapTransport { + endpoint_host: server_host, + }) + } + nuncio_core::AccountProtocol::CalDav => { + nuncio_core::Transport::Dav(nuncio_core::DavTransport { + collection_url: collection_url.unwrap_or_default(), + }) + } + }; nuncio_core::AccountConfig { id, name, email_address, - protocol, - server_host, - server_port: server_port as u16, - smtp_host: resolved_smtp_host, - smtp_port: resolved_smtp_port, - imap_tls_mode: tls_mode_from_db(&imap_tls_mode), - smtp_tls_mode: tls_mode_from_db(&smtp_tls_mode), keyring_secret_key, sync_interval_secs: sync_interval_secs as u64, - collection_url: collection_url.unwrap_or_default(), + transport, } }, ) @@ -2767,16 +2831,16 @@ mod tests { id: "acct-contended-1".to_string(), name: "Contended Account".to_string(), email_address: "contended@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-contended-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; engine.save_account(&acct).await.unwrap(); engine.close().await; @@ -3346,16 +3410,16 @@ mod tests { id: "acct-test-1".to_string(), name: "Work Account".to_string(), email_address: "work@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-test-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; engine @@ -3369,8 +3433,9 @@ mod tests { assert_eq!(accounts.len(), 1); assert_eq!(accounts[0].id, "acct-test-1"); assert_eq!(accounts[0].email_address, "work@nuncio.mx"); - assert_eq!(accounts[0].smtp_host, "smtp.nuncio.mx"); - assert_eq!(accounts[0].smtp_port, 465); + let t = accounts[0].imap_smtp().expect("imap-smtp transport"); + assert_eq!(t.smtp_host, "smtp.nuncio.mx"); + assert_eq!(t.smtp_port, 465); } /// Regression proof that the per-protocol TLS transport modes are @@ -3389,32 +3454,34 @@ mod tests { id: "acct-tls-1".to_string(), name: "Mixed TLS Account".to_string(), email_address: "mixed@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 143, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 25, - imap_tls_mode: nuncio_core::TlsMode::StartTls, - smtp_tls_mode: nuncio_core::TlsMode::Plain, keyring_secret_key: "nuncio/acct-tls-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 143, + imap_tls_mode: nuncio_core::TlsMode::StartTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 25, + smtp_tls_mode: nuncio_core::TlsMode::Plain, + }), }; engine.save_account(&acct).await.expect("save succeeds"); let listed = engine.list_accounts().await.expect("list succeeds"); assert_eq!(listed.len(), 1); - assert_eq!(listed[0].imap_tls_mode, nuncio_core::TlsMode::StartTls); - assert_eq!(listed[0].smtp_tls_mode, nuncio_core::TlsMode::Plain); + let lt = listed[0].imap_smtp().expect("imap-smtp transport"); + assert_eq!(lt.imap_tls_mode, nuncio_core::TlsMode::StartTls); + assert_eq!(lt.smtp_tls_mode, nuncio_core::TlsMode::Plain); let fetched = engine .get_account("acct-tls-1") .await .expect("get succeeds") .expect("account present"); - assert_eq!(fetched.imap_tls_mode, nuncio_core::TlsMode::StartTls); - assert_eq!(fetched.smtp_tls_mode, nuncio_core::TlsMode::Plain); + let ft = fetched.imap_smtp().expect("imap-smtp transport"); + assert_eq!(ft.imap_tls_mode, nuncio_core::TlsMode::StartTls); + assert_eq!(ft.smtp_tls_mode, nuncio_core::TlsMode::Plain); assert_eq!(fetched, acct); } @@ -3429,16 +3496,11 @@ mod tests { id: "acct-caldav-store-1".to_string(), name: "Work Calendar".to_string(), email_address: "cal@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::CalDav, - server_host: String::new(), - server_port: 0, - smtp_host: String::new(), - smtp_port: 0, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-caldav-store-1".to_string(), sync_interval_secs: 300, - collection_url: "https://dav.example.com/calendars/user/work/".to_string(), + transport: nuncio_core::Transport::Dav(nuncio_core::DavTransport { + collection_url: "https://dav.example.com/calendars/user/work/".to_string(), + }), }; engine.save_account(&caldav).await.expect("save succeeds"); @@ -3448,13 +3510,77 @@ mod tests { .expect("get succeeds") .expect("account present"); assert_eq!(fetched, caldav); - assert_eq!(fetched.protocol, nuncio_core::AccountProtocol::CalDav); + assert_eq!(fetched.protocol(), nuncio_core::AccountProtocol::CalDav); assert_eq!( - fetched.collection_url, - "https://dav.example.com/calendars/user/work/" + fetched.dav_collection_url(), + Some("https://dav.example.com/calendars/user/work/") ); } + /// Regression proof that accounts of DIFFERENT transports coexist on disk + /// and each loads back as its exact original transport variant -- the flat + /// physical columns must not blur one transport into another on reload. + #[tokio::test] + async fn distinct_transport_accounts_round_trip_to_their_own_variant() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let imap = nuncio_core::AccountConfig { + id: "acct-mix-imap".to_string(), + name: "IMAP Account".to_string(), + email_address: "imap@nuncio.mx".to_string(), + keyring_secret_key: "nuncio/acct-mix-imap".to_string(), + sync_interval_secs: 60, + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 143, + imap_tls_mode: nuncio_core::TlsMode::StartTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 587, + smtp_tls_mode: nuncio_core::TlsMode::StartTls, + }), + }; + let jmap = nuncio_core::AccountConfig { + id: "acct-mix-jmap".to_string(), + name: "JMAP Account".to_string(), + email_address: "jmap@nuncio.mx".to_string(), + keyring_secret_key: "nuncio/acct-mix-jmap".to_string(), + sync_interval_secs: 120, + transport: nuncio_core::Transport::Jmap(nuncio_core::JmapTransport { + endpoint_host: "jmap.nuncio.mx".to_string(), + }), + }; + let dav = nuncio_core::AccountConfig { + id: "acct-mix-dav".to_string(), + name: "DAV Account".to_string(), + email_address: "dav@nuncio.mx".to_string(), + keyring_secret_key: "nuncio/acct-mix-dav".to_string(), + sync_interval_secs: 300, + transport: nuncio_core::Transport::Dav(nuncio_core::DavTransport { + collection_url: "https://dav.example.com/cal/".to_string(), + }), + }; + + for acct in [&imap, &jmap, &dav] { + engine.save_account(acct).await.expect("save succeeds"); + } + + let reload = |id: &str| { + let engine = &engine; + let id = id.to_string(); + async move { + engine + .get_account(&id) + .await + .expect("get succeeds") + .expect("account present") + } + }; + + assert_eq!(reload("acct-mix-imap").await, imap); + assert_eq!(reload("acct-mix-jmap").await, jmap); + assert_eq!(reload("acct-mix-dav").await, dav); + } + /// Proves `get_account` resolves a saved account by id and returns `None` /// for an unknown id, and that `delete_account` genuinely removes the row /// (and is idempotent for an already-absent id). @@ -3466,16 +3592,16 @@ mod tests { id: "acct-del-1".to_string(), name: "Deletable".to_string(), email_address: "del@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-del-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; engine.save_account(&acct).await.expect("save succeeds"); @@ -3640,8 +3766,9 @@ mod tests { assert_eq!(acct.id, "acct-pre-migration"); // Falls back to the IMAP/JMAP endpoint since smtp_host/smtp_port // were NULL for this pre-existing row. - assert_eq!(acct.smtp_host, "imap.nuncio.mx"); - assert_eq!(acct.smtp_port, 993); + let acct_t = acct.imap_smtp().expect("imap-smtp transport"); + assert_eq!(acct_t.smtp_host, "imap.nuncio.mx"); + assert_eq!(acct_t.smtp_port, 993); // Step 3: re-running migrate() (e.g. a second daemon startup against // the same file) must be a no-op, not an error. @@ -3657,16 +3784,16 @@ mod tests { id: "acct-post-migration".to_string(), name: "Post Migration Account".to_string(), email_address: "post@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 587, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::StartTls, keyring_secret_key: "nuncio/acct-post-migration".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 587, + smtp_tls_mode: nuncio_core::TlsMode::StartTls, + }), }; engine .save_account(&new_acct) @@ -3677,8 +3804,9 @@ mod tests { .iter() .find(|a| a.id == "acct-post-migration") .expect("newly saved account present"); - assert_eq!(post.smtp_host, "smtp.nuncio.mx"); - assert_eq!(post.smtp_port, 587); + let post_t = post.imap_smtp().expect("imap-smtp transport"); + assert_eq!(post_t.smtp_host, "smtp.nuncio.mx"); + assert_eq!(post_t.smtp_port, 587); engine.close().await; } diff --git a/crates/nuncio-store/src/recovery.rs b/crates/nuncio-store/src/recovery.rs index 30a1fa0..177793b 100644 --- a/crates/nuncio-store/src/recovery.rs +++ b/crates/nuncio-store/src/recovery.rs @@ -422,20 +422,38 @@ impl SqliteRecoveryEngine { Self::parse_salvaged_tls_mode(&id, "imap_tls_mode", &imap_tls_mode_raw); let smtp_tls_mode = Self::parse_salvaged_tls_mode(&id, "smtp_tls_mode", &smtp_tls_mode_raw); + // Reconstruct the transport variant from the salvaged + // `protocol` discriminator, mirroring + // `DatabaseEngine::list_accounts`. + let transport = match protocol { + nuncio_core::AccountProtocol::ImapSmtp => { + nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: server_host, + imap_port: server_port as u16, + imap_tls_mode, + smtp_host: resolved_smtp_host, + smtp_port: resolved_smtp_port, + smtp_tls_mode, + }) + } + nuncio_core::AccountProtocol::Jmap => { + nuncio_core::Transport::Jmap(nuncio_core::JmapTransport { + endpoint_host: server_host, + }) + } + nuncio_core::AccountProtocol::CalDav => { + nuncio_core::Transport::Dav(nuncio_core::DavTransport { + collection_url: collection_url.unwrap_or_default(), + }) + } + }; nuncio_core::AccountConfig { id, name, email_address, - protocol, - server_host, - server_port: server_port as u16, - smtp_host: resolved_smtp_host, - smtp_port: resolved_smtp_port, - imap_tls_mode, - smtp_tls_mode, keyring_secret_key, sync_interval_secs: sync_interval_secs as u64, - collection_url: collection_url.unwrap_or_default(), + transport, } }, ) @@ -598,16 +616,16 @@ mod tests { id: "acct-test-1".to_string(), name: "Work Account".to_string(), email_address: "work@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-test-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; engine.save_account(&acct).await.unwrap(); assert!(engine.check_integrity().await.unwrap()); @@ -659,16 +677,16 @@ mod tests { id: "acct-salvage-1".to_string(), name: "Salvage Account".to_string(), email_address: "salvage@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-salvage-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; engine.save_account(&acct).await.unwrap(); @@ -796,16 +814,16 @@ mod tests { id: "acct-starttls-1".to_string(), name: "STARTTLS Account".to_string(), email_address: "starttls@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 143, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 587, - imap_tls_mode: nuncio_core::TlsMode::StartTls, - smtp_tls_mode: nuncio_core::TlsMode::Plain, keyring_secret_key: "nuncio/acct-starttls-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 143, + imap_tls_mode: nuncio_core::TlsMode::StartTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 587, + smtp_tls_mode: nuncio_core::TlsMode::Plain, + }), }; engine.save_account(&acct).await.unwrap(); } @@ -824,8 +842,9 @@ mod tests { .unwrap(); let accounts = fresh.list_accounts().await.unwrap(); assert_eq!(accounts.len(), 1); - assert_eq!(accounts[0].imap_tls_mode, nuncio_core::TlsMode::StartTls); - assert_eq!(accounts[0].smtp_tls_mode, nuncio_core::TlsMode::Plain); + let salvaged_t = accounts[0].imap_smtp().expect("imap-smtp transport"); + assert_eq!(salvaged_t.imap_tls_mode, nuncio_core::TlsMode::StartTls); + assert_eq!(salvaged_t.smtp_tls_mode, nuncio_core::TlsMode::Plain); } // NOTE: there is deliberately no test combining *header* corruption (bytes 0-16 -- the diff --git a/crates/nunciod/src/calendar_sync.rs b/crates/nunciod/src/calendar_sync.rs index 0eb7f3d..3b9c52f 100644 --- a/crates/nunciod/src/calendar_sync.rs +++ b/crates/nunciod/src/calendar_sync.rs @@ -74,7 +74,7 @@ pub async fn sync_caldav_account( ) -> Result { let client = CalDavClient::new(CalDavAccountConfig { account_id: account.id.clone(), - caldav_url: account.collection_url.clone(), + caldav_url: account.dav_collection_url().unwrap_or_default().to_string(), username: account.email_address.clone(), auth_token: password.to_string(), }); diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 913e3bc..8fdf6cd 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -14,6 +14,7 @@ use nuncio_core::{CoreCommand, CoreEvent, EventBus}; use nuncio_filter::{FilterEngine, NsqlParser, NsqlValidator, ValidationOptions}; use nuncio_mail::{ImapEngine, MailBackend, MessageSender, SmtpTransportEngine}; use nuncio_proto::errors; +use nuncio_proto::v1::account_config::Transport as TransportProto; use nuncio_proto::v1::accounts_server::{Accounts, AccountsServer}; use nuncio_proto::v1::audit_server::{Audit, AuditServer}; use nuncio_proto::v1::calendar_server::{Calendar, CalendarServer}; @@ -25,25 +26,25 @@ use nuncio_proto::v1::mail_server::{Mail, MailServer}; use nuncio_proto::v1::system_server::{System, SystemServer}; use nuncio_proto::v1::ErrorReason; use nuncio_proto::v1::{ - export_request, AccountConfig as AccountConfigProto, AccountProtocol as AccountProtocolProto, - AddAccountRequest, AddAccountResponse, Attachment as AttachmentProto, - AuditRecord as AuditRecordProto, BatchFilterProgress, CalendarEvent as CalendarEventProto, - CalendarSyncRequest, CalendarSyncResponse, Contact as ContactProto, - ContactEmail as ContactEmailProto, ContactPhone as ContactPhoneProto, ContactsSyncRequest, - ContactsSyncResponse, CreateContactRequest, CreateContactResponse, CreateRuleRequest, - CreateRuleResponse, DatabaseRecovered, DeleteRuleRequest, DeleteRuleResponse, Event, - EventError, ExportFormat as ExportFormatProto, ExportRequest, ExportResponse, - ExportRulesRequest, ExportRulesResponse, FilterExecuted, + export_request, AccountConfig as AccountConfigProto, AddAccountRequest, AddAccountResponse, + Attachment as AttachmentProto, AuditRecord as AuditRecordProto, BatchFilterProgress, + CalendarEvent as CalendarEventProto, CalendarSyncRequest, CalendarSyncResponse, + Contact as ContactProto, ContactEmail as ContactEmailProto, ContactPhone as ContactPhoneProto, + ContactsSyncRequest, ContactsSyncResponse, CreateContactRequest, CreateContactResponse, + CreateRuleRequest, CreateRuleResponse, DatabaseRecovered, DavTransport as DavTransportProto, + DeleteRuleRequest, DeleteRuleResponse, Event, EventError, ExportFormat as ExportFormatProto, + ExportRequest, ExportResponse, ExportRulesRequest, ExportRulesResponse, FilterExecuted, FilterExecutionLog as FilterExecutionLogProto, FilterRule as FilterRuleProto, Folder as FolderProto, GetContactRequest, GetContactResponse, GetEventRequest, GetEventResponse, GetExecutionLogsRequest, GetExecutionLogsResponse, GetMessageRequest, - GetMessageResponse, GetStatusRequest, GetStatusResponse, ImportRulesRequest, - ImportRulesResponse, ListAccountsRequest, ListAccountsResponse, ListContactsRequest, - ListContactsResponse, ListEventsRequest, ListEventsResponse, ListFoldersRequest, - ListFoldersResponse, ListMessagesRequest, ListMessagesResponse, ListRecordsRequest, - ListRecordsResponse, ListRulesRequest, ListRulesResponse, MarkReadRequest, MarkReadResponse, - Message as MessageProto, MessageFlagsChanged, MessageSearchHit, PreviewRuleRequest, - PreviewRuleResponse, RemoveAccountRequest, RemoveAccountResponse, + GetMessageResponse, GetStatusRequest, GetStatusResponse, + ImapSmtpTransport as ImapSmtpTransportProto, ImportRulesRequest, ImportRulesResponse, + JmapTransport as JmapTransportProto, ListAccountsRequest, ListAccountsResponse, + ListContactsRequest, ListContactsResponse, ListEventsRequest, ListEventsResponse, + ListFoldersRequest, ListFoldersResponse, ListMessagesRequest, ListMessagesResponse, + ListRecordsRequest, ListRecordsResponse, ListRulesRequest, ListRulesResponse, MarkReadRequest, + MarkReadResponse, Message as MessageProto, MessageFlagsChanged, MessageSearchHit, + PreviewRuleRequest, PreviewRuleResponse, RemoveAccountRequest, RemoveAccountResponse, RuleExportFormat as RuleExportFormatProto, SearchMessagesRequest, SearchMessagesResponse, SendMessageRequest, SendMessageResponse, ShuttingDown, SubscribeRequest, SyncCompleted, SyncRequest, SyncResponse, SyncStarted, TestAccountConnectionRequest, @@ -275,33 +276,6 @@ impl System for SystemGrpcService { } } -/// Maps a `nuncio_core::AccountProtocol` onto its wire-format -/// `nuncio.v1.AccountProtocol` enum value. -fn map_account_protocol_to_proto(protocol: nuncio_core::AccountProtocol) -> AccountProtocolProto { - match protocol { - nuncio_core::AccountProtocol::Jmap => AccountProtocolProto::Jmap, - nuncio_core::AccountProtocol::ImapSmtp => AccountProtocolProto::ImapSmtp, - nuncio_core::AccountProtocol::CalDav => AccountProtocolProto::Caldav, - } -} - -/// Maps a wire-format `nuncio.v1.AccountProtocol` enum value back onto -/// `nuncio_core::AccountProtocol`. `Unspecified` is treated as an invalid -/// request rather than silently defaulting to a protocol the caller never -/// asked for. -fn map_account_protocol_from_proto( - protocol: AccountProtocolProto, -) -> Result { - match protocol { - AccountProtocolProto::Jmap => Ok(nuncio_core::AccountProtocol::Jmap), - AccountProtocolProto::ImapSmtp => Ok(nuncio_core::AccountProtocol::ImapSmtp), - AccountProtocolProto::Caldav => Ok(nuncio_core::AccountProtocol::CalDav), - AccountProtocolProto::Unspecified => { - Err(Status::invalid_argument("account protocol is required")) - } - } -} - /// Maps a `nuncio_core::TlsMode` onto its wire-format `nuncio.v1.TlsMode` /// enum value. fn map_tls_mode_to_proto(mode: nuncio_core::TlsMode) -> TlsModeProto { @@ -330,56 +304,93 @@ fn map_tls_mode_from_proto(mode: TlsModeProto) -> Result AccountConfigProto { + let transport = match config.transport { + nuncio_core::Transport::ImapSmtp(t) => TransportProto::ImapSmtp(ImapSmtpTransportProto { + imap_host: t.imap_host, + imap_port: u32::from(t.imap_port), + imap_tls_mode: map_tls_mode_to_proto(t.imap_tls_mode).into(), + smtp_host: t.smtp_host, + smtp_port: u32::from(t.smtp_port), + smtp_tls_mode: map_tls_mode_to_proto(t.smtp_tls_mode).into(), + }), + nuncio_core::Transport::Jmap(t) => TransportProto::Jmap(JmapTransportProto { + endpoint_host: t.endpoint_host, + }), + nuncio_core::Transport::Dav(t) => TransportProto::Dav(DavTransportProto { + collection_url: t.collection_url, + }), + }; AccountConfigProto { id: config.id, name: config.name, email_address: config.email_address, - protocol: map_account_protocol_to_proto(config.protocol).into(), - server_host: config.server_host, - server_port: u32::from(config.server_port), - imap_tls_mode: map_tls_mode_to_proto(config.imap_tls_mode).into(), - smtp_tls_mode: map_tls_mode_to_proto(config.smtp_tls_mode).into(), keyring_secret_key: config.keyring_secret_key, sync_interval: Some(nuncio_proto::time::duration_from_secs( config.sync_interval_secs, )), - smtp_host: config.smtp_host, - smtp_port: u32::from(config.smtp_port), - collection_url: config.collection_url, + transport: Some(transport), } } /// Maps a wire-format `nuncio.v1.AccountConfig` request payload back onto -/// `nuncio_core::AccountConfig`. Rejects a `server_port` outside `1..=65535` -/// with `Status::invalid_argument` rather than silently truncating it. +/// `nuncio_core::AccountConfig`. An unset `transport` oneof is a typed +/// `VALIDATION_FAILED` error rather than a panic or a silent default; a port +/// outside `1..=65535` is likewise rejected rather than silently truncated. fn map_account_config_from_proto( config: AccountConfigProto, ) -> Result { - let protocol = map_account_protocol_from_proto(config.protocol())?; - let imap_tls_mode = map_tls_mode_from_proto(config.imap_tls_mode())?; - let smtp_tls_mode = map_tls_mode_from_proto(config.smtp_tls_mode())?; - let server_port = u16::try_from(config.server_port) - .map_err(|_| Status::invalid_argument("server_port must be in range 1..=65535"))?; - let smtp_port = u16::try_from(config.smtp_port) - .map_err(|_| Status::invalid_argument("smtp_port must be in range 1..=65535"))?; - let sync_interval = config - .sync_interval - .ok_or_else(|| Status::invalid_argument("sync_interval is required"))?; + let sync_interval = config.sync_interval.ok_or_else(|| { + errors::status(ErrorReason::ValidationFailed, "sync_interval is required") + })?; + + let transport = match config.transport { + Some(TransportProto::ImapSmtp(t)) => { + let imap_port = u16::try_from(t.imap_port).map_err(|_| { + errors::status( + ErrorReason::ValidationFailed, + "imap_port must be in range 1..=65535", + ) + })?; + let smtp_port = u16::try_from(t.smtp_port).map_err(|_| { + errors::status( + ErrorReason::ValidationFailed, + "smtp_port must be in range 1..=65535", + ) + })?; + // Resolve the TLS-mode enums (which borrow `t`) before moving the + // owned host strings out of `t`. + let imap_tls_mode = map_tls_mode_from_proto(t.imap_tls_mode())?; + let smtp_tls_mode = map_tls_mode_from_proto(t.smtp_tls_mode())?; + nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: t.imap_host, + imap_port, + imap_tls_mode, + smtp_host: t.smtp_host, + smtp_port, + smtp_tls_mode, + }) + } + Some(TransportProto::Jmap(t)) => nuncio_core::Transport::Jmap(nuncio_core::JmapTransport { + endpoint_host: t.endpoint_host, + }), + Some(TransportProto::Dav(t)) => nuncio_core::Transport::Dav(nuncio_core::DavTransport { + collection_url: t.collection_url, + }), + None => { + return Err(errors::status( + ErrorReason::ValidationFailed, + "account transport is required (one of imap_smtp, jmap, dav must be set)", + )) + } + }; Ok(nuncio_core::AccountConfig { id: config.id, name: config.name, email_address: config.email_address, - protocol, - server_host: config.server_host, - server_port, - smtp_host: config.smtp_host, - smtp_port, - imap_tls_mode, - smtp_tls_mode, keyring_secret_key: config.keyring_secret_key, sync_interval_secs: nuncio_proto::time::duration_to_secs(&sync_interval), - collection_url: config.collection_url, + transport, }) } @@ -452,44 +463,50 @@ impl AccountConnectionTester for RealAccountConnectionTester { config: &nuncio_core::AccountConfig, password: &str, ) -> AccountConnectionReport { - let imap = match config.protocol { - nuncio_core::AccountProtocol::ImapSmtp => { - let engine = ImapEngine::with_credentials( - &config.id, - &config.server_host, - config.server_port, - config.imap_tls_mode, - &config.email_address, - password, - ); - match engine.probe_connection().await { - Ok(()) => ProtocolProbe::ok(), - Err(e) => ProtocolProbe::failed(e.to_string()), + // Only an IMAP/SMTP account has mail endpoints to probe. A JMAP or DAV + // account has no IMAP/SMTP legs, so report that honestly rather than + // dialing fields that do not exist for it. + let transport = match &config.transport { + nuncio_core::Transport::ImapSmtp(t) => t, + nuncio_core::Transport::Jmap(_) => { + return AccountConnectionReport { + imap: ProtocolProbe::failed( + "IMAP connection probe is not applicable to a JMAP account".to_string(), + ), + smtp: ProtocolProbe::failed( + "SMTP connection probe is not applicable to a JMAP account".to_string(), + ), + } + } + nuncio_core::Transport::Dav(_) => { + return AccountConnectionReport { + imap: ProtocolProbe::failed( + "IMAP connection probe is not applicable to a CalDAV account".to_string(), + ), + smtp: ProtocolProbe::failed( + "SMTP connection probe is not applicable to a CalDAV account".to_string(), + ), } } - nuncio_core::AccountProtocol::Jmap => ProtocolProbe::failed( - "IMAP connection probe is not applicable to a JMAP account".to_string(), - ), - nuncio_core::AccountProtocol::CalDav => ProtocolProbe::failed( - "IMAP connection probe is not applicable to a CalDAV account".to_string(), - ), }; - // A CalDAV account has no SMTP endpoint to probe; report that honestly - // rather than dialing the (empty) mail endpoint fields. - if config.protocol == nuncio_core::AccountProtocol::CalDav { - return AccountConnectionReport { - imap, - smtp: ProtocolProbe::failed( - "SMTP connection probe is not applicable to a CalDAV account".to_string(), - ), - }; - } + let engine = ImapEngine::with_credentials( + &config.id, + &transport.imap_host, + transport.imap_port, + transport.imap_tls_mode, + &config.email_address, + password, + ); + let imap = match engine.probe_connection().await { + Ok(()) => ProtocolProbe::ok(), + Err(e) => ProtocolProbe::failed(e.to_string()), + }; let smtp = match SmtpTransportEngine::new( - &config.smtp_host, - config.smtp_port, - config.smtp_tls_mode, + &transport.smtp_host, + transport.smtp_port, + transport.smtp_tls_mode, &config.email_address, password, ) { @@ -1037,7 +1054,7 @@ impl CalendarGrpcService { .await .map_err(|e| Status::internal(format!("failed to list accounts: {e}")))? .into_iter() - .filter(|a| a.protocol == nuncio_core::AccountProtocol::CalDav) + .filter(|a| a.is_dav()) .filter(|a| req.account_id.is_empty() || a.id == req.account_id) .collect(); @@ -3493,19 +3510,38 @@ mod tests { id: id.to_string(), name: "Restart Test Account".to_string(), email_address: format!("{id}@nuncio.mx"), - protocol: AccountProtocolProto::ImapSmtp.into(), - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - imap_tls_mode: TlsModeProto::ImplicitTls.into(), - smtp_tls_mode: TlsModeProto::ImplicitTls.into(), keyring_secret_key: keyring_secret_key.to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), - collection_url: String::new(), - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, + transport: Some(TransportProto::ImapSmtp(ImapSmtpTransportProto { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: TlsModeProto::ImplicitTls.into(), + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: TlsModeProto::ImplicitTls.into(), + })), } } + #[test] + fn account_config_from_proto_rejects_unset_transport_as_validation_failed() { + let config = AccountConfigProto { + id: "acct-no-transport".to_string(), + name: "No Transport".to_string(), + email_address: "no-transport@nuncio.mx".to_string(), + keyring_secret_key: "nuncio/acct-no-transport".to_string(), + sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), + transport: None, + }; + let err = map_account_config_from_proto(config) + .expect_err("an account with no transport must be rejected"); + assert_eq!(err.code(), tonic::Code::InvalidArgument); + assert_eq!( + nuncio_proto::errors::error_reason(&err), + Some(ErrorReason::ValidationFailed), + ); + } + #[tokio::test] async fn add_account_rejects_missing_bearer_token() { // Confirms `Accounts` is mounted behind its own `BearerAuthInterceptor` @@ -3885,16 +3921,16 @@ mod tests { id: "acct-rotate-1".to_string(), name: "Rotate Account".to_string(), email_address: "rotate@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: key.to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; // Force `save_account` to fail for real by closing the pool, exactly @@ -3928,16 +3964,16 @@ mod tests { id: "acct-rotate-2".to_string(), name: "Rotate Account 2".to_string(), email_address: "rotate2@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: key.to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; db.close().await; @@ -4542,16 +4578,16 @@ mod tests { id: "acct-send-1".to_string(), name: "Send Test Account".to_string(), email_address: "sender@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "127.0.0.1".to_string(), - smtp_port: 1, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-send-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "127.0.0.1".to_string(), + smtp_port: 1, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; db.save_account(&config).await.expect("save account"); secrets @@ -4710,16 +4746,16 @@ mod tests { id: "acct-injected-send-1".to_string(), name: "Injected Send Test Account".to_string(), email_address: "sender@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-injected-send-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; // Deliberately never stores a keyring credential -- this path must // never need one. @@ -4826,16 +4862,16 @@ mod tests { id: "acct-injected-send-fail-1".to_string(), name: "Injected Send Failure Test Account".to_string(), email_address: "sender@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-injected-send-fail-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; db.save_account(&config).await.expect("save account"); diff --git a/crates/nunciod/src/orchestrator.rs b/crates/nunciod/src/orchestrator.rs index 5147754..24af9aa 100644 --- a/crates/nunciod/src/orchestrator.rs +++ b/crates/nunciod/src/orchestrator.rs @@ -152,16 +152,16 @@ mod tests { id: "acct-orch-1".to_string(), name: "Orch Account".to_string(), email_address: "orch@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: "nuncio/acct-orch-1".to_string(), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), }; engine.save_account(&acct).await.unwrap(); engine.close().await; diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 0d5bb8b..813a659 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -470,10 +470,13 @@ impl RemoteExecutionEnv for ProductionExecutionEnv { ) -> Result, OutboxExecuteError> { let config = self.account(account_id).await?; let password = self.secrets.get_secret(&config.keyring_secret_key)?; + let transport = config + .imap_smtp() + .ok_or_else(|| OutboxExecuteError::NotAMailAccount(account_id.to_string()))?; let engine = SmtpTransportEngine::new( - &config.smtp_host, - config.smtp_port, - config.smtp_tls_mode, + &transport.smtp_host, + transport.smtp_port, + transport.smtp_tls_mode, &config.email_address, &password, )?; diff --git a/crates/nunciod/src/send.rs b/crates/nunciod/src/send.rs index ee800db..eebcba4 100644 --- a/crates/nunciod/src/send.rs +++ b/crates/nunciod/src/send.rs @@ -68,6 +68,11 @@ pub enum SendError { /// rather than silently sending from a different account. #[error("no account configured with id {0:?}")] AccountNotFound(String), + /// The resolved account does not have an SMTP outbound endpoint (only an + /// IMAP/SMTP account can send outbound mail over SMTP; a JMAP or DAV + /// account cannot). + #[error("account {0:?} has no SMTP transport and cannot send outbound mail")] + NotAnSmtpAccount(String), /// Failed to read the account's credential from the secret vault. #[error("failed to read account credential from the secret vault: {0}")] Vault(#[from] VaultError), @@ -120,10 +125,13 @@ fn build_smtp_sender( config: &AccountConfig, password: &str, ) -> Result { + let transport = config + .imap_smtp() + .ok_or_else(|| SendError::NotAnSmtpAccount(config.id.clone()))?; let engine = SmtpTransportEngine::new( - &config.smtp_host, - config.smtp_port, - config.smtp_tls_mode, + &transport.smtp_host, + transport.smtp_port, + transport.smtp_tls_mode, &config.email_address, password, )?; @@ -233,16 +241,16 @@ mod tests { id: id.to_string(), name: "Send Test Account".to_string(), email_address: format!("{id}@nuncio.mx"), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "127.0.0.1".to_string(), - smtp_port: 1, - imap_tls_mode: TlsMode::ImplicitTls, - smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: TlsMode::ImplicitTls, + smtp_host: "127.0.0.1".to_string(), + smtp_port: 1, + smtp_tls_mode: TlsMode::ImplicitTls, + }), } } diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index df2cf81..0c5036a 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -4,8 +4,8 @@ //! / `CoreCommand::SyncAccount` only moved `EngineStatus` to `Syncing` and //! published `SyncStarted`, and nothing ever fetched a real message from a //! real server into the store. This module closes that gap: for a configured -//! account it connects a real [`MailBackend`] (a real [`ImapEngine`] for -//! [`AccountProtocol::ImapSmtp`], built from [`AccountConfig`] plus the +//! account it connects a real [`MailBackend`] (a real [`ImapEngine`] for an +//! IMAP/SMTP transport, built from [`AccountConfig`] plus the //! account's keyring-stored password), fetches folders and messages, and //! persists every message through [`DatabaseEngine::save_email`]. //! @@ -20,7 +20,7 @@ //! all-accounts sync fans these per-account runs out through //! [`crate::sync_dispatcher::sync_all_configured`], so concurrency and the //! per-account timeout are bounded in one place rather than serialized here. -use nuncio_core::{AccountConfig, AccountProtocol, CoreCommand, CoreEvent, EventBus}; +use nuncio_core::{AccountConfig, CoreCommand, CoreEvent, EventBus, Transport}; use nuncio_filter::{FilterEngine, OutboxManager, RuleAction}; use nuncio_mail::{ImapEngine, JmapEngine, MailBackend, MailError}; use nuncio_store::db::DatabaseError; @@ -347,24 +347,24 @@ pub(crate) fn build_mail_backend( config: &AccountConfig, password: &str, ) -> Result, SyncError> { - match config.protocol { - AccountProtocol::ImapSmtp => Ok(Box::new(ImapEngine::with_credentials( + match &config.transport { + Transport::ImapSmtp(t) => Ok(Box::new(ImapEngine::with_credentials( &config.id, - &config.server_host, - config.server_port, - config.imap_tls_mode, + &t.imap_host, + t.imap_port, + t.imap_tls_mode, &config.email_address, password, ))), - AccountProtocol::Jmap => Ok(Box::new(JmapEngine::with_credentials( + Transport::Jmap(t) => Ok(Box::new(JmapEngine::with_credentials( &config.id, - &config.server_host, + &t.endpoint_host, &config.email_address, password, ))), // A DAV-protocol account (e.g. CalDAV) has no inbound mail backend; // it is synced through its own domain service, not the mail path. - AccountProtocol::CalDav => Err(SyncError::NotAMailAccount(config.id.clone())), + Transport::Dav(_) => Err(SyncError::NotAMailAccount(config.id.clone())), } } @@ -436,16 +436,11 @@ mod tests { id: id.to_string(), name: "JMAP Test Account".to_string(), email_address: format!("{id}@nuncio.mx"), - protocol: AccountProtocol::Jmap, - server_host: host.to_string(), - server_port: 443, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: TlsMode::ImplicitTls, - smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, - collection_url: String::new(), + transport: Transport::Jmap(nuncio_core::JmapTransport { + endpoint_host: host.to_string(), + }), } } @@ -527,16 +522,16 @@ mod tests { id: id.to_string(), name: "IMAP Test Account".to_string(), email_address: format!("{id}@nuncio.mx"), - protocol: AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: TlsMode::ImplicitTls, - smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: TlsMode::ImplicitTls, + }), } } diff --git a/crates/nunciod/src/sync_dispatcher.rs b/crates/nunciod/src/sync_dispatcher.rs index 29718f5..2278c64 100644 --- a/crates/nunciod/src/sync_dispatcher.rs +++ b/crates/nunciod/src/sync_dispatcher.rs @@ -362,7 +362,7 @@ pub async fn sync_all_configured( let account_ids = match db.list_accounts().await { Ok(accounts) => accounts .into_iter() - .filter(|a| !a.protocol.is_dav()) + .filter(|a| !a.is_dav()) .map(|a| a.id) .collect::>(), Err(e) => { @@ -381,7 +381,7 @@ pub async fn sync_all_configured( #[cfg(test)] mod tests { use super::*; - use nuncio_core::{AccountConfig, AccountProtocol, CoreEvent, TlsMode}; + use nuncio_core::{AccountConfig, CoreEvent, DavTransport, JmapTransport, Transport}; use std::collections::HashSet; use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::time::{sleep, Duration}; @@ -628,22 +628,19 @@ mod tests { id: id.to_string(), name: "Test".to_string(), email_address: format!("{id}@nuncio.mx"), - protocol: AccountProtocol::Jmap, - server_host: "jmap.nuncio.mx".to_string(), - server_port: 443, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: TlsMode::ImplicitTls, - smtp_tls_mode: TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, - collection_url: String::new(), + transport: Transport::Jmap(JmapTransport { + endpoint_host: "jmap.nuncio.mx".to_string(), + }), } } fn caldav_account(id: &str) -> AccountConfig { AccountConfig { - protocol: AccountProtocol::CalDav, + transport: Transport::Dav(DavTransport { + collection_url: "https://dav.nuncio.mx/cal/".to_string(), + }), ..jmap_account(id) } } diff --git a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs index 68c53d8..020bae1 100644 --- a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs +++ b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs @@ -29,7 +29,7 @@ use nuncio_core::EventBus; use nuncio_filter::FilterEngine; use nuncio_proto::v1::{ - AccountConfig, AccountProtocol, AddAccountRequest, ListAccountsRequest, RemoveAccountRequest, + AccountConfig, AddAccountRequest, ListAccountsRequest, RemoveAccountRequest, TestAccountConnectionRequest, TlsMode, UpdateAccountRequest, }; use nuncio_store::db::DatabaseEngine; @@ -42,6 +42,15 @@ use nunciod::grpc::{ use std::sync::Arc; use tokio::net::TcpListener; +/// Extract the IMAP/SMTP transport sub-message from a wire `AccountConfig`, +/// panicking if the account is not an IMAP/SMTP account (test-only helper). +fn imap_smtp_transport(config: &AccountConfig) -> &nuncio_proto::v1::ImapSmtpTransport { + match &config.transport { + Some(nuncio_proto::v1::account_config::Transport::ImapSmtp(t)) => t, + _ => panic!("expected an IMAP/SMTP account"), + } +} + const ACCOUNT_ID: &str = "acct-lifecycle-e2e-1"; const ACCOUNT_EMAIL: &str = "lifecycle-e2e@nuncio.mx"; const KEYRING_KEY: &str = "nuncio/acct-lifecycle-e2e-1"; @@ -79,16 +88,18 @@ fn sample_account_config() -> AccountConfig { id: ACCOUNT_ID.to_string(), name: "Lifecycle E2E Account".to_string(), email_address: ACCOUNT_EMAIL.to_string(), - protocol: AccountProtocol::ImapSmtp.into(), - server_host: "imap.nuncio.mx".to_string(), - server_port: 143, - imap_tls_mode: TlsMode::StartTls.into(), - smtp_tls_mode: TlsMode::Plain.into(), keyring_secret_key: KEYRING_KEY.to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), - smtp_host: "127.0.0.1".to_string(), - smtp_port: 25, - collection_url: String::new(), + transport: Some(nuncio_proto::v1::account_config::Transport::ImapSmtp( + nuncio_proto::v1::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 143, + imap_tls_mode: TlsMode::StartTls.into(), + smtp_host: "127.0.0.1".to_string(), + smtp_port: 25, + smtp_tls_mode: TlsMode::Plain.into(), + }, + )), } } @@ -179,13 +190,18 @@ async fn account_lifecycle_round_trips_over_authenticated_grpc_offline() { .into_inner() .accounts; assert_eq!(listed.len(), 1); - assert_eq!(listed[0].imap_tls_mode(), TlsMode::StartTls); - assert_eq!(listed[0].smtp_tls_mode(), TlsMode::Plain); + let listed_t = imap_smtp_transport(&listed[0]); + assert_eq!(listed_t.imap_tls_mode(), TlsMode::StartTls); + assert_eq!(listed_t.smtp_tls_mode(), TlsMode::Plain); // ---- (c) UpdateAccount genuinely mutates the persisted config ---- let mut updated = sample_account_config(); updated.name = "Renamed Lifecycle Account".to_string(); - updated.imap_tls_mode = TlsMode::ImplicitTls.into(); + if let Some(nuncio_proto::v1::account_config::Transport::ImapSmtp(t)) = + updated.transport.as_mut() + { + t.imap_tls_mode = TlsMode::ImplicitTls.into(); + } accounts_client .update_account(UpdateAccountRequest { config: Some(updated), @@ -202,9 +218,10 @@ async fn account_lifecycle_round_trips_over_authenticated_grpc_offline() { .accounts; assert_eq!(relisted.len(), 1); assert_eq!(relisted[0].name, "Renamed Lifecycle Account"); - assert_eq!(relisted[0].imap_tls_mode(), TlsMode::ImplicitTls); + let relisted_t = imap_smtp_transport(&relisted[0]); + assert_eq!(relisted_t.imap_tls_mode(), TlsMode::ImplicitTls); // Untouched fields survive the update. - assert_eq!(relisted[0].smtp_tls_mode(), TlsMode::Plain); + assert_eq!(relisted_t.smtp_tls_mode(), TlsMode::Plain); // ---- (d) TestAccountConnection reports the GENUINE per-protocol result ---- let report = accounts_client diff --git a/crates/nunciod/tests/caldav_production_e2e_test.rs b/crates/nunciod/tests/caldav_production_e2e_test.rs index 566c911..8576be5 100644 --- a/crates/nunciod/tests/caldav_production_e2e_test.rs +++ b/crates/nunciod/tests/caldav_production_e2e_test.rs @@ -20,8 +20,7 @@ use nuncio_core::EventBus; use nuncio_filter::FilterEngine; use nuncio_proto::v1::{ - AccountConfig as AccountConfigProto, AccountProtocol as AccountProtocolProto, - AddAccountRequest, CalendarSyncRequest, ListEventsRequest, TlsMode as TlsModeProto, + AccountConfig as AccountConfigProto, AddAccountRequest, CalendarSyncRequest, ListEventsRequest, }; use nuncio_store::db::DatabaseEngine; use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT}; @@ -158,16 +157,13 @@ async fn caldav_production_sync_builds_real_client_and_persists_events() { id: ACCOUNT_ID.to_string(), name: "Work Calendar".to_string(), email_address: ACCOUNT_EMAIL.to_string(), - protocol: AccountProtocolProto::Caldav.into(), - server_host: String::new(), - server_port: 0, - imap_tls_mode: TlsModeProto::ImplicitTls.into(), - smtp_tls_mode: TlsModeProto::ImplicitTls.into(), keyring_secret_key: KEYRING_KEY.to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(300)), - smtp_host: String::new(), - smtp_port: 0, - collection_url: collection_url.clone(), + transport: Some(nuncio_proto::v1::account_config::Transport::Dav( + nuncio_proto::v1::DavTransport { + collection_url: collection_url.clone(), + }, + )), }), password: CALDAV_PASSWORD.to_string(), }) diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index d4a4e4d..5fccdcd 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -99,16 +99,16 @@ fn sample_account() -> nuncio_core::AccountConfig { id: ACCOUNT_ID.to_string(), name: "Outbox E2E Account".to_string(), email_address: "owner@nuncio.mx".to_string(), - protocol: nuncio_core::AccountProtocol::ImapSmtp, - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, keyring_secret_key: format!("nuncio/{ACCOUNT_ID}"), sync_interval_secs: 60, - collection_url: String::new(), + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), } } diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index be1b8d2..b15c2ae 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -37,8 +37,8 @@ use nuncio_filter::FilterEngine; use nuncio_mail::{MockMailBackend, MockMessageSender}; use nuncio_proto::v1::event::Kind; use nuncio_proto::v1::{ - AccountConfig, AccountProtocol, AddAccountRequest, GetMessageRequest, ListAccountsRequest, - ListMessagesRequest, MarkReadRequest, SendMessageRequest, SyncRequest, TlsMode, + AccountConfig, AddAccountRequest, GetMessageRequest, ListAccountsRequest, ListMessagesRequest, + MarkReadRequest, SendMessageRequest, SyncRequest, TlsMode, }; use nuncio_store::db::DatabaseEngine; use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT}; @@ -58,16 +58,18 @@ fn sample_account_config() -> AccountConfig { id: ACCOUNT_ID.to_string(), name: "Spine E2E Test Account".to_string(), email_address: ACCOUNT_EMAIL.to_string(), - protocol: AccountProtocol::ImapSmtp.into(), - server_host: "imap.nuncio.mx".to_string(), - server_port: 993, - imap_tls_mode: TlsMode::ImplicitTls.into(), - smtp_tls_mode: TlsMode::ImplicitTls.into(), keyring_secret_key: KEYRING_KEY.to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 465, - collection_url: String::new(), + transport: Some(nuncio_proto::v1::account_config::Transport::ImapSmtp( + nuncio_proto::v1::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: TlsMode::ImplicitTls.into(), + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: TlsMode::ImplicitTls.into(), + }, + )), } } -- 2.54.0 From 2a385b984cb6a359327efbffe3750711c87d2c00 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 18:34:51 -0500 Subject: [PATCH 38/95] fix(filter): surface rule-eval timeouts and instrument engine + webhook egress (#369) Replace the silent unwrap_or_default() on the ReDoS evaluation timeout with an explicit, per-rule timeout that logs a warn! carrying the rule's identity instead of quietly folding a timed-out evaluation into "no match". The per-rule evaluation now runs on a blocking-pool task so the timeout can actually preempt a slow synchronous evaluation, rather than racing a timer a non-yielding future would never let fire; a timed-out rule is documented and logged as non-matching, and the rest of the batch still runs. Add DEBUG rule-fire logging (rule id/name + action kinds, never the destination data an action carries) and INFO on reload_rules. Wrap webhook dispatch in a tracing span logging the target host, and log every SSRF/CIDR/DNS-rebind block decision as warn! with host/blocked-ip and reason; allowed egress logs at debug. No message bodies, secrets, or full URLs are logged. Refs #360 Co-authored-by: James Maes --- crates/nuncio-filter/src/engine.rs | 192 ++++++++++++++++-- crates/nuncio-filter/src/webhook.rs | 300 ++++++++++++++++++++++------ 2 files changed, 414 insertions(+), 78 deletions(-) diff --git a/crates/nuncio-filter/src/engine.rs b/crates/nuncio-filter/src/engine.rs index 6037db1..d5d1230 100644 --- a/crates/nuncio-filter/src/engine.rs +++ b/crates/nuncio-filter/src/engine.rs @@ -11,6 +11,7 @@ use regex::Regex; use sha2::Sha256; use std::sync::Arc; use std::time::{Duration, Instant}; +use tracing::{debug, info, warn}; /// Pre-compiled single filter rule optimizing regex and predicate execution. #[derive(Clone)] @@ -233,6 +234,26 @@ impl CompiledFilterSet { } } +/// Map each action to its variant name for observability logging, deliberately +/// dropping any destination data (folder name, forward address, webhook URL) +/// that the action variants carry. +fn action_kinds(actions: &[RuleAction]) -> Vec<&'static str> { + actions + .iter() + .map(|action| match action { + RuleAction::MoveTo(_) => "MOVE_TO", + RuleAction::CopyTo(_) => "COPY_TO", + RuleAction::MarkRead => "MARK_READ", + RuleAction::MarkUnread => "MARK_UNREAD", + RuleAction::Flag => "FLAG", + RuleAction::Unflag => "UNFLAG", + RuleAction::Delete => "DELETE", + RuleAction::ForwardTo(_) => "FORWARD_TO", + RuleAction::CallWebhook(_) => "CALL_WEBHOOK", + }) + .collect() +} + /// Lock-free Filter Engine wrapping `ArcSwap`. pub struct FilterEngine { cache: Arc>, @@ -250,7 +271,9 @@ impl FilterEngine { /// Swap / reload active filter rules atomically (<5ns latencies). pub fn reload_rules(&self, rules: Vec) -> Result<(), String> { let set = CompiledFilterSet::new(rules)?; + let rule_count = set.filters.len(); self.cache.store(Arc::new(set)); + info!(rule_count, "filter engine rules reloaded"); Ok(()) } @@ -266,6 +289,12 @@ impl FilterEngine { for filter in &guard.filters { if filter.rule.matches_account(&email.account_id) && filter.evaluate_condition(email) { + debug!( + rule_id = %filter.rule.id, + rule_name = %filter.rule.name, + actions = ?action_kinds(&filter.rule.actions), + "rule matched; dispatching actions" + ); results.push((filter.rule.clone(), filter.rule.actions.clone())); } } @@ -274,28 +303,66 @@ impl FilterEngine { } /// Evaluate with a Tokio hard timeout for ReDoS safety. + /// + /// Each rule's condition evaluation is timed independently (on a + /// blocking-pool task, so a real timeout can actually preempt a + /// long-running synchronous evaluation rather than merely racing a timer + /// that a purely synchronous future would never yield to). A rule whose + /// evaluation exceeds `timeout_duration` is treated as **non-matching** + /// for this message — the documented, fail-closed outcome — but unlike a + /// bare `unwrap_or_default()`, the timeout is never silent: it is logged + /// with the rule's identity via `warn!`, and evaluation continues with + /// the remaining rules rather than aborting the whole batch. pub async fn evaluate_with_timeout( &self, email: &Email, timeout_duration: Duration, ) -> Vec<(FilterRule, Vec)> { - let email_clone = email.clone(); - let engine_cache = self.cache.clone(); + let guard = self.cache.load(); + let mut results = Vec::new(); - tokio::time::timeout(timeout_duration, async move { - let guard = engine_cache.load(); - let mut results = Vec::new(); - for filter in &guard.filters { - if filter.rule.matches_account(&email_clone.account_id) - && filter.evaluate_condition(&email_clone) - { + for filter in &guard.filters { + if !filter.rule.matches_account(&email.account_id) { + continue; + } + + let filter_for_task = filter.clone(); + let email_for_task = email.clone(); + let eval_task = tokio::task::spawn_blocking(move || { + filter_for_task.evaluate_condition(&email_for_task) + }); + + match tokio::time::timeout(timeout_duration, eval_task).await { + Ok(Ok(true)) => { + debug!( + rule_id = %filter.rule.id, + rule_name = %filter.rule.name, + actions = ?action_kinds(&filter.rule.actions), + "rule matched; dispatching actions" + ); results.push((filter.rule.clone(), filter.rule.actions.clone())); } + Ok(Ok(false)) => {} + Ok(Err(join_error)) => { + warn!( + rule_id = %filter.rule.id, + rule_name = %filter.rule.name, + error = %join_error, + "rule evaluation task failed to complete; treating rule as non-matching" + ); + } + Err(_elapsed) => { + warn!( + rule_id = %filter.rule.id, + rule_name = %filter.rule.name, + timeout_ms = timeout_duration.as_millis(), + "rule evaluation timed out (ReDoS guard); treating rule as non-matching" + ); + } } - results - }) - .await - .unwrap_or_default() + } + + results } /// Dry-run preview evaluation returning detailed microsecond traces. @@ -367,6 +434,51 @@ impl FilterEngine { mod tests { use super::*; use nuncio_core::model::Email; + use std::sync::Mutex; + use tracing::field::{Field, Visit}; + use tracing::span; + + /// Minimal `tracing::Subscriber` that records a formatted line per event + /// so tests can assert on emitted level + fields without pulling in + /// `tracing-subscriber`'s registry machinery. + struct CapturingSubscriber { + events: Arc>>, + } + + struct LineVisitor<'a>(&'a mut String); + + impl Visit for LineVisitor<'_> { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } + } + + impl tracing::Subscriber for CapturingSubscriber { + fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool { + true + } + + fn new_span(&self, _span: &span::Attributes<'_>) -> span::Id { + span::Id::from_u64(1) + } + + fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {} + + fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {} + + fn event(&self, event: &tracing::Event<'_>) { + let mut line = format!("{}", event.metadata().level()); + let mut visitor = LineVisitor(&mut line); + event.record(&mut visitor); + if let Ok(mut events) = self.events.lock() { + events.push(line); + } + } + + fn enter(&self, _span: &span::Id) {} + + fn exit(&self, _span: &span::Id) {} + } fn test_email(account_id: &str, subject: &str, folder_id: &str) -> Email { Email { @@ -576,4 +688,58 @@ mod tests { let email = test_email("acct-1", "Anything", "inbox"); assert_eq!(engine.evaluate(&email).len(), 1); } + + #[tokio::test] + async fn test_evaluate_with_timeout_logs_warn_and_treats_timeout_as_non_match() { + // A crafted, slow-to-evaluate condition (large body scanned by a + // CONTAINS check) paired with a near-zero timeout budget forces the + // per-rule timeout path deterministically: the evaluation runs on a + // real blocking-pool thread while the timer races it on the async + // task, so the timer reliably wins for any evaluation slower than a + // few microseconds. + let events: Arc>> = Arc::new(Mutex::new(Vec::new())); + let subscriber = CapturingSubscriber { + events: events.clone(), + }; + let _guard = tracing::subscriber::set_default(subscriber); + + let rule = FilterRule { + id: "slow-rule".to_string(), + name: "Slow Rule".to_string(), + target_account: "*".to_string(), + priority: 1, + enabled: true, + nsql_text: String::new(), + conditions: ConditionNode::Leaf(ConditionLeaf { + field: FilterField::Body, + operator: FilterOperator::Contains, + value: FilterValue::String("nonexistent-token".to_string()), + }), + actions: vec![RuleAction::MarkRead], + created_at: 0, + updated_at: 0, + }; + let engine = FilterEngine::new(vec![rule]).unwrap(); + + let mut email = test_email("acct-1", "Anything", "inbox"); + email.body_plain = Some("A".repeat(60_000_000)); + + let results = engine + .evaluate_with_timeout(&email, Duration::from_millis(1)) + .await; + + assert!( + results.is_empty(), + "documented outcome: a timed-out rule must be treated as non-matching, \ + not fabricated as a match" + ); + + let captured = events.lock().unwrap(); + assert!( + captured.iter().any(|line| line.contains("WARN") + && line.contains("timed out") + && line.contains("slow-rule")), + "expected a WARN log carrying the rule id for the timed-out evaluation, got: {captured:?}" + ); + } } diff --git a/crates/nuncio-filter/src/webhook.rs b/crates/nuncio-filter/src/webhook.rs index 7d0e6da..123f82e 100644 --- a/crates/nuncio-filter/src/webhook.rs +++ b/crates/nuncio-filter/src/webhook.rs @@ -10,8 +10,20 @@ use std::net::{IpAddr, SocketAddr}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use thiserror::Error; use tokio::net::lookup_host; +use tracing::{debug, warn, Instrument}; use zeroize::Zeroizing; +/// Extract just the host for logging (never the path/query, which may carry +/// caller-supplied tokens), falling back to a placeholder when the URL itself +/// cannot be parsed. +fn webhook_host_for_log(url: &str) -> String { + reqwest::Url::parse(url) + .ok() + .and_then(|parsed| parsed.host_str().map(str::to_string)) + .map(|h| h.trim_start_matches('[').trim_end_matches(']').to_string()) + .unwrap_or_else(|| "unparsable-url".to_string()) +} + type HmacSha256 = Hmac; /// Returns the first address in `addrs` that is a blocked webhook target, if @@ -97,75 +109,111 @@ impl WebhookDispatcher { sender: &str, opts: &ValidationOptions, ) -> Result { - let action = RuleAction::CallWebhook(url.to_string()); - NsqlValidator::pass6_action_security(&[action], opts) - .map_err(|e| WebhookError::SecurityViolation(e.to_string()))?; + let host_for_log = webhook_host_for_log(url); + let span = + tracing::info_span!("webhook_dispatch", host = %host_for_log, rule_id = %rule_id); - // Pass 6 only sees literal-IP hosts. Resolve the target here and, - // when the policy is enabled, reject it if it points into a blocked - // range - closing the DNS-rebind class of SSRF bypass that a - // substring/literal check on the URL cannot catch. The resolved - // address is then pinned below for the actual connection, so the - // dispatch cannot re-resolve the hostname and land on a different, - // unchecked address: the address that was checked is the only one - // `reqwest` is permitted to connect to. - let parsed = reqwest::Url::parse(url) - .map_err(|e| WebhookError::SecurityViolation(format!("invalid webhook URL: {e}")))?; - let host = parsed - .host_str() - .ok_or_else(|| WebhookError::SecurityViolation("webhook URL has no host".to_string()))? - // `host_str` brackets IPv6 literals (`[::1]`); strip them so the - // pair passes to the resolver cleanly. - .trim_start_matches('[') - .trim_end_matches(']') - .to_string(); - let port = parsed.port_or_known_default().unwrap_or(0); + async move { + let action = RuleAction::CallWebhook(url.to_string()); + if let Err(e) = NsqlValidator::pass6_action_security(&[action], opts) { + warn!( + host = %host_for_log, + reason = %e, + "webhook egress blocked by SSRF/security policy" + ); + return Err(WebhookError::SecurityViolation(e.to_string())); + } - let resolved = resolve_host(&host, port).await?; - if opts.block_private_webhooks { - if let Some(blocked) = first_blocked_address(&resolved) { - return Err(WebhookError::SecurityViolation(format!( - "webhook host '{host}' resolves to blocked address {blocked}" - ))); + // Pass 6 only sees literal-IP hosts. Resolve the target here and, + // when the policy is enabled, reject it if it points into a blocked + // range - closing the DNS-rebind class of SSRF bypass that a + // substring/literal check on the URL cannot catch. The resolved + // address is then pinned below for the actual connection, so the + // dispatch cannot re-resolve the hostname and land on a different, + // unchecked address: the address that was checked is the only one + // `reqwest` is permitted to connect to. + let parsed = reqwest::Url::parse(url).map_err(|e| { + WebhookError::SecurityViolation(format!("invalid webhook URL: {e}")) + })?; + let host = parsed + .host_str() + .ok_or_else(|| { + WebhookError::SecurityViolation("webhook URL has no host".to_string()) + })? + // `host_str` brackets IPv6 literals (`[::1]`); strip them so the + // pair passes to the resolver cleanly. + .trim_start_matches('[') + .trim_end_matches(']') + .to_string(); + let port = parsed.port_or_known_default().unwrap_or(0); + + let resolved = resolve_host(&host, port).await?; + if opts.block_private_webhooks { + if let Some(blocked) = first_blocked_address(&resolved) { + warn!( + host = %host, + blocked_ip = %blocked, + "webhook egress blocked: resolved address is in a disallowed range (DNS-rebind guard)" + ); + return Err(WebhookError::SecurityViolation(format!( + "webhook host '{host}' resolves to blocked address {blocked}" + ))); + } + } + let pinned_addr = resolved[0]; + debug!( + host = %host, + resolved_ip = %pinned_addr, + "webhook egress allowed by SSRF/security policy" + ); + let client = pinned_client(&host, pinned_addr, port)?; + + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + + let payload = json!({ + "event": "nuncio.filter.matched", + "timestamp": timestamp, + "rule_id": rule_id, + "message_id": message_id, + "subject": subject, + "sender": sender, + }); + + let payload_str = payload.to_string(); + + let mut mac = HmacSha256::new_from_slice(self.secret_key.as_bytes()) + .map_err(|e| WebhookError::SigningError(e.to_string()))?; + mac.update(format!("{timestamp}.{payload_str}").as_bytes()); + let signature = hex::encode(mac.finalize().into_bytes()); + + let send_result = client + .post(url) + .header("Content-Type", "application/json") + .header( + "X-Nuncio-Signature", + format!("t={timestamp},v1={signature}"), + ) + .body(payload_str) + .send() + .await; + + match send_result { + Ok(response) => { + let status = response.status().as_u16(); + debug!(host = %host, status, "webhook dispatched"); + Ok(status) + } + Err(e) => { + warn!(host = %host, error = %e, "webhook dispatch network error"); + Err(WebhookError::NetworkError(e.to_string())) + } } } - let pinned_addr = resolved[0]; - let client = pinned_client(&host, pinned_addr, port)?; - - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - - let payload = json!({ - "event": "nuncio.filter.matched", - "timestamp": timestamp, - "rule_id": rule_id, - "message_id": message_id, - "subject": subject, - "sender": sender, - }); - - let payload_str = payload.to_string(); - - let mut mac = HmacSha256::new_from_slice(self.secret_key.as_bytes()) - .map_err(|e| WebhookError::SigningError(e.to_string()))?; - mac.update(format!("{timestamp}.{payload_str}").as_bytes()); - let signature = hex::encode(mac.finalize().into_bytes()); - - let response = client - .post(url) - .header("Content-Type", "application/json") - .header( - "X-Nuncio-Signature", - format!("t={timestamp},v1={signature}"), - ) - .body(payload_str) - .send() - .await - .map_err(|e| WebhookError::NetworkError(e.to_string()))?; - - Ok(response.status().as_u16()) + .instrument(span) + .await } /// Dispatch a `CALL WEBHOOK` action with default production security options. @@ -192,9 +240,54 @@ impl WebhookDispatcher { #[cfg(test)] mod tests { use super::*; + use std::sync::Mutex; + use tracing::field::{Field, Visit}; + use tracing::span; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; + /// Minimal `tracing::Subscriber` that records a formatted line per event + /// so tests can assert on emitted level + fields without pulling in + /// `tracing-subscriber`'s registry machinery. + struct CapturingSubscriber { + events: std::sync::Arc>>, + } + + struct LineVisitor<'a>(&'a mut String); + + impl Visit for LineVisitor<'_> { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } + } + + impl tracing::Subscriber for CapturingSubscriber { + fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool { + true + } + + fn new_span(&self, _span: &span::Attributes<'_>) -> span::Id { + span::Id::from_u64(1) + } + + fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {} + + fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {} + + fn event(&self, event: &tracing::Event<'_>) { + let mut line = format!("{}", event.metadata().level()); + let mut visitor = LineVisitor(&mut line); + event.record(&mut visitor); + if let Ok(mut events) = self.events.lock() { + events.push(line); + } + } + + fn enter(&self, _span: &span::Id) {} + + fn exit(&self, _span: &span::Id) {} + } + #[tokio::test] async fn test_pinned_client_connects_to_checked_ip_not_dns() { // `.invalid` is reserved by RFC 2606 to never resolve: any real DNS @@ -297,4 +390,81 @@ mod tests { ]; assert_eq!(first_blocked_address(&addrs), None); } + + #[tokio::test] + async fn test_blocked_webhook_logs_warn_with_host_and_reason() { + let events: std::sync::Arc>> = + std::sync::Arc::new(Mutex::new(Vec::new())); + let subscriber = CapturingSubscriber { + events: events.clone(), + }; + let _guard = tracing::subscriber::set_default(subscriber); + + let dispatcher = WebhookDispatcher::new("secret_key_123"); + let result = dispatcher + .dispatch( + "http://169.254.169.254/latest/meta-data", + "rule_1", + "msg_1", + "Test", + "a@b.com", + ) + .await; + + assert!(matches!(result, Err(WebhookError::SecurityViolation(_)))); + + let captured = events.lock().unwrap(); + assert!( + captured.iter().any(|line| line.contains("WARN") + && line.contains("blocked") + && line.contains("169.254.169.254")), + "expected a WARN log carrying the blocked host and reason, got: {captured:?}" + ); + assert!( + !captured.iter().any(|line| line.contains("secret_key_123")), + "the webhook HMAC secret must never appear in a log line" + ); + } + + #[tokio::test] + async fn test_allowed_webhook_logs_debug_not_warn() { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/hook")) + .respond_with(ResponseTemplate::new(200)) + .mount(&mock_server) + .await; + + let events: std::sync::Arc>> = + std::sync::Arc::new(Mutex::new(Vec::new())); + let subscriber = CapturingSubscriber { + events: events.clone(), + }; + let _guard = tracing::subscriber::set_default(subscriber); + + let dispatcher = WebhookDispatcher::new("secret_key_123"); + let opts = ValidationOptions { + available_folders: None, + allowed_forward_domains: None, + block_private_webhooks: false, + }; + let url = format!("{}/hook", mock_server.uri()); + let status = dispatcher + .dispatch_with_options(&url, "rule_1", "msg_1", "Test", "a@b.com", &opts) + .await + .expect("dispatch to the mock server must succeed"); + assert_eq!(status, 200); + + let captured = events.lock().unwrap(); + assert!( + captured + .iter() + .any(|line| line.contains("DEBUG") && line.contains("allowed")), + "expected a DEBUG log for the allowed egress decision, got: {captured:?}" + ); + assert!( + !captured.iter().any(|line| line.contains("WARN")), + "an allowed dispatch must not emit a WARN, got: {captured:?}" + ); + } } -- 2.54.0 From 9f39fb095ea1e667ebe337ce40e10daa5dde14b2 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 18:39:01 -0500 Subject: [PATCH 39/95] feat(nunciod): per-RPC tracing span with request-id correlation and auth logging (#371) Add a tower Layer on the tonic server that wraps every RPC in a root span carrying a generated request_id, the gRPC method, and the peer address. The inner service (routing, the bearer-auth interceptor, and the handler) runs inside that span, so all events emitted while handling a request inherit the request_id. The span opens with a DEBUG event and closes, once the response body drains, with an INFO event carrying the terminal gRPC status and elapsed time. Propagate the correlation span into the SyncDispatcher's detached sync tasks so a sync triggered by Mail.Sync logs under the triggering request_id. Log bearer-auth accept (DEBUG) and reject (WARN, with a fixed reason) in the interceptor, carrying only the peer -- never the token, the authorization header, or anything derived from them. The constant-time compare is unchanged. Refs #358 Co-authored-by: James Maes --- Cargo.lock | 4 + Cargo.toml | 4 + crates/nunciod/Cargo.toml | 4 + crates/nunciod/src/grpc.rs | 132 ++++++++- crates/nunciod/src/lib.rs | 3 + crates/nunciod/src/rpc_trace.rs | 401 ++++++++++++++++++++++++++ crates/nunciod/src/sync_dispatcher.rs | 52 +++- crates/nunciod/src/test_tracing.rs | 133 +++++++++ 8 files changed, 711 insertions(+), 22 deletions(-) create mode 100644 crates/nunciod/src/rpc_trace.rs create mode 100644 crates/nunciod/src/test_tracing.rs diff --git a/Cargo.lock b/Cargo.lock index 320c097..b4131eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2679,6 +2679,9 @@ dependencies = [ "bytes", "fs4", "hex", + "http", + "http-body", + "http-body-util", "nuncio-cal", "nuncio-cli", "nuncio-contacts", @@ -2696,6 +2699,7 @@ dependencies = [ "tokio", "tokio-stream", "tonic", + "tower", "tracing", "tracing-appender", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index b045994..b926cbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,10 @@ flate2 = "1.0" tar = "0.4" zip = { version = "8.6", default-features = false, features = ["deflate"] } tonic = "0.14" +tower = "0.5" +http = "1" +http-body = "1" +http-body-util = "0.1" tonic-prost = "0.14" prost = "0.14" prost-types = "0.14" diff --git a/crates/nunciod/Cargo.toml b/crates/nunciod/Cargo.toml index fd79f30..f47c835 100644 --- a/crates/nunciod/Cargo.toml +++ b/crates/nunciod/Cargo.toml @@ -30,6 +30,9 @@ tracing-subscriber.workspace = true tracing-appender.workspace = true tempfile.workspace = true tonic.workspace = true +tower.workspace = true +http.workspace = true +http-body.workspace = true async-trait.workspace = true subtle.workspace = true hex.workspace = true @@ -41,4 +44,5 @@ fs4.workspace = true nuncio-cli = { path = "../nuncio-cli" } sqlx.workspace = true wiremock.workspace = true +http-body-util.workspace = true tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 8fdf6cd..6562824 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -2458,25 +2458,41 @@ impl BearerAuthInterceptor { impl tonic::service::Interceptor for BearerAuthInterceptor { fn call(&mut self, request: Request<()>) -> Result, Status> { + // Only ever log the peer and a fixed reason string. The token value, + // the raw `authorization` header, and anything derived from either + // (length, prefix, substring) are NEVER logged -- doing so would leak + // the bearer credential into the logs. This runs inside the per-RPC + // span (see `rpc_trace`), so these events carry the request_id. + let peer = request + .remote_addr() + .map(|addr| addr.to_string()) + .unwrap_or_else(|| "unknown".to_string()); + + let reject = |reason: &'static str| { + tracing::warn!(peer = %peer, reason, "gRPC auth rejected"); + Status::unauthenticated(reason) + }; + let header = request .metadata() .get("authorization") - .ok_or_else(|| Status::unauthenticated("missing authorization metadata"))?; + .ok_or_else(|| reject("missing authorization metadata"))?; let header_str = header .to_str() - .map_err(|_| Status::unauthenticated("authorization metadata is not valid ASCII"))?; - let provided = header_str.strip_prefix("Bearer ").ok_or_else(|| { - Status::unauthenticated("expected 'Bearer ' authorization scheme") - })?; + .map_err(|_| reject("authorization metadata is not valid ASCII"))?; + let provided = header_str + .strip_prefix("Bearer ") + .ok_or_else(|| reject("expected 'Bearer ' authorization scheme"))?; let matches: bool = provided .as_bytes() .ct_eq(self.expected_token.as_bytes()) .into(); if matches { + tracing::debug!(peer = %peer, "gRPC auth accepted"); Ok(request) } else { - Err(Status::unauthenticated("invalid bearer token")) + Err(reject("invalid bearer token")) } } } @@ -2861,6 +2877,12 @@ pub async fn serve_on_listener_with_overrides_and_shutdown( ); Server::builder() + // Wrap every RPC in a correlation span (request_id + method + peer) + // before the resource-limit tuning below, so the span brackets the + // whole request -- routing, the per-service `BearerAuthInterceptor`, + // and the handler -- and its request_id is inherited by every event + // emitted while the request is handled. + .layer(crate::rpc_trace::RpcTraceLayer) .concurrency_limit_per_connection(GRPC_CONCURRENCY_LIMIT_PER_CONNECTION) .timeout(GRPC_REQUEST_TIMEOUT) .max_concurrent_streams(GRPC_MAX_CONCURRENT_STREAMS) @@ -2888,6 +2910,104 @@ mod tests { use nuncio_proto::v1::system_client::SystemClient; use tonic::Code; + use crate::test_tracing::with_recorder; + use tonic::service::Interceptor; + use tracing::Level; + + /// The bearer token used by the auth-logging tests. A test asserts this + /// exact value never appears in any captured log field. + const TEST_BEARER_TOKEN: &str = "s3cr3t-bearer-token-value"; + + fn bearer_request(header: Option<&str>) -> Request<()> { + let mut request = Request::new(()); + if let Some(value) = header { + request.metadata_mut().insert( + "authorization", + value.parse().expect("valid ASCII metadata value"), + ); + } + request + } + + #[test] + fn auth_accept_logs_debug_without_leaking_the_token() { + let (recorder, result) = with_recorder(|| { + let mut interceptor = BearerAuthInterceptor::new(TEST_BEARER_TOKEN); + interceptor.call(bearer_request(Some(&format!("Bearer {TEST_BEARER_TOKEN}")))) + }); + assert!(result.is_ok(), "a valid bearer token is accepted"); + + let events = recorder.events(); + let accept = events + .iter() + .find(|e| e.message() == "gRPC auth accepted") + .expect("an accept must be logged"); + assert_eq!(accept.level, Level::DEBUG, "accept is logged at DEBUG"); + + assert!( + recorder + .all_field_values() + .iter() + .all(|v| !v.contains(TEST_BEARER_TOKEN)), + "the bearer token must never appear in any log field" + ); + } + + #[test] + fn auth_reject_bad_token_logs_warn_without_leaking_the_token() { + let presented = "attacker-guess-token"; + let (recorder, result) = with_recorder(|| { + let mut interceptor = BearerAuthInterceptor::new(TEST_BEARER_TOKEN); + interceptor.call(bearer_request(Some(&format!("Bearer {presented}")))) + }); + let status = result.expect_err("a mismatched token is rejected"); + assert_eq!(status.code(), Code::Unauthenticated); + + let events = recorder.events(); + let reject = events + .iter() + .find(|e| e.message() == "gRPC auth rejected") + .expect("a reject must be logged"); + assert_eq!(reject.level, Level::WARN, "reject is logged at WARN"); + assert_eq!( + reject.fields.get("reason").map(String::as_str), + Some("invalid bearer token"), + "the reject must carry a fixed reason string" + ); + + // Neither the real token nor the presented guess may appear anywhere. + assert!( + recorder + .all_field_values() + .iter() + .all(|v| !v.contains(TEST_BEARER_TOKEN) && !v.contains(presented)), + "no token material may appear in any log field" + ); + } + + #[test] + fn auth_reject_missing_header_logs_warn_reason() { + let (recorder, result) = with_recorder(|| { + let mut interceptor = BearerAuthInterceptor::new(TEST_BEARER_TOKEN); + interceptor.call(bearer_request(None)) + }); + assert!( + result.is_err(), + "a missing authorization header is rejected" + ); + + let events = recorder.events(); + let reject = events + .iter() + .find(|e| e.message() == "gRPC auth rejected") + .expect("a reject must be logged"); + assert_eq!(reject.level, Level::WARN); + assert_eq!( + reject.fields.get("reason").map(String::as_str), + Some("missing authorization metadata") + ); + } + /// Proves a stored second-granularity instant survives the proto /// boundary exactly: `map_email_to_proto`'s `received_at` decodes back /// to the identical unix-seconds value via `timestamp_to_unix_secs`. diff --git a/crates/nunciod/src/lib.rs b/crates/nunciod/src/lib.rs index b37803f..a87fa7e 100644 --- a/crates/nunciod/src/lib.rs +++ b/crates/nunciod/src/lib.rs @@ -11,9 +11,12 @@ pub mod logging; pub mod orchestrator; pub mod outbox; pub mod pagination; +pub mod rpc_trace; pub mod send; pub mod sync; pub mod sync_dispatcher; +#[cfg(test)] +pub(crate) mod test_tracing; pub use orchestrator::SelfHealingSyncOrchestrator; /// Default PERSISTENT database path for `nunciod`: `~/.nuncio/nuncio.db` diff --git a/crates/nunciod/src/rpc_trace.rs b/crates/nunciod/src/rpc_trace.rs new file mode 100644 index 0000000..0931742 --- /dev/null +++ b/crates/nunciod/src/rpc_trace.rs @@ -0,0 +1,401 @@ +//! Per-RPC tracing span and request-id correlation for the gRPC transport. +//! +//! This is a [`tower::Layer`] mounted on the tonic `Server` so that every +//! inbound RPC is wrapped in a single root span carrying: +//! +//! * a generated `request_id` -- a lightweight per-process identifier +//! (process-start tag + monotonic counter) that lets an operator follow one +//! request across every log line it produced, without pulling in a heavier +//! UUID dependency; and +//! * the gRPC method path and the peer socket address. +//! +//! The inner service (routing + the per-service `BearerAuthInterceptor` + the +//! handler future) is entered inside that span, so *every* event emitted while +//! the request is handled -- including the auth accept/reject logs and any +//! event a handler emits -- inherits the `request_id` automatically. The span +//! opens with a DEBUG "received" event and closes, once the response body has +//! been fully drained, with an INFO event carrying the terminal gRPC status +//! code and the elapsed wall-clock time. +//! +//! ## Why the response body is wrapped +//! +//! A gRPC call's final status lives in the HTTP/2 *trailers* (`grpc-status`), +//! which are only produced once the response body finishes streaming (an +//! immediate error is instead a trailers-only response with `grpc-status` in +//! the leading headers). The layer therefore wraps the response body in +//! [`TracedBody`], which observes the terminal status as it flows past and +//! emits the completion event when the stream ends (or when the body is +//! dropped, e.g. on client disconnect), so a long-lived server-streaming RPC +//! is not logged as "complete" until it actually ends. + +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::OnceLock; +use std::task::{Context, Poll}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use http::{HeaderMap, Request, Response}; +use http_body::{Body, Frame, SizeHint}; +use tonic::body::Body as TonicBody; +use tonic::transport::server::TcpConnectInfo; +use tonic::Code; +use tower::{Layer, Service}; +use tracing::{Instrument, Span}; + +/// HTTP/2 trailer (or leading header, for a trailers-only error response) +/// carrying the final gRPC status code. +const GRPC_STATUS_HEADER: &str = "grpc-status"; + +/// Generates a correlation id unique within this daemon process. +/// +/// The id is `-`: the process tag is the daemon's start +/// time in unix nanoseconds (captured once), and the counter increments per +/// call. This is deliberately not a UUID -- it needs only to be unique within +/// a single running daemon so its logs can be correlated, which a per-process +/// tag plus a monotonic counter satisfies without a new dependency. +fn next_request_id() -> String { + static COUNTER: AtomicU64 = AtomicU64::new(0); + static PROCESS_TAG: OnceLock = OnceLock::new(); + + let tag = *PROCESS_TAG.get_or_init(|| { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0) + }); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + format!("{tag:016x}-{n:08x}") +} + +/// Reads the `grpc-status` code from a header/trailer map, if present. +fn grpc_status_code(headers: &HeaderMap) -> Option { + headers + .get(GRPC_STATUS_HEADER) + .map(|value| Code::from_bytes(value.as_bytes())) +} + +/// [`tower::Layer`] that wraps each RPC in a correlation span. Mounted on the +/// tonic `Server` via `Server::builder().layer(RpcTraceLayer)`. +#[derive(Clone, Copy, Debug, Default)] +pub struct RpcTraceLayer; + +impl Layer for RpcTraceLayer { + type Service = RpcTrace; + + fn layer(&self, inner: S) -> Self::Service { + RpcTrace { inner } + } +} + +/// The [`RpcTraceLayer`]'s service: opens the span, runs the inner service +/// (routing, auth interceptor, handler) inside it, and wraps the response body +/// so the terminal status can be observed at stream end. +#[derive(Clone, Debug)] +pub struct RpcTrace { + inner: S, +} + +impl Service> for RpcTrace +where + S: Service, Response = Response> + Clone + Send + 'static, + S::Future: Send + 'static, + S::Error: Send + 'static, +{ + type Response = Response>; + type Error = S::Error; + type Future = + Pin> + Send + 'static>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, req: Request) -> Self::Future { + // `poll_ready` was called on `self.inner`, so it -- not the fresh + // clone -- is the readied service that must handle this request. Swap + // the readied service out and leave the clone behind for the next + // call (the standard tower clone-and-replace pattern that avoids + // calling an un-readied clone). + let clone = self.inner.clone(); + let mut inner = std::mem::replace(&mut self.inner, clone); + + let method = req.uri().path().to_string(); + let peer = req + .extensions() + .get::() + .and_then(TcpConnectInfo::remote_addr) + .map(|addr| addr.to_string()) + .unwrap_or_else(|| "unknown".to_string()); + let request_id = next_request_id(); + + let span = tracing::info_span!( + "grpc.request", + request_id = %request_id, + rpc.method = %method, + peer = %peer, + ); + let start = Instant::now(); + + Box::pin(async move { + // Enter the span for the synchronous routing + interceptor call so + // the auth accept/reject logs inherit the request_id, and emit the + // open event under it. + let call = span.in_scope(|| { + tracing::debug!("gRPC request received"); + inner.call(req) + }); + // Instrument the handler future so events emitted across its await + // points also inherit the span. + let response = call.instrument(span.clone()).await?; + + // A trailers-only error response carries `grpc-status` in the + // leading headers; a normal response carries it in the trailers, + // observed by `TracedBody` as the body drains. + let initial_status = grpc_status_code(response.headers()); + let (parts, body) = response.into_parts(); + let traced = TracedBody::new(body, span, start, initial_status); + Ok(Response::from_parts(parts, traced)) + }) + } +} + +/// Response body wrapper that observes the terminal `grpc-status` and logs the +/// request's completion (status + elapsed) exactly once, when the body stream +/// ends or is dropped. +/// +/// All fields are `Unpin`, so the pin projection in [`Body::poll_frame`] is a +/// safe `Pin::new` over the inner body rather than an `unsafe` projection. +#[derive(Debug)] +pub struct TracedBody { + inner: B, + span: Span, + start: Instant, + status: Option, + logged: bool, +} + +impl TracedBody { + fn new(inner: B, span: Span, start: Instant, initial_status: Option) -> Self { + Self { + inner, + span, + start, + status: initial_status, + logged: false, + } + } + + /// Emits the terminal completion event once. Idempotent: subsequent calls + /// (e.g. a `Drop` after the stream already ended) are no-ops. + fn log_completion(&mut self) { + if self.logged { + return; + } + self.logged = true; + let elapsed_ms = self.start.elapsed().as_millis() as u64; + let status = self.status.unwrap_or(Code::Unknown); + self.span.in_scope(|| { + tracing::info!( + grpc.status = ?status, + elapsed_ms, + "gRPC request completed" + ); + }); + } +} + +impl Body for TracedBody +where + B: Body + Unpin, +{ + type Data = B::Data; + type Error = B::Error; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + // Safe because `Self: Unpin` (all fields are `Unpin`). + let this = self.get_mut(); + let polled = Pin::new(&mut this.inner).poll_frame(cx); + match &polled { + Poll::Ready(Some(Ok(frame))) => { + if let Some(trailers) = frame.trailers_ref() { + if let Some(code) = grpc_status_code(trailers) { + this.status = Some(code); + } + } + } + Poll::Ready(Some(Err(_))) | Poll::Ready(None) => this.log_completion(), + Poll::Pending => {} + } + polled + } + + fn is_end_stream(&self) -> bool { + self.inner.is_end_stream() + } + + fn size_hint(&self) -> SizeHint { + self.inner.size_hint() + } +} + +impl Drop for TracedBody { + fn drop(&mut self) { + // Covers stream ends that do not surface a final frame here (client + // disconnect, an outer layer dropping the body): the request is still + // logged as completed exactly once. + self.log_completion(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_tracing::with_recorder; + use http_body_util::BodyExt; + use std::collections::HashSet; + use std::convert::Infallible; + use tower::ServiceExt; + use tracing::Level; + + /// Minimal inner service that returns a response with `grpc-status` set in + /// the leading headers (a trailers-only response, tonic's shape for an + /// immediate status), so the layer's terminal-status extraction can be + /// exercised without a live server. + #[derive(Clone)] + struct StatusService { + code: i32, + } + + impl Service> for StatusService { + type Response = Response; + type Error = Infallible; + type Future = + Pin> + Send + 'static>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _req: Request) -> Self::Future { + let code = self.code; + Box::pin(async move { + // A handler-emitted event: it must inherit the span so it + // carries the request_id set by the layer. + tracing::info!("handler ran"); + let mut response = Response::new(TonicBody::empty()); + response + .headers_mut() + .insert(GRPC_STATUS_HEADER, code.to_string().parse().unwrap()); + Ok(response) + }) + } + } + + fn request() -> Request { + Request::builder() + .uri("/nuncio.v1.Mail/Sync") + .body(TonicBody::empty()) + .unwrap() + } + + #[test] + fn rpc_produces_span_with_request_id_and_logs_terminal_status() { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + let (recorder, ()) = with_recorder(|| { + runtime.block_on(async { + let mut svc = RpcTraceLayer.layer(StatusService { code: 5 }); + let response = svc.ready().await.unwrap().call(request()).await.unwrap(); + // Drain the body so the completion event fires at stream end. + response.into_body().collect().await.unwrap(); + }); + }); + + let spans = recorder.spans(); + let rpc_span = spans + .iter() + .find(|s| s.name == "grpc.request") + .expect("a grpc.request span must be created"); + let request_id = rpc_span + .fields + .get("request_id") + .expect("the span must carry a request_id field"); + assert!(!request_id.is_empty(), "request_id must be populated"); + assert_eq!( + rpc_span.fields.get("rpc.method").map(String::as_str), + Some("/nuncio.v1.Mail/Sync"), + "the span must carry the gRPC method path" + ); + + let events = recorder.events(); + let completion = events + .iter() + .find(|e| e.message() == "gRPC request completed") + .expect("the terminal completion event must be logged"); + assert_eq!( + completion.level, + Level::INFO, + "the completion event is logged at INFO" + ); + assert_eq!( + completion.fields.get("grpc.status").map(String::as_str), + Some("NotFound"), + "the completion event must carry the terminal gRPC status code" + ); + assert!( + completion.fields.contains_key("elapsed_ms"), + "the completion event must carry the elapsed time" + ); + + // The handler ran inside the span, so its own event was captured too + // (proving handler events inherit the correlation context). + assert!( + events.iter().any(|e| e.message() == "handler ran"), + "a handler-emitted event must be observed under the request span" + ); + } + + #[test] + fn ok_status_is_recorded_at_completion() { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let (recorder, ()) = with_recorder(|| { + runtime.block_on(async { + let mut svc = RpcTraceLayer.layer(StatusService { code: 0 }); + let response = svc.ready().await.unwrap().call(request()).await.unwrap(); + response.into_body().collect().await.unwrap(); + }); + }); + + let events = recorder.events(); + let completion = events + .iter() + .find(|e| e.message() == "gRPC request completed") + .expect("completion event"); + assert_eq!( + completion.fields.get("grpc.status").map(String::as_str), + Some("Ok") + ); + } + + #[test] + fn grpc_status_code_parses_known_codes_and_ignores_absent() { + let mut headers = HeaderMap::new(); + assert!(grpc_status_code(&headers).is_none()); + headers.insert(GRPC_STATUS_HEADER, "3".parse().unwrap()); + assert_eq!(grpc_status_code(&headers), Some(Code::InvalidArgument)); + } + + #[test] + fn request_ids_are_unique_within_the_process() { + let ids: HashSet = (0..1000).map(|_| next_request_id()).collect(); + assert_eq!(ids.len(), 1000, "request ids must not collide"); + } +} diff --git a/crates/nunciod/src/sync_dispatcher.rs b/crates/nunciod/src/sync_dispatcher.rs index 2278c64..8e46819 100644 --- a/crates/nunciod/src/sync_dispatcher.rs +++ b/crates/nunciod/src/sync_dispatcher.rs @@ -48,6 +48,7 @@ use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use thiserror::Error; use tokio::sync::{watch, Semaphore}; +use tracing::Instrument; /// Environment variable bounding how many account syncs may run at once. pub const CONCURRENCY_ENV_VAR: &str = "NUNCIO_SYNC_CONCURRENCY"; @@ -285,18 +286,30 @@ impl SyncDispatcher { let inner = self.inner.clone(); let id = account_id.to_string(); - tokio::spawn(async move { - let outcome = inner.run_guarded(&id).await; - // Remove the in-flight entry before publishing so a request - // arriving after completion starts a fresh sync rather than - // joining an already-finished one. - inner - .inflight - .lock() - .unwrap_or_else(PoisonError::into_inner) - .remove(&id); - let _ = tx.send(Some(outcome)); - }); + // The actual sync runs in a detached task, which does NOT + // inherit the caller's span automatically. Capture the current + // span (the RPC's correlation span when a `Mail/Sync` triggered + // this) and instrument the task with it, so the sync's logs are + // correlatable to the request that kicked it off. A request + // that COALESCES onto an already-running sync joins the owner's + // task and is therefore correlated to the owner's request_id, + // not its own. + let span = tracing::Span::current(); + tokio::spawn( + async move { + let outcome = inner.run_guarded(&id).await; + // Remove the in-flight entry before publishing so a + // request arriving after completion starts a fresh sync + // rather than joining an already-finished one. + inner + .inflight + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&id); + let _ = tx.send(Some(outcome)); + } + .instrument(span), + ); rx } }; @@ -319,12 +332,19 @@ impl SyncDispatcher { /// production syncer additionally reports them on the event bus). pub async fn sync_all(&self, account_ids: Vec) -> usize { let mut set = tokio::task::JoinSet::new(); + // Propagate the caller's span (the RPC correlation span for an + // all-accounts `Mail/Sync`) into each per-account task so the fan-out + // stays correlated to the triggering request. + let span = tracing::Span::current(); for account_id in account_ids { let dispatcher = self.clone(); - set.spawn(async move { - let result = dispatcher.sync_account(&account_id).await; - (account_id, result) - }); + set.spawn( + async move { + let result = dispatcher.sync_account(&account_id).await; + (account_id, result) + } + .instrument(span.clone()), + ); } let mut total = 0usize; diff --git a/crates/nunciod/src/test_tracing.rs b/crates/nunciod/src/test_tracing.rs new file mode 100644 index 0000000..9c68c9b --- /dev/null +++ b/crates/nunciod/src/test_tracing.rs @@ -0,0 +1,133 @@ +//! Shared in-test `tracing` capture helper. +//! +//! Installs a thread-local subscriber that records every span (with its +//! fields) and every event (with its level, fields, and message) into an +//! in-memory buffer, so tests can assert on what the code logged -- including +//! the negative assertion that a secret never appears in any recorded field. +//! Deterministic and fully offline: no global subscriber, no I/O. + +#![cfg(test)] + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use tracing::field::{Field, Visit}; +use tracing::span::Attributes; +use tracing::{Event, Id, Level, Subscriber}; +use tracing_subscriber::layer::Context as LayerContext; +use tracing_subscriber::prelude::*; +use tracing_subscriber::registry::LookupSpan; + +/// A captured span: its name and the string-rendered fields it opened with. +#[derive(Debug, Clone)] +pub(crate) struct RecordedSpan { + pub name: String, + pub fields: HashMap, +} + +/// A captured event: its level and its string-rendered fields (including the +/// `message`). +#[derive(Debug, Clone)] +pub(crate) struct RecordedEvent { + pub level: Level, + pub fields: HashMap, +} + +impl RecordedEvent { + /// The event's `message` field, or the empty string if it had none. + pub fn message(&self) -> &str { + self.fields.get("message").map(String::as_str).unwrap_or("") + } +} + +/// In-memory sink of everything the subscriber captured during a test. +#[derive(Debug, Default)] +pub(crate) struct Recorder { + spans: Mutex>, + events: Mutex>, +} + +impl Recorder { + /// A snapshot of every captured span. + pub fn spans(&self) -> Vec { + self.spans.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + /// A snapshot of every captured event. + pub fn events(&self) -> Vec { + self.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + /// Every field value recorded across all spans and events, so a test can + /// assert that a secret never appears anywhere in the telemetry. + pub fn all_field_values(&self) -> Vec { + let mut values = Vec::new(); + for span in self.spans().iter() { + values.extend(span.fields.values().cloned()); + } + for event in self.events().iter() { + values.extend(event.fields.values().cloned()); + } + values + } +} + +struct FieldVisitor(HashMap); + +impl Visit for FieldVisitor { + fn record_str(&mut self, field: &Field, value: &str) { + self.0.insert(field.name().to_string(), value.to_string()); + } + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0 + .entry(field.name().to_string()) + .or_insert_with(|| format!("{value:?}")); + } +} + +struct CaptureLayer(Arc); + +impl tracing_subscriber::Layer for CaptureLayer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_new_span(&self, attrs: &Attributes<'_>, _id: &Id, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(HashMap::new()); + attrs.record(&mut visitor); + self.0 + .spans + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedSpan { + name: attrs.metadata().name().to_string(), + fields: visitor.0, + }); + } + + fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(HashMap::new()); + event.record(&mut visitor); + self.0 + .events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedEvent { + level: *event.metadata().level(), + fields: visitor.0, + }); + } +} + +/// Runs `f` with a thread-local capturing subscriber installed and returns the +/// recorder alongside `f`'s result. Because the subscriber is thread-local, +/// any async work driven inside `f` must run on the calling thread (e.g. a +/// current-thread runtime's `block_on`) for its telemetry to be captured. +pub(crate) fn with_recorder(f: impl FnOnce() -> T) -> (Arc, T) { + let recorder = Arc::new(Recorder::default()); + let subscriber = tracing_subscriber::registry().with(CaptureLayer(recorder.clone())); + let value = tracing::subscriber::with_default(subscriber, f); + (recorder, value) +} -- 2.54.0 From da072bf25d1bb368ce088366c758f080055a1185 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 18:59:44 -0500 Subject: [PATCH 40/95] feat(nunciod): domain + lifecycle log coverage across handlers, syncs, outbox, shutdown (#372) Add meaningful domain events to the 8 gRPC services (account add/update/remove/probe, mark-read, sync dispatch, contact create, rule create/update/delete/import/export, triage, export, audit verify), instrument the previously-silent send/calendar_sync/contacts_sync routines, log outbox mutation attempt+result at the executor decision level, and trace every graceful-shutdown drain step plus the ShuttingDown event emission. Domain events inherit the OBS-2 per-RPC span (request_id) rather than duplicating its entry/exit lines. Only ids, counts, transport/op/rule kinds, folder names, and status are logged -- never bodies, subjects, addresses, credentials, or tokens. Refs #359 Co-authored-by: James Maes --- crates/nunciod/src/calendar_sync.rs | 72 ++++++- crates/nunciod/src/contacts_sync.rs | 59 +++++- crates/nunciod/src/grpc.rs | 286 +++++++++++++++++++++++----- crates/nunciod/src/lifecycle.rs | 28 +++ crates/nunciod/src/main.rs | 19 +- crates/nunciod/src/outbox.rs | 181 +++++++++++++++++- crates/nunciod/src/send.rs | 130 ++++++++++++- 7 files changed, 717 insertions(+), 58 deletions(-) diff --git a/crates/nunciod/src/calendar_sync.rs b/crates/nunciod/src/calendar_sync.rs index 3b9c52f..6b698a7 100644 --- a/crates/nunciod/src/calendar_sync.rs +++ b/crates/nunciod/src/calendar_sync.rs @@ -43,14 +43,33 @@ pub async fn sync_with_backend( start_window: i64, end_window: i64, ) -> Result { + tracing::debug!( + calendar_id = %calendar_id, + start_window, + end_window, + "calendar sync: fetching events from backend" + ); let events = backend .fetch_events(calendar_id, start_window, end_window) - .await?; + .await + .inspect_err(|e| { + tracing::warn!( + calendar_id = %calendar_id, + "calendar sync: backend fetch failed: {e}" + ); + })?; + let fetched = events.len(); let mut synced = 0usize; for event in events { db.save_calendar_event(&event).await?; synced += 1; } + tracing::info!( + calendar_id = %calendar_id, + fetched, + synced, + "calendar sync: persisted fetched events" + ); Ok(synced) } @@ -72,6 +91,11 @@ pub async fn sync_caldav_account( start_window: i64, end_window: i64, ) -> Result { + tracing::info!( + account_id = %account.id, + calendar_id = %calendar_id, + "calendar sync: dispatching CalDAV sync for account" + ); let client = CalDavClient::new(CalDavAccountConfig { account_id: account.id.clone(), caldav_url: account.dav_collection_url().unwrap_or_default().to_string(), @@ -170,4 +194,50 @@ mod tests { assert!(matches!(err, CalendarSyncError::Backend(_))); assert!(err.to_string().contains("calendar backend error")); } + + /// A calendar sync must emit a followable INFO event carrying the calendar + /// id and the persisted count, so an operator can trace what a sync did. + #[test] + fn sync_with_backend_logs_calendar_id_and_synced_count() { + use crate::test_tracing::with_recorder; + use tracing::Level; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + + let (recorder, ()) = with_recorder(|| { + runtime.block_on(async { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let mock = MockCalendarBackend::new(); + mock.add_event(mock_event( + "evt-1", + "cal-work", + 1_700_000_000, + 1_700_003_600, + )); + sync_with_backend(&db, &mock, "cal-work", 0, i64::MAX) + .await + .expect("sync succeeds"); + }); + }); + + let events = recorder.events(); + let persisted = events + .iter() + .find(|e| e.message() == "calendar sync: persisted fetched events") + .expect("a domain event for the persisted sync must be logged"); + assert_eq!(persisted.level, Level::INFO); + assert_eq!( + persisted.fields.get("calendar_id").map(String::as_str), + Some("cal-work") + ); + assert_eq!( + persisted.fields.get("synced").map(String::as_str), + Some("1") + ); + } } diff --git a/crates/nunciod/src/contacts_sync.rs b/crates/nunciod/src/contacts_sync.rs index b5cb883..640f8f7 100644 --- a/crates/nunciod/src/contacts_sync.rs +++ b/crates/nunciod/src/contacts_sync.rs @@ -43,12 +43,28 @@ pub async fn sync_with_backend( backend: &dyn ContactsBackend, account_id: &str, ) -> Result { - let contacts = backend.fetch_contacts(account_id).await?; + tracing::info!( + account_id = %account_id, + "contacts sync: dispatching sync for account" + ); + let contacts = backend.fetch_contacts(account_id).await.inspect_err(|e| { + tracing::warn!( + account_id = %account_id, + "contacts sync: backend fetch failed: {e}" + ); + })?; + let fetched = contacts.len(); let mut synced = 0usize; for contact in contacts { db.save_contact(&contact).await?; synced += 1; } + tracing::info!( + account_id = %account_id, + fetched, + synced, + "contacts sync: persisted fetched contacts" + ); Ok(synced) } @@ -120,4 +136,45 @@ mod tests { assert!(matches!(err, ContactsSyncError::Backend(_))); assert!(err.to_string().contains("contacts backend error")); } + + /// A contacts sync must emit a followable INFO event carrying the account + /// id and the persisted count. + #[test] + fn sync_with_backend_logs_account_id_and_synced_count() { + use crate::test_tracing::with_recorder; + use tracing::Level; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + + let (recorder, ()) = with_recorder(|| { + runtime.block_on(async { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let mock = MockContactsBackend::new(); + mock.add_contact(mock_contact("ct-1", "acct-contacts-1", "Alice")); + sync_with_backend(&db, &mock, "acct-contacts-1") + .await + .expect("sync succeeds"); + }); + }); + + let events = recorder.events(); + let persisted = events + .iter() + .find(|e| e.message() == "contacts sync: persisted fetched contacts") + .expect("a domain event for the persisted sync must be logged"); + assert_eq!(persisted.level, Level::INFO); + assert_eq!( + persisted.fields.get("account_id").map(String::as_str), + Some("acct-contacts-1") + ); + assert_eq!( + persisted.fields.get("synced").map(String::as_str), + Some("1") + ); + } } diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 6562824..00659cf 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -68,6 +68,7 @@ use tokio_stream::{Stream, StreamExt}; use tonic::service::interceptor::InterceptedService; use tonic::transport::Server; use tonic::{Request, Response, Status}; +use tracing::Instrument; // Resource bounds applied to the tonic `Server` builder in // `serve_on_listener_with_overrides_and_shutdown`. These exist so a @@ -298,6 +299,17 @@ fn map_tls_mode_from_proto(mode: TlsModeProto) -> Result &'static str { + match transport { + nuncio_core::Transport::ImapSmtp(_) => "imap_smtp", + nuncio_core::Transport::Jmap(_) => "jmap", + nuncio_core::Transport::Dav(_) => "dav", + } +} + /// Maps a `nuncio_core::AccountConfig` onto its wire-format /// `nuncio.v1.AccountConfig` representation for `ListAccounts` responses. /// The password credential is NEVER part of `AccountConfig` (it lives only @@ -662,6 +674,12 @@ impl Accounts for AccountsGrpcService { return Err(Status::internal(message)); } + tracing::info!( + account_id = %config.id, + transport = transport_kind(&config.transport), + "Accounts: account added" + ); + Ok(Response::new(AddAccountResponse { config: Some(map_account_config_to_proto(config)), })) @@ -671,7 +689,7 @@ impl Accounts for AccountsGrpcService { &self, _request: Request, ) -> Result, Status> { - let accounts = self + let accounts: Vec = self .db .list_accounts() .await @@ -680,6 +698,8 @@ impl Accounts for AccountsGrpcService { .map(map_account_config_to_proto) .collect(); + tracing::debug!(count = accounts.len(), "Accounts: listed accounts"); + Ok(Response::new(ListAccountsResponse { accounts })) } @@ -712,6 +732,7 @@ impl Accounts for AccountsGrpcService { ))); } + let password_rotated = matches!(&req.password, Some(p) if !p.is_empty()); match req.password { // A password rotation writes the new credential first, then // persists. Unlike `add_account` (where no prior secret exists, @@ -735,6 +756,13 @@ impl Accounts for AccountsGrpcService { } } + tracing::info!( + account_id = %config.id, + transport = transport_kind(&config.transport), + password_rotated, + "Accounts: account updated" + ); + Ok(Response::new(UpdateAccountResponse {})) } @@ -777,6 +805,8 @@ impl Accounts for AccountsGrpcService { ); } + tracing::info!(account_id = %req.account_id, "Accounts: account removed"); + Ok(Response::new(RemoveAccountResponse {})) } @@ -809,6 +839,13 @@ impl Accounts for AccountsGrpcService { let report = self.connection_tester.probe(&config, &password).await; + tracing::info!( + account_id = %config.id, + imap_ok = report.imap.ok, + smtp_ok = report.smtp.ok, + "Accounts: connection probe finished" + ); + Ok(Response::new(TestAccountConnectionResponse { imap_ok: report.imap.ok, smtp_ok: report.smtp.ok, @@ -1137,6 +1174,12 @@ impl Calendar for CalendarGrpcService { } }; + tracing::info!( + calendar_id = %req.calendar_id, + synced, + "Calendar: sync finished" + ); + Ok(Response::new(CalendarSyncResponse { synced_count: synced as u64, })) @@ -1306,6 +1349,12 @@ impl Contacts for ContactsGrpcService { } }; + tracing::info!( + account_id = %req.account_id, + synced, + "Contacts: sync finished" + ); + Ok(Response::new(ContactsSyncResponse { synced_count: synced as u64, })) @@ -1421,6 +1470,12 @@ impl Contacts for ContactsGrpcService { .await .map_err(|e| Status::internal(format!("failed to save contact: {e}")))?; + tracing::info!( + contact_id = %contact.id, + account_id = ?contact.account_id, + "Contacts: contact created" + ); + Ok(Response::new(CreateContactResponse { contact: Some(map_contact_to_proto(contact)), })) @@ -1601,6 +1656,12 @@ impl Mail for MailGrpcService { ) })?; + tracing::info!( + message_id = %req.message_id, + read = req.read, + "Mail: mark-read applied" + ); + self.event_bus.process_command(CoreCommand::MarkRead { message_id: req.message_id, read: req.read, @@ -1724,6 +1785,11 @@ impl Mail for MailGrpcService { async fn sync(&self, request: Request) -> Result, Status> { let account_id = request.into_inner().account_id; + match &account_id { + Some(id) => tracing::info!(account_id = %id, "Mail: dispatching sync for account"), + None => tracing::info!("Mail: dispatching sync for all configured accounts"), + } + let synced = if let Some(backend) = &self.overrides.mail_backend { crate::sync::sync_with_backend( &self.db, @@ -1743,6 +1809,8 @@ impl Mail for MailGrpcService { sync_all_configured(&self.sync_dispatcher, &self.db, &self.event_bus).await }; + tracing::info!(synced, "Mail: sync finished"); + Ok(Response::new(SyncResponse { synced_count: synced as u64, })) @@ -1906,6 +1974,8 @@ impl Filters for FiltersGrpcService { self.reload_engine_from_store().await; + tracing::info!(rule_id = %rule.id, "Filters: rule created"); + Ok(Response::new(CreateRuleResponse { rule: Some(map_filter_rule_to_proto(rule)), })) @@ -1957,6 +2027,8 @@ impl Filters for FiltersGrpcService { self.reload_engine_from_store().await; + tracing::info!(rule_id = %req.rule_id, "Filters: rule deleted"); + Ok(Response::new(DeleteRuleResponse {})) } @@ -2082,6 +2154,8 @@ impl Filters for FiltersGrpcService { self.reload_engine_from_store().await; + tracing::info!(rule_id = %rule.id, "Filters: rule updated"); + Ok(Response::new(UpdateRuleResponse { rule: Some(map_filter_rule_to_proto(rule)), })) @@ -2117,6 +2191,12 @@ impl Filters for FiltersGrpcService { } }; + tracing::info!( + format = ?format, + rule_count = rules.len(), + "Filters: rules exported" + ); + Ok(Response::new(ExportRulesResponse { content })) } @@ -2156,6 +2236,12 @@ impl Filters for FiltersGrpcService { self.reload_engine_from_store().await; + tracing::info!( + imported_count, + error_count = errors.len(), + "Filters: rules imported" + ); + Ok(Response::new(ImportRulesResponse { imported_count, errors, @@ -2214,66 +2300,80 @@ impl Filters for FiltersGrpcService { req.chunk_size as usize }; + tracing::info!(chunk_size, "Filters: triage rescan started"); + let (tx, rx) = tokio::sync::mpsc::channel::>(16); let db = self.db.clone(); let engine = self.filter_engine.clone(); + // Carry the per-RPC span into the detached worker so its completion + // event inherits the same request_id as the RPC that started the scan. + let triage_span = tracing::Span::current(); - tokio::spawn(async move { - let mut last_id = String::new(); - let mut scanned: u64 = 0; - let mut matched: u64 = 0; - let mut applied: u64 = 0; + tokio::spawn( + async move { + let mut last_id = String::new(); + let mut scanned: u64 = 0; + let mut matched: u64 = 0; + let mut applied: u64 = 0; - loop { - let batch = match db.get_message_chunk(&last_id, chunk_size).await { - Ok(batch) => batch, - Err(e) => { - let _ = tx - .send(Err(Status::internal(format!( - "failed to read message chunk during triage: {e}" - )))) - .await; + loop { + let batch = match db.get_message_chunk(&last_id, chunk_size).await { + Ok(batch) => batch, + Err(e) => { + let _ = tx + .send(Err(Status::internal(format!( + "failed to read message chunk during triage: {e}" + )))) + .await; + return; + } + }; + let is_last_page = batch.len() < chunk_size; + + for email in &batch { + let actions_for_email = + crate::sync::apply_filter_actions(&db, &engine, email).await; + scanned += 1; + applied += actions_for_email as u64; + if actions_for_email > 0 { + matched += 1; + } + } + + if let Some(last) = batch.last() { + last_id = last.id.clone(); + } + + let done = is_last_page; + if tx + .send(Ok(TriageProgress { + scanned_count: scanned, + matched_count: matched, + actions_applied_count: applied, + last_message_id: last_id.clone(), + done, + })) + .await + .is_err() + { + // Receiver dropped: the caller disconnected, so there is + // no one left to report progress to. return; } - }; - let is_last_page = batch.len() < chunk_size; - for email in &batch { - let actions_for_email = - crate::sync::apply_filter_actions(&db, &engine, email).await; - scanned += 1; - applied += actions_for_email as u64; - if actions_for_email > 0 { - matched += 1; + if is_last_page { + tracing::info!( + scanned, + matched, + applied, + "Filters: triage rescan finished" + ); + return; } } - - if let Some(last) = batch.last() { - last_id = last.id.clone(); - } - - let done = is_last_page; - if tx - .send(Ok(TriageProgress { - scanned_count: scanned, - matched_count: matched, - actions_applied_count: applied, - last_message_id: last_id.clone(), - done, - })) - .await - .is_err() - { - // Receiver dropped: the caller disconnected, so there is - // no one left to report progress to. - return; - } - - if is_last_page { - return; - } } - }); + .instrument(triage_span), + ); Ok(Response::new(Box::pin(ReceiverStream::new(rx)))) } @@ -2323,6 +2423,13 @@ impl Export for ExportGrpcService { } let format = map_export_format_from_proto(req.format())?; + let scope_kind = match &req.scope { + Some(export_request::Scope::AccountId(_)) => "account", + Some(export_request::Scope::FolderId(_)) => "folder", + None => "all", + }; + tracing::info!(format = ?format, scope = scope_kind, "Export: mailbox export started"); + let messages = match req.scope { Some(export_request::Scope::AccountId(account_id)) => { self.db @@ -2345,6 +2452,12 @@ impl Export for ExportGrpcService { .await .map_err(|e| Status::internal(format!("export failed: {e}")))?; + tracing::info!( + message_count = summary.message_count, + bytes_written = summary.bytes_written, + "Export: mailbox export finished" + ); + Ok(Response::new(ExportResponse { output_path: summary.output_path, message_count: summary.message_count as u64, @@ -2427,6 +2540,12 @@ impl Audit for AuditGrpcService { .await .map_err(|e| Status::internal(format!("failed to verify audit chain: {e}")))?; + tracing::info!( + valid = report.valid, + record_count = report.record_count, + "Audit: chain verification finished" + ); + Ok(Response::new(VerifyChainResponse { valid: report.valid, record_count: report.record_count as u64, @@ -2929,6 +3048,73 @@ mod tests { request } + /// A state-changing RPC must emit its own domain event, and (because every + /// handler event inherits the OBS-2 per-RPC span) that event must carry the + /// request-scoped `request_id`. Here the `Filters::create_rule` handler is + /// invoked inside a stand-in request span to prove the domain event both + /// fires and inherits the correlation id -- without duplicating the layer's + /// bare entry/exit lines. + #[test] + fn create_rule_handler_logs_domain_event_under_request_scope() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + + let (recorder, ()) = with_recorder(|| { + runtime.block_on(async { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let db = Arc::new(db); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rules")); + let service = FiltersGrpcService { db, filter_engine }; + + // Stand in for the RpcTraceLayer's per-RPC span so the handler's + // domain event inherits a request_id, exactly as in production. + let span = tracing::info_span!("grpc.request", request_id = "test-req-42"); + async { + service + .create_rule(Request::new(CreateRuleRequest { + name: "Domain Event Rule".to_string(), + priority: 1, + nsql: SAMPLE_RULE_NSQL.to_string(), + })) + .await + .expect("rule creation succeeds"); + } + .instrument(span) + .await; + }); + }); + + let events = recorder.events(); + let created = events + .iter() + .find(|e| e.message() == "Filters: rule created") + .expect("the create_rule handler must emit its domain event"); + assert_eq!(created.level, Level::INFO); + assert!( + created.fields.contains_key("rule_id"), + "the domain event must carry the rule id" + ); + + // The domain event was emitted while the handler ran inside the stand-in + // request span, so an operator can correlate it back to that request via + // the span's `request_id` (the same mechanism the RpcTraceLayer provides + // in production -- see `rpc_trace`'s own test). + let request_span = recorder + .spans() + .into_iter() + .find(|s| s.name == "grpc.request") + .expect("the request span must be recorded"); + assert_eq!( + request_span.fields.get("request_id").map(String::as_str), + Some("test-req-42"), + "the request span must carry the request-scoped correlation id" + ); + } + #[test] fn auth_accept_logs_debug_without_leaking_the_token() { let (recorder, result) = with_recorder(|| { diff --git a/crates/nunciod/src/lifecycle.rs b/crates/nunciod/src/lifecycle.rs index 107fd09..ce108bf 100644 --- a/crates/nunciod/src/lifecycle.rs +++ b/crates/nunciod/src/lifecycle.rs @@ -88,7 +88,12 @@ impl ShutdownController { /// Initiates shutdown. Idempotent: calling it more than once (e.g. a /// second Ctrl+C while already draining) is a no-op beyond the first. pub fn trigger(&self) { + // Drives the event bus to `ShuttingDown` and publishes the + // `CoreEvent::ShuttingDown` event to every subscriber (e.g. clients on + // `System/Subscribe`), so a graceful shutdown is observable over the + // API and not just in local logs. self.event_bus.process_command(CoreCommand::Shutdown); + tracing::info!("shutdown: ShuttingDown event published to subscribers"); let _ = self.tx.send(true); } } @@ -166,6 +171,29 @@ mod tests { assert!(late_clone.is_requested()); } + /// Triggering shutdown must both publish the `ShuttingDown` event (flipping + /// bus state) and emit a traceable lifecycle log line, so a clean shutdown + /// is observable in the logs and not only over the API. + #[test] + fn trigger_emits_a_traceable_shutting_down_lifecycle_log() { + use crate::test_tracing::with_recorder; + + let (recorder, ()) = with_recorder(|| { + let event_bus = Arc::new(EventBus::new()); + let (controller, _signal) = ShutdownController::new(event_bus.clone()); + controller.trigger(); + assert_eq!(event_bus.current_state().status, EngineStatus::ShuttingDown); + }); + + assert!( + recorder + .events() + .iter() + .any(|e| e.message() == "shutdown: ShuttingDown event published to subscribers"), + "trigger must emit the ShuttingDown lifecycle log" + ); + } + #[tokio::test] async fn trigger_is_idempotent() { let event_bus = Arc::new(EventBus::new()); diff --git a/crates/nunciod/src/main.rs b/crates/nunciod/src/main.rs index 3b9a892..134a805 100644 --- a/crates/nunciod/src/main.rs +++ b/crates/nunciod/src/main.rs @@ -308,10 +308,17 @@ async fn main() -> Result<(), Box> { } }; + // From here down is the graceful-shutdown drain: the gRPC server has + // stopped accepting new connections, so log each remaining teardown step + // to make a clean shutdown fully traceable end to end. + tracing::info!( + "shutdown: gRPC server stopped accepting connections; draining background workers" + ); + // Bound how long the background workers get to notice the shutdown // signal and exit their loops before the database is closed out from // under them. - let _ = tokio::time::timeout(SHUTDOWN_GRACE_PERIOD, async { + let workers_drained = tokio::time::timeout(SHUTDOWN_GRACE_PERIOD, async { if let Some(task) = sync_command_task { let _ = task.await; } @@ -321,12 +328,22 @@ async fn main() -> Result<(), Box> { } }) .await; + match workers_drained { + Ok(()) => tracing::info!("shutdown: background workers (sync, outbox, update) drained"), + Err(_) => tracing::warn!( + "shutdown: background workers did not all exit within the {:?} grace period; \ + proceeding to close the database", + SHUTDOWN_GRACE_PERIOD + ), + } signal_task.abort(); // Force the WAL checkpoint before the process exits so no committed // page is left for SQLite's own deferred "last connection closes" // teardown to race (see `DatabaseEngine::close`'s doc comment). + tracing::info!("shutdown: closing database and checkpointing the WAL"); db.close().await; + tracing::info!("shutdown: complete"); grpc_result.map_err(|e| format!("nunciod gRPC server failed: {e}"))?; diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 813a659..7de867e 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -155,6 +155,16 @@ pub async fn execute_pending_mutations( let mut summary = ExecutionSummary::default(); for item in pending { + // Log the executor's decision-level attempt before issuing the real + // remote op: the mutation identity, its operation, and the surrogate + // message id it targets (never the message body/subject/recipients). + tracing::info!( + mutation_id = %item.id, + op = %item.mutation_type, + message_id = %item.message_id, + retry_count = item.retry_count, + "outbox: attempting mutation" + ); let outcome = tokio::select! { biased; _ = shutdown.wait() => { @@ -174,7 +184,14 @@ pub async fn execute_pending_mutations( }; let next_retry = item.retry_count + 1; let (status, retry, tally): (&str, i32, fn(&mut ExecutionSummary)) = match disposition { - Disposition::Completed => ("completed", item.retry_count, |s| s.completed += 1), + Disposition::Completed => { + tracing::info!( + mutation_id = %item.id, + op = %item.mutation_type, + "outbox: mutation completed" + ); + ("completed", item.retry_count, |s| s.completed += 1) + } Disposition::Permanent(reason) => { tracing::warn!( mutation_id = %item.id, @@ -298,6 +315,14 @@ async fn dispatch_mailbox_mutation( kind, }; + tracing::debug!( + op = %mutation.mutation_type, + remote_id = %spec.remote_id, + folder_id = %spec.folder_id, + uid_validity = %spec.uid_validity, + "outbox: issuing mailbox mutation against remote backend" + ); + match backend.apply_mutation(&spec).await { Ok(()) => Disposition::Completed, Err(e) => Disposition::Retry(format!("remote mutation failed: {e}")), @@ -505,3 +530,157 @@ impl RemoteExecutionEnv for ProductionExecutionEnv { .await } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::lifecycle::{ShutdownController, ShutdownSignal}; + use crate::test_tracing::with_recorder; + use nuncio_core::model::Email; + use nuncio_core::EventBus; + use nuncio_filter::MutationPayload; + use nuncio_mail::MockMailBackend; + use tracing::Level; + + /// Minimal test [`RemoteExecutionEnv`] whose mailbox mutations always + /// genuinely succeed against a recording [`MockMailBackend`]; the forward + /// and webhook paths are unused by these tests. + struct SucceedingEnv { + backend: MockMailBackend, + } + + #[async_trait] + impl RemoteExecutionEnv for SucceedingEnv { + async fn mail_backend( + &self, + _account_id: &str, + ) -> Result, OutboxExecuteError> { + Ok(Box::new(self.backend.clone())) + } + + async fn message_sender( + &self, + account_id: &str, + ) -> Result, OutboxExecuteError> { + // The FLAG-only tests never take the forward path; surface an error + // rather than a panic if that assumption ever changes. + Err(OutboxExecuteError::NotAMailAccount(account_id.to_string())) + } + + async fn dispatch_webhook( + &self, + _url: &str, + _rule_id: &str, + _message_id: &str, + _subject: &str, + _sender: &str, + ) -> Result { + // The FLAG-only tests never take the webhook path. + Err(WebhookError::SigningError( + "webhook path unused in test".into(), + )) + } + } + + fn no_shutdown() -> (ShutdownController, ShutdownSignal) { + ShutdownController::new(Arc::new(EventBus::new())) + } + + fn sample_email() -> Email { + Email { + id: "msg-outbox-log".to_string(), + account_id: "acct-outbox-log".to_string(), + folder_id: "INBOX".to_string(), + remote_id: "77".to_string(), + uid_validity: "9".to_string(), + subject: "Confidential outbox subject".to_string(), + sender: "alice@example.com".to_string(), + recipient: "owner@nuncio.mx".to_string(), + received_at: 1_700_000_000, + read: false, + body_plain: Some("secret outbox body".to_string()), + body_html: None, + attachments: Vec::new(), + } + } + + fn flag_mutation() -> PendingRemoteMutation { + PendingRemoteMutation { + id: "mut-1".to_string(), + rule_id: "rule-1".to_string(), + message_id: "msg-outbox-log".to_string(), + mutation_type: "FLAG".to_string(), + payload: serde_json::to_string(&MutationPayload { + action_type: "FLAG".to_string(), + target: None, + }) + .expect("serialize payload"), + status: "pending".to_string(), + retry_count: 0, + created_at: 1_700_000_000, + } + } + + /// An outbox drain must log the executor's decision-level attempt BEFORE + /// issuing the remote op and the Completed result AFTER, carrying the + /// mutation id and operation -- and never the message body or subject. + #[test] + fn drain_logs_attempt_and_completion_without_leaking_message_content() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + + let (recorder, ()) = with_recorder(|| { + runtime.block_on(async { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + db.save_folder_sync_state("acct-outbox-log", "INBOX", "9:100") + .await + .expect("save folder checkpoint"); + db.save_email(&sample_email()).await.expect("save email"); + db.save_pending_mutation(&flag_mutation()) + .await + .expect("save pending mutation"); + + let env = SucceedingEnv { + backend: MockMailBackend::new(), + }; + let (_ctl, mut shutdown) = no_shutdown(); + let summary = execute_pending_mutations(&db, &env, 10, &mut shutdown).await; + assert_eq!(summary.completed, 1, "the mutation must complete"); + }); + }); + + let events = recorder.events(); + let attempt = events + .iter() + .find(|e| e.message() == "outbox: attempting mutation") + .expect("an attempt event must be logged before issuing the op"); + assert_eq!(attempt.level, Level::INFO); + assert_eq!(attempt.fields.get("op").map(String::as_str), Some("FLAG")); + assert_eq!( + attempt.fields.get("mutation_id").map(String::as_str), + Some("mut-1") + ); + + let completed = events + .iter() + .find(|e| e.message() == "outbox: mutation completed") + .expect("a completion event must be logged after a genuine success"); + assert_eq!(completed.level, Level::INFO); + assert_eq!(completed.fields.get("op").map(String::as_str), Some("FLAG")); + + for value in recorder.all_field_values() { + assert!( + !value.contains("Confidential outbox subject"), + "subject leaked into telemetry: {value}" + ); + assert!( + !value.contains("secret outbox body"), + "body leaked into telemetry: {value}" + ); + } + } +} diff --git a/crates/nunciod/src/send.rs b/crates/nunciod/src/send.rs index eebcba4..de07d15 100644 --- a/crates/nunciod/src/send.rs +++ b/crates/nunciod/src/send.rs @@ -183,6 +183,15 @@ pub async fn send_message_for_account( request: ComposeRequest, ) -> Result { let config = resolve_sending_account(db, request.account_id.as_deref()).await?; + let account_id = config.id.clone(); + let attachment_count = request.attachments.len(); + tracing::info!( + account_id = %account_id, + transport = "smtp", + has_cc = request.cc.is_some(), + attachment_count, + "mail send: dispatching outbound message over real SMTP transport" + ); let password = secrets.get_secret(&config.keyring_secret_key)?; let sender = build_smtp_sender(&config, &password)?; @@ -196,8 +205,24 @@ pub async fn send_message_for_account( attachments: request.attachments, }; - send_with_sender(&sender, &outbound).await?; - Ok(generate_message_id()) + match send_with_sender(&sender, &outbound).await { + Ok(()) => { + let message_id = generate_message_id(); + tracing::info!( + account_id = %account_id, + message_id = %message_id, + "mail send: outbound message accepted by SMTP transport" + ); + Ok(message_id) + } + Err(e) => { + tracing::warn!( + account_id = %account_id, + "mail send: SMTP transport rejected the outbound message: {e}" + ); + Err(e) + } + } } /// Test/E2E entry point for the `SendMessage` RPC when a @@ -215,6 +240,14 @@ pub async fn send_message_with_injected_sender( request: ComposeRequest, ) -> Result { let config = resolve_sending_account(db, request.account_id.as_deref()).await?; + let account_id = config.id.clone(); + tracing::info!( + account_id = %account_id, + transport = "injected", + has_cc = request.cc.is_some(), + attachment_count = request.attachments.len(), + "mail send: dispatching outbound message via injected sender" + ); let outbound = OutboundMessage { from: config.email_address, @@ -226,8 +259,24 @@ pub async fn send_message_with_injected_sender( attachments: request.attachments, }; - send_with_sender(sender, &outbound).await?; - Ok(generate_message_id()) + match send_with_sender(sender, &outbound).await { + Ok(()) => { + let message_id = generate_message_id(); + tracing::info!( + account_id = %account_id, + message_id = %message_id, + "mail send: outbound message accepted by injected sender" + ); + Ok(message_id) + } + Err(e) => { + tracing::warn!( + account_id = %account_id, + "mail send: injected sender rejected the outbound message: {e}" + ); + Err(e) + } + } } #[cfg(test)] @@ -534,4 +583,77 @@ mod tests { assert!(matches!(err, SendError::AccountNotFound(id) if id == "acct-does-not-exist")); assert!(mock.sent_messages().is_empty()); } + + /// A send must be followable in the log: a dispatch event before the send + /// and an acceptance event after, both scoped to the account id -- and the + /// recipient address, subject, and body must never appear in any logged + /// field. + #[test] + fn send_logs_dispatch_and_acceptance_without_leaking_recipient_or_subject() { + use crate::test_tracing::with_recorder; + use tracing::Level; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + + let (recorder, ()) = with_recorder(|| { + runtime.block_on(async { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let config = sample_account("acct-log"); + db.save_account(&config).await.expect("save account"); + + let mock = MockMessageSender::new(); + send_message_with_injected_sender( + &db, + &mock, + ComposeRequest { + to: "secret-recipient@example.com".to_string(), + cc: None, + subject: "Confidential Subject Line".to_string(), + body_text: Some("secret body".to_string()), + body_html: None, + attachments: Vec::new(), + account_id: None, + }, + ) + .await + .expect("send succeeds"); + }); + }); + + let events = recorder.events(); + let dispatch = events + .iter() + .find(|e| e.message() == "mail send: dispatching outbound message via injected sender") + .expect("dispatch event must be logged"); + assert_eq!(dispatch.level, Level::INFO); + assert_eq!( + dispatch.fields.get("account_id").map(String::as_str), + Some("acct-log") + ); + assert!(events + .iter() + .any(|e| e.message() == "mail send: outbound message accepted by injected sender")); + + // Secret-safety: no recipient, subject, or body text may appear in any + // recorded telemetry field. + for value in recorder.all_field_values() { + assert!( + !value.contains("secret-recipient@example.com"), + "recipient leaked into telemetry: {value}" + ); + assert!( + !value.contains("Confidential Subject Line"), + "subject leaked into telemetry: {value}" + ); + assert!( + !value.contains("secret body"), + "body leaked into telemetry: {value}" + ); + } + } } -- 2.54.0 From ba2eb726d9b2439308e9deba1d93e51f81404aef Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 19:17:45 -0500 Subject: [PATCH 41/95] fix(cal): surface dropped VEVENTs; instrument DAV syncs + store lifecycle; redact DAV config Debug (#373) An unparseable VEVENT inside a CalDAV multistatus response was silently discarded, so a partial sync failure looked identical to a calendar with fewer events. It now logs a WARN with the VEVENT's UID (if extractable) and a short parse-error reason, never the raw calendar body. CalDAV and CardDAV round trips are wrapped in spans and log completion counts (fetched/parsed/dropped) keyed by account/calendar id only. Additive SQLite column migrations and first-use vault key minting now log at INFO, naming the table/column or key purpose without ever exposing key material. CalDavAccountConfig and CardDavAccountConfig held an auth_token behind a derived Debug, which would have printed the raw token verbatim from any future debug! log or panic message. Both now use a manual Debug impl that redacts the token. Refs #362 Co-authored-by: James Maes --- Cargo.lock | 2 + crates/nuncio-cal/Cargo.toml | 1 + crates/nuncio-cal/src/caldav.rs | 246 ++++++++++++++++++++- crates/nuncio-cal/src/lib.rs | 2 + crates/nuncio-cal/src/test_tracing.rs | 134 +++++++++++ crates/nuncio-contacts/Cargo.toml | 1 + crates/nuncio-contacts/src/carddav.rs | 113 +++++++++- crates/nuncio-contacts/src/lib.rs | 2 + crates/nuncio-contacts/src/test_tracing.rs | 135 +++++++++++ crates/nuncio-store/src/db.rs | 136 ++++++++++++ crates/nuncio-store/src/vault.rs | 57 +++++ 11 files changed, 821 insertions(+), 8 deletions(-) create mode 100644 crates/nuncio-cal/src/test_tracing.rs create mode 100644 crates/nuncio-contacts/src/test_tracing.rs diff --git a/Cargo.lock b/Cargo.lock index b4131eb..1bde572 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2507,6 +2507,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tracing", + "tracing-subscriber", "wiremock", ] @@ -2552,6 +2553,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tracing", + "tracing-subscriber", "wiremock", ] diff --git a/crates/nuncio-cal/Cargo.toml b/crates/nuncio-cal/Cargo.toml index 4696177..f329efd 100644 --- a/crates/nuncio-cal/Cargo.toml +++ b/crates/nuncio-cal/Cargo.toml @@ -26,3 +26,4 @@ quick-xml.workspace = true [dev-dependencies] tempfile.workspace = true wiremock.workspace = true +tracing-subscriber.workspace = true diff --git a/crates/nuncio-cal/src/caldav.rs b/crates/nuncio-cal/src/caldav.rs index 3d1f888..055334e 100644 --- a/crates/nuncio-cal/src/caldav.rs +++ b/crates/nuncio-cal/src/caldav.rs @@ -3,6 +3,7 @@ use async_trait::async_trait; use nuncio_core::model::CalendarEvent; use serde::{Deserialize, Serialize}; +use tracing::{debug, info, instrument, warn}; use crate::backend::CalendarBackend; use crate::parser::{CalendarError, IcalParserAdapter}; @@ -13,7 +14,7 @@ use crate::parser::{CalendarError, IcalParserAdapter}; /// `caldav_url` must already resolve to a specific calendar collection (e.g. /// `https://caldav.example.com/dav/calendars/user/jmaes/work/`) -- PROPFIND-based /// `calendar-home-set` auto-discovery is out of scope for this client. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct CalDavAccountConfig { /// Nuncio account identifier this calendar collection belongs to. pub account_id: String, @@ -26,6 +27,36 @@ pub struct CalDavAccountConfig { pub auth_token: String, } +impl std::fmt::Debug for CalDavAccountConfig { + /// Manual `Debug` impl that redacts `auth_token` -- a derived impl would print the raw + /// secret verbatim the moment anything (a `debug!` log, an assertion failure message, + /// a panic payload) formats this config with `{:?}`. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CalDavAccountConfig") + .field("account_id", &self.account_id) + .field("caldav_url", &self.caldav_url) + .field("username", &self.username) + .field("auth_token", &"") + .finish() + } +} + +/// Best-effort extraction of the `UID` property from a raw (possibly malformed) VEVENT/ +/// VCALENDAR block, used only to correlate a dropped-VEVENT log line with the source event -- +/// never returns the summary, attendees, description, or any other calendar content. +fn extract_vevent_uid(raw_ics: &str) -> Option { + raw_ics.lines().find_map(|line| { + let line = line.trim(); + let (name, value) = line.split_once(':')?; + let bare_name = name.split(';').next().unwrap_or(name); + if bare_name.eq_ignore_ascii_case("UID") { + Some(value.trim().to_string()) + } else { + None + } + }) +} + /// CalDAV client protocol engine managing WebDAV REPORT queries against a real server. pub struct CalDavClient { config: CalDavAccountConfig, @@ -66,32 +97,59 @@ impl CalDavClient { } /// Parse CalDAV WebDAV XML `` response containing embedded VEVENT data. + /// + /// A VEVENT block that fails to parse is dropped from the returned events but never + /// silently: it is logged at `warn` (with its UID, if extractable, and a short parse-error + /// reason -- never the raw VEVENT body) so a sync that quietly lost calendar data is + /// visible in telemetry instead of just showing up as a smaller-than-expected event count. pub fn parse_multistatus_response( &self, calendar_id: &str, raw_xml: &str, ) -> Result, CalendarError> { + let (events, _dropped) = self.parse_multistatus_response_counted(calendar_id, raw_xml); + Ok(events) + } + + /// Same parse as [`Self::parse_multistatus_response`], additionally reporting how many + /// VEVENT blocks were dropped as unparseable so callers can log round-trip counts. + fn parse_multistatus_response_counted( + &self, + calendar_id: &str, + raw_xml: &str, + ) -> (Vec, usize) { let mut events = Vec::new(); + let mut dropped = 0usize; // Extract or text blocks for block in raw_xml.split("") { if let Some((ics_data, _)) = block.split_once("") { let clean_ics = ics_data.trim(); if !clean_ics.is_empty() { - let event_id = format!("caldav-evt-{}", events.len() + 1); - if let Ok(event) = IcalParserAdapter::parse_ical( + let event_id = format!("caldav-evt-{}", events.len() + dropped + 1); + match IcalParserAdapter::parse_ical( &event_id, &self.config.account_id, calendar_id, clean_ics, ) { - events.push(event); + Ok(event) => events.push(event), + Err(err) => { + dropped += 1; + warn!( + account_id = %self.config.account_id, + calendar_id = %calendar_id, + uid = %extract_vevent_uid(clean_ics).unwrap_or_else(|| "unknown".to_string()), + reason = %err, + "dropped unparseable VEVENT" + ); + } } } } } - Ok(events) + (events, dropped) } /// Format a unix timestamp as the `YYYYMMDDTHHMMSSZ` form RFC 4791's @@ -113,6 +171,11 @@ impl CalDavClient { /// returned. A server or network failure surfaces as [`CalendarError::TransportFailed`]; /// a malformed TZID inside a returned VEVENT surfaces as /// [`CalendarError::UnresolvableTimezone`] via [`IcalParserAdapter::parse_ical`]. + #[instrument( + name = "caldav_sync", + skip(self), + fields(account_id = %self.config.account_id, calendar_id = %calendar_id) + )] pub async fn fetch_remote_events( &self, calendar_id: &str, @@ -128,6 +191,12 @@ impl CalDavClient { let report_method = reqwest::Method::from_bytes(b"REPORT") .map_err(|e| CalendarError::TransportFailed(format!("invalid HTTP method: {e}")))?; + debug!( + account_id = %self.config.account_id, + calendar_id = %calendar_id, + "issuing CalDAV REPORT calendar-query" + ); + let response = self .http .request(report_method, &self.config.caldav_url) @@ -153,7 +222,17 @@ impl CalDavClient { .await .map_err(|e| CalendarError::TransportFailed(e.to_string()))?; - self.parse_multistatus_response(calendar_id, &raw_xml) + let (events, dropped) = self.parse_multistatus_response_counted(calendar_id, &raw_xml); + info!( + account_id = %self.config.account_id, + calendar_id = %calendar_id, + fetched = events.len() + dropped, + parsed = events.len(), + dropped, + "completed CalDAV sync" + ); + + Ok(events) } } @@ -221,10 +300,165 @@ END:VCALENDAR assert_eq!(events[0].location, Some("Boardroom".to_string())); } + #[test] + fn parse_multistatus_response_surfaces_a_warn_for_a_dropped_vevent_instead_of_silence() { + use crate::test_tracing::with_recorder; + + let client = CalDavClient::new(test_config()); + // A TZID that cannot be resolved to a real IANA zone; the VEVENT fails to parse but + // the one alongside it (plain UTC) must still come through. + let xml_response = r#" + + + + + BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +UID:evt-unresolvable-tzid +SUMMARY:Confidential Merger Talks +DTSTART;TZID=Not/ARealZone:20240101T090000 +DTEND;TZID=Not/ARealZone:20240101T100000 +END:VEVENT +END:VCALENDAR + + + + + + + BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +SUMMARY:Good Event +DTSTART:20240301T120000Z +DTEND:20240301T130000Z +END:VEVENT +END:VCALENDAR + + + + "#; + + let (recorder, result) = + with_recorder(|| client.parse_multistatus_response("cal-work", xml_response)); + let events = result.expect("the unparseable VEVENT is dropped, not fatal"); + assert_eq!(events.len(), 1); + assert_eq!(events[0].summary, "Good Event"); + + let warnings: Vec<_> = recorder + .events() + .into_iter() + .filter(|e| e.level == tracing::Level::WARN) + .collect(); + assert_eq!(warnings.len(), 1, "exactly one VEVENT should be dropped"); + assert_eq!( + warnings[0].fields.get("uid").map(String::as_str), + Some("evt-unresolvable-tzid") + ); + assert!(warnings[0].message().contains("dropped")); + + // The dropped VEVENT's summary (potential PII) must never appear in telemetry. + for value in recorder.all_field_values() { + assert!(!value.contains("Confidential Merger Talks")); + } + } + #[test] fn format_query_timestamp_produces_ical_utc_form() { let formatted = CalDavClient::format_query_timestamp(1_704_067_200).expect("valid timestamp"); assert_eq!(formatted, "20240101T000000Z"); } + + #[test] + fn debug_impl_redacts_auth_token() { + let config = test_config(); + let rendered = format!("{config:?}"); + assert!(!rendered.contains("app-token-secret")); + assert!(rendered.contains("")); + } + + #[test] + fn fetch_remote_events_logs_completion_counts_including_a_drop() { + use crate::test_tracing::with_recorder; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime"); + + let (recorder, result) = with_recorder(|| { + runtime.block_on(async { + let mock_server = MockServer::start().await; + let xml_response = r#" + + + + + BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +UID:evt-bad +DTSTART;TZID=Not/ARealZone:20240101T090000 +END:VEVENT +END:VCALENDAR + + + + + + + BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +SUMMARY:Good Event +DTSTART:20240301T120000Z +DTEND:20240301T130000Z +END:VEVENT +END:VCALENDAR + + + + "#; + + Mock::given(method("REPORT")) + .and(path("/calendars/work/")) + .respond_with(ResponseTemplate::new(207).set_body_string(xml_response)) + .mount(&mock_server) + .await; + + let mut config = test_config(); + config.caldav_url = format!("{}/calendars/work/", mock_server.uri()); + let client = CalDavClient::new(config); + + client + .fetch_remote_events("cal-work", 1_700_000_000, 1_800_000_000) + .await + }) + }); + + let events = result.expect("one good VEVENT still comes through"); + assert_eq!(events.len(), 1); + + let completion = recorder + .events() + .into_iter() + .find(|e| e.message().contains("completed CalDAV sync")) + .expect("a completion event is logged"); + assert_eq!( + completion.fields.get("fetched").map(String::as_str), + Some("2") + ); + assert_eq!( + completion.fields.get("parsed").map(String::as_str), + Some("1") + ); + assert_eq!( + completion.fields.get("dropped").map(String::as_str), + Some("1") + ); + } } diff --git a/crates/nuncio-cal/src/lib.rs b/crates/nuncio-cal/src/lib.rs index 5943a0a..058ccaa 100644 --- a/crates/nuncio-cal/src/lib.rs +++ b/crates/nuncio-cal/src/lib.rs @@ -11,6 +11,8 @@ pub mod nlp; pub mod parser; pub mod rrule; pub mod scheduling; +#[cfg(test)] +mod test_tracing; pub use backend::CalendarBackend; pub use caldav::{CalDavAccountConfig, CalDavClient}; diff --git a/crates/nuncio-cal/src/test_tracing.rs b/crates/nuncio-cal/src/test_tracing.rs new file mode 100644 index 0000000..178898c --- /dev/null +++ b/crates/nuncio-cal/src/test_tracing.rs @@ -0,0 +1,134 @@ +//! Shared in-test `tracing` capture helper. +//! +//! Installs a thread-local subscriber that records every span (with its +//! fields) and every event (with its level, fields, and message) into an +//! in-memory buffer, so tests can assert on what the code logged -- including +//! the negative assertion that a secret never appears in any recorded field. +//! Deterministic and fully offline: no global subscriber, no I/O. + +#![cfg(test)] + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use tracing::field::{Field, Visit}; +use tracing::span::Attributes; +use tracing::{Event, Id, Level, Subscriber}; +use tracing_subscriber::layer::Context as LayerContext; +use tracing_subscriber::prelude::*; +use tracing_subscriber::registry::LookupSpan; + +/// A captured span: its name and the string-rendered fields it opened with. +#[derive(Debug, Clone)] +pub(crate) struct RecordedSpan { + #[allow(dead_code)] + pub name: String, + pub fields: HashMap, +} + +/// A captured event: its level and its string-rendered fields (including the +/// `message`). +#[derive(Debug, Clone)] +pub(crate) struct RecordedEvent { + pub level: Level, + pub fields: HashMap, +} + +impl RecordedEvent { + /// The event's `message` field, or the empty string if it had none. + pub fn message(&self) -> &str { + self.fields.get("message").map(String::as_str).unwrap_or("") + } +} + +/// In-memory sink of everything the subscriber captured during a test. +#[derive(Debug, Default)] +pub(crate) struct Recorder { + spans: Mutex>, + events: Mutex>, +} + +impl Recorder { + /// A snapshot of every captured span. + pub fn spans(&self) -> Vec { + self.spans.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + /// A snapshot of every captured event. + pub fn events(&self) -> Vec { + self.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + /// Every field value recorded across all spans and events, so a test can + /// assert that a secret never appears anywhere in the telemetry. + pub fn all_field_values(&self) -> Vec { + let mut values = Vec::new(); + for span in self.spans().iter() { + values.extend(span.fields.values().cloned()); + } + for event in self.events().iter() { + values.extend(event.fields.values().cloned()); + } + values + } +} + +struct FieldVisitor(HashMap); + +impl Visit for FieldVisitor { + fn record_str(&mut self, field: &Field, value: &str) { + self.0.insert(field.name().to_string(), value.to_string()); + } + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0 + .entry(field.name().to_string()) + .or_insert_with(|| format!("{value:?}")); + } +} + +struct CaptureLayer(Arc); + +impl tracing_subscriber::Layer for CaptureLayer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_new_span(&self, attrs: &Attributes<'_>, _id: &Id, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(HashMap::new()); + attrs.record(&mut visitor); + self.0 + .spans + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedSpan { + name: attrs.metadata().name().to_string(), + fields: visitor.0, + }); + } + + fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(HashMap::new()); + event.record(&mut visitor); + self.0 + .events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedEvent { + level: *event.metadata().level(), + fields: visitor.0, + }); + } +} + +/// Runs `f` with a thread-local capturing subscriber installed and returns the +/// recorder alongside `f`'s result. Because the subscriber is thread-local, +/// any async work driven inside `f` must run on the calling thread (e.g. a +/// current-thread runtime's `block_on`) for its telemetry to be captured. +pub(crate) fn with_recorder(f: impl FnOnce() -> T) -> (Arc, T) { + let recorder = Arc::new(Recorder::default()); + let subscriber = tracing_subscriber::registry().with(CaptureLayer(recorder.clone())); + let value = tracing::subscriber::with_default(subscriber, f); + (recorder, value) +} diff --git a/crates/nuncio-contacts/Cargo.toml b/crates/nuncio-contacts/Cargo.toml index c91f4c9..3f38356 100644 --- a/crates/nuncio-contacts/Cargo.toml +++ b/crates/nuncio-contacts/Cargo.toml @@ -26,3 +26,4 @@ async-trait = { workspace = true } [dev-dependencies] tempfile = { workspace = true } wiremock = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/crates/nuncio-contacts/src/carddav.rs b/crates/nuncio-contacts/src/carddav.rs index 0fe496b..3d6c7a1 100644 --- a/crates/nuncio-contacts/src/carddav.rs +++ b/crates/nuncio-contacts/src/carddav.rs @@ -3,6 +3,7 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use thiserror::Error; +use tracing::{debug, info, instrument}; use crate::backend::ContactsBackend; use crate::models::Contact; @@ -27,7 +28,7 @@ pub enum CardDavError { /// `carddav_url` must already resolve to a specific address book collection (e.g. /// `https://carddav.example.com/dav/addressbooks/user/jmaes/contacts/`) -- PROPFIND-based /// `addressbook-home-set` auto-discovery is out of scope for this client. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct CardDavAccountConfig { /// Nuncio account identifier this address book collection belongs to. pub account_id: String, @@ -40,6 +41,20 @@ pub struct CardDavAccountConfig { pub auth_token: String, } +impl std::fmt::Debug for CardDavAccountConfig { + /// Manual `Debug` impl that redacts `auth_token` -- a derived impl would print the raw + /// secret verbatim the moment anything (a `debug!` log, an assertion failure message, + /// a panic payload) formats this config with `{:?}`. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CardDavAccountConfig") + .field("account_id", &self.account_id) + .field("carddav_url", &self.carddav_url) + .field("username", &self.username) + .field("auth_token", &"") + .finish() + } +} + /// CardDAV client protocol engine managing address book `REPORT` queries against a real server. pub struct CardDavClient { config: CardDavAccountConfig, @@ -100,6 +115,7 @@ impl CardDavClient { /// This performs a real network request -- no canned or fabricated data is ever /// returned. A server or network failure surfaces as [`CardDavError::TransportFailed`]; a /// malformed vCard inside a returned response surfaces as [`CardDavError::ParseFailed`]. + #[instrument(name = "carddav_sync", skip(self), fields(account_id = %account_id))] pub async fn fetch_remote_vcards( &self, account_id: &str, @@ -111,6 +127,8 @@ impl CardDavClient { let report_method = reqwest::Method::from_bytes(b"REPORT") .map_err(|e| CardDavError::TransportFailed(format!("invalid HTTP method: {e}")))?; + debug!(account_id = %account_id, "issuing CardDAV REPORT addressbook-query"); + let response = self .http .request(report_method, &self.config.carddav_url) @@ -136,7 +154,19 @@ impl CardDavClient { .await .map_err(|e| CardDavError::TransportFailed(e.to_string()))?; - self.parse_multistatus_response(account_id, &raw_xml) + // `parse_multistatus_response` fails fast on the first unparseable vCard (unlike the + // CalDAV client, which drops individual bad VEVENTs), so a successful return here means + // every card that was fetched parsed cleanly -- `dropped` is always 0 on this path. + let contacts = self.parse_multistatus_response(account_id, &raw_xml)?; + info!( + account_id = %account_id, + fetched = contacts.len(), + parsed = contacts.len(), + dropped = 0, + "completed CardDAV sync" + ); + + Ok(contacts) } } @@ -207,4 +237,83 @@ END:VCARD .expect("parse succeeds"); assert!(contacts.is_empty()); } + + #[test] + fn debug_impl_redacts_auth_token() { + let config = test_config(); + let rendered = format!("{config:?}"); + assert!(!rendered.contains("app-token-secret")); + assert!(rendered.contains("")); + } + + #[test] + fn fetch_remote_vcards_logs_completion_counts() { + use crate::test_tracing::with_recorder; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("current-thread runtime"); + + let (recorder, result) = with_recorder(|| { + runtime.block_on(async { + let mock_server = MockServer::start().await; + let xml_response = r#" + + + + + BEGIN:VCARD +VERSION:4.0 +FN:Alice Dev +EMAIL:alice@nuncio.mx +END:VCARD + + + + "#; + + Mock::given(method("REPORT")) + .and(path("/addressbooks/contacts/")) + .respond_with(ResponseTemplate::new(207).set_body_string(xml_response)) + .mount(&mock_server) + .await; + + let mut config = test_config(); + config.carddav_url = format!("{}/addressbooks/contacts/", mock_server.uri()); + let client = CardDavClient::new(config); + + client.fetch_remote_vcards("acct-1").await + }) + }); + + let contacts = result.expect("fetch succeeds against the wiremock server"); + assert_eq!(contacts.len(), 1); + + let completion = recorder + .events() + .into_iter() + .find(|e| e.message().contains("completed CardDAV sync")) + .expect("a completion event is logged"); + assert_eq!( + completion.fields.get("fetched").map(String::as_str), + Some("1") + ); + assert_eq!( + completion.fields.get("parsed").map(String::as_str), + Some("1") + ); + assert_eq!( + completion.fields.get("dropped").map(String::as_str), + Some("0") + ); + + // vCard PII must never appear in telemetry. + for value in recorder.all_field_values() { + assert!(!value.contains("Alice Dev")); + assert!(!value.contains("alice@nuncio.mx")); + } + } } diff --git a/crates/nuncio-contacts/src/lib.rs b/crates/nuncio-contacts/src/lib.rs index 3911bfc..15eddd9 100644 --- a/crates/nuncio-contacts/src/lib.rs +++ b/crates/nuncio-contacts/src/lib.rs @@ -10,6 +10,8 @@ pub mod db; pub mod mock; pub mod models; pub mod parser; +#[cfg(test)] +mod test_tracing; pub use backend::ContactsBackend; pub use carddav::{CardDavAccountConfig, CardDavClient, CardDavError}; diff --git a/crates/nuncio-contacts/src/test_tracing.rs b/crates/nuncio-contacts/src/test_tracing.rs new file mode 100644 index 0000000..7e7c4bf --- /dev/null +++ b/crates/nuncio-contacts/src/test_tracing.rs @@ -0,0 +1,135 @@ +//! Shared in-test `tracing` capture helper. +//! +//! Installs a thread-local subscriber that records every span (with its +//! fields) and every event (with its level, fields, and message) into an +//! in-memory buffer, so tests can assert on what the code logged -- including +//! the negative assertion that a secret never appears in any recorded field. +//! Deterministic and fully offline: no global subscriber, no I/O. + +#![cfg(test)] + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use tracing::field::{Field, Visit}; +use tracing::span::Attributes; +use tracing::{Event, Id, Level, Subscriber}; +use tracing_subscriber::layer::Context as LayerContext; +use tracing_subscriber::prelude::*; +use tracing_subscriber::registry::LookupSpan; + +/// A captured span: its name and the string-rendered fields it opened with. +#[derive(Debug, Clone)] +pub(crate) struct RecordedSpan { + #[allow(dead_code)] + pub name: String, + pub fields: HashMap, +} + +/// A captured event: its level and its string-rendered fields (including the +/// `message`). +#[derive(Debug, Clone)] +pub(crate) struct RecordedEvent { + #[allow(dead_code)] + pub level: Level, + pub fields: HashMap, +} + +impl RecordedEvent { + /// The event's `message` field, or the empty string if it had none. + pub fn message(&self) -> &str { + self.fields.get("message").map(String::as_str).unwrap_or("") + } +} + +/// In-memory sink of everything the subscriber captured during a test. +#[derive(Debug, Default)] +pub(crate) struct Recorder { + spans: Mutex>, + events: Mutex>, +} + +impl Recorder { + /// A snapshot of every captured span. + pub fn spans(&self) -> Vec { + self.spans.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + /// A snapshot of every captured event. + pub fn events(&self) -> Vec { + self.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + /// Every field value recorded across all spans and events, so a test can + /// assert that a secret never appears anywhere in the telemetry. + pub fn all_field_values(&self) -> Vec { + let mut values = Vec::new(); + for span in self.spans().iter() { + values.extend(span.fields.values().cloned()); + } + for event in self.events().iter() { + values.extend(event.fields.values().cloned()); + } + values + } +} + +struct FieldVisitor(HashMap); + +impl Visit for FieldVisitor { + fn record_str(&mut self, field: &Field, value: &str) { + self.0.insert(field.name().to_string(), value.to_string()); + } + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0 + .entry(field.name().to_string()) + .or_insert_with(|| format!("{value:?}")); + } +} + +struct CaptureLayer(Arc); + +impl tracing_subscriber::Layer for CaptureLayer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_new_span(&self, attrs: &Attributes<'_>, _id: &Id, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(HashMap::new()); + attrs.record(&mut visitor); + self.0 + .spans + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedSpan { + name: attrs.metadata().name().to_string(), + fields: visitor.0, + }); + } + + fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(HashMap::new()); + event.record(&mut visitor); + self.0 + .events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedEvent { + level: *event.metadata().level(), + fields: visitor.0, + }); + } +} + +/// Runs `f` with a thread-local capturing subscriber installed and returns the +/// recorder alongside `f`'s result. Because the subscriber is thread-local, +/// any async work driven inside `f` must run on the calling thread (e.g. a +/// current-thread runtime's `block_on`) for its telemetry to be captured. +pub(crate) fn with_recorder(f: impl FnOnce() -> T) -> (Arc, T) { + let recorder = Arc::new(Recorder::default()); + let subscriber = tracing_subscriber::registry().with(CaptureLayer(recorder.clone())); + let value = tracing::subscriber::with_default(subscriber, f); + (recorder, value) +} diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 898bfa5..a7095d8 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -764,12 +764,22 @@ impl DatabaseEngine { .execute(&mut *tx) .await .map_err(DatabaseError::Query)?; + tracing::info!( + table = "messages", + column = "remote_id", + "applying schema migration" + ); } if !existing_columns.iter().any(|c| c == "uid_validity") { sqlx::query("ALTER TABLE messages ADD COLUMN uid_validity TEXT NOT NULL DEFAULT ''") .execute(&mut *tx) .await .map_err(DatabaseError::Query)?; + tracing::info!( + table = "messages", + column = "uid_validity", + "applying schema migration" + ); } // Seed any legacy row still carrying the empty-string default with a @@ -823,6 +833,11 @@ impl DatabaseEngine { .execute(&self.pool) .await .map_err(DatabaseError::Query)?; + tracing::info!( + table = "accounts", + column = "imap_tls_mode", + "applying schema migration" + ); } if !existing_columns.iter().any(|c| c == "smtp_tls_mode") { sqlx::query( @@ -831,6 +846,11 @@ impl DatabaseEngine { .execute(&self.pool) .await .map_err(DatabaseError::Query)?; + tracing::info!( + table = "accounts", + column = "smtp_tls_mode", + "applying schema migration" + ); } Ok(()) @@ -865,12 +885,22 @@ impl DatabaseEngine { .execute(&self.pool) .await .map_err(DatabaseError::Query)?; + tracing::info!( + table = "accounts", + column = "smtp_host", + "applying schema migration" + ); } if !existing_columns.iter().any(|c| c == "smtp_port") { sqlx::query("ALTER TABLE accounts ADD COLUMN smtp_port INTEGER") .execute(&self.pool) .await .map_err(DatabaseError::Query)?; + tracing::info!( + table = "accounts", + column = "smtp_port", + "applying schema migration" + ); } Ok(()) @@ -902,6 +932,11 @@ impl DatabaseEngine { .execute(&self.pool) .await .map_err(DatabaseError::Query)?; + tracing::info!( + table = "accounts", + column = "collection_url", + "applying schema migration" + ); } Ok(()) @@ -3811,6 +3846,107 @@ mod tests { engine.close().await; } + /// Applying an additive column migration against a pre-existing (old-schema) + /// database must be visible in telemetry -- an `INFO` log naming the table and + /// column -- rather than only observable indirectly via the resulting schema. + #[tokio::test] + async fn migrate_logs_when_an_additive_column_migration_actually_runs() { + use std::sync::{Arc, Mutex}; + use tracing_subscriber::fmt::MakeWriter; + + #[derive(Clone, Default)] + struct CapturedLogs(Arc>>); + + impl std::io::Write for CapturedLogs { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 + .lock() + .expect("log buffer lock") + .extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> MakeWriter<'a> for CapturedLogs { + type Writer = CapturedLogs; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } + } + + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("pre_dav_migration.db"); + let secrets = crate::vault::SecretManager::mock(); + + // An old-schema `accounts` table predating `collection_url`. + { + let url = format!("sqlite://{}", db_path.to_string_lossy()); + let options = sqlx::sqlite::SqliteConnectOptions::from_str(&url) + .expect("valid sqlite url") + .create_if_missing(true); + let pool = sqlx::SqlitePool::connect_with(options) + .await + .expect("connect to fresh old-schema db"); + + sqlx::query( + r#" + CREATE TABLE accounts ( + id TEXT PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + email_address TEXT NOT NULL, + protocol TEXT NOT NULL, + server_host TEXT NOT NULL, + server_port INTEGER NOT NULL, + use_tls INTEGER NOT NULL, + keyring_secret_key TEXT NOT NULL, + sync_interval_secs INTEGER NOT NULL, + smtp_host TEXT, + smtp_port INTEGER, + imap_tls_mode TEXT NOT NULL DEFAULT 'implicit_tls', + smtp_tls_mode TEXT NOT NULL DEFAULT 'implicit_tls' + ) + "#, + ) + .execute(&pool) + .await + .expect("create old-schema accounts table"); + + pool.close().await; + } + + let logs = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_ansi(false) + .finish(); + + // `set_default` returns a guard that stays active across `.await` + // points as long as this test stays on the single `#[tokio::test]` + // current-thread executor, which is the default runtime flavor and + // exactly what is used here. + let guard = tracing::subscriber::set_default(subscriber); + let engine = DatabaseEngine::connect_file(&db_path, &secrets) + .await + .expect("opening an old-schema database must never error"); + drop(guard); + + let captured = String::from_utf8(logs.0.lock().expect("log buffer lock").clone()) + .expect("captured log is valid utf8"); + assert!( + captured.contains("collection_url"), + "expected the migrated column to be named in the INFO log, got: {captured}" + ); + assert!( + captured.to_ascii_uppercase().contains("INFO"), + "expected an INFO-level log for the applied migration, got: {captured}" + ); + + engine.close().await; + } + #[test] fn database_error_display() { let err = DatabaseError::PoolCreation("failed".to_string()); diff --git a/crates/nuncio-store/src/vault.rs b/crates/nuncio-store/src/vault.rs index ca60afd..26ec80c 100644 --- a/crates/nuncio-store/src/vault.rs +++ b/crates/nuncio-store/src/vault.rs @@ -231,6 +231,9 @@ impl SecretManager { let mut bytes = vec![0u8; len]; OsRng.fill_bytes(&mut bytes); self.set_secret(account, &hex::encode(&bytes))?; + // Lifecycle visibility only: which purpose got a key minted, and how long it + // is, never the key material itself. + tracing::info!(purpose = account, len, "minted key material on first use"); Ok(bytes) } Err(e) => Err(e), @@ -360,6 +363,60 @@ mod tests { assert!(matches!(err, VaultError::StorageFailed(_))); } + #[test] + fn get_or_create_key_bytes_logs_minting_without_leaking_the_key() { + use std::sync::{Arc, Mutex}; + use tracing_subscriber::fmt::MakeWriter; + + #[derive(Clone, Default)] + struct CapturedLogs(Arc>>); + + impl std::io::Write for CapturedLogs { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 + .lock() + .expect("log buffer lock") + .extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> MakeWriter<'a> for CapturedLogs { + type Writer = CapturedLogs; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } + } + + let manager = SecretManager::mock(); + let logs = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_ansi(false) + .finish(); + + let guard = tracing::subscriber::set_default(subscriber); + let key = manager + .get_or_create_key_bytes(WEBHOOK_SIGNING_KEY_ACCOUNT, 32) + .expect("key provisioned from vault"); + drop(guard); + + let captured = String::from_utf8(logs.0.lock().expect("log buffer lock").clone()) + .expect("captured log is valid utf8"); + assert!( + captured.contains(WEBHOOK_SIGNING_KEY_ACCOUNT), + "expected the key's purpose to be named in the INFO log, got: {captured}" + ); + assert!(captured.to_ascii_uppercase().contains("INFO")); + + // The minted key bytes must never appear in the log, in any encoding. + let hex_key = hex::encode(&key); + assert!(!captured.contains(&hex_key)); + } + #[test] fn os_keyring_provider_can_be_constructed_for_production_use() { // Construction alone must not touch the OS credential store (lazy `Entry` creation -- 2.54.0 From a668c802e1b4deb55ee88f5e4562c929d5a97eab Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 19:19:17 -0500 Subject: [PATCH 42/95] feat(mail): instrument protocol syncs and log destructive IMAP ops (#374) Wrap IMAP/JMAP folder and message sync in tracing spans keyed by account and folder, logging completion counts (fetched/oversized/skipped) so a sync run's outcome is visible without a debugger. SMTP send attempts and their outcome are logged with the recipient COUNT only, never addresses. Every destructive IMAP mutation (MOVE, DELETE/UID-scoped EXPUNGE) is logged with its UID set and the UIDVALIDITY guard's pass/fail result BEFORE the command reaches the server, so an audit trail exists even when the guard then refuses to act. JMAP mirrors the same pre-logging for its Email/set move/destroy, keyed by the JMAP object id instead of a UID (JMAP has no UIDVALIDITY-equivalent to guard). Refs #361 Co-authored-by: James Maes --- Cargo.lock | 22 ++++ crates/nuncio-mail/Cargo.toml | 1 + crates/nuncio-mail/src/imap.rs | 191 +++++++++++++++++++++++++++++++-- crates/nuncio-mail/src/jmap.rs | 55 +++++++++- crates/nuncio-mail/src/smtp.rs | 63 +++++++++-- 5 files changed, 312 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1bde572..042c091 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2628,6 +2628,7 @@ dependencies = [ "tokio-rustls", "tokio-stream", "tracing", + "tracing-test", "webpki-roots", "wiremock", ] @@ -4746,6 +4747,27 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tracing-test" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a4c448db514d4f24c5ddb9f73f2ee71bfb24c526cf0c570ba142d1119e0051" +dependencies = [ + "tracing-core", + "tracing-subscriber", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/crates/nuncio-mail/Cargo.toml b/crates/nuncio-mail/Cargo.toml index 644a401..1045a59 100644 --- a/crates/nuncio-mail/Cargo.toml +++ b/crates/nuncio-mail/Cargo.toml @@ -31,6 +31,7 @@ reqwest.workspace = true [dev-dependencies] tempfile.workspace = true wiremock.workspace = true +tracing-test = "0.2" # `test-util` (not in tokio's `full`) enables paused-clock tests so the # bounded per-item FETCH timeout can be asserted without a real-time wait. tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index e022380..2de83f7 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -716,6 +716,11 @@ impl ImapEngine { /// 2. Each queue is drained in explicit UID-set batches of at most /// `batch_size`, so the full bodies held in memory at once never exceed /// one batch. + #[tracing::instrument( + skip(self, session), + fields(account_id = %self.account_id, folder_id = %folder_id), + err + )] async fn sync_folder_messages_batched( &self, folder_id: &str, @@ -770,10 +775,12 @@ impl ImapEngine { let mut max_uid_seen: u32 = 0; let mut body_uids: Vec = Vec::new(); let mut oversized_uids: Vec = Vec::new(); + let mut skipped_no_uid: usize = 0; for item in &enumeration { // A message with no UID cannot be addressed by a follow-up fetch, // so it is skipped rather than issued a UID-less request. let Some(uid) = item.uid.filter(|u| *u >= 1) else { + skipped_no_uid += 1; continue; }; max_uid_seen = max_uid_seen.max(uid); @@ -857,6 +864,15 @@ impl ImapEngine { .unwrap_or_else(|| "full-resync-required".to_string()), }; + tracing::info!( + account_id = %self.account_id, + folder_id, + fetched = emails.len(), + oversized = oversized_uids.len(), + skipped = skipped_no_uid, + "imap folder sync complete" + ); + Ok((emails, new_checkpoint)) } @@ -1066,19 +1082,50 @@ impl ImapEngine { // Refuse to act unless the mailbox's current UIDVALIDITY provably // matches the one the stored UID was captured under. let stored_validity = parse_uid_validity(&spec.uid_validity); - match (stored_validity, mailbox.uid_validity) { - (Some(stored), Some(current)) if stored == current => {} - (stored, current) => { - return Err(MailError::ImapError(format!( - "refusing to mutate message '{}' in folder '{}': UIDVALIDITY guard failed \ - (stored {stored:?}, current {current:?}); the stored UID may no longer \ - address the intended message", - spec.message_id, spec.folder_id - ))); - } - } + let guard_passed = matches!( + (stored_validity, mailbox.uid_validity), + (Some(stored), Some(current)) if stored == current + ); let uid_set = uid.to_string(); + + // Every destructive op (MOVE, DELETE -- which issues STORE \Deleted + // followed by a UID-scoped EXPUNGE) is logged BEFORE any command is + // sent to the server, so an audit trail exists even when the + // UIDVALIDITY guard below then refuses to proceed. + match &spec.kind { + RemoteMutationKind::Move { to_folder } => { + tracing::info!( + op = "MOVE", + folder_id = %spec.folder_id, + to_folder = %to_folder, + uid_set = %uid_set, + uidvalidity_guard_passed = guard_passed, + "issuing destructive IMAP MOVE" + ); + } + RemoteMutationKind::Delete => { + tracing::warn!( + op = "DELETE", + folder_id = %spec.folder_id, + uid_set = %uid_set, + uidvalidity_guard_passed = guard_passed, + "issuing destructive IMAP DELETE (STORE \\Deleted + UID-scoped EXPUNGE)" + ); + } + RemoteMutationKind::SetFlagged { .. } | RemoteMutationKind::Copy { .. } => {} + } + + if !guard_passed { + let current_validity = mailbox.uid_validity; + return Err(MailError::ImapError(format!( + "refusing to mutate message '{}' in folder '{}': UIDVALIDITY guard failed \ + (stored {stored_validity:?}, current {current_validity:?}); the stored UID may \ + no longer address the intended message", + spec.message_id, spec.folder_id + ))); + } + match &spec.kind { RemoteMutationKind::SetFlagged { value } => { let op = if *value { "+FLAGS" } else { "-FLAGS" }; @@ -1336,6 +1383,7 @@ impl ImapEngine { #[async_trait] impl MailBackend for ImapEngine { + #[tracing::instrument(skip(self), fields(account_id = %self.account_id), err)] async fn sync_folders(&self) -> Result, MailError> { let (u, p) = match (&self.username, &self.password) { (Some(u), Some(p)) => (u.as_str(), p.as_str()), @@ -1365,6 +1413,11 @@ impl MailBackend for ImapEngine { } drop(mailboxes); let _ = session.logout().await; + tracing::info!( + account_id = %self.account_id, + folders = folders.len(), + "imap folder list sync complete" + ); // An empty mailbox is a genuine result, not a signal to fall back to // placeholder folders -- the caller gets exactly what the server reported. Ok(folders) @@ -1386,6 +1439,7 @@ impl MailBackend for ImapEngine { #[cfg(test)] mod tests { use super::*; + use tracing_test::traced_test; #[tokio::test] async fn imap_dual_socket_manager_lifecycle() -> Result<(), MailError> { @@ -2510,6 +2564,121 @@ mod tests { Ok(()) } + #[traced_test] + #[tokio::test] + async fn imap_destructive_delete_logs_before_issuing_even_when_uidvalidity_guard_fails() { + // The stored checkpoint's UIDVALIDITY (1) does not match the scripted + // server's current UIDVALIDITY (2), so the UIDVALIDITY guard MUST + // refuse the mutation and no protocol command may reach the server + // (proven below via `commands`). A WARN log naming the op, the target + // UID set, and the failed guard result must still exist -- proving the + // pre-log fires before dispatch, not merely on a successful path. + let (result, commands) = run_scripted_mutation( + &imap_spec(RemoteMutationKind::Delete, Some("1:105")), + false, + true, + 2, + ) + .await; + + let err = result.expect_err("a UIDVALIDITY mismatch must be refused"); + assert!(matches!(err, MailError::ImapError(_))); + assert!( + !commands.iter().any(|c| { + let u = c.to_ascii_uppercase(); + u.contains("STORE") || u.contains("EXPUNGE") || u.contains("COPY") + }), + "no mutating command may be issued on a guard failure, got: {commands:?}" + ); + + assert!(logs_contain("issuing destructive IMAP DELETE")); + assert!(logs_contain("uid_set=42")); + assert!(logs_contain("uidvalidity_guard_passed=false")); + } + + #[traced_test] + #[tokio::test] + async fn imap_destructive_move_logs_before_issuing_with_uid_set_and_passing_guard() { + // A guard-passing MOVE still logs before the command is issued, at + // INFO (MOVE is reversible/non-permanent, unlike DELETE/EXPUNGE). + let (result, _commands) = run_scripted_mutation( + &imap_spec( + RemoteMutationKind::Move { + to_folder: "Archive".to_string(), + }, + Some("1:105"), + ), + true, + true, + 1, + ) + .await; + result.expect("move mutation succeeds"); + + assert!(logs_contain("issuing destructive IMAP MOVE")); + assert!(logs_contain("uid_set=42")); + assert!(logs_contain("uidvalidity_guard_passed=true")); + } + + #[traced_test] + #[tokio::test] + async fn imap_folder_sync_logs_completion_counts() { + // A folder sync run must log its completion counts (fetched, and by + // extension the oversized/skipped counters this run does not + // exercise), so an operator can see sync outcomes without a debugger. + let (client_io, server_io) = tokio::io::duplex(8192); + let server = tokio::spawn(async move { + let mut io = server_io; + while let Some(line) = read_scripted_line(&mut io).await { + let tag = line.split_whitespace().next().unwrap_or("").to_string(); + let upper = line.to_ascii_uppercase(); + if upper.contains("LOGIN") { + let _ = io + .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) + .await; + } else if upper.contains("SELECT") { + let resp = format!( + "* FLAGS (\\Seen)\r\n* 0 EXISTS\r\n* 0 RECENT\r\n\ + * OK [UIDVALIDITY 1] ok\r\n* OK [UIDNEXT 1] ok\r\n\ + {tag} OK [READ-WRITE] SELECT done\r\n" + ); + let _ = io.write_all(resp.as_bytes()).await; + } else if upper.contains("UID FETCH") { + let _ = io + .write_all(format!("{tag} OK FETCH completed\r\n").as_bytes()) + .await; + } else if upper.contains("LOGOUT") { + let _ = io + .write_all(format!("* BYE\r\n{tag} OK LOGOUT\r\n").as_bytes()) + .await; + break; + } + } + }); + + let client = async_imap::Client::new(client_io); + let mut session = client + .login("user", "pass") + .await + .map_err(|(e, _)| e) + .expect("scripted login succeeds"); + let engine = ImapEngine::new("acct-completion", "example.test", 993); + + let (emails, _checkpoint) = engine + .sync_folder_messages_with_session("INBOX", None, &mut session) + .await + .expect("sync succeeds"); + assert!(emails.is_empty()); + + drop(session); + let _ = server.await; + + assert!(logs_contain("imap folder sync complete")); + assert!(logs_contain("fetched=0")); + assert!(logs_contain("oversized=0")); + assert!(logs_contain("skipped=0")); + } + #[test] fn imap_fetch_command_query_construction() { let query = build_fetch_command_query(); diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index cfbc0fa..80319b7 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -524,6 +524,7 @@ impl JmapEngine { #[async_trait] impl MailBackend for JmapEngine { + #[tracing::instrument(skip(self), fields(account_id = %self.account_id), err)] async fn sync_folders(&self) -> Result, MailError> { if !self.has_credentials() { return Err(MailError::AuthError( @@ -536,7 +537,7 @@ impl MailBackend for JmapEngine { let raw = self.post_jmap(&session.api_url, &request).await?; let mailboxes = Self::parse_mailbox_get_response(&raw)?; - Ok(mailboxes + let folders: Vec = mailboxes .into_iter() .map(|mb| Folder { id: mb.id, @@ -544,9 +545,22 @@ impl MailBackend for JmapEngine { total_messages: mb.total_emails as usize, unread_messages: mb.unread_emails as usize, }) - .collect()) + .collect(); + + tracing::info!( + account_id = %self.account_id, + folders = folders.len(), + "jmap folder list sync complete" + ); + + Ok(folders) } + #[tracing::instrument( + skip(self, _since_state), + fields(account_id = %self.account_id, folder_id = %folder_id), + err + )] async fn sync_messages( &self, folder_id: &str, @@ -566,15 +580,50 @@ impl MailBackend for JmapEngine { let get_request = Self::build_email_get_request(&account_id, Some(ids)); let get_raw = self.post_jmap(&session.api_url, &get_request).await?; - self.parse_email_get_response(&get_raw) + let (emails, state) = self.parse_email_get_response(&get_raw)?; + + tracing::info!( + account_id = %self.account_id, + folder_id, + fetched = emails.len(), + "jmap message sync complete" + ); + + Ok((emails, state)) } + #[tracing::instrument(skip(self), fields(account_id = %self.account_id), err)] async fn apply_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError> { if !self.has_credentials() { return Err(MailError::AuthError( "JMAP remote mutation requires credentials".to_string(), )); } + + // JMAP has no UIDVALIDITY-equivalent guard (an `Email/set` targets a + // stable object id per RFC 8621), but a destructive mutation is still + // logged BEFORE the request is issued, mirroring the IMAP audit trail. + match &spec.kind { + RemoteMutationKind::Move { to_folder } => { + tracing::info!( + op = "MOVE", + folder_id = %spec.folder_id, + to_folder = %to_folder, + remote_id = %spec.remote_id, + "issuing destructive JMAP mailbox move (Email/set update)" + ); + } + RemoteMutationKind::Delete => { + tracing::warn!( + op = "DELETE", + folder_id = %spec.folder_id, + remote_id = %spec.remote_id, + "issuing destructive JMAP delete (Email/set destroy)" + ); + } + RemoteMutationKind::SetFlagged { .. } | RemoteMutationKind::Copy { .. } => {} + } + let session = self.discover_session().await?; let account_id = self.resolve_account_id(&session); diff --git a/crates/nuncio-mail/src/smtp.rs b/crates/nuncio-mail/src/smtp.rs index 9cef1f0..894ae6c 100644 --- a/crates/nuncio-mail/src/smtp.rs +++ b/crates/nuncio-mail/src/smtp.rs @@ -88,21 +88,49 @@ impl SmtpTransportEngine { /// Send a composed [`OutboundMessage`] using the inner transport client. /// Returns `Ok(())` only when the transport genuinely accepted the - /// message. + /// message. Logs the send attempt and outcome with the recipient COUNT + /// only (`to` plus `cc` when present) -- never the addresses themselves. pub async fn send_message(&self, message: &OutboundMessage) -> Result<(), MailError> { + let recipient_count = + 1 + usize::from(message.cc.as_ref().is_some_and(|cc| !cc.trim().is_empty())); + tracing::info!(recipient_count, "smtp send attempt"); + let msg = Self::build_outbound_mime_message(message)?; - self.transport.send(msg).await.map_err(MailError::from)?; - Ok(()) + match self.transport.send(msg).await { + Ok(_) => { + tracing::info!(recipient_count, "smtp send succeeded"); + Ok(()) + } + Err(e) => { + let err = MailError::from(e); + tracing::warn!(recipient_count, error = %err, "smtp send failed"); + Err(err) + } + } } - /// Send an email message using a provided [`AsyncSmtpTransport`] client instance. + /// Send an email message using a provided [`AsyncSmtpTransport`] client + /// instance. Logs the send attempt and outcome with the recipient COUNT + /// only -- never the address itself. pub async fn send_email_with_transport( transport: &AsyncSmtpTransport, email: &Email, ) -> Result<(), MailError> { + let recipient_count = 1; + tracing::info!(recipient_count, "smtp send attempt"); + let msg = Self::build_mime_message(email)?; - transport.send(msg).await.map_err(MailError::from)?; - Ok(()) + match transport.send(msg).await { + Ok(_) => { + tracing::info!(recipient_count, "smtp send succeeded"); + Ok(()) + } + Err(e) => { + let err = MailError::from(e); + tracing::warn!(recipient_count, error = %err, "smtp send failed"); + Err(err) + } + } } /// Build an RFC 5322 [`lettre::Message`] from a Nuncio [`Email`] entity. @@ -323,6 +351,7 @@ mod tests { use super::*; use bytes::Bytes; use nuncio_core::model::Attachment; + use tracing_test::traced_test; fn sample_email() -> Email { Email { @@ -608,6 +637,28 @@ mod tests { )); } + #[traced_test] + #[tokio::test] + async fn send_message_logs_attempt_and_failure_with_recipient_count_never_addresses() { + let engine = SmtpTransportEngine::new("127.0.0.1", 1, TlsMode::ImplicitTls, "user", "pass") + .expect("valid config"); + let mut message = sample_outbound_message(); + message.cc = Some("carol@nuncio.mx".to_string()); + let _ = engine + .send_message(&message) + .await + .expect_err("delivery to port 1 should fail"); + + assert!(logs_contain("smtp send attempt")); + assert!(logs_contain("recipient_count=2")); + assert!(logs_contain("smtp send failed")); + + assert!(!logs_contain("alice@nuncio.mx")); + assert!(!logs_contain("bob@nuncio.mx")); + assert!(!logs_contain("carol@nuncio.mx")); + assert!(!logs_contain("Status Update")); + } + #[tokio::test] async fn message_sender_trait_impl_delegates_to_send_message() { // Proves `SmtpTransportEngine`'s `MessageSender` trait impl is wired -- 2.54.0 From 58f321451dd4edf931dd8f3ea98cdf05844a8a63 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 19:40:52 -0500 Subject: [PATCH 43/95] feat(proto): real GetStatus health/readiness surface (#376) Turn System.GetStatus into a live health/readiness surface backed by real daemon state: process uptime, accounts_loaded, unread_count, last_error, outbox_depth, per-account sync state (idle/syncing/error with last-synced/last-error), and top-level ready/db_healthy. - proto: extend GetStatusResponse; add AccountSyncState message + SyncState enum; regenerate the descriptor golden. - store: add cheap ping (SELECT 1), unread-message count, and pending outbox-mutation count. - sync dispatcher: retain each account's last completed outcome and expose a live per-account sync-status snapshot. - daemon: populate every field from live state; DB read failures degrade honestly (reflected via db_healthy), never fabricated. - cli: render the full health surface in text and JSON. Refs #364 Co-authored-by: James Maes --- crates/nuncio-cli/src/runner.rs | 120 +++++++++- .../proto/nuncio/v1/descriptor.bin | Bin 15645 -> 16326 bytes .../nuncio-proto/proto/nuncio/v1/nuncio.proto | 57 ++++- crates/nuncio-proto/src/lib.rs | 13 ++ crates/nuncio-proto/src/time.rs | 11 + crates/nuncio-store/src/db.rs | 33 +++ crates/nunciod/src/grpc.rs | 210 ++++++++++++++++-- crates/nunciod/src/sync_dispatcher.rs | 158 ++++++++++++- 8 files changed, 570 insertions(+), 32 deletions(-) diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index a74ef3a..65571d5 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -2786,17 +2786,80 @@ impl HeadlessRunner { /// NOT auto-spawn the daemon. async fn handle_system_status(&self, json_mode: bool) -> String { match self.query_daemon_status().await { - Ok((engine_status, version)) => { + Ok(status) => { + let uptime_secs = status + .uptime + .as_ref() + .map(nuncio_proto::time::duration_to_secs) + .unwrap_or(0); if json_mode { + let accounts: Vec<_> = status + .account_sync_states + .iter() + .map(|a| { + json!({ + "account_id": a.account_id, + "state": sync_state_label(a.state), + "last_synced": a + .last_synced + .as_ref() + .map(nuncio_proto::time::timestamp_to_unix_secs), + "last_error": a.last_error, + }) + }) + .collect(); format_json(&json!({ - "engine_status": engine_status, - "version": version, + "engine_status": status.engine_status, + "version": status.version, + "ready": status.ready, + "db_healthy": status.db_healthy, + "uptime_secs": uptime_secs, + "accounts_loaded": status.accounts_loaded, + "unread_count": status.unread_count, + "outbox_depth": status.outbox_depth, + "last_error": status.last_error, + "accounts": accounts, })) } else { - format!( - "Nuncio daemon status: {engine_status} (nunciod v{version}, {})", - self.grpc_addr - ) + let mut out = format!( + "Nuncio daemon status: {} (nunciod v{}, {})\n", + status.engine_status, status.version, self.grpc_addr + ); + out.push_str(&format!( + " ready: {} db_healthy: {}\n", + status.ready, status.db_healthy + )); + out.push_str(&format!(" uptime: {uptime_secs}s\n")); + out.push_str(&format!( + " accounts_loaded: {} unread: {} outbox_depth: {}\n", + status.accounts_loaded, status.unread_count, status.outbox_depth + )); + out.push_str(&format!( + " last_error: {}", + status.last_error.as_deref().unwrap_or("") + )); + if status.account_sync_states.is_empty() { + out.push_str("\n accounts: "); + } else { + out.push_str("\n accounts:"); + for a in &status.account_sync_states { + out.push_str(&format!( + "\n {}: {}", + a.account_id, + sync_state_label(a.state) + )); + if let Some(ts) = a.last_synced.as_ref() { + out.push_str(&format!( + " (last synced: {})", + nuncio_proto::time::timestamp_to_unix_secs(ts) + )); + } + if let Some(err) = a.last_error.as_deref() { + out.push_str(&format!(" [error: {err}]")); + } + } + } + out } } Err(e) => { @@ -2810,10 +2873,10 @@ impl HeadlessRunner { } /// Reads the gRPC bearer token from the injected vault, connects to the - /// daemon over gRPC, and calls `GetStatus`. Returns `(engine_status, - /// version)` on success, or a human-readable error string describing + /// daemon over gRPC, and calls `GetStatus`. Returns the full live health + /// response on success, or a human-readable error string describing /// exactly what failed (vault, connection, or the RPC itself). - async fn query_daemon_status(&self) -> Result<(String, String), String> { + async fn query_daemon_status(&self) -> Result { let token_bytes = self .secrets .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) @@ -2835,7 +2898,21 @@ impl HeadlessRunner { })? .into_inner(); - Ok((response.engine_status, response.version)) + Ok(response) + } +} + +/// Maps a `nuncio.v1.SyncState` wire discriminant onto a short human label +/// for the CLI's `system status` rendering. An unrecognized value renders as +/// "unknown" rather than failing. +fn sync_state_label(state: i32) -> &'static str { + match nuncio_proto::v1::SyncState::try_from(state) + .unwrap_or(nuncio_proto::v1::SyncState::Unspecified) + { + nuncio_proto::v1::SyncState::Idle => "idle", + nuncio_proto::v1::SyncState::Syncing => "syncing", + nuncio_proto::v1::SyncState::Error => "error", + nuncio_proto::v1::SyncState::Unspecified => "unknown", } } @@ -3056,6 +3133,21 @@ mod tests { Ok(tonic::Response::new(GetStatusResponse { engine_status: "Ready".to_string(), version: "9.9.9".to_string(), + uptime: Some(nuncio_proto::time::duration_from_secs(5)), + accounts_loaded: 2, + unread_count: 3, + last_error: None, + outbox_depth: 1, + account_sync_states: vec![nuncio_proto::v1::AccountSyncState { + account_id: "acct-1".to_string(), + state: nuncio_proto::v1::SyncState::Idle as i32, + last_synced: Some(nuncio_proto::time::timestamp_from_unix_secs( + 1_700_000_000, + )), + last_error: None, + }], + ready: true, + db_healthy: true, })) } @@ -3106,6 +3198,12 @@ mod tests { assert!(out.contains(r#""status":"ok""#)); assert!(out.contains(r#""engine_status":"Ready""#)); assert!(out.contains(r#""version":"9.9.9""#)); + assert!(out.contains(r#""ready":true"#)); + assert!(out.contains(r#""db_healthy":true"#)); + assert!(out.contains(r#""accounts_loaded":2"#)); + assert!(out.contains(r#""unread_count":3"#)); + assert!(out.contains(r#""outbox_depth":1"#)); + assert!(out.contains(r#""account_id":"acct-1""#)); let text_out = runner .execute_command( diff --git a/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin b/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin index f86c1175b676558b7b70987758876f4ac86bf1d1..d555779ca64587964e879d5272f5d2b66c622cc0 100644 GIT binary patch delta 652 zcmZ9JK~LK-6vykN?Z`{l22-JE)10xx81mK1N9>y?Q2ZhCt2)tRo0u|&(Ht&e}DhH{55$%n)Y-x)3rXT|3P3K_&!fU z;f_P!6`o*IZ=>;U3revpY^{Mdt9$iUn1ucrZ++iwEgDJIg7?83d9k2zs=$Kj<3{sy zw?IHL*~@|(gm4oSpe%}bWEWFaql<_kn;ePKp2I3sXin+yIha|k9yRavDbYyo5NE-@W(FJOV1QzX@Y_*hZ=8Y)Q`F)yHMbGwwfDH1bdZ5hUr z-LkS<_}{LS;fWYb_yrBvxft5dG93x|MMon10oLZjRXkLM8rv%kebd}~nM-x~7s%@T z-6HN7*P$d`k8L9zv_i51zkLM|4Cs(~BQYG?pze%d(9OV7inBZY_x9u~7zuXF5J2(JUqL3!-;-& Duration { } } +/// Converts a `std::time::Duration` (e.g. a process-uptime measurement taken +/// with `Instant::elapsed`) into a wire `Duration`, preserving sub-second +/// precision. Saturates rather than overflowing on an implausibly large span. +#[must_use] +pub fn duration_from_std(d: std::time::Duration) -> Duration { + Duration { + seconds: i64::try_from(d.as_secs()).unwrap_or(i64::MAX), + nanos: d.subsec_nanos() as i32, + } +} + /// Extracts a whole-second duration from a `Duration`, truncating any /// sub-second `nanos` component. A negative `seconds` (never produced by /// this crate's own mappers) reads back as zero rather than panicking. diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index a7095d8..c22ba55 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -438,6 +438,39 @@ impl DatabaseEngine { } } + /// Cheap liveness probe: runs `SELECT 1` against the pool and returns + /// `Ok(())` only if it genuinely round-trips. Unlike `check_integrity` + /// (a full `PRAGMA quick_check`), this is a constant-time readiness ping + /// suitable for a health endpoint served on every status request. + pub async fn ping(&self) -> Result<(), DatabaseError> { + let _: (i64,) = sqlx::query_as("SELECT 1") + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; + Ok(()) + } + + /// Total unread messages across every folder, read live from the store. + pub async fn count_unread_messages(&self) -> Result { + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages WHERE read_flag = 0") + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; + Ok(count.max(0) as u64) + } + + /// Number of outbound mutations still queued in the outbox (status + /// `pending`) -- the outbox depth reported by the daemon's health surface. + pub async fn count_pending_mutations(&self) -> Result { + let (count,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM pending_remote_mutations WHERE status = 'pending'", + ) + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; + Ok(count.max(0) as u64) + } + /// Cryptographic hash-chain audit ledger verification (`verify_chain_integrity()`) /// detecting log tampering or corrupted `filter_execution_logs`, using the ledger /// HMAC key provisioned for this engine. diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 00659cf..8d387bf 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -7,10 +7,12 @@ use crate::lifecycle::ShutdownSignal; use crate::pagination::{self, CursorField}; -use crate::sync_dispatcher::{sync_all_configured, ProductionAccountSyncer, SyncDispatcher}; +use crate::sync_dispatcher::{ + sync_all_configured, AccountSyncPhase, ProductionAccountSyncer, SyncDispatcher, +}; use nuncio_cal::CalendarBackend; use nuncio_contacts::ContactsBackend; -use nuncio_core::{CoreCommand, CoreEvent, EventBus}; +use nuncio_core::{CoreCommand, CoreEvent, EngineStatus, EventBus}; use nuncio_filter::{FilterEngine, NsqlParser, NsqlValidator, ValidationOptions}; use nuncio_mail::{ImapEngine, MailBackend, MessageSender, SmtpTransportEngine}; use nuncio_proto::errors; @@ -26,14 +28,15 @@ use nuncio_proto::v1::mail_server::{Mail, MailServer}; use nuncio_proto::v1::system_server::{System, SystemServer}; use nuncio_proto::v1::ErrorReason; use nuncio_proto::v1::{ - export_request, AccountConfig as AccountConfigProto, AddAccountRequest, AddAccountResponse, - Attachment as AttachmentProto, AuditRecord as AuditRecordProto, BatchFilterProgress, - CalendarEvent as CalendarEventProto, CalendarSyncRequest, CalendarSyncResponse, - Contact as ContactProto, ContactEmail as ContactEmailProto, ContactPhone as ContactPhoneProto, - ContactsSyncRequest, ContactsSyncResponse, CreateContactRequest, CreateContactResponse, - CreateRuleRequest, CreateRuleResponse, DatabaseRecovered, DavTransport as DavTransportProto, - DeleteRuleRequest, DeleteRuleResponse, Event, EventError, ExportFormat as ExportFormatProto, - ExportRequest, ExportResponse, ExportRulesRequest, ExportRulesResponse, FilterExecuted, + export_request, AccountConfig as AccountConfigProto, AccountSyncState as AccountSyncStateProto, + AddAccountRequest, AddAccountResponse, Attachment as AttachmentProto, + AuditRecord as AuditRecordProto, BatchFilterProgress, CalendarEvent as CalendarEventProto, + CalendarSyncRequest, CalendarSyncResponse, Contact as ContactProto, + ContactEmail as ContactEmailProto, ContactPhone as ContactPhoneProto, ContactsSyncRequest, + ContactsSyncResponse, CreateContactRequest, CreateContactResponse, CreateRuleRequest, + CreateRuleResponse, DatabaseRecovered, DavTransport as DavTransportProto, DeleteRuleRequest, + DeleteRuleResponse, Event, EventError, ExportFormat as ExportFormatProto, ExportRequest, + ExportResponse, ExportRulesRequest, ExportRulesResponse, FilterExecuted, FilterExecutionLog as FilterExecutionLogProto, FilterRule as FilterRuleProto, Folder as FolderProto, GetContactRequest, GetContactResponse, GetEventRequest, GetEventResponse, GetExecutionLogsRequest, GetExecutionLogsResponse, GetMessageRequest, @@ -47,18 +50,18 @@ use nuncio_proto::v1::{ PreviewRuleRequest, PreviewRuleResponse, RemoveAccountRequest, RemoveAccountResponse, RuleExportFormat as RuleExportFormatProto, SearchMessagesRequest, SearchMessagesResponse, SendMessageRequest, SendMessageResponse, ShuttingDown, SubscribeRequest, SyncCompleted, - SyncRequest, SyncResponse, SyncStarted, TestAccountConnectionRequest, - TestAccountConnectionResponse, TlsMode as TlsModeProto, TriageProgress, TriageRequest, - UpdateAccountRequest, UpdateAccountResponse, UpdateAvailable, UpdateRuleRequest, - UpdateRuleResponse, ValidateRuleRequest, ValidateRuleResponse, VerifyChainRequest, - VerifyChainResponse, + SyncRequest, SyncResponse, SyncStarted, SyncState as SyncStateProto, + TestAccountConnectionRequest, TestAccountConnectionResponse, TlsMode as TlsModeProto, + TriageProgress, TriageRequest, UpdateAccountRequest, UpdateAccountResponse, UpdateAvailable, + UpdateRuleRequest, UpdateRuleResponse, ValidateRuleRequest, ValidateRuleResponse, + VerifyChainRequest, VerifyChainResponse, }; use nuncio_store::db::DatabaseEngine; use nuncio_store::search::SearchEngine; use nuncio_store::vault::SecretManager; use std::pin::Pin; use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use subtle::ConstantTimeEq; use thiserror::Error; use tokio::net::TcpListener; @@ -216,10 +219,36 @@ fn map_core_event(event: CoreEvent) -> Option { Some(Event { kind: Some(kind) }) } +/// Maps a dispatcher [`AccountSyncPhase`] onto its wire `nuncio.v1.SyncState`. +fn map_sync_phase_to_proto(phase: AccountSyncPhase) -> SyncStateProto { + match phase { + AccountSyncPhase::Idle => SyncStateProto::Idle, + AccountSyncPhase::Syncing => SyncStateProto::Syncing, + AccountSyncPhase::Error => SyncStateProto::Error, + } +} + +/// Converts a wall-clock [`SystemTime`] (an account's last successful sync +/// instant) into a wire `Timestamp` at second granularity. A pre-epoch time +/// (never produced here) maps to `None` rather than a negative instant. +fn system_time_to_timestamp(t: SystemTime) -> Option { + t.duration_since(UNIX_EPOCH) + .ok() + .and_then(|d| i64::try_from(d.as_secs()).ok()) + .map(nuncio_proto::time::timestamp_from_unix_secs) +} + /// `nuncio.v1.System` gRPC service implementation backed by the daemon's live -/// [`EventBus`] state. +/// [`EventBus`] state, message store, and sync dispatcher. struct SystemGrpcService { event_bus: Arc, + db: Arc, + sync_dispatcher: Arc, + /// Captured once when the service is constructed at daemon startup; + /// `GetStatus` reports the elapsed time since as process uptime. An + /// `Instant` (monotonic) rather than a wall clock, so a system time + /// adjustment can never make uptime jump or go negative. + started_at: Instant, } #[tonic::async_trait] @@ -229,9 +258,68 @@ impl System for SystemGrpcService { _request: Request, ) -> Result, Status> { let state = self.event_bus.current_state(); + + // Cheap `SELECT 1` liveness probe; the sole source of `db_healthy`. + let db_healthy = self.db.ping().await.is_ok(); + + // Enumerate configured accounts live from the store: their count is + // `accounts_loaded`, and each one's live sync phase comes from the + // shared dispatcher's in-flight/last-result tracking. A read failure + // degrades to an empty set (already reflected by `db_healthy`) rather + // than fabricating a count. + let accounts = match self.db.list_accounts().await { + Ok(accounts) => accounts, + Err(e) => { + tracing::warn!("GetStatus: failed to list accounts for health surface: {e}"); + Vec::new() + } + }; + let accounts_loaded = accounts.len() as u64; + let account_sync_states = accounts + .iter() + .map(|account| { + let status = self.sync_dispatcher.sync_status(&account.id); + AccountSyncStateProto { + account_id: account.id.clone(), + state: map_sync_phase_to_proto(status.phase) as i32, + last_synced: status.last_success_at.and_then(system_time_to_timestamp), + last_error: status.last_error, + } + }) + .collect(); + + let unread_count = match self.db.count_unread_messages().await { + Ok(count) => count, + Err(e) => { + tracing::warn!("GetStatus: failed to count unread messages: {e}"); + 0 + } + }; + let outbox_depth = match self.db.count_pending_mutations().await { + Ok(depth) => depth, + Err(e) => { + tracing::warn!("GetStatus: failed to count pending outbox mutations: {e}"); + 0 + } + }; + + // Ready to serve normally: the database answered its liveness probe + // and the engine is not tearing down. + let ready = db_healthy && state.status != EngineStatus::ShuttingDown; + Ok(Response::new(GetStatusResponse { engine_status: format!("{:?}", state.status), version: env!("CARGO_PKG_VERSION").to_string(), + uptime: Some(nuncio_proto::time::duration_from_std( + self.started_at.elapsed(), + )), + accounts_loaded, + unread_count, + last_error: state.last_error, + outbox_depth, + account_sync_states, + ready, + db_healthy, })) } @@ -2890,6 +2978,9 @@ pub async fn serve_on_listener_with_overrides_and_shutdown( let system_service = SystemGrpcService { event_bus: event_bus.clone(), + db: db.clone(), + sync_dispatcher: sync_dispatcher.clone(), + started_at: Instant::now(), }; let system_interceptor = BearerAuthInterceptor::new(token.clone()); let system_svc = InterceptedService::new( @@ -3521,6 +3612,91 @@ mod tests { .into_inner(); assert_eq!(response.engine_status, "Syncing"); assert_eq!(response.version, env!("CARGO_PKG_VERSION")); + // Live health surface against a fresh in-memory daemon: the database + // answered its liveness probe, uptime is genuinely elapsing, and with + // no accounts configured the counts are honestly zero. + assert!(response.db_healthy, "ephemeral db must answer SELECT 1"); + assert!(response.ready, "healthy, non-shutting-down daemon is ready"); + let uptime = response.uptime.expect("uptime is always populated"); + assert!( + uptime.seconds > 0 || uptime.nanos > 0, + "uptime must be strictly positive, got {uptime:?}" + ); + assert_eq!(response.accounts_loaded, 0); + assert_eq!(response.unread_count, 0); + assert_eq!(response.outbox_depth, 0); + assert!(response.account_sync_states.is_empty()); + } + + #[tokio::test] + async fn get_status_reports_live_accounts_and_outbox_depth() { + // Back the health numbers with real persisted state: two configured + // accounts and one queued outbox mutation must be reflected exactly. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + let db = Arc::new(db); + + for id in ["acct-h1", "acct-h2"] { + let account = nuncio_core::AccountConfig { + id: id.to_string(), + name: "Health".to_string(), + email_address: format!("{id}@nuncio.mx"), + keyring_secret_key: format!("nuncio/{id}"), + sync_interval_secs: 60, + transport: nuncio_core::Transport::Jmap(nuncio_core::JmapTransport { + endpoint_host: "jmap.nuncio.mx".to_string(), + }), + }; + db.save_account(&account).await.expect("save account"); + } + + db.save_pending_mutation(&nuncio_filter::PendingRemoteMutation { + id: "mut-1".to_string(), + rule_id: "rule-1".to_string(), + message_id: "msg-1".to_string(), + mutation_type: "MOVE".to_string(), + payload: "{}".to_string(), + status: "pending".to_string(), + retry_count: 0, + created_at: 0, + }) + .await + .expect("queue an outbox mutation"); + + let event_bus = Arc::new(EventBus::new()); + let secrets = Arc::new(SecretManager::mock()); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + let (addr, _handle) = + spawn_test_server_with(event_bus, db, filter_engine, secrets, "correct-token").await; + + let mut client = SystemClient::connect(format!("http://{addr}")) + .await + .expect("client connects"); + let mut request = Request::new(GetStatusRequest {}); + request.metadata_mut().insert( + "authorization", + "Bearer correct-token" + .parse() + .expect("valid ascii metadata value"), + ); + let response = client + .get_status(request) + .await + .expect("valid token is accepted") + .into_inner(); + + assert!(response.db_healthy); + assert_eq!(response.accounts_loaded, 2); + assert_eq!(response.outbox_depth, 1); + // Each configured account has a per-account sync state; with no sync + // run yet every one is IDLE with no last-synced/last-error. + assert_eq!(response.account_sync_states.len(), 2); + for state in &response.account_sync_states { + assert_eq!(state.state, SyncStateProto::Idle as i32); + assert!(state.last_synced.is_none()); + assert!(state.last_error.is_none()); + } } #[tokio::test] diff --git a/crates/nunciod/src/sync_dispatcher.rs b/crates/nunciod/src/sync_dispatcher.rs index 8e46819..4526da0 100644 --- a/crates/nunciod/src/sync_dispatcher.rs +++ b/crates/nunciod/src/sync_dispatcher.rs @@ -45,7 +45,7 @@ use nuncio_store::db::DatabaseEngine; use nuncio_store::vault::SecretManager; use std::collections::HashMap; use std::sync::{Arc, Mutex, PoisonError}; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use thiserror::Error; use tokio::sync::{watch, Semaphore}; use tracing::Instrument; @@ -96,6 +96,42 @@ pub enum SyncDispatchError { Sync(Arc), } +/// Live sync phase of a single account, derived from the dispatcher's +/// in-flight map and last-result tracking for the daemon health surface. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AccountSyncPhase { + /// No sync in flight; the last attempt (if any) succeeded. + Idle, + /// A sync for the account is currently running. + Syncing, + /// No sync in flight, but the most recent attempt failed. + Error, +} + +/// A snapshot of one account's sync state, combining its current phase with +/// the outcome of its most recent completed attempt. `last_success_at` +/// survives a subsequent failure so a transient error does not erase the +/// last known-good sync time. +#[derive(Debug, Clone)] +pub struct AccountSyncStatus { + /// Current phase. + pub phase: AccountSyncPhase, + /// When the account last completed a SUCCESSFUL sync, if ever since start. + pub last_success_at: Option, + /// The most recent attempt's failure detail, present only when the last + /// attempt failed. + pub last_error: Option, +} + +/// The recorded outcome of an account's most recent completed sync attempt. +/// A success clears `last_error` and stamps `last_success_at`; a failure sets +/// `last_error` while leaving any earlier `last_success_at` intact. +#[derive(Default, Clone)] +struct AccountSyncRecord { + last_success_at: Option, + last_error: Option, +} + /// Injectable seam for the actual work of syncing one account. Production /// uses [`ProductionAccountSyncer`]; tests inject an in-memory mock. #[async_trait] @@ -179,6 +215,10 @@ struct Inner { /// that publishes the sync's outcome once. A second request for the same /// account clones the receiver and joins the in-flight sync. inflight: Mutex>>>, + /// The most recent completed outcome per account, retained after the + /// in-flight entry is cleared so the health surface can report each + /// account's last sync time and last error. + last_results: Mutex>, per_account_timeout: Duration, shutdown: ShutdownSignal, } @@ -246,6 +286,7 @@ impl SyncDispatcher { syncer, semaphore: Arc::new(Semaphore::new(concurrency)), inflight: Mutex::new(HashMap::new()), + last_results: Mutex::new(HashMap::new()), per_account_timeout, shutdown, }), @@ -298,6 +339,25 @@ impl SyncDispatcher { tokio::spawn( async move { let outcome = inner.run_guarded(&id).await; + // Record this attempt's outcome for the health surface + // before clearing the in-flight entry: a success stamps + // the last-synced time and clears any prior error; a + // failure records the error while preserving the last + // known-good sync time. + { + let mut results = inner + .last_results + .lock() + .unwrap_or_else(PoisonError::into_inner); + let record = results.entry(id.clone()).or_default(); + match &outcome { + Ok(_) => { + record.last_success_at = Some(SystemTime::now()); + record.last_error = None; + } + Err(e) => record.last_error = Some(e.to_string()), + } + } // Remove the in-flight entry before publishing so a // request arriving after completion starts a fresh sync // rather than joining an already-finished one. @@ -325,6 +385,41 @@ impl SyncDispatcher { published.unwrap_or(Err(SyncDispatchError::Cancelled)) } + /// Report the live sync state of a single account for the daemon health + /// surface: `Syncing` while a sync is in flight, otherwise `Error` if the + /// most recent attempt failed, else `Idle`. `last_success_at`/`last_error` + /// carry the genuine outcome of the last completed attempt (never + /// fabricated); an account that has not synced since daemon start reports + /// `Idle` with neither set. + pub fn sync_status(&self, account_id: &str) -> AccountSyncStatus { + let syncing = self + .inner + .inflight + .lock() + .unwrap_or_else(PoisonError::into_inner) + .contains_key(account_id); + let record = self + .inner + .last_results + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(account_id) + .cloned() + .unwrap_or_default(); + let phase = if syncing { + AccountSyncPhase::Syncing + } else if record.last_error.is_some() { + AccountSyncPhase::Error + } else { + AccountSyncPhase::Idle + }; + AccountSyncStatus { + phase, + last_success_at: record.last_success_at, + last_error: record.last_error, + } + } + /// Dispatch syncs for several accounts concurrently, each through the same /// bounded/coalesced/timed per-account path, and return the total number /// of messages synced across the accounts that succeeded. Per-account @@ -598,6 +693,67 @@ mod tests { ); } + #[tokio::test] + async fn sync_status_reports_idle_for_an_unknown_account() { + let syncer = Arc::new(MockSyncer::new()); + let dispatcher = dispatcher_with(syncer, 2, Duration::from_secs(30)); + + let status = dispatcher.sync_status("never-seen"); + assert_eq!(status.phase, AccountSyncPhase::Idle); + assert!(status.last_success_at.is_none()); + assert!(status.last_error.is_none()); + } + + #[tokio::test] + async fn sync_status_reports_syncing_while_a_sync_is_in_flight() { + let syncer = Arc::new(MockSyncer::new().blocking("acct-live")); + let dispatcher = dispatcher_with(syncer.clone(), 2, Duration::from_secs(30)); + + let d = dispatcher.clone(); + let handle = tokio::spawn(async move { d.sync_account("acct-live").await }); + + // Wait until the mock has actually entered the sync (it blocks there). + while syncer.call_count() == 0 { + sleep(Duration::from_millis(5)).await; + } + assert_eq!( + dispatcher.sync_status("acct-live").phase, + AccountSyncPhase::Syncing + ); + + syncer.gate.notify_one(); + let _ = handle.await; + } + + #[tokio::test] + async fn sync_status_reports_idle_with_last_synced_after_success() { + let syncer = Arc::new(MockSyncer::new()); + let dispatcher = dispatcher_with(syncer, 2, Duration::from_secs(30)); + + assert!(matches!(dispatcher.sync_account("acct-ok").await, Ok(1))); + + let status = dispatcher.sync_status("acct-ok"); + assert_eq!(status.phase, AccountSyncPhase::Idle); + assert!(status.last_success_at.is_some()); + assert!(status.last_error.is_none()); + } + + #[tokio::test] + async fn sync_status_reports_error_after_a_failed_attempt() { + let syncer = Arc::new(MockSyncer::new().failing("acct-bad")); + let dispatcher = dispatcher_with(syncer, 2, Duration::from_secs(30)); + + assert!(matches!( + dispatcher.sync_account("acct-bad").await, + Err(SyncDispatchError::Sync(_)) + )); + + let status = dispatcher.sync_status("acct-bad"); + assert_eq!(status.phase, AccountSyncPhase::Error); + assert!(status.last_error.is_some()); + assert!(status.last_success_at.is_none()); + } + #[tokio::test] async fn rejects_new_work_once_shutdown_is_requested() { let event_bus = Arc::new(EventBus::new()); -- 2.54.0 From ee63a62f17256fa4d0bf1afe63e3c0974c142180 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 19:41:49 -0500 Subject: [PATCH 44/95] feat(cli): wire verbosity flags and render typed daemon errors (#377) Wire -v/-vv/-vvv to a real stderr tracing subscriber (previously parsed but never used), with NUNCIO_LOG/RUST_LOG override precedence matching the daemon. Decode the WS-A1 ErrorInfo from a failed RPC's status details and render the gRPC code plus the typed reason alongside the message, in both text and --json output, instead of dropping the code or dumping a raw Status. Document daemon log levels, the rotating log file location, JSON log output, and request_id correlation in docs/RUNNING.md. Refs #365 Co-authored-by: James Maes --- Cargo.lock | 1 + crates/nuncio-cli/Cargo.toml | 1 + crates/nuncio-cli/src/args.rs | 17 +- crates/nuncio-cli/src/lib.rs | 1 + crates/nuncio-cli/src/logging.rs | 136 +++++++++ crates/nuncio-cli/src/main.rs | 8 +- crates/nuncio-cli/src/output.rs | 96 ++++++- crates/nuncio-cli/src/runner.rs | 463 +++++++++++++++---------------- docs/RUNNING.md | 84 ++++++ 9 files changed, 556 insertions(+), 251 deletions(-) create mode 100644 crates/nuncio-cli/src/logging.rs diff --git a/Cargo.lock b/Cargo.lock index 042c091..0e56aca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2534,6 +2534,7 @@ dependencies = [ "tokio-stream", "tonic", "tracing", + "tracing-subscriber", ] [[package]] diff --git a/crates/nuncio-cli/Cargo.toml b/crates/nuncio-cli/Cargo.toml index f680cdc..31eb58e 100644 --- a/crates/nuncio-cli/Cargo.toml +++ b/crates/nuncio-cli/Cargo.toml @@ -25,6 +25,7 @@ serde.workspace = true serde_json.workspace = true thiserror.workspace = true tracing.workspace = true +tracing-subscriber.workspace = true rpassword.workspace = true chrono.workspace = true hex.workspace = true diff --git a/crates/nuncio-cli/src/args.rs b/crates/nuncio-cli/src/args.rs index 4994093..b4059af 100644 --- a/crates/nuncio-cli/src/args.rs +++ b/crates/nuncio-cli/src/args.rs @@ -70,14 +70,18 @@ pub struct Cli { #[arg(long, global = true, help = "Target a specific configured account ID")] pub account: Option, - /// Enable verbose log output to stderr. + /// Increase log verbosity on stderr. Repeatable: `-v` = info, `-vv` = + /// debug, `-vvv` = trace. Omitted entirely, the CLI logs only warnings. + /// `NUNCIO_LOG`/`RUST_LOG`, if set, override this (see + /// [`crate::logging`]). #[arg( short, long, global = true, - help = "Enable detailed verbose execution logs on stderr" + action = clap::ArgAction::Count, + help = "Increase log verbosity on stderr (-v info, -vv debug, -vvv trace)" )] - pub verbose: bool, + pub verbose: u8, /// Subcommand resource noun. #[command(subcommand)] @@ -594,7 +598,7 @@ mod tests { fn parse_pure_noun_verb_mail_commands() { let cli_sync = Cli::parse_from(["nuncio", "--json", "--verbose", "mail", "sync"]); assert!(cli_sync.json); - assert!(cli_sync.verbose); + assert_eq!(cli_sync.verbose, 1); assert_eq!( cli_sync.command, Commands::Mail { @@ -602,6 +606,11 @@ mod tests { } ); + // `-v` is repeatable and counts occurrences, mapped to a log level by + // `nuncio_cli::logging::verbosity_directive`. + let cli_vv = Cli::parse_from(["nuncio", "-vv", "mail", "sync"]); + assert_eq!(cli_vv.verbose, 2); + let cli_read = Cli::parse_from(["nuncio", "mail", "read", "--id", "msg-123"]); assert_eq!( cli_read.command, diff --git a/crates/nuncio-cli/src/lib.rs b/crates/nuncio-cli/src/lib.rs index a5b9de4..3b35cac 100644 --- a/crates/nuncio-cli/src/lib.rs +++ b/crates/nuncio-cli/src/lib.rs @@ -3,6 +3,7 @@ #![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::panic))] pub mod args; +pub mod logging; pub mod output; pub mod runner; diff --git a/crates/nuncio-cli/src/logging.rs b/crates/nuncio-cli/src/logging.rs new file mode 100644 index 0000000..e97dfa7 --- /dev/null +++ b/crates/nuncio-cli/src/logging.rs @@ -0,0 +1,136 @@ +//! CLI-local `tracing` subscriber. +//! +//! This controls `nuncio-cli`'s OWN log output -- it is entirely separate +//! from `nunciod`'s own layered subscriber (`nunciod::logging`), which +//! governs the daemon process the CLI merely talks to over gRPC. The CLI +//! prints its normal command output on stdout regardless of this +//! configuration; this subscriber only ever writes to stderr, so scripts that +//! parse stdout (especially `--json` mode) are never polluted by log lines. +//! +//! ## Verbosity mapping +//! +//! `-v`/`--verbose` is a repeatable flag (`clap`'s `ArgAction::Count`), not a +//! boolean: `nuncio ` with none is quiet (`warn`), `-v` is `info`, `-vv` +//! is `debug`, and `-vvv` or more is `trace`. [`verbosity_directive`] is the +//! pure mapping and is unit-tested directly, without touching the process's +//! single global subscriber slot. +//! +//! ## Environment override +//! +//! `NUNCIO_LOG` (if set and non-blank) wins over `RUST_LOG`, which wins over +//! the `-v` count -- mirroring `nunciod`'s own precedence so the two +//! processes behave consistently for an operator who sets `NUNCIO_LOG` +//! globally. + +use tracing_subscriber::EnvFilter; + +/// Environment variable that overrides the log filter, taking precedence over +/// `RUST_LOG` and the `-v` count. +pub const LOG_FILTER_ENV_VAR: &str = "NUNCIO_LOG"; + +/// Maps a `-v` occurrence count to a filter directive. `0` is the quiet +/// default (`warn`); each additional `-v` steps up one level, capping at +/// `trace`. +#[must_use] +pub fn verbosity_directive(verbosity: u8) -> &'static str { + match verbosity { + 0 => "warn", + 1 => "info", + 2 => "debug", + _ => "trace", + } +} + +/// Resolves the effective filter directive with precedence `NUNCIO_LOG` > +/// `RUST_LOG` > the `-v` count (via [`verbosity_directive`]). +/// +/// An env var set to an empty/whitespace-only string is treated as unset so a +/// stray `NUNCIO_LOG=` does not silently override an explicit `-v`. +#[must_use] +pub fn resolve_filter_directive(verbosity: u8) -> String { + env_directive(LOG_FILTER_ENV_VAR) + .or_else(|| env_directive("RUST_LOG")) + .unwrap_or_else(|| verbosity_directive(verbosity).to_string()) +} + +fn env_directive(var: &str) -> Option { + match std::env::var(var) { + Ok(value) if !value.trim().is_empty() => Some(value), + _ => None, + } +} + +/// Installs a stderr-only `tracing` subscriber for the CLI process, driven by +/// [`resolve_filter_directive`]. +/// +/// Never panics: a malformed filter directive (from a hand-edited +/// `NUNCIO_LOG`/`RUST_LOG`) degrades to the `-v`-derived default rather than +/// crashing the CLI before it can even report an error. Idempotent -- if a +/// subscriber is already installed (e.g. a test harness), this is a silent +/// no-op rather than a panic, since `nuncio-cli`'s own tests run many +/// commands in the same process. +pub fn init(verbosity: u8) { + let directive = resolve_filter_directive(verbosity); + let filter = EnvFilter::try_new(&directive) + .unwrap_or_else(|_| EnvFilter::new(verbosity_directive(verbosity))); + let _ = tracing_subscriber::fmt() + .with_env_filter(filter) + .with_writer(std::io::stderr) + .try_init(); +} + +#[cfg(test)] +mod tests { + use super::*; + + // The process environment is global mutable state; serialize every test + // that reads or writes the log env vars so they cannot interleave. + static ENV_GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn clear_env() { + std::env::remove_var(LOG_FILTER_ENV_VAR); + std::env::remove_var("RUST_LOG"); + } + + #[test] + fn verbosity_count_maps_to_expected_level() { + assert_eq!(verbosity_directive(0), "warn"); + assert_eq!(verbosity_directive(1), "info"); + assert_eq!(verbosity_directive(2), "debug"); + assert_eq!(verbosity_directive(3), "trace"); + // Anything beyond -vvv still caps at trace rather than erroring. + assert_eq!(verbosity_directive(255), "trace"); + } + + #[test] + fn nuncio_log_beats_rust_log_beats_verbosity_count() { + let _guard = ENV_GUARD.lock().unwrap_or_else(|e| e.into_inner()); + clear_env(); + + // No env override: falls back to the -v count mapping. + assert_eq!(resolve_filter_directive(0), "warn"); + assert_eq!(resolve_filter_directive(2), "debug"); + + // RUST_LOG wins over the -v count. + std::env::set_var("RUST_LOG", "nuncio_cli=warn"); + assert_eq!(resolve_filter_directive(2), "nuncio_cli=warn"); + + // NUNCIO_LOG wins over RUST_LOG. + std::env::set_var(LOG_FILTER_ENV_VAR, "nuncio_cli=trace"); + assert_eq!(resolve_filter_directive(0), "nuncio_cli=trace"); + + // An empty NUNCIO_LOG is treated as unset, so RUST_LOG applies again. + std::env::set_var(LOG_FILTER_ENV_VAR, " "); + assert_eq!(resolve_filter_directive(0), "nuncio_cli=warn"); + + clear_env(); + } + + #[test] + fn init_never_panics_even_when_a_subscriber_is_already_installed() { + // Calling init() twice must not panic: the second call finds the + // global subscriber slot occupied and silently no-ops. + init(1); + init(1); + } +} diff --git a/crates/nuncio-cli/src/main.rs b/crates/nuncio-cli/src/main.rs index cf58f64..15d87e7 100644 --- a/crates/nuncio-cli/src/main.rs +++ b/crates/nuncio-cli/src/main.rs @@ -1,10 +1,16 @@ use clap::Parser; -use nuncio_cli::{args::Cli, AccountSubcommand, Commands, HeadlessRunner, PasswordArg}; +use nuncio_cli::{args::Cli, logging, AccountSubcommand, Commands, HeadlessRunner, PasswordArg}; #[tokio::main] async fn main() -> Result<(), Box> { let mut cli = Cli::parse(); + // Wire the CLI's own `-v`/`-vv`/`-vvv` flag (or `NUNCIO_LOG`/`RUST_LOG`) + // to a real stderr subscriber before anything else runs, so every RPC + // this process makes is followable. This is the CLI's own log stream -- + // entirely separate from the daemon's file-based logging. + logging::init(cli.verbose); + // `account add`'s password is deliberately never a Clap-parsed CLI // flag (see `PasswordArg`'s doc comment): read it here, interactively // and without echo, immediately after parsing argv and before any diff --git a/crates/nuncio-cli/src/output.rs b/crates/nuncio-cli/src/output.rs index 7281ea5..789b311 100644 --- a/crates/nuncio-cli/src/output.rs +++ b/crates/nuncio-cli/src/output.rs @@ -1,5 +1,7 @@ //! Standardized JSON machine-readable output formatters for Unix scripting. +use std::collections::BTreeMap; + use serde::Serialize; /// Standardized JSON response envelope for CLI machine output. @@ -13,6 +15,18 @@ pub struct JsonResponse { /// Error message string (present if status is "error"). #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, + /// The gRPC status code name (e.g. `"NotFound"`), when the error came + /// from a daemon RPC. + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, + /// The typed `nuncio.v1.ErrorReason` name (e.g. `"ACCOUNT_NOT_FOUND"`), + /// decoded from the daemon's `ErrorInfo` status details, when present. + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// Structured error context (e.g. `{"account_id": "acct-1"}`) carried in + /// the daemon's `ErrorInfo`, when present and non-empty. + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata: Option>, } impl JsonResponse { @@ -22,17 +36,47 @@ impl JsonResponse { status: "ok", data: Some(data), error: None, + code: None, + reason: None, + metadata: None, } } } impl JsonResponse<()> { - /// Construct an error JSON response wrapping an error string. + /// Construct an error JSON response wrapping an error string, with no + /// typed gRPC code/reason/metadata (e.g. a purely client-side failure + /// such as an unreachable daemon). pub fn error(message: &str) -> Self { Self { status: "error", data: None, error: Some(message.to_string()), + code: None, + reason: None, + metadata: None, + } + } + + /// Construct an error JSON response carrying the gRPC code and, when the + /// daemon attached a WS-A1 `ErrorInfo`, the typed reason and metadata. + pub fn error_with_info( + message: &str, + code: Option, + reason: Option, + metadata: BTreeMap, + ) -> Self { + Self { + status: "error", + data: None, + error: Some(message.to_string()), + code, + reason, + metadata: if metadata.is_empty() { + None + } else { + Some(metadata) + }, } } } @@ -51,6 +95,20 @@ pub fn format_json_error(message: &str) -> String { .unwrap_or_else(|_| r#"{"status":"error","error":"JSON serialization failed"}"#.to_string()) } +/// Serialize a typed daemon RPC error (gRPC code, optional `ErrorReason` +/// name, and optional metadata) to a formatted JSON string. See +/// [`JsonResponse::error_with_info`]. +pub fn format_json_error_with_info( + message: &str, + code: Option, + reason: Option, + metadata: BTreeMap, +) -> String { + let response = JsonResponse::error_with_info(message, code, reason, metadata); + serde_json::to_string(&response) + .unwrap_or_else(|_| r#"{"status":"error","error":"JSON serialization failed"}"#.to_string()) +} + /// Print ANSI color ASCII art splash banner for CLI startup. pub fn print_splash_banner() { let banner = get_splash_banner_text(); @@ -94,6 +152,42 @@ mod tests { let err_str = format_json_error("account not found"); assert!(err_str.contains(r#""status":"error""#)); assert!(err_str.contains(r#""error":"account not found""#)); + // A plain error carries no typed code/reason/metadata. + assert!(!err_str.contains("\"code\"")); + assert!(!err_str.contains("\"reason\"")); + assert!(!err_str.contains("\"metadata\"")); + } + + #[test] + fn format_json_error_with_info_carries_code_reason_and_metadata() { + let mut metadata = BTreeMap::new(); + metadata.insert("account_id".to_string(), "acct-1".to_string()); + + let out = format_json_error_with_info( + "nunciod daemon rejected account_show: account 'acct-1' not found", + Some("NotFound".to_string()), + Some("ACCOUNT_NOT_FOUND".to_string()), + metadata, + ); + assert!(out.contains(r#""status":"error""#)); + assert!(out.contains(r#""code":"NotFound""#)); + assert!(out.contains(r#""reason":"ACCOUNT_NOT_FOUND""#)); + assert!(out.contains(r#""account_id":"acct-1""#)); + } + + #[test] + fn format_json_error_with_info_omits_reason_and_metadata_when_absent() { + // Falls back cleanly when the status carried no typed `ErrorInfo`: + // still structured (code present), but no fabricated reason/metadata. + let out = format_json_error_with_info( + "daemon unreachable", + Some("Unavailable".to_string()), + None, + BTreeMap::new(), + ); + assert!(out.contains(r#""code":"Unavailable""#)); + assert!(!out.contains("\"reason\"")); + assert!(!out.contains("\"metadata\"")); } #[test] diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index 65571d5..cf063c7 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -12,7 +12,7 @@ use crate::args::{ FilterSubcommand, FolderSubcommand, MailSubcommand, SystemSubcommand, UpdateSubcommand, }; -use crate::output::{format_json, format_json_error}; +use crate::output::{format_json, format_json_error, format_json_error_with_info}; /// Parses a `--imap-mode`/`--smtp-mode` CLI string into `nuncio_core::TlsMode`. /// Rejects anything else rather than silently defaulting, since silently @@ -777,13 +777,7 @@ impl HeadlessRunner { format!("Synchronization complete: {synced_count} message(s) synced") } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected sync: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("sync", &status, json_mode), } } @@ -820,13 +814,7 @@ impl HeadlessRunner { page_token = page.next_page_token; } Err(status) => { - return Self::render_error( - &format!( - "nunciod daemon rejected list_messages: {}", - Self::clean_status_message(&status) - ), - json_mode, - ); + return Self::render_status_error("list_messages", &status, json_mode); } } } @@ -892,13 +880,7 @@ impl HeadlessRunner { ) } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected send_message: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("send_message", &status, json_mode), } } @@ -932,13 +914,7 @@ impl HeadlessRunner { page_token = page.next_page_token; } Err(status) => { - return Self::render_error( - &format!( - "nunciod daemon rejected search_messages: {}", - Self::clean_status_message(&status) - ), - json_mode, - ); + return Self::render_status_error("search_messages", &status, json_mode); } } } @@ -991,13 +967,7 @@ impl HeadlessRunner { page_token = page.next_page_token; } Err(status) => { - return Self::render_error( - &format!( - "nunciod daemon rejected list_folders: {}", - Self::clean_status_message(&status) - ), - json_mode, - ); + return Self::render_status_error("list_folders", &status, json_mode); } } } @@ -1055,13 +1025,7 @@ impl HeadlessRunner { Err(status) if status.code() == tonic::Code::NotFound => { Self::render_error(&format!("message '{}' not found", id), json_mode) } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected get_message: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("get_message", &status, json_mode), } } @@ -1111,13 +1075,7 @@ impl HeadlessRunner { Err(status) if status.code() == tonic::Code::NotFound => { Self::render_error(&format!("message '{}' not found", id), json_mode) } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected mark_read: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("mark_read", &status, json_mode), } } @@ -1181,13 +1139,7 @@ impl HeadlessRunner { ) } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected export_mailbox: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("export_mailbox", &status, json_mode), } } @@ -1238,13 +1190,7 @@ impl HeadlessRunner { page_token = page.next_page_token; } Err(status) => { - return Self::render_error( - &format!( - "nunciod daemon rejected list_records: {}", - Self::clean_status_message(&status) - ), - json_mode, - ); + return Self::render_status_error("list_records", &status, json_mode); } } } @@ -1305,13 +1251,7 @@ impl HeadlessRunner { ) } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected verify_chain: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("verify_chain", &status, json_mode), } } @@ -1363,13 +1303,7 @@ impl HeadlessRunner { page_token = page.next_page_token; } Err(status) => { - return Self::render_error( - &format!( - "nunciod daemon rejected list_rules: {}", - Self::clean_status_message(&status) - ), - json_mode, - ); + return Self::render_status_error("list_rules", &status, json_mode); } } } @@ -1431,16 +1365,7 @@ impl HeadlessRunner { json_mode, ), }, - Err(status) if status.code() == tonic::Code::InvalidArgument => { - Self::render_error(status.message(), json_mode) - } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected create_rule: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("create_rule", &status, json_mode), } } @@ -1467,13 +1392,7 @@ impl HeadlessRunner { format!("✓ Filter rule '{}' deleted.", id) } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected delete_rule: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("delete_rule", &status, json_mode), } } @@ -1505,13 +1424,7 @@ impl HeadlessRunner { format!("Validation Error: {}", response.error) } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected validate_rule: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("validate_rule", &status, json_mode), } } @@ -1560,16 +1473,7 @@ impl HeadlessRunner { ) } } - Err(status) if status.code() == tonic::Code::InvalidArgument => { - Self::render_error(status.message(), json_mode) - } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected preview_rule: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("preview_rule", &status, json_mode), } } @@ -1612,19 +1516,7 @@ impl HeadlessRunner { json_mode, ), }, - Err(status) - if status.code() == tonic::Code::InvalidArgument - || status.code() == tonic::Code::NotFound => - { - Self::render_error(status.message(), json_mode) - } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected update_rule: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("update_rule", &status, json_mode), } } @@ -1657,13 +1549,7 @@ impl HeadlessRunner { content } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected export_rules: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("export_rules", &status, json_mode), } } @@ -1711,13 +1597,7 @@ impl HeadlessRunner { out } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected import_rules: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("import_rules", &status, json_mode), } } @@ -1762,13 +1642,7 @@ impl HeadlessRunner { out } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected get_execution_logs: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("get_execution_logs", &status, json_mode), } } @@ -1796,15 +1670,7 @@ impl HeadlessRunner { .await { Ok(response) => response.into_inner(), - Err(status) => { - return Self::render_error( - &format!( - "nunciod daemon rejected triage: {}", - Self::clean_status_message(&status) - ), - json_mode, - ) - } + Err(status) => return Self::render_status_error("triage", &status, json_mode), }; let mut updates = Vec::new(); @@ -1829,13 +1695,7 @@ impl HeadlessRunner { } Ok(None) => break, Err(status) => { - return Self::render_error( - &format!( - "nunciod daemon triage stream failed: {}", - Self::clean_status_message(&status) - ), - json_mode, - ) + return Self::render_status_error("triage stream", &status, json_mode) } } } @@ -2041,13 +1901,7 @@ impl HeadlessRunner { ) } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected add_account: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("add_account", &status, json_mode), } } @@ -2101,13 +1955,7 @@ impl HeadlessRunner { out } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected list_accounts: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("list_accounts", &status, json_mode), } } @@ -2124,12 +1972,7 @@ impl HeadlessRunner { let accounts = client .list_accounts(nuncio_proto::v1::ListAccountsRequest {}) .await - .map_err(|status| { - format!( - "nunciod daemon rejected list_accounts: {}", - Self::clean_status_message(&status) - ) - })? + .map_err(|status| Self::describe_status_error("list_accounts", &status))? .into_inner() .accounts; accounts @@ -2262,13 +2105,7 @@ impl HeadlessRunner { format!("Account '{id}' updated via nunciod daemon.") } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected update_account: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("update_account", &status, json_mode), } } @@ -2295,13 +2132,7 @@ impl HeadlessRunner { format!("Account '{id}' removed via nunciod daemon.") } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected remove_account: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("remove_account", &status, json_mode), } } @@ -2352,13 +2183,7 @@ impl HeadlessRunner { format!("Connection test for '{id}'\n {imap}\n {smtp}") } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected test_account_connection: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("test_account_connection", &status, json_mode), } } @@ -2455,13 +2280,7 @@ impl HeadlessRunner { page_token = page.next_page_token; } Err(status) => { - return Self::render_error( - &format!( - "nunciod daemon rejected list_events: {}", - Self::clean_status_message(&status) - ), - json_mode, - ); + return Self::render_status_error("list_events", &status, json_mode); } } } @@ -2517,13 +2336,7 @@ impl HeadlessRunner { format!("Calendar synchronization complete: {synced_count} event(s) synced") } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected calendar sync: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("calendar sync", &status, json_mode), } } @@ -2557,13 +2370,7 @@ impl HeadlessRunner { let contacts = match self.fetch_all_contacts(&mut client, account).await { Ok(contacts) => contacts, Err(status) => { - return Self::render_error( - &format!( - "nunciod daemon rejected list_contacts: {}", - Self::clean_status_message(&status) - ), - json_mode, - ); + return Self::render_status_error("list_contacts", &status, json_mode); } }; @@ -2620,13 +2427,7 @@ impl HeadlessRunner { let contacts = match self.fetch_all_contacts(&mut client, account).await { Ok(contacts) => contacts, Err(status) => { - return Self::render_error( - &format!( - "nunciod daemon rejected list_contacts: {}", - Self::clean_status_message(&status) - ), - json_mode, - ); + return Self::render_status_error("list_contacts", &status, json_mode); } }; @@ -2702,13 +2503,7 @@ impl HeadlessRunner { json_mode, ), }, - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected create_contact: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("create_contact", &status, json_mode), } } @@ -2739,13 +2534,7 @@ impl HeadlessRunner { format!("Contacts synchronization complete: {synced_count} contact(s) synced") } } - Err(status) => Self::render_error( - &format!( - "nunciod daemon rejected contacts sync: {}", - Self::clean_status_message(&status) - ), - json_mode, - ), + Err(status) => Self::render_status_error("contacts sync", &status, json_mode), } } @@ -2775,6 +2564,64 @@ impl HeadlessRunner { } } + /// Strips the `ERROR_REASON_` prefix off a decoded [`v1::ErrorReason`]'s + /// generated `as_str_name()`, so the CLI renders the short, readable form + /// (`ACCOUNT_NOT_FOUND`) rather than the wire enum's full name. + fn reason_name(reason: nuncio_proto::v1::ErrorReason) -> String { + reason + .as_str_name() + .trim_start_matches("ERROR_REASON_") + .to_string() + } + + /// Builds the full human-readable text for a failed daemon RPC: the + /// server-provided message, the gRPC status code, and -- when the daemon + /// attached a WS-A1 [`nuncio_proto::v1::ErrorInfo`] to the status details + /// -- the typed [`nuncio_proto::v1::ErrorReason`]. Falls back to just the + /// code when no typed details are present (e.g. a status raised by a + /// library this client talks to that predates `ErrorInfo`). + fn describe_status_error(context: &str, status: &tonic::Status) -> String { + let message = Self::clean_status_message(status); + let code = status.code(); + match nuncio_proto::errors::error_reason(status) { + Some(reason) => format!( + "nunciod daemon rejected {context}: {message} (code: {code:?}, reason: {})", + Self::reason_name(reason) + ), + None => format!("nunciod daemon rejected {context}: {message} (code: {code:?})"), + } + } + + /// Renders a failed daemon RPC for both text and `--json` output modes, + /// decoding the WS-A1 `ErrorInfo` from the status details when present. + /// + /// Text mode gets the human message, the gRPC code, and the typed reason + /// (see [`Self::describe_status_error`]). JSON mode gets a coherent + /// `{error, code, reason, metadata}` shape via + /// [`format_json_error_with_info`] -- `reason`/`metadata` are omitted + /// (rather than fabricated) when the status carried no typed + /// `ErrorInfo`, so a caller can tell "no reason given" apart from an + /// actual reason. + fn render_status_error(context: &str, status: &tonic::Status, json_mode: bool) -> String { + if json_mode { + let info = nuncio_proto::errors::error_info(status); + let message = format!( + "nunciod daemon rejected {context}: {}", + Self::clean_status_message(status) + ); + let reason = info.as_ref().map(|i| Self::reason_name(i.reason())); + let metadata = info.map(|i| i.metadata).unwrap_or_default(); + format_json_error_with_info( + &message, + Some(format!("{:?}", status.code())), + reason, + metadata.into_iter().collect(), + ) + } else { + format!("Error: {}", Self::describe_status_error(context, status)) + } + } + /// `system status`: a real thin gRPC client of the running `nunciod` /// daemon's `nuncio.v1.System` API. /// @@ -3951,6 +3798,132 @@ mod tests { assert!(!out.contains("status:")); } + /// Proves the CLI decodes a WS-A1 `ErrorInfo` from a daemon RPC's status + /// details and renders the human message, the gRPC code, and the typed + /// reason -- in both text and `--json` mode -- instead of dropping the + /// code or the reason the daemon went to the trouble of attaching. + #[tokio::test] + async fn typed_error_info_renders_code_reason_and_metadata_in_text_and_json() { + use nuncio_proto::v1::mail_server::{Mail as MailService, MailServer}; + use nuncio_proto::v1::ErrorReason; + use nuncio_proto::v1::{ + GetMessageRequest, GetMessageResponse, ListFoldersRequest, ListFoldersResponse, + ListMessagesRequest, ListMessagesResponse, MarkReadRequest, MarkReadResponse, + SearchMessagesRequest, SearchMessagesResponse, SendMessageRequest, SendMessageResponse, + SyncRequest, SyncResponse, + }; + + /// Minimal test-only stub of `nuncio.v1.Mail` whose `ListFolders` + /// always fails with a typed `ErrorInfo` (account-not-found), so this + /// test can assert on exactly how the CLI decodes and renders it. + struct StubMailTypedError; + + #[tonic::async_trait] + impl MailService for StubMailTypedError { + async fn list_folders( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(nuncio_proto::errors::status_with_metadata( + ErrorReason::AccountNotFound, + "account 'acct-missing' not found", + [("account_id".to_string(), "acct-missing".to_string())], + )) + } + + async fn list_messages( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("unused in this test")) + } + + async fn get_message( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("unused in this test")) + } + + async fn mark_read( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("unused in this test")) + } + + async fn search_messages( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("unused in this test")) + } + + async fn send_message( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("unused in this test")) + } + + async fn sync( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::internal("unused in this test")) + } + } + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("listener has local addr"); + tokio::spawn(async move { + let _ = tonic::transport::Server::builder() + .add_service(MailServer::new(StubMailTypedError)) + .serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new(listener)) + .await; + }); + + let runner = + HeadlessRunner::ephemeral_with(Arc::new(SecretManager::mock()), addr.to_string()) + .await + .expect("ephemeral runner initializes"); + + // Text mode: human message, gRPC code, and the typed reason -- not a + // raw status dump, and not silently dropping the code/reason. + let text_out = runner + .execute_command( + &Commands::Folder { + action: FolderSubcommand::List, + }, + false, + ) + .await; + assert!(text_out.contains("account 'acct-missing' not found")); + assert!(text_out.contains("code: NotFound")); + assert!(text_out.contains("reason: ACCOUNT_NOT_FOUND")); + + // JSON mode: a coherent {error, code, reason, metadata} shape. + let json_out = runner + .execute_command( + &Commands::Folder { + action: FolderSubcommand::List, + }, + true, + ) + .await; + let parsed: serde_json::Value = serde_json::from_str(&json_out).expect("valid JSON output"); + assert_eq!(parsed["status"], "error"); + assert!(parsed["error"] + .as_str() + .expect("error is a string") + .contains("account 'acct-missing' not found")); + assert_eq!(parsed["code"], "NotFound"); + assert_eq!(parsed["reason"], "ACCOUNT_NOT_FOUND"); + assert_eq!(parsed["metadata"]["account_id"], "acct-missing"); + } + /// Reference-client proof: boots a stub `nuncio.v1.Calendar` gRPC server /// (mirroring the `Mail` stub pattern above) and drives the real /// `HeadlessRunner`'s `cal list`/`cal sync` gRPC client paths against it. diff --git a/docs/RUNNING.md b/docs/RUNNING.md index b8b9dfc..481523d 100644 --- a/docs/RUNNING.md +++ b/docs/RUNNING.md @@ -69,6 +69,90 @@ nuncio-cli.exe folder list Add `--json` to any command for machine-readable output. `nuncio-cli.exe --help` lists every verb and flag. +## Following & auditing the daemon + +`nunciod` runs a layered `tracing` subscriber (`crates/nunciod/src/logging.rs`); +`nuncio-cli` runs its own, separate, stderr-only subscriber for its own logs. +Neither ever writes a bearer token or an account password to a log line. + +### Daemon log levels + +`nunciod`'s console (stderr) and rotating file sinks share one filter, +resolved once at startup with this precedence: + +1. **`NUNCIO_LOG`** — a `tracing-subscriber` `EnvFilter` directive (e.g. + `info`, `debug`, `nunciod=debug,tower=warn`). Wins over `RUST_LOG` if set + to a non-blank value. +2. **`RUST_LOG`** — same directive syntax; used when `NUNCIO_LOG` is + unset/blank. +3. Default: `info`. + +A malformed directive never crashes the daemon — it falls back to the +default and logs a `warn!` once the subscriber is live. + +``` +:: run the daemon at debug level for one session +set NUNCIO_LOG=debug +target\release\nunciod.exe +``` + +### Log file location + +The rotating file sink writes daily-rotated files next to the database, under +a `logs/` sibling directory: `\logs\nunciod.log.` (e.g. +`%USERPROFILE%\.nuncio\logs\nunciod.log.2026-07-30`). If the log directory +cannot be created, the daemon degrades to stderr-only logging (with a +`warn!`) rather than failing to start. + +### JSON log output + +Set `NUNCIO_LOG_FORMAT=json` to switch the **file** sink to one JSON object +per line, for machine ingestion (e.g. shipping to a log aggregator). The +console sink always stays human-readable, since it's for an operator watching +a terminal. Any other value (or unset) keeps the file sink human-readable too. + +### Correlating one request's logs (`request_id`) + +Every gRPC call `nunciod` handles opens a root tracing span carrying a +per-process `request_id` (`-`), the RPC method +path, and the peer address; every log line emitted while that call is +handled — auth accept/reject, handler-level events, the terminal +`gRPC request completed` line with status + elapsed time — inherits it. To +follow one request end-to-end, grep the daemon log for its `request_id`: + +``` +:: human-readable file sink +findstr "a1b2c3d4e5f6-00000017" "%USERPROFILE%\.nuncio\logs\nunciod.log.2026-07-30" + +:: JSON file sink (NUNCIO_LOG_FORMAT=json), pick a request_id from the +:: completion line's fields, then filter every line for it: +findstr "\"request_id\":\"a1b2c3d4e5f6-00000017\"" "%USERPROFILE%\.nuncio\logs\nunciod.log.2026-07-30" +``` + +(On macOS/Linux, use `grep` in place of `findstr`.) There is currently no way +to read a request's `request_id` back from the CLI's own output — it is +visible only in the daemon's logs, since it is a server-side correlation +identifier assigned when the RPC arrives. + +### CLI verbosity (`-v`/`-vv`/`-vvv`) + +`nuncio-cli` logs to stderr only — command output (including `--json` +payloads) always goes to stdout, so redirecting or piping stdout is never +polluted by log lines. By default the CLI logs only warnings. Repeat `-v` to +raise the level: + +``` +nuncio-cli.exe mail sync :: warn only (default) +nuncio-cli.exe -v mail sync :: info +nuncio-cli.exe -vv mail sync :: debug +nuncio-cli.exe -vvv mail sync :: trace +``` + +`NUNCIO_LOG`/`RUST_LOG`, if set (same precedence as the daemon: `NUNCIO_LOG` +beats `RUST_LOG`), override the `-v` count entirely — useful for scoping a +directive to a specific module (e.g. `NUNCIO_LOG=nuncio_cli=trace`) without +turning on trace logging for every dependency `-vvv` would also enable. + ## The dogfooding loop Use it for real; when something breaks or feels wrong, tell me (or I'll hit it -- 2.54.0 From 99351310ec81cb7c12826f9f2c069232c8e8b4cb Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 31 Jul 2026 20:07:03 -0500 Subject: [PATCH 45/95] feat(core): redaction policy, Redacted, and no-secrets-in-logs canary (#378) Add a Redacted newtype in nuncio-core whose Debug, Display, and serde render a fixed placeholder, so a wrapped credential cannot leak through a derived Debug or serde_json; the value is reachable only via an explicit expose_secret(). Migrate the CalDAV/CardDAV account auth tokens to it and drop their hand-written redacting Debug impls. Document the logging redaction policy (what must never be logged vs what is allowed) in docs/LOGGING.md and the redact module doc. Add a cross-crate no-secrets canary that installs a capturing subscriber, drives auth reject, account add, mock and JMAP-wiremock sync, mark-read, send, a filter rule-fire, a webhook dispatch, and CalDAV/CardDAV sync with sentinel secrets/PII, and asserts none appear in captured telemetry. Refs #363 Co-authored-by: James Maes --- crates/nuncio-cal/src/caldav.rs | 30 +- .../nuncio-cal/tests/wiremock_caldav_test.rs | 4 +- crates/nuncio-contacts/src/carddav.rs | 30 +- .../tests/wiremock_carddav_test.rs | 6 +- crates/nuncio-core/src/lib.rs | 2 + crates/nuncio-core/src/redact.rs | 203 +++++++ crates/nunciod/src/calendar_sync.rs | 2 +- .../nunciod/tests/no_secrets_canary_test.rs | 537 ++++++++++++++++++ docs/LOGGING.md | 56 ++ 9 files changed, 826 insertions(+), 44 deletions(-) create mode 100644 crates/nuncio-core/src/redact.rs create mode 100644 crates/nunciod/tests/no_secrets_canary_test.rs create mode 100644 docs/LOGGING.md diff --git a/crates/nuncio-cal/src/caldav.rs b/crates/nuncio-cal/src/caldav.rs index 055334e..3c5ee2a 100644 --- a/crates/nuncio-cal/src/caldav.rs +++ b/crates/nuncio-cal/src/caldav.rs @@ -2,6 +2,7 @@ use async_trait::async_trait; use nuncio_core::model::CalendarEvent; +use nuncio_core::redact::Redacted; use serde::{Deserialize, Serialize}; use tracing::{debug, info, instrument, warn}; @@ -14,7 +15,7 @@ use crate::parser::{CalendarError, IcalParserAdapter}; /// `caldav_url` must already resolve to a specific calendar collection (e.g. /// `https://caldav.example.com/dav/calendars/user/jmaes/work/`) -- PROPFIND-based /// `calendar-home-set` auto-discovery is out of scope for this client. -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct CalDavAccountConfig { /// Nuncio account identifier this calendar collection belongs to. pub account_id: String, @@ -23,22 +24,10 @@ pub struct CalDavAccountConfig { /// Basic-auth username (or app-specific username, per provider). pub username: String, /// Basic-auth secret (password or app-specific token) resolved from the OS keyring by the - /// caller -- never stored anywhere else in plaintext. - pub auth_token: String, -} - -impl std::fmt::Debug for CalDavAccountConfig { - /// Manual `Debug` impl that redacts `auth_token` -- a derived impl would print the raw - /// secret verbatim the moment anything (a `debug!` log, an assertion failure message, - /// a panic payload) formats this config with `{:?}`. - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CalDavAccountConfig") - .field("account_id", &self.account_id) - .field("caldav_url", &self.caldav_url) - .field("username", &self.username) - .field("auth_token", &"") - .finish() - } + /// caller -- never stored anywhere else in plaintext. Wrapped in [`Redacted`] so a derived + /// `Debug`, a `Display`, or `serde` can never disclose it to a log; the raw value is + /// reachable only through an explicit `expose_secret` call. + pub auth_token: Redacted, } /// Best-effort extraction of the `UID` property from a raw (possibly malformed) VEVENT/ @@ -202,7 +191,10 @@ impl CalDavClient { .request(report_method, &self.config.caldav_url) .header("Content-Type", "application/xml; charset=utf-8") .header("Depth", "1") - .basic_auth(&self.config.username, Some(&self.config.auth_token)) + .basic_auth( + &self.config.username, + Some(self.config.auth_token.expose_secret()), + ) .body(body) .send() .await @@ -258,7 +250,7 @@ mod tests { account_id: "acct-1".to_string(), caldav_url: "https://caldav.example.com/calendars/work/".to_string(), username: "jmaes".to_string(), - auth_token: "app-token-secret".to_string(), + auth_token: Redacted::new("app-token-secret".to_string()), } } diff --git a/crates/nuncio-cal/tests/wiremock_caldav_test.rs b/crates/nuncio-cal/tests/wiremock_caldav_test.rs index d8df94b..d9023c3 100644 --- a/crates/nuncio-cal/tests/wiremock_caldav_test.rs +++ b/crates/nuncio-cal/tests/wiremock_caldav_test.rs @@ -60,7 +60,7 @@ async fn wiremock_caldav_report_fetches_and_resolves_tzid_events() { account_id: "acct-wm-1".to_string(), caldav_url: format!("{}/calendars/work/", mock_server.uri()), username: "james.maes".to_string(), - auth_token: "wiremock-app-token".to_string(), + auth_token: "wiremock-app-token".to_string().into(), }; let client = CalDavClient::new(config); @@ -106,7 +106,7 @@ async fn wiremock_caldav_report_server_error_surfaces_as_transport_failure() { account_id: "acct-wm-2".to_string(), caldav_url: format!("{}/calendars/broken/", mock_server.uri()), username: "james.maes".to_string(), - auth_token: "wiremock-app-token".to_string(), + auth_token: "wiremock-app-token".to_string().into(), }; let client = CalDavClient::new(config); diff --git a/crates/nuncio-contacts/src/carddav.rs b/crates/nuncio-contacts/src/carddav.rs index 3d6c7a1..23ddfd1 100644 --- a/crates/nuncio-contacts/src/carddav.rs +++ b/crates/nuncio-contacts/src/carddav.rs @@ -1,6 +1,7 @@ //! CardDAV (RFC 6352) `REPORT` client and multistatus response parser. use async_trait::async_trait; +use nuncio_core::redact::Redacted; use serde::{Deserialize, Serialize}; use thiserror::Error; use tracing::{debug, info, instrument}; @@ -28,7 +29,7 @@ pub enum CardDavError { /// `carddav_url` must already resolve to a specific address book collection (e.g. /// `https://carddav.example.com/dav/addressbooks/user/jmaes/contacts/`) -- PROPFIND-based /// `addressbook-home-set` auto-discovery is out of scope for this client. -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct CardDavAccountConfig { /// Nuncio account identifier this address book collection belongs to. pub account_id: String, @@ -37,22 +38,10 @@ pub struct CardDavAccountConfig { /// Basic-auth username (or app-specific username, per provider). pub username: String, /// Basic-auth secret (password or app-specific token) resolved from the OS keyring by the - /// caller -- never stored anywhere else in plaintext. - pub auth_token: String, -} - -impl std::fmt::Debug for CardDavAccountConfig { - /// Manual `Debug` impl that redacts `auth_token` -- a derived impl would print the raw - /// secret verbatim the moment anything (a `debug!` log, an assertion failure message, - /// a panic payload) formats this config with `{:?}`. - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CardDavAccountConfig") - .field("account_id", &self.account_id) - .field("carddav_url", &self.carddav_url) - .field("username", &self.username) - .field("auth_token", &"") - .finish() - } + /// caller -- never stored anywhere else in plaintext. Wrapped in [`Redacted`] so a derived + /// `Debug`, a `Display`, or `serde` can never disclose it to a log; the raw value is + /// reachable only through an explicit `expose_secret` call. + pub auth_token: Redacted, } /// CardDAV client protocol engine managing address book `REPORT` queries against a real server. @@ -134,7 +123,10 @@ impl CardDavClient { .request(report_method, &self.config.carddav_url) .header("Content-Type", "application/xml; charset=utf-8") .header("Depth", "1") - .basic_auth(&self.config.username, Some(&self.config.auth_token)) + .basic_auth( + &self.config.username, + Some(self.config.auth_token.expose_secret()), + ) .body(body) .send() .await @@ -186,7 +178,7 @@ mod tests { account_id: "acct-1".to_string(), carddav_url: "https://carddav.example.com/addressbooks/contacts/".to_string(), username: "jmaes".to_string(), - auth_token: "app-token-secret".to_string(), + auth_token: Redacted::new("app-token-secret".to_string()), } } diff --git a/crates/nuncio-contacts/tests/wiremock_carddav_test.rs b/crates/nuncio-contacts/tests/wiremock_carddav_test.rs index a9c3ee4..dd64c64 100644 --- a/crates/nuncio-contacts/tests/wiremock_carddav_test.rs +++ b/crates/nuncio-contacts/tests/wiremock_carddav_test.rs @@ -57,7 +57,7 @@ async fn wiremock_carddav_report_fetches_and_parses_vcards() { account_id: "acct-wm-1".to_string(), carddav_url: format!("{}/addressbooks/contacts/", mock_server.uri()), username: "james.maes".to_string(), - auth_token: "wiremock-app-token".to_string(), + auth_token: "wiremock-app-token".to_string().into(), }; let client = CardDavClient::new(config); @@ -102,7 +102,7 @@ async fn wiremock_carddav_report_server_error_surfaces_as_transport_failure() { account_id: "acct-wm-2".to_string(), carddav_url: format!("{}/addressbooks/broken/", mock_server.uri()), username: "james.maes".to_string(), - auth_token: "wiremock-app-token".to_string(), + auth_token: "wiremock-app-token".to_string().into(), }; let client = CardDavClient::new(config); @@ -143,7 +143,7 @@ END:VCARD account_id: "acct-wm-3".to_string(), carddav_url: format!("{}/addressbooks/malformed/", mock_server.uri()), username: "james.maes".to_string(), - auth_token: "wiremock-app-token".to_string(), + auth_token: "wiremock-app-token".to_string().into(), }; let client = CardDavClient::new(config); diff --git a/crates/nuncio-core/src/lib.rs b/crates/nuncio-core/src/lib.rs index f72c4b7..b91cb1b 100644 --- a/crates/nuncio-core/src/lib.rs +++ b/crates/nuncio-core/src/lib.rs @@ -7,6 +7,7 @@ pub mod config; pub mod export; pub mod mcp_policy; pub mod model; +pub mod redact; pub mod update; pub mod worm_audit; @@ -17,6 +18,7 @@ pub use config::{ pub use export::{ExportEngine, ExportError, ExportFormat, ExportSummary}; pub use mcp_policy::{AgentPermissions, DataType, McpAgentPolicy}; pub use model::{Attachment, CalendarEvent, Contact, DaemonTelemetry, Email, Folder}; +pub use redact::Redacted; pub use update::{ReleaseInfo, UpdateCheckResult, UpdateEngine, UpdateError}; pub use worm_audit::{verify_worm_chain, WormAuditError, WormAuditRecord}; diff --git a/crates/nuncio-core/src/redact.rs b/crates/nuncio-core/src/redact.rs new file mode 100644 index 0000000..357d788 --- /dev/null +++ b/crates/nuncio-core/src/redact.rs @@ -0,0 +1,203 @@ +//! Secret-safe value wrapper and the logging redaction policy it enforces. +//! +//! # Logging redaction policy +//! +//! Telemetry (`tracing` spans, events, and any `Debug`/`Display`/`serde` +//! rendering that can reach a log sink) is a plaintext side channel. The +//! following values MUST NEVER appear in it, in any encoding: +//! +//! - message bodies, subjects, and snippets; +//! - sender/recipient email addresses and any other message correspondents; +//! - account credentials (passwords, app-specific passwords/tokens); +//! - gRPC bearer tokens; +//! - keyring secret *values* (the material stored in the OS vault) and any +//! derived key material (encryption keys, HMAC signing keys); +//! - vCard / iCal personal data (contact names, phones, addresses; event +//! summaries, locations, attendees, descriptions). +//! +//! The following identifying, non-sensitive values ARE allowed in telemetry, +//! because operating the daemon is impossible without them and none of them +//! reveals content or grants access: +//! +//! - opaque ids (account id, message id, rule id, folder id) and UIDs; +//! - counts, sizes, and durations; +//! - operation / rule / transport / action *kinds* (the discriminant, never +//! its payload); +//! - folder names, hostnames, ports, and the per-request `request_id`; +//! - the keyring *lookup key* (e.g. `nuncio/`) — an identifier +//! that names a vault entry but is not itself the stored secret. +//! +//! [`Redacted`] is the mechanical enforcement point for the first list: a +//! field whose type is `Redacted` cannot leak through a derived `Debug`, a +//! `Display`, or `serde` serialization, because each of those renders a fixed +//! placeholder instead of the inner value. The secret is reachable only via an +//! explicit [`Redacted::expose_secret`] / [`Redacted::into_inner`] call, which +//! is easy to audit and never happens implicitly. + +use std::fmt; + +use serde::de::{Deserialize, Deserializer}; +use serde::ser::{Serialize, Serializer}; + +/// The single placeholder rendered wherever a redacted value would otherwise +/// appear (`Debug`, `Display`, and `serde`). +const PLACEHOLDER: &str = ""; + +/// A wrapper for a sensitive value that never renders its contents through +/// `Debug`, `Display`, or `serde`. +/// +/// Wrapping a credential/token/key-material field in `Redacted` makes it +/// impossible to leak that field into telemetry by accident: the compiler still +/// lets a struct derive `Debug`/`Serialize`, but the wrapped field renders as +/// ``. Reaching the real value is deliberate and greppable — it +/// requires calling [`expose_secret`](Self::expose_secret) or +/// [`into_inner`](Self::into_inner). +/// +/// Equality is derived over the inner value so wrapped fields remain comparable +/// in tests and config diffs; it is intentionally not constant-time and is not +/// meant for authenticating a secret against an attacker-supplied guess. +#[derive(Clone, PartialEq, Eq)] +pub struct Redacted(T); + +impl Redacted { + /// Wrap `value` so it is protected from accidental disclosure in telemetry. + pub fn new(value: T) -> Self { + Self(value) + } + + /// Borrow the wrapped secret. Naming the call `expose_secret` keeps every + /// deliberate disclosure of the value greppable and reviewable. + pub fn expose_secret(&self) -> &T { + &self.0 + } + + /// Consume the wrapper and return the owned secret. + pub fn into_inner(self) -> T { + self.0 + } +} + +impl From for Redacted { + fn from(value: T) -> Self { + Self(value) + } +} + +/// Renders `` regardless of the inner value, so a derived `Debug` on +/// any containing struct can never print the secret. +impl fmt::Debug for Redacted { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(PLACEHOLDER) + } +} + +/// Renders ``, so interpolating the value with `{}` (e.g. into a +/// log message or error) discloses nothing. +impl fmt::Display for Redacted { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(PLACEHOLDER) + } +} + +/// Serializes the placeholder, never the inner value. This closes the +/// `serde_json`/structured-logging leak path: even serializing a whole struct +/// that contains a `Redacted` field emits `` for it. As a result +/// serialization is deliberately lossy — a `Redacted` must never be used for +/// a field whose real value has to survive a serialization round trip. +impl Serialize for Redacted { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(PLACEHOLDER) + } +} + +/// Deserializes transparently into `Redacted` by deserializing the inner +/// `T`. This exists so structs that hold a `Redacted` can still derive +/// `Deserialize`; it does not reverse [`Serialize`], which is lossy by design. +impl<'de, T> Deserialize<'de> for Redacted +where + T: Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(Self(T::deserialize(deserializer)?)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SENTINEL: &str = "super-secret-app-token-9f3a"; + + #[test] + fn debug_renders_placeholder_not_the_secret() { + let wrapped = Redacted::new(SENTINEL.to_string()); + let rendered = format!("{wrapped:?}"); + assert_eq!(rendered, PLACEHOLDER); + assert!(!rendered.contains(SENTINEL)); + } + + #[test] + fn display_renders_placeholder_not_the_secret() { + let wrapped = Redacted::new(SENTINEL.to_string()); + let rendered = format!("{wrapped}"); + assert_eq!(rendered, PLACEHOLDER); + assert!(!rendered.contains(SENTINEL)); + } + + #[test] + fn derived_debug_of_a_containing_struct_does_not_leak() { + // Fields are exercised only through the derived `Debug`, which + // dead-code analysis deliberately ignores; the test asserts on that + // rendering, so silence the false-positive unread-field lint. + #[derive(Debug)] + #[allow(dead_code)] + struct Config { + username: String, + auth_token: Redacted, + } + let cfg = Config { + username: "jmaes".to_string(), + auth_token: Redacted::new(SENTINEL.to_string()), + }; + let rendered = format!("{cfg:?}"); + assert!(rendered.contains("jmaes")); + assert!(!rendered.contains(SENTINEL)); + assert!(rendered.contains(PLACEHOLDER)); + } + + #[test] + fn serialize_emits_placeholder_not_the_secret() { + let wrapped = Redacted::new(SENTINEL.to_string()); + let json = serde_json::to_string(&wrapped).expect("serialize"); + assert!(!json.contains(SENTINEL)); + assert_eq!(json, format!("\"{PLACEHOLDER}\"")); + } + + #[test] + fn expose_and_into_inner_return_the_real_value() { + let wrapped = Redacted::new(SENTINEL.to_string()); + assert_eq!(wrapped.expose_secret(), SENTINEL); + assert_eq!(wrapped.into_inner(), SENTINEL); + } + + #[test] + fn deserialize_wraps_the_inner_value() { + let wrapped: Redacted = + serde_json::from_str(&format!("\"{SENTINEL}\"")).expect("deserialize"); + assert_eq!(wrapped.expose_secret(), SENTINEL); + } + + #[test] + fn from_and_equality() { + let a: Redacted = SENTINEL.to_string().into(); + let b = Redacted::new(SENTINEL.to_string()); + assert_eq!(a, b); + assert_ne!(a, Redacted::new("other".to_string())); + } +} diff --git a/crates/nunciod/src/calendar_sync.rs b/crates/nunciod/src/calendar_sync.rs index 6b698a7..d0a5c45 100644 --- a/crates/nunciod/src/calendar_sync.rs +++ b/crates/nunciod/src/calendar_sync.rs @@ -100,7 +100,7 @@ pub async fn sync_caldav_account( account_id: account.id.clone(), caldav_url: account.dav_collection_url().unwrap_or_default().to_string(), username: account.email_address.clone(), - auth_token: password.to_string(), + auth_token: nuncio_core::redact::Redacted::new(password.to_string()), }); sync_with_backend(db, &client, calendar_id, start_window, end_window).await } diff --git a/crates/nunciod/tests/no_secrets_canary_test.rs b/crates/nunciod/tests/no_secrets_canary_test.rs new file mode 100644 index 0000000..3113464 --- /dev/null +++ b/crates/nunciod/tests/no_secrets_canary_test.rs @@ -0,0 +1,537 @@ +//! Cross-crate "no secrets in logs" canary. +//! +//! This is the regression net for the logging redaction policy documented in +//! `nuncio_core::redact`. It installs a capturing `tracing` subscriber, seeds a +//! set of unmistakable SENTINEL secrets/PII, then drives a representative slice +//! of the now-instrumented code paths across the daemon and the library crates +//! using only mock backends and loopback wiremock servers (no live network): +//! +//! - gRPC auth reject (a wrong bearer token) and authenticated calls; +//! - an account add carrying an account password; +//! - a mock inbound mail sync carrying a sentinel subject/body/sender/recipient; +//! - a real JMAP protocol sync (wiremock) carrying the same sentinel content +//! plus sentinel credentials; +//! - an outbox-affecting mark-read and an outbound send carrying sentinel content; +//! - a filter rule firing on a sentinel message; +//! - a webhook dispatch signed with a sentinel HMAC key; +//! - a CalDAV and a CardDAV sync authenticated with a sentinel token and +//! carrying sentinel event/contact PII. +//! +//! It then asserts that NONE of the sentinel strings appear anywhere in the +//! captured telemetry, while also proving the log-emitting paths actually ran +//! (so the test can never pass by capturing nothing). + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::sync::{Arc, Mutex}; + +use nuncio_cal::{CalDavAccountConfig, CalDavClient, CalendarBackend}; +use nuncio_contacts::{CardDavAccountConfig, CardDavClient, ContactsBackend}; +use nuncio_core::model::{Email, Folder}; +use nuncio_core::EventBus; +use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher}; +use nuncio_mail::{JmapEngine, MailBackend, MockMailBackend, MockMessageSender}; +use nuncio_proto::v1::{ + AccountConfig, AddAccountRequest, GetStatusRequest, MarkReadRequest, SendMessageRequest, + SyncRequest, TlsMode, +}; +use nuncio_store::db::DatabaseEngine; +use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT}; +use nunciod::grpc::{ + serve_on_listener_with_overrides, AccountsEngineOverrides, CalendarEngineOverrides, + ContactsEngineOverrides, MailEngineOverrides, +}; +use tokio::net::TcpListener; +use tracing::field::{Field, Visit}; +use tracing::span::Attributes; +use tracing::{Event, Id, Subscriber}; +use tracing_subscriber::layer::Context as LayerContext; +use tracing_subscriber::prelude::*; +use tracing_subscriber::registry::LookupSpan; +use wiremock::matchers::{body_partial_json, method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +// ---- Sentinels: none of these may EVER appear in captured telemetry. ---- +const SENTINEL_BEARER: &str = "SENTINELbearerTokenAaBbCc001"; +const SENTINEL_PASSWORD: &str = "SENTINELaccountPasswordDdEe002"; +const SENTINEL_SUBJECT: &str = "SENTINELmessageSubjectFfGg003"; +const SENTINEL_BODY: &str = "SENTINELmessageBodyHhIi004"; +const SENTINEL_SENDER: &str = "sentinel-sender-jjkk005@example.invalid"; +const SENTINEL_RECIPIENT: &str = "sentinel-recipient-llmm006@example.invalid"; +const SENTINEL_WEBHOOK_KEY: &str = "SENTINELwebhookHmacKeyNnOo007"; +const SENTINEL_DAV_TOKEN: &str = "SENTINELdavAuthTokenPpQq008"; +const SENTINEL_EVENT_SUMMARY: &str = "SENTINELeventSummaryRrSs009"; +const SENTINEL_CONTACT_NAME: &str = "SENTINELcontactNameTtUu010"; + +const ACCOUNT_ID: &str = "acct-canary-1"; +const ACCOUNT_EMAIL: &str = "canary@nuncio.mx"; +const KEYRING_KEY: &str = "nuncio/acct-canary-1"; + +/// Thread-safe sink recording every rendered field value from every captured +/// span and event, so a single global subscriber can collect telemetry emitted +/// on any daemon/runtime thread. +#[derive(Default)] +struct Sink(Mutex>); + +impl Sink { + fn push_all(&self, values: Vec) { + if let Ok(mut guard) = self.0.lock() { + guard.extend(values); + } + } + + fn snapshot(&self) -> Vec { + self.0.lock().map(|g| g.clone()).unwrap_or_default() + } +} + +struct FieldVisitor(Vec); + +impl Visit for FieldVisitor { + fn record_str(&mut self, field: &Field, value: &str) { + self.0.push(format!("{}={}", field.name(), value)); + } + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0.push(format!("{}={:?}", field.name(), value)); + } +} + +struct CaptureLayer(Arc); + +impl tracing_subscriber::Layer for CaptureLayer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_new_span(&self, attrs: &Attributes<'_>, _id: &Id, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(vec![format!("span={}", attrs.metadata().name())]); + attrs.record(&mut visitor); + self.0.push_all(visitor.0); + } + + fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(vec![format!("event={}", event.metadata().target())]); + event.record(&mut visitor); + self.0.push_all(visitor.0); + } +} + +fn sample_proto_config() -> AccountConfig { + AccountConfig { + id: ACCOUNT_ID.to_string(), + name: "Canary Account".to_string(), + email_address: ACCOUNT_EMAIL.to_string(), + keyring_secret_key: KEYRING_KEY.to_string(), + sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), + transport: Some(nuncio_proto::v1::account_config::Transport::ImapSmtp( + nuncio_proto::v1::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: TlsMode::ImplicitTls.into(), + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: TlsMode::ImplicitTls.into(), + }, + )), + } +} + +fn sentinel_inbound_email() -> Email { + Email { + id: "msg-canary-1".to_string(), + account_id: ACCOUNT_ID.to_string(), + folder_id: "inbox".to_string(), + remote_id: "msg-canary-1".to_string(), + uid_validity: "1".to_string(), + subject: SENTINEL_SUBJECT.to_string(), + sender: SENTINEL_SENDER.to_string(), + recipient: SENTINEL_RECIPIENT.to_string(), + received_at: 1_700_000_000, + read: false, + body_plain: Some(SENTINEL_BODY.to_string()), + body_html: None, + attachments: Vec::new(), + } +} + +/// Boots the real daemon gRPC server (all services, bearer-authenticated) on an +/// ephemeral loopback port, backed by a real temp file DB and the given mail +/// overrides + filter engine. +async fn boot_daemon( + secrets: Arc, + overrides: MailEngineOverrides, + filter_engine: Arc, +) -> (std::net::SocketAddr, String, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("temp dir for db"); + let db_path = dir.path().join("canary.db"); + let db = Arc::new( + DatabaseEngine::connect_file(&db_path, &secrets) + .await + .expect("open temp file db"), + ); + let token = hex::encode( + secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("mint bearer token from mock vault"), + ); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("listener has addr"); + + let event_bus = Arc::new(EventBus::new()); + let server_secrets = secrets.clone(); + let server_token = token.clone(); + tokio::spawn(async move { + let _ = serve_on_listener_with_overrides( + listener, + event_bus, + db, + filter_engine, + server_secrets, + server_token, + overrides, + CalendarEngineOverrides::default(), + ContactsEngineOverrides::default(), + AccountsEngineOverrides::default(), + ) + .await; + }); + (addr, token, dir) +} + +/// Drives the daemon over authenticated gRPC: an auth-reject with a sentinel +/// bearer token, an account add with a sentinel password, an inbound sync +/// carrying sentinel content, a mark-read, and an outbound send. +async fn drive_daemon() { + let secrets = Arc::new(SecretManager::mock()); + + let mock_backend = MockMailBackend::new(); + mock_backend.add_folder(Folder { + id: "inbox".to_string(), + name: "Inbox".to_string(), + total_messages: 1, + unread_messages: 1, + }); + mock_backend.add_message(sentinel_inbound_email()); + let mock_sender = MockMessageSender::new(); + let overrides = MailEngineOverrides { + mail_backend: Some(Arc::new(mock_backend)), + message_sender: Some(Arc::new(mock_sender)), + }; + + // A rule that fires on the sentinel subject, so the daemon's filter path is + // exercised against sentinel content during any evaluation it performs. + let rule = NsqlParser::parse_rule( + "Canary Rule", + 1, + &format!("WHERE subject CONTAINS '{SENTINEL_SUBJECT}' ACTION MARK READ"), + ) + .expect("parse canary rule"); + let filter_engine = Arc::new(FilterEngine::new(vec![rule]).expect("compile rule")); + + let (addr, token, _db_dir) = boot_daemon(secrets, overrides, filter_engine).await; + let addr_str = addr.to_string(); + + // (1) Auth reject: a call bearing the WRONG (sentinel) token is rejected, + // and that token must never be echoed to a log. + let mut unauth = + nuncio_proto::v1::system_client::SystemClient::connect(format!("http://{addr}")) + .await + .expect("system client connects"); + let mut bad = tonic::Request::new(GetStatusRequest {}); + bad.metadata_mut().insert( + "authorization", + format!("Bearer {SENTINEL_BEARER}") + .parse() + .expect("ascii metadata"), + ); + let err = unauth + .get_status(bad) + .await + .expect_err("wrong bearer token must be rejected"); + assert_eq!(err.code(), tonic::Code::Unauthenticated); + + let mut accounts = nuncio_proto::client::connect_accounts(&addr_str, &token) + .await + .expect("accounts client"); + let mut mail = nuncio_proto::client::connect_mail(&addr_str, &token) + .await + .expect("mail client"); + + // (2) Account add carrying a sentinel password. + accounts + .add_account(AddAccountRequest { + config: Some(sample_proto_config()), + password: SENTINEL_PASSWORD.to_string(), + }) + .await + .expect("add_account succeeds"); + + // (3) Inbound sync over the mock backend carrying sentinel content. + let synced = mail + .sync(SyncRequest { account_id: None }) + .await + .expect("sync succeeds") + .into_inner(); + assert_eq!(synced.synced_count, 1); + + // (4) Outbox-affecting mark-read. + mail.mark_read(MarkReadRequest { + message_id: "msg-canary-1".to_string(), + read: true, + }) + .await + .expect("mark_read succeeds"); + + // (5) Outbound send carrying sentinel content. + mail.send_message(SendMessageRequest { + to: SENTINEL_RECIPIENT.to_string(), + cc: None, + subject: SENTINEL_SUBJECT.to_string(), + body_text: SENTINEL_BODY.to_string(), + body_html: None, + attachments: Vec::new(), + account_id: None, + }) + .await + .expect("send_message succeeds"); +} + +/// Drives the filter engine (a rule fires on the sentinel email) and a webhook +/// dispatch signed with a sentinel HMAC key against a loopback wiremock server. +async fn drive_filter_and_webhook() { + let rule = NsqlParser::parse_rule( + "Canary Webhook Rule", + 1, + &format!("WHERE subject CONTAINS '{SENTINEL_SUBJECT}' ACTION MARK READ"), + ) + .expect("parse rule"); + let engine = FilterEngine::new(vec![rule]).expect("compile rule"); + let matches = engine.evaluate(&sentinel_inbound_email()); + assert_eq!(matches.len(), 1, "rule must fire on the sentinel message"); + + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/hook")) + .respond_with(ResponseTemplate::new(200)) + .mount(&mock_server) + .await; + + let dispatcher = WebhookDispatcher::new(SENTINEL_WEBHOOK_KEY); + let opts = ValidationOptions { + available_folders: None, + allowed_forward_domains: None, + block_private_webhooks: false, + }; + let url = format!("{}/hook", mock_server.uri()); + let status = dispatcher + .dispatch_with_options( + &url, + "rule-canary", + "msg-canary-1", + SENTINEL_SUBJECT, + SENTINEL_SENDER, + &opts, + ) + .await + .expect("webhook dispatch to loopback mock succeeds"); + assert_eq!(status, 200); +} + +/// Drives a CalDAV and a CardDAV sync authenticated with the sentinel token and +/// carrying sentinel PII, against loopback wiremock servers. +async fn drive_dav() { + // CalDAV. + let cal_server = MockServer::start().await; + let cal_body = format!( + r#" + + + /calendars/work/e.ics + BEGIN:VCALENDAR +VERSION:2.0 +BEGIN:VEVENT +SUMMARY:{SENTINEL_EVENT_SUMMARY} +DTSTART:20240301T120000Z +DTEND:20240301T130000Z +END:VEVENT +END:VCALENDAR + +"# + ); + Mock::given(method("REPORT")) + .and(path("/calendars/work/")) + .respond_with(ResponseTemplate::new(207).set_body_string(cal_body)) + .mount(&cal_server) + .await; + let cal_client = CalDavClient::new(CalDavAccountConfig { + account_id: ACCOUNT_ID.to_string(), + caldav_url: format!("{}/calendars/work/", cal_server.uri()), + username: ACCOUNT_EMAIL.to_string(), + auth_token: SENTINEL_DAV_TOKEN.to_string().into(), + }); + let events = + CalendarBackend::fetch_events(&cal_client, "cal-work", 1_700_000_000, 1_800_000_000) + .await + .expect("caldav fetch succeeds"); + assert_eq!(events.len(), 1); + + // CardDAV. + let card_server = MockServer::start().await; + let card_body = format!( + r#" + + + /addressbooks/contacts/c.vcf + BEGIN:VCARD +VERSION:4.0 +FN:{SENTINEL_CONTACT_NAME} +EMAIL:{SENTINEL_SENDER} +END:VCARD + +"# + ); + Mock::given(method("REPORT")) + .and(path("/addressbooks/contacts/")) + .respond_with(ResponseTemplate::new(207).set_body_string(card_body)) + .mount(&card_server) + .await; + let card_client = CardDavClient::new(CardDavAccountConfig { + account_id: ACCOUNT_ID.to_string(), + carddav_url: format!("{}/addressbooks/contacts/", card_server.uri()), + username: ACCOUNT_EMAIL.to_string(), + auth_token: SENTINEL_DAV_TOKEN.to_string().into(), + }); + let contacts = ContactsBackend::fetch_contacts(&card_client, ACCOUNT_ID) + .await + .expect("carddav fetch succeeds"); + assert_eq!(contacts.len(), 1); +} + +/// Drives a real JMAP protocol sync (session discovery + Mailbox/get + +/// Email/query + Email/get) with sentinel credentials, carrying sentinel +/// content in the fetched message. +async fn drive_jmap() { + let server = MockServer::start().await; + let session_body = serde_json::json!({ + "username": "canary@nuncio.mx", + "primaryAccounts": { "urn:ietf:params:jmap:mail": "acct-100" }, + "apiUrl": format!("{}/jmap/api", server.uri()), + "state": "session-state-1" + }); + Mock::given(method("GET")) + .and(path("/.well-known/jmap")) + .respond_with(ResponseTemplate::new(200).set_body_json(&session_body)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(serde_json::json!({ + "methodCalls": [["Mailbox/get", {}, "c1"]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "methodResponses": [["Mailbox/get", { + "accountId": "acct-100", + "list": [{"id": "mb-inbox", "name": "Inbox", "totalEmails": 1, "unreadEmails": 1}] + }, "c1"]] + }))) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(serde_json::json!({ + "methodCalls": [["Email/query", {}, "c1"]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "methodResponses": [["Email/query", {"ids": ["msg-jmap-1"]}, "c1"]] + }))) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(serde_json::json!({ + "methodCalls": [["Email/get", {}, "c1"]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "methodResponses": [["Email/get", { + "accountId": "acct-100", + "state": "sync-state-1", + "list": [{ + "id": "msg-jmap-1", + "subject": SENTINEL_SUBJECT, + "from": [{ "email": SENTINEL_SENDER }], + "to": [{ "email": SENTINEL_RECIPIENT }], + "receivedAt": 1700002000i64, + "isUnread": true, + "bodySnippet": SENTINEL_BODY + }] + }, "c1"]] + }))) + .mount(&server) + .await; + + let engine = + JmapEngine::with_credentials("acct-100", &server.uri(), ACCOUNT_EMAIL, SENTINEL_PASSWORD); + let folders = engine.sync_folders().await.expect("jmap sync_folders"); + assert_eq!(folders.len(), 1); + let (emails, _state) = engine + .sync_messages("mb-inbox", None) + .await + .expect("jmap sync_messages"); + assert_eq!(emails.len(), 1); +} + +#[tokio::test] +async fn no_sentinel_secret_appears_in_any_captured_telemetry() { + let sink = Arc::new(Sink::default()); + let subscriber = tracing_subscriber::registry().with(CaptureLayer(sink.clone())); + tracing::subscriber::set_global_default(subscriber) + .expect("install the global capturing subscriber"); + + drive_daemon().await; + drive_filter_and_webhook().await; + drive_dav().await; + drive_jmap().await; + + let captured = sink.snapshot(); + + // Prove the log-emitting paths actually ran: the test must never pass by + // capturing nothing. + assert!( + !captured.is_empty(), + "no telemetry captured -- the canary exercised nothing" + ); + let haystack = captured.join("\n"); + assert!( + haystack.contains("webhook"), + "expected the webhook dispatch path to have logged; got:\n{haystack}" + ); + assert!( + haystack.contains("rule matched") || haystack.contains("dispatching actions"), + "expected the filter rule-fire path to have logged; got:\n{haystack}" + ); + + // The core assertion: no sentinel secret or PII may appear anywhere. + let sentinels = [ + ("bearer token", SENTINEL_BEARER), + ("account password", SENTINEL_PASSWORD), + ("message subject", SENTINEL_SUBJECT), + ("message body", SENTINEL_BODY), + ("sender address", SENTINEL_SENDER), + ("recipient address", SENTINEL_RECIPIENT), + ("webhook signing key", SENTINEL_WEBHOOK_KEY), + ("DAV auth token", SENTINEL_DAV_TOKEN), + ("calendar event summary", SENTINEL_EVENT_SUMMARY), + ("contact name", SENTINEL_CONTACT_NAME), + ]; + for (label, sentinel) in sentinels { + let leaked: Vec<&String> = captured + .iter() + .filter(|line| line.contains(sentinel)) + .collect(); + assert!( + leaked.is_empty(), + "SECRET LEAK: the {label} sentinel appeared in telemetry: {leaked:?}" + ); + } +} diff --git a/docs/LOGGING.md b/docs/LOGGING.md new file mode 100644 index 0000000..0d47739 --- /dev/null +++ b/docs/LOGGING.md @@ -0,0 +1,56 @@ +# Logging & redaction policy + +Nuncio's telemetry (`tracing` spans and events, plus anything that reaches a log +sink through a `Debug` / `Display` / `serde` rendering) is a plaintext side +channel. This document is the authoritative statement of what may and may not +appear in it. It is enforced mechanically by two things: + +- **`nuncio_core::redact::Redacted`** — a newtype whose `Debug`, `Display`, + and `serde` all render the fixed placeholder ``, so a secret field + wrapped in it cannot leak even through a *derived* `Debug` on the containing + struct. The inner value is reachable only via an explicit, greppable + `expose_secret()` / `into_inner()` call. The module doc comment restates this + policy next to the type. +- **the no-secrets canary** — `crates/nunciod/tests/no_secrets_canary_test.rs`, + a workspace test that drives a representative slice of the instrumented code + paths with known sentinel secrets and asserts none of them appear in captured + telemetry. It is the regression net: if a future change starts logging a + secret, this test fails. + +## MUST NEVER be logged (in any encoding) + +- Message bodies, subjects, and snippets. +- Sender / recipient email addresses and any other message correspondents. +- Account credentials: passwords and app-specific passwords/tokens. +- gRPC bearer tokens. +- Keyring secret **values** (the material stored in the OS vault) and any + derived key material (encryption keys, HMAC signing keys). +- vCard / iCal personal data: contact names, phone numbers, addresses; event + summaries, locations, attendees, and descriptions. + +## MAY be logged (identifying but non-sensitive) + +Operating the daemon is impossible without correlation identifiers, and none of +the following reveals content or grants access: + +- Opaque ids: account id, message id, rule id, folder id — and UIDs. +- Counts, sizes, and durations. +- Operation / rule / transport / action **kinds** (the discriminant only, never + its payload). +- Folder names, hostnames, ports, and the per-request `request_id`. +- The keyring **lookup key** (e.g. `nuncio/`) — an identifier that + names a vault entry but is not itself the stored secret. + +## Guidance for contributors + +- Prefer logging an **id or a kind** over the value it points at. If you need to + correlate a log line with a message/event/contact, log its id/UID, never its + content. +- When a struct must hold a credential/token/key, make the field type + `Redacted` rather than relying on a hand-written `Debug`. Reach the value + only at the point of use, via `expose_secret()`. +- Never interpolate a whole config/request struct into a log or error with + `{:?}` unless every sensitive field it holds is `Redacted` (or otherwise + self-redacting). +- If you add an instrumented path that could plausibly touch any of the + "MUST NEVER" values, extend the canary to drive it with a sentinel. -- 2.54.0 From 418aa7052e0d7da8f510381184cd75a08419891d Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 7 Aug 2026 12:51:12 -0500 Subject: [PATCH 46/95] docs: reconcile roadmap and handoff with M6.5, WS and OBS Roadmap and its rendered page carried no mention of M6.5, the WS-A..WS-F workstreams, or the completed OBS epic. AGENTS.md becomes a pointer to CLAUDE.md instead of a near-verbatim copy. --- AGENTS.md | 11 +++++ docs/HANDOFF.md | 41 ++++++++++++++---- docs/ROADMAP.md | 93 ++++++++++++++++++++++++++++++++++------- docs/roadmap/index.html | 50 ++++++++++++++++++---- 4 files changed, 163 insertions(+), 32 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..93f96e7 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,11 @@ +# AGENTS.md + +The guidance for this repository lives in [`CLAUDE.md`](CLAUDE.md). It is +tool-agnostic despite the filename: what it documents is the repo's +architecture, build and quality gates, conventions, and testing standards. + +**Read [`CLAUDE.md`](CLAUDE.md) before working here**, whichever agent you are. + +This file is deliberately a pointer, not a copy. It previously held a +near-verbatim duplicate of `CLAUDE.md` differing only in the title line, which +would have drifted out of sync the first time either file was edited. diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index c6dbec5..02bc4ec 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -23,7 +23,7 @@ machine (or a new LLM) picks up the work cold. it to the definition of done, and open a PR. Do not merge your own work; do not start a second story until yours is merged or handed back. -## Current state of the code (2026-07-30) +## Current state of the code (2026-08-07) - **Phases 0–2 complete.** The daemon boots gRPC-only on loopback `127.0.0.1:9420` behind a keyring bearer token, serving eight services (System, Accounts, Mail, Filters, Calendar, Contacts, Export, Audit), each behind the @@ -44,11 +44,29 @@ machine (or a new LLM) picks up the work cold. full re-fetch on UIDVALIDITY change and a per-item FETCH timeout, explicit `SendMessage` account selection, and keyring rollback on failed `AddAccount`) are all done. -- **Next milestone: M6 — Security & release hardening** (fail-closed +- **M6 — Security & release hardening is 13 of 16 stories closed.** Fail-closed encryption, zeroized key material, WORM/ledger key-length validation, - non-loopback gRPC bind rejection + an auth-coverage canary, a fail-closed - updater on missing `SHA256SUMS`, real CIDR/DNS SSRF checks for webhooks, - inbound HTML email sanitization). Then M7 (API freeze / client-readiness). + non-loopback gRPC bind rejection plus the auth-coverage canary, the + fail-closed updater on missing `SHA256SUMS`, and real CIDR/DNS SSRF checks for + webhooks have all landed. Open: inbound HTML sanitization (#204), Rust static + analysis in CodeQL (#250), assertion-free test theater (#251). +- **Work reorganized after M5 into M6.5, WS-A … WS-F, and OBS.** M6.5 + (de-fabrication, 3/11) and the six WS workstreams (22/66) are the pre-freeze + reshape and now carry most of the remaining effort; **WS-A gates M7**. The + **OBS observability epic is COMPLETE** (OBS-1 … OBS-9, #357–#365, merged + without a milestone): tracing subscriber, per-RPC spans with request-id + correlation, domain/lifecycle coverage, protocol instrumentation, a real + `System.GetStatus`, CLI `-v`/`-vv` with typed errors, and a `Redacted` + redaction policy with a no-secrets-in-logs canary. See `docs/LOGGING.md`. +- **M7 (API freeze) has not started** — 5 stories open, 0 closed. +- **Gate verified green on 2026-08-07** under the pinned 1.97.1 toolchain: + `cargo fmt --all -- --check` clean, `cargo check-all` clean, `cargo test-all` + 596 passed / 0 failed / 0 ignored across 37 suites. +- **`dev` is a superset of `main` again.** `main` had drifted ahead by one real + commit, `157c7c2` (rustls 0.23.42 → 0.23.43), which was never back-merged; + back-merged 2026-08-07 and the gate re-verified green on the new lockfile. + Watch for this: a dependency bump merged straight onto `main` leaves `dev` + building on the older crate until someone merges back. - Full detail and the path to a client-ready API: `docs/ROADMAP.md`. ## Branches & preserved WIP @@ -85,8 +103,10 @@ commit/push, review-before-close, honest-error over green-lie.** 3. Run it: `cargo run -p nunciod` (daemon), then drive with `cargo run -p nuncio-cli -- system status`. See `docs/RUNNING.md` / `docs/DEVELOPING-IN-RUSTROVER.md`. -4. Claim an open story issue (M1 first — M1–M5 build the feature surface, M6 - hardens security, M7 freezes the API). Follow `docs/STORY-WORKFLOW.md`. +4. Claim an open story issue. M1–M5 (feature surface) and OBS (observability) + are done; M6 is 13/16. The open work is **M6.5**, **WS-A … WS-F**, and the + M6 remainder — start with **WS-A**, since it gates the M7 freeze. Follow + `docs/STORY-WORKFLOW.md`. ## Coordination rules - **One proto-touching story in flight at a time** — several stories add to the @@ -102,7 +122,10 @@ GitHub milestones **M1–M7** mirror `docs/ROADMAP.md`. Each milestone holds `story`-labelled issues with a full, self-contained spec. Order: M1 → M7 (M1–M5 independent-ish feature work; M6 security; M7 the client-readiness gate). -**M1–M5 are DELIVERED — all stories merged to `dev`.** M6 is next. +**M1–M5 are DELIVERED — all stories merged to `dev`.** M6 is 13/16 closed. The +live status for any track is `gh issue list --milestone ""`; the +per-story lists below are a point-in-time snapshot of M1–M7 only and do not +cover M6.5, WS-A … WS-F, or OBS (see `docs/ROADMAP.md` for those). **M1 — Finish Calendar (CalDAV)** · milestone #23 · **DONE** - #185 — Add Calendar API vertical (proto, daemon, CLI, E2E) · **[PROTO]** · merged @@ -127,7 +150,7 @@ independent-ish feature work; M6 security; M7 the client-readiness gate). - #196 — SendMessage selects an explicit account instead of the first configured · **[PROTO]** · merged - #197 — Roll back orphaned keyring secret when AddAccount fails to persist · merged -**M6 — Security & release hardening** · milestone #28 ← **next** +**M6 — Security & release hardening** · milestone #28 · **13/16 closed** (open: #204, #250, #251) - #198 — Fail closed on encryption failure in `PayloadCipher::encrypt_text_at_rest` - #199 — Zeroize long-lived engine key material on drop - #200 — Validate WORM/ledger key length from the vault, fail closed on mismatch diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e3eac3d..c564ae6 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -41,7 +41,7 @@ These are non-negotiable and apply to every item below: --- -## Where we are today (2026-07-30) +## Where we are today (2026-08-07) **Phases 0–2 are complete and on `dev`.** The daemon boots gRPC-only on loopback `127.0.0.1:9420` behind a keyring-minted bearer token, and serves **eight @@ -85,18 +85,56 @@ next): wrong-account fallback), and `AddAccount` rolls back an orphaned keyring secret if persistence fails. -**Next up: M6 — Security & release hardening** (fail-closed encryption, -zeroized key material, WORM/ledger key-length validation, a rejection of -non-loopback gRPC binds plus an auth-coverage canary, a fail-closed updater on -missing `SHA256SUMS`, real CIDR/DNS SSRF checks for webhooks, inbound HTML -email sanitization). Then M7 (API freeze / client-readiness). +**M6 — Security & release hardening is nearly complete** (13 of 16 stories +closed). Landed: fail-closed encryption, zeroized key material, WORM/ledger +key-length validation, non-loopback gRPC bind rejection plus the auth-coverage +canary, the fail-closed updater on missing `SHA256SUMS`, and real CIDR/DNS SSRF +checks on webhook egress. Still open: inbound HTML email sanitization (#204), +Rust static analysis in CodeQL (#250 — CodeQL currently scans JS and actions +only), and removal of assertion-free test theater (#251). + +**Work reorganized after M5 into three tracks that now carry the bulk of the +remaining effort.** These are GitHub milestones in their own right; they are not +part of the original M1–M7 numbering: + +- **M6.5 — Feature completeness & de-fabrication** (3 closed / 8 open). + Eliminates the remaining fabricated or silently-wrong paths: production-wiring + Contacts sync with per-account CardDAV config (#255), namespace-aware DAV XML + parsing to replace substring splitting and silent drops (#257), the + `tls_mode_from_db` silent `ImplicitTls` fallback (#271), the + `NaturalLanguageScheduler` that returns wrong times (#258), plus CLI and audit + cleanups. +- **WS-A … WS-F — the pre-freeze reshape** (22 closed / 44 open). Six tracks + that must land before `nuncio.v1` can be frozen: **WS-A** contract hardening + (typed errors, `Timestamp`/`Duration` unification, keyset pagination, id + conventions, enum hygiene, account transport `oneof`); **WS-B** mail model and + mutations; **WS-C** sync, push and lifecycle; **WS-D** security to 9/10 + (whole-DB SQLCipher encryption, OS-permissioned UDS / named-pipe IPC replacing + loopback TCP, signed updates, HTML sanitization); **WS-E** calendar and + contacts write-back plus structured search; **WS-F** ops, usability and + maintainability, including the behavior-preserving `grpc.rs` (#333) and + `DatabaseEngine` (#334) decompositions. +- **OBS-1 … OBS-9 — observability — COMPLETE.** All nine stories merged + (#357–#365; no milestone was assigned). Delivered: a layered `tracing` + subscriber with `EnvFilter`, rotating file and JSON output; per-RPC spans with + request-id correlation and auth accept/reject logging; domain and lifecycle + coverage across handlers, syncs, outbox and shutdown; protocol-sync + instrumentation for mail, calendar and contacts; a real `System.GetStatus` + health and readiness surface; CLI `-v`/`-vv` with typed `ErrorInfo` rendering; + and a redaction policy with a `Redacted<T>` newtype plus a no-secrets-in-logs + canary. Reference: [`docs/LOGGING.md`](LOGGING.md). + +**M7 (API freeze) has not started** — 5 stories open, 0 closed. It gates on WS-A +finishing the contract reshape. A handful of follow-ups were filed during M1–M5 and remain open, tracked as their own backlog items rather than blocking completion: calendar windowing/predicate refinement (#216), contacts notes handling (#219), a `filter edit` bug where re-saving a rule silently re-enables it and resets `created_at` (#225), and further mail sync/SMTP robustness polish identified -during M5 (#233). +during M5 (#233). Also open and worth early attention: the ReDoS guard +`evaluate_with_timeout` exists but no production caller uses it, so the live +sync path still calls the unbounded `evaluate` (#370). **Explicitly deferred** (out of scope until the backend is client-ready and each is re-justified as a genuine engine capability): OpenPGP/S-MIME E2EE (#79), @@ -106,12 +144,18 @@ WASM/QuickJS plugins (#80), local-LLM summarization (#81). ## The path to client-ready -The remaining work is organized into seven milestones. M1–M5 complete the -**feature surface**; M6 hardens **security**; M7 is the **API-freeze gate** that -declares the backend ready for front-end repos. Milestones are ordered but -M1–M4 are largely independent (they touch different engine crates); the only -hard serialization is the shared `nuncio.proto` (and its golden), so proto -additions land one at a time. +The original plan was seven milestones: M1–M5 complete the **feature surface**; +M6 hardens **security**; M7 is the **API-freeze gate** that declares the backend +ready for front-end repos. Milestones are ordered but M1–M4 are largely +independent (they touch different engine crates); the only hard serialization is +the shared `nuncio.proto` (and its golden), so proto additions land one at a +time. + +That structure still holds, with one amendment made after M5: the gap between +"the feature surface exists" and "the contract can be frozen" turned out to be +larger than M6 alone, so **M6.5**, the **WS-A … WS-F** workstreams, and the +**OBS** observability epic were opened between M6 and M7. They are described +under [M6.5, WS and OBS](#m65-ws-and-obs--the-work-between-m6-and-m7) below. ### M1 — Finish Calendar (closes #178) Complete the Calendar vertical behind the API. @@ -181,6 +225,25 @@ The engine becomes production-solid. - **Exit:** a security review passes; no fail-open crypto, update, or egress path remains. +### M6.5, WS and OBS — the work between M6 and M7 + +Opened after M5, when the reshape needed before an honest `v1` freeze proved +larger than M6. Status as of 2026-08-07: + +| Track | Closed / Open | Scope | +| :--- | :---: | :--- | +| **M6.5** Feature completeness & de-fabrication | 3 / 8 | Remove the remaining fabricated or silently-wrong paths (Contacts sync wiring #255, DAV XML namespaces #257, TLS-mode fallback #271, NL scheduler #258) | +| **WS-A** Contract hardening (pre-freeze reshape) | 6 / 3 | Typed errors, `Timestamp`/`Duration`, keyset pagination, id conventions, enum hygiene, account transport `oneof` — **gates M7** | +| **WS-B** Mail model & mutations | 1 / 9 | Message identity, drafts, folder management, attachments on receive | +| **WS-C** Sync, push & lifecycle | 4 / 4 | Autonomous scheduler, IMAP IDLE, resumable event stream, graceful shutdown | +| **WS-D** Security to 9/10 | 4 / 8 | Whole-DB SQLCipher (#299), UDS / named-pipe IPC (#305), signed updates (#300), HTML sanitization (#302) | +| **WS-E** Calendar/Contacts write & Search | 2 / 10 | DAV write-back with conditional PUT/DELETE, recurrence editing, structured Search service | +| **WS-F** Ops, usability & maintainability | 5 / 10 | CLI polish, `System.Backup`, and the `grpc.rs` (#333) / `DatabaseEngine` (#334) decompositions | +| **OBS-1 … OBS-9** Observability | 9 / 0 | **COMPLETE.** Tracing subscriber, RPC spans + request-id, domain/lifecycle coverage, protocol instrumentation, `GetStatus`, CLI verbosity, redaction canary | + +The OBS stories (#357–#365) were merged without a milestone assignment; the +epic is complete regardless, and `docs/LOGGING.md` is its reference. + ### M7 — API freeze & client-readiness gate ← **the goal line** Declare the backend ready for front-end repos. - **Freeze `nuncio.v1`:** every backend capability is represented; the @@ -208,7 +271,9 @@ scheduled here or folded back into the engine as they earn their place. ## Tracking -Each milestone maps to GitHub issues (numbers above). Every capability is built +Each milestone maps to GitHub issues (numbers above); M6.5 and WS-A … WS-F are +GitHub milestones too, so `gh issue list --milestone "<title>"` is the live +status for any track in the table above. Every capability is built via one implementer + a spec/quality review gate + an offline E2E, committed only when the local gate (fmt + clippy `-D warnings` + tests) is green and the proto golden is consistent. Issues close only after the review step — never on a green diff --git a/docs/roadmap/index.html b/docs/roadmap/index.html index 524df65..ac57958 100644 --- a/docs/roadmap/index.html +++ b/docs/roadmap/index.html @@ -133,17 +133,22 @@ <hr class="rule"> - <h2>Where we are · 2026-07-30</h2> + <h2>Where we are · 2026-08-07</h2> <div class="band"> - <h3>Phases 0–2 complete, feature surface (Phase 3–M5) done — M1–M5 all merged</h3> + <h3>Phases 0–2 complete, feature surface (Phase 3–M5) done, M6 nearly done, OBS complete</h3> <p>The daemon boots gRPC-only on loopback <span class="chip">127.0.0.1:9420</span> behind a keyring-minted bearer token. <code>IMAP → store → read → SMTP send</code> runs end-to-end over gRPC, driven by the <span class="chip">nuncio-cli</span> reference client, tested fully offline. The legacy JSON-RPC IPC is gone, and <span class="chip">nuncio.v1</span> is published with a byte-deterministic contract-stability golden. Milestones M1–M5 are now complete on <span class="chip">dev</span> (M1–M4 already promoted to <span class="chip">main</span>): Calendar, Contacts, JMAP, live Filters, and mail lifecycle + incremental - sync are all real, wired, and API-complete. <b style="color:var(--ink)">Next up: M6</b> (security & - release hardening).</p> + sync are all real, wired, and API-complete. <b style="color:var(--ink)">M6 (security & release + hardening) is 13 of 16 stories closed</b>, and the <b style="color:var(--ink)">OBS observability epic + is complete</b> (tracing subscriber, per-RPC spans with request-id correlation, a real + <code>GetStatus</code> health surface, CLI verbosity, and a no-secrets-in-logs redaction canary). + Work after M5 reorganized into <b style="color:var(--ink)">M6.5</b> (de-fabrication) and the + <b style="color:var(--ink)">WS-A … WS-F</b> pre-freeze reshape, which now carry the bulk of the + remaining effort. <b style="color:var(--ink)">M7 has not started</b>; it gates on WS-A.</p> <div class="chips"> <span class="chip svc">System</span><span class="chip svc">Accounts</span><span class="chip svc">Mail</span> <span class="chip svc">Filters</span><span class="chip svc">Calendar</span><span class="chip svc">Contacts</span> @@ -160,12 +165,13 @@ <span class="donetag">M3 · JMAP</span> <span class="donetag">M4 · Filters made live</span> <span class="donetag">M5 · Mail lifecycle + incremental sync</span> + <span class="donetag">OBS · Observability (OBS-1 … OBS-9)</span> </div> </div> <hr class="rule"> - <h2>The path · seven milestones to the goal line</h2> + <h2>The path · the milestones to the goal line</h2> <div class="miles"> <div class="m done"> @@ -252,13 +258,38 @@ <div class="card"> <div class="chead"> <h3 class="ctitle"><span class="code">M6</span>Security & release hardening</h3> - <span class="stat prog">Next</span> + <span class="stat prog">13 / 16 done</span> </div> <p class="cbody">Fail-<i>closed</i> everywhere: fix the silent empty-ciphertext data-loss path; a signature-verifying updater that refuses to apply without checksums; zeroize key material; validate vault - key lengths; gRPC bind hardening with an interceptor guard; real CIDR-based SSRF checks; sandboxed HTML rendering.</p> + key lengths; gRPC bind hardening with an interceptor guard; real CIDR-based SSRF checks; sandboxed HTML rendering. + <b>All of the above have landed except</b> inbound HTML sanitization (#204), Rust static analysis in CodeQL + (#250 — CodeQL scans JS and actions only today), and removal of assertion-free test theater (#251).</p> <div class="exit"><b>Exit:</b> a security review passes — no fail-open crypto, update, or egress path remains.</div> - <div class="issues"><a class="iss">#162</a><a class="iss">#163</a><a class="iss">#164</a><a class="iss">#165</a></div> + <div class="issues"><a class="iss">#162</a><a class="iss">#163</a><a class="iss">#164</a><a class="iss">#165</a><a class="iss">#204</a><a class="iss">#250</a><a class="iss">#251</a></div> + </div> + </div> + + <div class="m prog"> + <div class="node">WS</div> + <div class="card"> + <div class="chead"> + <h3 class="ctitle"><span class="code">M6.5 · WS · OBS</span>The work between M6 and M7</h3> + <span class="stat prog">In progress</span> + </div> + <p class="cbody">Opened after M5, when the reshape needed before an honest <code>v1</code> freeze proved larger + than M6 alone. <b>M6.5</b> de-fabrication (3/11) removes the remaining silently-wrong paths: Contacts sync + wiring (#255), namespace-aware DAV XML (#257), the TLS-mode fallback (#271), the NL scheduler (#258). + <b>WS-A … WS-F</b> (22/66) is the pre-freeze reshape: contract hardening (WS-A, <b>gates M7</b>), mail model + & mutations (WS-B), sync/push/lifecycle (WS-C), security to 9/10 (WS-D — SQLCipher #299, UDS & + named-pipe IPC #305, signed updates #300), calendar/contacts write-back & search (WS-E), and ops, + usability & maintainability (WS-F — including the <code>grpc.rs</code> #333 and <code>DatabaseEngine</code> + #334 decompositions). <b>OBS-1 … OBS-9 is complete</b> (9/9, #357–#365): layered tracing subscriber, per-RPC + spans with request-id correlation, domain and lifecycle coverage, protocol instrumentation, a real + <code>System.GetStatus</code>, CLI <code>-v</code>/<code>-vv</code> with typed errors, and a + <code>Redacted<T></code> policy with a no-secrets-in-logs canary.</p> + <div class="exit"><b>Exit:</b> WS-A closes the contract reshape, unblocking the M7 freeze.</div> + <div class="issues"><a class="iss">M6.5</a><a class="iss">WS-A</a><a class="iss">WS-B</a><a class="iss">WS-C</a><a class="iss">WS-D</a><a class="iss">WS-E</a><a class="iss">WS-F</a><span class="iss">OBS — done</span></div> </div> </div> @@ -273,7 +304,8 @@ golden covers the full surface, and the additive-only versioning policy is enforced by test. Verify Swift / C# / TypeScript codegen with a “hello, daemon” per language. <b>The CLI reference client exercises every RPC in offline E2E</b> — the proof the contract is complete and usable. Replace the gamed coverage gate; - CI green; tag an honest pre-alpha backend release.</p> + CI green; tag an honest pre-alpha backend release. <b>Not started</b> — 5 stories open, 0 closed; + gates on WS-A completing the contract reshape.</p> <div class="exit"><b>Exit / client-ready:</b> the daemon publishes a <b>frozen, documented, contract-tested gRPC API</b> that the CLI fully exercises offline. <b>Front-end repos may now begin.</b></div> <div class="issues"><a class="iss">#143</a></div> -- 2.54.0 From d54720439e480b4e71c4387c464de0810c436957 Mon Sep 17 00:00:00 2001 From: James Maes <jmaes@dmdbrands.com> Date: Fri, 7 Aug 2026 12:51:59 -0500 Subject: [PATCH 47/95] chore: drop dead npm dependabot path, ignore scratchpad The Tauri GUI npm tree moved to _reference/ and is out of the workspace, so /crates/nuncio-gui/ui no longer exists for dependabot to update. --- .github/dependabot.yml | 17 ++++------------- .gitignore | 1 + 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1295c0c..11a9183 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,20 +13,11 @@ updates: commit-message: prefix: "deps(cargo)" - # 2. Tauri v2 Desktop GUI UI (npm) - - package-ecosystem: "npm" - directory: "/crates/nuncio-gui/ui" - schedule: - interval: "daily" - time: "04:00" - open-pull-requests-limit: 10 - labels: - - "dependencies" - - "gui" - commit-message: - prefix: "deps(npm)" + # The Tauri GUI's npm tree moved out of the workspace to `_reference/` and is + # being rebuilt as a separate client repo, so there is no npm ecosystem to + # update here. Re-add an npm entry in the client repo, not this one. - # 3. GitHub Actions Workflows + # 2. GitHub Actions Workflows - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.gitignore b/.gitignore index b311c8e..5c29c04 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ _reference/**/gen/ **/node_modules/ **/dist/ trivy_err*.log +scratchpad/ -- 2.54.0 From 1e5f19c6ae95421c215edbe0dfcf413ef59c2248 Mon Sep 17 00:00:00 2001 From: James Maes <jmaes@dmdbrands.com> Date: Fri, 7 Aug 2026 13:36:46 -0500 Subject: [PATCH 48/95] docs: add Nuncio Monitor design spec In-repo dev instrument (crates/nuncio-monitor) plus the three engine changes it needs: System.Shutdown, pending_remote_mutations.account_id, and System.GetHealth (implements #296). --- .../specs/2026-08-07-nuncio-monitor-design.md | 219 ++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-07-nuncio-monitor-design.md diff --git a/docs/superpowers/specs/2026-08-07-nuncio-monitor-design.md b/docs/superpowers/specs/2026-08-07-nuncio-monitor-design.md new file mode 100644 index 0000000..6da06bd --- /dev/null +++ b/docs/superpowers/specs/2026-08-07-nuncio-monitor-design.md @@ -0,0 +1,219 @@ +# Nuncio Monitor — design + +Status: approved 2026-08-07. Implementation plan not yet written. + +## Purpose + +A Windows system-tray instrument for watching a running `nunciod` during +development: start and stop the engine, follow its logs live with structured +filtering, and see configured accounts with their per-account sync state and +queue depth. + +It is a **development and operations tool**, not a product client. The native +macOS and Windows GUIs remain separate repositories per +[`docs/adr/0001-engine-first-grpc-architecture.md`](../../adr/0001-engine-first-grpc-architecture.md); +this crate does not replace or preempt them. It holds no business logic and no +data store, so the library-first boundary in `CLAUDE.md` is preserved. + +## Decisions + +| Decision | Choice | Rationale | +| :--- | :--- | :--- | +| What it is | In-repo dev instrument | Fastest to something useful; free to read the log file and control the process, which a contract-only client could not do. | +| Where it lives | Workspace member `crates/nuncio-monitor` | The gate (`cargo check-all`, `cargo test-all`, `-D warnings`, `unwrap_used`/`panic` denies) covers it automatically. A tool outside the gate rots. | +| Engine lifetime | Independent | The daemon outlives the monitor. Closing the tray never stops mail sync; a monitor crash cannot take the engine down. `start` spawns detached. | +| Stop mechanism | New `System.Shutdown` RPC | Windows has no `SIGTERM`. An authenticated RPC is cross-platform, testable offline, serves future clients, and avoids OS-specific daemon code. | +| Log view | Structured JSON viewer | Level filter, text search, and click-to-follow `request_id` across an RPC. The `request_id` correlation is the payoff of the OBS epic. | +| Build order | All pieces in one pass | Monitor plus both engine changes designed and planned together. One design and one plan, sequenced per "Build order" below, not one commit. | + +## Scope + +In scope: engine start/stop/liveness, structured live log viewing, account list +with per-account sync state and queue depth, tray icon encoding coarse state. + +Out of scope: adding or editing accounts, triggering syncs, retrying the outbox, +reading mail, autostart at login. The monitor observes and controls lifecycle; +it does not operate the mail engine. + +## Architecture + +Three data sources, deliberately separate. The monitor never touches SQLite. + +| Source | Provides | Why this source | +| :--- | :--- | :--- | +| gRPC on `127.0.0.1:9420` | Engine status, per-account state, queue depths, live events, account list | The contract. | +| `%USERPROFILE%\.nuncio\logs\nunciod.log.<date>` | The log viewer | Logs are not in the contract, and a log-streaming RPC is a far larger design. Tailing the file the daemon already writes is the honest cheap path for a dev tool. | +| `<db_path>.lock` and OS process control | Liveness and starting the engine | A stopped daemon cannot answer an RPC about itself. The single-instance advisory lock is already the authoritative "is one running" signal. | + +### Engine changes + +All three serialize through the PO's proto queue, since two touch `nuncio.v1` +and `HANDOFF.md` allows one proto-touching story in flight at a time. + +1. **`System.Shutdown`** — new RPC on the `System` service, behind the existing + bearer-auth interceptor, triggering the shutdown signal in + `crates/nunciod/src/lifecycle.rs`. Any client holding the token can stop the + daemon; the token already grants full mail access, so this adds little real + privilege, but it is a genuine new capability in the contract and is recorded + as such. +2. **`pending_remote_mutations.account_id`** — additive column plus a backfill + joining `message_id` to `messages.account_id`, following the existing + `migrate_backfills_smtp_columns_for_a_pre_existing_accounts_table` precedent. + The table today carries `id, rule_id, message_id, mutation_type, payload, + status, retry_count, created_at` and has no account column, so per-account + outbox counts are not currently derivable. This is a prerequisite, not a + nicety. +3. **`System.GetHealth`** — implements existing open story **#296 (WS-C8)** + rather than inventing a parallel surface: outbox pending and failed per + account, per-account last-sync outcome, WAL size. #296 is claimed for this + work and annotated with the `account_id` prerequisite. Its own note to + de-duplicate with WS-F10 still applies. + + "Queue depth" throughout this document means **two counts per account, + reported separately**: `pending` and `failed`, matching the `status` column + on `pending_remote_mutations` and #296's own wording. A single merged number + would hide a stuck outbox behind a healthy-looking total, which the + three-state display rule below exists to prevent. + +### Build order + +"One pass" means one design and one plan, not one commit. `HANDOFF.md` allows +only one proto-touching story in flight at a time, so the work has a required +sequence: + +1. `pending_remote_mutations.account_id` plus backfill. Store-only, touches no + proto, so it can land first and unblocks the rest. +2. `System.Shutdown`. Proto change one. Small and self-contained. +3. `System.GetHealth` (#296). Proto change two, depends on step 1 for its + per-account counts. +4. `crates/nuncio-monitor`. Depends on steps 2 and 3 for stop and queue depth, + but its `LogTailer`, `EngineController` liveness, and `ui` shell depend on + none of them and can be built in parallel with the earlier steps. + +`System.GetStatus` already supplies `engine_status`, `version`, `uptime`, +`accounts_loaded`, `unread_count`, `last_error`, a global `outbox_depth`, +`ready`, `db_healthy`, and repeated `account_sync_states` carrying `account_id`, +`state`, `last_synced`, and `last_error`. `System.Subscribe` already provides a +live server-streaming event feed. Neither needs changing. + +### Consuming an unfrozen contract + +M7 has not started and WS-A is still reshaping `nuncio.v1`, so anything the +monitor consumes may move under it. For a dev instrument that cost is +acceptable, but it argues for keeping the client surface small: four RPCs +(`GetStatus`, `GetHealth`, `Subscribe`, `Shutdown`) plus `ListAccounts`, not +thirty. + +## Components + +Five, each independently testable. + +- **`EngineController`** — owns lifecycle. `is_running()` reads the lock file. + `start()` spawns `nunciod.exe` detached with `NUNCIO_LOG_FORMAT=json`, then + polls `GetStatus` until `ready`. `stop()` calls `System.Shutdown` and waits for + the lock to release. Parses no logs and renders nothing. +- **`StatusPoller`** — owns the authenticated channel. Reads the bearer token + from the keyring under `nuncio_store::vault::GRPC_TOKEN_ACCOUNT` + (`"grpc-bearer-token"`), hex-encodes it, and dials through the existing + `nuncio_proto::client::connect_system` / `connect_accounts` helpers, the same + path `nuncio-cli` uses. Runs a timed `GetStatus`/`GetHealth` poll for current + values alongside a `Subscribe` stream for push. Both are needed: the stream + says when something happened, the poll gives current depth and readiness. +- **`LogTailer`** — follows the current day's file, seeking to the end rather + than replaying history. Handles daily rotation by reopening when the dated + filename changes. Parses one JSON object per line into a structured record + (timestamp, level, target, `request_id`, fields, message) and falls back to + opaque text when the daemon was not started in JSON mode. +- **`ui`** — status header, accounts table, and log table with level filter, + text search, and click-a-`request_id`-to-filter. Pure rendering. +- **`tray`** — icon and menu; the icon encodes coarse state (stopped, running, + error) so a glance at the taskbar is meaningful without opening the window. + +### Data flow + +``` +lock file ──────────► EngineController ──┐ + ├──► AppState ──► ui / tray +gRPC (poll + stream) ► StatusPoller ─────┤ + │ +log file (JSON) ─────► LogTailer ────────┘ +``` + +`egui` runs on the main thread, a Tokio runtime on a worker, communicating over +channels. `AppState` is the single thing the UI reads. No component calls +another directly, which is what makes them testable without a running daemon. + +## Error handling + +The governing rule is the repo's anti-theater guardrail: no fabricated success. +Applied here, **every value has three states, not one** — a value, unknown, or +error. A monitor rendering `0 queued` when it actually failed to reach the +daemon is exactly the class of lie this codebase was rebuilt to eliminate. +Unknown must be visually distinct from zero. + +| Condition | Behaviour | +| :--- | :--- | +| No lock file | Stopped. Not an error. Start enabled; panes show unknown, not zeros. | +| Lock held, gRPC refuses | Starting or wedged. Distinct from stopped: show starting, with a timeout escalating to "not responding" rather than spinning forever. | +| No token in keyring | The daemon has never run on this machine, so no token was minted. Expected fresh-machine state; say so actionably. | +| `Unauthenticated` | Token rotated. Re-read the keyring once and redial; if it fails again, report plainly. | +| Log directory missing | The daemon degrades to stderr-only rather than failing to start. Surface that; do not render an empty pane as healthy. | +| Log not JSON | Fall back to opaque text and state why filtering is unavailable. | +| `Subscribe` drops | Reconnect with backoff; mark the stream stale so the UI does not imply live data. | +| `Shutdown` returns, process persists | Wait, then report honestly. Force-kill exists only as a separate, explicitly labelled action, never a silent fallback, since that would reintroduce the ungraceful exit `lifecycle.rs` removed. | +| Start blocked by instance lock | `InstanceLock` already returns a typed `AlreadyHeld`; surface it rather than a generic failure. | + +## Security + +The monitor renders log content, making it a surface where a redaction failure +becomes visible rather than merely present. It inherits the guarantee in +[`docs/LOGGING.md`](../../LOGGING.md) and `no_secrets_canary_test`: message +bodies, addresses, credentials, and tokens must never reach a log line at all. + +The monitor deliberately adds **no redaction of its own**, because a second +redaction layer would mask exactly the canary regressions worth catching. A +secret appearing in the monitor is a real engine bug and should look like one. + +## Testing + +All logic lives outside `ui`, because `egui` is impractical to assert against. +If something needs a test, it does not belong in the render layer. The crate is +a workspace member, so `-D warnings` and the `unwrap_used` / `expect_used` / +`panic` denies apply, which matters more than usual since GUI code reaches for +`unwrap` constantly. + +- **`LogTailer`** — fixture files: well-formed JSON, malformed lines mid-stream, + non-JSON fallback, midnight rotation handoff. No daemon, no network. +- **`EngineController`** — liveness against temporary lock files in `tempfile` + directories; the spawn path tested through an injected launcher rather than by + starting a real `nunciod.exe`. +- **`StatusPoller`** — against a test-local gRPC server with `MockKeyring`, + mirroring the seam `nuncio-cli`'s runner already uses. +- **`System.Shutdown`** — offline E2E asserting the daemon actually exits and + that an unauthenticated call is rejected. +- **`account_id` migration** — backfill test following the existing SMTP-column + precedent. +- **`System.GetHealth`** — per #296's own definition of done. + +Live network calls remain forbidden throughout, per `docs/STORY-WORKFLOW.md`. + +## Accepted costs + +- **Dependency surface.** `eframe`, `egui`, and `tray-icon` enter every + `cargo test-all` and every CI run, on a project that currently has no + dependency-advisory scanning: no `cargo-audit`, no `cargo-deny`, no + `deny.toml`, no supply-chain CI job, and CodeQL scanning JS and actions only + (#250). The monitor roughly doubles the third-party surface such a scanner + would watch, which strengthens the case for landing `cargo-deny` soon. +- **CI time.** GUI dependencies slow the shared build. If that bites, moving the + crate to a nested workspace under `tools/` is a directory move plus a CI job. +- **Contract churn.** Consuming `nuncio.v1` before the M7 freeze means the + monitor may need updating as WS-A reshapes it. + +## Definition of done + +Per `CLAUDE.md`: engine, proto, CLI command, and offline E2E all exist together, +with the local gate green. For the two engine changes that means proto plus +daemon plus a `nuncio-cli` command plus an offline E2E each. The monitor itself +is a GUI and has no CLI surface; its equivalent bar is that every component +outside `ui` is unit-tested and the crate passes the workspace gate. -- 2.54.0 From a997220a98e70fa1f7df8052188a6cbfac1efd40 Mon Sep 17 00:00:00 2001 From: James Maes <jmaes@dmdbrands.com> Date: Fri, 7 Aug 2026 13:46:05 -0500 Subject: [PATCH 49/95] docs: add Nuncio Monitor implementation plan Twelve tasks across four phases: outbox account attribution, System.Shutdown, System.GetHealth, then the monitor crate. --- .../plans/2026-08-07-nuncio-monitor.md | 975 ++++++++++++++++++ 1 file changed, 975 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-07-nuncio-monitor.md diff --git a/docs/superpowers/plans/2026-08-07-nuncio-monitor.md b/docs/superpowers/plans/2026-08-07-nuncio-monitor.md new file mode 100644 index 0000000..2190024 --- /dev/null +++ b/docs/superpowers/plans/2026-08-07-nuncio-monitor.md @@ -0,0 +1,975 @@ +# Nuncio Monitor Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a Windows system-tray dev instrument (`crates/nuncio-monitor`) that starts and stops `nunciod`, follows its logs live with structured filtering, and shows configured accounts with per-account sync state and outbox depth — plus the three engine changes it needs. + +**Architecture:** The monitor is a thin client reading from three separate sources: gRPC on `127.0.0.1:9420` for facts, the daemon's rotating JSON log file for the log viewer, and the `<db_path>.lock` single-instance lock for liveness. It never touches SQLite. Engine-side work adds a `System.Shutdown` RPC, an `account_id` column on `pending_remote_mutations`, and `System.GetHealth` (issue #296). + +**Tech Stack:** Rust 1.97.1 (pinned), `eframe`/`egui` for the window, `tray-icon` for the tray, `tonic`/`prost` via the existing `nuncio-proto` client helpers, `tokio`, `serde_json`, `sqlx` (store only). + +Design: [`docs/superpowers/specs/2026-08-07-nuncio-monitor-design.md`](../specs/2026-08-07-nuncio-monitor-design.md) + +## Global Constraints + +- Toolchain is pinned to **1.97.1** by `rust-toolchain.toml`. Do not bump it. +- Workspace lints are **deny**: `warnings`, `unsafe_code`, `unused_must_use`, and clippy's `unwrap_used`, `expect_used`, `panic`, `todo`, `unimplemented`, `unreachable`. This applies to the GUI crate too. In tests these are allowed; in production code they are compile errors. +- The gate is three commands, run in order: `cargo fmt --all -- --check`, `cargo check-all`, `cargo test-all`. The pre-commit hook runs exactly this. +- **No live network in tests.** Mock with `wiremock` or the `Mock*Backend` traits; mock the keyring with `SecretManager::mock()`. +- **No fabricated success.** A path that cannot do the real thing returns an honest error, never canned data. In the monitor UI this means unknown must render differently from zero. +- **The monitor adds no redaction of its own.** Secrets are kept out of logs at the source by `Redacted<T>` and `no_secrets_canary_test` (see `docs/LOGGING.md`). A second redaction layer in the viewer would mask exactly the canary regressions worth catching, so a secret appearing in the monitor must look like the engine bug it is. Do not add filtering or masking to `log_tail.rs` or `ui.rs`. +- Conventional Commits, imperative, subject under 72 chars, **no AI attribution**. +- Do not put issue numbers, story ids, or `Phase N` breadcrumbs in `.rs` or `.proto` comments. That belongs in git history and `docs/`. +- **Only one proto-touching task may be in flight at a time** (`docs/HANDOFF.md`). Tasks 2 and 5 both touch `nuncio.v1`; do not start Task 5 until Task 2 is merged. +- Any change to `crates/nuncio-proto/proto/nuncio/v1/nuncio.proto` requires regenerating the committed descriptor golden `descriptor.bin`, or the contract-stability test fails. +- Kill any running `nunciod.exe` before building on Windows: a live daemon holds `target\debug\nunciod.exe` and the link step fails with `Access is denied. (os error 5)`. + +## File Structure + +**Phase 1 — store** +- Modify: `crates/nuncio-store/src/db.rs` — add `ensure_pending_mutations_account_column()`, call it from `migrate()`, add `count_pending_mutations_by_account()`. + +**Phase 2 — Shutdown RPC** +- Modify: `crates/nuncio-proto/proto/nuncio/v1/nuncio.proto` — `rpc Shutdown` on `System`, plus request/response messages. +- Modify: `crates/nuncio-proto/proto/nuncio/v1/descriptor.bin` — regenerated golden. +- Modify: `crates/nuncio-proto/src/client.rs` — nothing new; `connect_system` already returns the client that gains the method. +- Modify: `crates/nunciod/src/grpc.rs` — `shutdown` field on `SystemGrpcService`, `shutdown()` handler. +- Modify: `crates/nunciod/src/main.rs` — share the `ShutdownController` with the gRPC server. +- Modify: `crates/nuncio-cli/src/args.rs`, `crates/nuncio-cli/src/runner.rs` — `system shutdown` command. +- Create: `crates/nunciod/tests/shutdown_e2e_test.rs`. + +**Phase 3 — GetHealth** +- Modify: `crates/nuncio-proto/proto/nuncio/v1/nuncio.proto` — `rpc GetHealth`, `GetHealthResponse`, `AccountQueueDepth`. +- Modify: `crates/nunciod/src/grpc.rs` — `get_health()` handler. +- Modify: `crates/nuncio-cli/src/args.rs`, `runner.rs` — `system health` command. +- Create: `crates/nunciod/tests/health_e2e_test.rs`. + +**Phase 4 — monitor** +- Create: `crates/nuncio-monitor/Cargo.toml`, `src/main.rs`, `src/log_tail.rs`, `src/engine.rs`, `src/status.rs`, `src/state.rs`, `src/ui.rs`, `src/tray.rs`. +- Modify: root `Cargo.toml` — add the workspace member and the three new workspace dependencies. + +One file per responsibility. `state.rs` holds `AppState`, the only thing `ui.rs` reads, which is what keeps the logic testable without a GUI harness. + +--- + +## Phase 1 — Store: per-account outbox + +### Task 1: Add `account_id` to `pending_remote_mutations` with backfill + +`pending_remote_mutations` today is `id, rule_id, message_id, mutation_type, payload, status, retry_count, created_at` (`crates/nuncio-store/src/db.rs:575`). There is no account column, so per-account outbox counts are not derivable. `messages.account_id` exists (`crates/nuncio-store/src/db.rs:505`), so the backfill joins through `message_id`. + +**Files:** +- Modify: `crates/nuncio-store/src/db.rs` (add method near `ensure_accounts_dav_columns` at line 954; call site at line 753; new query near `count_pending_mutations` at line 464) +- Test: `crates/nuncio-store/src/db.rs` (inline `mod tests`, following `migrate_backfills_smtp_columns_for_a_pre_existing_accounts_table` at line 3771) + +**Interfaces:** +- Consumes: nothing. +- Produces: `DatabaseEngine::count_pending_mutations_by_account() -> Result<Vec<(String, u64, u64)>, DatabaseError>` returning `(account_id, pending, failed)`. Task 5 calls this. + +- [ ] **Step 1: Write the failing test** + +Add to the `mod tests` block in `crates/nuncio-store/src/db.rs`: + +```rust +#[tokio::test] +async fn migrate_backfills_account_id_on_a_pre_existing_outbox_table() { + let dir = tempfile::tempdir().expect("tempdir"); + let db_path = dir.path().join("pre_account_id.db"); + let secrets = crate::vault::SecretManager::mock(); + + // OLD-schema outbox table: no account_id column. + { + let engine = DatabaseEngine::new(&db_path, secrets.clone()) + .await + .expect("engine"); + sqlx::query("DROP TABLE IF EXISTS pending_remote_mutations") + .execute(&engine.pool) + .await + .expect("drop"); + sqlx::query( + "CREATE TABLE pending_remote_mutations ( + id TEXT PRIMARY KEY NOT NULL, + rule_id TEXT NOT NULL, + message_id TEXT NOT NULL, + mutation_type TEXT NOT NULL, + payload TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + retry_count INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL + )", + ) + .execute(&engine.pool) + .await + .expect("create old table"); + + sqlx::query( + "INSERT INTO messages (id, account_id, folder_id, remote_id) + VALUES ('msg-1', 'acct-A', 'inbox', 'r1')", + ) + .execute(&engine.pool) + .await + .expect("seed message"); + + sqlx::query( + "INSERT INTO pending_remote_mutations + (id, rule_id, message_id, mutation_type, payload, status, retry_count, created_at) + VALUES ('mut-1', 'rule-1', 'msg-1', 'move', '{}', 'pending', 0, 0)", + ) + .execute(&engine.pool) + .await + .expect("seed mutation"); + } + + // Reopening runs migrate(), which must add and backfill account_id. + let engine = DatabaseEngine::new(&db_path, secrets) + .await + .expect("reopen"); + + let counts = engine + .count_pending_mutations_by_account() + .await + .expect("counts"); + + assert_eq!(counts, vec![("acct-A".to_string(), 1, 0)]); +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cargo test -p nuncio-store migrate_backfills_account_id_on_a_pre_existing_outbox_table` +Expected: FAIL — `no method named 'count_pending_mutations_by_account'`. + +- [ ] **Step 3: Add the migration** + +In `crates/nuncio-store/src/db.rs`, after `ensure_accounts_dav_columns`, mirroring its `PRAGMA table_info` shape exactly: + +```rust + /// Additive, backfill-safe migration adding `account_id` to a pre-existing + /// `pending_remote_mutations` table. + /// + /// The outbox originally recorded only the message a mutation targets, so + /// queued work could not be attributed to an account. The column is added + /// as `NULL`-able and backfilled by joining `message_id` to + /// `messages.account_id`; a mutation whose message row is already gone + /// keeps a `NULL` account and is reported separately rather than being + /// silently dropped from the totals. SQLite has no `ADD COLUMN IF NOT + /// EXISTS`, so presence is checked via `PRAGMA table_info` first, making + /// this safe to run on every daemon startup. + async fn ensure_pending_mutations_account_column(&self) -> Result<(), DatabaseError> { + let existing_columns: Vec<String> = + sqlx::query("PRAGMA table_info(pending_remote_mutations)") + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)? + .iter() + .map(|row| row.get::<String, _>("name")) + .collect(); + + if !existing_columns.iter().any(|c| c == "account_id") { + sqlx::query("ALTER TABLE pending_remote_mutations ADD COLUMN account_id TEXT") + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + tracing::info!( + table = "pending_remote_mutations", + column = "account_id", + "applying schema migration" + ); + + sqlx::query( + "UPDATE pending_remote_mutations + SET account_id = ( + SELECT m.account_id FROM messages m + WHERE m.id = pending_remote_mutations.message_id + ) + WHERE account_id IS NULL", + ) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + } + + Ok(()) + } +``` + +Add `account_id TEXT` to the `CREATE TABLE IF NOT EXISTS pending_remote_mutations` block at line 575 so fresh databases get it directly, and add the call after line 753: + +```rust + self.ensure_pending_mutations_account_column().await?; +``` + +- [ ] **Step 4: Add the per-account count query** + +After `count_pending_mutations` at line 464: + +```rust + /// Per-account outbox depth as `(account_id, pending, failed)`, ordered by + /// account id. Mutations whose `account_id` is `NULL` (their message row + /// was deleted before the backfill ran) are grouped under the empty string + /// rather than being dropped, so the reported totals always reconcile with + /// [`Self::count_pending_mutations`]. + pub async fn count_pending_mutations_by_account( + &self, + ) -> Result<Vec<(String, u64, u64)>, DatabaseError> { + let rows: Vec<(Option<String>, i64, i64)> = sqlx::query_as( + "SELECT account_id, + SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END), + SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) + FROM pending_remote_mutations + GROUP BY account_id + ORDER BY account_id", + ) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map(|(id, pending, failed)| { + ( + id.unwrap_or_default(), + pending.max(0) as u64, + failed.max(0) as u64, + ) + }) + .collect()) + } +``` + +- [ ] **Step 5: Run the test to verify it passes** + +Run: `cargo test -p nuncio-store migrate_backfills_account_id_on_a_pre_existing_outbox_table` +Expected: PASS. + +- [ ] **Step 6: Run the full gate** + +Run: `cargo fmt --all -- --check && cargo check-all && cargo test-all` +Expected: all green, 596+ tests passing. + +- [ ] **Step 7: Commit** + +```bash +git add crates/nuncio-store/src/db.rs +git commit -m "feat(store): attribute outbox mutations to an account" +``` + +--- + +## Phase 2 — `System.Shutdown` + +> Proto-touching. Do not start Task 5 until this phase is merged. + +### Task 2: Add the `Shutdown` RPC and handler + +**Files:** +- Modify: `crates/nuncio-proto/proto/nuncio/v1/nuncio.proto:78-90` +- Modify: `crates/nuncio-proto/proto/nuncio/v1/descriptor.bin` (regenerated) +- Modify: `crates/nunciod/src/grpc.rs:243` (struct), `:255` (impl), `:2979` (construction) +- Modify: `crates/nunciod/src/main.rs:52-54` +- Test: `crates/nunciod/src/grpc.rs` inline tests + +**Interfaces:** +- Consumes: `ShutdownController::trigger()` from `crates/nunciod/src/lifecycle.rs:90`. +- Produces: `nuncio.v1.System/Shutdown` taking `ShutdownRequest{}` and returning `ShutdownResponse{}`. Tasks 3 and 11 call it. + +- [ ] **Step 1: Write the failing test** + +Add to the `mod tests` block in `crates/nunciod/src/grpc.rs`, alongside `get_status_rejects_missing_bearer_token` at line 3532: + +```rust +#[tokio::test] +async fn shutdown_rejects_missing_bearer_token() { + let event_bus = Arc::new(EventBus::new()); + let (addr, _handle, _dir) = spawn_test_server(event_bus, "correct-token").await; + + let mut client = nuncio_proto::v1::system_client::SystemClient::connect(format!("http://{addr}")) + .await + .expect("connect"); + + let status = client + .shutdown(nuncio_proto::v1::ShutdownRequest {}) + .await + .expect_err("must reject unauthenticated shutdown"); + + assert_eq!(status.code(), tonic::Code::Unauthenticated); +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cargo test -p nunciod shutdown_rejects_missing_bearer_token` +Expected: FAIL — `ShutdownRequest` does not exist. + +- [ ] **Step 3: Add the proto surface** + +In `crates/nuncio-proto/proto/nuncio/v1/nuncio.proto`, inside `service System` (line 78): + +```proto + // Shutdown asks the daemon to begin a graceful shutdown: in-flight work is + // allowed to finish, then the process exits. Returns before the process is + // gone, so a caller that needs to observe the exit should poll the + // single-instance lock rather than assume completion on response. + // + // This is the only way to stop the daemon cleanly on platforms without a + // POSIX signal for it. It is guarded by the same bearer-auth interceptor as + // every other RPC. + rpc Shutdown(ShutdownRequest) returns (ShutdownResponse); +``` + +And after `GetStatusRequest` (line 90): + +```proto +// ShutdownRequest carries no parameters. A future revision may add a drain +// deadline. +message ShutdownRequest {} + +// ShutdownResponse is returned once shutdown has been *requested*, not once it +// has completed. +message ShutdownResponse {} +``` + +- [ ] **Step 4: Regenerate the descriptor golden** + +Run: `cargo build -p nuncio-proto` +Then follow the regeneration step the crate documents (check `crates/nuncio-proto/README.md` or the golden test's failure message for the exact command). +Expected: `descriptor.bin` updated; the contract-stability test passes. + +- [ ] **Step 5: Wire the controller into the service** + +In `crates/nunciod/src/grpc.rs`, add to `SystemGrpcService` (line 243): + +```rust + /// Triggers graceful shutdown when `Shutdown` is called. Optional because + /// test servers are spawned without a real lifecycle controller; a `None` + /// here makes `Shutdown` return `Unavailable` rather than pretending to + /// succeed. + shutdown: Option<Arc<ShutdownController>>, +``` + +Add the handler inside `impl System for SystemGrpcService`: + +```rust + async fn shutdown( + &self, + _request: Request<ShutdownRequest>, + ) -> Result<Response<ShutdownResponse>, Status> { + match &self.shutdown { + Some(controller) => { + tracing::info!("shutdown requested over gRPC"); + controller.trigger(); + Ok(Response::new(ShutdownResponse {})) + } + None => Err(Status::unavailable( + "this daemon instance has no lifecycle controller wired", + )), + } + } +``` + +Set `shutdown: None` in the test-server construction paths and thread the real controller through `serve_on_listener_with_overrides` to the construction at line 2979. + +- [ ] **Step 6: Share the controller in main.rs** + +In `crates/nunciod/src/main.rs`, wrap the controller in an `Arc` at line 52 and clone it: one clone into `install_signal_handlers`, one into the gRPC serve call. + +- [ ] **Step 7: Run the tests** + +Run: `cargo test -p nunciod shutdown_rejects_missing_bearer_token && cargo test -p nuncio-proto` +Expected: PASS, including the contract-stability golden. + +- [ ] **Step 8: Commit** + +```bash +git add crates/nuncio-proto crates/nunciod/src/grpc.rs crates/nunciod/src/main.rs +git commit -m "feat(proto): add System.Shutdown for graceful stop" +``` + +### Task 3: CLI command and offline E2E for `Shutdown` + +Definition of done in `CLAUDE.md` requires engine, proto, CLI command, and offline E2E together. + +**Files:** +- Modify: `crates/nuncio-cli/src/args.rs`, `crates/nuncio-cli/src/runner.rs` +- Create: `crates/nunciod/tests/shutdown_e2e_test.rs` + +**Interfaces:** +- Consumes: `nuncio.v1.System/Shutdown` from Task 2; `nuncio_proto::client::connect_system(addr, token)`. +- Produces: `nuncio system shutdown` CLI command. + +- [ ] **Step 1: Write the failing E2E** + +Create `crates/nunciod/tests/shutdown_e2e_test.rs`, following the shape of the existing `crates/nunciod/tests/graceful_shutdown_test.rs`: + +```rust +//! Proves an authenticated `System.Shutdown` actually stops the daemon, using +//! the same wiring `main.rs` assembles: one `ShutdownController` shared by the +//! gRPC server and the lifecycle path. Never opens a socket off 127.0.0.1. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use nuncio_core::EventBus; +use nuncio_filter::FilterEngine; +use nuncio_proto::v1::system_client::SystemClient; +use nuncio_proto::v1::ShutdownRequest; +use nuncio_store::db::DatabaseEngine; +use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT}; +use nunciod::grpc::serve_on_listener_with_shutdown; +use nunciod::lifecycle::ShutdownController; +use std::sync::Arc; +use std::time::Duration; +use tokio::net::TcpListener; +use tonic::metadata::MetadataValue; +use tonic::Request; + +const JOIN_TIMEOUT: Duration = Duration::from_secs(5); + +#[tokio::test] +async fn authenticated_shutdown_stops_the_serving_task() { + let event_bus = Arc::new(EventBus::new()); + let (controller, shutdown_signal) = ShutdownController::new(event_bus.clone()); + let controller = Arc::new(controller); + + let secrets = Arc::new(SecretManager::mock()); + let token = hex::encode( + secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("token provisioned from mock vault"), + ); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + let db = Arc::new(db); + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("listener has local addr"); + + let serve = tokio::spawn(serve_on_listener_with_shutdown( + listener, + event_bus, + db, + filter_engine, + secrets, + token.clone(), + Arc::clone(&controller), + async move { + let mut sig = shutdown_signal; + sig.wait().await; + }, + )); + + let mut client = SystemClient::connect(format!("http://{addr}")) + .await + .expect("connect"); + + let mut request = Request::new(ShutdownRequest {}); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {token}")).expect("valid metadata"), + ); + + client.shutdown(request).await.expect("shutdown accepted"); + + // The RPC returns once shutdown is REQUESTED; the serve task must then + // finish on its own. A timeout here means the trigger never reached the + // lifecycle signal. + let joined = tokio::time::timeout(JOIN_TIMEOUT, serve).await; + assert!( + joined.is_ok(), + "gRPC serve task did not stop within {JOIN_TIMEOUT:?} of Shutdown" + ); +} +``` + +Adjust the `serve_on_listener_with_shutdown` argument list to match whatever signature Task 2 Step 5 settled on; the existing call in `crates/nunciod/tests/graceful_shutdown_test.rs` is the reference for every argument except the new controller. + +- [ ] **Step 2: Run it to verify it fails** + +Run: `cargo test -p nunciod --test shutdown_e2e_test` +Expected: FAIL — no CLI/serve wiring yet, or an empty test body asserting nothing. + +- [ ] **Step 3: Add the CLI command** + +In `crates/nuncio-cli/src/args.rs`, add a `Shutdown` variant to the `system` subcommand enum. In `runner.rs`, implement it next to the existing status handler, reusing the `connect_system` helper and the token-from-vault pattern at `crates/nuncio-cli/src/runner.rs:1152`. + +- [ ] **Step 4: Run the tests** + +Run: `cargo test -p nunciod --test shutdown_e2e_test && cargo test -p nuncio-cli` +Expected: PASS. + +- [ ] **Step 5: Run the full gate and commit** + +```bash +cargo fmt --all -- --check && cargo check-all && cargo test-all +git add crates/nuncio-cli crates/nunciod/tests/shutdown_e2e_test.rs +git commit -m "feat(cli): add system shutdown command" +``` + +--- + +## Phase 3 — `System.GetHealth` (closes #296) + +> Proto-touching. Start only after Phase 2 is merged. + +### Task 4: Add the `GetHealth` RPC and handler + +**Files:** +- Modify: `crates/nuncio-proto/proto/nuncio/v1/nuncio.proto`, `descriptor.bin` +- Modify: `crates/nunciod/src/grpc.rs` +- Test: inline tests in `grpc.rs` + +**Interfaces:** +- Consumes: `DatabaseEngine::count_pending_mutations_by_account()` from Task 1. +- Produces: `nuncio.v1.System/GetHealth` returning `GetHealthResponse`. Tasks 6 and 10 call it. + +- [ ] **Step 1: Write the failing test** + +```rust +#[tokio::test] +async fn get_health_reports_per_account_queue_depth() { + // Seed two accounts, one with 2 pending and 1 failed mutation, one with none. + // Call GetHealth with a valid token. + // Assert both accounts appear, with exact pending/failed counts, and that + // an account with no queued work is present with zeros rather than absent. +} +``` + +Fill the body using `spawn_test_server_with` (`crates/nunciod/src/grpc.rs:3392`) so you can inject a seeded `DatabaseEngine`. + +- [ ] **Step 2: Run it to verify it fails** + +Run: `cargo test -p nunciod get_health_reports_per_account_queue_depth` +Expected: FAIL — `GetHealthResponse` does not exist. + +- [ ] **Step 3: Add the proto surface** + +```proto + // GetHealth reports operational depth: per-account outbox backlog, the + // per-account last-sync outcome, and database size signals. Distinct from + // GetStatus, which answers "is the engine up"; this answers "is it keeping + // up". Every value is read live when the RPC is served. + rpc GetHealth(GetHealthRequest) returns (GetHealthResponse); +``` + +```proto +message GetHealthRequest {} + +// Outbox backlog for one account. `pending` and `failed` are reported +// SEPARATELY and must never be summed into one number by a caller: a merged +// total hides a stuck queue behind a healthy-looking figure. +message AccountQueueDepth { + string account_id = 1; + uint64 pending = 2; + uint64 failed = 3; +} + +message GetHealthResponse { + repeated AccountQueueDepth account_queues = 1; + uint64 wal_size_bytes = 2; + bool db_healthy = 3; +} +``` + +- [ ] **Step 4: Regenerate the golden and implement the handler** + +Regenerate `descriptor.bin` as in Task 2 Step 4. Implement `get_health` on `SystemGrpcService`, calling `count_pending_mutations_by_account()`. Return an honest error on a store read failure rather than an empty list, since an empty list means "no queued work" and a failure means "unknown". + +- [ ] **Step 5: Run the tests** + +Run: `cargo test -p nunciod get_health && cargo test -p nuncio-proto` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/nuncio-proto crates/nunciod/src/grpc.rs +git commit -m "feat(proto): add System.GetHealth operational surface" +``` + +### Task 5: CLI command and offline E2E for `GetHealth` + +**Files:** +- Modify: `crates/nuncio-cli/src/args.rs`, `runner.rs` +- Create: `crates/nunciod/tests/health_e2e_test.rs` + +**Interfaces:** +- Consumes: `System/GetHealth` from Task 4. +- Produces: `nuncio system health` CLI command. + +- [ ] **Step 1: Write the failing E2E** + +Create `crates/nunciod/tests/health_e2e_test.rs` asserting an authenticated `GetHealth` returns the seeded per-account depths, and that an unauthenticated call is rejected with `Unauthenticated`. + +- [ ] **Step 2: Run it to verify it fails** + +Run: `cargo test -p nunciod --test health_e2e_test` +Expected: FAIL. + +- [ ] **Step 3: Add the CLI command** + +Mirror Task 3 Step 3: a `Health` variant plus a runner arm reusing `connect_system`. + +- [ ] **Step 4: Run the gate and commit** + +```bash +cargo fmt --all -- --check && cargo check-all && cargo test-all +git add crates/nuncio-cli crates/nunciod/tests/health_e2e_test.rs +git commit -m "feat(cli): add system health command + +Closes #296" +``` + +--- + +## Phase 4 — The monitor crate + +Tasks 6 through 8 depend on nothing in Phases 1 to 3 and may be built in parallel with them. + +### Task 6: Scaffold the crate and add dependencies + +**Files:** +- Create: `crates/nuncio-monitor/Cargo.toml`, `crates/nuncio-monitor/src/main.rs` +- Modify: root `Cargo.toml` (members list and `[workspace.dependencies]`) + +**Interfaces:** +- Produces: a buildable `nuncio-monitor` binary that the gate covers. + +- [ ] **Step 1: Add the workspace member and dependencies** + +In the root `Cargo.toml`, add `"crates/nuncio-monitor"` to `members`, and to `[workspace.dependencies]`: + +```toml +eframe = "0.29" +egui = "0.29" +egui_extras = "0.29" +tray-icon = "0.19" +``` + +Verify current versions against crates.io before pinning; these must compile on Rust 1.97.1. + +- [ ] **Step 2: Create the crate manifest** + +```toml +[package] +name = "nuncio-monitor" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +nuncio-proto = { path = "../nuncio-proto" } +nuncio-store = { path = "../nuncio-store" } +tokio = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +eframe = { workspace = true } +egui = { workspace = true } +egui_extras = { workspace = true } +tray-icon = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +``` + +- [ ] **Step 3: Create a minimal main that compiles** + +```rust +//! Nuncio Monitor: a development instrument for watching a running `nunciod`. + +fn main() { + println!("nuncio-monitor"); +} +``` + +- [ ] **Step 4: Verify the gate covers it** + +Run: `cargo check-all` +Expected: `nuncio-monitor` appears in the checked crates and passes. + +- [ ] **Step 5: Commit** + +```bash +git add Cargo.toml crates/nuncio-monitor +git commit -m "feat(monitor): scaffold the nuncio-monitor crate" +``` + +### Task 7: `LogTailer` + +**Files:** +- Create: `crates/nuncio-monitor/src/log_tail.rs` +- Test: same file, inline `mod tests` + +**Interfaces:** +- Consumes: `nunciod::logging::log_dir_for_db_path` semantics (logs live at `<data_dir>/logs/nunciod.log.<date>`). Do not depend on the `nunciod` crate; take the directory as a parameter. +- Produces: `LogRecord { timestamp: String, level: String, target: String, request_id: Option<String>, message: String, raw: String }`; `LogAvailability { Ok, DirectoryMissing, NotJson }`; `LogTailer::new(dir: PathBuf) -> Self`, `LogTailer::poll(&mut self) -> Vec<LogRecord>`, `LogTailer::availability(&self) -> LogAvailability`. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn parses_a_json_line_into_a_record() { + let line = r#"{"timestamp":"2026-08-07T12:00:00Z","level":"INFO","target":"nunciod","fields":{"message":"started","request_id":"abc"}}"#; + let rec = LogRecord::parse(line); + assert_eq!(rec.level, "INFO"); + assert_eq!(rec.request_id.as_deref(), Some("abc")); + assert_eq!(rec.message, "started"); +} + +#[test] +fn a_non_json_line_becomes_an_opaque_record_not_an_error() { + let rec = LogRecord::parse("2026-08-07 plain text log line"); + assert_eq!(rec.level, "RAW"); + assert_eq!(rec.raw, "2026-08-07 plain text log line"); +} + +#[test] +fn a_malformed_json_line_mid_stream_does_not_abort_the_tail() { + let lines = ["{\"level\":\"INFO\"}", "{not json", "{\"level\":\"WARN\"}"]; + let recs: Vec<_> = lines.iter().map(|l| LogRecord::parse(l)).collect(); + assert_eq!(recs.len(), 3); + assert_eq!(recs[1].level, "RAW"); +} +``` + +- [ ] **Step 2: Run them to verify they fail** + +Run: `cargo test -p nuncio-monitor log_tail` +Expected: FAIL — `LogRecord` not defined. + +- [ ] **Step 3: Implement `LogRecord::parse` and the tailer** + +Implement parsing with `serde_json::Value` so an unexpected shape degrades to `RAW` instead of erroring. Implement `LogTailer` holding the directory, the currently open file, its dated filename, and the last read offset. `poll()` reopens when the dated filename changes (midnight rotation) and reads only from the stored offset. Seek to end on first open so history is not replayed. + +- [ ] **Step 4: Add a rotation test** + +```rust +#[test] +fn rotating_to_a_new_dated_file_continues_the_tail() { + // Write nunciod.log.2026-08-07, poll, then write nunciod.log.2026-08-08 + // and poll again; assert records from both files are returned in order. +} +``` + +- [ ] **Step 5: Handle a missing log directory honestly** + +`docs/LOGGING.md` states the daemon degrades to stderr-only logging (with a `warn!`) when the log directory cannot be created, rather than failing to start. So an absent directory means "this daemon is not writing logs", **not** "no logs yet", and the UI must say so instead of rendering an empty pane as if healthy. + +```rust +#[test] +fn a_missing_log_directory_is_reported_not_silently_empty() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut tailer = LogTailer::new(dir.path().join("does-not-exist")); + assert_eq!(tailer.poll(), Vec::new()); + assert_eq!(tailer.availability(), LogAvailability::DirectoryMissing); +} +``` + +Add `LogAvailability { Ok, DirectoryMissing, NotJson }` and a `LogTailer::availability(&self)` accessor. Task 11 renders it. + +- [ ] **Step 6: Run the tests and commit** + +```bash +cargo test -p nuncio-monitor +git add crates/nuncio-monitor/src/log_tail.rs +git commit -m "feat(monitor): tail and parse the daemon's rotating JSON log" +``` + +### Task 8: `EngineController` + +**Files:** +- Create: `crates/nuncio-monitor/src/engine.rs` +- Test: same file, inline `mod tests` + +**Interfaces:** +- Consumes: `System/Shutdown` from Task 2. +- Produces: `EngineState { Stopped, Starting, Running, NotResponding }` and `EngineController::liveness(&self) -> EngineState`, `EngineController::start(&self) -> Result<(), EngineError>`, `EngineController::stop(&self) -> Result<(), EngineError>`. + +- [ ] **Step 1: Write the failing liveness tests** + +```rust +#[test] +fn no_lock_file_means_stopped() { + let dir = tempfile::tempdir().expect("tempdir"); + let db = dir.path().join("nuncio.db"); + let ctl = EngineController::new(db, Launcher::noop()); + assert_eq!(ctl.liveness(), EngineState::Stopped); +} + +#[test] +fn a_held_lock_file_means_not_stopped() { + // Create <db>.lock and hold an exclusive OS lock on it, then assert + // liveness() is not Stopped. +} +``` + +- [ ] **Step 2: Run them to verify they fail** + +Run: `cargo test -p nuncio-monitor engine` +Expected: FAIL — `EngineController` not defined. + +- [ ] **Step 3: Implement liveness and an injected launcher** + +Liveness reads `<db_path>.lock` and attempts a non-blocking shared lock: success means no daemon holds it (Stopped), failure means one does. Define `Launcher` as a trait with a `spawn(&self) -> Result<(), EngineError>` method so tests inject a no-op and production spawns `nunciod.exe` detached with `NUNCIO_LOG_FORMAT=json` set. + +- [ ] **Step 4: Implement `stop` against the Shutdown RPC** + +`stop()` calls `System/Shutdown`, then polls liveness until Stopped or a bounded timeout elapses. On timeout return an error; **do not** fall back to killing the process. + +- [ ] **Step 5: Run the tests and commit** + +```bash +cargo test -p nuncio-monitor +git add crates/nuncio-monitor/src/engine.rs +git commit -m "feat(monitor): detect engine liveness and control lifecycle" +``` + +### Task 9: `StatusPoller` + +**Files:** +- Create: `crates/nuncio-monitor/src/status.rs` + +**Interfaces:** +- Consumes: `nuncio_proto::client::connect_system`, `connect_accounts`; `System/GetStatus`, `System/GetHealth`, `System/Subscribe`, `Accounts/ListAccounts`; `nuncio_store::vault::GRPC_TOKEN_ACCOUNT` (`"grpc-bearer-token"`). +- Produces: `StatusPoller::spawn(addr, secrets) -> mpsc::Receiver<StatusUpdate>` where `StatusUpdate` carries `Option<GetStatusResponse>`, `Option<GetHealthResponse>`, `Vec<AccountConfig>`, and a `stream_stale: bool`. + +- [ ] **Step 1: Write the failing test** + +```rust +#[tokio::test] +async fn a_missing_token_is_an_honest_error_not_an_empty_status() { + let secrets = nuncio_store::vault::SecretManager::mock(); + let err = StatusPoller::connect("127.0.0.1:9420", &secrets) + .await + .expect_err("must fail without a minted token"); + assert!(matches!(err, StatusError::NoToken)); +} +``` + +- [ ] **Step 2: Run it to verify it fails** + +Run: `cargo test -p nuncio-monitor status` +Expected: FAIL — `StatusPoller` not defined. + +- [ ] **Step 3: Implement token resolution and dialing** + +Read the token from the vault under `GRPC_TOKEN_ACCOUNT`, hex-encode it, and dial via `connect_system`. A missing vault entry returns `StatusError::NoToken`, distinct from a connection failure, because the two need different messages in the UI. + +- [ ] **Step 4: Implement the poll and stream loop** + +Spawn a task running a timed `GetStatus` + `GetHealth` poll alongside a `Subscribe` stream. On stream error, mark `stream_stale` and reconnect with capped exponential backoff. Never let a failed poll emit zeros; emit `None` so the UI renders unknown. + +- [ ] **Step 5: Run the tests and commit** + +```bash +cargo test -p nuncio-monitor +git add crates/nuncio-monitor/src/status.rs +git commit -m "feat(monitor): poll daemon status and follow the event stream" +``` + +### Task 10: `AppState` + +**Files:** +- Create: `crates/nuncio-monitor/src/state.rs` + +**Interfaces:** +- Consumes: `LogRecord` (Task 7), `EngineState` (Task 8), `StatusUpdate` (Task 9). +- Produces: `AppState` with `engine: EngineState`, `status: Option<GetStatusResponse>`, `health: Option<GetHealthResponse>`, `accounts: Vec<AccountConfig>`, `logs: VecDeque<LogRecord>`, `log_filter: LogFilter`, `stream_stale: bool`; and `AppState::visible_logs(&self) -> Vec<&LogRecord>`. + +- [ ] **Step 1: Write the failing filter tests** + +```rust +#[test] +fn filtering_by_request_id_returns_only_that_requests_lines() { + let mut st = AppState::default(); + st.push_log(rec_with_request_id("a")); + st.push_log(rec_with_request_id("b")); + st.log_filter.request_id = Some("a".into()); + assert_eq!(st.visible_logs().len(), 1); +} + +#[test] +fn the_log_buffer_is_bounded_and_drops_oldest_first() { + let mut st = AppState::default(); + for i in 0..(AppState::MAX_LOG_LINES + 10) { + st.push_log(rec_with_message(&i.to_string())); + } + assert_eq!(st.logs.len(), AppState::MAX_LOG_LINES); + assert_eq!(st.logs.front().map(|r| r.message.as_str()), Some("10")); +} +``` + +- [ ] **Step 2: Run them to verify they fail** + +Run: `cargo test -p nuncio-monitor state` +Expected: FAIL — `AppState` not defined. + +- [ ] **Step 3: Implement `AppState`** + +Bound `logs` with a `VecDeque` and a `MAX_LOG_LINES` constant (start at 5000) so a debug-level sync cannot exhaust memory. Implement `visible_logs` applying level, text, and `request_id` filters in that order. + +- [ ] **Step 4: Run the tests and commit** + +```bash +cargo test -p nuncio-monitor +git add crates/nuncio-monitor/src/state.rs +git commit -m "feat(monitor): add bounded, filterable application state" +``` + +### Task 11: UI and tray, wired together + +**Files:** +- Create: `crates/nuncio-monitor/src/ui.rs`, `crates/nuncio-monitor/src/tray.rs` +- Modify: `crates/nuncio-monitor/src/main.rs` + +**Interfaces:** +- Consumes: everything from Tasks 6 to 10. +- Produces: the running application. + +- [ ] **Step 1: Implement the tray** + +Icon plus menu: Show, Start Engine, Stop Engine, Quit. Icon colour encodes `EngineState`. Start and Stop are disabled when not applicable rather than failing when clicked. + +- [ ] **Step 2: Implement the UI panes** + +Status header rendering `EngineState`, version, uptime, readiness, and a stale-stream indicator. Accounts table with one row per account showing sync state, last sync, last error, and `pending`/`failed` **as separate columns**. Log table via `egui_extras::TableBuilder` with a level dropdown, a search box, and a clickable `request_id` cell that sets the filter. + +- [ ] **Step 3: Render unknown distinctly from zero** + +Every numeric cell renders an em dash when its source is `None`. Add a manual check: stop the daemon and confirm no pane shows `0`. + +- [ ] **Step 4: Wire main.rs** + +Start the Tokio runtime on a worker thread, spawn `StatusPoller` and `LogTailer`, run `eframe` on the main thread, and drain channels into `AppState` each frame. + +- [ ] **Step 5: Run the gate** + +```bash +cargo fmt --all -- --check && cargo check-all && cargo test-all +``` + +- [ ] **Step 6: Manual verification** + +Start the daemon, confirm the tray icon goes green, logs stream, accounts populate. Stop it from the tray and confirm a graceful exit (the log shows the shutdown path, not an abrupt end). + +- [ ] **Step 7: Commit** + +```bash +git add crates/nuncio-monitor +git commit -m "feat(monitor): add tray icon and status, accounts, and log panes" +``` + +--- + +## Documentation + +- [ ] Add a Nuncio Monitor section to `docs/RUNNING.md` covering how to launch it, that it needs `NUNCIO_LOG_FORMAT=json` for filtering, and that it attaches to an already-running daemon. +- [ ] Update `docs/ROADMAP.md` and `docs/roadmap/index.html` **in the same commit** if this changes milestone status. #296 moving to closed changes the WS-C counts. -- 2.54.0 From ac017ba3c4baaf763191fbdc47c34de34c55e7ca Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Mon, 10 Aug 2026 13:57:12 -0500 Subject: [PATCH 50/95] test(mail): add a stateful multi-connection IMAP server for offline tests The IMAP client had no mock. wiremock is HTTP-only, so JMAP and the DAV protocols have one and IMAP does not; what stood in for it was a scripted responder rewritten inline in each test, holding one connection and no mailbox state. That shape cannot express the property the sync work needs to assert: two clients observing each other's writes through shared server state. It also cannot reproduce the response forms that decide whether a mutation actually applied -- COPYUID, VANISHED, and a tagged [MODIFIED ...] -- so a client that reports success for a no-op cannot currently be caught by a test. The server binds a real loopback listener, shares mailbox state across every accepted connection, and speaks IMAP4rev1 plus the parts of RFC 7162, RFC 4315, and RFC 6851 the engine exercises. Capability advertisement is selectable, because the sync ladder branches on it and a suite that only ever runs against a QRESYNC server proves nothing about the path Gmail and Exchange take. Behind a `test-support` feature so it stays out of production builds while remaining available to dependent crates' integration suites. Covers: shared state across connections; a move of an already-moved UID returning OK with no COPYUID; UNCHANGEDSINCE reporting MODIFIED in the tagged response; QRESYNC SELECT reporting VANISHED; QRESYNC refused without ENABLE and on a CONDSTORE-only server; VANISHED versus EXPUNGE on delete. --- crates/nuncio-mail/Cargo.toml | 5 + crates/nuncio-mail/src/lib.rs | 6 + crates/nuncio-mail/src/test_server.rs | 1257 +++++++++++++++++++++++++ 3 files changed, 1268 insertions(+) create mode 100644 crates/nuncio-mail/src/test_server.rs diff --git a/crates/nuncio-mail/Cargo.toml b/crates/nuncio-mail/Cargo.toml index 1045a59..72a10cc 100644 --- a/crates/nuncio-mail/Cargo.toml +++ b/crates/nuncio-mail/Cargo.toml @@ -9,6 +9,11 @@ repository.workspace = true [lints] workspace = true +[features] +# Exposes `test_server`, a stateful loopback IMAP server, to dependent crates' +# test suites. Never enabled in a production build. +test-support = [] + [dependencies] nuncio-core = { path = "../nuncio-core" } tokio.workspace = true diff --git a/crates/nuncio-mail/src/lib.rs b/crates/nuncio-mail/src/lib.rs index cc8bfc9..4689ea8 100644 --- a/crates/nuncio-mail/src/lib.rs +++ b/crates/nuncio-mail/src/lib.rs @@ -10,6 +10,12 @@ pub mod mock; pub mod parser; pub mod smtp; +// A real loopback IMAP server for offline tests. Behind a feature so it never +// enters a production build, but visible to this crate's own tests and to +// dependents that opt in (integration suites in `nunciod`). +#[cfg(any(test, feature = "test-support"))] +pub mod test_server; + pub use backend::{ MailBackend, MessageSender, OutboundMessage, RemoteMutationKind, RemoteMutationSpec, }; diff --git a/crates/nuncio-mail/src/test_server.rs b/crates/nuncio-mail/src/test_server.rs new file mode 100644 index 0000000..bb659f3 --- /dev/null +++ b/crates/nuncio-mail/src/test_server.rs @@ -0,0 +1,1257 @@ +//! A stateful, multi-connection IMAP server for offline tests. +//! +//! `wiremock` only speaks HTTP, so JMAP and the DAV protocols have a mock and +//! IMAP does not. What existed instead was a scripted responder re-written +//! inline in each test: one connection, no mailbox state, and a hand-rolled +//! answer per command. That shape cannot express the behaviour this module +//! exists to test -- two clients observing each other's writes through shared +//! server state. +//! +//! The server binds a real loopback `TcpListener` and speaks enough of +//! IMAP4rev1 (plus RFC 7162 CONDSTORE/QRESYNC, RFC 4315 UIDPLUS, and RFC 6851 +//! MOVE) for a client to be driven end to end over plain TCP. Mailbox state is +//! shared across every accepted connection, so a mutation issued on one +//! connection is visible to the next command on another -- which is the only +//! way to reproduce, offline, what happens when several engines sync the same +//! account. +//! +//! Capability advertisement is selectable ([`ServerProfile`]) because the sync +//! ladder branches on it: Fastmail/Cyrus and Dovecot advertise QRESYNC, Gmail +//! advertises CONDSTORE without QRESYNC, and Exchange advertises neither. A +//! test that only ever runs against a QRESYNC server proves nothing about the +//! path most real accounts take. +//! +//! Deliberately not a conformant server. It implements the commands the engine +//! issues, in the response shapes the relevant RFCs require, and answers `BAD` +//! to everything else. + +use std::collections::{BTreeMap, BTreeSet}; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex, MutexGuard}; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +/// Which capabilities the server advertises, and therefore which rung of the +/// sync ladder a client can negotiate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ServerProfile { + /// No CONDSTORE and no QRESYNC: full UID-set reconciliation is the only + /// option. Models Exchange. + Basic, + /// CONDSTORE but no QRESYNC: `CHANGEDSINCE` works, `SELECT (QRESYNC ...)` + /// does not. Models Gmail, which is the common case and the one a + /// QRESYNC-only test suite silently skips. + Condstore, + /// CONDSTORE and QRESYNC. Models Fastmail/Cyrus and Dovecot. + Qresync, +} + +impl ServerProfile { + /// The `CAPABILITY` atom list for this profile. `UIDPLUS` and `MOVE` are + /// advertised by all three because every profile we model supports them. + fn capability_line(self) -> String { + let mut caps = vec!["IMAP4rev1", "UIDPLUS", "MOVE", "ENABLE", "LIST-STATUS"]; + match self { + ServerProfile::Basic => {} + ServerProfile::Condstore => caps.push("CONDSTORE"), + ServerProfile::Qresync => { + caps.push("CONDSTORE"); + caps.push("QRESYNC"); + } + } + caps.join(" ") + } + + fn supports_condstore(self) -> bool { + matches!(self, ServerProfile::Condstore | ServerProfile::Qresync) + } + + fn supports_qresync(self) -> bool { + matches!(self, ServerProfile::Qresync) + } +} + +/// One message as the server holds it. +#[derive(Debug, Clone)] +pub struct MockMessage { + pub uid: u32, + pub modseq: u64, + pub flags: BTreeSet<String>, + /// Full RFC822 octets served for `BODY[]`. + pub body: String, +} + +/// A mailbox: its UID space, its mod-sequence, and the tombstones needed to +/// answer `VANISHED (EARLIER)`. +#[derive(Debug, Clone)] +pub struct MockMailbox { + pub uid_validity: u32, + pub uid_next: u32, + pub highest_modseq: u64, + pub messages: Vec<MockMessage>, + /// `(uid, modseq_at_removal)` for every message removed from this mailbox. + /// QRESYNC needs removal history; a client resyncing from an older modseq + /// must be told what disappeared, which is exactly the information a + /// high-water-mark sync can never recover. + pub vanished: Vec<(u32, u64)>, +} + +impl MockMailbox { + fn new(uid_validity: u32) -> Self { + Self { + uid_validity, + uid_next: 1, + highest_modseq: 1, + messages: Vec::new(), + vanished: Vec::new(), + } + } + + fn next_modseq(&mut self) -> u64 { + self.highest_modseq += 1; + self.highest_modseq + } + + fn append(&mut self, body: &str, flags: &[&str]) -> u32 { + let uid = self.uid_next; + self.uid_next += 1; + let modseq = self.next_modseq(); + self.messages.push(MockMessage { + uid, + modseq, + flags: flags.iter().map(|f| (*f).to_string()).collect(), + body: body.to_string(), + }); + uid + } + + /// Remove `uid`, recording a tombstone at a fresh modseq. + fn remove(&mut self, uid: u32) -> bool { + let Some(pos) = self.messages.iter().position(|m| m.uid == uid) else { + return false; + }; + let modseq = self.next_modseq(); + self.messages.remove(pos); + self.vanished.push((uid, modseq)); + true + } + + /// 1-based sequence number of `uid`, as untagged responses require. + fn seq_of(&self, uid: u32) -> usize { + self.messages + .iter() + .position(|m| m.uid == uid) + .map_or(0, |p| p + 1) + } +} + +/// Server state shared by every connection. +#[derive(Debug)] +pub struct ServerState { + pub profile: ServerProfile, + pub mailboxes: BTreeMap<String, MockMailbox>, + /// Every command line received, across all connections, in arrival order. + /// Tests assert on this to prove *which* protocol path ran -- e.g. that a + /// second sync narrowed its FETCH rather than re-fetching everything. + pub commands: Vec<String>, + next_uid_validity: u32, +} + +impl ServerState { + fn new(profile: ServerProfile) -> Self { + Self { + profile, + mailboxes: BTreeMap::new(), + commands: Vec::new(), + next_uid_validity: 1000, + } + } +} + +/// Handle to a running mock server. Dropping it aborts the accept loop; the +/// bound port is released with it. +#[derive(Debug)] +pub struct MockImapServer { + addr: SocketAddr, + state: Arc<Mutex<ServerState>>, + accept_task: tokio::task::JoinHandle<()>, +} + +impl Drop for MockImapServer { + fn drop(&mut self) { + self.accept_task.abort(); + } +} + +/// Lock the shared state, recovering from a poisoned mutex rather than +/// panicking: a panic in one connection task must not cascade into every other +/// assertion in the test. +fn lock_state(state: &Arc<Mutex<ServerState>>) -> MutexGuard<'_, ServerState> { + match state.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } +} + +impl MockImapServer { + /// Bind an ephemeral loopback port and start accepting connections. + pub async fn start(profile: ServerProfile) -> std::io::Result<Self> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let state = Arc::new(Mutex::new(ServerState::new(profile))); + + let accept_state = Arc::clone(&state); + let accept_task = tokio::spawn(async move { + loop { + let Ok((socket, _)) = listener.accept().await else { + return; + }; + let conn_state = Arc::clone(&accept_state); + tokio::spawn(async move { + // A dropped client mid-command is normal in these tests; + // the connection task simply ends. + let _ = handle_connection(socket, conn_state).await; + }); + } + }); + + Ok(Self { + addr, + state, + accept_task, + }) + } + + pub fn port(&self) -> u16 { + self.addr.port() + } + + pub fn host(&self) -> &'static str { + "127.0.0.1" + } + + /// Create a mailbox with a fresh UIDVALIDITY and return its name. + pub fn create_mailbox(&self, name: &str) { + let mut state = lock_state(&self.state); + let uid_validity = state.next_uid_validity; + state.next_uid_validity += 1; + state + .mailboxes + .insert(name.to_string(), MockMailbox::new(uid_validity)); + } + + /// Append a message, creating the mailbox if absent. Returns its UID. + pub fn append_message(&self, mailbox: &str, body: &str, flags: &[&str]) -> u32 { + let mut state = lock_state(&self.state); + let uid_validity = state.next_uid_validity; + let entry = state.mailboxes.entry(mailbox.to_string()); + let mailbox = entry.or_insert_with(|| { + // Only consumes a UIDVALIDITY when the mailbox is actually new. + MockMailbox::new(uid_validity) + }); + let fresh = mailbox.uid_next == 1 && mailbox.messages.is_empty(); + let uid = mailbox.append(body, flags); + if fresh { + state.next_uid_validity += 1; + } + uid + } + + /// Simulate another client removing a message -- the case a forward-only + /// high-water-mark sync can never observe. + pub fn expunge(&self, mailbox: &str, uid: u32) -> bool { + let mut state = lock_state(&self.state); + state + .mailboxes + .get_mut(mailbox) + .is_some_and(|mb| mb.remove(uid)) + } + + /// Force a UIDVALIDITY change, as a server does when a mailbox is + /// recreated. UIDs restart at 1. + pub fn bump_uid_validity(&self, mailbox: &str) { + let mut state = lock_state(&self.state); + if let Some(mb) = state.mailboxes.get_mut(mailbox) { + mb.uid_validity += 1; + mb.uid_next = 1; + mb.messages.clear(); + mb.vanished.clear(); + } + } + + /// UIDs currently present in `mailbox`. + pub fn uids(&self, mailbox: &str) -> Vec<u32> { + let state = lock_state(&self.state); + state + .mailboxes + .get(mailbox) + .map(|mb| mb.messages.iter().map(|m| m.uid).collect()) + .unwrap_or_default() + } + + /// Flags currently set on `uid`, or `None` if it is not present. + pub fn flags(&self, mailbox: &str, uid: u32) -> Option<Vec<String>> { + let state = lock_state(&self.state); + state.mailboxes.get(mailbox).and_then(|mb| { + mb.messages + .iter() + .find(|m| m.uid == uid) + .map(|m| m.flags.iter().cloned().collect()) + }) + } + + /// Every command line the server has received, in arrival order. + pub fn commands(&self) -> Vec<String> { + lock_state(&self.state).commands.clone() + } + + /// Commands containing `needle` (case-insensitive). + pub fn commands_matching(&self, needle: &str) -> Vec<String> { + let upper = needle.to_ascii_uppercase(); + self.commands() + .into_iter() + .filter(|c| c.to_ascii_uppercase().contains(&upper)) + .collect() + } +} + +/// Per-connection session state. Only `selected` and `qresync_enabled` are +/// connection-scoped; everything else lives in the shared `ServerState`. +struct Session { + selected: Option<String>, + qresync_enabled: bool, +} + +async fn handle_connection( + mut socket: TcpStream, + state: Arc<Mutex<ServerState>>, +) -> std::io::Result<()> { + let greeting = { + let st = lock_state(&state); + format!( + "* OK [CAPABILITY {}] Nuncio mock IMAP ready\r\n", + st.profile.capability_line() + ) + }; + socket.write_all(greeting.as_bytes()).await?; + + let mut session = Session { + selected: None, + qresync_enabled: false, + }; + + while let Some(line) = read_line(&mut socket).await { + if line.trim().is_empty() { + continue; + } + let response = { + let mut st = lock_state(&state); + st.commands.push(line.clone()); + dispatch(&line, &mut session, &mut st) + }; + socket.write_all(response.text.as_bytes()).await?; + if response.close { + break; + } + } + Ok(()) +} + +struct Reply { + text: String, + close: bool, +} + +impl Reply { + fn open(text: String) -> Self { + Self { text, close: false } + } + fn closing(text: String) -> Self { + Self { text, close: true } + } +} + +fn dispatch(line: &str, session: &mut Session, st: &mut ServerState) -> Reply { + let mut parts = line.split_whitespace(); + let tag = parts.next().unwrap_or("*").to_string(); + let command = parts.next().unwrap_or("").to_ascii_uppercase(); + let rest: Vec<String> = parts.map(str::to_string).collect(); + + match command.as_str() { + "CAPABILITY" => Reply::open(format!( + "* CAPABILITY {}\r\n{tag} OK CAPABILITY completed\r\n", + st.profile.capability_line() + )), + "LOGIN" => Reply::open(format!( + "{tag} OK [CAPABILITY {}] Logged in\r\n", + st.profile.capability_line() + )), + "ENABLE" => handle_enable(&tag, &rest, session, st), + "NOOP" => Reply::open(format!("{tag} OK NOOP completed\r\n")), + "LIST" => handle_list(&tag, st), + "STATUS" => handle_status(&tag, &rest, st), + "SELECT" | "EXAMINE" => handle_select(&tag, line, &rest, session, st), + "UID" => handle_uid(&tag, line, &rest, session, st), + "LOGOUT" => Reply::closing(format!( + "* BYE Logging out\r\n{tag} OK LOGOUT completed\r\n" + )), + _ => Reply::open(format!("{tag} BAD unsupported command\r\n")), + } +} + +fn handle_enable(tag: &str, rest: &[String], session: &mut Session, st: &ServerState) -> Reply { + let wants_qresync = rest.iter().any(|a| a.eq_ignore_ascii_case("QRESYNC")); + if wants_qresync && st.profile.supports_qresync() { + session.qresync_enabled = true; + return Reply::open(format!( + "* ENABLED QRESYNC\r\n{tag} OK ENABLE completed\r\n" + )); + } + Reply::open(format!("{tag} OK ENABLE completed\r\n")) +} + +fn handle_list(tag: &str, st: &ServerState) -> Reply { + let mut out = String::new(); + for name in st.mailboxes.keys() { + out.push_str(&format!("* LIST () \"/\" {name}\r\n")); + } + out.push_str(&format!("{tag} OK LIST completed\r\n")); + Reply::open(out) +} + +fn handle_status(tag: &str, rest: &[String], st: &ServerState) -> Reply { + let Some(name) = rest.first().map(|n| n.trim_matches('"').to_string()) else { + return Reply::open(format!("{tag} BAD STATUS needs a mailbox\r\n")); + }; + let Some(mb) = st.mailboxes.get(&name) else { + return Reply::open(format!("{tag} NO no such mailbox\r\n")); + }; + let mut items = format!( + "MESSAGES {} UIDNEXT {} UIDVALIDITY {}", + mb.messages.len(), + mb.uid_next, + mb.uid_validity + ); + if st.profile.supports_condstore() { + items.push_str(&format!(" HIGHESTMODSEQ {}", mb.highest_modseq)); + } + Reply::open(format!( + "* STATUS {name} ({items})\r\n{tag} OK STATUS completed\r\n" + )) +} + +fn handle_select( + tag: &str, + line: &str, + rest: &[String], + session: &mut Session, + st: &ServerState, +) -> Reply { + let Some(name) = rest.first().map(|n| n.trim_matches('"').to_string()) else { + return Reply::open(format!("{tag} BAD SELECT needs a mailbox\r\n")); + }; + let Some(mb) = st.mailboxes.get(&name) else { + return Reply::open(format!("{tag} NO no such mailbox\r\n")); + }; + + let upper = line.to_ascii_uppercase(); + let asked_qresync = upper.contains("QRESYNC"); + // RFC 7162 3.2.3: a server MUST reply BAD to a QRESYNC SELECT parameter + // when the client has not issued `ENABLE QRESYNC` on this connection. + // Enforced so a client that skips ENABLE fails here rather than silently + // degrading in production. + if asked_qresync && !session.qresync_enabled { + return Reply::open(format!("{tag} BAD QRESYNC not enabled\r\n")); + } + + session.selected = Some(name.clone()); + + let mut out = String::new(); + out.push_str(&format!("* {} EXISTS\r\n", mb.messages.len())); + out.push_str("* 0 RECENT\r\n"); + out.push_str("* FLAGS (\\Seen \\Flagged \\Deleted \\Draft)\r\n"); + out.push_str("* OK [PERMANENTFLAGS (\\Seen \\Flagged \\Deleted \\Draft \\*)] Limited\r\n"); + out.push_str(&format!( + "* OK [UIDVALIDITY {}] UIDs valid\r\n", + mb.uid_validity + )); + out.push_str(&format!( + "* OK [UIDNEXT {}] Predicted next\r\n", + mb.uid_next + )); + if st.profile.supports_condstore() { + out.push_str(&format!( + "* OK [HIGHESTMODSEQ {}] Highest\r\n", + mb.highest_modseq + )); + } + + if asked_qresync { + if let Some(since) = parse_qresync_modseq(line) { + let gone: Vec<String> = mb + .vanished + .iter() + .filter(|(_, m)| *m > since) + .map(|(u, _)| u.to_string()) + .collect(); + if !gone.is_empty() { + out.push_str(&format!("* VANISHED (EARLIER) {}\r\n", gone.join(","))); + } + for msg in mb.messages.iter().filter(|m| m.modseq > since) { + out.push_str(&fetch_line(mb, msg, false)); + } + } + } + + out.push_str(&format!("{tag} OK [READ-WRITE] SELECT completed\r\n")); + Reply::open(out) +} + +/// Pull the mod-sequence out of `SELECT x (QRESYNC (uidvalidity modseq ...))`. +fn parse_qresync_modseq(line: &str) -> Option<u64> { + let upper = line.to_ascii_uppercase(); + let at = upper.find("QRESYNC")?; + let tail = &line[at + "QRESYNC".len()..]; + let nums: Vec<u64> = tail + .split(|c: char| !c.is_ascii_digit()) + .filter(|s| !s.is_empty()) + .filter_map(|s| s.parse().ok()) + .collect(); + // (uidvalidity modseq [known-uids...]) -- the second number is the modseq. + nums.get(1).copied() +} + +fn handle_uid( + tag: &str, + line: &str, + rest: &[String], + session: &Session, + st: &mut ServerState, +) -> Reply { + let sub = rest + .first() + .map(|s| s.to_ascii_uppercase()) + .unwrap_or_default(); + let Some(selected) = session.selected.clone() else { + return Reply::open(format!("{tag} BAD no mailbox selected\r\n")); + }; + match sub.as_str() { + "FETCH" => handle_uid_fetch(tag, line, rest, &selected, st), + "STORE" => handle_uid_store(tag, line, rest, &selected, st), + "COPY" => handle_uid_copy(tag, rest, &selected, st, false), + "MOVE" => handle_uid_copy(tag, rest, &selected, st, true), + "EXPUNGE" => handle_uid_expunge(tag, rest, &selected, session, st), + _ => Reply::open(format!("{tag} BAD unsupported UID subcommand\r\n")), + } +} + +/// Expand an IMAP `uid-set` (`1,3:5,10:*`) against the UIDs present. +fn expand_uid_set(spec: &str, present: &[u32], uid_next: u32) -> Vec<u32> { + let mut out = Vec::new(); + for part in spec.split(',') { + if let Some((lo, hi)) = part.split_once(':') { + let lo: u32 = lo.trim().parse().unwrap_or(0); + let hi: u32 = if hi.trim() == "*" { + uid_next.saturating_sub(1).max(lo) + } else { + hi.trim().parse().unwrap_or(0) + }; + let (lo, hi) = if lo <= hi { (lo, hi) } else { (hi, lo) }; + out.extend(present.iter().copied().filter(|u| *u >= lo && *u <= hi)); + } else if let Ok(single) = part.trim().parse::<u32>() { + // RFC 3501 6.4.8: a non-existent UID is silently ignored, never an + // error. Filtering against `present` reproduces that exactly -- + // which is the behaviour that makes an unverified mutation look + // like a success. + out.extend(present.iter().copied().filter(|u| *u == single)); + } + } + out.sort_unstable(); + out.dedup(); + out +} + +fn format_flags(flags: &BTreeSet<String>) -> String { + flags.iter().cloned().collect::<Vec<_>>().join(" ") +} + +fn fetch_line(mb: &MockMailbox, msg: &MockMessage, include_body: bool) -> String { + let seq = mb.seq_of(msg.uid); + let mut items = format!( + "UID {} FLAGS ({}) MODSEQ ({})", + msg.uid, + format_flags(&msg.flags), + msg.modseq + ); + if include_body { + items.push_str(&format!(" BODY[] {{{}}}\r\n{}", msg.body.len(), msg.body)); + } + format!("* {seq} FETCH ({items})\r\n") +} + +fn handle_uid_fetch( + tag: &str, + line: &str, + rest: &[String], + selected: &str, + st: &mut ServerState, +) -> Reply { + let condstore = st.profile.supports_condstore(); + let Some(mb) = st.mailboxes.get(selected) else { + return Reply::open(format!("{tag} NO mailbox vanished\r\n")); + }; + let Some(spec) = rest.get(1) else { + return Reply::open(format!("{tag} BAD UID FETCH needs a set\r\n")); + }; + + let upper = line.to_ascii_uppercase(); + let include_body = upper.contains("BODY[]") || upper.contains("RFC822"); + let changed_since = if condstore { + parse_changed_since(&upper) + } else { + None + }; + + let present: Vec<u32> = mb.messages.iter().map(|m| m.uid).collect(); + let wanted = expand_uid_set(spec, &present, mb.uid_next); + + let mut out = String::new(); + for msg in mb.messages.iter().filter(|m| wanted.contains(&m.uid)) { + if let Some(since) = changed_since { + if msg.modseq <= since { + continue; + } + } + out.push_str(&fetch_line(mb, msg, include_body)); + } + + // RFC 7162 3.2.6: with CHANGEDSINCE ... VANISHED the server reports + // removals in the same pass. This is the whole point of the mechanism -- + // it is the only way a client learns a message is gone. + if changed_since.is_some() && upper.contains("VANISHED") { + if let Some(since) = changed_since { + let gone: Vec<String> = mb + .vanished + .iter() + .filter(|(_, m)| *m > since) + .map(|(u, _)| u.to_string()) + .collect(); + if !gone.is_empty() { + out.push_str(&format!("* VANISHED (EARLIER) {}\r\n", gone.join(","))); + } + } + } + + out.push_str(&format!("{tag} OK UID FETCH completed\r\n")); + Reply::open(out) +} + +fn parse_changed_since(upper: &str) -> Option<u64> { + let at = upper.find("CHANGEDSINCE")?; + upper[at + "CHANGEDSINCE".len()..] + .split(|c: char| !c.is_ascii_digit()) + .find(|s| !s.is_empty()) + .and_then(|s| s.parse().ok()) +} + +fn parse_unchanged_since(upper: &str) -> Option<u64> { + let at = upper.find("UNCHANGEDSINCE")?; + upper[at + "UNCHANGEDSINCE".len()..] + .split(|c: char| !c.is_ascii_digit()) + .find(|s| !s.is_empty()) + .and_then(|s| s.parse().ok()) +} + +fn handle_uid_store( + tag: &str, + line: &str, + rest: &[String], + selected: &str, + st: &mut ServerState, +) -> Reply { + let condstore = st.profile.supports_condstore(); + let Some(mb) = st.mailboxes.get_mut(selected) else { + return Reply::open(format!("{tag} NO mailbox vanished\r\n")); + }; + let Some(spec) = rest.get(1) else { + return Reply::open(format!("{tag} BAD UID STORE needs a set\r\n")); + }; + + let upper = line.to_ascii_uppercase(); + let silent = upper.contains(".SILENT"); + let removing = line.contains("-FLAGS"); + let unchanged_since = if condstore { + parse_unchanged_since(&upper) + } else { + None + }; + + let flags = parse_flag_list(line); + let present: Vec<u32> = mb.messages.iter().map(|m| m.uid).collect(); + let targets = expand_uid_set(spec, &present, mb.uid_next); + + let mut modified: Vec<u32> = Vec::new(); + let mut touched: Vec<u32> = Vec::new(); + for uid in targets { + let Some(idx) = mb.messages.iter().position(|m| m.uid == uid) else { + continue; + }; + // RFC 7162 3.1.3: a message changed since the client's mod-sequence is + // reported in MODIFIED and left untouched -- the compare-and-swap that + // makes a lost update detectable instead of silent. + if let Some(since) = unchanged_since { + if mb.messages[idx].modseq > since { + modified.push(uid); + continue; + } + } + let next = mb.next_modseq(); + if let Some(msg) = mb.messages.get_mut(idx) { + for flag in &flags { + if removing { + msg.flags.remove(flag); + } else { + msg.flags.insert(flag.clone()); + } + } + msg.modseq = next; + } + touched.push(uid); + } + + let mut out = String::new(); + if !silent { + for uid in &touched { + if let Some(msg) = mb.messages.iter().find(|m| m.uid == *uid) { + out.push_str(&fetch_line(mb, msg, false)); + } + } + } + // RFC 7162 3.1.3 puts MODIFIED in the *tagged* response -- the response + // whose code async-imap currently discards. + if modified.is_empty() { + out.push_str(&format!("{tag} OK UID STORE completed\r\n")); + } else { + let list: Vec<String> = modified.iter().map(u32::to_string).collect(); + out.push_str(&format!( + "{tag} OK [MODIFIED {}] Conditional STORE failed\r\n", + list.join(",") + )); + } + Reply::open(out) +} + +/// Extract flag atoms from the parenthesised argument of a STORE. +fn parse_flag_list(line: &str) -> Vec<String> { + let Some(open) = line.find('(') else { + return Vec::new(); + }; + let Some(close) = line[open..].find(')') else { + return Vec::new(); + }; + line[open + 1..open + close] + .split_whitespace() + .map(str::to_string) + .collect() +} + +fn handle_uid_copy( + tag: &str, + rest: &[String], + selected: &str, + st: &mut ServerState, + is_move: bool, +) -> Reply { + let qresync = st.profile.supports_qresync(); + let (Some(spec), Some(dest_raw)) = (rest.get(1), rest.get(2)) else { + return Reply::open(format!("{tag} BAD UID COPY needs a set and a mailbox\r\n")); + }; + let dest = dest_raw.trim_matches('"').to_string(); + if !st.mailboxes.contains_key(&dest) { + return Reply::open(format!("{tag} NO [TRYCREATE] no such destination\r\n")); + } + + let (source_uids, source_validity, bodies) = { + let Some(mb) = st.mailboxes.get(selected) else { + return Reply::open(format!("{tag} NO mailbox vanished\r\n")); + }; + let present: Vec<u32> = mb.messages.iter().map(|m| m.uid).collect(); + let uids = expand_uid_set(spec, &present, mb.uid_next); + let bodies: Vec<(u32, String, BTreeSet<String>)> = mb + .messages + .iter() + .filter(|m| uids.contains(&m.uid)) + .map(|m| (m.uid, m.body.clone(), m.flags.clone())) + .collect(); + (uids, mb.uid_validity, bodies) + }; + + // RFC 3501 6.4.8 again: a UID set that matches nothing is a successful + // no-op. No COPYUID is emitted, and the client cannot distinguish this + // from a copy it did not observe -- which is precisely why an absent + // COPYUID must mean "unknown", never "applied" and never "conflict". + if source_uids.is_empty() { + return Reply::open(format!("{tag} OK UID {} completed\r\n", verb(is_move))); + } + + let mut dest_uids = Vec::new(); + let dest_validity = { + let Some(dst) = st.mailboxes.get_mut(&dest) else { + return Reply::open(format!("{tag} NO destination vanished\r\n")); + }; + for (_, body, flags) in &bodies { + let refs: Vec<&str> = flags.iter().map(String::as_str).collect(); + dest_uids.push(dst.append(body, &refs)); + } + dst.uid_validity + }; + + let mut out = String::new(); + // RFC 6851 4.3 advises COPYUID in an *untagged* OK for MOVE, and RFC 4315 + // 3 defines the code. A client reading only the tagged OK misses it. + out.push_str(&format!( + "* OK [COPYUID {} {} {}] Copied\r\n", + dest_validity, + join_uids(&source_uids), + join_uids(&dest_uids) + )); + + if is_move { + let Some(src) = st.mailboxes.get_mut(selected) else { + return Reply::open(format!("{tag} NO mailbox vanished\r\n")); + }; + let mut seqs = Vec::new(); + for uid in &source_uids { + seqs.push(src.seq_of(*uid)); + src.remove(*uid); + } + if qresync { + // RFC 6851 4.4: under QRESYNC the server reports VANISHED rather + // than EXPUNGE. A client proving its move by counting EXPUNGE + // responses sees zero and wrongly concludes it failed. + out.push_str(&format!("* VANISHED {}\r\n", join_uids(&source_uids))); + } else { + // Descending, so earlier removals do not renumber later ones. + seqs.sort_unstable(); + for seq in seqs.iter().rev() { + out.push_str(&format!("* {seq} EXPUNGE\r\n")); + } + } + let _ = source_validity; + } + + out.push_str(&format!("{tag} OK UID {} completed\r\n", verb(is_move))); + Reply::open(out) +} + +fn verb(is_move: bool) -> &'static str { + if is_move { + "MOVE" + } else { + "COPY" + } +} + +fn join_uids(uids: &[u32]) -> String { + uids.iter() + .map(|u| u.to_string()) + .collect::<Vec<_>>() + .join(",") +} + +fn handle_uid_expunge( + tag: &str, + rest: &[String], + selected: &str, + session: &Session, + st: &mut ServerState, +) -> Reply { + let qresync = st.profile.supports_qresync(); + let Some(mb) = st.mailboxes.get_mut(selected) else { + return Reply::open(format!("{tag} NO mailbox vanished\r\n")); + }; + let Some(spec) = rest.get(1) else { + return Reply::open(format!("{tag} BAD UID EXPUNGE needs a set\r\n")); + }; + + let present: Vec<u32> = mb.messages.iter().map(|m| m.uid).collect(); + // UID EXPUNGE removes only \Deleted messages within the given set. + let targets: Vec<u32> = expand_uid_set(spec, &present, mb.uid_next) + .into_iter() + .filter(|uid| { + mb.messages + .iter() + .any(|m| m.uid == *uid && m.flags.iter().any(|f| f == "\\Deleted")) + }) + .collect(); + + let mut out = String::new(); + let mut seqs = Vec::new(); + for uid in &targets { + seqs.push(mb.seq_of(*uid)); + mb.remove(*uid); + } + + if qresync && session.qresync_enabled { + if !targets.is_empty() { + out.push_str(&format!("* VANISHED {}\r\n", join_uids(&targets))); + } + } else { + seqs.sort_unstable(); + for seq in seqs.iter().rev() { + out.push_str(&format!("* {seq} EXPUNGE\r\n")); + } + } + out.push_str(&format!("{tag} OK UID EXPUNGE completed\r\n")); + Reply::open(out) +} + +/// Read one CRLF-terminated command line, returning `None` at EOF. +async fn read_line(socket: &mut TcpStream) -> Option<String> { + let mut buf = Vec::new(); + let mut byte = [0u8; 1]; + loop { + match socket.read(&mut byte).await { + Ok(0) => { + return if buf.is_empty() { + None + } else { + Some(String::from_utf8_lossy(&buf).into_owned()) + } + } + Ok(_) => { + if byte[0] == b'\n' { + return Some(String::from_utf8_lossy(&buf).into_owned()); + } + if byte[0] != b'\r' { + buf.push(byte[0]); + } + } + Err(_) => return None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::BufReader; + + /// A raw line-oriented IMAP client. The harness is tested at the wire + /// level on purpose: `async-imap` cannot express ENABLE, a QRESYNC SELECT, + /// or read a tagged response code, so an engine-driven test could not + /// observe most of what this server exists to provide. + struct RawClient { + reader: BufReader<tokio::net::tcp::OwnedReadHalf>, + writer: tokio::net::tcp::OwnedWriteHalf, + counter: usize, + } + + impl RawClient { + async fn connect(server: &MockImapServer) -> Self { + let stream = TcpStream::connect((server.host(), server.port())) + .await + .expect("connect to mock server"); + let (r, w) = stream.into_split(); + let mut client = Self { + reader: BufReader::new(r), + writer: w, + counter: 0, + }; + let greeting = client.read_line().await; + assert!(greeting.starts_with("* OK"), "greeting was {greeting:?}"); + client + } + + async fn read_line(&mut self) -> String { + use tokio::io::AsyncBufReadExt; + let mut line = String::new(); + let _ = self.reader.read_line(&mut line).await; + line.trim_end().to_string() + } + + /// Send a command and collect every response line up to the tagged one. + async fn send(&mut self, command: &str) -> Vec<String> { + self.counter += 1; + let tag = format!("a{}", self.counter); + let _ = self + .writer + .write_all(format!("{tag} {command}\r\n").as_bytes()) + .await; + let mut lines = Vec::new(); + loop { + let line = self.read_line().await; + if line.is_empty() { + break; + } + let done = line.starts_with(&format!("{tag} ")); + lines.push(line); + if done { + break; + } + } + lines + } + + async fn login_select(&mut self, mailbox: &str) -> Vec<String> { + let _ = self.send("LOGIN user pass").await; + self.send(&format!("SELECT {mailbox}")).await + } + } + + fn joined(lines: &[String]) -> String { + lines.join("\n") + } + + #[tokio::test] + async fn two_connections_observe_each_others_writes_through_shared_state() { + // The property the old inline duplex responders could not express, and + // the reason this harness exists: engine B must see what engine A did. + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.create_mailbox("Archive"); + let uid = server.append_message("INBOX", "Subject: one\r\n\r\nbody", &[]); + + let mut engine_a = RawClient::connect(&server).await; + let mut engine_b = RawClient::connect(&server).await; + + engine_a.login_select("INBOX").await; + engine_b.login_select("INBOX").await; + + // B sees the message before A moves it. + let before = engine_b.send("UID FETCH 1:* (FLAGS)").await; + assert!( + joined(&before).contains(&format!("UID {uid}")), + "B should see the message first: {before:?}" + ); + + let moved = engine_a.send(&format!("UID MOVE {uid} Archive")).await; + assert!( + joined(&moved).contains("COPYUID"), + "a real move must emit COPYUID: {moved:?}" + ); + + // The same command on B now returns nothing -- shared state, not a + // per-connection script. + let after = engine_b.send("UID FETCH 1:* (FLAGS)").await; + assert!( + !joined(&after).contains(&format!("UID {uid} ")), + "B must not still see the moved message: {after:?}" + ); + assert_eq!(server.uids("INBOX"), Vec::<u32>::new()); + assert_eq!(server.uids("Archive").len(), 1); + } + + #[tokio::test] + async fn a_move_of_an_already_moved_uid_succeeds_silently_without_copyuid() { + // RFC 3501 6.4.8. This is the exact race that makes an unverified + // mutation report success: the losing engine's MOVE returns OK and + // does nothing. The harness must reproduce it, or no test can catch it. + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.create_mailbox("Archive"); + server.create_mailbox("Trash"); + let uid = server.append_message("INBOX", "Subject: contested\r\n\r\nx", &[]); + + let mut engine_a = RawClient::connect(&server).await; + let mut engine_b = RawClient::connect(&server).await; + engine_a.login_select("INBOX").await; + engine_b.login_select("INBOX").await; + + let winner = engine_a.send(&format!("UID MOVE {uid} Archive")).await; + assert!(joined(&winner).contains("COPYUID")); + + let loser = engine_b.send(&format!("UID MOVE {uid} Trash")).await; + let loser_text = joined(&loser); + assert!( + loser_text.contains("OK UID MOVE completed"), + "the losing move must report OK: {loser:?}" + ); + assert!( + !loser_text.contains("COPYUID"), + "a no-op move must NOT emit COPYUID: {loser:?}" + ); + assert!(server.uids("Trash").is_empty(), "nothing moved to Trash"); + } + + #[tokio::test] + async fn unchangedsince_reports_modified_in_the_tagged_response() { + // RFC 7162 3.1.3. MODIFIED rides on the tagged OK -- the response whose + // code async-imap discards, which is why chunk 3 needs a raw layer. + let server = MockImapServer::start(ServerProfile::Condstore) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + let uid = server.append_message("INBOX", "Subject: flags\r\n\r\nx", &[]); + + let mut engine_a = RawClient::connect(&server).await; + let mut engine_b = RawClient::connect(&server).await; + engine_a.login_select("INBOX").await; + engine_b.login_select("INBOX").await; + + // B captures the current modseq, then A changes the message. + let observed = engine_b.send(&format!("UID FETCH {uid} (FLAGS)")).await; + let modseq = observed + .iter() + .find_map(|l| { + let at = l.find("MODSEQ (")?; + l[at + "MODSEQ (".len()..] + .split(')') + .next()? + .parse::<u64>() + .ok() + }) + .expect("a MODSEQ is reported"); + + let _ = engine_a + .send(&format!("UID STORE {uid} +FLAGS (\\Seen)")) + .await; + + let conflicted = engine_b + .send(&format!( + "UID STORE {uid} (UNCHANGEDSINCE {modseq}) +FLAGS (\\Flagged)" + )) + .await; + assert!( + joined(&conflicted).contains(&format!("[MODIFIED {uid}]")), + "B's conditional store must be refused: {conflicted:?}" + ); + let flags = server.flags("INBOX", uid).unwrap_or_default(); + assert!( + !flags.iter().any(|f| f == "\\Flagged"), + "the refused store must not have applied: {flags:?}" + ); + } + + #[tokio::test] + async fn qresync_select_reports_vanished_for_messages_removed_since_the_modseq() { + // The mechanism that fixes ghost messages. A forward-only high-water + // mark cannot learn this; VANISHED states it directly. + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + let keep = server.append_message("INBOX", "Subject: keep\r\n\r\nx", &[]); + let drop = server.append_message("INBOX", "Subject: drop\r\n\r\nx", &[]); + + let mut client = RawClient::connect(&server).await; + let _ = client.send("LOGIN user pass").await; + let _ = client.send("ENABLE QRESYNC").await; + let selected = client.send("SELECT INBOX").await; + let modseq = selected + .iter() + .find_map(|l| { + let at = l.find("HIGHESTMODSEQ ")?; + l[at + "HIGHESTMODSEQ ".len()..] + .split(']') + .next()? + .trim() + .parse::<u64>() + .ok() + }) + .expect("HIGHESTMODSEQ is advertised"); + + // Another client removes a message while we are away. + assert!(server.expunge("INBOX", drop)); + + let resync = client + .send(&format!("SELECT INBOX (QRESYNC (1000 {modseq}))")) + .await; + let text = joined(&resync); + assert!( + text.contains("VANISHED (EARLIER)") && text.contains(&drop.to_string()), + "the removal must be reported: {resync:?}" + ); + assert_eq!(server.uids("INBOX"), vec![keep]); + } + + #[tokio::test] + async fn qresync_select_is_rejected_without_enable_and_on_a_condstore_only_server() { + // RFC 7162 3.2.3 requires BAD when QRESYNC was not enabled. Modelling + // Gmail (CONDSTORE, no QRESYNC) keeps the ladder's rung 2 honest. + let qresync = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + qresync.create_mailbox("INBOX"); + let mut client = RawClient::connect(&qresync).await; + let _ = client.send("LOGIN user pass").await; + let no_enable = client.send("SELECT INBOX (QRESYNC (1000 1))").await; + assert!( + joined(&no_enable).contains("BAD"), + "QRESYNC without ENABLE must be BAD: {no_enable:?}" + ); + + let gmail = MockImapServer::start(ServerProfile::Condstore) + .await + .expect("server starts"); + gmail.create_mailbox("INBOX"); + let mut gclient = RawClient::connect(&gmail).await; + let caps = gclient.send("CAPABILITY").await; + let caps_text = joined(&caps); + assert!(caps_text.contains("CONDSTORE"), "{caps:?}"); + assert!( + !caps_text.contains("QRESYNC"), + "the Gmail profile must not advertise QRESYNC: {caps:?}" + ); + let _ = gclient.send("LOGIN user pass").await; + let enabled = gclient.send("ENABLE QRESYNC").await; + assert!( + !joined(&enabled).contains("ENABLED QRESYNC"), + "a CONDSTORE-only server must not enable QRESYNC: {enabled:?}" + ); + } + + #[tokio::test] + async fn expunge_reports_vanished_under_qresync_and_expunge_otherwise() { + // RFC 6851 4.4. A client proving a delete by counting EXPUNGE lines + // sees none under QRESYNC and would wrongly infer failure. + for (profile, expect_vanished) in [ + (ServerProfile::Qresync, true), + (ServerProfile::Condstore, false), + ] { + let server = MockImapServer::start(profile).await.expect("server starts"); + server.create_mailbox("INBOX"); + let uid = server.append_message("INBOX", "Subject: gone\r\n\r\nx", &["\\Deleted"]); + + let mut client = RawClient::connect(&server).await; + let _ = client.send("LOGIN user pass").await; + if expect_vanished { + let _ = client.send("ENABLE QRESYNC").await; + } + let _ = client.send("SELECT INBOX").await; + let out = client.send(&format!("UID EXPUNGE {uid}")).await; + let text = joined(&out); + + if expect_vanished { + assert!(text.contains("* VANISHED"), "expected VANISHED: {out:?}"); + assert!(!text.contains("EXPUNGE\r"), "{out:?}"); + } else { + assert!(text.contains("EXPUNGE"), "expected EXPUNGE: {out:?}"); + assert!(!text.contains("VANISHED"), "{out:?}"); + } + assert!(server.uids("INBOX").is_empty()); + } + } + + #[tokio::test] + async fn the_command_log_records_every_connections_commands_in_order() { + let server = MockImapServer::start(ServerProfile::Basic) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + let mut a = RawClient::connect(&server).await; + let mut b = RawClient::connect(&server).await; + a.login_select("INBOX").await; + let _ = b.send("LOGIN user pass").await; + let _ = b.send("CAPABILITY").await; + + let fetches = server.commands_matching("SELECT"); + assert_eq!(fetches.len(), 1, "{fetches:?}"); + assert!(server.commands().len() >= 4, "{:?}", server.commands()); + } +} -- 2.54.0 From 83636efcdb4255666ce96c8c43c3c1ee6ead578e Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Mon, 10 Aug 2026 14:14:41 -0500 Subject: [PATCH 51/95] fix(filter): stop the webhook log-capture tests failing at random The two webhook telemetry tests installed a hand-rolled tracing::Subscriber with set_default and asserted on what it recorded. Roughly three runs in ten they captured nothing at all and failed. Callsite Interest is cached process-globally. A raw Subscriber installed thread-locally does not participate in the way tracing_subscriber's registry handles that cache, so whichever sibling test reached the dispatch callsites first decided whether these two could ever see them -- an order dependency, and therefore a coin flip under a parallel test harness. Two narrower fixes were tried and measured, and both made it worse: rebuild_interest_cache() after set_default failed five runs in six, because the rebuild is itself global and concurrent capture tests clobber each other; returning Interest::sometimes() from register_callsite failed nine in ten, because callsite registration consults the global dispatcher and never asks a thread-local subscriber at all. Adopt the registry-backed capture helper nuncio-cal and nuncio-contacts already use, which does not exhibit the problem. The two tests become plain #[test] driving a current-thread runtime built inside the capture scope, since a #[tokio::test] creates its runtime outside it. Assertions now match on level and named fields rather than substrings of a formatted line, so "the host was logged" no longer passes because the host happened to appear in some other field. Measured: 0 failures in 12 stress runs, against 3 in 10 before. --- crates/nuncio-filter/Cargo.toml | 1 + crates/nuncio-filter/src/lib.rs | 3 + crates/nuncio-filter/src/test_tracing.rs | 166 +++++++++++++++++++++++ crates/nuncio-filter/src/webhook.rs | 161 ++++++++-------------- 4 files changed, 229 insertions(+), 102 deletions(-) create mode 100644 crates/nuncio-filter/src/test_tracing.rs diff --git a/crates/nuncio-filter/Cargo.toml b/crates/nuncio-filter/Cargo.toml index 3eb1acb..341443a 100644 --- a/crates/nuncio-filter/Cargo.toml +++ b/crates/nuncio-filter/Cargo.toml @@ -31,3 +31,4 @@ aes-gcm.workspace = true [dev-dependencies] tempfile.workspace = true wiremock.workspace = true +tracing-subscriber.workspace = true diff --git a/crates/nuncio-filter/src/lib.rs b/crates/nuncio-filter/src/lib.rs index 7056052..2314110 100644 --- a/crates/nuncio-filter/src/lib.rs +++ b/crates/nuncio-filter/src/lib.rs @@ -11,6 +11,9 @@ pub mod parser; pub mod validator; pub mod webhook; +#[cfg(test)] +mod test_tracing; + pub use ast::*; pub use codegen::*; pub use engine::*; diff --git a/crates/nuncio-filter/src/test_tracing.rs b/crates/nuncio-filter/src/test_tracing.rs new file mode 100644 index 0000000..03729b2 --- /dev/null +++ b/crates/nuncio-filter/src/test_tracing.rs @@ -0,0 +1,166 @@ +//! Shared in-test `tracing` capture helper. +//! +//! Installs a thread-local subscriber that records every span (with its +//! fields) and every event (with its level, fields, and message) into an +//! in-memory buffer, so tests can assert on what the code logged -- including +//! the negative assertion that a secret never appears in any recorded field. +//! Deterministic and fully offline: no global subscriber, no I/O. +//! +//! This mirrors the identical helper in `nuncio-cal` and `nuncio-contacts`. +//! The webhook tests previously used a hand-rolled `tracing::Subscriber` +//! installed with `set_default`; that captured nothing whenever a sibling test +//! reached the same callsites first, because callsite `Interest` is cached +//! process-globally and a raw subscriber does not participate in the +//! `tracing_subscriber` registry's interest handling. The result was an +//! order-dependent empty capture that failed roughly three runs in ten. + +#![cfg(test)] + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use tracing::field::{Field, Visit}; +use tracing::span::Attributes; +use tracing::{Event, Id, Level, Subscriber}; +use tracing_subscriber::layer::Context as LayerContext; +use tracing_subscriber::prelude::*; +use tracing_subscriber::registry::LookupSpan; + +/// A captured span: its name and the string-rendered fields it opened with. +#[derive(Debug, Clone)] +pub(crate) struct RecordedSpan { + #[allow(dead_code)] + pub name: String, + pub fields: HashMap<String, String>, +} + +/// A captured event: its level and its string-rendered fields (including the +/// `message`). +#[derive(Debug, Clone)] +pub(crate) struct RecordedEvent { + pub level: Level, + pub fields: HashMap<String, String>, +} + +impl RecordedEvent { + /// The event's `message` field, or the empty string if it had none. + pub fn message(&self) -> &str { + self.fields.get("message").map(String::as_str).unwrap_or("") + } + + /// The string-rendered value of `name`, if the event carried that field. + pub fn field(&self, name: &str) -> Option<&str> { + self.fields.get(name).map(String::as_str) + } +} + +/// In-memory sink of everything the subscriber captured during a test. +#[derive(Debug, Default)] +pub(crate) struct Recorder { + spans: Mutex<Vec<RecordedSpan>>, + events: Mutex<Vec<RecordedEvent>>, +} + +impl Recorder { + /// A snapshot of every captured span. + #[allow(dead_code)] + pub fn spans(&self) -> Vec<RecordedSpan> { + self.spans.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + /// A snapshot of every captured event. + pub fn events(&self) -> Vec<RecordedEvent> { + self.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + /// Every field value recorded across all spans and events, so a test can + /// assert that a secret never appears anywhere in the telemetry. + pub fn all_field_values(&self) -> Vec<String> { + let mut values = Vec::new(); + for span in self.spans().iter() { + values.extend(span.fields.values().cloned()); + } + for event in self.events().iter() { + values.extend(event.fields.values().cloned()); + } + values + } +} + +struct FieldVisitor(HashMap<String, String>); + +impl Visit for FieldVisitor { + fn record_str(&mut self, field: &Field, value: &str) { + self.0.insert(field.name().to_string(), value.to_string()); + } + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0 + .entry(field.name().to_string()) + .or_insert_with(|| format!("{value:?}")); + } +} + +struct CaptureLayer(Arc<Recorder>); + +impl<S> tracing_subscriber::Layer<S> for CaptureLayer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_new_span(&self, attrs: &Attributes<'_>, _id: &Id, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(HashMap::new()); + attrs.record(&mut visitor); + self.0 + .spans + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedSpan { + name: attrs.metadata().name().to_string(), + fields: visitor.0, + }); + } + + fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(HashMap::new()); + event.record(&mut visitor); + self.0 + .events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedEvent { + level: *event.metadata().level(), + fields: visitor.0, + }); + } +} + +/// Runs `f` with a thread-local capturing subscriber installed and returns the +/// recorder alongside `f`'s result. Because the subscriber is thread-local, +/// any async work driven inside `f` must run on the calling thread (e.g. a +/// current-thread runtime's `block_on`) for its telemetry to be captured -- +/// see [`block_on_captured`]. +pub(crate) fn with_recorder<T>(f: impl FnOnce() -> T) -> (Arc<Recorder>, T) { + let recorder = Arc::new(Recorder::default()); + let subscriber = tracing_subscriber::registry().with(CaptureLayer(recorder.clone())); + let value = tracing::subscriber::with_default(subscriber, f); + (recorder, value) +} + +/// Drives `future` to completion on a current-thread runtime built inside the +/// capture scope, so every event it emits lands on the thread holding the +/// subscriber. This is the async counterpart to [`with_recorder`]; a +/// `#[tokio::test]` cannot be used here because its runtime is created outside +/// the capture scope. +pub(crate) fn block_on_captured<F: std::future::Future>(future: F) -> (Arc<Recorder>, F::Output) { + with_recorder(|| { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build(); + match runtime { + Ok(rt) => rt.block_on(future), + Err(e) => panic!("building a current-thread runtime must succeed: {e}"), + } + }) +} diff --git a/crates/nuncio-filter/src/webhook.rs b/crates/nuncio-filter/src/webhook.rs index 123f82e..3cf543f 100644 --- a/crates/nuncio-filter/src/webhook.rs +++ b/crates/nuncio-filter/src/webhook.rs @@ -240,54 +240,11 @@ impl WebhookDispatcher { #[cfg(test)] mod tests { use super::*; - use std::sync::Mutex; - use tracing::field::{Field, Visit}; - use tracing::span; + use crate::test_tracing::block_on_captured; + use tracing::Level; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; - /// Minimal `tracing::Subscriber` that records a formatted line per event - /// so tests can assert on emitted level + fields without pulling in - /// `tracing-subscriber`'s registry machinery. - struct CapturingSubscriber { - events: std::sync::Arc<Mutex<Vec<String>>>, - } - - struct LineVisitor<'a>(&'a mut String); - - impl Visit for LineVisitor<'_> { - fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { - self.0.push_str(&format!(" {}={:?}", field.name(), value)); - } - } - - impl tracing::Subscriber for CapturingSubscriber { - fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool { - true - } - - fn new_span(&self, _span: &span::Attributes<'_>) -> span::Id { - span::Id::from_u64(1) - } - - fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {} - - fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {} - - fn event(&self, event: &tracing::Event<'_>) { - let mut line = format!("{}", event.metadata().level()); - let mut visitor = LineVisitor(&mut line); - event.record(&mut visitor); - if let Ok(mut events) = self.events.lock() { - events.push(line); - } - } - - fn enter(&self, _span: &span::Id) {} - - fn exit(&self, _span: &span::Id) {} - } - #[tokio::test] async fn test_pinned_client_connects_to_checked_ip_not_dns() { // `.invalid` is reserved by RFC 2606 to never resolve: any real DNS @@ -391,80 +348,80 @@ mod tests { assert_eq!(first_blocked_address(&addrs), None); } - #[tokio::test] - async fn test_blocked_webhook_logs_warn_with_host_and_reason() { - let events: std::sync::Arc<Mutex<Vec<String>>> = - std::sync::Arc::new(Mutex::new(Vec::new())); - let subscriber = CapturingSubscriber { - events: events.clone(), - }; - let _guard = tracing::subscriber::set_default(subscriber); - - let dispatcher = WebhookDispatcher::new("secret_key_123"); - let result = dispatcher - .dispatch( - "http://169.254.169.254/latest/meta-data", - "rule_1", - "msg_1", - "Test", - "a@b.com", - ) - .await; + // Not `#[tokio::test]`: the capture subscriber is thread-local, so the + // runtime has to be built inside the capture scope for the dispatch's + // telemetry to land on the thread holding it. + #[test] + fn test_blocked_webhook_logs_warn_with_host_and_reason() { + let (recorder, result) = block_on_captured(async { + let dispatcher = WebhookDispatcher::new("secret_key_123"); + dispatcher + .dispatch( + "http://169.254.169.254/latest/meta-data", + "rule_1", + "msg_1", + "Test", + "a@b.com", + ) + .await + }); assert!(matches!(result, Err(WebhookError::SecurityViolation(_)))); - let captured = events.lock().unwrap(); + let events = recorder.events(); assert!( - captured.iter().any(|line| line.contains("WARN") - && line.contains("blocked") - && line.contains("169.254.169.254")), - "expected a WARN log carrying the blocked host and reason, got: {captured:?}" + events.iter().any(|e| e.level == Level::WARN + && e.message().contains("blocked") + && e.field("host") + .is_some_and(|h| h.contains("169.254.169.254"))), + "expected a WARN log carrying the blocked host and reason, got: {events:?}" ); assert!( - !captured.iter().any(|line| line.contains("secret_key_123")), + !recorder + .all_field_values() + .iter() + .any(|v| v.contains("secret_key_123")), "the webhook HMAC secret must never appear in a log line" ); } - #[tokio::test] - async fn test_allowed_webhook_logs_debug_not_warn() { - let mock_server = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/hook")) - .respond_with(ResponseTemplate::new(200)) - .mount(&mock_server) - .await; + // See the sibling capture test for why this is not a `#[tokio::test]`. + // The wiremock server is started inside the same runtime so the whole + // exchange runs on the capturing thread. + #[test] + fn test_allowed_webhook_logs_debug_not_warn() { + let (recorder, status) = block_on_captured(async { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/hook")) + .respond_with(ResponseTemplate::new(200)) + .mount(&mock_server) + .await; - let events: std::sync::Arc<Mutex<Vec<String>>> = - std::sync::Arc::new(Mutex::new(Vec::new())); - let subscriber = CapturingSubscriber { - events: events.clone(), - }; - let _guard = tracing::subscriber::set_default(subscriber); - - let dispatcher = WebhookDispatcher::new("secret_key_123"); - let opts = ValidationOptions { - available_folders: None, - allowed_forward_domains: None, - block_private_webhooks: false, - }; - let url = format!("{}/hook", mock_server.uri()); - let status = dispatcher - .dispatch_with_options(&url, "rule_1", "msg_1", "Test", "a@b.com", &opts) - .await - .expect("dispatch to the mock server must succeed"); + let dispatcher = WebhookDispatcher::new("secret_key_123"); + let opts = ValidationOptions { + available_folders: None, + allowed_forward_domains: None, + block_private_webhooks: false, + }; + let url = format!("{}/hook", mock_server.uri()); + dispatcher + .dispatch_with_options(&url, "rule_1", "msg_1", "Test", "a@b.com", &opts) + .await + .expect("dispatch to the mock server must succeed") + }); assert_eq!(status, 200); - let captured = events.lock().unwrap(); + let events = recorder.events(); assert!( - captured + events .iter() - .any(|line| line.contains("DEBUG") && line.contains("allowed")), - "expected a DEBUG log for the allowed egress decision, got: {captured:?}" + .any(|e| e.level == Level::DEBUG && e.message().contains("allowed")), + "expected a DEBUG log for the allowed egress decision, got: {events:?}" ); assert!( - !captured.iter().any(|line| line.contains("WARN")), - "an allowed dispatch must not emit a WARN, got: {captured:?}" + !events.iter().any(|e| e.level == Level::WARN), + "an allowed dispatch must not emit a WARN, got: {events:?}" ); } } -- 2.54.0 From 85c5d46ba0f5e2ca5e2e7d18b0668f769895af66 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Mon, 10 Aug 2026 14:26:59 -0500 Subject: [PATCH 52/95] docs(adr): record the convergent multi-engine sync model The engine syncs as though it is the only client of an account: a forward-only UID high-water mark that cannot observe deletion, and fire-and-forget mutations that report success when a UID command silently did nothing. A second client -- another machine, a phone, or webmail -- makes both assumptions false, so the local view diverges and nothing detects it. ADR 0002 decides how a daemon behaves as one of several clients. The core choice is to build no coordination layer: IMAP and WebDAV already coordinate concurrent clients, so the work is finishing the sync model rather than adding multi-engine support, and correctness at ten engines follows from correctness at one. The exception is side-effecting filter actions, which land on third parties with no shared resource to compare against; those get a single owner. A draft was reviewed independently for protocol correctness, codebase fit, and adversarially. All three falsified load-bearing claims, and the corrections are material enough to record rather than summarise: Gmail has no QRESYNC and Exchange has neither QRESYNC nor CONDSTORE, so the ladder needs four rungs selected per mailbox; an absent COPYUID is conforming on success and cannot mean conflict; keying identity on a Message-ID hash would have let a forged message overwrite a trusted body; and the proposed convergence test could not have detected the design's largest hole. The review evidence lands alongside the ADR because several findings cost real effort to establish and would otherwise be rediscovered expensively. Documentation only. No code or contract changes. --- docs/adr/0002-multi-engine-sync-model.md | 221 +++++++++++ .../2026-08-10-multi-engine-design-review.md | 346 ++++++++++++++++++ 2 files changed, 567 insertions(+) create mode 100644 docs/adr/0002-multi-engine-sync-model.md create mode 100644 docs/reviews/2026-08-10-multi-engine-design-review.md diff --git a/docs/adr/0002-multi-engine-sync-model.md b/docs/adr/0002-multi-engine-sync-model.md new file mode 100644 index 0000000..04a8cc4 --- /dev/null +++ b/docs/adr/0002-multi-engine-sync-model.md @@ -0,0 +1,221 @@ +# ADR 0002 — Convergent sync model for multiple concurrent engines + +- **Status:** Accepted +- **Date:** 2026-08-10 +- **Deciders:** James Maes +- **Extends:** [ADR 0001](0001-engine-first-grpc-architecture.md). Nothing here + changes the engine-first, contract-published architecture; this decides how a + daemon behaves as one of several clients of the same mail account. + +## Context + +ADR 0001 establishes one daemon owning all state. It does not say what happens +when **several** daemons own state derived from the **same** upstream account — +which is the normal case for a local-first product the moment a user runs it on +a laptop and a desktop, and is already the case today whenever the user also has +a phone mail app or Gmail open in a browser. + +The engine currently implements a **single-writer, append-only** model: + +- Mail sync is a forward-only per-folder high-water mark (`{uidnext}:*`), and + `fetch_and_persist` only ever calls `save_email`. Nothing observes deletion. +- CalDAV/CardDAV sync re-queries a full window every time; there is no + sync-token, no CTag, no ETag use, and no write-back at all. +- Local identity is `surrogate_id(account, folder, uidvalidity, uid)`, so the + same message in a different folder is a different message. +- Mutations are fire-and-forget: `apply_mutation` returns `Result<()>` and + `Ok(())` is recorded as `completed` with no check that anything happened. + +The consequences are not hypothetical. A message moved or deleted by any other +client becomes a permanent local ghost. A mutation that lost a race against +another client is reported as success, because a UID command against a UID that +no longer exists returns `OK` having done nothing (RFC 3501 §6.4.8). Two engines +diverge silently; ten diverge identically and confidently. + +A design was drafted, then reviewed independently for protocol correctness, +codebase fit, and adversarially. The reviews falsified four load-bearing claims +in the draft. This ADR records the corrected decision; the evidence is in +[`docs/reviews/2026-08-10-multi-engine-design-review.md`](../reviews/2026-08-10-multi-engine-design-review.md). + +## Decision + +### 1. The servers are the coordinator. We do not build a distributed system. + +No engine-to-engine messaging, no leader election, no shared database, no vector +clocks or CRDTs, no broker, no custom sync protocol. IMAP and WebDAV are designed +for many concurrent clients against one collection and ship the primitives for +it. If each engine is a **correct** client, N engines is **O(N) server load and +O(1) design complexity**. + +The work is therefore not "add multi-engine support." It is "finish the sync +model"; multi-engine falls out of it. + +### 2. There is one bounded exception: side-effecting actions. + +The servers coordinate *state*, not *actions*. `MOVE`/`COPY`/`FLAG` converge +under at-least-once execution — ten moves produce one net effect. **`FORWARD` +and `CALL WEBHOOK` do not.** They land on third parties with no shared resource +to compare-and-set against, and every engine independently classifies the same +message as new. Ten engines page an on-call engineer ten times per alert. + +- A per-account `filters_enabled` setting, **default off**, gives exactly one + engine ownership of filter execution. This converts a silent duplication bug + into an explicit, documented limitation, and requires no protocol work. +- Where CONDSTORE and custom keywords are available, an engine may instead claim + a message with `UID STORE <uid> +FLAGS.SILENT (UNCHANGEDSINCE <modseq>) + ($NuncioFiltered)`; a `MODIFIED` response means another engine won the claim. + This is the same compare-and-set primitive as §4, so it adds no new machinery. +- Rule identity becomes a content hash of normalized NSQL rather than random + bytes, and side-effecting actions carry `Idempotency-Key: + hash(rule_key, message_key)`, so receivers can dedup what we cannot make + once-only. + +### 3. Change enumeration replaces the high-water mark — four rungs, per mailbox. + +`CollectionSync::enumerate_changes(collection, since) -> {upserts, removals, +next_token}`. The `removals` set is the entire fix for ghost messages, and it +fixes the single-engine case too. + +| Rung | Mechanism | Providers | +| --- | --- | --- | +| 1 | QRESYNC: `ENABLE QRESYNC` + `SELECT (QRESYNC …)` → `VANISHED` | Fastmail/Cyrus, Dovecot | +| 2 | CONDSTORE: `UID FETCH 1:* (FLAGS) (CHANGEDSINCE n)` + periodic UID-set diff | **Gmail** | +| 3 | `STATUS`/`LIST-STATUS` change gate, then ESEARCH UID-set diff | **Exchange**, Yahoo | +| 4 | Full resync | last resort | + +**Gmail advertises CONDSTORE but not QRESYNC; Exchange advertises neither.** +Detection is per *mailbox*, not per account, because `NOMODSEQ` is a mailbox +property. A two-rung ladder would put the largest provider on the most expensive +path. + +CalDAV and CardDAV both use RFC 6578 `sync-collection` (identical shape), with +`sync-level: infinite`, 507 truncation handling, loop-to-convergence, and +`calendar-multiget`/`addressbook-multiget` to fetch bodies for changed hrefs. +CTag is a vendor extension, not a peer mechanism, and is a fallback only. + +Sync state is structured, not a string: `(uidvalidity, highestmodseq, +known-uid-set, seq-match-sample)`. QRESYNC **never signals modseq expiry** — a +too-old modseq produces a large `VANISHED`, not an error — so a full resync is +always a client-side derivation from a UIDVALIDITY mismatch, `NOMODSEQ`, or a +HIGHESTMODSEQ regression. A periodic full UID-set reconciliation is retained +even on rung 1 as a safety net. + +**Sync tokens carry a scheme tag** (`v2:qresync:…`). The existing checkpoint is +`{uidvalidity}{DELIM}{uid}` — the same shape as `uidvalidity:highestmodseq` with +a different meaning. Untagged reuse would hand a UIDNEXT to a server as a +MODSEQ; because UIDNEXT is typically the larger number, the server would answer +"nothing changed" and everything below it would never be enumerated again. +Unknown schemes force a full resync. + +### 4. Mutations carry a precondition and must prove their effect. Three outcomes. + +``` +MutationOutcome = Applied { token: Option<Token> } | Conflict { observed } | Unknown +``` + +`Conflict` requires **positive evidence**: `[MODIFIED …]` in the tagged +response, or HTTP 412. Anything ambiguous is `Unknown`, which triggers +re-enumeration of the affected collections rather than a verdict. + +This matters because the obvious proofs are unreliable: + +- `COPYUID` is only *SHOULD* for MOVE (RFC 6851 §4.3), is advised in an + **untagged** OK, has no empty form, and is legitimately omitted for + UIDNOTSTICKY destinations. Present ⇒ applied; absent ⇒ **unknown**. +- Under QRESYNC, `UID EXPUNGE` returns `VANISHED`, not `EXPUNGE` (RFC 6851 §4.4), + so counting `EXPUNGE` responses reports failure on every successful delete. +- `+FLAGS.SILENT` suppresses the untagged FETCH (RFC 3501 §6.4.6) and is the + normal form for conditional stores, so "no FETCH ⇒ conflict" fires on the most + common flag operation. +- A DAV server that rewrites a submitted object MUST NOT return an ETag + (RFC 4791 §5.3.4), and normalisation is the norm; `Applied` must tolerate a + missing token and schedule a re-read. + +Flags use per-flag last-writer-wins with benign merge. RFC 7162 §3.1.3 only +*SHOULD*s avoiding spurious `MODIFIED` on single-modseq servers, so a blanket +hard-conflict policy would make routine concurrent flag activity a constant +conflict stream. + +### 5. Conflicts are durable and queryable, not a notification. + +Mutations model **desired end-state on an identity**, not an imperative +operation on a placement, so re-enumerate-and-retarget preserves user intent; +re-targets are capped, then a hard conflict is raised. Conflicts persist in a +table with RPCs to list and resolve them. Events become an optimization over a +queryable source of truth — necessary because `Subscribe` is a broadcast stream +that silently drops events on lag, so no event-driven guarantee is currently +sound without a sequence number and resumption. + +### 6. Identity is content-addressed and account-scoped. + +Precedence: `EMAILID` (RFC 8474 OBJECTID) → `X-GM-MSGID` → `(account_id, +normalized Message-ID, content_hash)` → `(account, folder, uidvalidity, uid)`. + +Bodies are keyed by a content hash of the full RFC822 octets, first-write-wins. +**A Message-ID hash alone must not be the key**: Message-IDs are public, echoed +in every `References:` header, and forgeable, so with `INSERT OR REPLACE` a +message reusing a known Message-ID would silently overwrite the body of a +message the user already trusts and would evade filters by never being seen as +new. That hole does not exist today and must not be introduced. The same +Message-ID with different content is two messages, which also covers +list-copy-vs-direct and drafts. + +A message is separated from its **placements** — `PRIMARY KEY(account_id, +folder_id, uidvalidity, uid)`, with `uidvalidity` in the key so UIDs restarting +after a bump cannot collide with old rows. DAV gets the same split: href = +placement, iCalendar/vCard UID = identity, ETag = version. + +### 7. Load is bounded by bandwidth, not connection count. + +Gmail publishes both a 15-simultaneous-connection and a 2,500 MB/day download +cap **per account**. On rungs 2 and 3, ten engines over five accounts reach the +byte cap long before the connection cap. Per-engine budget derives from a +`fleet_size` setting, since cross-engine rate limiting is impossible without the +coordination layer we decline to build. `Retry-After` is honoured *with* jitter +proportional to `fleet_size`, or ten engines retry in the same instant. + +One mutex-guarded connection per (engine, account) with mutation priority; sync +work is chunked per folder so it yields to user actions. + +## Consequences + +- **This gates the M7 API freeze.** Mutation RPCs do not exist (`Mail` has no + Move/Delete/Flag); `Conflict` must be a typed error, not a response field, so + old clients take the error path; `Message` needs identity and placements. + Most individual additions are wire-additive, but redefining `Message.id` + changes the *value* of every existing id while staying wire-compatible — + invisible to the byte-exact descriptor golden, and that token appears at ten + other wire positions. A model that cannot express deletion or conflict must + not be frozen. +- **Shared OAuth accounts across engines are out of scope.** Gmail and + Microsoft 365 mandate OAuth; providers that rotate refresh tokens on use would + have ten engines invalidating each other. A token broker is exactly the + coordination layer this ADR declines to build. Documented limitation. +- **Migration is forward-only.** `message_key` cannot be derived for existing + rows: Message-ID is never parsed, never stored, and raw bytes are discarded + after parse. Ingest must capture identity first and let it accumulate. Legacy + rows keep their surrogate as `message_key` with one placement each and receive + no retroactive dedup. +- **The audit ledger gains a namespace boundary.** + `filter_execution_logs.message_id` is an input to an HMAC hash chain verified + by a live `Audit.VerifyChain`, so historical ids cannot be rewritten. +- **A new class of test is required.** Convergence cannot be shown by one engine + against one mock. Verification needs ≥3 engines against one stateful server, + plus side-effect-counting SMTP and webhook mocks — final-state placement + convergence is blind to duplicate forwards, so the largest risk in this ADR + would pass a naive convergence test. +- **`async-imap` 0.11.3 cannot express any of this**: no `ENABLE`, its `SELECT` + parser drops `VANISHED`, and `check_status_ok` discards the tagged response + code where `COPYUID` and `[MODIFIED …]` live. A fork-versus-hand-rolled + decision gates the protocol work. + +## Alternatives considered + +- **A coordination layer** (broker, leader election, shared state). Rejected: it + would be the largest and least reliable component in the repo, and it solves a + problem the servers already solve for everything except side-effecting actions. +- **CRDTs / vector clocks.** Rejected: the server holds authoritative state, so + there is no merge to compute — only a state to observe and preconditions to + respect. +- **Freezing `nuncio.v1` first and adding conflict semantics later.** Rejected: + not expressible additively. diff --git a/docs/reviews/2026-08-10-multi-engine-design-review.md b/docs/reviews/2026-08-10-multi-engine-design-review.md new file mode 100644 index 0000000..1e5d4a2 --- /dev/null +++ b/docs/reviews/2026-08-10-multi-engine-design-review.md @@ -0,0 +1,346 @@ +# Design review — multi-engine sync model (2026-08-10) + +Evidence behind [ADR 0002](../adr/0002-multi-engine-sync-model.md). A draft +design was reviewed independently along three axes, each instructed to find +errors rather than confirm the draft, and to say plainly when an attack failed. +All three found load-bearing defects. Reviewed against `dev` @ `a997220`. + +Findings are recorded here because several are non-obvious, several cost real +effort to establish, and at least two would otherwise be rediscovered the +expensive way. + +--- + +## A. Protocol correctness + +### A1. The provider capability matrix in the draft was wrong + +The draft claimed Gmail, Fastmail, Dovecot, and Exchange all support CONDSTORE +and QRESYNC. Verified against live capability strings: + +| Provider | CONDSTORE | QRESYNC | +| --- | --- | --- | +| Fastmail (Cyrus) | yes | yes (also RFC 8474 OBJECTID) | +| Dovecot | yes | yes, but `NOMODSEQ` per mailbox where persistent modseqs were never enabled | +| **Gmail** | yes | **no** — not planned | +| **Exchange / Exchange Online** | **no** | **no** | + +Two of four named providers cannot use the draft's headline mechanism. This is +what forced the ladder from two rungs to four, and capability detection from +per-account to per-mailbox (RFC 7162 §3.1.2.2 makes `NOMODSEQ` a mailbox +property). + +The draft also omitted the CONDSTORE-only middle rung entirely — RFC 7162 §3.2.5.1 +states QRESYNC's `SELECT` is semantically equivalent to `UID FETCH … (CHANGEDSINCE +… VANISHED)`, i.e. a round-trip optimisation over CONDSTORE, not a different +capability class. Skipping it put Gmail on full flag sweeps against a published +2,500 MB/day per-account cap. + +### A2. QRESYNC has no modseq-expiry error + +RFC 7162 §3.2.6: a server receiving a mod-sequence older than it remembers +**MUST behave as if asked to report all expunged messages from the provided UID +set**. The failure mode is a bandwidth blowup, not a refusal. `full_resync_required` +can therefore never be *received*; it must be derived client-side from a +UIDVALIDITY mismatch (§3.2.5), `NOMODSEQ` (§3.1.2.2), or a HIGHESTMODSEQ +regression. + +`uidvalidity:highestmodseq` is also insufficient state: §3.2.5.1 and §3.2.5.2 +make the known-UID set and sequence-match data part of what the client supplies. +And §3.2.3 requires a tagged **BAD** if `QRESYNC` is used without `ENABLE QRESYNC` +on the connection. + +### A3. "Absent COPYUID means the mutation failed" is wrong three times over + +- RFC 4315 §3 ABNF has no empty `uid-set`; the states are *present* or *absent*. +- Absence is conforming on success: the client may lack SELECT rights on the + destination, or the destination may be UIDNOTSTICKY. +- RFC 6851 §4.3 makes COPYUID only *SHOULD* for MOVE, and advises sending it in + an **untagged** OK — so a client reading only the tagged OK misses it on every + conforming MOVE. + +Additionally, RFC 6851 §4.4: under QRESYNC the server sends `VANISHED` rather +than `EXPUNGE` for `UID EXPUNGE`, so a delete proved by counting `EXPUNGE` +responses reports conflict on every success. + +Consequence: a third outcome, `Unknown`, is required. `Conflict` needs positive +evidence. + +### A4. MODIFIED does not cover the case the draft cared about + +RFC 7162 §3.1.3 puts `MODIFIED` in the **tagged** response, and only *SHOULD*s +that servers avoid spurious `MODIFIED` on single-modseq servers — so engine B +setting `\Seen` can fail engine A's unrelated `\Flagged` store. More +importantly, `MODIFIED` means "changed by someone else"; a UID another engine +already *moved or expunged* is simply ignored (RFC 3501 §6.4.8) and never +appears in `MODIFIED`. Presence detection needs a separate mechanism. + +Relatedly, the draft's flag proof-of-effect ("untagged FETCH present") is unsound: +`.SILENT` suppresses it (RFC 3501 §6.4.6) and is the normal form for conditional +stores. + +### A5. Message-ID is not a content identifier + +RFC 5322 §3.6.4 makes it *SHOULD*, not MUST. One Message-ID legitimately maps to +different octets for mailing-list-plus-direct copies (Mailman preserves it while +adding `List-*` headers and a footer) and for drafts (Gmail and Exchange keep one +Message-ID across saves). The draft's "hash of raw headers" fallback is also +unstable across exactly the placements it exists to unify, because `Received:` +chains differ. + +Missed primitives that solve this properly: **`EMAILID`** (RFC 8474, Fastmail/Cyrus, +Dovecot) and **`X-GM-MSGID`** (Gmail) are server-assigned identities stable across +folders. + +### A6. DAV specifics the draft omitted + +RFC 6578 §3.6 truncation (207 + `507` + `DAV:number-of-matches-within-limits`) — +"Clients MUST handle the 507 status"; §3.1 loop-to-convergence; `sync-level: +infinite` (§3.5.1 warns `sync-level: 1` is not reliable); discovery via +`DAV:supported-report-set`; and `calendar-multiget`/`addressbook-multiget` +(RFC 4791 §7.9, RFC 6352 §8.7) because `sync-collection` returns hrefs and ETags, +not content. RFC 4791 §5.3.4: a server that rewrites the submitted object MUST NOT +return an ETag. RFC 6638 scheduling is a write hazard — PUTting a VEVENT with +ATTENDEEs on a scheduling-aware server emails real people. + +`getctag` is a Calendar Server vendor extension, not an RFC, and was +mis-presented as a peer of RFC 6578. + +### A7. Other omitted primitives + +`STATUS (… HIGHESTMODSEQ)` (RFC 7162 §3.1.7) and `LIST-STATUS` (RFC 5819, which +Gmail advertises) give a whole-account change probe in one round trip. +`COMPRESS=DEFLATE` (RFC 4978) directly reduces metered bytes. `NOTIFY` (RFC 5465) +watches many mailboxes on one connection, which moots the draft's +"IDLE costs a connection per mailbox" reasoning. ESEARCH `SEARCH RETURN (ALL)` +is far cheaper than `UID FETCH 1:* (FLAGS)` for a vanished-set diff. +`UIDNOTSTICKY` (RFC 4315) invalidates both COPYUID proof and the placement key +and must be detected. + +--- + +## B. Codebase fit + +### B1. `message_key` cannot be derived — the blocker is ingest, not migration + +`MimeParserAdapter::parse_mime` never calls `message_id()`; `Email` has no such +field; the `messages` table has no column for it; and `build_email_from_fetch` +discards the `BODY[]` bytes after parsing. The IMAP `ENVELOPE` does carry +`message-id`, but only `subject`/`from`/`to` are read. + +So in-place migration of a populated database is impossible — there is neither a +Message-ID nor raw headers to hash. Either re-fetch every message (offline-hostile, +impossible for a removed account) or set `message_key := old surrogate`, which +yields one placement per message and no retroactive dedup. Capture must ship +first and accumulate. + +### B2. The audit hash chain pins historical ids permanently + +`filter_execution_logs.message_id` is an input to `compute_log_hash`, verified by +`verify_execution_log_chain`, reachable from the live `Audit.VerifyChain` RPC. +Rewriting historical ids breaks verification permanently. `worm_audit_records` is +harder still — `BEFORE UPDATE`/`BEFORE DELETE` `RAISE(ABORT)` triggers. A +documented namespace boundary with a schema-version marker is the only option. + +### B3. Placements break the filter engine's evaluation contract + +`FilterField::Folder` and `FilterField::Account` evaluate scalar fields on +`Email`; `WHERE FOLDER = 'INBOX'` has no meaning against a multi-placement +message. `evaluate(&Email)` must become per-placement. + +More seriously: **the fire-once guard is `existing_message_ids`, not the +execution log.** Under `message_key`, a genuinely new arrival that already exists +in another folder becomes "not new" and **silently never fires any filter** — a +new correctness bug the identity change introduces. The guard must become +placement-aware *and* gain a real `(rule_id, message_key)` dedup. + +`set_message_read` is `UPDATE messages … WHERE id = ?`; since IMAP `\Seen` is +per-mailbox, read state becomes per-placement and `MARK READ` must name one. + +### B4. FTS orphans are a confidentiality problem + +`messages_fts` holds **plaintext** bodies keyed on `messages.id`, with a +delete-only trigger. Under placements, dropping *a* placement must not delete the +FTS row while others reference the body; dropping the *last* must. Inverted, this +either loses search or leaks plaintext bodies in orphaned rows — in the one table +the schema comments already document as unencrypted. + +Also: there is no `folders` table. Folders are derived from `messages GROUP BY +folder_id` and are currently global across accounts, which placements will expose. + +### B5. `async-imap` 0.11.3 cannot express the design + +| Needed | Status in the dependency | +| --- | --- | +| `COPYUID` | `uid_copy`/`uid_mv` return `Result<()>`; `check_status_ok` discards the response code | +| `[MODIFIED …]` | no `UNCHANGEDSINCE` variant; tagged code discarded | +| `SELECT (QRESYNC …)` | no `select_qresync`; no `ENABLE` command at all | +| `VANISHED` | parsed by `imap-proto` but dropped by async-imap's `select()` parser | +| untagged FETCH / expunged UIDs | reachable; both currently drained and discarded | + +`run_command`/`read_response` are public, so a hand-rolled tagged-response +collector is possible (~80–150 LOC plus per-command parsers) — but it is a +prerequisite, and it is subject to the workspace's `-D warnings` / no-`unwrap` +regime. Fork-versus-hand-roll is a gate on the protocol work. + +### B6. There is no IMAP mock, and no scheduler, and no backoff + +- `wiremock` is HTTP-only. IMAP had only ad-hoc inline `duplex` responders: + single connection, no shared state. *(Addressed by PR #384.)* +- `OutboxManager::calculate_backoff_ms` is **dead code**; retries run on a fixed + 5s tick with `MAX_RETRIES = 5`, so a 25-second blip permanently fails every + queued mutation fleet-wide. +- `sync_interval_secs` is stored and validated but **nothing polls**. There is no + periodic sync at all; `main.rs` spawns only the outbox drain (5s) and an opt-in + update check (24h). +- `SyncProgress` never reaches the wire — it is dropped at the gRPC boundary. +- `Mail.MarkRead` and `Contacts.CreateContact` are silently **local-only** and + report success. +- `AccountProtocol` has no `CardDav` variant, so a CardDAV account is not + representable — contacts sync is unwireable, not merely unwired. +- The DAV "parsers" are substring splits on `<c:calendar-data>`; a different + namespace prefix yields zero events reported as a successful sync. DAV ids are + positional, so re-syncs reshuffle which local row holds which remote object. + +### B7. The proto claim was overstated in the draft's favour and against it + +Three of five changes *are* wire-additive with free tags (mutation RPCs, sync +tokens in `GetStatus`, new `Event` oneof arms). The genuinely dangerous one is +invisible: redefining `Message.id` changes the *value* of every existing id while +staying wire-compatible — the byte-exact descriptor golden cannot see it, and the +token appears at ten other wire positions. + +`STORY-WORKFLOW.md` permits one proto-touching story in flight; since four work +items touch the proto, a single "proto chunk" is not schedulable and must +dissolve into a serialized tail on each. + +### B8. Sizing + +The draft's M/L/M/S/M (~4–6 weeks) is roughly 3× optimistic. Measured against the +tree: **5,800–9,300 production LOC + 4,000–6,600 test LOC across ~90–120 file +touches** — about a quarter of focused work. Chunk 1 alone is ~120–160 edit sites +across ~25–30 files. This substantially *is* WS-B + WS-C + WS-E + parts of M6.5. + +--- + +## C. Adversarial + +### C1. Filter fan-out — the largest hole, absent from the draft entirely + +All ten engines classify the same message as new (per-engine store), and +`pending_remote_mutations` has no uniqueness constraint. State-mutating actions +converge; `FORWARD` and `CALL WEBHOOK` do not — the on-call engineer is paged ten +times per alert. Rule IDs are `OsRng`-random per engine, so the "same" rule has +ten identities and no receiver can dedup from rule identity even in principle. + +Adding a `Conflict` outcome makes the UX *worse* here, not better: the nine losing +MOVEs stop silently fabricating success and instead emit conflict events to two +clients each — eighteen spurious notifications per filtered message. "Server wins, +loudly" is calibrated for rare conflicts; filter fan-out makes conflict routine. + +The draft's one sentence that sounds like it addresses this ("filters key on +`message_key`, so they stop re-firing on the destination copy") is about *self* +re-firing after a move — a different bug. + +### C2. The draft's identity scheme introduces a spoofing primitive + +Message-IDs are public and echoed in `References:`. With `INSERT OR REPLACE` and a +key of `hash(Message-ID)`, an attacker reusing a known Message-ID silently +overwrites the body shown under the original message's placement, and evades +filters entirely by never being classified new. **This hole does not exist today**, +because the current `surrogate_id` includes the UID. The identity change would +have created it. + +### C3. Mixed-version fleets fight on the shared server + +Old engines key identity on folder, so a new engine's move looks like a brand-new +message to them — they re-evaluate the rule set and re-move it. Old and new +engines trade the message back and forth. There is no `PRAGMA user_version`, no +down path, and an old binary opening a new database "works" while writing the old +model, leaving two mutually invisible truths in one file. + +Worst of these: the sync-token shape collision described in ADR 0002 §3, which +fails *silently* rather than loudly. + +### C4. "Server wins, loudly" strands intent and leaves clients nothing to act on + +A user pressing Delete on engine 7 for a message engine 2 already archived has +their intent discarded; re-enumeration removes the placement, so from their seat +the delete appeared to work. Retry is *safe* here — `message_key` is stable and +the new location is known — and the blanket prohibition follows from modelling +mutations as imperative operations on placements rather than desired end-state. + +Clients get no event payload contract, no RPC to act on a conflict, no listing, no +GC, and `Subscribe` silently drops events on lag with no sequence number or +resumption. "Loudly" reduces to a log line that sometimes reaches a UI. + +### C5. Connection math measures the wrong quantity + +The outbox opens a **fresh connection per mutation**: worst case 50 connect/login +cycles per 5s per engine, ×10 engines = 500 authentications per 5 seconds against +one account. Providers rate-limit *authentications* far harder than concurrent +connections, and some respond with a temporary lockout. Sync and outbox also +contend for the single pooled connection the draft specifies — share it and a +mark-read queues behind a 300-second per-account sync timeout; don't, and it is +2 per engine, which the draft's own table marks as over cap. + +Jitter alone does not solve correlated wake-ups (workday start, VPN up, power +restored), and honouring `Retry-After` without jitter creates a synchronized herd. +Rate limiting across engines is impossible without coordination, so an engine must +know its `fleet_size` and derive its budget from it. + +### C6. Placement schema and mutation addressing + +The draft's `PRIMARY KEY(account_id, folder_id, uid)` collides after a UIDVALIDITY +bump, when UIDs restart at 1. And a mutation has no placement coordinate: a +message in both INBOX and Archive with a rule saying MOVE TO Archive has no +defined source. The outbox currently recovers addressing from the *message* row, +which ceases to exist under the split. + +Drafts are unaddressed and will churn the model (APPEND-then-delete per save, new +UID each time, same or new Message-ID depending on client). + +### C7. The proposed verification cannot detect C1 + +Final-state placement convergence is exactly blind to duplicate side effects — ten +forwards and one forward produce identical placements. N=2 also shows a single +duplicate, which reads as an off-by-one. Requires N≥3, side-effect-counting SMTP +and webhook mocks asserting "exactly one forward across the fleet", and an +interleaving harness that can hold one engine's mutation in flight while another +acts on stale state. + +### C8. OAuth breaks the shared-account premise + +Gmail and Microsoft 365 mandate OAuth; there is none in the tree. Providers that +rotate refresh tokens on use would have ten engines invalidating each other in a +loop, and with no backoff that becomes a lockout. A token broker is the +coordination layer the design declines to build — so this is a scope boundary, not +a TODO. + +### C9. Smaller items + +Clock skew across ten machines makes any wall-clock tiebreak unreliable (outbox +ordering, audit ledger). "Body stored once" is per-engine, not fleet-wide. The +append-only hash-chained execution log has no retention policy and will orphan +entries. One poison collection that always fails burns the account's 300s timeout +every cycle. And the draft never stated a consistency guarantee at all — several +of the above decisions fall out of writing one down. + +--- + +## What the reviews agreed the draft got right + +- **"Don't build a distributed system" is correct for state convergence.** For + placements, flags, and DAV objects, the servers genuinely are the coordinator, + and O(N) load with O(1) design complexity is the right trade. The thesis fails + only for side-effecting actions (C1) — a bounded exception, not a repudiation. +- **`CollectionSync` with `{upserts, removals, next_token}` is the right shape**, + and the RFC 7162 / 6578 mapping is sound. +- **"An empty effect must not be reported as success" is correct and important**; + the objection is to the verdict drawn from it, not to detecting it. +- **The M7-freeze argument is correct and understated** — C4 and B6 strengthen it, + since the freeze must also cover a durable conflict surface, event sequence + numbers, and the fact that `MarkRead` will come to mean something different. +- **Change enumeration ships first and alone**, and fixes ghost messages for + single-engine users independently of everything else. The draft was wrong that + it depends on the identity split. -- 2.54.0 From 3de75493d5655554f495922a1c90ea6fdefe775f Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Mon, 10 Aug 2026 14:48:37 -0500 Subject: [PATCH 53/95] feat(mail): capture Message-ID and a content hash at ingest Identity cannot currently be reconstructed for a stored message. The parser never reads Message-ID, the Email model has nowhere to put it, the messages table has no column for it, and the raw octets are dropped once parsed. Any later attempt to give a message a folder-independent identity therefore has nothing to work from except a re-download of the entire mailbox. Capture both now, as nullable columns nothing reads yet, so the data starts accruing while the design that consumes it is still being built. Normalization is fixed and shared rather than left to each call site: strip all ASCII whitespace (a msg-id cannot legally contain any, so this unfolds a wrapped header without knowing how the transport folded it), strip one pair of angle brackets, and preserve case, since only the domain half is case-insensitive and lowercasing wholesale would merge ids the origin considers distinct. Engines that normalize differently would hold the same message under different bytes, and no comparison between their caches would mean anything. Captured at every ingest path, including the two that do not parse a full message. The envelope-only path takes Message-ID from ENVELOPE rather than skipping capture. The headers-only path keeps the id recovered from those headers but discards the hash parse_mime computed over them, because content_hash is defined over a message's full octets and publishing a header-only digest under that name would quietly mean something else. The hash exists to keep Message-ID honest: it is public, echoed in every References header, and reused legitimately -- a mailing-list copy and a direct copy share one, and a draft keeps one across every save -- so the pair distinguishes messages that the id alone would merge. No backfill. Legacy rows stay NULL, which reads as "not captured" rather than "this message has no Message-ID"; inventing a value would be worse than admitting the gap. Refs #386 --- crates/nuncio-core/src/export.rs | 4 + crates/nuncio-core/src/model.rs | 116 ++++++++++++ crates/nuncio-filter/src/engine.rs | 4 + crates/nuncio-mail/src/imap.rs | 18 ++ crates/nuncio-mail/src/jmap.rs | 44 ++++- crates/nuncio-mail/src/mock.rs | 2 + crates/nuncio-mail/src/parser.rs | 38 ++++ crates/nuncio-mail/src/smtp.rs | 2 + crates/nuncio-store/src/db.rs | 167 +++++++++++++++++- crates/nuncio-store/src/search.rs | 2 + crates/nunciod/src/grpc.rs | 16 ++ crates/nunciod/src/outbox.rs | 2 + crates/nunciod/src/sync.rs | 2 + .../nunciod/tests/filters_triage_e2e_test.rs | 2 + .../nunciod/tests/no_secrets_canary_test.rs | 2 + .../nunciod/tests/outbox_executor_e2e_test.rs | 2 + crates/nunciod/tests/spine_e2e_test.rs | 2 + 17 files changed, 416 insertions(+), 9 deletions(-) diff --git a/crates/nuncio-core/src/export.rs b/crates/nuncio-core/src/export.rs index d180291..8c5ad0b 100644 --- a/crates/nuncio-core/src/export.rs +++ b/crates/nuncio-core/src/export.rs @@ -195,6 +195,8 @@ mod tests { body_plain: Some("Hello world body text".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }, Email { id: "msg-101".to_string(), @@ -210,6 +212,8 @@ mod tests { body_plain: Some("Second body text".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }, ] } diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index b6e5034..9796db4 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -104,6 +104,26 @@ pub struct Email { pub body_html: Option<String>, /// List of attached files. pub attachments: Vec<Attachment>, + /// The message's RFC 5322 `Message-ID`, normalized by + /// [`Email::normalize_message_id`], or `None` when the message carried + /// none (the header is `SHOULD`, not `MUST`) or the transport did not + /// surface it. + /// + /// Captured but not yet used for identity. It is a *hint* toward a + /// folder-independent identity, never an identity on its own: the value is + /// public (it is echoed in the `References` of every reply) and forgeable, + /// so keying storage on it alone would let a crafted message address, and + /// overwrite, one the user already trusts. + pub message_id: Option<String>, + /// Hex SHA-256 over the full RFC822 octets the message was parsed from, or + /// `None` when only headers or an envelope were available. + /// + /// Pairs with [`Email::message_id`]: the same `Message-ID` carrying + /// different content is two messages, not one. That case is ordinary -- + /// a mailing-list copy and a direct copy of the same mail share a + /// `Message-ID` while differing in headers and footer, and a draft keeps + /// one `Message-ID` across every save. + pub content_hash: Option<String>, } impl Email { @@ -131,6 +151,51 @@ impl Email { } hex::encode(hasher.finalize()) } + + /// Canonicalize a raw `Message-ID` header value. + /// + /// Every engine syncing an account must derive byte-identical values from + /// the same header, or two caches that agree in substance will disagree in + /// bytes and no comparison between them can be trusted. The rules are + /// therefore fixed and deliberately small: + /// + /// 1. Remove **all** ASCII whitespace. A `msg-id` cannot legally contain + /// any, so this unfolds a wrapped header without needing to know how the + /// transport folded it -- and transports differ. + /// 2. Strip one surrounding pair of angle brackets, which servers include + /// or omit inconsistently between `ENVELOPE` and a raw header. + /// 3. Preserve case. Only the domain half is case-insensitive per RFC 5322 + /// §3.6.4; lowercasing the whole value would merge ids that the + /// originating server considers distinct, and the same server returns + /// the same bytes to every engine anyway. + /// + /// Returns `None` for an absent or empty value so a missing header stays + /// distinguishable from a present-but-empty one. + pub fn normalize_message_id(raw: &str) -> Option<String> { + let stripped: String = raw.chars().filter(|c| !c.is_ascii_whitespace()).collect(); + let stripped = stripped + .strip_prefix('<') + .and_then(|s| s.strip_suffix('>')) + .unwrap_or(&stripped); + if stripped.is_empty() { + None + } else { + Some(stripped.to_string()) + } + } + + /// Hex SHA-256 over a message's full RFC822 octets, for the + /// `content_hash` field. + /// + /// Deterministic for identical bytes, which is what makes it usable as the + /// tiebreaker against a duplicated `Message-ID`. It is *not* stable across + /// servers that re-serialize a message, so it distinguishes content within + /// an account rather than identifying a message globally. + pub fn content_hash_of(raw_bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(raw_bytes); + hex::encode(hasher.finalize()) + } } /// Mailbox folder entity. @@ -247,4 +312,55 @@ mod tests { Email::surrogate_id("ab", "c", "1", "1"), ); } + + #[test] + fn normalize_message_id_unfolds_and_unwraps_to_one_canonical_form() { + // Every spelling a transport might hand us for the same id must + // collapse to identical bytes, or two engines holding the same message + // cannot be shown to agree. + let canonical = "abc123@mail.example.com"; + for raw in [ + "<abc123@mail.example.com>", + "abc123@mail.example.com", + " <abc123@mail.example.com> ", + "<abc123@mail.\r\n example.com>", + "<abc123@mail.\t example.com>", + ] { + assert_eq!( + Email::normalize_message_id(raw).as_deref(), + Some(canonical), + "input {raw:?} did not canonicalize" + ); + } + } + + #[test] + fn normalize_message_id_preserves_case_and_rejects_empty() { + // Only the domain half is case-insensitive per RFC 5322 3.6.4; + // lowercasing wholesale would merge ids the origin considers distinct. + assert_eq!( + Email::normalize_message_id("<AbC@Example.COM>").as_deref(), + Some("AbC@Example.COM") + ); + assert_eq!(Email::normalize_message_id(""), None); + assert_eq!(Email::normalize_message_id("<>"), None); + assert_eq!(Email::normalize_message_id(" "), None); + } + + #[test] + fn content_hash_distinguishes_identical_message_ids_with_different_bodies() { + // The case the hash exists for: a list copy and a direct copy share a + // Message-ID but are not the same message. + let direct = b"Message-ID: <shared@example.com>\r\n\r\nbody"; + let via_list = b"Message-ID: <shared@example.com>\r\nList-Id: l\r\n\r\nbody\n--\nfooter"; + assert_ne!( + Email::content_hash_of(direct), + Email::content_hash_of(via_list) + ); + assert_eq!( + Email::content_hash_of(direct), + Email::content_hash_of(direct), + "the hash must be deterministic for identical octets" + ); + } } diff --git a/crates/nuncio-filter/src/engine.rs b/crates/nuncio-filter/src/engine.rs index d5d1230..569823c 100644 --- a/crates/nuncio-filter/src/engine.rs +++ b/crates/nuncio-filter/src/engine.rs @@ -495,6 +495,8 @@ mod tests { body_plain: Some("Hello".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } @@ -518,6 +520,8 @@ mod tests { body_plain: Some("Hello".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; let results = engine.evaluate(&email); diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index 2de83f7..afae823 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -931,6 +931,14 @@ impl ImapEngine { let received_at = fetch_data.internal_date().map_or(0, |dt| dt.timestamp()); + // No body was returned, so there are no octets to hash -- but the + // ENVELOPE carries `message-id` (RFC 3501 section 7.4.2), so identity + // capture does not have to be skipped on this path. + let message_id = fetch_data + .envelope() + .and_then(|env| env.message_id.as_ref()) + .and_then(|raw| Email::normalize_message_id(&String::from_utf8_lossy(raw))); + Ok(Email { id: email_id, account_id: self.account_id.clone(), @@ -945,6 +953,8 @@ impl ImapEngine { body_plain: None, body_html: None, attachments: Vec::new(), + message_id, + content_hash: None, }) } @@ -982,6 +992,12 @@ impl ImapEngine { email.body_plain = None; email.body_html = None; email.attachments = Vec::new(); + // `parse_mime` hashed the header section it was handed, but + // `content_hash` is defined over a message's FULL octets. + // Keeping that value would publish a hash that silently means + // something else, so drop it; the `Message-ID` it recovered + // from those headers is still correct and is kept. + email.content_hash = None; return email; } } @@ -1004,6 +1020,8 @@ impl ImapEngine { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }) } diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index 80319b7..187f257 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -92,6 +92,10 @@ pub struct JmapEmail { pub received_at: Option<i64>, pub is_unread: Option<bool>, pub body_snippet: Option<String>, + /// RFC 8621 models `messageId` as a list, because a message may carry more + /// than one `Message-ID` header. The first entry is the one used. + #[serde(default)] + pub message_id: Option<Vec<String>>, } /// JMAP email address object. @@ -184,7 +188,7 @@ impl JmapEngine { { "accountId": account_id, "ids": ids, - "properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet"] + "properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet", "messageId"] }, "c1" ] @@ -418,6 +422,15 @@ impl JmapEngine { body_plain: item.body_snippet, body_html: None, attachments: Vec::new(), + message_id: item + .message_id + .as_ref() + .and_then(|ids| ids.first()) + .and_then(|raw| Email::normalize_message_id(raw)), + // `Email/get` returns a `bodySnippet`, not the message's + // octets, so there is nothing to hash here without an + // extra download per message. + content_hash: None, } }) .collect(); @@ -705,6 +718,35 @@ mod tests { assert_eq!(new_state, "s-2"); } + #[test] + fn parse_email_get_response_captures_normalized_message_id() { + // RFC 8621 models `messageId` as a list; the first entry is taken and + // normalized to the same canonical form the IMAP paths produce, so two + // engines syncing one account via different protocols agree. + let raw = r#"{ + "methodResponses": [ + ["Email/get", {"state": "s-1", "list": [ + {"id": "m-1", "subject": "Hi", "messageId": ["<Abc.123@mail.nuncio.mx>"]}, + {"id": "m-2", "subject": "No id"} + ]}, "c1"] + ] + }"#; + let engine = JmapEngine::new("acct-1"); + let (emails, _state) = engine.parse_email_get_response(raw).expect("parse get"); + + assert_eq!( + emails[0].message_id.as_deref(), + Some("Abc.123@mail.nuncio.mx") + ); + assert_eq!( + emails[1].message_id, None, + "an absent messageId must stay None, not become empty" + ); + // `Email/get` returns a snippet, never the octets, so there is nothing + // honest to hash on this path. + assert_eq!(emails[0].content_hash, None); + } + #[test] fn parse_invalid_jmap_json_fails() { let engine = JmapEngine::new("acct-1"); diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index 487d09a..706f6ba 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -254,6 +254,8 @@ mod tests { body_plain: Some("Mock body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; mock.add_message(email.clone()); diff --git a/crates/nuncio-mail/src/parser.rs b/crates/nuncio-mail/src/parser.rs index 95e19f7..e2c051e 100644 --- a/crates/nuncio-mail/src/parser.rs +++ b/crates/nuncio-mail/src/parser.rs @@ -95,6 +95,9 @@ impl MimeParserAdapter { let received_at = msg.date().map_or(0, |d| d.to_timestamp()); + let message_id = msg.message_id().and_then(Email::normalize_message_id); + let content_hash = Some(Email::content_hash_of(raw_bytes)); + let body_plain = msg.body_text(0).map(|b| b.to_string()); let body_html = msg.body_html(0).map(|b| b.to_string()); @@ -141,6 +144,8 @@ impl MimeParserAdapter { body_plain, body_html, attachments, + message_id, + content_hash, }) } } @@ -170,6 +175,39 @@ mod tests { ); } + #[test] + fn parse_captures_normalized_message_id_and_content_hash() { + let raw = b"From: Alice <alice@nuncio.mx>\r\n\ + To: Bob <bob@nuncio.mx>\r\n\ + Message-ID: <Abc.123@mail.nuncio.mx>\r\n\ + Subject: Test\r\n\ + \r\n\ + body"; + + let email = + MimeParserAdapter::parse_mime("msg-1", "acct-1", "inbox", raw).expect("parse succeeds"); + assert_eq!(email.message_id.as_deref(), Some("Abc.123@mail.nuncio.mx")); + assert_eq!( + email.content_hash, + Some(nuncio_core::model::Email::content_hash_of(raw)), + "the hash must cover the octets actually parsed" + ); + } + + #[test] + fn parse_reports_no_message_id_when_the_header_is_absent() { + // RFC 5322 3.6.4 makes Message-ID a SHOULD. An absent header must read + // as "none present", never as an invented or empty value. + let raw = b"From: Alice <alice@nuncio.mx>\r\n\ + Subject: No id\r\n\ + \r\n\ + body"; + let email = + MimeParserAdapter::parse_mime("msg-2", "acct-1", "inbox", raw).expect("parse succeeds"); + assert_eq!(email.message_id, None); + assert!(email.content_hash.is_some()); + } + #[test] fn parse_invalid_mime_returns_error() { let raw = b""; diff --git a/crates/nuncio-mail/src/smtp.rs b/crates/nuncio-mail/src/smtp.rs index 894ae6c..25e7a80 100644 --- a/crates/nuncio-mail/src/smtp.rs +++ b/crates/nuncio-mail/src/smtp.rs @@ -368,6 +368,8 @@ mod tests { body_plain: Some("Plaintext status update".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index c22ba55..cbce742 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -512,7 +512,13 @@ impl DatabaseEngine { received_at INTEGER NOT NULL, read_flag INTEGER NOT NULL DEFAULT 0, body_plain TEXT, - body_html TEXT + body_html TEXT, + -- Captured at ingest, not yet read. Both are nullable because + -- neither is always available: `Message-ID` is SHOULD, not + -- MUST (RFC 5322 3.6.4), and a headers-only or envelope-only + -- fetch has no full octets to hash. + message_id TEXT, + content_hash TEXT ); CREATE TABLE IF NOT EXISTS calendar_events ( @@ -751,11 +757,53 @@ impl DatabaseEngine { self.ensure_accounts_tls_mode_columns().await?; self.ensure_accounts_dav_columns().await?; self.ensure_messages_identity_columns().await?; + self.ensure_messages_provenance_columns().await?; self.backfill_message_fts().await?; Ok(()) } + /// Additive migration adding the `message_id` / `content_hash` provenance + /// columns to a pre-existing `messages` table. + /// + /// Both are nullable and there is **no backfill**, deliberately. The values + /// cannot be reconstructed for rows already stored: the raw octets are + /// discarded after parsing and the `Message-ID` header was never persisted, + /// so the only ways to populate history would be to re-download every + /// message or to invent a value. Legacy rows therefore stay `NULL`, which + /// reads as "not captured" rather than as "this message has no + /// `Message-ID`" -- a distinction any future consumer has to respect. + /// + /// A fresh database gets both columns from `CREATE TABLE IF NOT EXISTS` + /// above, so the `PRAGMA table_info` check finds them present and this is a + /// no-op. SQLite has no `ADD COLUMN IF NOT EXISTS`, hence the explicit + /// check, which makes this safe to run on every daemon startup. + async fn ensure_messages_provenance_columns(&self) -> Result<(), DatabaseError> { + let existing_columns: Vec<String> = sqlx::query("PRAGMA table_info(messages)") + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)? + .iter() + .map(|row| row.get::<String, _>("name")) + .collect(); + + for column in ["message_id", "content_hash"] { + if existing_columns.iter().any(|c| c == column) { + continue; + } + sqlx::query(&format!("ALTER TABLE messages ADD COLUMN {column} TEXT")) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + tracing::info!( + column, + "migrated messages table: added message provenance column" + ); + } + + Ok(()) + } + /// Additive, backfill-safe migration that adds the `remote_id` / /// `uid_validity` message-identity columns to a pre-existing `messages` /// table that predated the opaque surrogate id, and installs the @@ -1295,8 +1343,8 @@ impl DatabaseEngine { sqlx::query( r#" INSERT OR REPLACE INTO messages - (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag, body_plain, body_html) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag, body_plain, body_html, message_id, content_hash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "#, ) .bind(&email.id) @@ -1311,6 +1359,8 @@ impl DatabaseEngine { .bind(if email.read { 1i64 } else { 0i64 }) .bind(&enc_plain) .bind(&enc_html) + .bind(&email.message_id) + .bind(&email.content_hash) .execute(&mut *tx) .await .map_err(DatabaseError::Query)?; @@ -1361,9 +1411,11 @@ impl DatabaseEngine { Option<String>, String, String, + Option<String>, + Option<String>, )> = sqlx::query_as( r#" - SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity + SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages WHERE folder_id = ? ORDER BY received_at DESC @@ -1391,6 +1443,8 @@ impl DatabaseEngine { body_html, remote_id, uid_validity, + message_id, + content_hash, )| { let dec_plain = body_plain .map(|p| { @@ -1424,6 +1478,8 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id, + content_hash, }) }, ) @@ -1445,7 +1501,7 @@ impl DatabaseEngine { ) -> Result<(Vec<nuncio_core::model::Email>, Option<(i64, String)>), DatabaseError> { let fetch = page_size.saturating_add(1); let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages WHERE folder_id = ", + "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages WHERE folder_id = ", ); builder.push_bind(folder_id.to_string()); if let Some((ts, id)) = &after { @@ -1474,6 +1530,8 @@ impl DatabaseEngine { Option<String>, String, String, + Option<String>, + Option<String>, )>() .fetch_all(&self.pool) .await @@ -1510,6 +1568,8 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id: r.12, + content_hash: r.13, }) }) .collect::<Result<Vec<_>, DatabaseError>>()?; @@ -1545,9 +1605,11 @@ impl DatabaseEngine { Option<String>, String, String, + Option<String>, + Option<String>, ) = sqlx::query_as( r#" - SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity + SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages WHERE id = ? "#, @@ -1582,6 +1644,8 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id: row.12, + content_hash: row.13, }) } @@ -2435,7 +2499,7 @@ impl DatabaseEngine { limit: usize, ) -> Result<Vec<nuncio_core::model::Email>, DatabaseError> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages " + "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages " ); if !last_id.is_empty() { builder.push("WHERE id > "); @@ -2457,6 +2521,8 @@ impl DatabaseEngine { Option<String>, String, String, + Option<String>, + Option<String>, )>(); let rows = query @@ -2492,6 +2558,8 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id: r.12, + content_hash: r.13, }) }) .collect::<Result<Vec<_>, DatabaseError>>() @@ -2509,7 +2577,7 @@ impl DatabaseEngine { folder_id: Option<&str>, ) -> Result<Vec<nuncio_core::model::Email>, DatabaseError> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages" + "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages" ); let mut has_filter = false; @@ -2541,6 +2609,8 @@ impl DatabaseEngine { Option<String>, String, String, + Option<String>, + Option<String>, )>(); let rows = query @@ -2576,6 +2646,8 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id: r.12, + content_hash: r.13, }) }) .collect::<Result<Vec<_>, DatabaseError>>() @@ -3084,6 +3156,8 @@ mod tests { body_plain: Some("Plaintext content".to_string()), body_html: Some("<p>HTML content</p>".to_string()), attachments: Vec::new(), + message_id: None, + content_hash: None, }; engine @@ -3133,6 +3207,8 @@ mod tests { body_plain: Some("Sensitive plaintext body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; engine .save_email(&email) @@ -3437,6 +3513,8 @@ mod tests { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; engine.save_email(&email).await.expect("save email"); @@ -4129,6 +4207,8 @@ mod tests { body_plain: Some("Hello".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; engine.save_email(&email).await.unwrap(); } @@ -4256,6 +4336,8 @@ mod tests { body_plain: Some("body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } @@ -4368,6 +4450,73 @@ mod tests { .expect("re-running an already-complete migration is a no-op"); } + #[tokio::test] + async fn message_provenance_round_trips_through_every_read_path() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let mut email = synced_email("acct-1", "INBOX", "42", "7"); + email.message_id = Some("abc123@mail.nuncio.mx".to_string()); + email.content_hash = Some(nuncio_core::model::Email::content_hash_of(b"raw octets")); + engine.save_email(&email).await.expect("save"); + + let by_id = engine.get_message(&email.id).await.expect("get_message"); + assert_eq!(by_id.message_id, email.message_id); + assert_eq!(by_id.content_hash, email.content_hash); + + let listed = engine.list_messages("INBOX", 10).await.expect("list"); + let found = listed + .iter() + .find(|m| m.id == email.id) + .expect("the saved message is listed"); + assert_eq!(found.message_id, email.message_id); + assert_eq!(found.content_hash, email.content_hash); + + // A message with no Message-ID stays None rather than becoming "". + let plain = synced_email("acct-1", "INBOX", "42", "8"); + assert_eq!(plain.message_id, None); + engine.save_email(&plain).await.expect("save"); + let fetched = engine.get_message(&plain.id).await.expect("get_message"); + assert_eq!(fetched.message_id, None); + assert_eq!(fetched.content_hash, None); + } + + #[tokio::test] + async fn provenance_migration_adds_nullable_columns_without_inventing_history() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Reproduce a pre-migration database: the columns cannot be dropped + // from a live table cheaply, so assert the property that matters -- + // a row written before capture existed reads back as "not captured". + sqlx::query( + "INSERT INTO messages \ + (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag) \ + VALUES ('legacy-1', 'acct-1', 'INBOX', '9', '42', 'subj', 'a@nuncio.mx', 'b@nuncio.mx', 0, 0)", + ) + .execute(engine.pool()) + .await + .unwrap(); + + engine + .ensure_messages_provenance_columns() + .await + .expect("migration is idempotent on an already-migrated database"); + + let legacy = engine.get_message("legacy-1").await.expect("get_message"); + assert_eq!( + legacy.message_id, None, + "a legacy row must read as not-captured, never as a fabricated id" + ); + assert_eq!(legacy.content_hash, None); + + // Running it twice more must stay a no-op. + for _ in 0..2 { + engine + .ensure_messages_provenance_columns() + .await + .expect("re-running is a no-op"); + } + } + fn export_test_email(id: &str, account_id: &str, folder_id: &str) -> nuncio_core::model::Email { nuncio_core::model::Email { id: id.to_string(), @@ -4383,6 +4532,8 @@ mod tests { body_plain: Some(format!("Body {id}")), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nuncio-store/src/search.rs b/crates/nuncio-store/src/search.rs index 92d63e1..36c6ac6 100644 --- a/crates/nuncio-store/src/search.rs +++ b/crates/nuncio-store/src/search.rs @@ -266,6 +266,8 @@ mod tests { body_plain: Some(body_plain.to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 8d387bf..0cb2ee9 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -1989,6 +1989,8 @@ fn synthetic_preview_email(id: &str) -> nuncio_core::model::Email { body_plain: Some("Sample body text".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } @@ -3303,6 +3305,8 @@ mod tests { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, remote_id: "remote-1".to_string(), uid_validity: "1".to_string(), }; @@ -4498,6 +4502,8 @@ mod tests { body_plain: Some(body.to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } @@ -5552,6 +5558,8 @@ mod tests { body_plain: Some("body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; // Before creating anything, the live engine matches nothing. @@ -5663,6 +5671,8 @@ mod tests { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }) .is_empty()); } @@ -5813,6 +5823,8 @@ mod tests { body_plain: Some("Please review the attached deck".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }) .await .expect("seed message"); @@ -5987,6 +5999,8 @@ mod tests { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }); assert_eq!(matches.len(), 1); assert_eq!(matches[0].0.id, rule_id); @@ -6168,6 +6182,8 @@ mod tests { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }); assert_eq!(matches.len(), 1); } diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 7de867e..a23dcd1 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -601,6 +601,8 @@ mod tests { body_plain: Some("secret outbox body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index 0c5036a..79e3fac 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -550,6 +550,8 @@ mod tests { body_plain: Some(format!("body for {id}")), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/tests/filters_triage_e2e_test.rs b/crates/nunciod/tests/filters_triage_e2e_test.rs index 60689d3..9fa2b36 100644 --- a/crates/nunciod/tests/filters_triage_e2e_test.rs +++ b/crates/nunciod/tests/filters_triage_e2e_test.rs @@ -53,6 +53,8 @@ fn sample_email(id: &str, subject: &str) -> Email { body_plain: Some("body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/tests/no_secrets_canary_test.rs b/crates/nunciod/tests/no_secrets_canary_test.rs index 3113464..e4150a9 100644 --- a/crates/nunciod/tests/no_secrets_canary_test.rs +++ b/crates/nunciod/tests/no_secrets_canary_test.rs @@ -150,6 +150,8 @@ fn sentinel_inbound_email() -> Email { body_plain: Some(SENTINEL_BODY.to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index 5fccdcd..933f84e 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -127,6 +127,8 @@ fn sample_email(subject: &str) -> Email { body_plain: Some("original body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index b15c2ae..93e28e1 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -88,6 +88,8 @@ fn mock_inbound_email(id: &str, subject: &str, body: &str) -> Email { body_plain: Some(body.to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } -- 2.54.0 From ae8815263abe6a66b4d524af0f62b74277b9ac88 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Mon, 10 Aug 2026 14:57:58 -0500 Subject: [PATCH 54/95] fix(filter): make filter execution single-owner per account, default off Every daemon syncing an account decides for itself that a message is new, because "new" means "not in this daemon's store". So every daemon fires the same rules against the same message. MOVE, COPY and FLAG survive that -- ten moves converge on one outcome -- but FORWARD and CALL WEBHOOK land on a third party with no shared state to compare against. Ten daemons send ten forwards and page an on-call engineer ten times per alert. Nothing in the outbox constrains this either: pending_remote_mutations has no uniqueness of any kind. The engine cannot fix this by itself. The servers coordinate state, not actions, so there is no resource to compare-and-set against for an effect that leaves the system. Until a server-side claim is available, ownership has to be declared: filters_enabled on the account, and exactly one daemon per account turns it on. Default off, including for accounts that already exist. An account already configured on several machines would otherwise have all of them start forwarding the moment they upgrade. Off costs one setting to correct; a sent forward cannot be recalled. The salvage path deliberately does not recover the flag either, for the same reason. A non-owning daemon still syncs and stores everything -- it is a full mail client, it simply does not act. Triage is bound by the same rule since it produces the identical side effects, and refuses outright when this daemon owns no account rather than streaming a scan that reports progress and applies nothing, which would read as "no rules matched". Webhooks now carry an Idempotency-Key stable across retries of a rule/message pair, so a redelivery caused by a lost response can be discarded rather than acted on twice. It is not yet stable across daemons -- rule ids are random per daemon -- which single-owner execution is what covers today. Refs #389 --- crates/nuncio-core/src/config.rs | 18 +++ crates/nuncio-filter/src/webhook.rs | 17 +++ crates/nuncio-store/src/db.rs | 64 ++++++++- crates/nuncio-store/src/recovery.rs | 10 ++ crates/nunciod/src/grpc.rs | 63 +++++++- crates/nunciod/src/orchestrator.rs | 1 + crates/nunciod/src/send.rs | 1 + crates/nunciod/src/sync.rs | 134 ++++++++++++++++-- crates/nunciod/src/sync_dispatcher.rs | 1 + .../nunciod/tests/filters_triage_e2e_test.rs | 78 ++++++++++ .../nunciod/tests/outbox_executor_e2e_test.rs | 1 + 11 files changed, 369 insertions(+), 19 deletions(-) diff --git a/crates/nuncio-core/src/config.rs b/crates/nuncio-core/src/config.rs index 8c5fc99..245ad11 100644 --- a/crates/nuncio-core/src/config.rs +++ b/crates/nuncio-core/src/config.rs @@ -154,6 +154,21 @@ pub struct AccountConfig { pub keyring_secret_key: String, /// Background sync polling interval in seconds (minimum 10s). pub sync_interval_secs: u64, + /// Whether this daemon executes filter rules for this account. + /// + /// **Defaults to `false`**, which is the conservative answer to a problem + /// the engine cannot solve on its own. Every daemon syncing an account + /// decides independently that a message is new, so every daemon fires the + /// same rules against it. `MOVE`/`COPY`/`FLAG` survive that -- ten moves + /// converge on one outcome -- but `FORWARD` and `CALL WEBHOOK` land on a + /// third party with no shared state to compare against, so N daemons send + /// N forwards and N webhook calls. + /// + /// Enabling this on exactly one daemon per account makes filter execution + /// single-owner. It is a deliberate, documented limitation rather than a + /// silent duplication bug; a server-side claim is the eventual fix where + /// the server supports one. + pub filters_enabled: bool, /// The account's single transport. pub transport: Transport, } @@ -343,6 +358,7 @@ mod tests { email_address: "user@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-123".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: Transport::ImapSmtp(valid_imap_smtp_transport()), } } @@ -354,6 +370,7 @@ mod tests { email_address: "user@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-jmap-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: Transport::Jmap(JmapTransport { endpoint_host: "jmap.nuncio.mx".to_string(), }), @@ -367,6 +384,7 @@ mod tests { email_address: "user@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-cal-1".to_string(), sync_interval_secs: 300, + filters_enabled: false, transport: Transport::Dav(DavTransport { collection_url: "https://caldav.example.com/calendars/user/work/".to_string(), }), diff --git a/crates/nuncio-filter/src/webhook.rs b/crates/nuncio-filter/src/webhook.rs index 123f82e..165ba0d 100644 --- a/crates/nuncio-filter/src/webhook.rs +++ b/crates/nuncio-filter/src/webhook.rs @@ -189,9 +189,26 @@ impl WebhookDispatcher { mac.update(format!("{timestamp}.{payload_str}").as_bytes()); let signature = hex::encode(mac.finalize().into_bytes()); + // Stable across every retry of this rule/message pair, so a + // receiver can discard a redelivery caused by a lost response + // rather than acting twice. It is NOT yet stable across daemons: + // `rule_id` is random per daemon, so two daemons executing the + // same rule produce different keys. Single-owner filter execution + // is what prevents that case today; a rule key derived from the + // normalized NSQL would make it dedupable directly. + let idempotency_key = { + let mut hasher = <Sha256 as sha2::Digest>::new(); + for field in [rule_id, message_id] { + sha2::Digest::update(&mut hasher, (field.len() as u64).to_le_bytes()); + sha2::Digest::update(&mut hasher, field.as_bytes()); + } + hex::encode(sha2::Digest::finalize(hasher)) + }; + let send_result = client .post(url) .header("Content-Type", "application/json") + .header("Idempotency-Key", &idempotency_key) .header( "X-Nuncio-Signature", format!("t={timestamp},v1={signature}"), diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index c22ba55..30d64b3 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -543,7 +543,13 @@ impl DatabaseEngine { smtp_port INTEGER, imap_tls_mode TEXT NOT NULL DEFAULT 'implicit_tls', smtp_tls_mode TEXT NOT NULL DEFAULT 'implicit_tls', - collection_url TEXT + collection_url TEXT, + -- Defaults to 0 (off), including for accounts written before + -- the column existed. Filter execution is not safe to run on + -- more than one daemon per account, and defaulting a shared + -- account to "every daemon forwards" would be the wrong way + -- to be wrong. + filters_enabled INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE IF NOT EXISTS filter_rules ( @@ -750,12 +756,49 @@ impl DatabaseEngine { self.ensure_accounts_smtp_columns().await?; self.ensure_accounts_tls_mode_columns().await?; self.ensure_accounts_dav_columns().await?; + self.ensure_accounts_filters_enabled_column().await?; self.ensure_messages_identity_columns().await?; self.backfill_message_fts().await?; Ok(()) } + /// Additive migration adding the `filters_enabled` column to a + /// pre-existing `accounts` table. + /// + /// Defaults to `0` for every existing row. That is the deliberate choice: + /// an account already configured on several machines would otherwise have + /// each of them start executing its rules at once, and the observable + /// result is duplicated forwards and duplicated webhook calls to third + /// parties. Turning it back on is one setting; unsending a forward is not + /// possible. + /// + /// SQLite has no `ADD COLUMN IF NOT EXISTS`, so presence is checked first, + /// making this safe to run on every daemon startup. + async fn ensure_accounts_filters_enabled_column(&self) -> Result<(), DatabaseError> { + let existing_columns: Vec<String> = sqlx::query("PRAGMA table_info(accounts)") + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)? + .iter() + .map(|row| row.get::<String, _>("name")) + .collect(); + + if existing_columns.iter().any(|c| c == "filters_enabled") { + return Ok(()); + } + + sqlx::query("ALTER TABLE accounts ADD COLUMN filters_enabled INTEGER NOT NULL DEFAULT 0") + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + tracing::info!( + "migrated accounts table: added filters_enabled (defaulting existing accounts to off)" + ); + + Ok(()) + } + /// Additive, backfill-safe migration that adds the `remote_id` / /// `uid_validity` message-identity columns to a pre-existing `messages` /// table that predated the opaque surrogate id, and installs the @@ -1071,8 +1114,8 @@ impl DatabaseEngine { sqlx::query( r#" INSERT OR REPLACE INTO accounts - (id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url, filters_enabled) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "#, ) .bind(&config.id) @@ -1095,6 +1138,7 @@ impl DatabaseEngine { .bind(tls_mode_to_db(imap_tls_mode)) .bind(tls_mode_to_db(smtp_tls_mode)) .bind(&collection_url) + .bind(i64::from(config.filters_enabled)) .execute(&self.pool) .await .map_err(DatabaseError::Query)?; @@ -1140,9 +1184,10 @@ impl DatabaseEngine { String, String, Option<String>, + i64, )> = sqlx::query_as( r#" - SELECT id, name, email_address, protocol, server_host, server_port, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url + SELECT id, name, email_address, protocol, server_host, server_port, keyring_secret_key, sync_interval_secs, smtp_host, smtp_port, imap_tls_mode, smtp_tls_mode, collection_url, filters_enabled FROM accounts "#, ) @@ -1167,6 +1212,7 @@ impl DatabaseEngine { imap_tls_mode, smtp_tls_mode, collection_url, + filters_enabled, )| { let protocol = serde_json::from_str(&protocol_str) .unwrap_or(nuncio_core::AccountProtocol::ImapSmtp); @@ -1213,6 +1259,7 @@ impl DatabaseEngine { email_address, keyring_secret_key, sync_interval_secs: sync_interval_secs as u64, + filters_enabled: filters_enabled != 0, transport, } }, @@ -2901,6 +2948,7 @@ mod tests { email_address: "contended@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-contended-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -3480,6 +3528,7 @@ mod tests { email_address: "work@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-test-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -3524,6 +3573,7 @@ mod tests { email_address: "mixed@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-tls-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 143, @@ -3566,6 +3616,7 @@ mod tests { email_address: "cal@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-caldav-store-1".to_string(), sync_interval_secs: 300, + filters_enabled: false, transport: nuncio_core::Transport::Dav(nuncio_core::DavTransport { collection_url: "https://dav.example.com/calendars/user/work/".to_string(), }), @@ -3598,6 +3649,7 @@ mod tests { email_address: "imap@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-mix-imap".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 143, @@ -3613,6 +3665,7 @@ mod tests { email_address: "jmap@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-mix-jmap".to_string(), sync_interval_secs: 120, + filters_enabled: false, transport: nuncio_core::Transport::Jmap(nuncio_core::JmapTransport { endpoint_host: "jmap.nuncio.mx".to_string(), }), @@ -3623,6 +3676,7 @@ mod tests { email_address: "dav@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-mix-dav".to_string(), sync_interval_secs: 300, + filters_enabled: false, transport: nuncio_core::Transport::Dav(nuncio_core::DavTransport { collection_url: "https://dav.example.com/cal/".to_string(), }), @@ -3662,6 +3716,7 @@ mod tests { email_address: "del@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-del-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -3854,6 +3909,7 @@ mod tests { email_address: "post@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-post-migration".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, diff --git a/crates/nuncio-store/src/recovery.rs b/crates/nuncio-store/src/recovery.rs index 177793b..64d82a2 100644 --- a/crates/nuncio-store/src/recovery.rs +++ b/crates/nuncio-store/src/recovery.rs @@ -453,6 +453,13 @@ impl SqliteRecoveryEngine { email_address, keyring_secret_key, sync_interval_secs: sync_interval_secs as u64, + // Salvage deliberately does not recover this flag. + // The column may be among what the corruption took, + // and re-enabling filter execution on a guess would + // start forwarding mail from a daemon the user never + // chose as the owner. Off is recoverable by one + // setting; wrongly on is not recoverable at all. + filters_enabled: false, transport, } }, @@ -618,6 +625,7 @@ mod tests { email_address: "work@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-test-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -679,6 +687,7 @@ mod tests { email_address: "salvage@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-salvage-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -816,6 +825,7 @@ mod tests { email_address: "starttls@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-starttls-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 143, diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 8d387bf..14425d0 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -490,6 +490,7 @@ fn map_account_config_from_proto( email_address: config.email_address, keyring_secret_key: config.keyring_secret_key, sync_interval_secs: nuncio_proto::time::duration_to_secs(&sync_interval), + filters_enabled: false, transport, }) } @@ -1879,12 +1880,29 @@ impl Mail for MailGrpcService { } let synced = if let Some(backend) = &self.overrides.mail_backend { + // Filter execution is owned by whichever daemon the account opts + // in (see `AccountConfig::filters_enabled`). Resolve it from the + // stored account rather than assuming: an injected backend must + // take the same path production does, or the gate would only ever + // be exercised in production. + let filters_enabled = match &account_id { + Some(id) => self + .db + .list_accounts() + .await + .map_err(|e| Status::internal(format!("sync failed: {e}")))? + .into_iter() + .find(|a| &a.id == id) + .is_some_and(|a| a.filters_enabled), + None => false, + }; crate::sync::sync_with_backend( &self.db, &self.event_bus, backend.as_ref(), &self.filter_engine, account_id, + filters_enabled, ) .await .map_err(|e| Status::internal(format!("sync failed: {e}")))? @@ -2388,7 +2406,34 @@ impl Filters for FiltersGrpcService { req.chunk_size as usize }; - tracing::info!(chunk_size, "Filters: triage rescan started"); + // Triage produces exactly the side effects a live sync produces, so it + // is bound by the same single-owner rule: only accounts this daemon + // owns filter execution for may be acted on. Resolved once here rather + // than per message. + let owning_accounts: std::collections::HashSet<String> = self + .db + .list_accounts() + .await + .map_err(|e| Status::internal(format!("failed to read accounts for triage: {e}")))? + .into_iter() + .filter(|a| a.filters_enabled) + .map(|a| a.id) + .collect(); + + if owning_accounts.is_empty() { + // An honest refusal beats streaming a scan that reports progress + // and applies nothing, which would read as "no rules matched". + return Err(Status::failed_precondition( + "no account on this daemon has filter execution enabled; \ + enable filters_enabled on exactly one daemon per account", + )); + } + + tracing::info!( + chunk_size, + owning_accounts = owning_accounts.len(), + "Filters: triage rescan started" + ); let (tx, rx) = tokio::sync::mpsc::channel::<Result<TriageProgress, Status>>(16); let db = self.db.clone(); @@ -2419,8 +2464,14 @@ impl Filters for FiltersGrpcService { let is_last_page = batch.len() < chunk_size; for email in &batch { - let actions_for_email = - crate::sync::apply_filter_actions(&db, &engine, email).await; + // Messages belonging to an account this daemon does not + // own are counted as scanned but never acted on, so the + // progress total still reflects the whole store. + let actions_for_email = if owning_accounts.contains(&email.account_id) { + crate::sync::apply_filter_actions(&db, &engine, email).await + } else { + 0 + }; scanned += 1; applied += actions_for_email as u64; if actions_for_email > 0 { @@ -3644,6 +3695,7 @@ mod tests { email_address: format!("{id}@nuncio.mx"), keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::Jmap(nuncio_core::JmapTransport { endpoint_host: "jmap.nuncio.mx".to_string(), }), @@ -4405,6 +4457,7 @@ mod tests { email_address: "rotate@nuncio.mx".to_string(), keyring_secret_key: key.to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -4448,6 +4501,7 @@ mod tests { email_address: "rotate2@nuncio.mx".to_string(), keyring_secret_key: key.to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -5062,6 +5116,7 @@ mod tests { email_address: "sender@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-send-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -5230,6 +5285,7 @@ mod tests { email_address: "sender@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-injected-send-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -5346,6 +5402,7 @@ mod tests { email_address: "sender@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-injected-send-fail-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, diff --git a/crates/nunciod/src/orchestrator.rs b/crates/nunciod/src/orchestrator.rs index 24af9aa..9b5bf5d 100644 --- a/crates/nunciod/src/orchestrator.rs +++ b/crates/nunciod/src/orchestrator.rs @@ -154,6 +154,7 @@ mod tests { email_address: "orch@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-orch-1".to_string(), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, diff --git a/crates/nunciod/src/send.rs b/crates/nunciod/src/send.rs index de07d15..27a24d2 100644 --- a/crates/nunciod/src/send.rs +++ b/crates/nunciod/src/send.rs @@ -292,6 +292,7 @@ mod tests { email_address: format!("{id}@nuncio.mx"), keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index 0c5036a..514158b 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -232,6 +232,7 @@ async fn fetch_and_persist( backend: &dyn MailBackend, filter_engine: &FilterEngine, account_id: Option<&str>, + filters_enabled: bool, ) -> Result<usize, SyncError> { let folders = backend.sync_folders().await?; let mut synced = 0usize; @@ -263,7 +264,12 @@ async fn fetch_and_persist( let is_new = !already_present.contains(&email.id) && seen_this_pass.insert(email.id.clone()); db.save_email(&email).await?; - if is_new { + // Filter execution is single-owner per account (see + // `AccountConfig::filters_enabled`). A daemon that is not the owner + // still syncs and stores the message; it just does not act on it, + // because the side-effecting actions -- FORWARD, CALL WEBHOOK -- + // are not idempotent across daemons. + if is_new && filters_enabled { apply_filter_actions(db, filter_engine, &email).await; } synced += 1; @@ -305,6 +311,7 @@ pub async fn sync_with_backend( backend: &dyn MailBackend, filter_engine: &FilterEngine, account_id: Option<String>, + filters_enabled: bool, ) -> Result<usize, SyncError> { match &account_id { Some(id) => event_bus.process_command(CoreCommand::SyncAccount { @@ -313,8 +320,15 @@ pub async fn sync_with_backend( None => event_bus.process_command(CoreCommand::SyncAll), } - let result = - fetch_and_persist(db, event_bus, backend, filter_engine, account_id.as_deref()).await; + let result = fetch_and_persist( + db, + event_bus, + backend, + filter_engine, + account_id.as_deref(), + filters_enabled, + ) + .await; if let Err(e) = &result { event_bus.process_command(CoreCommand::ReportError { @@ -386,18 +400,20 @@ pub async fn run_account_sync( let setup = async { let config = find_account(db, account_id).await?; let password = secrets.get_secret(&config.keyring_secret_key)?; - build_mail_backend(&config, &password) + let filters_enabled = config.filters_enabled; + build_mail_backend(&config, &password).map(|backend| (backend, filters_enabled)) } .await; match setup { - Ok(backend) => { + Ok((backend, filters_enabled)) => { sync_with_backend( db, event_bus, backend.as_ref(), filter_engine, Some(account_id.to_string()), + filters_enabled, ) .await } @@ -438,6 +454,7 @@ mod tests { email_address: format!("{id}@nuncio.mx"), keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, + filters_enabled: false, transport: Transport::Jmap(nuncio_core::JmapTransport { endpoint_host: host.to_string(), }), @@ -524,6 +541,7 @@ mod tests { email_address: format!("{id}@nuncio.mx"), keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -578,6 +596,7 @@ mod tests { &mock, &filter_engine, Some("acct-mock-1".to_string()), + true, ) .await .expect("sync succeeds"); @@ -634,7 +653,7 @@ mod tests { }); let filter_engine = empty_filter_engine(); - let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) + let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None, true) .await .expect("sync succeeds"); assert_eq!(synced, 0); @@ -677,6 +696,7 @@ mod tests { &mock, &filter_engine, Some("acct-fail".to_string()), + true, ) .await .expect_err("sync fails"); @@ -896,7 +916,7 @@ mod tests { }); mock.add_message(mock_email("m-urgent", "inbox", "Urgent: server down")); - let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) + let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None, true) .await .expect("sync succeeds"); assert_eq!(synced, 1); @@ -924,6 +944,94 @@ mod tests { ); } + #[tokio::test] + async fn a_daemon_that_does_not_own_filters_stores_the_mail_but_fires_nothing() { + // The single-owner rule. A non-owning daemon must still sync and store + // -- it is a full mail client -- while producing none of the side + // effects that would be duplicated across the fleet. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let event_bus = EventBus::new(); + + let rule = nuncio_filter::NsqlParser::parse_rule( + "Urgent Auto-Read", + 1, + "WHERE subject CONTAINS 'Urgent' ACTION MARK READ", + ) + .expect("parse rule"); + let filter_engine = FilterEngine::new(vec![rule]).expect("compile rule"); + + let mock = MockMailBackend::new(); + mock.add_folder(Folder { + id: "inbox".to_string(), + name: "Inbox".to_string(), + total_messages: 1, + unread_messages: 1, + }); + mock.add_message(mock_email("m-urgent", "inbox", "Urgent: server down")); + + let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None, false) + .await + .expect("sync succeeds"); + + assert_eq!(synced, 1, "the message must still be synced and stored"); + let stored = db.get_message("m-urgent").await.expect("message persisted"); + assert!(!stored.read, "a non-owning daemon must not apply MARK READ"); + assert!( + db.list_filter_execution_logs(10) + .await + .expect("list logs") + .is_empty(), + "no rule ran, so nothing may be logged as executed" + ); + assert!( + db.list_pending_mutations(10) + .await + .expect("list mutations") + .is_empty(), + "a non-owning daemon must enqueue no remote mutation" + ); + } + + #[tokio::test] + async fn run_account_sync_takes_filter_ownership_from_the_stored_account() { + // The flag has to come from the account row, not a caller's guess -- + // otherwise the gate is only ever exercised by tests that pass it. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + + let mut account = sample_imap_account("acct-owner"); + assert!( + !account.filters_enabled, + "a freshly constructed account must default to not owning filters" + ); + db.save_account(&account).await.expect("save account"); + let reloaded = db + .list_accounts() + .await + .expect("list accounts") + .into_iter() + .find(|a| a.id == "acct-owner") + .expect("account round-trips"); + assert!(!reloaded.filters_enabled); + + account.filters_enabled = true; + db.save_account(&account).await.expect("update account"); + let reloaded = db + .list_accounts() + .await + .expect("list accounts") + .into_iter() + .find(|a| a.id == "acct-owner") + .expect("account round-trips"); + assert!( + reloaded.filters_enabled, + "opting one daemon in must persist" + ); + } + #[tokio::test] async fn sync_with_backend_enqueues_outbox_mutation_for_remote_action_on_match() { let (db, _dir) = DatabaseEngine::connect_ephemeral() @@ -948,7 +1056,7 @@ mod tests { }); mock.add_message(mock_email("m-archive", "inbox", "Please Archive Me")); - let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) + let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None, true) .await .expect("sync succeeds"); assert_eq!(synced, 1); @@ -1002,7 +1110,7 @@ mod tests { }); mock.add_message(mock_email("m-plain", "inbox", "Just a normal update")); - let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) + let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None, true) .await .expect("sync succeeds"); assert_eq!(synced, 1); @@ -1062,7 +1170,7 @@ mod tests { mock.add_message(mock_email("m-repeat", "inbox", "Urgent: server down")); // First sync: the message is genuinely new, so both rules must fire. - let synced_first = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) + let synced_first = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None, true) .await .expect("first sync succeeds"); assert_eq!(synced_first, 1); @@ -1094,7 +1202,7 @@ mod tests { // Second sync: the mock backend reports the SAME message again // (unchanged id), simulating a non-incremental re-sync. - let synced_second = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) + let synced_second = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None, true) .await .expect("second sync succeeds"); assert_eq!(synced_second, 1); @@ -1171,7 +1279,7 @@ mod tests { mock.add_message(mock_email("m-new-1", "inbox", "Urgent: brand new")); mock.add_message(mock_email("m-new-2", "inbox", "Urgent: also new")); - let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) + let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None, true) .await .expect("sync succeeds"); assert_eq!(synced, 3, "every fetched message is still persisted"); @@ -1235,6 +1343,7 @@ mod tests { &mock, &filter_engine, Some("acct-inc-1".to_string()), + true, ) .await .expect("first sync succeeds"); @@ -1256,6 +1365,7 @@ mod tests { &mock, &filter_engine, Some("acct-inc-1".to_string()), + true, ) .await .expect("second sync succeeds"); diff --git a/crates/nunciod/src/sync_dispatcher.rs b/crates/nunciod/src/sync_dispatcher.rs index 4526da0..52bdd6c 100644 --- a/crates/nunciod/src/sync_dispatcher.rs +++ b/crates/nunciod/src/sync_dispatcher.rs @@ -806,6 +806,7 @@ mod tests { email_address: format!("{id}@nuncio.mx"), keyring_secret_key: format!("nuncio/{id}"), sync_interval_secs: 60, + filters_enabled: false, transport: Transport::Jmap(JmapTransport { endpoint_host: "jmap.nuncio.mx".to_string(), }), diff --git a/crates/nunciod/tests/filters_triage_e2e_test.rs b/crates/nunciod/tests/filters_triage_e2e_test.rs index 60689d3..48d1038 100644 --- a/crates/nunciod/tests/filters_triage_e2e_test.rs +++ b/crates/nunciod/tests/filters_triage_e2e_test.rs @@ -56,6 +56,27 @@ fn sample_email(id: &str, subject: &str) -> Email { } } +/// The account the seeded messages belong to, opted in as this daemon's +/// filter-execution owner. +fn owning_account() -> nuncio_core::AccountConfig { + nuncio_core::AccountConfig { + id: ACCOUNT_ID.to_string(), + name: "Triage E2E".to_string(), + email_address: "triage-e2e@nuncio.mx".to_string(), + keyring_secret_key: format!("nuncio/{ACCOUNT_ID}"), + sync_interval_secs: 60, + filters_enabled: true, + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), + } +} + /// Boots the real daemon gRPC server (bearer-token authenticated) on an /// ephemeral loopback port, backed by a real temp file-backed /// `DatabaseEngine` and the given live `FilterEngine`. Returns the server @@ -79,6 +100,13 @@ async fn boot_daemon( .expect("open temp file-backed database"), ); + // Triage only acts on accounts this daemon owns filter execution for, so + // the account must exist and be opted in -- otherwise the RPC honestly + // refuses rather than scanning and applying nothing. + db.save_account(&owning_account()) + .await + .expect("seed the triage account"); + let token_bytes = secrets .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) .expect("mint gRPC bearer token from mock vault"); @@ -267,3 +295,53 @@ async fn triage_streams_multiple_chunks_for_a_store_bigger_than_one_chunk() { prev_scanned = update.scanned_count; } } + +/// A daemon that owns filter execution for no account must refuse `Triage` +/// outright. Streaming a scan that reports progress and applies nothing would +/// be indistinguishable, from the caller's side, from "no rules matched" -- +/// and the whole point of single-owner execution is that the non-owner stays +/// visibly inert rather than quietly so. +#[tokio::test] +async fn triage_refuses_when_this_daemon_owns_no_account() { + let secrets = Arc::new(SecretManager::mock()); + let rule = NsqlParser::parse_rule( + "Urgent Auto-Read", + 1, + "WHERE subject CONTAINS 'Urgent' ACTION MARK READ", + ) + .expect("parse rule"); + let filter_engine = Arc::new(FilterEngine::new(vec![rule]).expect("compile rule")); + + let (addr, token, db, _dir) = boot_daemon(secrets, filter_engine).await; + + // Take ownership away: the account exists, but this daemon is not its + // filter owner. + let mut account = owning_account(); + account.filters_enabled = false; + db.save_account(&account) + .await + .expect("opt the account out"); + + db.save_email(&sample_email("m-1", "Urgent: server down")) + .await + .expect("seed a matching message"); + + let mut client = FiltersClient::connect(format!("http://{addr}")) + .await + .expect("connect to the daemon"); + let status = client + .triage(authed_request(&token, TriageRequest::default())) + .await + .expect_err("triage must refuse without an owning account"); + + assert_eq!(status.code(), tonic::Code::FailedPrecondition); + assert!( + status.message().contains("filter execution"), + "the refusal must say why, got: {}", + status.message() + ); + + // And it must genuinely have applied nothing. + let stored = db.get_message("m-1").await.expect("message persisted"); + assert!(!stored.read, "a refused triage must not apply any action"); +} diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index 5fccdcd..7388dd6 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -101,6 +101,7 @@ fn sample_account() -> nuncio_core::AccountConfig { email_address: "owner@nuncio.mx".to_string(), keyring_secret_key: format!("nuncio/{ACCOUNT_ID}"), sync_interval_secs: 60, + filters_enabled: false, transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, -- 2.54.0 From c6eca910c43e28d11c6f65ee3f9205a4cac4f27a Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Mon, 10 Aug 2026 15:29:47 -0500 Subject: [PATCH 55/95] docs(adr): decide the pre-release posture and drop the migration compromise The first draft reasoned like a shipped product: preserve every stored row, never change behaviour without a path, keep old and new daemons interoperable. Nothing is released, so none of that is owed -- and paying for it distorted the model in one place that matters. Message identity cannot be derived for rows already stored, because Message-ID was never persisted and the octets are discarded after parse. Honouring those rows meant keeping them under their old surrogate forever: permanently un-deduped, invisible to every cross-folder guarantee the design exists to provide, and leaving the store two-tier for as long as the install survives. Resetting the message store and re-syncing from the server -- which is authoritative regardless -- costs a developer one sync and buys a single coherent model. The audit ledger follows the same reasoning rather than gaining a permanent namespace boundary: a hash chain whose entries reference ids that no longer resolve has no audit value, so it is reinitialised with the store it describes. This also removes the dependency that shaped the build order. Placements waited on captured identity accumulating only because history had to be preserved; now capture need only land one merge ahead of the model consuming it. The order is recorded explicitly, along with the note that change enumeration still ships alone and still fixes ghost messages for a single engine. The posture section is written to be deleted at the first real release, when every one of these questions becomes a genuine obligation. --- docs/adr/0002-multi-engine-sync-model.md | 77 +++++++++++++++++++++--- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/docs/adr/0002-multi-engine-sync-model.md b/docs/adr/0002-multi-engine-sync-model.md index 04a8cc4..2e45fa1 100644 --- a/docs/adr/0002-multi-engine-sync-model.md +++ b/docs/adr/0002-multi-engine-sync-model.md @@ -37,6 +37,30 @@ codebase fit, and adversarially. The reviews falsified four load-bearing claims in the draft. This ADR records the corrected decision; the evidence is in [`docs/reviews/2026-08-10-multi-engine-design-review.md`](../reviews/2026-08-10-multi-engine-design-review.md). +## Pre-release posture + +Nothing is released. There is no installed base, no data anyone is entitled to +keep, and no compatibility obligation. **Getting this model right outranks +continuity of any store that exists today**, and the decisions below are made on +that basis. + +Concretely, and in contrast to how a shipped product would have to reason: + +- **A local store is disposable.** Where a correct model needs data that cannot + be reconstructed from what is persisted, the answer is to reset the store and + re-sync from the server -- never to carry a degraded identity forward forever + so that old rows survive. A permanently un-deduped subset of history is a + worse outcome than a one-time re-sync on a developer's machine. +- **Behaviour may change without a deprecation path.** A default that is wrong + gets corrected, not preserved behind a flag. +- **Mixed-version fleets are not a supported configuration.** Upgrade all + daemons for an account together. `PRAGMA user_version` still earns its place + as cheap insurance against a stale binary silently writing an old model, but + the design owes nothing to interoperating with an older one. + +This section exists to be **deleted at the first real release**, at which point +every migration question below becomes a genuine obligation. + ## Decision ### 1. The servers are the coordinator. We do not build a distributed system. @@ -191,14 +215,21 @@ work is chunked per folder so it yields to user actions. Microsoft 365 mandate OAuth; providers that rotate refresh tokens on use would have ten engines invalidating each other. A token broker is exactly the coordination layer this ADR declines to build. Documented limitation. -- **Migration is forward-only.** `message_key` cannot be derived for existing - rows: Message-ID is never parsed, never stored, and raw bytes are discarded - after parse. Ingest must capture identity first and let it accumulate. Legacy - rows keep their surrogate as `message_key` with one placement each and receive - no retroactive dedup. -- **The audit ledger gains a namespace boundary.** - `filter_execution_logs.message_id` is an input to an HMAC hash chain verified - by a live `Audit.VerifyChain`, so historical ids cannot be rewritten. +- **The store resets rather than migrating into a degraded identity.** + `message_key` cannot be derived for rows already stored: Message-ID is never + parsed, never persisted, and the raw octets are discarded after parse. A + shipped product would have to keep those rows under their old surrogate, + permanently un-deduped and invisible to every cross-folder guarantee the model + provides. Pre-release that trade is not worth making: identity-bearing schema + changes bump `PRAGMA user_version`, the message store is rebuilt, and the + account re-syncs from the server, which is authoritative anyway. Account + configuration and credentials are preserved; cached mail is not. +- **The audit ledger is reset with the store, not namespaced.** + `filter_execution_logs.message_id` feeds an HMAC hash chain verified by a live + `Audit.VerifyChain`, so historical ids genuinely cannot be rewritten in place. + Rather than carry a permanently mixed-namespace ledger, the chain is + reinitialised alongside the message store it refers to. A ledger whose entries + point at ids that no longer resolve has no audit value. - **A new class of test is required.** Convergence cannot be shown by one engine against one mock. Verification needs ≥3 engines against one stateful server, plus side-effect-counting SMTP and webhook mocks — final-state placement @@ -219,3 +250,33 @@ work is chunked per folder so it yields to user actions. respect. - **Freezing `nuncio.v1` first and adding conflict semantics later.** Rejected: not expressible additively. +- **Preserving existing local stores across the identity change.** Rejected for + now — see [Pre-release posture](#pre-release-posture). It buys nothing anyone + is owed and costs a permanently two-tier message store. + +## Build order + +The dependency that originally forced this sequence — placements waiting for +captured identity to accumulate, because history could not be re-keyed — does +not exist once the store is resettable. Capture still has to land before the +model that consumes it, but only by one merge, not by a soak period. + +| # | Work | Depends on | +| --- | --- | --- | +| P0 | Stateful multi-connection IMAP mock harness | — | +| P1 | Namespace-aware DAV XML reader + href/ETag identity | — | +| P2 | Message-ID + content-hash capture at ingest | — | +| P3 | `user_version` gating + scheme-tagged sync tokens | — | +| P4 | Raw IMAP command layer (`ENABLE`, tagged codes, `VANISHED`) | P0 | +| 1 | Message/placement identity split | P2, P3 | +| 2a | IMAP change enumeration, four-rung ladder | P0, P4 | +| 2b | DAV `sync-collection` + multiget + 507 paging | P1 | +| 3a | Three-state mutation outcomes + `Conflict` | P4 | +| 3b | Mutation RPCs + DAV conditional write-back | 1, 3a | +| 4 | Scheduler, session pooling, rate limiting, backoff | 3a (partly) | +| V | ≥3-engine convergence harness with side-effect counting | P0 | + +**2a remains the highest value per unit of work** and still ships independently: +it fixes ghost messages for a single engine, against either schema. + +Filter-execution ownership (§1) is independent of all of it and ships first. -- 2.54.0 From 1ec1e90772ea782df5438705fbd0921b7b2e2e36 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Mon, 10 Aug 2026 15:34:24 -0500 Subject: [PATCH 56/95] feat(store): gate the identity schema and tag sync tokens with their scheme Two silent-failure paths, both cheap to close and both load-bearing for the sync work that follows. The schema is built entirely from CREATE TABLE IF NOT EXISTS, so an older binary opens a newer database successfully and writes the old model into it. Nothing detects that; the file ends up holding two mutually invisible truths. A user_version generation now gates opening: newer than this build is a fail-closed error, and older rebuilds the cached message store. Rebuilding rather than migrating is the pre-release decision recorded in ADR 0002. Message identity cannot be recomputed for rows already stored, so the alternative is keeping them under the old model forever -- a store where half the mail is invisible to every guarantee the new model provides. Accounts, credentials and rules are kept; cached mail, checkpoints and the execution ledger are rebuilt from the server, which is authoritative regardless. The ledger goes with them because its hash chain is keyed on message ids that would no longer resolve, and a chain pointing at absent rows proves nothing. Separately, folder checkpoints now carry a scheme tag. The stored value was "{uidvalidity}:{uidnext}" and the QRESYNC checkpoint that replaces it is "{uidvalidity}:{highestmodseq}" -- identical shape, different meaning. An upgraded engine reading one as the other hands the server a UIDNEXT where it expects a MODSEQ; UIDNEXT is normally the larger number, so the server answers "nothing changed since" and every message below that point is never enumerated again. That is silent, permanent mail loss with no error anywhere. Any token whose scheme this build does not recognise, including an untagged one, now forces a full re-fetch: over-fetching is recoverable, misreading is not. Refs #387 --- crates/nuncio-mail/src/imap.rs | 98 ++++++++++++--- crates/nuncio-store/src/db.rs | 214 +++++++++++++++++++++++++++++++++ 2 files changed, 296 insertions(+), 16 deletions(-) diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index 2de83f7..ca91725 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -86,6 +86,22 @@ fn parse_uid_validity(uid_validity: &str) -> Option<u32> { /// first turns that into a safe full re-fetch instead. const CHECKPOINT_DELIM: char = ':'; +/// Scheme tag prefixing every checkpoint this build writes. +/// +/// The stored value is `"v1:uidnext:{uidvalidity}:{uidnext}"`. The tag exists +/// because the *next* checkpoint scheme -- a QRESYNC `(uidvalidity, +/// highestmodseq)` pair -- has exactly the same shape as this one and a wholly +/// different meaning. Untagged, an upgraded engine would hand a stored UIDNEXT +/// to a server as a MODSEQ. UIDNEXT is normally the larger number, so the +/// server would answer "nothing changed since" and every message below that +/// point would never be enumerated again: silent, permanent mail loss with no +/// error anywhere. +/// +/// Anything that does not carry a scheme this build recognises -- an unknown +/// tag, or a bare `"42:105"` written before tagging existed -- resolves to a +/// full re-fetch. Over-fetching is recoverable; misreading a token is not. +const CHECKPOINT_SCHEME_UIDNEXT: &str = "v1:uidnext"; + /// The `UID FETCH` sequence-set to issue for a folder sync, resolved from the /// stored checkpoint. Kept as a small typed value (rather than collapsing to a /// bare range string) so the distinct fall-back reasons stay observable: all @@ -98,6 +114,11 @@ enum FetchRange { /// A checkpoint was stored but could not be parsed (malformed, or a legacy /// bare-UID checkpoint predating UIDVALIDITY tracking). Falls back to a /// full fetch, surfaced (logged) rather than silently ignored. + /// + /// Also covers a checkpoint whose scheme tag this build does not recognise + /// (see [`CHECKPOINT_SCHEME_UIDNEXT`]) -- including an untagged one, which + /// cannot be distinguished from a future scheme that happens to share its + /// shape. CorruptCheckpoint, /// The stored checkpoint's UIDVALIDITY no longer matches the mailbox's /// current UIDVALIDITY (the mailbox was renumbered) or the server did not @@ -140,7 +161,17 @@ fn resolve_fetch_range(since_state: Option<&str>, current_uid_validity: Option<u let Some(raw) = since_state else { return FetchRange::Full; }; - let parsed = raw + // Require the scheme tag. A value without one is either pre-tagging or from + // a scheme this build cannot interpret; both must re-fetch rather than be + // guessed at. + let Some(body) = raw + .trim() + .strip_prefix(CHECKPOINT_SCHEME_UIDNEXT) + .and_then(|rest| rest.strip_prefix(CHECKPOINT_DELIM)) + else { + return FetchRange::CorruptCheckpoint; + }; + let parsed = body .split_once(CHECKPOINT_DELIM) .and_then(|(validity, uid)| { Some(( @@ -858,7 +889,10 @@ impl ImapEngine { let boundary_uid = server_uid_next .or_else(|| (max_uid_seen > 0).then_some(max_uid_seen.saturating_add(1))); let new_checkpoint = match (server_uid_validity, boundary_uid) { - (Some(validity), Some(uid)) => format!("{}{}{}", validity, CHECKPOINT_DELIM, uid), + (Some(validity), Some(uid)) => format!( + "{}{}{}{}{}", + CHECKPOINT_SCHEME_UIDNEXT, CHECKPOINT_DELIM, validity, CHECKPOINT_DELIM, uid + ), _ => since_state .map(str::to_string) .unwrap_or_else(|| "full-resync-required".to_string()), @@ -1534,15 +1568,15 @@ mod tests { // Valid checkpoint whose UIDVALIDITY still matches: narrowed fetch. assert_eq!( - resolve_fetch_range(Some("1:105"), Some(1)), + resolve_fetch_range(Some("v1:uidnext:1:105"), Some(1)), FetchRange::Incremental(105) ); assert_eq!( - resolve_fetch_range(Some("1:105"), Some(1)).sequence_set(), + resolve_fetch_range(Some("v1:uidnext:1:105"), Some(1)).sequence_set(), "105:*" ); assert_eq!( - resolve_fetch_range(Some(" 7 : 42 "), Some(7)), + resolve_fetch_range(Some("v1:uidnext: 7 : 42 "), Some(7)), FetchRange::Incremental(42) ); @@ -1550,18 +1584,18 @@ mod tests { // to compare against: MUST fall back to a full fetch, distinctly, so a // renumber can never silently skip the low new UIDs. assert_eq!( - resolve_fetch_range(Some("1:105"), Some(2)), + resolve_fetch_range(Some("v1:uidnext:1:105"), Some(2)), FetchRange::UidValidityChanged { stored: 1, current: Some(2) } ); assert_eq!( - resolve_fetch_range(Some("1:105"), Some(2)).sequence_set(), + resolve_fetch_range(Some("v1:uidnext:1:105"), Some(2)).sequence_set(), "1:*" ); assert_eq!( - resolve_fetch_range(Some("1:105"), None), + resolve_fetch_range(Some("v1:uidnext:1:105"), None), FetchRange::UidValidityChanged { stored: 1, current: None @@ -1580,7 +1614,7 @@ mod tests { "a legacy bare-number checkpoint must not be read as a UID boundary" ); assert_eq!( - resolve_fetch_range(Some("1:0"), Some(1)), + resolve_fetch_range(Some("v1:uidnext:1:0"), Some(1)), FetchRange::CorruptCheckpoint ); assert_eq!( @@ -1679,7 +1713,7 @@ mod tests { .await .expect("first sync succeeds"); assert!(first.is_empty()); - assert_eq!(checkpoint, "1:105"); + assert_eq!(checkpoint, "v1:uidnext:1:105"); // Second sync: feed back the checkpoint; UIDVALIDITY still matches // (1), so the fetch is narrowed. @@ -1687,7 +1721,7 @@ mod tests { .sync_folder_messages_with_session("INBOX", Some(&checkpoint), &mut session) .await .expect("second sync succeeds"); - assert_eq!(checkpoint2, "1:105"); + assert_eq!(checkpoint2, "v1:uidnext:1:105"); drop(session); let _ = server.await; @@ -1868,7 +1902,7 @@ mod tests { let (emails, checkpoint, cmds) = run_batched_sync(messages, 1, 6, None, 2).await; assert_eq!(emails.len(), 5, "every message must be synced"); - assert_eq!(checkpoint, "1:6"); + assert_eq!(checkpoint, "v1:uidnext:1:6"); let body_cmds = body_fetch_commands(&cmds); assert_eq!( @@ -1957,7 +1991,7 @@ mod tests { let (first, checkpoint, first_cmds) = run_batched_sync(messages.clone(), 1, 4, None, 2).await; assert_eq!(first.len(), 3); - assert_eq!(checkpoint, "1:4"); + assert_eq!(checkpoint, "v1:uidnext:1:4"); assert!( first_cmds.iter().any(|c| c.contains("1:*")), "first sync enumerates the full range, got: {first_cmds:?}" @@ -1971,7 +2005,7 @@ mod tests { "no new mail arrived, so an incremental sync returns nothing" ); assert_eq!( - checkpoint2, "1:4", + checkpoint2, "v1:uidnext:1:4", "checkpoint stays stable across an empty pass" ); assert!( @@ -2046,7 +2080,7 @@ mod tests { .sync_folder_messages_with_session("INBOX", None, &mut session) .await .expect("first sync succeeds"); - assert_eq!(checkpoint, "1:105"); + assert_eq!(checkpoint, "v1:uidnext:1:105"); // Second sync: server now reports UIDVALIDITY 2. The stored checkpoint // is stale, so the fetch MUST be a full `1:*`, and the checkpoint is @@ -2056,7 +2090,7 @@ mod tests { .await .expect("second sync succeeds"); assert_eq!( - checkpoint2, "2:105", + checkpoint2, "v1:uidnext:2:105", "checkpoint must be rewritten under the new UIDVALIDITY" ); @@ -2146,6 +2180,38 @@ mod tests { assert_eq!(parse_imap_uid("jmap-object-id"), None); } + #[test] + fn an_untagged_or_unknown_scheme_checkpoint_forces_a_full_refetch() { + // The hazard this tag exists for. A QRESYNC checkpoint is + // `(uidvalidity, highestmodseq)` -- the same shape as this scheme's + // `(uidvalidity, uidnext)` and a different meaning. Read one as the + // other and the server is handed a UIDNEXT where it expects a MODSEQ; + // since UIDNEXT is normally the larger number it answers "nothing + // changed" and everything below is never enumerated again. Refusing + // to interpret an unrecognised token is the only safe reading. + for token in [ + // Pre-tagging, written by an older build. + "1:105", + // A scheme from a future build. + "v2:qresync:1:900", + // Tag-shaped but not ours. + "v1:modseq:1:105", + // Truncated tag. + "v1:1:105", + ] { + assert_eq!( + resolve_fetch_range(Some(token), Some(1)), + FetchRange::CorruptCheckpoint, + "token {token:?} must not be interpreted as a UIDNEXT boundary" + ); + assert_eq!( + resolve_fetch_range(Some(token), Some(1)).sequence_set(), + "1:*", + "an uninterpretable token must re-fetch everything" + ); + } + } + #[test] fn parse_uid_validity_accepts_only_a_decimal_value() { assert_eq!(parse_uid_validity("7"), Some(7)); diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index c22ba55..c34829a 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -29,6 +29,22 @@ pub enum DatabaseError { /// Recovery salvage operation failed. #[error("database recovery error: {0}")] RecoveryFailed(String), + /// The database was written by a newer binary than this one. + /// + /// Fail-closed: the schema is built from `CREATE TABLE IF NOT EXISTS`, so + /// an older binary would otherwise open a newer database successfully and + /// write the old model into it, leaving two mutually invisible truths in + /// one file. There is no safe way to continue. + #[error( + "database identity schema v{on_disk} is newer than this build supports (v{supported}); \ + upgrade nunciod rather than running an older binary against it" + )] + SchemaTooNew { + /// The generation recorded in the database's `user_version`. + on_disk: i64, + /// The newest generation this binary understands. + supported: i64, + }, /// Audit chain verification failed. #[error("audit chain integrity error: {0}")] ChainIntegrityFailed(String), @@ -496,6 +512,102 @@ impl DatabaseEngine { &self.pool } + /// The identity-schema generation this binary understands. + /// + /// Distinct from the additive column migrations elsewhere in this module. + /// Those add nullable columns and leave every stored row meaningful. This + /// counter tracks changes to what a stored message *is* -- how it is keyed + /// and what a sync checkpoint means -- where old rows cannot be reinterpreted + /// under the new model at all. + /// + /// Bump this only for a change of that kind, and only together with the + /// reset it implies (see [`DatabaseEngine::reconcile_schema_version`]). + pub const IDENTITY_SCHEMA_VERSION: i64 = 1; + + /// Reconcile the on-disk identity-schema generation with this binary's. + /// + /// Three outcomes: + /// + /// - **Same version** (and the common case): nothing happens. + /// - **Database older**: the cached message store is rebuilt and the account + /// re-syncs from the server. Accounts, credentials and filter rules are + /// preserved; cached mail, sync checkpoints and the execution ledger are + /// not. This is deliberate and pre-release only -- see ADR 0002. Message + /// identity cannot be recomputed for rows already stored, so the + /// alternative is a permanently two-tier store in which old messages are + /// invisible to every guarantee the new model provides. The server holds + /// the authoritative copy either way. + /// - **Database newer**: refuse to open. A `CREATE TABLE IF NOT EXISTS` + /// schema means an older binary can open a newer database and quietly + /// write the old model into it, leaving two mutually invisible truths in + /// one file. Failing loudly is the only way that does not corrupt. + pub async fn reconcile_schema_version(&self) -> Result<(), DatabaseError> { + let (on_disk,): (i64,) = sqlx::query_as("PRAGMA user_version") + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + if on_disk > Self::IDENTITY_SCHEMA_VERSION { + return Err(DatabaseError::SchemaTooNew { + on_disk, + supported: Self::IDENTITY_SCHEMA_VERSION, + }); + } + + // 0 is both "brand new database" and "predates this counter". Neither + // has anything worth preserving that a re-sync will not restore, and + // the reset is a no-op on an empty store. + if on_disk < Self::IDENTITY_SCHEMA_VERSION { + if on_disk > 0 { + tracing::warn!( + on_disk, + supported = Self::IDENTITY_SCHEMA_VERSION, + "message identity schema changed: rebuilding the cached message store; \ + accounts and rules are preserved and mail will re-sync from the server" + ); + } + self.reset_cached_message_store().await?; + sqlx::query(&format!( + "PRAGMA user_version = {}", + Self::IDENTITY_SCHEMA_VERSION + )) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + } + + Ok(()) + } + + /// Drop everything derived from the server, keeping everything the user + /// configured. + /// + /// Cleared: messages and their FTS index, per-folder sync checkpoints, and + /// the filter execution ledger. The ledger goes because its hash chain is + /// keyed on message ids that will no longer resolve, and a chain pointing + /// at absent rows has no audit value -- reinitialising it is more honest + /// than carrying a permanently mixed-namespace log. + /// + /// Kept: accounts, credentials (which live in the OS keyring regardless), + /// filter rules, and the WORM audit records, whose triggers forbid deletion + /// and whose entries do not reference message identity. + async fn reset_cached_message_store(&self) -> Result<(), DatabaseError> { + let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + for statement in [ + "DELETE FROM messages", + "DELETE FROM messages_fts", + "DELETE FROM folder_sync_state", + "DELETE FROM filter_execution_logs", + ] { + sqlx::query(statement) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + } + tx.commit().await.map_err(DatabaseError::Query)?; + Ok(()) + } + /// Execute initial database migrations creating core envelope tables. pub async fn migrate(&self) -> Result<(), DatabaseError> { sqlx::query( @@ -750,6 +862,10 @@ impl DatabaseEngine { self.ensure_accounts_smtp_columns().await?; self.ensure_accounts_tls_mode_columns().await?; self.ensure_accounts_dav_columns().await?; + // Runs after the tables exist (the reset deletes from them) and after + // the additive column migrations, so a rebuild lands on the current + // shape rather than an intermediate one. + self.reconcile_schema_version().await?; self.ensure_messages_identity_columns().await?; self.backfill_message_fts().await?; @@ -4314,6 +4430,104 @@ mod tests { /// the empty-string default -- with the UNIQUE index not yet built. The /// migration must complete that backfill on the next open and build the /// index without a collision, rather than wedging the daemon. + #[tokio::test] + async fn opening_a_newer_database_fails_closed_instead_of_writing_the_old_model() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + sqlx::query(&format!( + "PRAGMA user_version = {}", + DatabaseEngine::IDENTITY_SCHEMA_VERSION + 1 + )) + .execute(engine.pool()) + .await + .unwrap(); + + let err = engine + .reconcile_schema_version() + .await + .expect_err("a database from a newer build must not be opened"); + assert!( + matches!(err, DatabaseError::SchemaTooNew { .. }), + "got {err:?}" + ); + // The schema is CREATE TABLE IF NOT EXISTS, so without this an older + // binary opens a newer database happily and writes the old model into + // it, leaving two mutually invisible truths in one file. + assert!(err.to_string().contains("newer than this build supports")); + } + + #[tokio::test] + async fn an_identity_schema_bump_rebuilds_cached_mail_and_keeps_configuration() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let account = nuncio_core::AccountConfig { + id: "acct-reset".to_string(), + name: "Reset Account".to_string(), + email_address: "reset@nuncio.mx".to_string(), + keyring_secret_key: "nuncio/acct-reset".to_string(), + sync_interval_secs: 60, + transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { + imap_host: "imap.nuncio.mx".to_string(), + imap_port: 993, + imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, + smtp_host: "smtp.nuncio.mx".to_string(), + smtp_port: 465, + smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls, + }), + }; + engine.save_account(&account).await.expect("save account"); + engine + .save_email(&synced_email("acct-reset", "INBOX", "42", "7")) + .await + .expect("save email"); + engine + .save_folder_sync_state("acct-reset", "INBOX", "v1:uidnext:42:8") + .await + .expect("save checkpoint"); + + // Pretend this store was written by the previous identity generation. + sqlx::query("PRAGMA user_version = 0") + .execute(engine.pool()) + .await + .unwrap(); + + engine + .reconcile_schema_version() + .await + .expect("an older store is rebuilt, not rejected"); + + // Server-derived state goes: it cannot be reinterpreted under a new + // identity model, and the server holds the authoritative copy. + let (message_count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(message_count, 0, "cached mail must be rebuilt"); + assert_eq!( + engine + .get_folder_sync_state("acct-reset", "INBOX") + .await + .unwrap(), + None, + "a checkpoint must not survive an identity change" + ); + + // Everything the user configured survives. + let accounts = engine.list_accounts().await.expect("list accounts"); + assert_eq!(accounts.len(), 1, "accounts must be preserved"); + assert_eq!(accounts[0].id, "acct-reset"); + + // The version is recorded, so the next open is a no-op. + let (version,): (i64,) = sqlx::query_as("PRAGMA user_version") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(version, DatabaseEngine::IDENTITY_SCHEMA_VERSION); + engine + .reconcile_schema_version() + .await + .expect("re-running at the current version is a no-op"); + } + #[tokio::test] async fn messages_identity_migration_completes_from_a_partial_run() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); -- 2.54.0 From bf19b9f3a6dad62667687b81a90e431fa8e74bd2 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Mon, 10 Aug 2026 16:08:45 -0500 Subject: [PATCH 57/95] feat(mail): add a raw IMAP layer for ENABLE, tagged codes, and VANISHED The typed async-imap API cannot express the four responses that decide whether a sync saw everything or whether a mutation actually happened. ENABLE does not exist, and RFC 7162 3.2.3 makes a server answer BAD to a QRESYNC SELECT without it. select()'s parser has no arm for VANISHED, so a deleted message is simply never learned about. And run_command_and_check_ok discards the tagged response code, which is exactly where RFC 4315 puts COPYUID and RFC 7162 puts MODIFIED. Hand-rolled rather than forked. run_command and read_response are public and imap-proto already parses Vanished and CopyUid, so what was missing is a collector that reads to the tagged response and keeps what arrived on the way. That is a hundred lines against maintaining a fork for the same result. Two subtleties the collector exists to get right. COPYUID arrives in an UNTAGGED OK for MOVE (RFC 6851 4.3 advises exactly that), so a collector reading only the tagged response misses it on every conforming server and then has no proof the move happened. And UID EXPUNGE answers with VANISHED rather than EXPUNGE once QRESYNC is enabled (4.4), so counting EXPUNGE alone reports failure on every successful delete; both are gathered. MODIFIED is the one real gap. imap-proto 0.16.7 has no ResponseCode variant for it, and ResponseData's raw bytes are behind a private module, so it cannot be read from the parse tree at all. Its resp_text is opt(resp_text_code) followed by free text, so an unrecognised code is not an error -- the whole "[MODIFIED 7,9]" survives in the information string, and that is where it is recovered from. Parsing a response code out of a human-readable field is not how this should work forever; upstreaming the variant is the real fix. It is deterministic, it is pinned by a test against a server emitting the real syntax, and the alternative is no conflict detection at all. Absent COPYUID is deliberately not treated as failure. A UID set matching nothing succeeds having done nothing (RFC 3501 6.4.8), and a conforming server may also omit the code for a UIDNOTSTICKY destination, so absence means unknown. Also fixes the mock server reading the first parenthesised group of a STORE as its flag list, which made a conditional store apply UNCHANGEDSINCE and the mod-sequence as flag names. The flag list is always the final argument. Refs #388 --- Cargo.lock | 2 + crates/nuncio-mail/Cargo.toml | 4 + crates/nuncio-mail/src/imap_raw.rs | 432 ++++++++++++++++++ crates/nuncio-mail/src/lib.rs | 1 + crates/nuncio-mail/src/test_server.rs | 10 +- .../nuncio-mail/tests/imap_raw_wire_test.rs | 301 ++++++++++++ 6 files changed, 748 insertions(+), 2 deletions(-) create mode 100644 crates/nuncio-mail/src/imap_raw.rs create mode 100644 crates/nuncio-mail/tests/imap_raw_wire_test.rs diff --git a/Cargo.lock b/Cargo.lock index 80c9ab8..ad178f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2604,6 +2604,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tracing", + "tracing-subscriber", "wiremock", "zeroize", ] @@ -2618,6 +2619,7 @@ dependencies = [ "lettre", "mail-parser", "nuncio-core", + "nuncio-mail", "reqwest", "rustls", "rustls-pki-types", diff --git a/crates/nuncio-mail/Cargo.toml b/crates/nuncio-mail/Cargo.toml index 72a10cc..0e98401 100644 --- a/crates/nuncio-mail/Cargo.toml +++ b/crates/nuncio-mail/Cargo.toml @@ -34,6 +34,10 @@ tokio-stream.workspace = true reqwest.workspace = true [dev-dependencies] +# Integration tests need the loopback IMAP server, which production builds +# must not carry. A self dev-dependency is how a crate enables its own feature +# for its own `tests/` targets. +nuncio-mail = { path = ".", features = ["test-support"] } tempfile.workspace = true wiremock.workspace = true tracing-test = "0.2" diff --git a/crates/nuncio-mail/src/imap_raw.rs b/crates/nuncio-mail/src/imap_raw.rs new file mode 100644 index 0000000..be2b321 --- /dev/null +++ b/crates/nuncio-mail/src/imap_raw.rs @@ -0,0 +1,432 @@ +//! Raw IMAP command layer: the responses `async-imap`'s typed API discards. +//! +//! The engine needs four things the typed client cannot express, and all four +//! decide whether a sync saw everything or whether a mutation actually +//! happened: +//! +//! | Needed | Why the typed API cannot give it | +//! | --- | --- | +//! | `ENABLE` | not implemented at all, and RFC 7162 3.2.3 makes a server reply `BAD` to a QRESYNC `SELECT` without it | +//! | `SELECT (QRESYNC …)` | no such method, and `select()`'s parser drops `VANISHED` | +//! | `COPYUID` | `uid_copy`/`uid_mv` return `Result<()>`; the tagged response *code* is discarded | +//! | `[MODIFIED …]` | same discard, and `imap-proto` has no `ResponseCode` variant for it at all | +//! +//! ## Why this is hand-rolled rather than a fork +//! +//! `Session::run_command` and `Session::read_response` are public, and +//! `imap-proto` already parses `Response::Vanished` and +//! `ResponseCode::CopyUid`. So the only thing missing is a collector that +//! reads responses until the tagged one and keeps what it saw -- roughly a +//! hundred lines, against maintaining a fork of the client for the same +//! result. The one genuine gap is `[MODIFIED …]`, handled below. +//! +//! ## The `MODIFIED` gap +//! +//! `imap-proto` 0.16.7 has no `ResponseCode::Modified`. Its `resp_text` parser +//! is `opt(resp_text_code)` followed by free text, so an unrecognised code is +//! not an error: `opt` yields `None` and the whole `[MODIFIED 7,9] …` is +//! carried through as the response's `information` string. That is where this +//! module recovers it from. Parsing a response code out of a human-readable +//! field is not how this should work forever -- upstreaming a `Modified` +//! variant is the real fix -- but it is deterministic, it is covered by tests +//! against a server that emits the real syntax, and the alternative is having +//! no conflict detection at all. + +use std::fmt::Debug; + +use async_imap::Session; +// Reached through `async-imap`'s re-export rather than a direct dependency, +// so the parser types can never version-skew against the client using them. +use async_imap::imap_proto::{Response, ResponseCode, Status, UidSetMember}; +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::parser::MailError; + +/// A `COPYUID` response code (RFC 4315 section 3): the destination mailbox's +/// UIDVALIDITY, the source UIDs, and the UIDs they became. +/// +/// Its presence is the only positive proof that a `COPY`/`MOVE` actually moved +/// something. Its **absence proves nothing**: RFC 6851 section 4.3 makes it +/// only `SHOULD` for `MOVE`, and RFC 4315 section 3 permits omitting it for a +/// `UIDNOTSTICKY` destination or one the client cannot `SELECT`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CopyUid { + /// UIDVALIDITY of the destination mailbox. + pub uid_validity: u32, + /// Source UIDs, expanded from the wire's set syntax. + pub source_uids: Vec<u32>, + /// Destination UIDs, positionally aligned with `source_uids`. + pub destination_uids: Vec<u32>, +} + +/// Everything one command's responses carried, tagged and untagged. +#[derive(Debug, Default, Clone)] +pub struct RawExchange { + /// `true` when the tagged response was `OK`. + pub ok: bool, + /// The tagged response's human text, which is also where an unparsed + /// response code such as `[MODIFIED …]` ends up. + pub information: Option<String>, + /// `COPYUID`, from wherever it arrived. RFC 6851 section 4.3 advises + /// servers to send it in an **untagged** `OK` for `MOVE`, so a collector + /// that only inspected the tagged response would miss it on every + /// conforming server. + pub copy_uid: Option<CopyUid>, + /// UIDs reported by `VANISHED` (RFC 7162), whether or not `(EARLIER)`. + pub vanished_uids: Vec<u32>, + /// Sequence numbers reported by untagged `EXPUNGE`. Servers that have + /// QRESYNC enabled send `VANISHED` instead (RFC 6851 section 4.4), so a + /// delete proved by counting these alone reports failure on success. + pub expunged_seqs: Vec<u32>, + /// UIDs named by untagged `FETCH` responses. + pub fetched_uids: Vec<u32>, + /// `HIGHESTMODSEQ`, from an untagged `OK` during `SELECT`. + pub highest_modseq: Option<u64>, + /// UIDVALIDITY, from an untagged `OK` during `SELECT`. + pub uid_validity: Option<u32>, +} + +impl RawExchange { + /// UIDs the server refused to update because they changed since the + /// client's mod-sequence (RFC 7162 section 3.1.3). + /// + /// `Some(vec![])` cannot occur: a `MODIFIED` code always names at least + /// one UID, so `None` means "no conflict reported" and `Some` means "these + /// specific UIDs conflicted". + pub fn modified_uids(&self) -> Option<Vec<u32>> { + self.information + .as_deref() + .and_then(parse_modified_response_code) + } +} + +/// Recover `[MODIFIED <uid-set>]` from a tagged response's text. +/// +/// See the module docs for why this is a string scan: `imap-proto` has no +/// `ResponseCode` variant for it, and its `opt(resp_text_code)` leaves the +/// unrecognised code in the free-text field rather than failing. +fn parse_modified_response_code(information: &str) -> Option<Vec<u32>> { + let start = information.find("[MODIFIED ")?; + let rest = &information[start + "[MODIFIED ".len()..]; + let end = rest.find(']')?; + let uids = expand_uid_set_text(&rest[..end]); + if uids.is_empty() { + None + } else { + Some(uids) + } +} + +/// Expand an IMAP `uid-set` in its wire text form (`7,9`, `1:4`, `1:3,7`). +fn expand_uid_set_text(spec: &str) -> Vec<u32> { + let mut out = Vec::new(); + for part in spec.split(',') { + let part = part.trim(); + if let Some((lo, hi)) = part.split_once(':') { + if let (Ok(lo), Ok(hi)) = (lo.trim().parse::<u32>(), hi.trim().parse::<u32>()) { + let (lo, hi) = if lo <= hi { (lo, hi) } else { (hi, lo) }; + out.extend(lo..=hi); + } + } else if let Ok(uid) = part.parse::<u32>() { + out.push(uid); + } + } + out.sort_unstable(); + out.dedup(); + out +} + +/// Flatten `imap-proto`'s uid-set representation into concrete UIDs. +fn expand_uid_set_members(members: &[UidSetMember]) -> Vec<u32> { + let mut out = Vec::new(); + for member in members { + match member { + UidSetMember::Uid(uid) => out.push(*uid), + UidSetMember::UidRange(range) => out.extend(range.clone()), + } + } + out +} + +/// Issue `command` and read responses until its tagged completion, keeping +/// everything that arrived on the way. +/// +/// This is the whole point of the module. `run_command_and_check_ok` reads to +/// the same place and throws away the untagged responses *and* the tagged +/// response code, which between them carry every fact about whether the +/// command did anything. +pub async fn run_collected<S>( + session: &mut Session<S>, + command: &str, +) -> Result<RawExchange, MailError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, +{ + let tag = session + .run_command(command) + .await + .map_err(|e| MailError::ImapError(format!("failed to issue '{command}': {e}")))?; + + let mut exchange = RawExchange::default(); + + loop { + let response = session + .read_response() + .await + .map_err(|e| MailError::ImapError(format!("reading response to '{command}': {e}")))? + .ok_or_else(|| { + MailError::ImapError(format!("connection closed before '{command}' completed")) + })?; + + let mut done = false; + match response.parsed() { + Response::Done { + tag: response_tag, + status, + code, + information, + } => { + // Only our own tag ends the exchange; another command's + // completion interleaved here would otherwise truncate ours. + if response_tag == &tag { + exchange.ok = matches!(status, Status::Ok); + exchange.information = information.as_ref().map(|i| i.to_string()); + absorb_code(&mut exchange, code.as_ref()); + done = true; + } + } + Response::Data { code, .. } => absorb_code(&mut exchange, code.as_ref()), + Response::Vanished { uids, .. } => { + for range in uids { + exchange.vanished_uids.extend(range.clone()); + } + } + Response::Expunge(seq) => exchange.expunged_seqs.push(*seq), + Response::Fetch(_, attributes) => { + for attribute in attributes { + if let async_imap::imap_proto::AttributeValue::Uid(uid) = attribute { + exchange.fetched_uids.push(*uid); + } + } + } + _ => {} + } + + if done { + break; + } + } + + Ok(exchange) +} + +/// Fold a response code into the exchange, from a tagged or untagged response. +fn absorb_code(exchange: &mut RawExchange, code: Option<&ResponseCode<'_>>) { + match code { + Some(ResponseCode::CopyUid(uid_validity, source, destination)) => { + exchange.copy_uid = Some(CopyUid { + uid_validity: *uid_validity, + source_uids: expand_uid_set_members(source), + destination_uids: expand_uid_set_members(destination), + }); + } + Some(ResponseCode::HighestModSeq(modseq)) => exchange.highest_modseq = Some(*modseq), + Some(ResponseCode::UidValidity(validity)) => exchange.uid_validity = Some(*validity), + _ => {} + } +} + +/// Issue `ENABLE` for `capabilities` (RFC 5161). +/// +/// Returns whether the server accepted the command. It deliberately does not +/// try to confirm *which* capabilities were enabled: the untagged `* ENABLED` +/// line has no `imap-proto` representation, and confirming it is unnecessary +/// -- RFC 7162 section 3.2.3 requires a server to answer `BAD` to a QRESYNC +/// `SELECT` that was never enabled, so a QRESYNC `SELECT` that succeeds is +/// itself the proof. +pub async fn enable<S>(session: &mut Session<S>, capabilities: &[&str]) -> Result<bool, MailError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, +{ + if capabilities.is_empty() { + return Ok(true); + } + let exchange = run_collected(session, &format!("ENABLE {}", capabilities.join(" "))).await?; + Ok(exchange.ok) +} + +/// `SELECT <mailbox> (QRESYNC (<uidvalidity> <modseq>))` (RFC 7162 section 3.2.5). +/// +/// The returned exchange carries the `VANISHED` set and the changed `FETCH` +/// UIDs the server volunteered, which together are the entire reason to use +/// QRESYNC: they are how a client learns what disappeared while it was away. +/// A plain `SELECT` can never report that. +/// +/// `known_uids` is optional; omitting it makes the server behave as if the +/// client knew `1:<UIDNEXT-1>` (section 3.2.5.1), which is correct but can +/// make the `VANISHED` set much larger than it needs to be. +pub async fn select_qresync<S>( + session: &mut Session<S>, + mailbox: &str, + uid_validity: u32, + modseq: u64, + known_uids: Option<&str>, +) -> Result<RawExchange, MailError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, +{ + let command = match known_uids { + Some(uids) => format!("SELECT {mailbox} (QRESYNC ({uid_validity} {modseq} {uids}))"), + None => format!("SELECT {mailbox} (QRESYNC ({uid_validity} {modseq}))"), + }; + let exchange = run_collected(session, &command).await?; + if !exchange.ok { + return Err(MailError::ImapError(format!( + "QRESYNC SELECT of '{mailbox}' was refused: {}", + exchange.information.as_deref().unwrap_or("no detail") + ))); + } + Ok(exchange) +} + +/// `UID STORE <set> (UNCHANGEDSINCE <modseq>) <op> (<flag>)` (RFC 7162 section 3.1.3). +/// +/// The compare-and-swap that turns a lost update into a detected one. Read the +/// result with [`RawExchange::modified_uids`]: `None` means every targeted UID +/// was updated, `Some(uids)` means those were left untouched because another +/// client changed them first. +/// +/// `.SILENT` is appended because the untagged `FETCH` responses a non-silent +/// store produces are not the proof of effect they look like -- RFC 3501 +/// section 6.4.6 lets a server suppress them, and a no-op store legitimately +/// produces none. `MODIFIED` is the signal that matters. +pub async fn uid_store_unchanged_since<S>( + session: &mut Session<S>, + uid_set: &str, + modseq: u64, + op: &str, + flag: &str, +) -> Result<RawExchange, MailError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, +{ + let command = format!("UID STORE {uid_set} (UNCHANGEDSINCE {modseq}) {op}.SILENT ({flag})"); + let exchange = run_collected(session, &command).await?; + // A tagged NO/BAD is a genuine failure; a tagged OK carrying MODIFIED is a + // conflict, which is a successful command with a reportable outcome. + if !exchange.ok && exchange.modified_uids().is_none() { + return Err(MailError::ImapError(format!( + "conditional UID STORE on '{uid_set}' failed: {}", + exchange.information.as_deref().unwrap_or("no detail") + ))); + } + Ok(exchange) +} + +/// `UID MOVE <set> <destination>` (RFC 6851), collecting the `COPYUID` proof. +/// +/// Check [`RawExchange::copy_uid`]: present means the move genuinely happened +/// and names the destination UIDs; **absent means unknown, not failed**. A UID +/// set matching nothing is a successful no-op per RFC 3501 section 6.4.8, and +/// a conforming server may also omit `COPYUID` for a `UIDNOTSTICKY` +/// destination. +pub async fn uid_move_collected<S>( + session: &mut Session<S>, + uid_set: &str, + destination: &str, +) -> Result<RawExchange, MailError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, +{ + let exchange = run_collected(session, &format!("UID MOVE {uid_set} {destination}")).await?; + if !exchange.ok { + return Err(MailError::ImapError(format!( + "UID MOVE of '{uid_set}' to '{destination}' failed: {}", + exchange.information.as_deref().unwrap_or("no detail") + ))); + } + Ok(exchange) +} + +/// `UID EXPUNGE <set>` (RFC 4315), collecting both removal reports. +/// +/// A server with QRESYNC enabled answers with `VANISHED` rather than +/// `EXPUNGE` (RFC 6851 section 4.4), so both are gathered and callers should +/// treat either as evidence. Counting only `EXPUNGE` reports failure on every +/// successful delete against a QRESYNC server. +pub async fn uid_expunge_collected<S>( + session: &mut Session<S>, + uid_set: &str, +) -> Result<RawExchange, MailError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, +{ + let exchange = run_collected(session, &format!("UID EXPUNGE {uid_set}")).await?; + if !exchange.ok { + return Err(MailError::ImapError(format!( + "UID EXPUNGE of '{uid_set}' failed: {}", + exchange.information.as_deref().unwrap_or("no detail") + ))); + } + Ok(exchange) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn modified_response_code_is_recovered_from_the_tagged_text() { + // imap-proto has no ResponseCode::Modified, and its opt(resp_text_code) + // leaves the unrecognised code in the free-text field. This is the + // contract that behaviour gives us; if a future imap-proto starts + // parsing MODIFIED, this test is what will notice. + let exchange = RawExchange { + information: Some("[MODIFIED 7,9] Conditional STORE failed".to_string()), + ..RawExchange::default() + }; + assert_eq!(exchange.modified_uids(), Some(vec![7, 9])); + } + + #[test] + fn modified_parses_ranges_and_reports_none_when_absent() { + assert_eq!( + parse_modified_response_code("[MODIFIED 1:3,7] failed"), + Some(vec![1, 2, 3, 7]) + ); + assert_eq!( + parse_modified_response_code("Conditional STORE failed"), + None + ); + assert_eq!(parse_modified_response_code("[MODIFIED ] weird"), None); + assert_eq!( + parse_modified_response_code("[MODIFIED 7 unterminated"), + None + ); + } + + #[test] + fn no_modified_means_no_conflict_not_an_empty_conflict() { + // The distinction matters: `Some(vec![])` would read as "a conflict + // affecting nothing", which is not a state the protocol can produce. + let exchange = RawExchange { + information: Some("STORE completed".to_string()), + ..RawExchange::default() + }; + assert_eq!(exchange.modified_uids(), None); + } + + #[test] + fn uid_sets_expand_from_both_wire_forms() { + assert_eq!(expand_uid_set_text("5"), vec![5]); + assert_eq!(expand_uid_set_text("1:4"), vec![1, 2, 3, 4]); + assert_eq!(expand_uid_set_text("9,1:3"), vec![1, 2, 3, 9]); + // Reversed ranges are normalized rather than yielding nothing. + assert_eq!(expand_uid_set_text("4:2"), vec![2, 3, 4]); + + assert_eq!( + expand_uid_set_members(&[UidSetMember::Uid(3), UidSetMember::UidRange(7..=9)]), + vec![3, 7, 8, 9] + ); + } +} diff --git a/crates/nuncio-mail/src/lib.rs b/crates/nuncio-mail/src/lib.rs index 4689ea8..fa913fe 100644 --- a/crates/nuncio-mail/src/lib.rs +++ b/crates/nuncio-mail/src/lib.rs @@ -5,6 +5,7 @@ pub mod backend; pub mod imap; +pub mod imap_raw; pub mod jmap; pub mod mock; pub mod parser; diff --git a/crates/nuncio-mail/src/test_server.rs b/crates/nuncio-mail/src/test_server.rs index bb659f3..9eaffea 100644 --- a/crates/nuncio-mail/src/test_server.rs +++ b/crates/nuncio-mail/src/test_server.rs @@ -742,9 +742,15 @@ fn handle_uid_store( Reply::open(out) } -/// Extract flag atoms from the parenthesised argument of a STORE. +/// Extract flag atoms from the parenthesised flag list of a STORE. +/// +/// Takes the **last** parenthesised group, not the first: a conditional store +/// is `UID STORE <set> (UNCHANGEDSINCE <n>) +FLAGS.SILENT (<flags>)`, so +/// reading the first group silently applies `UNCHANGEDSINCE` and the +/// mod-sequence as though they were flag names. The flag list is always the +/// final argument of a STORE. fn parse_flag_list(line: &str) -> Vec<String> { - let Some(open) = line.find('(') else { + let Some(open) = line.rfind('(') else { return Vec::new(); }; let Some(close) = line[open..].find(')') else { diff --git a/crates/nuncio-mail/tests/imap_raw_wire_test.rs b/crates/nuncio-mail/tests/imap_raw_wire_test.rs new file mode 100644 index 0000000..8793b3d --- /dev/null +++ b/crates/nuncio-mail/tests/imap_raw_wire_test.rs @@ -0,0 +1,301 @@ +//! Wire-level tests for the raw IMAP command layer. +//! +//! These drive the real `async-imap` client over real TCP against +//! [`MockImapServer`], because the whole claim being tested is that responses +//! the typed API discards can be recovered from the wire. A unit test over a +//! hand-built struct could not show that: it would assert on data this crate +//! produced rather than data a server sent. +//! +//! Each test pins one response that decides whether a sync saw everything, or +//! whether a mutation actually happened. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use nuncio_mail::imap_raw; +use nuncio_mail::test_server::{MockImapServer, ServerProfile}; + +/// Connect the real client to the mock server and authenticate. +async fn login( + server: &MockImapServer, +) -> async_imap::Session<tokio::io::BufWriter<tokio::net::TcpStream>> { + let stream = tokio::net::TcpStream::connect((server.host(), server.port())) + .await + .expect("connect to the mock server"); + let client = async_imap::Client::new(tokio::io::BufWriter::new(stream)); + client + .login("user", "pass") + .await + .map_err(|(e, _)| e) + .expect("login succeeds") +} + +#[tokio::test] +async fn qresync_select_recovers_vanished_uids_the_typed_select_would_drop() { + // The mechanism that fixes ghost messages. `async-imap`'s own `select()` + // parser has no arm for VANISHED, so this is the difference between + // learning a message was deleted and never learning it. + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + let keep = server.append_message("INBOX", "Subject: keep\r\n\r\nx", &[]); + let gone = server.append_message("INBOX", "Subject: gone\r\n\r\nx", &[]); + + let mut session = login(&server).await; + assert!( + imap_raw::enable(&mut session, &["QRESYNC"]) + .await + .expect("ENABLE succeeds"), + "the server advertises QRESYNC, so ENABLE must be accepted" + ); + + let initial = imap_raw::run_collected(&mut session, "SELECT INBOX") + .await + .expect("plain SELECT"); + let modseq = initial + .highest_modseq + .expect("a CONDSTORE server reports HIGHESTMODSEQ"); + let uid_validity = initial.uid_validity.expect("SELECT reports UIDVALIDITY"); + + // Another client removes a message while we are away. + assert!(server.expunge("INBOX", gone)); + + let resync = imap_raw::select_qresync(&mut session, "INBOX", uid_validity, modseq, None) + .await + .expect("QRESYNC SELECT succeeds"); + + assert!( + resync.vanished_uids.contains(&gone), + "the removal must be reported, got {:?}", + resync.vanished_uids + ); + assert!( + !resync.vanished_uids.contains(&keep), + "a surviving message must not be reported as vanished" + ); +} + +#[tokio::test] +async fn a_conditional_store_that_loses_a_race_reports_modified() { + // RFC 7162 3.1.3. MODIFIED rides on the tagged OK, which + // `run_command_and_check_ok` discards -- so without this layer a lost + // update is indistinguishable from a successful one. + let server = MockImapServer::start(ServerProfile::Condstore) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + let uid = server.append_message("INBOX", "Subject: contested\r\n\r\nx", &[]); + + let mut writer = login(&server).await; + let mut loser = login(&server).await; + imap_raw::run_collected(&mut writer, "SELECT INBOX") + .await + .expect("writer selects"); + imap_raw::run_collected(&mut loser, "SELECT INBOX") + .await + .expect("loser selects"); + + // The loser captures the current mod-sequence, then the writer changes the + // message underneath it. + let observed = imap_raw::run_collected(&mut loser, &format!("UID FETCH {uid} (FLAGS)")) + .await + .expect("fetch flags"); + assert_eq!(observed.fetched_uids, vec![uid]); + let modseq = imap_raw::run_collected(&mut loser, "SELECT INBOX") + .await + .expect("re-select") + .highest_modseq + .expect("HIGHESTMODSEQ reported"); + + imap_raw::run_collected(&mut writer, &format!("UID STORE {uid} +FLAGS (\\Seen)")) + .await + .expect("the writer wins the race"); + + let outcome = imap_raw::uid_store_unchanged_since( + &mut loser, + &uid.to_string(), + modseq, + "+FLAGS", + "\\Flagged", + ) + .await + .expect("a refused conditional store is not a transport failure"); + + assert_eq!( + outcome.modified_uids(), + Some(vec![uid]), + "the conflict must name the UID; information was {:?}", + outcome.information + ); + + // And the refused flag genuinely did not apply. + let flags = server.flags("INBOX", uid).unwrap_or_default(); + assert!( + !flags.iter().any(|f| f == "\\Flagged"), + "a MODIFIED store must leave the message untouched, got {flags:?}" + ); +} + +#[tokio::test] +async fn a_conditional_store_that_wins_reports_no_conflict() { + let server = MockImapServer::start(ServerProfile::Condstore) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + let uid = server.append_message("INBOX", "Subject: uncontested\r\n\r\nx", &[]); + + let mut session = login(&server).await; + let modseq = imap_raw::run_collected(&mut session, "SELECT INBOX") + .await + .expect("select") + .highest_modseq + .expect("HIGHESTMODSEQ reported"); + + let outcome = imap_raw::uid_store_unchanged_since( + &mut session, + &uid.to_string(), + modseq, + "+FLAGS", + "\\Flagged", + ) + .await + .expect("store succeeds"); + + assert_eq!( + outcome.modified_uids(), + None, + "an uncontested store must report no conflict" + ); + assert!(server + .flags("INBOX", uid) + .unwrap_or_default() + .iter() + .any(|f| f == "\\Flagged")); +} + +#[tokio::test] +async fn a_move_reports_copyuid_from_an_untagged_ok() { + // RFC 6851 4.3 advises COPYUID in an *untagged* OK for MOVE. A collector + // that only read the tagged response would miss it on every conforming + // server, and then have no proof the move happened. + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.create_mailbox("Archive"); + let uid = server.append_message("INBOX", "Subject: moving\r\n\r\nx", &[]); + + let mut session = login(&server).await; + imap_raw::run_collected(&mut session, "SELECT INBOX") + .await + .expect("select"); + + let outcome = imap_raw::uid_move_collected(&mut session, &uid.to_string(), "Archive") + .await + .expect("move succeeds"); + + let copy_uid = outcome + .copy_uid + .expect("a real move must be provable by COPYUID"); + assert_eq!(copy_uid.source_uids, vec![uid]); + assert_eq!(copy_uid.destination_uids.len(), 1); + assert_eq!(server.uids("INBOX"), Vec::<u32>::new()); + assert_eq!(server.uids("Archive"), copy_uid.destination_uids); +} + +#[tokio::test] +async fn a_move_of_an_already_moved_uid_is_unknown_not_applied() { + // RFC 3501 6.4.8: a UID command matching nothing succeeds having done + // nothing. The command reports OK, so only the *absence* of COPYUID + // distinguishes it -- and absence means unknown, never applied. + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.create_mailbox("Archive"); + server.create_mailbox("Trash"); + let uid = server.append_message("INBOX", "Subject: contested\r\n\r\nx", &[]); + + let mut winner = login(&server).await; + let mut loser = login(&server).await; + imap_raw::run_collected(&mut winner, "SELECT INBOX") + .await + .expect("select"); + imap_raw::run_collected(&mut loser, "SELECT INBOX") + .await + .expect("select"); + + imap_raw::uid_move_collected(&mut winner, &uid.to_string(), "Archive") + .await + .expect("the winner moves it"); + + let outcome = imap_raw::uid_move_collected(&mut loser, &uid.to_string(), "Trash") + .await + .expect("the losing move still reports OK"); + + assert!( + outcome.ok, + "a UID set matching nothing is a successful no-op, not an error" + ); + assert_eq!( + outcome.copy_uid, None, + "nothing moved, so there is no COPYUID to prove one did" + ); + assert!( + server.uids("Trash").is_empty(), + "and genuinely nothing arrived at the destination" + ); +} + +#[tokio::test] +async fn expunge_is_provable_under_both_qresync_and_plain_servers() { + // RFC 6851 4.4: a QRESYNC server answers UID EXPUNGE with VANISHED rather + // than EXPUNGE. Counting only EXPUNGE reports failure on every successful + // delete against such a server, so both must be collected. + for profile in [ServerProfile::Qresync, ServerProfile::Condstore] { + let server = MockImapServer::start(profile).await.expect("server starts"); + server.create_mailbox("INBOX"); + let uid = server.append_message("INBOX", "Subject: gone\r\n\r\nx", &["\\Deleted"]); + + let mut session = login(&server).await; + if profile == ServerProfile::Qresync { + imap_raw::enable(&mut session, &["QRESYNC"]) + .await + .expect("ENABLE succeeds"); + } + imap_raw::run_collected(&mut session, "SELECT INBOX") + .await + .expect("select"); + + let outcome = imap_raw::uid_expunge_collected(&mut session, &uid.to_string()) + .await + .expect("expunge succeeds"); + + let reported = outcome.vanished_uids.len() + outcome.expunged_seqs.len(); + assert_eq!( + reported, 1, + "the removal must be provable on {profile:?}: vanished={:?} expunged={:?}", + outcome.vanished_uids, outcome.expunged_seqs + ); + assert!(server.uids("INBOX").is_empty()); + } +} + +#[tokio::test] +async fn a_qresync_select_without_enable_is_refused() { + // RFC 7162 3.2.3 requires a tagged BAD. Surfacing it as an error keeps a + // client that skipped ENABLE from silently degrading in production. + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + + let mut session = login(&server).await; + let err = imap_raw::select_qresync(&mut session, "INBOX", 1000, 1, None) + .await + .expect_err("QRESYNC without ENABLE must fail"); + assert!( + err.to_string().contains("refused"), + "the refusal must be surfaced, got: {err}" + ); +} -- 2.54.0 From 1f75527563afff7965b0be949d6cd232c6ae9be1 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Mon, 10 Aug 2026 16:50:48 -0500 Subject: [PATCH 58/95] feat(sync): enumerate IMAP changes over a four-rung capability ladder Sync could only ever add. It resolved a UID boundary, fetched forward from it, and stored what came back; nothing ever asked what stopped being there. A message moved or deleted by any other client -- another daemon, a phone, webmail -- stayed in the local store forever, and every account accumulated ghosts. This is the defect, and it affects a single engine just as much as ten. A folder pass now reports what changed, what vanished, and honestly whether it was able to observe absence at all. The last part is why `present` is an Option: None means the pass was incremental and cannot speak to absence, while Some(empty) means the folder is genuinely empty. Conflating those deletes a mailbox. Four rungs, chosen per mailbox rather than per account, because NOMODSEQ is a mailbox property (RFC 7162 3.1.2.2) and one folder can be unable to answer mod-sequence queries on a server that otherwise can. QRESYNC has the server state removals outright. CONDSTORE narrows the fetch but never volunteers them, so absence comes from enumerating what remains -- this is Gmail's rung, which advertises CONDSTORE and not QRESYNC, and a two-rung ladder would have pushed the largest provider onto full sweeps against a published daily byte cap. Neither extension leaves a UID boundary for arrivals plus an enumeration for removals; that is Exchange. No usable checkpoint fetches everything. The enumeration passes ask for UID only -- no bodies, no flags -- so detecting removals costs one short line per message rather than a download. A mailbox that can supply a mod-sequence gets a mod-sequence checkpoint even after a full pass, so the next sync starts on a higher rung than this one did. A UIDVALIDITY change is not a downgrade to a lesser rung, it is a full pass: UIDs restart, so every stored boundary names a different message. The dangerous reading would be to enumerate the new UID space, find none of the stored ids in it, and conclude the folder was emptied. The raw collector also gains a bounded read. A server that accepts a command and then stops answering must surface as a stall rather than hanging the sync, which is the same failure the typed fetch path already bounds. Refs #390 --- crates/nuncio-mail/src/backend.rs | 50 +- crates/nuncio-mail/src/imap.rs | 841 +++++++++++++----- crates/nuncio-mail/src/imap_raw.rs | 19 +- crates/nuncio-mail/src/jmap.rs | 15 +- crates/nuncio-mail/src/mock.rs | 21 +- .../nuncio-mail/tests/imap_raw_wire_test.rs | 179 ++++ crates/nuncio-store/src/db.rs | 45 + crates/nunciod/src/sync.rs | 37 +- 8 files changed, 966 insertions(+), 241 deletions(-) diff --git a/crates/nuncio-mail/src/backend.rs b/crates/nuncio-mail/src/backend.rs index cb999bd..c776170 100644 --- a/crates/nuncio-mail/src/backend.rs +++ b/crates/nuncio-mail/src/backend.rs @@ -65,19 +65,63 @@ pub struct RemoteMutationSpec { pub kind: RemoteMutationKind, } +/// What one folder-sync pass observed on the server. +/// +/// The reason this is not just a list of messages: a sync that can only report +/// what *is* there can never report what stopped being there. A message moved +/// or deleted by any other client -- another Nuncio daemon, a phone, webmail -- +/// simply stays in the local store forever. Expressing absence is the whole +/// point of the type. +#[derive(Debug, Default, Clone)] +pub struct FolderChanges { + /// Messages to store, whether newly arrived or changed. + pub upserts: Vec<Email>, + /// Surrogate ids the server **explicitly reported as gone**, via QRESYNC + /// `VANISHED`. Always safe to delete. + pub removals: Vec<String>, + /// Surrogate ids of everything the folder currently holds, when this pass + /// enumerated the complete UID set. The caller may delete anything it + /// stores for this folder that is absent from this list. + /// + /// `None` and `Some(vec![])` mean different things, and conflating them + /// deletes a mailbox. `None` is "this pass was incremental and cannot + /// speak to absence"; `Some(vec![])` is "the folder is genuinely empty". + pub present: Option<Vec<String>>, + /// The checkpoint to resume from next time. + pub next_state: String, +} + /// Protocol-agnostic mail backend engine trait implemented by JMAP and IMAP engines. #[async_trait] pub trait MailBackend: Send + Sync { /// Synchronize and list available mailbox folders. async fn sync_folders(&self) -> Result<Vec<Folder>, MailError>; - /// Synchronize message envelopes for a specific folder since a checkpoint state. - /// Returns the updated email list and the new server state checkpoint. + /// Enumerate what changed in a folder since `since_state`. + /// + /// Replaces a fetch-only sync: implementations must report removals when + /// the protocol can express them, and must say honestly (via + /// [`FolderChanges::present`]) whether the pass was able to observe + /// absence at all. + async fn sync_changes( + &self, + folder_id: &str, + since_state: Option<&str>, + ) -> Result<FolderChanges, MailError>; + + /// Fetch-only view of [`MailBackend::sync_changes`], for callers that + /// genuinely only want the messages. + /// + /// Provided rather than required so no implementation can satisfy the + /// trait by supplying this and quietly never reporting a removal. async fn sync_messages( &self, folder_id: &str, since_state: Option<&str>, - ) -> Result<(Vec<Email>, String), MailError>; + ) -> Result<(Vec<Email>, String), MailError> { + let changes = self.sync_changes(folder_id, since_state).await?; + Ok((changes.upserts, changes.next_state)) + } /// Apply a remote mutation (move/copy/flag/delete) to a single message /// against the real server. MUST return `Ok(())` only when the server diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index ca91725..6e883bc 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -102,95 +102,147 @@ const CHECKPOINT_DELIM: char = ':'; /// full re-fetch. Over-fetching is recoverable; misreading a token is not. const CHECKPOINT_SCHEME_UIDNEXT: &str = "v1:uidnext"; -/// The `UID FETCH` sequence-set to issue for a folder sync, resolved from the -/// stored checkpoint. Kept as a small typed value (rather than collapsing to a -/// bare range string) so the distinct fall-back reasons stay observable: all -/// of `Full`/`CorruptCheckpoint`/`UidValidityChanged` fetch `1:*`, but only the -/// latter two are worth flagging, and tests can assert on which branch fired. +/// Scheme for a mod-sequence checkpoint: `"v2:modseq:{uidvalidity}:{modseq}"`. +/// +/// Written by both mod-sequence rungs. QRESYNC and CONDSTORE differ in how +/// changes are *requested*, not in what has to be remembered -- both resume +/// from `(UIDVALIDITY, HIGHESTMODSEQ)` -- so one scheme covers both and a +/// mailbox can move between them without invalidating its checkpoint. +const CHECKPOINT_SCHEME_MODSEQ: &str = "v2:modseq"; + +/// Which mechanism a folder sync can use, in descending order of precision. +/// +/// Chosen **per mailbox, not per account**: `NOMODSEQ` is a mailbox property +/// (RFC 7162 section 3.1.2.2), so one folder on a CONDSTORE server can still be +/// unable to answer mod-sequence queries. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum FetchRange { - /// No prior checkpoint (first-ever sync of this folder): fetch everything. +pub(crate) enum SyncRung { + /// `SELECT (QRESYNC …)` returns `VANISHED` plus changed messages in one + /// round trip. The server reports removals directly. + /// Fastmail/Cyrus, Dovecot. + Qresync, + /// `CHANGEDSINCE` narrows the fetch, but the server never volunteers + /// removals, so absence is found by diffing the full UID set. + /// **Gmail lives here**: it advertises CONDSTORE and not QRESYNC. + Condstore, + /// Neither extension. Changes and removals both come from a full UID-set + /// enumeration. **Exchange lives here.** + UidSetDiff, + /// No usable checkpoint: fetch everything. Also where an unreadable or + /// UIDVALIDITY-invalidated checkpoint lands. Full, - /// A checkpoint was stored but could not be parsed (malformed, or a legacy - /// bare-UID checkpoint predating UIDVALIDITY tracking). Falls back to a - /// full fetch, surfaced (logged) rather than silently ignored. - /// - /// Also covers a checkpoint whose scheme tag this build does not recognise - /// (see [`CHECKPOINT_SCHEME_UIDNEXT`]) -- including an untagged one, which - /// cannot be distinguished from a future scheme that happens to share its - /// shape. - CorruptCheckpoint, - /// The stored checkpoint's UIDVALIDITY no longer matches the mailbox's - /// current UIDVALIDITY (the mailbox was renumbered) or the server did not - /// report a current UIDVALIDITY to compare against. Stored UIDs are no - /// longer meaningful, so this falls back to a safe full re-fetch. - UidValidityChanged { - /// UIDVALIDITY recorded in the stored checkpoint. - stored: u32, - /// UIDVALIDITY the server reports now, if any. - current: Option<u32>, - }, - /// A valid checkpoint whose UIDVALIDITY still matches: fetch only UIDs at - /// or above the stored boundary, i.e. messages that arrived since. - Incremental(u32), } -impl FetchRange { - /// Render the IMAP `UID FETCH` sequence-set. Every non-incremental variant - /// fetches all UIDs (`1:*`); an incremental checkpoint `n` fetches `n:*`, - /// which the server bounds to UIDs `>= n` -- a genuinely narrower fetch. - fn sequence_set(self) -> String { +/// What the server and this mailbox can actually do. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct MailboxSyncCapabilities { + /// Server advertises `QRESYNC` **and** this connection enabled it. + pub qresync_enabled: bool, + /// Server advertises `CONDSTORE`. + pub condstore: bool, + /// This mailbox reported a `HIGHESTMODSEQ`. A mailbox answering + /// `NOMODSEQ` has no mod-sequences to compare against no matter what the + /// server advertises. + pub mailbox_has_modseq: bool, +} + +/// One folder-sync pass, resolved before any body is fetched. +#[derive(Debug, Clone)] +pub(crate) struct FolderSyncPlan { + /// The mechanism this pass negotiated. + pub rung: SyncRung, + pub uid_validity: Option<u32>, + pub uid_next: Option<u32>, + pub highest_modseq: Option<u64>, + /// UID set whose bodies this pass fetches. Empty means nothing changed. + pub body_sequence_set: String, + /// UIDs the server reported as gone (QRESYNC only). + pub vanished_uids: Vec<u32>, + /// The folder's complete UID set, when this pass enumerated it. + pub present_uids: Option<Vec<u32>>, +} + +/// A checkpoint decoded into the resume point it names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Checkpoint { + /// `v1:uidnext` -- resume from a UID boundary. + UidNext { uid_validity: u32, uid_next: u32 }, + /// `v2:modseq` -- resume from a mod-sequence. + ModSeq { uid_validity: u32, modseq: u64 }, +} + +impl Checkpoint { + fn uid_validity(self) -> u32 { match self { - FetchRange::Full - | FetchRange::CorruptCheckpoint - | FetchRange::UidValidityChanged { .. } => "1:*".to_string(), - FetchRange::Incremental(uid) => format!("{}:*", uid), + Checkpoint::UidNext { uid_validity, .. } | Checkpoint::ModSeq { uid_validity, .. } => { + uid_validity + } } } } -/// Resolve the fetch range from a stored `since_state` checkpoint and the -/// mailbox's CURRENT UIDVALIDITY (as just reported by `SELECT`). `None` means -/// no prior sync (full fetch). A checkpoint in the `"{uidvalidity}:{uidnext}"` -/// format proceeds incrementally only when its UIDVALIDITY still matches the -/// server's current one; a mismatch (mailbox renumbered) or a missing current -/// UIDVALIDITY forces a safe full re-fetch. A malformed or legacy bare-number -/// checkpoint is treated as corrupt (full fetch, flagged), never misread as a -/// live UID boundary. -fn resolve_fetch_range(since_state: Option<&str>, current_uid_validity: Option<u32>) -> FetchRange { - let Some(raw) = since_state else { - return FetchRange::Full; - }; - // Require the scheme tag. A value without one is either pre-tagging or from - // a scheme this build cannot interpret; both must re-fetch rather than be - // guessed at. - let Some(body) = raw - .trim() - .strip_prefix(CHECKPOINT_SCHEME_UIDNEXT) - .and_then(|rest| rest.strip_prefix(CHECKPOINT_DELIM)) - else { - return FetchRange::CorruptCheckpoint; - }; - let parsed = body - .split_once(CHECKPOINT_DELIM) - .and_then(|(validity, uid)| { - Some(( - validity.trim().parse::<u32>().ok()?, - uid.trim().parse::<u32>().ok()?, - )) +/// Decode a stored checkpoint, or `None` when it carries no scheme this build +/// recognises. An unrecognised token is never guessed at -- see +/// [`CHECKPOINT_SCHEME_UIDNEXT`] for why that would lose mail silently. +pub(crate) fn parse_checkpoint(raw: &str) -> Option<Checkpoint> { + let raw = raw.trim(); + if let Some(body) = raw + .strip_prefix(CHECKPOINT_SCHEME_MODSEQ) + .and_then(|r| r.strip_prefix(CHECKPOINT_DELIM)) + { + let (validity, modseq) = body.split_once(CHECKPOINT_DELIM)?; + return Some(Checkpoint::ModSeq { + uid_validity: validity.trim().parse().ok()?, + modseq: modseq.trim().parse().ok()?, }); - let Some((stored_validity, uid)) = parsed else { - return FetchRange::CorruptCheckpoint; - }; - if uid < 1 { - return FetchRange::CorruptCheckpoint; } - match current_uid_validity { - Some(current) if current == stored_validity => FetchRange::Incremental(uid), - current => FetchRange::UidValidityChanged { - stored: stored_validity, - current, - }, + let body = raw + .strip_prefix(CHECKPOINT_SCHEME_UIDNEXT) + .and_then(|r| r.strip_prefix(CHECKPOINT_DELIM))?; + let (validity, uid) = body.split_once(CHECKPOINT_DELIM)?; + let uid_next: u32 = uid.trim().parse().ok()?; + if uid_next < 1 { + return None; + } + Some(Checkpoint::UidNext { + uid_validity: validity.trim().parse().ok()?, + uid_next, + }) +} + +/// Pick the rung for this pass. +/// +/// A checkpoint whose UIDVALIDITY no longer matches the mailbox is not a +/// downgrade to a lesser rung -- it is [`SyncRung::Full`]. UIDs restart after a +/// UIDVALIDITY change, so every stored boundary refers to a different message +/// than it did (RFC 3501 section 2.3.1.1). +pub(crate) fn choose_rung( + checkpoint: Option<Checkpoint>, + current_uid_validity: Option<u32>, + caps: MailboxSyncCapabilities, +) -> SyncRung { + let modseq_usable = caps.mailbox_has_modseq && (caps.qresync_enabled || caps.condstore); + + let Some(checkpoint) = checkpoint else { + // No checkpoint means everything must be fetched regardless of what + // the server can do. + return SyncRung::Full; + }; + let (Some(current), stored) = (current_uid_validity, checkpoint.uid_validity()) else { + return SyncRung::Full; + }; + if current != stored { + return SyncRung::Full; + } + + match checkpoint { + Checkpoint::ModSeq { .. } if modseq_usable && caps.qresync_enabled => SyncRung::Qresync, + Checkpoint::ModSeq { .. } if modseq_usable => SyncRung::Condstore, + // A mod-sequence checkpoint against a mailbox that can no longer + // answer mod-sequence queries (index loss, or a move to a server + // without the extension) has nothing to resume from. + Checkpoint::ModSeq { .. } => SyncRung::Full, + Checkpoint::UidNext { .. } => SyncRung::UidSetDiff, } } @@ -719,7 +771,7 @@ impl ImapEngine { folder_id: &str, since_state: Option<&str>, session: &mut async_imap::Session<S>, - ) -> Result<(Vec<Email>, String), MailError> + ) -> Result<crate::backend::FolderChanges, MailError> where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { @@ -758,35 +810,18 @@ impl ImapEngine { since_state: Option<&str>, session: &mut async_imap::Session<S>, batch_size: usize, - ) -> Result<(Vec<Email>, String), MailError> + ) -> Result<crate::backend::FolderChanges, MailError> where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { - let mailbox = session.select(folder_id).await.map_err(|e| { - MailError::ImapError(format!("failed to select folder '{}': {}", folder_id, e)) - })?; - let server_uid_next = mailbox.uid_next; - let server_uid_validity = mailbox.uid_validity; - - let range = resolve_fetch_range(since_state, server_uid_validity); - match range { - FetchRange::CorruptCheckpoint => tracing::warn!( - folder_id, - checkpoint = since_state.unwrap_or_default(), - "ignoring unparseable folder sync checkpoint; falling back to a full fetch" - ), - FetchRange::UidValidityChanged { stored, current } => tracing::warn!( - folder_id, - stored_uidvalidity = stored, - current_uidvalidity = current, - "mailbox UIDVALIDITY changed (renumbered); the stored UID checkpoint is no \ - longer valid, so re-fetching in full to avoid silently skipping messages" - ), - FetchRange::Full | FetchRange::Incremental(_) => {} - } + let plan = self + .plan_folder_sync(folder_id, since_state, session) + .await?; + let server_uid_next = plan.uid_next; + let server_uid_validity = plan.uid_validity; let batch_size = batch_size.max(1); - let sequence_set = range.sequence_set(); + let sequence_set = plan.body_sequence_set.clone(); // Phase 1: enumerate UIDs and sizes only -- no bodies transferred, so // this stays small even for an enormous mailbox. @@ -888,8 +923,15 @@ impl ImapEngine { // safely falls back to a full fetch. let boundary_uid = server_uid_next .or_else(|| (max_uid_seen > 0).then_some(max_uid_seen.saturating_add(1))); - let new_checkpoint = match (server_uid_validity, boundary_uid) { - (Some(validity), Some(uid)) => format!( + // A mod-sequence checkpoint is preferred whenever the mailbox can + // supply one, even after a full pass: writing it is what lets the NEXT + // sync use a higher rung than this one did. + let new_checkpoint = match (server_uid_validity, plan.highest_modseq, boundary_uid) { + (Some(validity), Some(modseq), _) => format!( + "{}{}{}{}{}", + CHECKPOINT_SCHEME_MODSEQ, CHECKPOINT_DELIM, validity, CHECKPOINT_DELIM, modseq + ), + (Some(validity), None, Some(uid)) => format!( "{}{}{}{}{}", CHECKPOINT_SCHEME_UIDNEXT, CHECKPOINT_DELIM, validity, CHECKPOINT_DELIM, uid ), @@ -898,16 +940,222 @@ impl ImapEngine { .unwrap_or_else(|| "full-resync-required".to_string()), }; + let removals: Vec<String> = plan + .vanished_uids + .iter() + .map(|uid| self.surrogate_for(folder_id, server_uid_validity, *uid)) + .collect(); + let present = plan.present_uids.as_ref().map(|uids| { + uids.iter() + .map(|uid| self.surrogate_for(folder_id, server_uid_validity, *uid)) + .collect() + }); + tracing::info!( account_id = %self.account_id, folder_id, + rung = ?plan.rung, fetched = emails.len(), + removed = removals.len(), + enumerated = plan.present_uids.as_ref().map_or(0, Vec::len), oversized = oversized_uids.len(), skipped = skipped_no_uid, "imap folder sync complete" ); - Ok((emails, new_checkpoint)) + Ok(crate::backend::FolderChanges { + upserts: emails, + removals, + present, + next_state: new_checkpoint, + }) + } + + /// Negotiate capabilities, select the folder by the best available means, + /// and decide what this pass must fetch and what it can say about absence. + /// + /// This is where the ladder is applied. Ordering matters: `ENABLE` must + /// precede `SELECT` (RFC 7162 section 3.2.3 makes a server answer `BAD` to + /// a QRESYNC `SELECT` otherwise), and the rung cannot be finalised until + /// the mailbox has reported whether it has mod-sequences at all. + async fn plan_folder_sync<S>( + &self, + folder_id: &str, + since_state: Option<&str>, + session: &mut async_imap::Session<S>, + ) -> Result<FolderSyncPlan, MailError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, + { + let (has_qresync, has_condstore) = { + let caps = session + .capabilities() + .await + .map_err(|e| MailError::ImapError(format!("CAPABILITY query failed: {e}")))?; + (caps.has_str("QRESYNC"), caps.has_str("CONDSTORE")) + }; + + let checkpoint = since_state.and_then(parse_checkpoint); + if since_state.is_some() && checkpoint.is_none() { + tracing::warn!( + folder_id, + checkpoint = since_state.unwrap_or_default(), + "folder sync checkpoint carries no scheme this build recognises; \ + re-fetching in full rather than guessing at its meaning" + ); + } + + // ENABLE before SELECT, and only when there is a mod-sequence + // checkpoint that a QRESYNC SELECT could actually resume from. + let qresync_enabled = match (has_qresync, checkpoint) { + (true, Some(Checkpoint::ModSeq { .. })) => { + crate::imap_raw::enable(session, &["QRESYNC"]).await? + } + _ => false, + }; + + // Try the QRESYNC SELECT when it is available. Per RFC 7162 section + // 3.2.5 a server whose UIDVALIDITY no longer matches simply ignores the + // QRESYNC parameters, so this is safe to attempt: the mismatch is + // detected below from the UIDVALIDITY it reports back. + let (selected, attempted_qresync) = match (qresync_enabled, checkpoint) { + ( + true, + Some(Checkpoint::ModSeq { + uid_validity, + modseq, + }), + ) => ( + crate::imap_raw::select_qresync(session, folder_id, uid_validity, modseq, None) + .await?, + true, + ), + _ => { + let exchange = + crate::imap_raw::run_collected(session, &format!("SELECT {folder_id}")).await?; + if !exchange.ok { + return Err(MailError::ImapError(format!( + "failed to select folder '{folder_id}': {}", + exchange.information.as_deref().unwrap_or("no detail") + ))); + } + (exchange, false) + } + }; + + let caps = MailboxSyncCapabilities { + qresync_enabled, + condstore: has_condstore, + mailbox_has_modseq: selected.highest_modseq.is_some(), + }; + let mut rung = choose_rung(checkpoint, selected.uid_validity, caps); + // The QRESYNC SELECT was issued optimistically; if the server answered + // with a different UIDVALIDITY it ignored our parameters, and anything + // it did report refers to a UID space that no longer exists. + if attempted_qresync && rung != SyncRung::Qresync { + tracing::warn!( + folder_id, + "QRESYNC resume was refused or the mailbox was renumbered; re-fetching in full" + ); + } + if rung == SyncRung::Qresync && !attempted_qresync { + rung = SyncRung::Full; + } + + let mut plan = FolderSyncPlan { + rung, + uid_validity: selected.uid_validity, + uid_next: selected.uid_next, + highest_modseq: selected.highest_modseq, + body_sequence_set: "1:*".to_string(), + vanished_uids: Vec::new(), + present_uids: None, + }; + + match rung { + SyncRung::Qresync => { + // The server volunteered both halves: what changed, and what + // disappeared. No enumeration of the folder is needed at all, + // which is the entire reason this rung is worth having. + plan.vanished_uids = selected.vanished_uids.clone(); + plan.body_sequence_set = if selected.fetched_uids.is_empty() { + // Nothing changed. An empty set would be a protocol error, + // so ask for a range that cannot match instead of skipping + // the fetch path. + String::new() + } else { + join_uid_batch(&selected.fetched_uids) + }; + } + SyncRung::Condstore => { + let modseq = match checkpoint { + Some(Checkpoint::ModSeq { modseq, .. }) => modseq, + _ => 0, + }; + let changed = self + .enumerate_uids(session, folder_id, Some(modseq)) + .await?; + plan.body_sequence_set = if changed.is_empty() { + String::new() + } else { + join_uid_batch(&changed) + }; + // CONDSTORE narrows the fetch but never reports removals, so + // absence has to be found by enumerating what remains. + plan.present_uids = Some(self.enumerate_uids(session, folder_id, None).await?); + } + SyncRung::UidSetDiff => { + // New arrivals come from the UID boundary; removals from the + // full UID set. Two cheap enumerations beat re-fetching bodies. + if let Some(Checkpoint::UidNext { uid_next, .. }) = checkpoint { + plan.body_sequence_set = format!("{uid_next}:*"); + } + plan.present_uids = Some(self.enumerate_uids(session, folder_id, None).await?); + } + SyncRung::Full => { + // Everything is fetched, so the same pass also establishes + // exactly what the folder holds. + plan.present_uids = Some(self.enumerate_uids(session, folder_id, None).await?); + } + } + + Ok(plan) + } + + /// List the folder's UIDs, optionally narrowed to those changed since + /// `changed_since`. Requests no bodies and no flags, so the cost is one + /// short line per message rather than a download. + async fn enumerate_uids<S>( + &self, + session: &mut async_imap::Session<S>, + folder_id: &str, + changed_since: Option<u64>, + ) -> Result<Vec<u32>, MailError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, + { + let command = match changed_since { + Some(modseq) => format!("UID FETCH 1:* (UID) (CHANGEDSINCE {modseq})"), + None => "UID FETCH 1:* (UID)".to_string(), + }; + let exchange = crate::imap_raw::run_collected(session, &command).await?; + if !exchange.ok { + return Err(MailError::ImapError(format!( + "UID enumeration of '{folder_id}' failed: {}", + exchange.information.as_deref().unwrap_or("no detail") + ))); + } + let mut uids = exchange.fetched_uids; + uids.sort_unstable(); + uids.dedup(); + Ok(uids) + } + + /// Surrogate id for a UID in this account/folder/UIDVALIDITY scope, so + /// removals name messages the same way stored rows do. + fn surrogate_for(&self, folder_id: &str, uid_validity: Option<u32>, uid: u32) -> String { + let validity = uid_validity.map(|v| v.to_string()).unwrap_or_default(); + Email::surrogate_id(&self.account_id, folder_id, &validity, &uid.to_string()) } /// Build an [`Email`] from a full-body (or envelope-only) FETCH item. When @@ -1049,7 +1297,7 @@ impl ImapEngine { &self, folder_id: &str, since_state: Option<&str>, - ) -> Result<(Vec<Email>, String), MailError> { + ) -> Result<crate::backend::FolderChanges, MailError> { let (username, password) = match (&self.username, &self.password) { (Some(u), Some(p)) => (u.as_str(), p.as_str()), _ => { @@ -1457,11 +1705,11 @@ impl MailBackend for ImapEngine { Ok(folders) } - async fn sync_messages( + async fn sync_changes( &self, folder_id: &str, since_state: Option<&str>, - ) -> Result<(Vec<Email>, String), MailError> { + ) -> Result<crate::backend::FolderChanges, MailError> { self.sync_folder_messages(folder_id, since_state).await } @@ -1526,7 +1774,16 @@ mod tests { while let Some(line) = read_scripted_line(&mut socket).await { let tag = line.split_whitespace().next().unwrap_or("").to_string(); let upper = line.to_ascii_uppercase(); - if upper.contains("LOGIN") { + if upper.contains("CAPABILITY") { + let _ = socket + .write_all( + format!( + "* CAPABILITY IMAP4rev1 UIDPLUS MOVE\r\n{tag} OK CAPABILITY\r\n" + ) + .as_bytes(), + ) + .await; + } else if upper.contains("LOGIN") { let _ = socket .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) .await; @@ -1560,76 +1817,8 @@ mod tests { assert_eq!(folders[1].id, "Archive"); } - #[test] - fn resolve_fetch_range_distinguishes_none_valid_corrupt_and_renumbered_checkpoints() { - // First-ever sync: full history (current UIDVALIDITY irrelevant). - assert_eq!(resolve_fetch_range(None, Some(1)), FetchRange::Full); - assert_eq!(resolve_fetch_range(None, None).sequence_set(), "1:*"); - - // Valid checkpoint whose UIDVALIDITY still matches: narrowed fetch. - assert_eq!( - resolve_fetch_range(Some("v1:uidnext:1:105"), Some(1)), - FetchRange::Incremental(105) - ); - assert_eq!( - resolve_fetch_range(Some("v1:uidnext:1:105"), Some(1)).sequence_set(), - "105:*" - ); - assert_eq!( - resolve_fetch_range(Some("v1:uidnext: 7 : 42 "), Some(7)), - FetchRange::Incremental(42) - ); - - // UIDVALIDITY changed (mailbox renumbered) or the server reported none - // to compare against: MUST fall back to a full fetch, distinctly, so a - // renumber can never silently skip the low new UIDs. - assert_eq!( - resolve_fetch_range(Some("v1:uidnext:1:105"), Some(2)), - FetchRange::UidValidityChanged { - stored: 1, - current: Some(2) - } - ); - assert_eq!( - resolve_fetch_range(Some("v1:uidnext:1:105"), Some(2)).sequence_set(), - "1:*" - ); - assert_eq!( - resolve_fetch_range(Some("v1:uidnext:1:105"), None), - FetchRange::UidValidityChanged { - stored: 1, - current: None - } - ); - - // Malformed or legacy bare-number (pre-UIDVALIDITY) checkpoints are - // corrupt -> full fetch, never misread as a live UID boundary. - assert_eq!( - resolve_fetch_range(Some(""), Some(1)), - FetchRange::CorruptCheckpoint - ); - assert_eq!( - resolve_fetch_range(Some("105"), Some(1)), - FetchRange::CorruptCheckpoint, - "a legacy bare-number checkpoint must not be read as a UID boundary" - ); - assert_eq!( - resolve_fetch_range(Some("v1:uidnext:1:0"), Some(1)), - FetchRange::CorruptCheckpoint - ); - assert_eq!( - resolve_fetch_range(Some("not:auid"), Some(1)), - FetchRange::CorruptCheckpoint - ); - assert_eq!( - resolve_fetch_range(Some("not-a-uid"), Some(1)).sequence_set(), - "1:*" - ); - } - /// Read a single CRLF-terminated line from a scripted-server stream, - /// returning `None` at EOF (client hung up). Used only by the in-memory - /// duplex tests below; the real fetch path never parses raw lines. + /// returning `None` at EOF (client hung up). async fn read_scripted_line<R: AsyncRead + Unpin>(reader: &mut R) -> Option<String> { let mut buf = Vec::new(); let mut byte = [0u8; 1]; @@ -1655,6 +1844,159 @@ mod tests { } } + /// The body/size fetches from a captured command list, excluding the + /// `(UID)`-only enumeration a non-QRESYNC rung issues to find removals. + /// That pass is expected and says nothing about how the body fetch was + /// narrowed, which is what these tests are about. + fn body_fetch_ranges(all: &[String]) -> Vec<String> { + all.iter() + .filter(|c| !c.to_ascii_uppercase().ends_with("(UID)")) + .cloned() + .collect() + } + + /// Capabilities that can reach every rung, so a test only has to vary the + /// one thing it is about. + fn full_caps() -> MailboxSyncCapabilities { + MailboxSyncCapabilities { + qresync_enabled: true, + condstore: true, + mailbox_has_modseq: true, + } + } + + #[test] + fn checkpoints_round_trip_through_their_scheme() { + assert_eq!( + parse_checkpoint("v2:modseq:42:900"), + Some(Checkpoint::ModSeq { + uid_validity: 42, + modseq: 900 + }) + ); + assert_eq!( + parse_checkpoint("v1:uidnext:42:105"), + Some(Checkpoint::UidNext { + uid_validity: 42, + uid_next: 105 + }) + ); + assert_eq!( + parse_checkpoint("v1:uidnext: 7 : 42 "), + Some(Checkpoint::UidNext { + uid_validity: 7, + uid_next: 42 + }) + ); + } + + #[test] + fn an_untagged_or_unknown_scheme_checkpoint_is_not_interpreted() { + // A QRESYNC checkpoint is (uidvalidity, highestmodseq) -- the same + // shape as (uidvalidity, uidnext) and a different meaning. Reading one + // as the other hands the server a UIDNEXT where it expects a MODSEQ; + // UIDNEXT is normally the larger number, so the server answers + // "nothing changed" and everything below is never enumerated again. + for token in [ + "42:105", // pre-tagging + "v3:future:42:105", // a scheme from a later build + "v1:modseq:42:105", // tag-shaped but not ours + "v1:42:105", // truncated tag + "v1:uidnext:42:0", // a UID boundary below 1 is not addressable + "", + ] { + assert_eq!( + parse_checkpoint(token), + None, + "token {token:?} must not be interpreted" + ); + assert_eq!( + choose_rung(parse_checkpoint(token), Some(42), full_caps()), + SyncRung::Full, + "an uninterpretable token must re-fetch everything" + ); + } + } + + #[test] + fn the_rung_is_chosen_from_the_checkpoint_and_what_the_mailbox_supports() { + let modseq = parse_checkpoint("v2:modseq:42:900"); + let uidnext = parse_checkpoint("v1:uidnext:42:105"); + + // Fastmail/Cyrus, Dovecot: the server states removals directly. + assert_eq!( + choose_rung(modseq, Some(42), full_caps()), + SyncRung::Qresync + ); + + // Gmail: CONDSTORE without QRESYNC. This is the rung a two-rung ladder + // would have skipped, pushing the largest provider onto full sweeps. + assert_eq!( + choose_rung( + modseq, + Some(42), + MailboxSyncCapabilities { + qresync_enabled: false, + condstore: true, + mailbox_has_modseq: true + } + ), + SyncRung::Condstore + ); + + // Exchange: neither extension, so a UID-set diff is all that is left. + assert_eq!( + choose_rung( + uidnext, + Some(42), + MailboxSyncCapabilities { + qresync_enabled: false, + condstore: false, + mailbox_has_modseq: false + } + ), + SyncRung::UidSetDiff + ); + + // NOMODSEQ is a MAILBOX property (RFC 7162 3.1.2.2): a mod-sequence + // checkpoint against a mailbox that cannot answer mod-sequence queries + // has nothing to resume from, whatever the server advertises. + assert_eq!( + choose_rung( + modseq, + Some(42), + MailboxSyncCapabilities { + qresync_enabled: true, + condstore: true, + mailbox_has_modseq: false + } + ), + SyncRung::Full + ); + + // No checkpoint at all. + assert_eq!(choose_rung(None, Some(42), full_caps()), SyncRung::Full); + } + + #[test] + fn a_uidvalidity_change_forces_a_full_pass_from_every_rung() { + // UIDs restart after a UIDVALIDITY change (RFC 3501 2.3.1.1), so every + // stored boundary now names a different message. This is not a + // downgrade to a lesser rung; nothing stored can be resumed from. + for token in ["v2:modseq:42:900", "v1:uidnext:42:105"] { + assert_eq!( + choose_rung(parse_checkpoint(token), Some(43), full_caps()), + SyncRung::Full, + "{token} must not survive a renumbering" + ); + assert_eq!( + choose_rung(parse_checkpoint(token), None, full_caps()), + SyncRung::Full, + "{token}: a server that reports no UIDVALIDITY cannot be resumed against" + ); + } + } + #[tokio::test] async fn incremental_sync_narrows_the_fetch_range_on_the_second_pass() { // Drives a real `async_imap::Session` over an in-memory duplex pair @@ -1677,6 +2019,17 @@ mod tests { let _ = io .write_all(format!("{tag} OK FETCH completed\r\n").as_bytes()) .await; + } else if upper.contains("CAPABILITY") { + // The planner negotiates capabilities before selecting; a + // script that never answers this blocks the client forever. + let _ = io + .write_all( + format!( + "* CAPABILITY IMAP4rev1 UIDPLUS MOVE\r\n{tag} OK CAPABILITY\r\n" + ) + .as_bytes(), + ) + .await; } else if upper.contains("LOGIN") { let _ = io .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) @@ -1708,26 +2061,33 @@ mod tests { // First sync: no checkpoint -> full history; returns // "{uidvalidity}:{uidnext}" from real server state. - let (first, checkpoint) = engine + let first_pass = engine .sync_folder_messages_with_session("INBOX", None, &mut session) .await .expect("first sync succeeds"); + let (first, checkpoint) = (first_pass.upserts, first_pass.next_state); assert!(first.is_empty()); assert_eq!(checkpoint, "v1:uidnext:1:105"); // Second sync: feed back the checkpoint; UIDVALIDITY still matches // (1), so the fetch is narrowed. - let (_second, checkpoint2) = engine + let __changes = engine .sync_folder_messages_with_session("INBOX", Some(&checkpoint), &mut session) .await .expect("second sync succeeds"); + let (_second, checkpoint2) = (__changes.upserts, __changes.next_state); assert_eq!(checkpoint2, "v1:uidnext:1:105"); drop(session); let _ = server.await; - let ranges = fetch_ranges.lock().expect("lock captured ranges"); - assert_eq!(ranges.len(), 2, "expected exactly two UID FETCH commands"); + let all = fetch_ranges.lock().expect("lock captured ranges"); + let ranges = body_fetch_ranges(&all); + assert_eq!( + ranges.len(), + 2, + "expected one body fetch per sync, got: {all:?}" + ); assert!( ranges[0].contains("1:*"), "first sync must fetch full history, got: {}", @@ -1842,6 +2202,17 @@ mod tests { } resp.push_str(&format!("{tag} OK FETCH completed\r\n")); let _ = io.write_all(resp.as_bytes()).await; + } else if upper.contains("CAPABILITY") { + // The planner negotiates capabilities before selecting; a + // script that never answers this blocks the client forever. + let _ = io + .write_all( + format!( + "* CAPABILITY IMAP4rev1 UIDPLUS MOVE\r\n{tag} OK CAPABILITY\r\n" + ) + .as_bytes(), + ) + .await; } else if upper.contains("LOGIN") { let _ = io .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) @@ -1869,10 +2240,11 @@ mod tests { .map_err(|(e, _)| e) .expect("scripted login succeeds"); let engine = ImapEngine::new("acct-1", "example.test", 993); - let (emails, checkpoint) = engine + let changes = engine .sync_folder_messages_batched("INBOX", since_state.as_deref(), &mut session, batch_size) .await .expect("batched sync succeeds"); + let (emails, checkpoint) = (changes.upserts, changes.next_state); drop(session); let _ = server.await; @@ -2008,13 +2380,23 @@ mod tests { checkpoint2, "v1:uidnext:1:4", "checkpoint stays stable across an empty pass" ); + let second_bodies = body_fetch_ranges(&second_cmds); assert!( - second_cmds.iter().any(|c| c.contains("4:*")), - "second sync must enumerate the narrowed range, got: {second_cmds:?}" + second_bodies.iter().any(|c| c.contains("4:*")), + "second sync must fetch only the narrowed range, got: {second_bodies:?}" ); assert!( - !second_cmds.iter().any(|c| c.contains("1:*")), - "second sync must NOT re-enumerate the full range, got: {second_cmds:?}" + !second_bodies.iter().any(|c| c.contains("1:*")), + "second sync must NOT re-fetch bodies for the full range, got: {second_bodies:?}" + ); + // The `1:*` that IS expected: a UID-only enumeration. Without QRESYNC + // there is no other way to learn a message was removed, and it costs + // one short line per message rather than a body. + assert!( + second_cmds + .iter() + .any(|c| c.to_ascii_uppercase().ends_with("(UID)") && c.contains("1:*")), + "a non-QRESYNC rung must enumerate the folder to detect removals, got: {second_cmds:?}" ); } @@ -2042,6 +2424,17 @@ mod tests { let _ = io .write_all(format!("{tag} OK FETCH completed\r\n").as_bytes()) .await; + } else if upper.contains("CAPABILITY") { + // The planner negotiates capabilities before selecting; a + // script that never answers this blocks the client forever. + let _ = io + .write_all( + format!( + "* CAPABILITY IMAP4rev1 UIDPLUS MOVE\r\n{tag} OK CAPABILITY\r\n" + ) + .as_bytes(), + ) + .await; } else if upper.contains("LOGIN") { let _ = io .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) @@ -2076,19 +2469,21 @@ mod tests { let engine = ImapEngine::new("acct-1", "example.test", 993); // First sync under UIDVALIDITY 1 stores "1:105". - let (_first, checkpoint) = engine + let __changes = engine .sync_folder_messages_with_session("INBOX", None, &mut session) .await .expect("first sync succeeds"); + let (_first, checkpoint) = (__changes.upserts, __changes.next_state); assert_eq!(checkpoint, "v1:uidnext:1:105"); // Second sync: server now reports UIDVALIDITY 2. The stored checkpoint // is stale, so the fetch MUST be a full `1:*`, and the checkpoint is // rewritten under the new UIDVALIDITY. - let (_second, checkpoint2) = engine + let __changes = engine .sync_folder_messages_with_session("INBOX", Some(&checkpoint), &mut session) .await .expect("second sync succeeds"); + let (_second, checkpoint2) = (__changes.upserts, __changes.next_state); assert_eq!( checkpoint2, "v1:uidnext:2:105", "checkpoint must be rewritten under the new UIDVALIDITY" @@ -2097,8 +2492,13 @@ mod tests { drop(session); let _ = server.await; - let ranges = fetch_ranges.lock().expect("lock captured ranges"); - assert_eq!(ranges.len(), 2, "expected exactly two UID FETCH commands"); + let all = fetch_ranges.lock().expect("lock captured ranges"); + let ranges = body_fetch_ranges(&all); + assert_eq!( + ranges.len(), + 2, + "expected one body fetch per sync, got: {all:?}" + ); assert!( ranges[0].contains("1:*"), "first sync fetches full history, got: {}", @@ -2127,6 +2527,17 @@ mod tests { if upper.contains("UID FETCH") { // Deliberately never respond: a mid-stream server stall. std::future::pending::<()>().await; + } else if upper.contains("CAPABILITY") { + // The planner negotiates capabilities before selecting; a + // script that never answers this blocks the client forever. + let _ = io + .write_all( + format!( + "* CAPABILITY IMAP4rev1 UIDPLUS MOVE\r\n{tag} OK CAPABILITY\r\n" + ) + .as_bytes(), + ) + .await; } else if upper.contains("LOGIN") { let _ = io .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) @@ -2180,38 +2591,6 @@ mod tests { assert_eq!(parse_imap_uid("jmap-object-id"), None); } - #[test] - fn an_untagged_or_unknown_scheme_checkpoint_forces_a_full_refetch() { - // The hazard this tag exists for. A QRESYNC checkpoint is - // `(uidvalidity, highestmodseq)` -- the same shape as this scheme's - // `(uidvalidity, uidnext)` and a different meaning. Read one as the - // other and the server is handed a UIDNEXT where it expects a MODSEQ; - // since UIDNEXT is normally the larger number it answers "nothing - // changed" and everything below is never enumerated again. Refusing - // to interpret an unrecognised token is the only safe reading. - for token in [ - // Pre-tagging, written by an older build. - "1:105", - // A scheme from a future build. - "v2:qresync:1:900", - // Tag-shaped but not ours. - "v1:modseq:1:105", - // Truncated tag. - "v1:1:105", - ] { - assert_eq!( - resolve_fetch_range(Some(token), Some(1)), - FetchRange::CorruptCheckpoint, - "token {token:?} must not be interpreted as a UIDNEXT boundary" - ); - assert_eq!( - resolve_fetch_range(Some(token), Some(1)).sequence_set(), - "1:*", - "an uninterpretable token must re-fetch everything" - ); - } - } - #[test] fn parse_uid_validity_accepts_only_a_decimal_value() { assert_eq!(parse_uid_validity("7"), Some(7)); @@ -2698,7 +3077,16 @@ mod tests { while let Some(line) = read_scripted_line(&mut io).await { let tag = line.split_whitespace().next().unwrap_or("").to_string(); let upper = line.to_ascii_uppercase(); - if upper.contains("LOGIN") { + if upper.contains("CAPABILITY") { + let _ = io + .write_all( + format!( + "* CAPABILITY IMAP4rev1 UIDPLUS MOVE\r\n{tag} OK CAPABILITY\r\n" + ) + .as_bytes(), + ) + .await; + } else if upper.contains("LOGIN") { let _ = io .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) .await; @@ -2730,10 +3118,11 @@ mod tests { .expect("scripted login succeeds"); let engine = ImapEngine::new("acct-completion", "example.test", 993); - let (emails, _checkpoint) = engine + let __changes = engine .sync_folder_messages_with_session("INBOX", None, &mut session) .await .expect("sync succeeds"); + let (emails, _checkpoint) = (__changes.upserts, __changes.next_state); assert!(emails.is_empty()); drop(session); diff --git a/crates/nuncio-mail/src/imap_raw.rs b/crates/nuncio-mail/src/imap_raw.rs index be2b321..cee5f64 100644 --- a/crates/nuncio-mail/src/imap_raw.rs +++ b/crates/nuncio-mail/src/imap_raw.rs @@ -42,6 +42,14 @@ use tokio::io::{AsyncRead, AsyncWrite}; use crate::parser::MailError; +/// Bounded wait for any single response line. +/// +/// A server that accepts a command and then stops answering must surface as a +/// stall, not as a sync that never returns. Mirrors the per-item FETCH timeout +/// the typed path already applies -- an unbounded read here would reintroduce +/// exactly the hang that bounded it. +const RESPONSE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + /// A `COPYUID` response code (RFC 4315 section 3): the destination mailbox's /// UIDVALIDITY, the source UIDs, and the UIDs they became. /// @@ -84,6 +92,8 @@ pub struct RawExchange { pub highest_modseq: Option<u64>, /// UIDVALIDITY, from an untagged `OK` during `SELECT`. pub uid_validity: Option<u32>, + /// UIDNEXT, from an untagged `OK` during `SELECT`. + pub uid_next: Option<u32>, } impl RawExchange { @@ -170,9 +180,13 @@ where let mut exchange = RawExchange::default(); loop { - let response = session - .read_response() + let response = tokio::time::timeout(RESPONSE_READ_TIMEOUT, session.read_response()) .await + .map_err(|_| { + MailError::FetchStalled(format!( + "no response to '{command}' within {RESPONSE_READ_TIMEOUT:?}" + )) + })? .map_err(|e| MailError::ImapError(format!("reading response to '{command}': {e}")))? .ok_or_else(|| { MailError::ImapError(format!("connection closed before '{command}' completed")) @@ -232,6 +246,7 @@ fn absorb_code(exchange: &mut RawExchange, code: Option<&ResponseCode<'_>>) { } Some(ResponseCode::HighestModSeq(modseq)) => exchange.highest_modseq = Some(*modseq), Some(ResponseCode::UidValidity(validity)) => exchange.uid_validity = Some(*validity), + Some(ResponseCode::UidNext(uid_next)) => exchange.uid_next = Some(*uid_next), _ => {} } } diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index 80319b7..7d63652 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -561,11 +561,11 @@ impl MailBackend for JmapEngine { fields(account_id = %self.account_id, folder_id = %folder_id), err )] - async fn sync_messages( + async fn sync_changes( &self, folder_id: &str, _since_state: Option<&str>, - ) -> Result<(Vec<Email>, String), MailError> { + ) -> Result<crate::backend::FolderChanges, MailError> { if !self.has_credentials() { return Err(MailError::AuthError( "JMAP message sync requires credentials".to_string(), @@ -589,7 +589,16 @@ impl MailBackend for JmapEngine { "jmap message sync complete" ); - Ok((emails, state)) + Ok(crate::backend::FolderChanges { + upserts: emails, + removals: Vec::new(), + // `Email/query` was filtered to one mailbox, but this engine does + // not yet use `Email/changes`, so it cannot claim to have seen the + // complete set. Reporting `None` keeps the caller from deleting + // anything on the strength of a partial view. + present: None, + next_state: state, + }) } #[tracing::instrument(skip(self), fields(account_id = %self.account_id), err)] diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index 487d09a..adbf97b 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -106,11 +106,11 @@ impl MailBackend for MockMailBackend { Ok(folders.clone()) } - async fn sync_messages( + async fn sync_changes( &self, folder_id: &str, since_state: Option<&str>, - ) -> Result<(Vec<Email>, String), MailError> { + ) -> Result<crate::backend::FolderChanges, MailError> { // Record the checkpoint arg before any early return, so even a failing // call is visible to a test asserting on how the caller resumes. if let Ok(mut calls) = self.since_state_calls.lock() { @@ -136,19 +136,30 @@ impl MailBackend for MockMailBackend { // is nothing new to hand back, so the incremental fetch is genuinely // narrower (empty) rather than a full re-report of every message. if since_state.is_some() { - return Ok((Vec::new(), returned_state)); + return Ok(crate::backend::FolderChanges { + next_state: returned_state, + ..Default::default() + }); } let messages = self .messages .lock() .map_err(|e| MailError::ParseFailed(e.to_string()))?; - let matches = messages + let matches: Vec<_> = messages .iter() .filter(|m| m.folder_id == folder_id) .cloned() .collect(); - Ok((matches, returned_state)) + // A full pass over the mock's store did see everything in the folder, + // so it can honestly report what is present. + let present = matches.iter().map(|m| m.id.clone()).collect(); + Ok(crate::backend::FolderChanges { + upserts: matches, + removals: Vec::new(), + present: Some(present), + next_state: returned_state, + }) } async fn apply_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError> { diff --git a/crates/nuncio-mail/tests/imap_raw_wire_test.rs b/crates/nuncio-mail/tests/imap_raw_wire_test.rs index 8793b3d..9d03e60 100644 --- a/crates/nuncio-mail/tests/imap_raw_wire_test.rs +++ b/crates/nuncio-mail/tests/imap_raw_wire_test.rs @@ -299,3 +299,182 @@ async fn a_qresync_select_without_enable_is_refused() { "the refusal must be surfaced, got: {err}" ); } + +/// Drive a real folder sync against the mock and return what it observed. +async fn sync_folder( + server: &MockImapServer, + since: Option<&str>, +) -> nuncio_mail::backend::FolderChanges { + use nuncio_mail::MailBackend; + let engine = nuncio_mail::ImapEngine::with_credentials( + "acct-1", + server.host(), + server.port(), + nuncio_core::TlsMode::Plain, + "user", + "pass", + ); + engine + .sync_changes("INBOX", since) + .await + .expect("folder sync succeeds") +} + +#[tokio::test] +async fn every_rung_reports_a_message_that_disappeared() { + // The defect this whole change exists for. A forward-only sync can only + // ever add, so a message removed by another client stays local forever. + // Each rung must be able to say it is gone -- QRESYNC because the server + // states it, the other two because the pass enumerated what remains. + for profile in [ + ServerProfile::Qresync, + ServerProfile::Condstore, + ServerProfile::Basic, + ] { + let server = MockImapServer::start(profile).await.expect("server starts"); + server.create_mailbox("INBOX"); + let keep = server.append_message("INBOX", "Subject: keep\r\n\r\nkeep", &[]); + let gone = server.append_message("INBOX", "Subject: gone\r\n\r\ngone", &[]); + + let first = sync_folder(&server, None).await; + assert_eq!(first.upserts.len(), 2, "{profile:?}: first pass sees both"); + + // Another client removes one while we are away. + assert!(server.expunge("INBOX", gone)); + + let second = sync_folder(&server, Some(&first.next_state)).await; + + // However the rung learned it, the caller must end up able to identify + // the removed message and only that one. + let mut reported: Vec<String> = second.removals.clone(); + if let Some(present) = &second.present { + let stored_gone = first + .upserts + .iter() + .map(|e| e.id.clone()) + .filter(|id| !present.contains(id)); + reported.extend(stored_gone); + } + reported.sort(); + reported.dedup(); + + assert_eq!( + reported.len(), + 1, + "{profile:?}: exactly one message must be reported gone, got {reported:?}" + ); + let surviving = first + .upserts + .iter() + .find(|e| e.remote_id == keep.to_string()) + .expect("the kept message was in the first pass"); + assert!( + !reported.contains(&surviving.id), + "{profile:?}: the surviving message must not be reported gone" + ); + } +} + +#[tokio::test] +async fn an_incremental_pass_that_cannot_see_absence_says_so() { + // `present: None` versus `Some(vec![])` is the difference between "I did + // not look" and "the folder is empty". Conflating them deletes a mailbox, + // so QRESYNC -- which never enumerates -- must report None. + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.append_message("INBOX", "Subject: one\r\n\r\nx", &[]); + + let first = sync_folder(&server, None).await; + let second = sync_folder(&server, Some(&first.next_state)).await; + + assert!( + second.next_state.starts_with("v2:modseq:"), + "a QRESYNC-capable mailbox must checkpoint by mod-sequence, got {:?}", + second.next_state + ); + assert_eq!( + second.present, None, + "a QRESYNC pass does not enumerate the folder, so it cannot claim to know what is present" + ); +} + +#[tokio::test] +async fn a_condstore_only_server_still_checkpoints_by_modseq_and_enumerates() { + // Gmail's rung. It cannot be told what vanished, so it must enumerate -- + // and it must still upgrade its checkpoint so the next pass is narrow. + let server = MockImapServer::start(ServerProfile::Condstore) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.append_message("INBOX", "Subject: one\r\n\r\nx", &[]); + + let first = sync_folder(&server, None).await; + assert!( + first.next_state.starts_with("v2:modseq:"), + "CONDSTORE alone is enough to checkpoint by mod-sequence, got {:?}", + first.next_state + ); + + let second = sync_folder(&server, Some(&first.next_state)).await; + assert!( + second.present.is_some(), + "without QRESYNC the only way to find removals is to enumerate" + ); + assert!( + second.upserts.is_empty(), + "nothing changed, so CHANGEDSINCE must fetch no bodies" + ); +} + +#[tokio::test] +async fn a_server_without_modseq_falls_back_to_a_uid_boundary_checkpoint() { + // Exchange's rung: no CONDSTORE, no QRESYNC. + let server = MockImapServer::start(ServerProfile::Basic) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.append_message("INBOX", "Subject: one\r\n\r\nx", &[]); + + let first = sync_folder(&server, None).await; + assert!( + first.next_state.starts_with("v1:uidnext:"), + "a mailbox with no mod-sequences must checkpoint by UID boundary, got {:?}", + first.next_state + ); + assert!(first.present.is_some()); +} + +#[tokio::test] +async fn a_uidvalidity_change_refetches_rather_than_deleting_the_folder() { + // After a renumbering every stored UID names a different message. The + // dangerous outcome would be enumerating the new UID space, finding none + // of the old ids in it, and concluding the whole folder was deleted. + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.append_message("INBOX", "Subject: before\r\n\r\nx", &[]); + + let first = sync_folder(&server, None).await; + assert_eq!(first.upserts.len(), 1); + + server.bump_uid_validity("INBOX"); + server.append_message("INBOX", "Subject: after\r\n\r\nx", &[]); + + let second = sync_folder(&server, Some(&first.next_state)).await; + + assert_eq!( + second.upserts.len(), + 1, + "a renumbered mailbox must be re-fetched in full" + ); + assert!( + second.removals.is_empty(), + "a renumbering is not a removal report" + ); + // The re-fetched message has a different surrogate, because UIDVALIDITY is + // part of identity -- so the caller replaces rather than merges. + assert_ne!(second.upserts[0].id, first.upserts[0].id); +} diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index c34829a..7bd02f9 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -1707,6 +1707,51 @@ impl DatabaseEngine { /// Lets a caller classify a whole fetched batch as new-vs-seen with one round trip instead /// of one existence lookup per id. Passing an empty slice short-circuits to an empty set /// without querying at all, since `IN ()` is not valid SQL. + /// Every message id currently stored for one folder of one account. + /// + /// The local half of a UID-set diff: a sync that enumerated the folder's + /// complete contents can subtract what the server reported from this to + /// find what disappeared. Scoped by account as well as folder because + /// folder ids are not globally unique -- two accounts both have an + /// `INBOX`, and diffing across them would delete one account's mail + /// because the other's server did not mention it. + pub async fn message_ids_in_folder( + &self, + account_id: &str, + folder_id: &str, + ) -> Result<Vec<String>, DatabaseError> { + let rows: Vec<(String,)> = + sqlx::query_as("SELECT id FROM messages WHERE account_id = ? AND folder_id = ?") + .bind(account_id) + .bind(folder_id) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + Ok(rows.into_iter().map(|(id,)| id).collect()) + } + + /// Delete stored messages by id, returning how many rows went. + /// + /// The FTS index follows via the `messages_ad` trigger, so a deleted + /// message cannot survive as a searchable plaintext orphan. + pub async fn delete_messages(&self, ids: &[String]) -> Result<u64, DatabaseError> { + if ids.is_empty() { + return Ok(0); + } + let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + let mut deleted = 0u64; + for id in ids { + let result = sqlx::query("DELETE FROM messages WHERE id = ?") + .bind(id) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + deleted += result.rows_affected(); + } + tx.commit().await.map_err(DatabaseError::Query)?; + Ok(deleted) + } + pub async fn existing_message_ids( &self, ids: &[String], diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index 0c5036a..aca69d5 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -245,10 +245,12 @@ async fn fetch_and_persist( Some(acct) => db.get_folder_sync_state(acct, &folder.id).await?, None => None, }; - let (emails, new_state) = backend - .sync_messages(&folder.id, last_state.as_deref()) + let changes = backend + .sync_changes(&folder.id, last_state.as_deref()) .await?; + let emails = changes.upserts; let fetched = emails.len(); + let new_state = changes.next_state; // One round trip classifies the whole chunk instead of one lookup per // message. A lookup failure that is not "not found" would be a @@ -268,6 +270,37 @@ async fn fetch_and_persist( } synced += 1; } + // Reconcile what the server no longer has. Until this existed, a sync + // could only ever add: a message moved or deleted by any other client + // -- another daemon, a phone, webmail -- stayed in the local store + // forever, and every account accumulated ghosts. + if let Some(acct) = account_id { + let mut gone: Vec<String> = changes.removals; + + // `present` is the folder's complete contents when the pass was + // able to enumerate them. `None` and `Some(vec![])` are NOT the + // same: `None` means the pass was incremental and cannot speak to + // absence, and treating it as "nothing is present" would delete + // the folder. + if let Some(present) = changes.present { + let present: std::collections::HashSet<String> = present.into_iter().collect(); + let stored = db.message_ids_in_folder(acct, &folder.id).await?; + gone.extend(stored.into_iter().filter(|id| !present.contains(id))); + } + + gone.sort_unstable(); + gone.dedup(); + if !gone.is_empty() { + let deleted = db.delete_messages(&gone).await?; + tracing::info!( + account_id = %acct, + folder_id = %folder.id, + deleted, + "removed messages that are no longer on the server" + ); + } + } + // Persist the returned checkpoint only AFTER this folder's messages // have landed, so the stored high-water mark can never advance past // work that actually reached the store. -- 2.54.0 From 5a03ab13e394ef8ddf802de1b5a842fdc85a9a68 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Mon, 10 Aug 2026 17:06:02 -0500 Subject: [PATCH 59/95] feat(mail): report what a mutation proved, not that it was accepted A mutation reported success whenever the server answered OK. IMAP answers OK for a UID command that matched nothing (RFC 3501 6.4.8), so a mutation that lost a race to another client -- the message already moved, already deleted -- was recorded as completed having done nothing. The user's intent was silently dropped and the outbox row was marked done. Two states cannot express this. "Succeeded" and "failed" leave out the case that actually occurs: the server accepted the command and its response does not establish what happened. Collapsing that into either neighbour is the bug, so mutations now return Applied, Conflict, or Unknown. Applied requires positive evidence. A COPYUID naming destination UIDs proves a move; a removal report proves a delete; an echoed FETCH proves a flag change. Conflict requires positive evidence too -- a MODIFIED response code -- because MODIFIED means "changed by someone else", never "gone". Everything else is Unknown, which is the honest reading of an absent COPYUID: only SHOULD for MOVE (RFC 6851 4.3), legitimately omitted for a UIDNOTSTICKY destination, and indistinguishable from a UID set that matched nothing. The outbox treats Unknown as retryable. A mutation that did apply and is re-attempted is a no-op; one that did not and is recorded complete is unrecoverable, so the asymmetry decides it. Conflict is terminal and visible instead: the intent was formed against state that no longer exists, and re-issuing it would either do nothing or overwrite someone else's change. The delete path now reads its proof through the collecting layer, because a QRESYNC-enabled server reports VANISHED rather than EXPUNGE (RFC 6851 4.4) and the typed expunge stream only yields the latter -- reading it alone reports nothing removed on every successful delete. Refs #391 --- crates/nuncio-mail/src/backend.rs | 58 ++++- crates/nuncio-mail/src/imap.rs | 217 ++++++++++++------ crates/nuncio-mail/src/jmap.rs | 14 +- crates/nuncio-mail/src/lib.rs | 3 +- crates/nuncio-mail/src/mock.rs | 11 +- .../nuncio-mail/tests/imap_raw_wire_test.rs | 134 +++++++++++ crates/nunciod/src/outbox.rs | 33 ++- 7 files changed, 390 insertions(+), 80 deletions(-) diff --git a/crates/nuncio-mail/src/backend.rs b/crates/nuncio-mail/src/backend.rs index c776170..f25e9bf 100644 --- a/crates/nuncio-mail/src/backend.rs +++ b/crates/nuncio-mail/src/backend.rs @@ -91,6 +91,50 @@ pub struct FolderChanges { pub next_state: String, } +/// What a mutation attempt actually established. +/// +/// Two states are not enough. A protocol that answers `OK` for a command that +/// did nothing -- which IMAP does, by design, for a UID that no longer exists +/// (RFC 3501 section 6.4.8) -- makes "succeeded" and "failed" an incomplete +/// partition. The missing third state is *the server accepted the command and +/// its response does not establish what happened*, and collapsing that into +/// either neighbour is how a lost mutation gets recorded as done. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MutationOutcome { + /// The server's response proves the change was made. + /// + /// Requires positive evidence: a `COPYUID` naming the destination UIDs, a + /// removal report for a delete, or an explicit acknowledgement for a flag + /// change. `token` carries whatever the server returned to address the + /// result afterwards (the destination UID for a move), when it returned + /// one at all -- a DAV server that rewrites the object MUST NOT return an + /// ETag (RFC 4791 section 5.3.4), so absence here is ordinary. + Applied { + /// Server-assigned handle for the result, when the response carried one. + token: Option<String>, + }, + /// Another client changed the message first, and the server said so. + /// + /// Requires positive evidence too: `[MODIFIED …]` (RFC 7162 section 3.1.3) + /// or an HTTP 412. A mutation that merely failed to find its target is + /// **not** a conflict -- `MODIFIED` reports "changed by someone else", + /// never "gone". + Conflict { + /// What the server reported about the conflicting state. + observed: String, + }, + /// The command was accepted and its response settles nothing. + /// + /// The honest reading of an absent `COPYUID` (only `SHOULD` for `MOVE` per + /// RFC 6851 section 4.3, and legitimately omitted for a `UIDNOTSTICKY` + /// destination), or of a UID set that matched nothing. The caller must + /// re-enumerate to find out, not guess. + Unknown { + /// Why the response was inconclusive. + reason: String, + }, +} + /// Protocol-agnostic mail backend engine trait implemented by JMAP and IMAP engines. #[async_trait] pub trait MailBackend: Send + Sync { @@ -124,11 +168,15 @@ pub trait MailBackend: Send + Sync { } /// Apply a remote mutation (move/copy/flag/delete) to a single message - /// against the real server. MUST return `Ok(())` only when the server - /// genuinely applied the change -- implementations must never fabricate - /// success, and must refuse to act (returning an error) when the target - /// message cannot be safely identified (e.g. an IMAP UIDVALIDITY mismatch). - async fn apply_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError>; + /// against the real server. + /// + /// Returns what the server's response actually proves. Implementations + /// must never report [`MutationOutcome::Applied`] on the strength of a + /// tagged `OK` alone, and must refuse to act (returning `Err`) when the + /// target cannot be safely identified -- e.g. an IMAP UIDVALIDITY + /// mismatch, where the stored UID now names a different message. + async fn apply_mutation(&self, spec: &RemoteMutationSpec) + -> Result<MutationOutcome, MailError>; } /// A composed outbound email message ready to send over SMTP. Deliberately diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index 6e883bc..ac273d7 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -52,7 +52,7 @@ const MAX_MESSAGES_PER_FETCH_BATCH: usize = 200; /// would ever accept is pointless to transfer only to reject. const MAX_MESSAGE_BODY_BYTES: u32 = 25 * 1024 * 1024; -use crate::backend::{MailBackend, RemoteMutationKind, RemoteMutationSpec}; +use crate::backend::{MailBackend, MutationOutcome, RemoteMutationKind, RemoteMutationSpec}; use crate::parser::{MailError, MimeParserAdapter}; /// Recover a raw IMAP UID from a [`RemoteMutationSpec::remote_id`] (a decimal @@ -102,6 +102,34 @@ const CHECKPOINT_DELIM: char = ':'; /// full re-fetch. Over-fetching is recoverable; misreading a token is not. const CHECKPOINT_SCHEME_UIDNEXT: &str = "v1:uidnext"; +/// Read a `COPY`/`MOVE` outcome from whether the server proved it. +/// +/// `COPYUID` present is proof and names the destination UIDs. Absent is +/// **unknown, never failure**: RFC 6851 section 4.3 makes it only `SHOULD` for +/// `MOVE`, RFC 4315 section 3 permits omitting it for a `UIDNOTSTICKY` +/// destination or one the client cannot `SELECT`, and a UID set matching +/// nothing succeeds having done nothing (RFC 3501 section 6.4.8). Those are +/// indistinguishable from here, and the caller has to re-enumerate to tell. +fn copy_outcome(copy_uid: Option<crate::imap_raw::CopyUid>, op: &str) -> MutationOutcome { + match copy_uid { + Some(proof) if !proof.destination_uids.is_empty() => MutationOutcome::Applied { + token: Some( + proof + .destination_uids + .iter() + .map(u32::to_string) + .collect::<Vec<_>>() + .join(","), + ), + }, + _ => MutationOutcome::Unknown { + reason: format!( + "the server accepted the UID {op} without a COPYUID, so nothing proves it moved" + ), + }, + } +} + /// Scheme for a mod-sequence checkpoint: `"v2:modseq:{uidvalidity}:{modseq}"`. /// /// Written by both mod-sequence rungs. QRESYNC and CONDSTORE differ in how @@ -1343,7 +1371,7 @@ impl ImapEngine { &self, spec: &RemoteMutationSpec, session: &mut async_imap::Session<S>, - ) -> Result<(), MailError> + ) -> Result<MutationOutcome, MailError> where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { @@ -1411,54 +1439,82 @@ impl ImapEngine { match &spec.kind { RemoteMutationKind::SetFlagged { value } => { let op = if *value { "+FLAGS" } else { "-FLAGS" }; - self.uid_store_flags(session, &uid_set, op, "\\Flagged") - .await + let exchange = crate::imap_raw::run_collected( + session, + &format!("UID STORE {uid_set} {op} (\\Flagged)"), + ) + .await?; + if let Some(conflicting) = exchange.modified_uids() { + return Ok(MutationOutcome::Conflict { + observed: format!("server reported MODIFIED for uid(s) {conflicting:?}"), + }); + } + if !exchange.ok { + return Err(MailError::ImapError(format!( + "UID STORE {op} for uid {uid_set} failed: {}", + exchange.information.as_deref().unwrap_or("no detail") + ))); + } + // An untagged FETCH echoing the new flags is proof. Its absence + // is not failure: RFC 3501 section 6.4.6 lets a server suppress + // it, and a store that changes nothing legitimately produces + // none -- so the honest answer is that nothing was established. + if exchange.fetched_uids.is_empty() { + Ok(MutationOutcome::Unknown { + reason: "the server acknowledged the store without echoing the message" + .to_string(), + }) + } else { + Ok(MutationOutcome::Applied { token: None }) + } } RemoteMutationKind::Copy { to_folder } => { - session.uid_copy(&uid_set, to_folder).await.map_err(|e| { - MailError::ImapError(format!( + let exchange = crate::imap_raw::run_collected( + session, + &format!("UID COPY {uid_set} {to_folder}"), + ) + .await?; + if !exchange.ok { + return Err(MailError::ImapError(format!( "UID COPY of '{}' to '{}' failed: {}", - spec.message_id, to_folder, e - )) - }) + spec.message_id, + to_folder, + exchange.information.as_deref().unwrap_or("no detail") + ))); + } + Ok(copy_outcome(exchange.copy_uid, "COPY")) } RemoteMutationKind::Move { to_folder } => { self.uid_move(session, &uid_set, to_folder, &spec.message_id) .await } RemoteMutationKind::Delete => { - self.uid_store_flags(session, &uid_set, "+FLAGS", "\\Deleted") - .await?; - self.uid_expunge_scoped(session, &uid_set).await + let store = crate::imap_raw::run_collected( + session, + &format!("UID STORE {uid_set} +FLAGS (\\Deleted)"), + ) + .await?; + if !store.ok { + return Err(MailError::ImapError(format!( + "UID STORE +FLAGS (\\Deleted) for uid {uid_set} failed: {}", + store.information.as_deref().unwrap_or("no detail") + ))); + } + let expunged = self.uid_expunge_scoped(session, &uid_set).await?; + // Under QRESYNC the removal is reported as VANISHED rather than + // EXPUNGE (RFC 6851 section 4.4), so counting only EXPUNGE + // would report failure on every successful delete. + if expunged { + Ok(MutationOutcome::Applied { token: None }) + } else { + Ok(MutationOutcome::Unknown { + reason: "the server reported no removal for the expunged uid".to_string(), + }) + } } } } - /// Issue a `UID STORE {uid} {op} ({flag})` and drain its untagged `FETCH` - /// responses, surfacing any protocol error. `op` is `+FLAGS`/`-FLAGS`. - async fn uid_store_flags<S>( - &self, - session: &mut async_imap::Session<S>, - uid_set: &str, - op: &str, - flag: &str, - ) -> Result<(), MailError> - where - S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, - { - let query = format!("{op} ({flag})"); - let updates = session.uid_store(uid_set, &query).await.map_err(|e| { - MailError::ImapError(format!("UID STORE {query} for uid {uid_set} failed: {e}")) - })?; - tokio::pin!(updates); - while let Some(item) = updates.next().await { - item.map_err(|e| { - MailError::ImapError(format!("reading UID STORE response failed: {e}")) - })?; - } - Ok(()) - } - /// Permanently remove ONLY the target `uid_set` from the selected folder /// via RFC 4315 `UID EXPUNGE`, draining its untagged `EXPUNGE` responses. /// @@ -1473,7 +1529,7 @@ impl ImapEngine { &self, session: &mut async_imap::Session<S>, uid_set: &str, - ) -> Result<(), MailError> + ) -> Result<bool, MailError> where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { @@ -1491,17 +1547,19 @@ impl ImapEngine { ))); } - let expunged = session - .uid_expunge(uid_set) - .await - .map_err(|e| MailError::ImapError(format!("UID EXPUNGE {uid_set} failed: {e}")))?; - tokio::pin!(expunged); - while let Some(item) = expunged.next().await { - item.map_err(|e| { - MailError::ImapError(format!("reading UID EXPUNGE response failed: {e}")) - })?; + // Collected rather than streamed, because a QRESYNC-enabled server + // answers with VANISHED instead of EXPUNGE (RFC 6851 section 4.4) and + // the typed expunge stream only yields the latter. Reading just that + // stream reports "nothing removed" on every successful delete. + let exchange = + crate::imap_raw::run_collected(session, &format!("UID EXPUNGE {uid_set}")).await?; + if !exchange.ok { + return Err(MailError::ImapError(format!( + "UID EXPUNGE {uid_set} failed: {}", + exchange.information.as_deref().unwrap_or("no detail") + ))); } - Ok(()) + Ok(!exchange.vanished_uids.is_empty() || !exchange.expunged_seqs.is_empty()) } /// Move a message by UID, preferring RFC 6851 `UID MOVE` and falling back @@ -1515,7 +1573,7 @@ impl ImapEngine { uid_set: &str, to_folder: &str, message_id: &str, - ) -> Result<(), MailError> + ) -> Result<MutationOutcome, MailError> where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { @@ -1527,27 +1585,53 @@ impl ImapEngine { drop(caps); if has_move { - return session.uid_mv(uid_set, to_folder).await.map_err(|e| { - MailError::ImapError(format!( - "UID MOVE of '{message_id}' to '{to_folder}' failed: {e}" - )) - }); + let exchange = + crate::imap_raw::run_collected(session, &format!("UID MOVE {uid_set} {to_folder}")) + .await?; + if !exchange.ok { + return Err(MailError::ImapError(format!( + "UID MOVE of '{message_id}' to '{to_folder}' failed: {}", + exchange.information.as_deref().unwrap_or("no detail") + ))); + } + return Ok(copy_outcome(exchange.copy_uid, "MOVE")); } - session.uid_copy(uid_set, to_folder).await.map_err(|e| { - MailError::ImapError(format!( - "UID COPY (move fallback) of '{message_id}' to '{to_folder}' failed: {e}" - )) - })?; - self.uid_store_flags(session, uid_set, "+FLAGS", "\\Deleted") - .await?; - self.uid_expunge_scoped(session, uid_set).await + let copied = + crate::imap_raw::run_collected(session, &format!("UID COPY {uid_set} {to_folder}")) + .await?; + if !copied.ok { + return Err(MailError::ImapError(format!( + "UID COPY (move fallback) of '{message_id}' to '{to_folder}' failed: {}", + copied.information.as_deref().unwrap_or("no detail") + ))); + } + // The copy half is what the destination proof refers to; the source + // removal below is separately verified, and a failure there leaves a + // duplicate rather than a lost message. + let outcome = copy_outcome(copied.copy_uid, "COPY"); + let stored = crate::imap_raw::run_collected( + session, + &format!("UID STORE {uid_set} +FLAGS (\\Deleted)"), + ) + .await?; + if !stored.ok { + return Err(MailError::ImapError(format!( + "UID STORE +FLAGS (\\Deleted) during move fallback failed: {}", + stored.information.as_deref().unwrap_or("no detail") + ))); + } + self.uid_expunge_scoped(session, uid_set).await?; + Ok(outcome) } /// Apply a remote mutation over a freshly authenticated session. See /// [`Self::apply_mutation_with_session`] for the UIDVALIDITY guard and the /// per-action protocol commands. - pub async fn apply_remote_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError> { + pub async fn apply_remote_mutation( + &self, + spec: &RemoteMutationSpec, + ) -> Result<MutationOutcome, MailError> { let (username, password) = match (&self.username, &self.password) { (Some(u), Some(p)) => (u.as_str(), p.as_str()), _ => { @@ -1713,7 +1797,10 @@ impl MailBackend for ImapEngine { self.sync_folder_messages(folder_id, since_state).await } - async fn apply_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError> { + async fn apply_mutation( + &self, + spec: &RemoteMutationSpec, + ) -> Result<MutationOutcome, MailError> { self.apply_remote_mutation(spec).await } } @@ -2612,7 +2699,7 @@ mod tests { advertise_move: bool, advertise_uidplus: bool, select_uid_validity: u32, - ) -> (Result<(), MailError>, Vec<String>) { + ) -> (Result<MutationOutcome, MailError>, Vec<String>) { let (client_io, server_io) = tokio::io::duplex(8192); let commands = Arc::new(std::sync::Mutex::new(Vec::<String>::new())); let server_commands = commands.clone(); diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index 7d63652..d09d12f 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -5,7 +5,7 @@ use nuncio_core::model::{Email, Folder}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use crate::backend::{MailBackend, RemoteMutationKind, RemoteMutationSpec}; +use crate::backend::{MailBackend, MutationOutcome, RemoteMutationKind, RemoteMutationSpec}; use crate::parser::MailError; /// Stand-in for the UIDVALIDITY scope on JMAP, which has no such concept: a @@ -602,7 +602,10 @@ impl MailBackend for JmapEngine { } #[tracing::instrument(skip(self), fields(account_id = %self.account_id), err)] - async fn apply_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError> { + async fn apply_mutation( + &self, + spec: &RemoteMutationSpec, + ) -> Result<MutationOutcome, MailError> { if !self.has_credentials() { return Err(MailError::AuthError( "JMAP remote mutation requires credentials".to_string(), @@ -639,7 +642,12 @@ impl MailBackend for JmapEngine { let request = Self::build_email_set_request(&account_id, spec); let raw = self.post_jmap(&session.api_url, &request).await?; let expect_destroy = matches!(spec.kind, RemoteMutationKind::Delete); - Self::confirm_email_set_applied(&raw, &spec.remote_id, expect_destroy) + // `Email/set` names the ids it updated or destroyed, so a confirmed + // response is genuine proof rather than a bare acknowledgement. JMAP + // has no COPYUID-style ambiguity here; the id is stable and the server + // either lists it or does not. + Self::confirm_email_set_applied(&raw, &spec.remote_id, expect_destroy)?; + Ok(MutationOutcome::Applied { token: None }) } } diff --git a/crates/nuncio-mail/src/lib.rs b/crates/nuncio-mail/src/lib.rs index fa913fe..9fc19ad 100644 --- a/crates/nuncio-mail/src/lib.rs +++ b/crates/nuncio-mail/src/lib.rs @@ -18,7 +18,8 @@ pub mod smtp; pub mod test_server; pub use backend::{ - MailBackend, MessageSender, OutboundMessage, RemoteMutationKind, RemoteMutationSpec, + FolderChanges, MailBackend, MessageSender, MutationOutcome, OutboundMessage, + RemoteMutationKind, RemoteMutationSpec, }; pub use imap::{IdleSocketState, ImapDualSocketManager, ImapEngine}; pub use jmap::JmapEngine; diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index adbf97b..f592192 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -4,7 +4,9 @@ use async_trait::async_trait; use nuncio_core::model::{Email, Folder}; use std::sync::{Arc, Mutex}; -use crate::backend::{MailBackend, MessageSender, OutboundMessage, RemoteMutationSpec}; +use crate::backend::{ + MailBackend, MessageSender, MutationOutcome, OutboundMessage, RemoteMutationSpec, +}; use crate::parser::MailError; /// Thread-safe mock mail backend for offline testing. @@ -162,7 +164,10 @@ impl MailBackend for MockMailBackend { }) } - async fn apply_mutation(&self, spec: &RemoteMutationSpec) -> Result<(), MailError> { + async fn apply_mutation( + &self, + spec: &RemoteMutationSpec, + ) -> Result<MutationOutcome, MailError> { let should_fail = self .should_fail .lock() @@ -179,7 +184,7 @@ impl MailBackend for MockMailBackend { .lock() .map_err(|e| MailError::ParseFailed(e.to_string()))?; applied.push(spec.clone()); - Ok(()) + Ok(MutationOutcome::Applied { token: None }) } } diff --git a/crates/nuncio-mail/tests/imap_raw_wire_test.rs b/crates/nuncio-mail/tests/imap_raw_wire_test.rs index 9d03e60..42a67a7 100644 --- a/crates/nuncio-mail/tests/imap_raw_wire_test.rs +++ b/crates/nuncio-mail/tests/imap_raw_wire_test.rs @@ -478,3 +478,137 @@ async fn a_uidvalidity_change_refetches_rather_than_deleting_the_folder() { // part of identity -- so the caller replaces rather than merges. assert_ne!(second.upserts[0].id, first.upserts[0].id); } + +/// Build an engine pointed at the mock and apply one mutation through the +/// real backend path. +async fn apply( + server: &MockImapServer, + kind: nuncio_mail::RemoteMutationKind, + uid: u32, + uid_validity: &str, +) -> nuncio_mail::MutationOutcome { + use nuncio_mail::MailBackend; + let engine = nuncio_mail::ImapEngine::with_credentials( + "acct-1", + server.host(), + server.port(), + nuncio_core::TlsMode::Plain, + "user", + "pass", + ); + engine + .apply_mutation(&nuncio_mail::RemoteMutationSpec { + message_id: "m-1".to_string(), + remote_id: uid.to_string(), + folder_id: "INBOX".to_string(), + uid_validity: uid_validity.to_string(), + kind, + }) + .await + .expect("the mutation attempt itself succeeds") +} + +/// UIDVALIDITY the mock assigns to the first mailbox created. +const FIRST_UID_VALIDITY: &str = "1000"; + +#[tokio::test] +async fn a_move_that_lost_a_race_is_unknown_rather_than_applied() { + // The silent-loss case. The server answers OK for a UID that no longer + // exists (RFC 3501 6.4.8), so only the absence of COPYUID distinguishes a + // real move from a no-op -- and absence cannot mean "applied". + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.create_mailbox("Archive"); + server.create_mailbox("Trash"); + let uid = server.append_message("INBOX", "Subject: contested\r\n\r\nx", &[]); + + let applied = apply( + &server, + nuncio_mail::RemoteMutationKind::Move { + to_folder: "Archive".to_string(), + }, + uid, + FIRST_UID_VALIDITY, + ) + .await; + assert!( + matches!( + applied, + nuncio_mail::MutationOutcome::Applied { token: Some(_) } + ), + "a real move must be Applied and name its destination UID, got {applied:?}" + ); + + let lost = apply( + &server, + nuncio_mail::RemoteMutationKind::Move { + to_folder: "Trash".to_string(), + }, + uid, + FIRST_UID_VALIDITY, + ) + .await; + assert!( + matches!(lost, nuncio_mail::MutationOutcome::Unknown { .. }), + "a move of an already-moved UID must be Unknown, never Applied, got {lost:?}" + ); + assert!( + server.uids("Trash").is_empty(), + "and nothing actually reached the destination" + ); +} + +#[tokio::test] +async fn a_delete_is_applied_only_when_the_server_reports_the_removal() { + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + let uid = server.append_message("INBOX", "Subject: gone\r\n\r\nx", &[]); + + let outcome = apply( + &server, + nuncio_mail::RemoteMutationKind::Delete, + uid, + FIRST_UID_VALIDITY, + ) + .await; + + // Under QRESYNC the proof arrives as VANISHED, not EXPUNGE. Reading only + // the typed expunge stream would report nothing removed here. + assert!( + matches!(outcome, nuncio_mail::MutationOutcome::Applied { .. }), + "a delete the server confirmed must be Applied, got {outcome:?}" + ); + assert!(server.uids("INBOX").is_empty()); +} + +#[tokio::test] +async fn a_delete_of_an_absent_uid_is_unknown() { + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.append_message("INBOX", "Subject: present\r\n\r\nx", &[]); + + // UID 999 does not exist; the server ignores it and answers OK. + let outcome = apply( + &server, + nuncio_mail::RemoteMutationKind::Delete, + 999, + FIRST_UID_VALIDITY, + ) + .await; + + assert!( + matches!(outcome, nuncio_mail::MutationOutcome::Unknown { .. }), + "a delete the server never confirmed must not be Applied, got {outcome:?}" + ); + assert_eq!( + server.uids("INBOX").len(), + 1, + "the real message must be untouched" + ); +} diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 7de867e..c46328d 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -36,8 +36,8 @@ use nuncio_filter::{ WebhookError, }; use nuncio_mail::{ - MailBackend, MailError, MessageSender, OutboundMessage, RemoteMutationKind, RemoteMutationSpec, - SmtpTransportEngine, + MailBackend, MailError, MessageSender, MutationOutcome, OutboundMessage, RemoteMutationKind, + RemoteMutationSpec, SmtpTransportEngine, }; use nuncio_store::db::{DatabaseEngine, DatabaseError}; use nuncio_store::vault::{SecretManager, VaultError, WEBHOOK_SIGNING_KEY_ACCOUNT}; @@ -113,6 +113,12 @@ pub trait RemoteExecutionEnv: Send + Sync { enum Disposition { /// The real operation genuinely succeeded; mark `completed`. Completed, + /// Another client changed the message first and the server said so. + /// + /// Terminal, and deliberately not a retry: the user's intent was formed + /// against state that no longer exists, so re-issuing it would either do + /// nothing or overwrite someone else's change. + Conflict(String), /// A transient failure; keep retrying until `MAX_RETRIES` is exceeded. Retry(String), /// A permanent failure nothing could retry away; mark `failed` now. @@ -192,6 +198,15 @@ pub async fn execute_pending_mutations( ); ("completed", item.retry_count, |s| s.completed += 1) } + Disposition::Conflict(observed) => { + tracing::warn!( + mutation_id = %item.id, + op = %item.mutation_type, + message_id = %item.message_id, + "outbox: mutation conflicted with another client: {observed}" + ); + ("conflicted", item.retry_count, |s| s.failed += 1) + } Disposition::Permanent(reason) => { tracing::warn!( mutation_id = %item.id, @@ -324,7 +339,19 @@ async fn dispatch_mailbox_mutation( ); match backend.apply_mutation(&spec).await { - Ok(()) => Disposition::Completed, + Ok(MutationOutcome::Applied { .. }) => Disposition::Completed, + // Another client won. Retrying would either do nothing or overwrite a + // change the user did not make, so this stops here and stays visible + // rather than being quietly re-attempted. + Ok(MutationOutcome::Conflict { observed }) => Disposition::Conflict(observed), + // The server accepted the command and proved nothing. Treated as + // retryable, because the alternative -- recording it as done -- is the + // exact silent-loss this outcome exists to prevent. A genuinely + // applied mutation re-attempted is a no-op; a lost one recorded as + // complete is unrecoverable. + Ok(MutationOutcome::Unknown { reason }) => { + Disposition::Retry(format!("mutation outcome unverified: {reason}")) + } Err(e) => Disposition::Retry(format!("remote mutation failed: {e}")), } } -- 2.54.0 From d138dbf65d06be0c004f791d6a5e441570138183 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 13:46:56 -0500 Subject: [PATCH 60/95] docs(plan): plan the message/placement identity split Refs #394 --- .../2026-08-11-message-placement-split.md | 2295 +++++++++++++++++ 1 file changed, 2295 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-11-message-placement-split.md diff --git a/docs/superpowers/plans/2026-08-11-message-placement-split.md b/docs/superpowers/plans/2026-08-11-message-placement-split.md new file mode 100644 index 0000000..abfdd78 --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-message-placement-split.md @@ -0,0 +1,2295 @@ +# Message/Placement Identity Split — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Separate a mail message's *identity* (content-addressed, folder-independent) from its *placements* (one row per folder the message occupies), so a message moved between folders stays one message. + +**Architecture:** `messages` becomes identity + immutable content, keyed by a content-addressed `message_key` derived through a four-tier precedence (`EMAILID` → `X-GM-MSGID` → `(account, Message-ID, content_hash)` → legacy surrogate). A new `placements` table holds `(account_id, folder_id, uidvalidity, uid)` as its primary key plus the per-mailbox read flag. The Rust `Email` struct **loses** its `folder_id` / `remote_id` / `uid_validity` / `read` scalars so the compiler flags every site that assumed a message lives in exactly one folder. Filter evaluation becomes per-placement, and fire-once becomes an atomic `(rule_id, message_key)` claim. + +**Tech Stack:** Rust 2021 (toolchain 1.97.1), `sqlx` + SQLite/WAL, `sha2`, `tonic`/`prost`, `tokio`. + +## Global Constraints + +- **Branch:** `story/message-placement-split`, already created off `dev` with `story/message-id-capture` (#386) and `story/schema-version-and-token-scheme` (#387) merged in. Baseline gate is green on it (627+ tests). Do not rebase onto the #401–#403 chain. +- **Gate (all three must pass before every commit):** `cargo fmt --all -- --check`, then `cargo check-all`, then `cargo test-all`. Warnings are hard errors; `unwrap_used` / `expect_used` / `panic` / `todo` are `deny` outside tests. +- **Commits:** Conventional Commits, imperative, <72-char subject, **no AI attribution**. `Refs #394` in the message body is fine. +- **Comments:** explain intent, constraints, rationale. **No issue numbers, GH refs, or `Phase N` breadcrumbs in `.rs` / `.proto` comments.** +- **No live network in tests.** IMAP/JMAP/SMTP/CalDAV/CardDAV all mocked; keyring via `MockKeyring`. Ephemeral SQLite (`tempfile` / `:memory:`) per test. +- **Never key identity on a Message-ID alone.** Tier 3 requires `Message-ID` **and** `content_hash` both present; if either is missing, fall through to the surrogate tier. First-write-wins on the body. +- **`uidvalidity` is in the placement primary key.** UIDs restart at 1 after a UIDVALIDITY bump and would otherwise collide with old rows. +- **FTS plaintext-orphan rule.** Dropping *a* placement must NOT delete the `messages_fts` row; dropping the *last* placement must. Inverted, this leaks plaintext bodies in orphaned FTS rows. +- **No `.proto` schema changes in this plan.** Only a doc-comment correction on `Message.id`. Placement fields on the wire are #395's scope, and the repo allows one proto-touching story in flight at a time. + +--- + +## File Structure + +| File | Responsibility | Change | +| --- | --- | --- | +| `crates/nuncio-core/src/model.rs` | `Email` (identity + content), new `Placement`, new `MessageIdentity` / `IdentitySource`, `derive_message_key` | Modify | +| `crates/nuncio-store/src/db.rs` | `placements` + `filter_fired` schema, `IDENTITY_SCHEMA_VERSION` bump, all message reads/writes/deletes | Modify | +| `crates/nuncio-store/src/search.rs` | FTS hit mapping onto `message_key` | Modify | +| `crates/nuncio-filter/src/engine.rs` | per-placement condition evaluation | Modify | +| `crates/nuncio-filter/src/validator.rs` | `Folder` / `Account` field validation unchanged, comments corrected | Modify | +| `crates/nuncio-mail/src/imap.rs` | emit identity + placement instead of a surrogate-keyed `Email` | Modify | +| `crates/nuncio-mail/src/jmap.rs` | same | Modify | +| `crates/nuncio-mail/src/mock.rs` | mock backend emits placements | Modify | +| `crates/nunciod/src/sync.rs` | placement-aware new-vs-seen, per-placement filter fan-out, placement removals | Modify | +| `crates/nunciod/src/grpc.rs` | present a placement-in-context on the wire | Modify | +| `crates/nuncio-proto/proto/nuncio/v1/nuncio.proto` | `Message.id` doc comment only | Modify | + +--- + +## Task 1: Core identity and placement types + +**Files:** +- Modify: `crates/nuncio-core/src/model.rs` +- Test: `crates/nuncio-core/src/model.rs` (inline `#[cfg(test)] mod tests`) + +**Interfaces:** +- Consumes: `Email::normalize_message_id`, `Email::content_hash_of` (both already exist from #386). +- Produces: + - `pub enum IdentitySource { EmailId, GmailMsgId, MessageIdContent, Surrogate }` with `as_str(&self) -> &'static str` and `pub fn from_str(s: &str) -> Option<Self>` + - `pub struct MessageIdentity { pub key: String, pub source: IdentitySource }` + - `pub struct RemoteIdentity<'a> { pub email_id: Option<&'a str>, pub gm_msgid: Option<&'a str>, pub message_id: Option<&'a str>, pub content_hash: Option<&'a str> }` + - `pub struct Placement { pub account_id: String, pub folder_id: String, pub uid_validity: String, pub remote_id: String, pub read: bool }` + - `Email::derive_message_key(account_id: &str, remote: RemoteIdentity<'_>, folder_id: &str, uid_validity: &str, remote_id: &str) -> MessageIdentity` + - `Email` keeps `id`, `account_id`, `subject`, `sender`, `recipient`, `received_at`, `body_plain`, `body_html`, `attachments`, `message_id`, `content_hash`; **loses** `folder_id`, `remote_id`, `uid_validity`, `read`. + +- [ ] **Step 1: Write the failing tests** + +Add to the existing `mod tests` in `crates/nuncio-core/src/model.rs`: + +```rust +#[test] +fn message_key_precedence_prefers_emailid_over_everything_below_it() { + let remote = RemoteIdentity { + email_id: Some("M00000001"), + gm_msgid: Some("1234567890"), + message_id: Some("a@b.example"), + content_hash: Some("deadbeef"), + }; + let id = Email::derive_message_key("acct-1", remote, "INBOX", "42", "5"); + assert_eq!(id.source, IdentitySource::EmailId); + + // Same EMAILID in a different folder is the SAME message. + let elsewhere = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: Some("M00000001"), gm_msgid: None, message_id: None, content_hash: None }, + "Archive", + "99", + "7", + ); + assert_eq!(id.key, elsewhere.key); +} + +#[test] +fn message_key_is_account_scoped_at_every_tier() { + let remote = || RemoteIdentity { + email_id: Some("M00000001"), + gm_msgid: None, + message_id: None, + content_hash: None, + }; + let a = Email::derive_message_key("acct-1", remote(), "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", remote(), "INBOX", "42", "5"); + assert_ne!(a.key, b.key, "the same EMAILID in two accounts must not collide"); +} + +#[test] +fn a_message_id_without_a_content_hash_never_reaches_the_content_tier() { + // The non-negotiable: a Message-ID is public and forgeable, so it must + // never key storage on its own. Missing content hash falls through to the + // folder-scoped surrogate rather than trusting the header alone. + let id = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: None, gm_msgid: None, message_id: Some("a@b.example"), content_hash: None }, + "INBOX", + "42", + "5", + ); + assert_eq!(id.source, IdentitySource::Surrogate); + + let surrogate_only = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: None, gm_msgid: None, message_id: None, content_hash: None }, + "INBOX", + "42", + "5", + ); + assert_eq!(id.key, surrogate_only.key); +} + +#[test] +fn the_same_message_id_with_different_content_is_two_messages() { + let mk = |hash: &str| { + Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: Some(hash), + }, + "INBOX", + "42", + "5", + ) + }; + let list_copy = mk("1111"); + let direct_copy = mk("2222"); + assert_eq!(list_copy.source, IdentitySource::MessageIdContent); + assert_ne!(list_copy.key, direct_copy.key); +} + +#[test] +fn tiers_are_domain_separated_so_a_shared_value_cannot_collide_across_them() { + let as_emailid = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: Some("X"), gm_msgid: None, message_id: None, content_hash: None }, + "INBOX", "42", "5", + ); + let as_gmsgid = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: None, gm_msgid: Some("X"), message_id: None, content_hash: None }, + "INBOX", "42", "5", + ); + assert_ne!(as_emailid.key, as_gmsgid.key); +} + +#[test] +fn identity_source_round_trips_through_its_stored_string() { + for source in [ + IdentitySource::EmailId, + IdentitySource::GmailMsgId, + IdentitySource::MessageIdContent, + IdentitySource::Surrogate, + ] { + assert_eq!(IdentitySource::from_str(source.as_str()), Some(source)); + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-core model::tests` +Expected: FAIL — `cannot find type RemoteIdentity`, `no function derive_message_key`. + +- [ ] **Step 3: Implement the identity types** + +In `crates/nuncio-core/src/model.rs`, add above `impl Email`: + +```rust +/// Which tier of the identity precedence produced a [`MessageIdentity`]. +/// +/// Persisted alongside the key so a later pass can tell a server-assigned +/// identity from a locally-derived fallback without recomputing it, and so a +/// store can be audited for how much of it rests on the weakest tier. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum IdentitySource { + /// RFC 8474 OBJECTID `EMAILID` -- server-assigned and stable across folders. + EmailId, + /// Gmail's `X-GM-MSGID`, the same guarantee by a vendor extension. + GmailMsgId, + /// `Message-ID` **and** a content hash together. Never the header alone. + MessageIdContent, + /// Folder-scoped last resort: the message is only identified by where it + /// currently sits, so the same mail in another folder is a second message. + Surrogate, +} + +impl IdentitySource { + /// Stable token persisted in `messages.identity_source`. + pub fn as_str(&self) -> &'static str { + match self { + Self::EmailId => "emailid", + Self::GmailMsgId => "gmsgid", + Self::MessageIdContent => "msgid_content", + Self::Surrogate => "surrogate", + } + } + + /// Parse a token written by [`Self::as_str`]. Returns `None` for anything + /// else so an unrecognised value fails loudly rather than defaulting to a + /// tier that would overstate how trustworthy the key is. + pub fn from_str(s: &str) -> Option<Self> { + match s { + "emailid" => Some(Self::EmailId), + "gmsgid" => Some(Self::GmailMsgId), + "msgid_content" => Some(Self::MessageIdContent), + "surrogate" => Some(Self::Surrogate), + _ => None, + } + } +} + +/// A content-addressed message key together with the tier that produced it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MessageIdentity { + /// Hex SHA-256 key identifying the message within its account. + pub key: String, + /// Which precedence tier produced `key`. + pub source: IdentitySource, +} + +/// Server-supplied identity hints for one message, in precedence order. +/// +/// Borrowed rather than owned because every caller already holds these as +/// fields on a parsed response and only needs them for the duration of the +/// key derivation. +#[derive(Debug, Clone, Copy, Default)] +pub struct RemoteIdentity<'a> { + /// RFC 8474 `EMAILID` from a `FETCH ... (EMAILID)`, if the server has OBJECTID. + pub email_id: Option<&'a str>, + /// Gmail `X-GM-MSGID`, if the server advertises `X-GM-EXT-1`. + pub gm_msgid: Option<&'a str>, + /// Normalized `Message-ID` (see [`Email::normalize_message_id`]). + pub message_id: Option<&'a str>, + /// Hex SHA-256 over the full RFC822 octets (see [`Email::content_hash_of`]). + pub content_hash: Option<&'a str>, +} + +/// One occupancy of a message in one mailbox. +/// +/// A message can sit in several folders at once (an IMAP `COPY`, a Gmail label, +/// a message that is both in a thread's folder and in `\All`). The addressing +/// coordinates and the read flag are properties of the *occupancy*, not of the +/// message: IMAP `\Seen` is per-mailbox, and a UID is only meaningful inside one +/// `uid_validity` scope of one folder. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Placement { + /// Account owning the mailbox. + pub account_id: String, + /// Mailbox folder identifier (e.g. "inbox"). + pub folder_id: String, + /// The IMAP UIDVALIDITY the `remote_id` was captured under, as a decimal + /// string; a stable sentinel for protocols without one (JMAP). In the + /// primary key because UIDs restart at 1 after a bump and would otherwise + /// collide with rows written before it. + pub uid_validity: String, + /// Protocol-native id addressing the message in this mailbox: the IMAP UID + /// as a decimal string, or the JMAP Email object id. + pub remote_id: String, + /// Read/unread state **in this mailbox**. + pub read: bool, +} +``` + +- [ ] **Step 4: Implement `derive_message_key` and reshape `Email`** + +Add to `impl Email`: + +```rust + /// Domain-separated, length-prefixed digest over an identity tier's inputs. + /// + /// The tier label participates in the hash so that a value appearing in two + /// different tiers (a server whose `EMAILID` happens to equal another's + /// `X-GM-MSGID`) cannot produce one key. Length prefixes stop two distinct + /// field tuples serializing to the same byte stream, exactly as in + /// [`Email::surrogate_id`]. + fn identity_digest(tier: &str, account_id: &str, parts: &[&str]) -> String { + let mut hasher = Sha256::new(); + for field in [tier, account_id].into_iter().chain(parts.iter().copied()) { + hasher.update((field.len() as u64).to_le_bytes()); + hasher.update(field.as_bytes()); + } + hex::encode(hasher.finalize()) + } + + /// Derive the account-scoped message key by identity precedence. + /// + /// `EMAILID` → `X-GM-MSGID` → (`Message-ID` + content hash) → surrogate. + /// Every tier is account-scoped: identity is only ever claimed within the + /// account that fetched it, so one account's server cannot mint a key that + /// addresses another account's mail. + /// + /// The third tier requires **both** a `Message-ID` and a content hash. A + /// `Message-ID` is public -- it is echoed in the `References` of every reply + /// -- and forgeable, so keying on it alone would let a crafted message + /// address, and with an upsert overwrite, a message the user already trusts, + /// and would evade filtering by never being classified as new. Pairing it + /// with the content hash also gets the ordinary cases right: a mailing-list + /// copy and a direct copy share a `Message-ID` but differ in content, and so + /// are correctly two messages. + /// + /// The surrogate tier is folder-scoped and therefore *not* stable across a + /// move; it is the honest answer when the server offered nothing better. + pub fn derive_message_key( + account_id: &str, + remote: RemoteIdentity<'_>, + folder_id: &str, + uid_validity: &str, + remote_id: &str, + ) -> MessageIdentity { + if let Some(email_id) = remote.email_id.filter(|v| !v.is_empty()) { + return MessageIdentity { + key: Self::identity_digest("emailid", account_id, &[email_id]), + source: IdentitySource::EmailId, + }; + } + if let Some(gm_msgid) = remote.gm_msgid.filter(|v| !v.is_empty()) { + return MessageIdentity { + key: Self::identity_digest("gmsgid", account_id, &[gm_msgid]), + source: IdentitySource::GmailMsgId, + }; + } + if let (Some(message_id), Some(content_hash)) = ( + remote.message_id.filter(|v| !v.is_empty()), + remote.content_hash.filter(|v| !v.is_empty()), + ) { + return MessageIdentity { + key: Self::identity_digest("msgid_content", account_id, &[message_id, content_hash]), + source: IdentitySource::MessageIdContent, + }; + } + MessageIdentity { + key: Self::identity_digest("surrogate", account_id, &[folder_id, uid_validity, remote_id]), + source: IdentitySource::Surrogate, + } + } +``` + +Then delete the `folder_id`, `remote_id`, `uid_validity`, and `read` fields from `struct Email`, and replace the doc comment on `Email::id` with: + +```rust + /// Opaque, account-scoped identity for the message itself. + /// + /// Derived by [`Email::derive_message_key`], so it is independent of which + /// folder the message currently occupies: moving a message between folders + /// does not change it. Where the message actually sits, and its read state + /// there, live in [`Placement`] rows instead. Callers must treat this as an + /// opaque token -- it carries no decodable structure. + pub id: String, +``` + +Keep `Email::surrogate_id` in place — the surrogate tier still uses the same +construction and existing tests cover it. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cargo test -p nuncio-core model::tests` +Expected: PASS. Other crates will not compile yet; that is expected and is fixed in later tasks. + +- [ ] **Step 6: Commit** + +```bash +git add crates/nuncio-core/src/model.rs +git commit -m "feat(core): separate message identity from mailbox placement" +``` + +--- + +## Task 2: Store schema — placements, fire-once claims, and the version bump + +**Files:** +- Modify: `crates/nuncio-store/src/db.rs` (`migrate`, `reset_cached_message_store`, `IDENTITY_SCHEMA_VERSION`) +- Test: `crates/nuncio-store/src/db.rs` inline tests + +**Interfaces:** +- Consumes: `DatabaseEngine::reconcile_schema_version` and `reset_cached_message_store` (both from #387). +- Produces: `placements` and `filter_fired` tables; `messages` reshaped to `message_key` + identity columns; `IDENTITY_SCHEMA_VERSION == 2`. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[tokio::test] +async fn placements_primary_key_includes_uidvalidity() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Same folder, same UID, different UIDVALIDITY: two distinct placements. + // After a UIDVALIDITY bump the server restarts UIDs at 1, so without + // uidvalidity in the key the new mail would overwrite the old. + for validity in ["42", "43"] { + sqlx::query( + "INSERT INTO placements (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES (?, ?, ?, ?, ?, 0)", + ) + .bind("acct-1") + .bind("INBOX") + .bind(validity) + .bind("1") + .bind(format!("key-{validity}")) + .execute(engine.pool()) + .await + .expect("both placements must insert"); + } + + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM placements") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(count, 2); +} + +#[tokio::test] +async fn a_filter_fired_claim_is_won_exactly_once() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + assert!(engine.claim_filter_fire("rule-1", "key-1").await.unwrap()); + assert!( + !engine.claim_filter_fire("rule-1", "key-1").await.unwrap(), + "a second claim on the same (rule, message) must lose" + ); + assert!( + engine.claim_filter_fire("rule-2", "key-1").await.unwrap(), + "a different rule still gets its own claim" + ); +} + +#[tokio::test] +async fn bumping_the_identity_schema_rebuilds_placements_and_claims_too() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("nuncio.db"); + let secrets = crate::vault::MockKeyring::new(); + + { + let engine = DatabaseEngine::connect_file(&db_path, &secrets).await.unwrap(); + sqlx::query( + "INSERT INTO placements (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES ('a', 'INBOX', '1', '1', 'k', 0)", + ) + .execute(engine.pool()) + .await + .unwrap(); + engine.claim_filter_fire("rule-1", "k").await.unwrap(); + // Pretend this file was written by the previous identity generation. + sqlx::query("PRAGMA user_version = 1").execute(engine.pool()).await.unwrap(); + engine.close().await; + } + + let engine = DatabaseEngine::connect_file(&db_path, &secrets).await.unwrap(); + let (placements,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM placements") + .fetch_one(engine.pool()) + .await + .unwrap(); + let (claims,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM filter_fired") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(placements, 0, "placements are derived from the server and must be rebuilt"); + assert_eq!(claims, 0, "fire-once claims name message keys that no longer resolve"); + engine.close().await; +} +``` + +If `DatabaseEngine` has no `pool()` accessor, add one gated to the crate: + +```rust + /// Test-only access to the underlying pool for schema-level assertions. + #[cfg(test)] + pub(crate) fn pool(&self) -> &sqlx::SqlitePool { + &self.pool + } +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-store db::tests::placements_primary_key_includes_uidvalidity db::tests::a_filter_fired_claim_is_won_exactly_once` +Expected: FAIL — `no such table: placements`, `no method claim_filter_fire`. + +- [ ] **Step 3: Add the schema** + +In `DatabaseEngine::migrate`, replace the `CREATE TABLE IF NOT EXISTS messages (...)` block with: + +```sql + -- Identity and immutable content, one row per message. Where the + -- message sits lives in `placements`; nothing here is folder-scoped. + CREATE TABLE IF NOT EXISTS messages ( + message_key TEXT PRIMARY KEY NOT NULL, + account_id TEXT NOT NULL, + subject TEXT NOT NULL, + sender TEXT NOT NULL, + recipient TEXT NOT NULL, + received_at INTEGER NOT NULL, + body_plain TEXT, + body_html TEXT, + message_id TEXT, + content_hash TEXT, + identity_source TEXT NOT NULL DEFAULT 'surrogate' + ); + + -- One row per (mailbox, message) occupancy. `uidvalidity` is in the + -- primary key because UIDs restart at 1 after a bump: without it the + -- first message of the new generation would overwrite the row of + -- whichever old message shared its UID. + CREATE TABLE IF NOT EXISTS placements ( + account_id TEXT NOT NULL, + folder_id TEXT NOT NULL, + uidvalidity TEXT NOT NULL, + uid TEXT NOT NULL, + message_key TEXT NOT NULL, + read_flag INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (account_id, folder_id, uidvalidity, uid) + ); + + -- Fire-once ledger for filter actions. Keyed on the message + -- *identity*, not a placement, so a message arriving in a second + -- folder does not re-fire a rule that already acted on it -- which + -- matters most for FORWARD and CALL WEBHOOK, whose effects land on + -- third parties and cannot be undone by convergence. + CREATE TABLE IF NOT EXISTS filter_fired ( + rule_id TEXT NOT NULL, + message_key TEXT NOT NULL, + fired_at INTEGER NOT NULL, + PRIMARY KEY (rule_id, message_key) + ); +``` + +Add alongside the existing indexes: + +```sql + CREATE INDEX IF NOT EXISTS idx_placements_message ON placements(message_key); + CREATE INDEX IF NOT EXISTS idx_placements_folder ON placements(account_id, folder_id); +``` + +The `messages_ad` trigger stays exactly as it is. Update only its comment to +record why it must remain on `messages`: + +```sql + -- Subject/sender are never encrypted in `messages`, so a delete-only trigger is + -- sufficient here: it just keeps the FTS index free of orphaned rows. Insertion and + -- update of `messages_fts` content happens explicitly in `upsert_message`, never via an + -- AFTER INSERT/UPDATE trigger, because such a trigger would only ever see the + -- ciphertext body column. + -- + -- This trigger MUST stay on `messages` and never move to `placements`. Dropping one + -- placement of a message that still sits in another folder must leave the body + -- searchable; only losing the last placement removes the `messages` row, and that is + -- what reaps the FTS row here. Firing on `placements` instead would delete the index + -- entry while the body remained -- and, inverted, leaving it off `messages` entirely + -- would strand plaintext bodies in `messages_fts` after the message itself was gone. + CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN + DELETE FROM messages_fts WHERE message_key = old.message_key; + END; +``` + +Change the FTS virtual table's key column to match: + +```sql + CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + message_key UNINDEXED, + subject, + sender, + body_plain, + tokenize = 'trigram' + ); +``` + +- [ ] **Step 4: Bump the version and extend the reset** + +```rust + pub const IDENTITY_SCHEMA_VERSION: i64 = 2; +``` + +Extend the statement list inside `reset_cached_message_store` to: + +```rust + for statement in [ + "DELETE FROM messages", + "DELETE FROM messages_fts", + "DELETE FROM placements", + "DELETE FROM filter_fired", + "DELETE FROM folder_sync_state", + "DELETE FROM filter_execution_logs", + ] { +``` + +and add to its doc comment, in the "Cleared" sentence, `mailbox placements` and +`the filter fire-once ledger`. + +Apply the same additions to the `for statement in [...]` list in the +existing `clear_cached_message_data`-style helper at `db.rs:596` if it is a +separate function. + +- [ ] **Step 5: Add `claim_filter_fire`** + +```rust + /// Atomically claim the right to run `rule_id`'s actions against + /// `message_key`, returning `true` only for the claim that won. + /// + /// The claim is the fire-once guard for side-effecting actions. It is keyed + /// on message identity rather than on a placement because the same message + /// legitimately arrives in several folders -- and, under the old + /// folder-scoped identity, each arrival looked like a different message and + /// fired the rule again. `INSERT ... ON CONFLICT DO NOTHING` makes winning + /// the claim a single atomic statement, so two passes racing over the same + /// message cannot both act. + pub async fn claim_filter_fire( + &self, + rule_id: &str, + message_key: &str, + ) -> Result<bool, DatabaseError> { + let result = sqlx::query( + "INSERT INTO filter_fired (rule_id, message_key, fired_at) VALUES (?, ?, ?) + ON CONFLICT (rule_id, message_key) DO NOTHING", + ) + .bind(rule_id) + .bind(message_key) + .bind(chrono::Utc::now().timestamp()) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(result.rows_affected() == 1) + } +``` + +Use whichever clock helper `db.rs` already uses for `created_at`/`matched_at` +rather than introducing a second one. + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cargo test -p nuncio-store db::tests::placements_primary_key_includes_uidvalidity db::tests::a_filter_fired_claim_is_won_exactly_once db::tests::bumping_the_identity_schema_rebuilds` +Expected: PASS. The rest of `nuncio-store` will not compile until Task 3. + +- [ ] **Step 7: Commit** + +```bash +git add crates/nuncio-store/src/db.rs +git commit -m "feat(store): add placements, fire-once claims, and bump identity schema" +``` + +--- + +## Task 3: Store writes — upsert message, upsert placement, first-write-wins + +**Files:** +- Modify: `crates/nuncio-store/src/db.rs` (`save_email` → `upsert_message` + `upsert_placement`, `backfill_message_fts`) +- Test: `crates/nuncio-store/src/db.rs` inline tests + +**Interfaces:** +- Consumes: `Email` and `Placement` from Task 1; `placements` schema from Task 2. +- Produces: + - `DatabaseEngine::upsert_message(&self, email: &Email, source: IdentitySource) -> Result<bool, DatabaseError>` — `true` when the row was newly inserted + - `DatabaseEngine::upsert_placement(&self, message_key: &str, placement: &Placement) -> Result<bool, DatabaseError>` — `true` when the placement was newly inserted + - `DatabaseEngine::save_email_at(&self, email: &Email, source: IdentitySource, placement: &Placement) -> Result<SaveOutcome, DatabaseError>` where `pub struct SaveOutcome { pub message_is_new: bool, pub placement_is_new: bool }` + +- [ ] **Step 1: Write the failing tests** + +```rust +#[tokio::test] +async fn a_second_placement_does_not_overwrite_the_stored_body() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let email = nuncio_core::model::Email { + id: "key-1".into(), + account_id: "acct-1".into(), + subject: "Original".into(), + sender: "a@nuncio.mx".into(), + recipient: "b@nuncio.mx".into(), + received_at: 1_000, + body_plain: Some("the trusted body".into()), + body_html: None, + attachments: Vec::new(), + message_id: Some("a@b.example".into()), + content_hash: Some("1111".into()), + }; + let inbox = nuncio_core::model::Placement { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + read: false, + }; + let outcome = engine + .save_email_at(&email, nuncio_core::model::IdentitySource::MessageIdContent, &inbox) + .await + .unwrap(); + assert!(outcome.message_is_new && outcome.placement_is_new); + + // The same key arriving from another folder, carrying a different body. + let impostor = nuncio_core::model::Email { subject: "Replaced".into(), body_plain: Some("attacker body".into()), ..email.clone() }; + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), ..inbox.clone() }; + let outcome = engine + .save_email_at(&impostor, nuncio_core::model::IdentitySource::MessageIdContent, &archive) + .await + .unwrap(); + assert!(!outcome.message_is_new, "identity already existed"); + assert!(outcome.placement_is_new, "but the Archive placement is new"); + + let stored = engine.get_message("key-1").await.unwrap(); + assert_eq!(stored.subject, "Original", "first write must win"); + assert_eq!(stored.body_plain.as_deref(), Some("the trusted body")); +} + +#[tokio::test] +async fn re_placing_the_same_message_leaves_exactly_one_fts_row() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), ..inbox.clone() }; + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &archive).await.unwrap(); + + let (rows,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(rows, 1, "the FTS index holds one row per message, not per placement"); +} + +#[tokio::test] +async fn read_state_is_tracked_per_placement() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), read: true, ..inbox.clone() }; + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &archive).await.unwrap(); + + let placements = engine.placements_of("key-1").await.unwrap(); + let inbox_read = placements.iter().find(|p| p.folder_id == "INBOX").unwrap().read; + let archive_read = placements.iter().find(|p| p.folder_id == "Archive").unwrap().read; + assert!(!inbox_read); + assert!(archive_read, "IMAP \\Seen is per-mailbox, so the flags differ"); +} +``` + +Add this test helper next to the tests: + +```rust + fn sample_message_and_placement( + key: &str, + folder: &str, + uid: &str, + ) -> (nuncio_core::model::Email, nuncio_core::model::Placement) { + ( + nuncio_core::model::Email { + id: key.into(), + account_id: "acct-1".into(), + subject: "Subject".into(), + sender: "a@nuncio.mx".into(), + recipient: "b@nuncio.mx".into(), + received_at: 1_000, + body_plain: Some("body".into()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::Placement { + account_id: "acct-1".into(), + folder_id: folder.into(), + uid_validity: "42".into(), + remote_id: uid.into(), + read: false, + }, + ) + } +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-store db::tests::a_second_placement_does_not_overwrite` +Expected: FAIL — `no method save_email_at`. + +- [ ] **Step 3: Implement the write path** + +Replace `save_email` with: + +```rust +/// What a [`DatabaseEngine::save_email_at`] call actually changed. +/// +/// The two flags answer different questions and a caller usually needs both: +/// `message_is_new` gates work that should happen once per message (indexing, +/// side-effecting filter actions), `placement_is_new` gates work that should +/// happen once per mailbox occupancy (per-folder counters, per-placement +/// evaluation). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SaveOutcome { + /// The message identity had not been stored before this call. + pub message_is_new: bool, + /// This mailbox occupancy had not been stored before this call. + pub placement_is_new: bool, +} + +impl DatabaseEngine { + /// Persist a message's identity and content, and record that it occupies + /// `placement`, in one transaction. + /// + /// The message row is **first-write-wins**: a second sighting of the same + /// key -- which is the normal case for a message that also exists in another + /// folder -- leaves the stored subject and body untouched. That is what + /// keeps a message whose identity rests on `Message-ID` + content hash from + /// being rewritten by a later fetch claiming the same key, and it is why the + /// FTS row is written only on the insert that actually created the message. + /// + /// The placement row is upserted rather than ignored, because its read flag + /// is genuinely mutable: `\Seen` changes in the mailbox and the store must + /// follow it. + pub async fn save_email_at( + &self, + email: &nuncio_core::model::Email, + source: nuncio_core::model::IdentitySource, + placement: &nuncio_core::model::Placement, + ) -> Result<SaveOutcome, DatabaseError> { + // Encryption failures MUST surface before any SQL runs: a swallowed error here would + // otherwise leave a row with an empty/placeholder body column, indistinguishable from + // a genuinely empty body on read-back. + let enc_plain = email + .body_plain + .as_ref() + .map(|p| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, p)) + .transpose()?; + let enc_html = email + .body_html + .as_ref() + .map(|h| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, h)) + .transpose()?; + + let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + + // Ask before writing. SQLite reports one row affected for both arms of + // an upsert, so after the fact there is no way to tell an inserted + // placement from an updated one -- and the caller needs that distinction + // to decide whether this is a first arrival worth filtering. + let (already_placed,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&placement.account_id) + .bind(&placement.folder_id) + .bind(&placement.uid_validity) + .bind(&placement.remote_id) + .fetch_one(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + let placement_is_new = already_placed == 0; + + let inserted = sqlx::query( + r#" + INSERT INTO messages + (message_key, account_id, subject, sender, recipient, received_at, + body_plain, body_html, message_id, content_hash, identity_source) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (message_key) DO NOTHING + "#, + ) + .bind(&email.id) + .bind(&email.account_id) + .bind(&email.subject) + .bind(&email.sender) + .bind(&email.recipient) + .bind(email.received_at) + .bind(&enc_plain) + .bind(&enc_html) + .bind(&email.message_id) + .bind(&email.content_hash) + .bind(source.as_str()) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + let message_is_new = inserted.rows_affected() == 1; + + // Index only on the insert that created the message. Re-indexing on + // every placement would either duplicate the row (FTS5 standalone tables + // have no upsert) or rewrite it with a body the first-write-wins rule + // just rejected. + if message_is_new { + sqlx::query( + "INSERT INTO messages_fts (message_key, subject, sender, body_plain) VALUES (?, ?, ?, ?)", + ) + .bind(&email.id) + .bind(&email.subject) + .bind(&email.sender) + .bind(email.body_plain.as_deref().unwrap_or("")) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + } + + sqlx::query( + r#" + INSERT INTO placements + (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (account_id, folder_id, uidvalidity, uid) + DO UPDATE SET message_key = excluded.message_key, read_flag = excluded.read_flag + "#, + ) + .bind(&placement.account_id) + .bind(&placement.folder_id) + .bind(&placement.uid_validity) + .bind(&placement.remote_id) + .bind(&email.id) + .bind(if placement.read { 1i64 } else { 0i64 }) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + tx.commit().await.map_err(DatabaseError::Query)?; + + Ok(SaveOutcome { message_is_new, placement_is_new }) + } +} +``` + +Add the placement reader used by the tests: + +```rust + /// Every mailbox this message currently occupies. + pub async fn placements_of( + &self, + message_key: &str, + ) -> Result<Vec<nuncio_core::model::Placement>, DatabaseError> { + let rows: Vec<(String, String, String, String, i64)> = sqlx::query_as( + "SELECT account_id, folder_id, uidvalidity, uid, read_flag + FROM placements WHERE message_key = ? ORDER BY folder_id ASC", + ) + .bind(message_key) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map(|(account_id, folder_id, uid_validity, remote_id, read_flag)| { + nuncio_core::model::Placement { + account_id, + folder_id, + uid_validity, + remote_id, + read: read_flag != 0, + } + }) + .collect()) + } +``` + +Update `backfill_message_fts` to select `message_key` instead of `id` and to +join nothing — it stays a `messages`-only backfill. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p nuncio-store db::tests` +Expected: the three new tests PASS. Pre-existing `save_email` tests fail to compile — fix them in Task 4 as part of the read-path migration. + +- [ ] **Step 5: Commit** + +```bash +git add crates/nuncio-store/src/db.rs +git commit -m "feat(store): write message identity and placement separately" +``` + +--- + +## Task 4: Store reads — placement-aware queries and folder derivation + +**Files:** +- Modify: `crates/nuncio-store/src/db.rs` (`get_message`, `list_messages`, `list_messages_page`, `message_ids_in_folder`, `set_message_read`, `list_folders`, `list_folders_page`, unread count at `db.rs:471`) +- Modify: `crates/nuncio-store/src/search.rs` +- Test: `crates/nuncio-store/src/db.rs` inline tests + +**Interfaces:** +- Consumes: `placements_of`, `SaveOutcome` from Task 3. +- Produces: + - `get_message(&self, message_key: &str) -> Result<Email, DatabaseError>` (identity only, no folder scalars) + - `list_messages(&self, account_id: &str, folder_id: &str, limit: usize) -> Result<Vec<(Email, Placement)>, DatabaseError>` + - `existing_placements(&self, keys: &[PlacementKey]) -> Result<HashSet<PlacementKey>, DatabaseError>` with: + + ```rust + /// The four coordinates that address one mailbox occupancy. + /// + /// `Ord` is derived because the sync path sorts and dedups batches of these + /// before deleting; `Hash`/`Eq` because it is also the key of the + /// already-present set used to classify a fetched chunk. + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + pub struct PlacementKey { + pub account_id: String, + pub folder_id: String, + pub uid_validity: String, + pub remote_id: String, + } + ``` +- Test helper: the tests below reuse `sample_message_and_placement`, added to `db.rs`'s test module in Task 3 Step 1. It returns an `(Email, Placement)` pair for `account_id = "acct-1"`, `uid_validity = "42"`. + - `set_placement_read(&self, key: &PlacementKey, read: bool) -> Result<(), DatabaseError>` + - `list_folders(&self, account_id: &str) -> Result<Vec<Folder>, DatabaseError>` — now account-scoped + +- [ ] **Step 1: Write the failing tests** + +```rust +#[tokio::test] +async fn folders_are_derived_from_placements_and_scoped_per_account() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + for (account, folder, uid) in [("acct-1", "INBOX", "1"), ("acct-1", "Archive", "2"), ("acct-2", "INBOX", "1")] { + let (mut email, mut placement) = sample_message_and_placement(&format!("key-{account}-{folder}"), folder, uid); + email.account_id = account.into(); + placement.account_id = account.into(); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &placement).await.unwrap(); + } + + let folders = engine.list_folders("acct-1").await.unwrap(); + let names: Vec<&str> = folders.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(names, vec!["Archive", "INBOX"]); + assert!( + !folders.iter().any(|f| f.total_messages > 1), + "acct-2's INBOX must not be folded into acct-1's" + ); +} + +#[tokio::test] +async fn one_message_in_two_folders_counts_once_in_each() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), ..inbox.clone() }; + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &archive).await.unwrap(); + + let folders = engine.list_folders("acct-1").await.unwrap(); + for folder in &folders { + assert_eq!(folder.total_messages, 1, "{} should hold the message once", folder.id); + } + let (messages,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages").fetch_one(engine.pool()).await.unwrap(); + assert_eq!(messages, 1, "but it is still one message"); +} + +#[tokio::test] +async fn existing_placements_reports_the_occupancy_not_the_message() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + + let in_inbox = PlacementKey { account_id: "acct-1".into(), folder_id: "INBOX".into(), uid_validity: "42".into(), remote_id: "5".into() }; + let in_archive = PlacementKey { folder_id: "Archive".into(), remote_id: "9".into(), ..in_inbox.clone() }; + + let found = engine.existing_placements(&[in_inbox.clone(), in_archive.clone()]).await.unwrap(); + assert!(found.contains(&in_inbox)); + assert!( + !found.contains(&in_archive), + "the same message arriving in a new folder is a new placement and must be seen as such" + ); +} + +#[tokio::test] +async fn marking_read_in_one_folder_leaves_the_other_unread() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), ..inbox.clone() }; + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &archive).await.unwrap(); + + let inbox_key = PlacementKey { account_id: "acct-1".into(), folder_id: "INBOX".into(), uid_validity: "42".into(), remote_id: "5".into() }; + engine.set_placement_read(&inbox_key, true).await.unwrap(); + + let placements = engine.placements_of("key-1").await.unwrap(); + assert!(placements.iter().find(|p| p.folder_id == "INBOX").unwrap().read); + assert!(!placements.iter().find(|p| p.folder_id == "Archive").unwrap().read); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-store db::tests::folders_are_derived_from_placements` +Expected: FAIL — `list_folders` takes no arguments; `PlacementKey` undefined. + +- [ ] **Step 3: Implement the read path** + +`get_message` — drop the folder scalars from both the `SELECT` and the +constructed `Email`: + +```rust + /// Retrieve a message's identity and content by its key. + /// + /// Returns the message alone. Where it sits, and whether it is read there, + /// are per-mailbox facts -- ask [`Self::placements_of`] for those. + pub async fn get_message( + &self, + message_key: &str, + ) -> Result<nuncio_core::model::Email, DatabaseError> { + let row: (String, String, String, String, String, i64, Option<String>, Option<String>, Option<String>, Option<String>) = + sqlx::query_as( + r#" + SELECT message_key, account_id, subject, sender, recipient, received_at, + body_plain, body_html, message_id, content_hash + FROM messages + WHERE message_key = ? + "#, + ) + .bind(message_key) + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + let dec_plain = row + .6 + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = row + .7 + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + + Ok(nuncio_core::model::Email { + id: row.0, + account_id: row.1, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, + body_plain: dec_plain, + body_html: dec_html, + attachments: Vec::new(), + message_id: row.8, + content_hash: row.9, + }) + } +``` + +`list_messages` and `list_messages_page` join through `placements` and return +pairs. Both gain an `account_id` parameter for the same reason +`message_ids_in_folder` already has one — folder ids are not globally unique: + +```rust + /// Messages occupying one folder of one account, newest first, paired with + /// the placement they were found through. + /// + /// The placement travels with the message because the caller almost always + /// needs the per-mailbox facts -- the read flag and the addressing UID -- + /// and re-deriving them would mean a second query per row. + #[allow(clippy::type_complexity)] + pub async fn list_messages( + &self, + account_id: &str, + folder_id: &str, + limit: usize, + ) -> Result<Vec<(nuncio_core::model::Email, nuncio_core::model::Placement)>, DatabaseError> { + let rows: Vec<( + String, + String, + String, + String, + String, + i64, + Option<String>, + Option<String>, + Option<String>, + Option<String>, + String, + String, + i64, + )> = sqlx::query_as( + r#" + SELECT m.message_key, m.account_id, m.subject, m.sender, m.recipient, + m.received_at, m.body_plain, m.body_html, m.message_id, m.content_hash, + p.uidvalidity, p.uid, p.read_flag + FROM messages m + JOIN placements p ON p.message_key = m.message_key + WHERE p.account_id = ? AND p.folder_id = ? + ORDER BY m.received_at DESC + LIMIT ? + "#, + ) + .bind(account_id) + .bind(folder_id) + .bind(limit as i64) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let dec_plain = row + .6 + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = row + .7 + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + + out.push(( + nuncio_core::model::Email { + id: row.0, + account_id: row.1, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, + body_plain: dec_plain, + body_html: dec_html, + attachments: Vec::new(), + message_id: row.8, + content_hash: row.9, + }, + nuncio_core::model::Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: row.10, + remote_id: row.11, + read: row.12 != 0, + }, + )); + } + Ok(out) + } +``` + +`list_messages_page` takes the same join and adds the existing keyset cursor +over `m.received_at`, fetching `page_size + 1` rows to detect a following page +exactly as it does today. + +`message_ids_in_folder` selects from `placements` and returns placement keys, since +that is what a UID-set diff subtracts: + +```rust + /// Every placement currently stored for one folder of one account. + /// + /// The local half of a UID-set diff. This returns placements rather than + /// message keys because what a server stops reporting is an occupancy, not a + /// message: the same mail may still exist in another folder and must survive + /// there. + pub async fn placements_in_folder( + &self, + account_id: &str, + folder_id: &str, + ) -> Result<Vec<PlacementKey>, DatabaseError> { + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT uidvalidity, uid FROM placements WHERE account_id = ? AND folder_id = ?", + ) + .bind(account_id) + .bind(folder_id) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map(|(uid_validity, remote_id)| PlacementKey { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity, + remote_id, + }) + .collect()) + } +``` + +`existing_placements` mirrors `existing_message_ids`, batching with a tuple +`IN` clause: + +```rust + /// The subset of `keys` already stored, in one round trip. + /// + /// SQLite supports row-value `IN` lists, so the whole batch is one query + /// rather than one existence check per placement. An empty slice + /// short-circuits, since `IN ()` is not valid SQL. + pub async fn existing_placements( + &self, + keys: &[PlacementKey], + ) -> Result<std::collections::HashSet<PlacementKey>, DatabaseError> { + if keys.is_empty() { + return Ok(std::collections::HashSet::new()); + } + + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT account_id, folder_id, uidvalidity, uid FROM placements + WHERE (account_id, folder_id, uidvalidity, uid) IN (", + ); + let mut separated = builder.separated(", "); + for key in keys { + separated.push("("); + separated.push_bind_unseparated(key.account_id.clone()); + separated.push_unseparated(", "); + separated.push_bind_unseparated(key.folder_id.clone()); + separated.push_unseparated(", "); + separated.push_bind_unseparated(key.uid_validity.clone()); + separated.push_unseparated(", "); + separated.push_bind_unseparated(key.remote_id.clone()); + separated.push_unseparated(")"); + } + builder.push(")"); + + let rows: Vec<(String, String, String, String)> = builder + .build_query_as() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map(|(account_id, folder_id, uid_validity, remote_id)| PlacementKey { + account_id, + folder_id, + uid_validity, + remote_id, + }) + .collect()) + } +``` + +> **Implementer note:** if `QueryBuilder`'s separated API makes the row-value +> list awkward, build the `IN` list with a plain loop over +> `builder.push("(").push_bind(..)` calls instead — the shape of the SQL is what +> matters, not the builder idiom. Verify the generated SQL against the test +> before moving on. + +`set_message_read` becomes `set_placement_read`, updating `placements` by its +full primary key and keeping the `RowNotFound` contract: + +```rust + /// Update one placement's read flag in place. + /// + /// Scoped to a single mailbox occupancy because IMAP `\Seen` is per-mailbox: + /// a message read in `INBOX` is not thereby read in `Archive`. Returns + /// `DatabaseError::Query(sqlx::Error::RowNotFound)` when no such placement + /// exists, so callers can still distinguish "flag flipped" from "never + /// there" rather than silently succeeding on a no-op update. + pub async fn set_placement_read( + &self, + key: &PlacementKey, + read: bool, + ) -> Result<(), DatabaseError> { + let result = sqlx::query( + "UPDATE placements SET read_flag = ? + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(if read { 1i64 } else { 0i64 }) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + if result.rows_affected() == 0 { + return Err(DatabaseError::Query(sqlx::Error::RowNotFound)); + } + Ok(()) + } +``` + +`list_folders` / `list_folders_page` group over `placements` and take an +`account_id`: + +```sql + SELECT folder_id, COUNT(*) as total, + SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread + FROM placements + WHERE account_id = ? + GROUP BY folder_id + ORDER BY folder_id ASC +``` + +Add to `list_folders`'s doc comment: + +```rust + /// Folders are derived from the placements that reference them rather than + /// stored in their own table, so a folder exists exactly as long as it holds + /// mail. Scoping by account is required, not cosmetic: folder ids are not + /// globally unique -- every account has an `INBOX` -- and grouping across + /// accounts would report one merged folder holding both accounts' mail. +``` + +Fix the unread count at `db.rs:471` to read +`SELECT COUNT(*) FROM placements WHERE read_flag = 0`. + +In `search.rs`, change the FTS hit column from `id` to `message_key` and have +hits resolve through `get_message`; a hit names a message, not a placement. + +- [ ] **Step 4: Migrate the existing tests in `db.rs`** + +Every pre-existing test constructing an `Email` with `folder_id` / `remote_id` / +`uid_validity` / `read` must move those into a `Placement` and call +`save_email_at`. Work through the compiler errors; there is no behaviour +decision to make in this step. Two that need real thought rather than +mechanical translation: + +- the corrupted-ciphertext test around `db.rs:3276` — it must still fail closed + on `get_message`, unchanged in intent +- `existing_message_ids_returns_only_the_ids_already_persisted` around + `db.rs:4649` — rewrite as the `existing_placements` test already added above, + and delete the original + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cargo test -p nuncio-store` +Expected: PASS, whole crate. + +- [ ] **Step 6: Commit** + +```bash +git add crates/nuncio-store/src/db.rs crates/nuncio-store/src/search.rs +git commit -m "feat(store): read messages through their placements" +``` + +--- + +## Task 5: Store deletes — the FTS plaintext-orphan rule + +**Files:** +- Modify: `crates/nuncio-store/src/db.rs` (`delete_messages` → `delete_placements`) +- Test: `crates/nuncio-store/src/db.rs` inline tests + +**Interfaces:** +- Consumes: `PlacementKey` from Task 4; `save_email_at` / `placements_of` from Task 3. +- Produces: `delete_placements(&self, keys: &[PlacementKey]) -> Result<DeleteOutcome, DatabaseError>` with `pub struct DeleteOutcome { pub placements_removed: u64, pub messages_reaped: u64 }` +- Test helper: `sample_message_and_placement`, added to `db.rs`'s test module in Task 3 Step 1 (returns an `(Email, Placement)` pair for `account_id = "acct-1"`, `uid_validity = "42"`). + +This task carries the non-negotiable that is easiest to get backwards, so its +tests are written to fail loudly in **both** directions. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[tokio::test] +async fn dropping_one_placement_keeps_the_body_searchable_from_the_other() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), ..inbox.clone() }; + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &archive).await.unwrap(); + + let inbox_key = PlacementKey { account_id: "acct-1".into(), folder_id: "INBOX".into(), uid_validity: "42".into(), remote_id: "5".into() }; + let outcome = engine.delete_placements(&[inbox_key]).await.unwrap(); + assert_eq!(outcome.placements_removed, 1); + assert_eq!(outcome.messages_reaped, 0, "the message still lives in Archive"); + + engine.get_message("key-1").await.expect("the message must survive"); + let (fts,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(fts, 1, "search must still find a message that still exists"); +} + +#[tokio::test] +async fn dropping_the_last_placement_reaps_the_message_and_its_plaintext_index() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + + let inbox_key = PlacementKey { account_id: "acct-1".into(), folder_id: "INBOX".into(), uid_validity: "42".into(), remote_id: "5".into() }; + let outcome = engine.delete_placements(&[inbox_key]).await.unwrap(); + assert_eq!(outcome.messages_reaped, 1); + + engine.get_message("key-1").await.expect_err("the message is gone"); + let (fts,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!( + fts, 0, + "an orphaned FTS row would leave the plaintext body readable to anyone \ + with filesystem access after the message itself was deleted" + ); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-store db::tests::dropping_one_placement db::tests::dropping_the_last_placement` +Expected: FAIL — `no method delete_placements`. + +- [ ] **Step 3: Implement the delete path** + +```rust +/// What a [`DatabaseEngine::delete_placements`] call removed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DeleteOutcome { + /// Mailbox occupancies removed. + pub placements_removed: u64, + /// Messages that lost their last placement and were removed with it. + pub messages_reaped: u64, +} + +impl DatabaseEngine { + /// Remove mailbox occupancies, reaping any message left with none. + /// + /// A server that stops reporting a UID has told us the message left *that + /// mailbox*, not that it ceased to exist -- a move reports exactly this in + /// the source folder while the message continues in the destination. So the + /// placement goes unconditionally and the message goes only when its last + /// placement does. + /// + /// The reap is what keeps the plaintext FTS index honest. `messages_fts` + /// holds decrypted bodies keyed on `message_key`, reaped by the + /// `messages_ad` trigger on `messages`. Deleting the message row is + /// therefore the only thing that clears the index: skip the reap and every + /// fully-deleted message leaves its body readable to anyone with filesystem + /// access to the database. + pub async fn delete_placements( + &self, + keys: &[PlacementKey], + ) -> Result<DeleteOutcome, DatabaseError> { + if keys.is_empty() { + return Ok(DeleteOutcome { placements_removed: 0, messages_reaped: 0 }); + } + + let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + let mut placements_removed = 0u64; + let mut touched: Vec<String> = Vec::with_capacity(keys.len()); + + for key in keys { + // Remember which message each placement pointed at before removing + // it -- afterwards there is nothing left to join through. + let owner: Option<(String,)> = sqlx::query_as( + "SELECT message_key FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .fetch_optional(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + let Some((message_key,)) = owner else { continue }; + + let result = sqlx::query( + "DELETE FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + placements_removed += result.rows_affected(); + touched.push(message_key); + } + + touched.sort_unstable(); + touched.dedup(); + + let mut messages_reaped = 0u64; + for message_key in touched { + // Reap only where nothing is left pointing at the message. The + // `messages_ad` trigger clears the FTS row as a consequence. + let result = sqlx::query( + "DELETE FROM messages WHERE message_key = ? + AND NOT EXISTS (SELECT 1 FROM placements WHERE message_key = ?)", + ) + .bind(&message_key) + .bind(&message_key) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + messages_reaped += result.rows_affected(); + } + + tx.commit().await.map_err(DatabaseError::Query)?; + Ok(DeleteOutcome { placements_removed, messages_reaped }) + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p nuncio-store` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/nuncio-store/src/db.rs +git commit -m "feat(store): reap a message only when its last placement goes" +``` + +--- + +## Task 6: Filter evaluation becomes per-placement + +**Files:** +- Modify: `crates/nuncio-filter/src/engine.rs` +- Modify: `crates/nuncio-filter/src/validator.rs` (comments only) +- Test: `crates/nuncio-filter/src/engine.rs` inline tests + +**Interfaces:** +- Consumes: `Email`, `Placement` from Task 1. +- Produces: + - `pub struct PlacedEmail<'a> { pub email: &'a Email, pub placement: &'a Placement }` + - `CompiledFilter::evaluate_condition(&self, placed: PlacedEmail<'_>) -> bool` + - `FilterEngine::evaluate(&self, placed: PlacedEmail<'_>) -> Vec<(FilterRule, Vec<RuleAction>)>` + - `FilterEngine::evaluate_with_timeout(&self, placed: PlacedEmail<'_>, timeout: Duration) -> Vec<(FilterRule, Vec<RuleAction>)>` + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn a_folder_condition_matches_only_the_placement_it_is_evaluated_against() { + let engine = FilterEngine::new(vec![rule_where_folder_is("INBOX")]).unwrap(); + let email = sample_email("key-1"); + let inbox = sample_placement("INBOX", "5"); + let archive = sample_placement("Archive", "9"); + + assert_eq!( + engine.evaluate(PlacedEmail { email: &email, placement: &inbox }).len(), + 1, + "the INBOX placement matches" + ); + assert_eq!( + engine.evaluate(PlacedEmail { email: &email, placement: &archive }).len(), + 0, + "the same message's Archive placement does not" + ); +} + +#[test] +fn an_account_condition_reads_the_placement_account() { + let engine = FilterEngine::new(vec![rule_where_account_is("acct-1")]).unwrap(); + let email = sample_email("key-1"); + let mine = sample_placement("INBOX", "5"); + let theirs = Placement { account_id: "acct-2".into(), ..sample_placement("INBOX", "5") }; + + assert_eq!(engine.evaluate(PlacedEmail { email: &email, placement: &mine }).len(), 1); + assert_eq!(engine.evaluate(PlacedEmail { email: &email, placement: &theirs }).len(), 0); +} +``` + +Add the test helpers `sample_email`, `sample_placement`, +`rule_where_folder_is`, `rule_where_account_is` next to the existing engine +test helpers, mirroring the rule-construction style already used at +`engine.rs:650`. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-filter engine::tests::a_folder_condition_matches_only` +Expected: FAIL — `cannot find type PlacedEmail`. + +- [ ] **Step 3: Implement per-placement evaluation** + +Add near the top of `engine.rs`: + +```rust +/// A message together with the one mailbox occupancy being evaluated. +/// +/// `WHERE FOLDER = 'INBOX'` has no answer for a message that sits in three +/// folders at once, so evaluation is defined against a single placement and the +/// caller fans out over however many the message has. Borrowing both halves +/// keeps that fan-out free of clones. +#[derive(Debug, Clone, Copy)] +pub struct PlacedEmail<'a> { + /// Identity and content -- the same for every placement. + pub email: &'a Email, + /// The occupancy this evaluation is about. + pub placement: &'a Placement, +} +``` + +Thread `PlacedEmail` through `evaluate_condition`, `eval_node`, and +`eval_leaf`, replacing the `email: &Email` parameter. In `eval_leaf`, the +content fields read `placed.email`, and the two placement-scoped fields become: + +```rust + FilterField::Folder => Self::eval_string_op( + &placed.placement.folder_id, + &leaf.operator, + &leaf.value, + regexes, + ), + FilterField::Account => Self::eval_string_op( + &placed.placement.account_id, + &leaf.operator, + &leaf.value, + regexes, + ), +``` + +In `evaluate` and `evaluate_with_timeout`, the account-scope gate reads the +placement: + +```rust + if filter.rule.matches_account(&placed.placement.account_id) && filter.evaluate_condition(placed) { +``` + +`evaluate_with_timeout` moves owned values into `spawn_blocking`, so it needs +owned clones inside the loop: + +```rust + let filter_for_task = filter.clone(); + let email_for_task = placed.email.clone(); + let placement_for_task = placed.placement.clone(); + let eval_task = tokio::task::spawn_blocking(move || { + filter_for_task.evaluate_condition(PlacedEmail { + email: &email_for_task, + placement: &placement_for_task, + }) + }); +``` + +Update `evaluate`'s doc comment to record the new contract: + +```rust + /// Evaluate one placement of a message, returning the matching rules' actions. + /// + /// Defined per placement, not per message: `FOLDER` and `ACCOUNT` are + /// properties of where the message sits, and a message can sit in several + /// places at once. Callers holding a multi-placement message evaluate each + /// placement and are responsible for not acting twice on one message -- see + /// the fire-once claim in the store. + /// + /// A rule only ever fires for the account it was created against + /// (`FilterRule::target_account`, set via `ON ACCOUNT` or `*` for all + /// accounts) — condition matching alone is not account-scoped, so this + /// check must happen before `evaluate_condition` runs. +``` + +In `validator.rs`, correct the comment above the `Folder | Account` arm at +line ~99 to say these validate against a placement's coordinates. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p nuncio-filter` +Expected: PASS. Existing engine tests need their `evaluate(&email)` calls +rewritten as `evaluate(PlacedEmail { .. })`; that is mechanical. + +- [ ] **Step 5: Commit** + +```bash +git add crates/nuncio-filter/src/engine.rs crates/nuncio-filter/src/validator.rs +git commit -m "feat(filter): evaluate rules against a placement, not a message" +``` + +--- + +## Task 7: Mail backends emit identity and placement + +**Files:** +- Modify: `crates/nuncio-mail/src/imap.rs` (~lines 1186, 1208, 1244–1310) +- Modify: `crates/nuncio-mail/src/jmap.rs` (~line 401) +- Modify: `crates/nuncio-mail/src/mock.rs` +- Modify: `crates/nuncio-mail/src/backend.rs` (`FolderChanges`) +- Test: `crates/nuncio-mail/tests/wiremock_jmap_test.rs`, inline `imap.rs` tests + +**Interfaces:** +- Consumes: `derive_message_key`, `RemoteIdentity`, `Placement`, `MessageIdentity`. +- Produces: `pub struct PlacedMessage { pub email: Email, pub source: IdentitySource, pub placement: Placement }`; `FolderChanges.upserts: Vec<PlacedMessage>`; `FolderChanges.removals: Vec<PlacementKey>`; `FolderChanges.present: Option<Vec<PlacementKey>>`. + +- [ ] **Step 1: Write the failing test** + +```rust +#[tokio::test] +async fn a_message_fetched_from_two_folders_keeps_one_identity() { + // Both folders return the same Message-ID and the same octets, so the + // content tier must produce one key from two different UIDs. + let raw = b"Message-ID: <a@b.example>\r\nSubject: Hi\r\n\r\nbody"; + let hash = Email::content_hash_of(raw); + let inbox = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: None, gm_msgid: None, message_id: Some("a@b.example"), content_hash: Some(&hash) }, + "INBOX", "42", "5", + ); + let archive = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: None, gm_msgid: None, message_id: Some("a@b.example"), content_hash: Some(&hash) }, + "Archive", "77", "9", + ); + assert_eq!(inbox.key, archive.key); + assert_eq!(inbox.source, IdentitySource::MessageIdContent); +} +``` + +Place this in `crates/nuncio-mail/src/imap.rs`'s test module so it guards the +backend's own use of the precedence, not just the core function. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p nuncio-mail a_message_fetched_from_two_folders` +Expected: FAIL to compile until `nuncio-mail` is migrated. + +- [ ] **Step 3: Migrate the IMAP backend** + +At each of `imap.rs:1186`, `1208`, and `1278`, the call currently reads +`Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id)`. +Replace with a helper on the backend: + +```rust + /// Build the identity and placement for one fetched message. + /// + /// `EMAILID` and `X-GM-MSGID` are only present when the server advertised + /// `OBJECTID` / `X-GM-EXT-1` and the FETCH asked for them; when neither is + /// available the pair of `Message-ID` and content hash carries the identity, + /// and only if both are missing does this fall back to the folder-scoped + /// surrogate -- which is the one tier that does not survive a move. + fn place( + &self, + folder_id: &str, + uid_validity: &str, + remote_id: &str, + email_id: Option<&str>, + gm_msgid: Option<&str>, + message_id: Option<&str>, + content_hash: Option<&str>, + read: bool, + ) -> (MessageIdentity, Placement) { + let identity = Email::derive_message_key( + &self.account_id, + RemoteIdentity { email_id, gm_msgid, message_id, content_hash }, + folder_id, + uid_validity, + remote_id, + ); + let placement = Placement { + account_id: self.account_id.clone(), + folder_id: folder_id.to_string(), + uid_validity: uid_validity.to_string(), + remote_id: remote_id.to_string(), + read, + }; + (identity, placement) + } +``` + +`EMAILID` and `X-GM-MSGID` are not fetched today. Pass `None` for both and +leave a comment saying so — wiring the FETCH items is #395/#396 territory and +adding it here would widen this change without a test server that serves them: + +```rust + // EMAILID (RFC 8474) and X-GM-MSGID are not requested by the current + // FETCH, so identity rests on the Message-ID/content pair where the + // message carried a Message-ID and on the folder-scoped surrogate + // otherwise. Adding those FETCH items lifts existing rows to a stronger + // tier on the next resync at no cost to this code path. +``` + +Change `FolderChanges` in `backend.rs`: + +```rust +/// One message as a backend surfaced it: identity, content, and the mailbox +/// occupancy it was found in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlacedMessage { + /// Identity and content. `email.id` is the derived message key. + pub email: Email, + /// Which precedence tier produced `email.id`. + pub source: IdentitySource, + /// The occupancy this fetch found it in. + pub placement: Placement, +} +``` + +and retype `upserts`, `removals`, and `present` as noted in the Interfaces +block above. `removals` and `present` become placement keys because what a +folder stops reporting is an occupancy. + +- [ ] **Step 4: Migrate the JMAP backend and the mock** + +At `jmap.rs:401`, JMAP has no UIDVALIDITY, so it keeps its existing sentinel as +`uid_validity` and uses the JMAP object id as `remote_id`. JMAP's object id is +server-assigned and account-stable, so pass it as `email_id` — that is exactly +what the top tier is for: + +```rust + // A JMAP Email id is server-assigned and stable across mailboxes within + // the account, which is the same guarantee RFC 8474 EMAILID gives, so it + // enters at the top tier rather than through the surrogate. +``` + +In `mock.rs`, have `MockMailBackend` emit `PlacedMessage` values so daemon +tests can construct multi-placement scenarios. + +Leave the JMAP `present: None` gap exactly as it is — JMAP still accumulates +ghosts until `Email/changes` lands, and pretending otherwise here would delete +mail. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cargo test -p nuncio-mail` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/nuncio-mail/ +git commit -m "feat(mail): derive message identity and emit placements" +``` + +--- + +## Task 8: Sync wiring — placement-aware fan-out and the fire-once claim + +**Files:** +- Modify: `crates/nunciod/src/sync.rs` (`fetch_and_persist`, `apply_filter_actions`) +- Test: `crates/nunciod/src/sync.rs` inline tests, `crates/nunciod/tests/filters_triage_e2e_test.rs` + +**Interfaces:** +- Consumes: `save_email_at`, `existing_placements`, `delete_placements`, `claim_filter_fire`, `PlacedEmail`, `PlacedMessage`. +- Produces: no new public API; `fetch_and_persist` keeps its signature and return type. + +This task carries the bug the issue calls out by name: under a message-scoped +"is new" test, a message arriving in a second folder looks like it was already +seen and **silently never fires any filter**. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[tokio::test] +async fn a_message_arriving_in_a_second_folder_still_gets_evaluated() { + // The regression this guards: with identity now folder-independent, a + // message already stored from INBOX has a known message_key, so a + // message-scoped "already present" test would classify its arrival in + // Archive as old and skip filtering it entirely. + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let backend = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]); + let engine = FilterEngine::new(vec![rule_matching_everything()]).unwrap(); + + let processed = fetch_and_persist(&db, &EventBus::new(), &backend, &engine, Some("acct-1")) + .await + .unwrap(); + + assert_eq!(processed, 2, "both placements are processed"); + let placements = db.placements_of(&expected_key()).await.unwrap(); + assert_eq!(placements.len(), 2); +} + +#[tokio::test] +async fn a_rule_fires_once_for_a_message_that_lands_in_two_folders() { + // MOVE and FLAG converge under repetition; FORWARD and CALL WEBHOOK do not. + // Two placements of one message must not page an on-call engineer twice. + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let backend = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]); + let engine = FilterEngine::new(vec![rule_matching_everything()]).unwrap(); + + fetch_and_persist(&db, &EventBus::new(), &backend, &engine, Some("acct-1")) + .await + .unwrap(); + + let (fires,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM filter_fired") + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(fires, 1, "one claim, therefore one set of actions"); +} +``` + +Add `MockMailBackend::with_same_message_in(folders: &[&str])` to `mock.rs`, +returning the same `email` (same derived key) under a different +`folder_id`/`remote_id` per folder. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nunciod sync::tests::a_message_arriving_in_a_second_folder` +Expected: FAIL to compile until the sync path is migrated. + +- [ ] **Step 3: Rewrite the classification and fan-out** + +In `fetch_and_persist`, replace the chunk-classification block: + +```rust + // One round trip classifies the whole chunk. This asks about + // *placements*, not messages: identity is folder-independent now, so a + // message already stored from another folder is still a first arrival + // here, and testing the message key instead would classify it as seen + // and skip filtering it entirely. + let chunk_keys: Vec<PlacementKey> = placed + .iter() + .map(|p| PlacementKey { + account_id: p.placement.account_id.clone(), + folder_id: p.placement.folder_id.clone(), + uid_validity: p.placement.uid_validity.clone(), + remote_id: p.placement.remote_id.clone(), + }) + .collect(); + let already_present = db.existing_placements(&chunk_keys).await?; + let mut seen_this_pass = std::collections::HashSet::new(); + + for message in placed { + let key = PlacementKey { + account_id: message.placement.account_id.clone(), + folder_id: message.placement.folder_id.clone(), + uid_validity: message.placement.uid_validity.clone(), + remote_id: message.placement.remote_id.clone(), + }; + let is_new = !already_present.contains(&key) && seen_this_pass.insert(key); + db.save_email_at(&message.email, message.source, &message.placement).await?; + if is_new { + apply_filter_actions(db, filter_engine, &message.email, &message.placement).await; + } + synced += 1; + } +``` + +Change `apply_filter_actions` to take the placement, evaluate against it, and +gate every action behind the claim: + +```rust +async fn apply_filter_actions( + db: &nuncio_store::db::DatabaseEngine, + filter_engine: &FilterEngine, + email: &nuncio_core::model::Email, + placement: &nuncio_core::model::Placement, +) { + let placed = nuncio_filter::PlacedEmail { email, placement }; + for (rule, actions) in filter_engine.evaluate(placed) { + // Claim before acting. The claim is keyed on message identity, so the + // same mail arriving in a second folder -- an ordinary MOVE, or a Gmail + // label -- does not run the actions again. That is load-bearing for + // FORWARD and CALL WEBHOOK, which land on third parties and have no + // shared state to converge against; MOVE and FLAG would survive the + // repetition, but there is no reason to pay for it. + match db.claim_filter_fire(&rule.id, &email.id).await { + Ok(true) => {} + Ok(false) => { + tracing::debug!( + rule_id = %rule.id, + message_key = %email.id, + folder_id = %placement.folder_id, + "rule already fired for this message; skipping actions for this placement" + ); + continue; + } + Err(e) => { + // Fail closed. An unverifiable claim must not become a licence + // to act: a lost claim check on a FORWARD is a duplicate mail + // the recipient cannot un-receive. + tracing::error!( + rule_id = %rule.id, + message_key = %email.id, + error = %e, + "could not claim the filter fire; skipping actions" + ); + continue; + } + } + + // ... existing per-action dispatch, unchanged apart from taking + // `placement` where it previously read `email.folder_id` / + // `email.remote_id` / `email.uid_validity` + } +} +``` + +Every `RemoteMutationSpec` built here now takes its `folder_id`, `remote_id`, +and `uid_validity` from `placement`, and its `message_id` from `email.id`. +`MARK READ` names the placement it was evaluated against — which is what makes +it well-defined at all, since `\Seen` is per-mailbox. + +Update the removals block to work in placements: + +```rust + if let Some(present) = changes.present { + let present: std::collections::HashSet<PlacementKey> = present.into_iter().collect(); + let stored = db.placements_in_folder(acct, &folder.id).await?; + gone.extend(stored.into_iter().filter(|key| !present.contains(key))); + } + + gone.sort_unstable(); + gone.dedup(); + if !gone.is_empty() { + let outcome = db.delete_placements(&gone).await?; + tracing::info!( + account_id = %acct, + folder_id = %folder.id, + placements_removed = outcome.placements_removed, + messages_reaped = outcome.messages_reaped, + "removed placements that are no longer on the server" + ); + } +``` + +`PlacementKey` needs `Ord` for the `sort_unstable`/`dedup` pair; derive +`PartialOrd, Ord` on it in Task 4. + +Rewrite the doc comment on `fetch_and_persist` so it describes placement +classification rather than message classification — the current text at +`sync.rs:200–228` explains the old model in detail and would be actively +misleading left as-is. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p nunciod` +Expected: PASS, including `filters_triage_e2e_test`. + +- [ ] **Step 5: Commit** + +```bash +git add crates/nunciod/src/sync.rs crates/nuncio-mail/src/mock.rs +git commit -m "feat(sync): classify arrivals per placement and claim fires once" +``` + +--- + +## Task 9: Daemon surface — gRPC, CLI, and the proto comment + +**Files:** +- Modify: `crates/nunciod/src/grpc.rs` +- Modify: `crates/nuncio-proto/proto/nuncio/v1/nuncio.proto` (comment only) +- Modify: `crates/nuncio-cli/src/runner.rs` (only where it fails to compile) +- Test: `crates/nunciod/tests/spine_e2e_test.rs`, `crates/nunciod/tests/typed_errors_e2e_test.rs` + +**Interfaces:** +- Consumes: everything above. +- Produces: no wire-schema change. `Message.folder_id` and `Message.read` are populated from the placement the message was reached through. + +- [ ] **Step 1: Write the failing test** + +```rust +#[tokio::test] +async fn list_messages_reports_the_folder_it_was_asked_about() { + // A message in two folders appears in both listings, each time carrying the + // placement that listing is about -- not an arbitrary one of the two. + let harness = SpineHarness::start().await; + harness.seed_message_in(&["INBOX", "Archive"]).await; + + for folder in ["INBOX", "Archive"] { + let listed = harness + .client() + .list_messages(ListMessagesRequest { folder_id: folder.into(), ..Default::default() }) + .await + .unwrap() + .into_inner(); + assert_eq!(listed.messages.len(), 1); + assert_eq!(listed.messages[0].folder_id, folder); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p nunciod --test spine_e2e_test list_messages_reports_the_folder` +Expected: FAIL to compile until `grpc.rs` is migrated. + +- [ ] **Step 3: Migrate the gRPC layer** + +`ListMessages` already carries a folder, so map each `(Email, Placement)` pair +straight onto the proto `Message`, taking `folder_id` and `read` from the +placement. + +`GetMessage` receives only an id, so it must choose. Resolve through +`placements_of` and take the first — `placements_of` orders by `folder_id ASC`, +so the choice is deterministic rather than arbitrary: + +```rust + // A message key can name several placements and this RPC's request + // carries no folder to disambiguate them. Report the first by folder id + // so the answer is at least stable between calls, and mark the response + // honestly: the full placement list needs a wire field this contract + // does not have yet. + let placements = db.placements_of(&request.message_id).await?; + let Some(primary) = placements.first() else { + return Err(tonic::Status::not_found( + "message has no placement; it is no longer in any folder", + )); + }; +``` + +Add the same note to the `.proto` beside `Message.folder_id`: + +```proto + // Mailbox folder identifier (e.g. "inbox"). + // + // A message can occupy several folders at once. This names the one the + // message was reached through: the requested folder for a listing, and the + // lowest-sorting folder id for a lookup by id. A field carrying the complete + // placement set is not yet part of this contract. + string folder_id = 3; +``` + +and correct the `Message.id` comment, whose current text describes the old +folder-scoped derivation: + +```proto + // Unique message identifier. OPAQUE and server-assigned: content-addressed + // from the message's strongest available identity, so it is independent of + // which folder the message currently occupies and survives a move. Never a + // raw upstream protocol id (e.g. an IMAP UID, which is only stable within a + // single IMAP session and is NOT a durable cross-session identifier). + // Clients MUST treat this as an opaque token -- pass it back verbatim in + // later requests, but never parse, construct, or assume any structure in + // its value. + string id = 1; +``` + +- [ ] **Step 4: Reconcile the descriptor golden** + +Run: `cargo test -p nuncio-proto descriptor_matches_committed_golden` + +If it passes, comments are not in the descriptor and nothing more is needed. +If it fails, the test prints the exact path of the freshly generated +descriptor; copy it over `crates/nuncio-proto/proto/nuncio/v1/descriptor.bin` +exactly as the failure message instructs, then re-run to confirm. + +- [ ] **Step 5: Fix the CLI** + +Work through `cargo check-all` errors in `crates/nuncio-cli/src/runner.rs`. +The CLI consumes the proto types, which have not changed shape, so the fixes +should be confined to test fixtures constructing `Message` values. + +- [ ] **Step 6: Run the full gate** + +```bash +cargo fmt --all -- --check +cargo check-all +cargo test-all +``` + +Expected: all three pass. Give the test command ~540s; `cargo test-all` can +exceed a 10-minute tool timeout on this workspace. + +- [ ] **Step 7: Commit** + +```bash +git add crates/nunciod/src/grpc.rs crates/nuncio-cli/src/runner.rs crates/nuncio-proto/ +git commit -m "feat(daemon): present the placement a message was reached through" +``` + +--- + +## Task 10: Documentation and PR + +**Files:** +- Modify: `docs/ROADMAP.md` and `docs/roadmap/index.html` (both, in one commit) +- Modify: `docs/HANDOFF.md` + +- [ ] **Step 1: Update the roadmap in both formats** + +Mark the message/placement split done in `docs/ROADMAP.md`, and make the +matching edit to `docs/roadmap/index.html`. The rendered page is published at +<https://koftwentytwo.github.io/nuncio/roadmap/> and must never drift from the +markdown; the repo requires both to change in the same commit. + +- [ ] **Step 2: Commit** + +```bash +git add docs/ +git commit -m "docs(roadmap): mark the message/placement split complete" +``` + +- [ ] **Step 3: Push and open the PR** + +SSH has no key in this environment; push over HTTPS with the `gh` credential +helper and leave the remote as SSH: + +```bash +git -c credential.helper='!gh auth git-credential' \ + push https://github.com/KofTwentyTwo/nuncio.git story/message-placement-split +``` + +Open the PR against `dev`. The body must carry the reasoning, not just the +what, and must state explicitly: + +- the branch merges #398 (#386) and #400 (#387) as its declared ADR + dependencies, so **those two PRs must land first** +- `IDENTITY_SCHEMA_VERSION` goes 1 → 2, so **every existing local store resets + its cached mail on next open**; accounts, credentials, and rules survive +- `EMAILID` / `X-GM-MSGID` are not yet fetched, so identity currently rests on + the Message-ID/content tier and the surrogate fallback; adding those FETCH + items lifts rows to a stronger tier on the next resync +- JMAP still reports `present: None` and therefore still accumulates ghost + messages — unchanged by this PR, and needs `Email/changes` +- the wire contract is unchanged; `Message.folder_id` now names the placement + the message was reached through, and the full placement set awaits #395 +- expect conflicts in `db.rs`, `imap.rs`, and `sync.rs` against the + #401–#403 chain + +--- + +## Deferred, deliberately + +Named here so they are not mistaken for oversights. + +- **`EMAILID` / `X-GM-MSGID` are not requested.** Wiring the FETCH items needs a + test server that serves them; the precedence handles their absence correctly + and picks them up for free once they arrive. +- **The full placement set is not on the wire.** `Message` gains it in #395, + which owns the proto surface. +- **`GetMessage` picks one placement.** Deterministic (lowest folder id) and + documented, but genuinely lossy until the contract can express the set. +- **JMAP ghost messages persist.** `present: None` is unchanged; the fix is + `Email/changes` and is not filed as an issue yet. -- 2.54.0 From 1aacf74509730032c6ceaf039fcc574750b032d4 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 13:56:45 -0500 Subject: [PATCH 61/95] docs(plan): record pre-flight rulings on gate, ledger access, scoping --- .../2026-08-11-message-placement-split.md | 95 +++++++++++++++++-- 1 file changed, 88 insertions(+), 7 deletions(-) diff --git a/docs/superpowers/plans/2026-08-11-message-placement-split.md b/docs/superpowers/plans/2026-08-11-message-placement-split.md index abfdd78..b2ab37a 100644 --- a/docs/superpowers/plans/2026-08-11-message-placement-split.md +++ b/docs/superpowers/plans/2026-08-11-message-placement-split.md @@ -11,7 +11,11 @@ ## Global Constraints - **Branch:** `story/message-placement-split`, already created off `dev` with `story/message-id-capture` (#386) and `story/schema-version-and-token-scheme` (#387) merged in. Baseline gate is green on it (627+ tests). Do not rebase onto the #401–#403 chain. -- **Gate (all three must pass before every commit):** `cargo fmt --all -- --check`, then `cargo check-all`, then `cargo test-all`. Warnings are hard errors; `unwrap_used` / `expect_used` / `panic` / `todo` are `deny` outside tests. +- **Gate.** Removing `Email`'s folder scalars breaks four crates until Task 9 lands, so the gate is staged: + - **Tasks 1–8:** `cargo fmt --all -- --check` **and** `cargo test -p <crate under change>` must pass. The workspace will not compile end-to-end during these tasks; that is expected and is not a reason to stop or to widen the task. + - **Task 9 onward, and the PR head:** the full gate — `cargo fmt --all -- --check`, then `cargo check-all`, then `cargo test-all`. + - Warnings are hard errors throughout; `unwrap_used` / `expect_used` / `panic` / `todo` are `deny` outside tests. Never silence a lint to get a commit through. + - `cargo test-all` can exceed a 10-minute tool timeout on this workspace; allow ~540s. - **Commits:** Conventional Commits, imperative, <72-char subject, **no AI attribution**. `Refs #394` in the message body is fine. - **Comments:** explain intent, constraints, rationale. **No issue numbers, GH refs, or `Phase N` breadcrumbs in `.rs` / `.proto` comments.** - **No live network in tests.** IMAP/JMAP/SMTP/CalDAV/CardDAV all mocked; keyring via `MockKeyring`. Ephemeral SQLite (`tempfile` / `:memory:`) per test. @@ -387,7 +391,12 @@ git commit -m "feat(core): separate message identity from mailbox placement" **Interfaces:** - Consumes: `DatabaseEngine::reconcile_schema_version` and `reset_cached_message_store` (both from #387). -- Produces: `placements` and `filter_fired` tables; `messages` reshaped to `message_key` + identity columns; `IDENTITY_SCHEMA_VERSION == 2`. +- Produces: `placements` and `filter_fired` tables; `messages` reshaped to `message_key` + identity columns; `IDENTITY_SCHEMA_VERSION == 2`; and three store methods — + - `claim_filter_fire(&self, rule_id: &str, message_key: &str) -> Result<bool, DatabaseError>` + - `has_filter_fired(&self, rule_id: &str, message_key: &str) -> Result<bool, DatabaseError>` + - `filter_fire_count(&self) -> Result<i64, DatabaseError>` + + The last two are `pub` (not test-only) because `nunciod` asserts on the ledger from another crate and cannot reach the pool. - [ ] **Step 1: Write the failing tests** @@ -631,6 +640,41 @@ separate function. Ok(result.rows_affected() == 1) } + + /// Whether `rule_id` has already acted on `message_key`. + /// + /// Public rather than test-only: the claim ledger answers a question + /// operators and the outbox both have a real reason to ask -- "did this rule + /// already act on this message" -- and callers in other crates cannot reach + /// the pool directly. + pub async fn has_filter_fired( + &self, + rule_id: &str, + message_key: &str, + ) -> Result<bool, DatabaseError> { + let found: Option<(i64,)> = sqlx::query_as( + "SELECT 1 FROM filter_fired WHERE rule_id = ? AND message_key = ?", + ) + .bind(rule_id) + .bind(message_key) + .fetch_optional(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(found.is_some()) + } + + /// How many distinct (rule, message) fires the ledger records. + /// + /// Counterpart to [`Self::has_filter_fired`] for callers that need to assert + /// on the ledger as a whole rather than one entry. + pub async fn filter_fire_count(&self) -> Result<i64, DatabaseError> { + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM filter_fired") + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; + Ok(count) + } ``` Use whichever clock helper `db.rs` already uses for `created_at`/`matched_at` @@ -1950,11 +1994,12 @@ async fn a_rule_fires_once_for_a_message_that_lands_in_two_folders() { .await .unwrap(); - let (fires,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM filter_fired") - .fetch_one(db.pool()) - .await - .unwrap(); - assert_eq!(fires, 1, "one claim, therefore one set of actions"); + assert_eq!( + db.filter_fire_count().await.unwrap(), + 1, + "one claim, therefore one set of actions" + ); + assert!(db.has_filter_fired(&rule_matching_everything().id, &expected_key()).await.unwrap()); } ``` @@ -2147,6 +2192,42 @@ Expected: FAIL to compile until `grpc.rs` is migrated. - [ ] **Step 3: Migrate the gRPC layer** +**Account scoping at the boundary.** The store methods now take an +`account_id`, but `ListFoldersRequest` and `ListMessagesRequest` carry none and +this plan does not change the wire schema. Bridge it by iterating the accounts +and merging, which reproduces exactly today's behaviour — no wire change, no +regression: + +```rust + // The request carries no account, so this reproduces the pre-existing + // account-blind view: every account's folders, merged by folder id. The + // store below it is account-scoped and correct; only this presentation + // is ambiguous, and it is ambiguous in precisely the way it already was. + // An account-scoped request field is the real fix and belongs with the + // rest of the placement surface. + let mut merged: std::collections::BTreeMap<String, nuncio_core::model::Folder> = + std::collections::BTreeMap::new(); + for account in db.list_accounts().await? { + for folder in db.list_folders(&account.id).await? { + let entry = merged.entry(folder.id.clone()).or_insert_with(|| { + nuncio_core::model::Folder { + id: folder.id.clone(), + name: folder.name.clone(), + total_messages: 0, + unread_messages: 0, + } + }); + entry.total_messages += folder.total_messages; + entry.unread_messages += folder.unread_messages; + } + } +``` + +Apply the same account loop to `ListMessages`, concatenating each account's +matches for the requested folder and then applying the existing ordering and +page size to the merged result. Use whatever account-listing method `db` +already exposes rather than adding one. + `ListMessages` already carries a folder, so map each `(Email, Placement)` pair straight onto the proto `Message`, taking `folder_id` and `read` from the placement. -- 2.54.0 From 590e9a40f22db2c781f38dbe898b65ff68f97be3 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 14:00:45 -0500 Subject: [PATCH 62/95] feat(core): separate message identity from mailbox placement --- crates/nuncio-core/src/export.rs | 12 +- crates/nuncio-core/src/model.rs | 342 ++++++++++++++++++++++++++++--- 2 files changed, 320 insertions(+), 34 deletions(-) diff --git a/crates/nuncio-core/src/export.rs b/crates/nuncio-core/src/export.rs index 8c5ad0b..89acdde 100644 --- a/crates/nuncio-core/src/export.rs +++ b/crates/nuncio-core/src/export.rs @@ -89,8 +89,8 @@ impl ExportEngine { // Headers let headers = format!( - "Message-ID: <{}>\r\nFrom: {}\r\nTo: {}\r\nSubject: {}\r\nDate: {}\r\nX-Nuncio-Account: {}\r\nX-Nuncio-Folder: {}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n", - email.id, email.sender, email.recipient, email.subject, date_str, email.account_id, email.folder_id + "Message-ID: <{}>\r\nFrom: {}\r\nTo: {}\r\nSubject: {}\r\nDate: {}\r\nX-Nuncio-Account: {}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n", + email.id, email.sender, email.recipient, email.subject, date_str, email.account_id ); writer.write_all(headers.as_bytes())?; total_bytes += headers.len() as u64; @@ -184,14 +184,10 @@ mod tests { Email { id: "msg-100".to_string(), account_id: "acct-1".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "100".to_string(), - uid_validity: "1".to_string(), subject: "Test Subject 1".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: false, body_plain: Some("Hello world body text".to_string()), body_html: None, attachments: Vec::new(), @@ -201,14 +197,10 @@ mod tests { Email { id: "msg-101".to_string(), account_id: "acct-1".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "101".to_string(), - uid_validity: "1".to_string(), subject: "Test Subject 2".to_string(), sender: "charlie@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000100, - read: true, body_plain: Some("Second body text".to_string()), body_html: None, attachments: Vec::new(), diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index 9796db4..666d105 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -61,33 +61,113 @@ pub struct Attachment { pub content: Bytes, } -/// Email message domain entity owned by Nuncio core. +/// Which tier of the identity precedence produced a [`MessageIdentity`]. +/// +/// Persisted alongside the key so a later pass can tell a server-assigned +/// identity from a locally-derived fallback without recomputing it, and so a +/// store can be audited for how much of it rests on the weakest tier. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum IdentitySource { + /// RFC 8474 OBJECTID `EMAILID` -- server-assigned and stable across folders. + EmailId, + /// Gmail's `X-GM-MSGID`, the same guarantee by a vendor extension. + GmailMsgId, + /// `Message-ID` **and** a content hash together. Never the header alone. + MessageIdContent, + /// Folder-scoped last resort: the message is only identified by where it + /// currently sits, so the same mail in another folder is a second message. + Surrogate, +} + +impl IdentitySource { + /// Stable token persisted in `messages.identity_source`. + pub fn as_str(&self) -> &'static str { + match self { + Self::EmailId => "emailid", + Self::GmailMsgId => "gmsgid", + Self::MessageIdContent => "msgid_content", + Self::Surrogate => "surrogate", + } + } + + /// Parse a token written by [`Self::as_str`]. Returns `None` for anything + /// else so an unrecognised value fails loudly rather than defaulting to a + /// tier that would overstate how trustworthy the key is. + pub fn from_str(s: &str) -> Option<Self> { + match s { + "emailid" => Some(Self::EmailId), + "gmsgid" => Some(Self::GmailMsgId), + "msgid_content" => Some(Self::MessageIdContent), + "surrogate" => Some(Self::Surrogate), + _ => None, + } + } +} + +/// A content-addressed message key together with the tier that produced it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MessageIdentity { + /// Hex SHA-256 key identifying the message within its account. + pub key: String, + /// Which precedence tier produced `key`. + pub source: IdentitySource, +} + +/// Server-supplied identity hints for one message, in precedence order. +/// +/// Borrowed rather than owned because every caller already holds these as +/// fields on a parsed response and only needs them for the duration of the +/// key derivation. +#[derive(Debug, Clone, Copy, Default)] +pub struct RemoteIdentity<'a> { + /// RFC 8474 `EMAILID` from a `FETCH ... (EMAILID)`, if the server has OBJECTID. + pub email_id: Option<&'a str>, + /// Gmail `X-GM-MSGID`, if the server advertises `X-GM-EXT-1`. + pub gm_msgid: Option<&'a str>, + /// Normalized `Message-ID` (see [`Email::normalize_message_id`]). + pub message_id: Option<&'a str>, + /// Hex SHA-256 over the full RFC822 octets (see [`Email::content_hash_of`]). + pub content_hash: Option<&'a str>, +} + +/// One occupancy of a message in one mailbox. +/// +/// A message can sit in several folders at once (an IMAP `COPY`, a Gmail label, +/// a message that is both in a thread's folder and in `\All`). The addressing +/// coordinates and the read flag are properties of the *occupancy*, not of the +/// message: IMAP `\Seen` is per-mailbox, and a UID is only meaningful inside one +/// `uid_validity` scope of one folder. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Email { - /// Opaque, deterministic surrogate identifier for the message. - /// - /// Derived from [`Email::surrogate_id`] over the account, folder, - /// UIDVALIDITY, and protocol-native id. It carries no decodable internal - /// encoding (callers must treat it as opaque) and is stable across - /// re-syncs of the same message, so persisting it upserts the same row - /// rather than duplicating. Distinct messages that happen to share a - /// protocol id across folders or accounts hash to distinct surrogates, so - /// one can never silently overwrite another. - pub id: String, - /// Account identifier owning the message. +pub struct Placement { + /// Account owning the mailbox. pub account_id: String, /// Mailbox folder identifier (e.g. "inbox"). pub folder_id: String, - /// Protocol-native message id used to address the message on its server: - /// the IMAP UID as a decimal string, or the JMAP Email object id. This is - /// what a remote mutation must use to identify the message on the wire -- - /// never the opaque [`Email::id`]. - pub remote_id: String, - /// The addressing scope the `remote_id` is only meaningful within: the IMAP - /// folder UIDVALIDITY as a decimal string. Protocols without a UIDVALIDITY - /// (JMAP) carry a stable sentinel instead, so the surrogate id stays - /// deterministic. + /// The IMAP UIDVALIDITY the `remote_id` was captured under, as a decimal + /// string; a stable sentinel for protocols without one (JMAP). In the + /// primary key because UIDs restart at 1 after a bump and would otherwise + /// collide with rows written before it. pub uid_validity: String, + /// Protocol-native id addressing the message in this mailbox: the IMAP UID + /// as a decimal string, or the JMAP Email object id. + pub remote_id: String, + /// Read/unread state **in this mailbox**. + pub read: bool, +} + +/// Email message domain entity owned by Nuncio core. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Email { + /// Opaque, account-scoped identity for the message itself. + /// + /// Derived by [`Email::derive_message_key`], so it is independent of which + /// folder the message currently occupies: moving a message between folders + /// does not change it. Where the message actually sits, and its read state + /// there, live in [`Placement`] rows instead. Callers must treat this as an + /// opaque token -- it carries no decodable structure. + pub id: String, + /// Account identifier owning the message. + pub account_id: String, /// Subject line. pub subject: String, /// Sender address (e.g. "alice@nuncio.mx"). @@ -96,8 +176,6 @@ pub struct Email { pub recipient: String, /// Unix timestamp of message arrival. pub received_at: i64, - /// Read/unread flag status. - pub read: bool, /// Plaintext message body. pub body_plain: Option<String>, /// HTML message body. @@ -196,6 +274,82 @@ impl Email { hasher.update(raw_bytes); hex::encode(hasher.finalize()) } + + /// Domain-separated, length-prefixed digest over an identity tier's inputs. + /// + /// The tier label participates in the hash so that a value appearing in two + /// different tiers (a server whose `EMAILID` happens to equal another's + /// `X-GM-MSGID`) cannot produce one key. Length prefixes stop two distinct + /// field tuples serializing to the same byte stream, exactly as in + /// [`Email::surrogate_id`]. + fn identity_digest(tier: &str, account_id: &str, parts: &[&str]) -> String { + let mut hasher = Sha256::new(); + for field in [tier, account_id].into_iter().chain(parts.iter().copied()) { + hasher.update((field.len() as u64).to_le_bytes()); + hasher.update(field.as_bytes()); + } + hex::encode(hasher.finalize()) + } + + /// Derive the account-scoped message key by identity precedence. + /// + /// `EMAILID` → `X-GM-MSGID` → (`Message-ID` + content hash) → surrogate. + /// Every tier is account-scoped: identity is only ever claimed within the + /// account that fetched it, so one account's server cannot mint a key that + /// addresses another account's mail. + /// + /// The third tier requires **both** a `Message-ID` and a content hash. A + /// `Message-ID` is public -- it is echoed in the `References` of every reply + /// -- and forgeable, so keying on it alone would let a crafted message + /// address, and with an upsert overwrite, a message the user already trusts, + /// and would evade filtering by never being classified as new. Pairing it + /// with the content hash also gets the ordinary cases right: a mailing-list + /// copy and a direct copy share a `Message-ID` but differ in content, and so + /// are correctly two messages. + /// + /// The surrogate tier is folder-scoped and therefore *not* stable across a + /// move; it is the honest answer when the server offered nothing better. + pub fn derive_message_key( + account_id: &str, + remote: RemoteIdentity<'_>, + folder_id: &str, + uid_validity: &str, + remote_id: &str, + ) -> MessageIdentity { + if let Some(email_id) = remote.email_id.filter(|v| !v.is_empty()) { + return MessageIdentity { + key: Self::identity_digest("emailid", account_id, &[email_id]), + source: IdentitySource::EmailId, + }; + } + if let Some(gm_msgid) = remote.gm_msgid.filter(|v| !v.is_empty()) { + return MessageIdentity { + key: Self::identity_digest("gmsgid", account_id, &[gm_msgid]), + source: IdentitySource::GmailMsgId, + }; + } + if let (Some(message_id), Some(content_hash)) = ( + remote.message_id.filter(|v| !v.is_empty()), + remote.content_hash.filter(|v| !v.is_empty()), + ) { + return MessageIdentity { + key: Self::identity_digest( + "msgid_content", + account_id, + &[message_id, content_hash], + ), + source: IdentitySource::MessageIdContent, + }; + } + MessageIdentity { + key: Self::identity_digest( + "surrogate", + account_id, + &[folder_id, uid_validity, remote_id], + ), + source: IdentitySource::Surrogate, + } + } } /// Mailbox folder entity. @@ -363,4 +517,144 @@ mod tests { "the hash must be deterministic for identical octets" ); } + + #[test] + fn message_key_precedence_prefers_emailid_over_everything_below_it() { + let remote = RemoteIdentity { + email_id: Some("M00000001"), + gm_msgid: Some("1234567890"), + message_id: Some("a@b.example"), + content_hash: Some("deadbeef"), + }; + let id = Email::derive_message_key("acct-1", remote, "INBOX", "42", "5"); + assert_eq!(id.source, IdentitySource::EmailId); + + // Same EMAILID in a different folder is the SAME message. + let elsewhere = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: Some("M00000001"), + gm_msgid: None, + message_id: None, + content_hash: None, + }, + "Archive", + "99", + "7", + ); + assert_eq!(id.key, elsewhere.key); + } + + #[test] + fn message_key_is_account_scoped_at_every_tier() { + let remote = || RemoteIdentity { + email_id: Some("M00000001"), + gm_msgid: None, + message_id: None, + content_hash: None, + }; + let a = Email::derive_message_key("acct-1", remote(), "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", remote(), "INBOX", "42", "5"); + assert_ne!( + a.key, b.key, + "the same EMAILID in two accounts must not collide" + ); + } + + #[test] + fn a_message_id_without_a_content_hash_never_reaches_the_content_tier() { + // The non-negotiable: a Message-ID is public and forgeable, so it must + // never key storage on its own. Missing content hash falls through to the + // folder-scoped surrogate rather than trusting the header alone. + let id = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: None, + }, + "INBOX", + "42", + "5", + ); + assert_eq!(id.source, IdentitySource::Surrogate); + + let surrogate_only = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: None, + content_hash: None, + }, + "INBOX", + "42", + "5", + ); + assert_eq!(id.key, surrogate_only.key); + } + + #[test] + fn the_same_message_id_with_different_content_is_two_messages() { + let mk = |hash: &str| { + Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: Some(hash), + }, + "INBOX", + "42", + "5", + ) + }; + let list_copy = mk("1111"); + let direct_copy = mk("2222"); + assert_eq!(list_copy.source, IdentitySource::MessageIdContent); + assert_ne!(list_copy.key, direct_copy.key); + } + + #[test] + fn tiers_are_domain_separated_so_a_shared_value_cannot_collide_across_them() { + let as_emailid = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: Some("X"), + gm_msgid: None, + message_id: None, + content_hash: None, + }, + "INBOX", + "42", + "5", + ); + let as_gmsgid = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: Some("X"), + message_id: None, + content_hash: None, + }, + "INBOX", + "42", + "5", + ); + assert_ne!(as_emailid.key, as_gmsgid.key); + } + + #[test] + fn identity_source_round_trips_through_its_stored_string() { + for source in [ + IdentitySource::EmailId, + IdentitySource::GmailMsgId, + IdentitySource::MessageIdContent, + IdentitySource::Surrogate, + ] { + assert_eq!(IdentitySource::from_str(source.as_str()), Some(source)); + } + } } -- 2.54.0 From 2c9ed5a218158bcaa181981f4d7035845ebc1081 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 14:06:12 -0500 Subject: [PATCH 63/95] docs(core): explain dropped mbox folder header, broaden test export_mbox stopped emitting X-Nuncio-Folder when folder_id moved off Email onto Placement, but the reason lived only in an SDD report, not the code. Add a comment at the removal site so a future reader knows it's intentional and what restoring it would require (threading a Placement into export_mbox's signature). Also broaden message_key_is_account_scoped_at_every_tier, which claimed to cover every identity tier but only exercised EmailId; it now asserts account-scoping at all four tiers. --- crates/nuncio-core/src/export.rs | 5 +++- crates/nuncio-core/src/model.rs | 45 +++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/crates/nuncio-core/src/export.rs b/crates/nuncio-core/src/export.rs index 89acdde..79a71e4 100644 --- a/crates/nuncio-core/src/export.rs +++ b/crates/nuncio-core/src/export.rs @@ -87,7 +87,10 @@ impl ExportEngine { writer.write_all(from_line.as_bytes())?; total_bytes += from_line.len() as u64; - // Headers + // Headers. No X-Nuncio-Folder header: the folder is a property of + // a placement now, not of the message, so `Email` has none to + // name here. Restoring it would mean threading the relevant + // `Placement` into this function's signature. let headers = format!( "Message-ID: <{}>\r\nFrom: {}\r\nTo: {}\r\nSubject: {}\r\nDate: {}\r\nX-Nuncio-Account: {}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n", email.id, email.sender, email.recipient, email.subject, date_str, email.account_id diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index 666d105..2fbffb0 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -547,18 +547,57 @@ mod tests { #[test] fn message_key_is_account_scoped_at_every_tier() { - let remote = || RemoteIdentity { + let emailid = || RemoteIdentity { email_id: Some("M00000001"), gm_msgid: None, message_id: None, content_hash: None, }; - let a = Email::derive_message_key("acct-1", remote(), "INBOX", "42", "5"); - let b = Email::derive_message_key("acct-2", remote(), "INBOX", "42", "5"); + let a = Email::derive_message_key("acct-1", emailid(), "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", emailid(), "INBOX", "42", "5"); assert_ne!( a.key, b.key, "the same EMAILID in two accounts must not collide" ); + + let gmsgid = || RemoteIdentity { + email_id: None, + gm_msgid: Some("1234567890"), + message_id: None, + content_hash: None, + }; + let a = Email::derive_message_key("acct-1", gmsgid(), "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", gmsgid(), "INBOX", "42", "5"); + assert_ne!( + a.key, b.key, + "the same X-GM-MSGID in two accounts must not collide" + ); + + let msgid_content = || RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: Some("deadbeef"), + }; + let a = Email::derive_message_key("acct-1", msgid_content(), "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", msgid_content(), "INBOX", "42", "5"); + assert_ne!( + a.key, b.key, + "the same Message-ID and content hash in two accounts must not collide" + ); + + let surrogate = RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: None, + content_hash: None, + }; + let a = Email::derive_message_key("acct-1", surrogate, "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", surrogate, "INBOX", "42", "5"); + assert_ne!( + a.key, b.key, + "the same folder/uidvalidity/remote-id in two accounts must not collide" + ); } #[test] -- 2.54.0 From 09c3069d57c52582c5a0cbdeaeb0f1990befc14c Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 14:29:11 -0500 Subject: [PATCH 64/95] feat(store): add placements, fire-once claims, and bump identity schema Split the messages table into identity/content only (message_key primary key, identity_source column) and a new placements table holding one row per (account, folder, uidvalidity, uid) mailbox occupancy. uidvalidity is part of the placements primary key because UIDs restart at 1 after a bump and would otherwise collide with rows from the prior generation. Add filter_fired as a fire-once ledger for filter actions, keyed on message identity so a message arriving in a second folder cannot re-trigger a side-effecting rule that already ran against it. Add claim_filter_fire (atomic INSERT ... ON CONFLICT DO NOTHING), has_filter_fired, and filter_fire_count as public store methods so other crates can observe and use the ledger without reaching the connection pool. Bump IDENTITY_SCHEMA_VERSION to 2 so reconcile_schema_version rebuilds the cached store on upgrade, and extend reset_cached_message_store to clear placements and filter_fired alongside messages. Retire the now -obsolete ensure_messages_identity_columns additive migration and its test: the remote_id/uid_validity columns and the identity index it backfilled no longer exist on messages, and placements' own primary key now provides the uniqueness guarantee it used to build. Update backfill_message_fts and the messages_ad trigger for the message_key rename, keeping the trigger on messages (not placements) so dropping one placement of a still-present message never orphans its FTS row. --- crates/nuncio-store/src/db.rs | 334 ++++++++++++++++++++-------------- 1 file changed, 194 insertions(+), 140 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index f2f6bfd..e323432 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -522,7 +522,7 @@ impl DatabaseEngine { /// /// Bump this only for a change of that kind, and only together with the /// reset it implies (see [`DatabaseEngine::reconcile_schema_version`]). - pub const IDENTITY_SCHEMA_VERSION: i64 = 1; + pub const IDENTITY_SCHEMA_VERSION: i64 = 2; /// Reconcile the on-disk identity-schema generation with this binary's. /// @@ -582,11 +582,12 @@ impl DatabaseEngine { /// Drop everything derived from the server, keeping everything the user /// configured. /// - /// Cleared: messages and their FTS index, per-folder sync checkpoints, and - /// the filter execution ledger. The ledger goes because its hash chain is - /// keyed on message ids that will no longer resolve, and a chain pointing - /// at absent rows has no audit value -- reinitialising it is more honest - /// than carrying a permanently mixed-namespace log. + /// Cleared: messages and their FTS index, mailbox placements, the filter + /// fire-once ledger, per-folder sync checkpoints, and the filter execution + /// ledger. The execution ledger goes because its hash chain is keyed on + /// message ids that will no longer resolve, and a chain pointing at absent + /// rows has no audit value -- reinitialising it is more honest than + /// carrying a permanently mixed-namespace log. /// /// Kept: accounts, credentials (which live in the OS keyring regardless), /// filter rules, and the WORM audit records, whose triggers forbid deletion @@ -596,6 +597,8 @@ impl DatabaseEngine { for statement in [ "DELETE FROM messages", "DELETE FROM messages_fts", + "DELETE FROM placements", + "DELETE FROM filter_fired", "DELETE FROM folder_sync_state", "DELETE FROM filter_execution_logs", ] { @@ -608,21 +611,82 @@ impl DatabaseEngine { Ok(()) } + /// Atomically claim the right to run `rule_id`'s actions against + /// `message_key`, returning `true` only for the claim that won. + /// + /// The claim is the fire-once guard for side-effecting actions. It is keyed + /// on message identity rather than on a placement because the same message + /// legitimately arrives in several folders -- and, under the old + /// folder-scoped identity, each arrival looked like a different message and + /// fired the rule again. `INSERT ... ON CONFLICT DO NOTHING` makes winning + /// the claim a single atomic statement, so two passes racing over the same + /// message cannot both act. + pub async fn claim_filter_fire( + &self, + rule_id: &str, + message_key: &str, + ) -> Result<bool, DatabaseError> { + let result = sqlx::query( + "INSERT INTO filter_fired (rule_id, message_key, fired_at) VALUES (?, ?, ?) + ON CONFLICT (rule_id, message_key) DO NOTHING", + ) + .bind(rule_id) + .bind(message_key) + .bind(chrono::Utc::now().timestamp()) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(result.rows_affected() == 1) + } + + /// Whether `rule_id` has already acted on `message_key`. + /// + /// Public rather than test-only: the claim ledger answers a question + /// operators and the outbox both have a real reason to ask -- "did this rule + /// already act on this message" -- and callers in other crates cannot reach + /// the pool directly. + pub async fn has_filter_fired( + &self, + rule_id: &str, + message_key: &str, + ) -> Result<bool, DatabaseError> { + let found: Option<(i64,)> = + sqlx::query_as("SELECT 1 FROM filter_fired WHERE rule_id = ? AND message_key = ?") + .bind(rule_id) + .bind(message_key) + .fetch_optional(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(found.is_some()) + } + + /// How many distinct (rule, message) fires the ledger records. + /// + /// Counterpart to [`Self::has_filter_fired`] for callers that need to assert + /// on the ledger as a whole rather than one entry. + pub async fn filter_fire_count(&self) -> Result<i64, DatabaseError> { + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM filter_fired") + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; + Ok(count) + } + /// Execute initial database migrations creating core envelope tables. pub async fn migrate(&self) -> Result<(), DatabaseError> { sqlx::query( r#" + -- Identity and immutable content, one row per message. Where the + -- message sits lives in `placements`; nothing here is folder-scoped. CREATE TABLE IF NOT EXISTS messages ( - id TEXT PRIMARY KEY NOT NULL, + message_key TEXT PRIMARY KEY NOT NULL, account_id TEXT NOT NULL, - folder_id TEXT NOT NULL, - remote_id TEXT NOT NULL DEFAULT '', - uid_validity TEXT NOT NULL DEFAULT '', subject TEXT NOT NULL, sender TEXT NOT NULL, recipient TEXT NOT NULL, received_at INTEGER NOT NULL, - read_flag INTEGER NOT NULL DEFAULT 0, body_plain TEXT, body_html TEXT, -- Captured at ingest, not yet read. Both are nullable because @@ -630,7 +694,34 @@ impl DatabaseEngine { -- MUST (RFC 5322 3.6.4), and a headers-only or envelope-only -- fetch has no full octets to hash. message_id TEXT, - content_hash TEXT + content_hash TEXT, + identity_source TEXT NOT NULL DEFAULT 'surrogate' + ); + + -- One row per (mailbox, message) occupancy. `uidvalidity` is in the + -- primary key because UIDs restart at 1 after a bump: without it the + -- first message of the new generation would overwrite the row of + -- whichever old message shared its UID. + CREATE TABLE IF NOT EXISTS placements ( + account_id TEXT NOT NULL, + folder_id TEXT NOT NULL, + uidvalidity TEXT NOT NULL, + uid TEXT NOT NULL, + message_key TEXT NOT NULL, + read_flag INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (account_id, folder_id, uidvalidity, uid) + ); + + -- Fire-once ledger for filter actions. Keyed on the message + -- *identity*, not a placement, so a message arriving in a second + -- folder does not re-fire a rule that already acted on it -- which + -- matters most for FORWARD and CALL WEBHOOK, whose effects land on + -- third parties and cannot be undone by convergence. + CREATE TABLE IF NOT EXISTS filter_fired ( + rule_id TEXT NOT NULL, + message_key TEXT NOT NULL, + fired_at INTEGER NOT NULL, + PRIMARY KEY (rule_id, message_key) ); CREATE TABLE IF NOT EXISTS calendar_events ( @@ -737,6 +828,8 @@ impl DatabaseEngine { CREATE INDEX IF NOT EXISTS idx_filter_logs_rule ON filter_execution_logs(rule_id, matched_at DESC); CREATE INDEX IF NOT EXISTS idx_pending_mutations_status ON pending_remote_mutations(status, created_at ASC); CREATE INDEX IF NOT EXISTS idx_worm_audit_seq ON worm_audit_records(sequence ASC); + CREATE INDEX IF NOT EXISTS idx_placements_message ON placements(message_key); + CREATE INDEX IF NOT EXISTS idx_placements_folder ON placements(account_id, folder_id); -- Full-text search indexes are created eagerly at migration time (not lazily on -- first search) so no message or event saved before the first search call is ever @@ -759,7 +852,7 @@ impl DatabaseEngine { -- encrypted search index, or whole-database encryption) is future work and is NOT -- provided today. CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( - id UNINDEXED, + message_key UNINDEXED, subject, sender, body_plain, @@ -768,11 +861,18 @@ impl DatabaseEngine { -- Subject/sender are never encrypted in `messages`, so a delete-only trigger is -- sufficient here: it just keeps the FTS index free of orphaned rows. Insertion and - -- update of `messages_fts` content happens explicitly in `save_email`, never via an + -- update of `messages_fts` content happens explicitly in `upsert_message`, never via an -- AFTER INSERT/UPDATE trigger, because such a trigger would only ever see the -- ciphertext body column. + -- + -- This trigger MUST stay on `messages` and never move to `placements`. Dropping one + -- placement of a message that still sits in another folder must leave the body + -- searchable; only losing the last placement removes the `messages` row, and that is + -- what reaps the FTS row here. Firing on `placements` instead would delete the index + -- entry while the body remained -- and, inverted, leaving it off `messages` entirely + -- would strand plaintext bodies in `messages_fts` after the message itself was gone. CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN - DELETE FROM messages_fts WHERE id = old.id; + DELETE FROM messages_fts WHERE message_key = old.message_key; END; -- Calendar event summary/location are never encrypted at rest, so trigger-based @@ -872,7 +972,6 @@ impl DatabaseEngine { // the additive column migrations, so a rebuild lands on the current // shape rather than an intermediate one. self.reconcile_schema_version().await?; - self.ensure_messages_identity_columns().await?; self.ensure_messages_provenance_columns().await?; self.backfill_message_fts().await?; @@ -920,88 +1019,6 @@ impl DatabaseEngine { Ok(()) } - /// Additive, backfill-safe migration that adds the `remote_id` / - /// `uid_validity` message-identity columns to a pre-existing `messages` - /// table that predated the opaque surrogate id, and installs the - /// `UNIQUE(account_id, folder_id, uid_validity, remote_id)` index that is - /// the real correctness guard against a shared protocol id (e.g. an IMAP - /// UID reused across folders/accounts) overwriting an unrelated message. - /// - /// A fresh database already gets both columns from `CREATE TABLE IF NOT - /// EXISTS messages` above, so on a fresh database the `PRAGMA table_info` - /// check finds them present and the backfill matches no rows -- only the - /// index is created. For a pre-existing database file the columns are added - /// with an empty-string default and every legacy row is backfilled to - /// `remote_id = id` (the old primary key, which was globally unique) so the - /// UNIQUE index can be built without collisions. SQLite has no `ADD COLUMN - /// IF NOT EXISTS`, so column presence is checked explicitly first, making - /// this safe to run on every daemon startup. - /// - /// The whole step runs in a single transaction, and the backfill is - /// unconditional (idempotent `WHERE remote_id = ''`) rather than gated on - /// having just added the column. Both properties make a partial run - /// self-healing: a crash mid-migration rolls back atomically, and even a - /// committed intermediate state (column present but a legacy row still - /// carrying the empty default) is completed by the next startup's backfill - /// before the index is built -- so the daemon can never wedge on an index - /// collision it could have avoided. - async fn ensure_messages_identity_columns(&self) -> Result<(), DatabaseError> { - let existing_columns: Vec<String> = sqlx::query("PRAGMA table_info(messages)") - .fetch_all(&self.pool) - .await - .map_err(DatabaseError::Query)? - .iter() - .map(|row| row.get::<String, _>("name")) - .collect(); - - let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; - - if !existing_columns.iter().any(|c| c == "remote_id") { - sqlx::query("ALTER TABLE messages ADD COLUMN remote_id TEXT NOT NULL DEFAULT ''") - .execute(&mut *tx) - .await - .map_err(DatabaseError::Query)?; - tracing::info!( - table = "messages", - column = "remote_id", - "applying schema migration" - ); - } - if !existing_columns.iter().any(|c| c == "uid_validity") { - sqlx::query("ALTER TABLE messages ADD COLUMN uid_validity TEXT NOT NULL DEFAULT ''") - .execute(&mut *tx) - .await - .map_err(DatabaseError::Query)?; - tracing::info!( - table = "messages", - column = "uid_validity", - "applying schema migration" - ); - } - - // Seed any legacy row still carrying the empty-string default with a - // unique, non-empty remote id (the old primary key) so the UNIQUE - // identity index is satisfiable. Unconditional so a partial prior run - // is always completed before the index is (re)built; a no-op on a fresh - // or already-migrated database. - sqlx::query("UPDATE messages SET remote_id = id WHERE remote_id = ''") - .execute(&mut *tx) - .await - .map_err(DatabaseError::Query)?; - - sqlx::query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_identity \ - ON messages (account_id, folder_id, uid_validity, remote_id)", - ) - .execute(&mut *tx) - .await - .map_err(DatabaseError::Query)?; - - tx.commit().await.map_err(DatabaseError::Query)?; - - Ok(()) - } - /// Additive, backfill-safe migration that adds the `imap_tls_mode` / /// `smtp_tls_mode` columns to a pre-existing `accounts` table that /// predates persisted per-protocol TLS transport modes. @@ -1150,16 +1167,16 @@ impl DatabaseEngine { async fn backfill_message_fts(&self) -> Result<(), DatabaseError> { let rows: Vec<(String, String, String, Option<String>)> = sqlx::query_as( r#" - SELECT m.id, m.subject, m.sender, m.body_plain + SELECT m.message_key, m.subject, m.sender, m.body_plain FROM messages m - WHERE NOT EXISTS (SELECT 1 FROM messages_fts f WHERE f.id = m.id) + WHERE NOT EXISTS (SELECT 1 FROM messages_fts f WHERE f.message_key = m.message_key) "#, ) .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; - for (id, subject, sender, body_plain) in rows { + for (message_key, subject, sender, body_plain) in rows { let dec_plain = body_plain .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) .transpose() @@ -1167,9 +1184,9 @@ impl DatabaseEngine { .unwrap_or_default(); sqlx::query( - "INSERT INTO messages_fts (id, subject, sender, body_plain) VALUES (?, ?, ?, ?)", + "INSERT INTO messages_fts (message_key, subject, sender, body_plain) VALUES (?, ?, ?, ?)", ) - .bind(&id) + .bind(&message_key) .bind(&subject) .bind(&sender) .bind(&dec_plain) @@ -4611,57 +4628,94 @@ mod tests { } #[tokio::test] - async fn messages_identity_migration_completes_from_a_partial_run() { + async fn placements_primary_key_includes_uidvalidity() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - // Reproduce the partial state: drop the identity index, then insert two - // rows in the same folder that both still hold the empty default. With - // the index gone this is permitted, and the two would collide on - // (account, folder, uid_validity='', remote_id='') if it were rebuilt now. - sqlx::query("DROP INDEX IF EXISTS idx_messages_identity") - .execute(engine.pool()) - .await - .unwrap(); - for id in ["legacy-a", "legacy-b"] { + // Same folder, same UID, different UIDVALIDITY: two distinct placements. + // After a UIDVALIDITY bump the server restarts UIDs at 1, so without + // uidvalidity in the key the new mail would overwrite the old. + for validity in ["42", "43"] { sqlx::query( - "INSERT INTO messages \ - (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag) \ - VALUES (?, 'acct-1', 'INBOX', '', '', 'subj', 'a@nuncio.mx', 'b@nuncio.mx', 0, 0)", + "INSERT INTO placements (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES (?, ?, ?, ?, ?, 0)", ) - .bind(id) + .bind("acct-1") + .bind("INBOX") + .bind(validity) + .bind("1") + .bind(format!("key-{validity}")) .execute(engine.pool()) .await - .unwrap(); + .expect("both placements must insert"); } - // Re-running the migration must finish the backfill and build the index. - engine - .ensure_messages_identity_columns() + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM placements") + .fetch_one(engine.pool()) .await - .expect("a partial migration must complete on the next open"); + .unwrap(); + assert_eq!(count, 2); + } - // Both legacy rows survive with a unique, non-empty backfilled remote id. + #[tokio::test] + async fn a_filter_fired_claim_is_won_exactly_once() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + assert!(engine.claim_filter_fire("rule-1", "key-1").await.unwrap()); + assert!( + !engine.claim_filter_fire("rule-1", "key-1").await.unwrap(), + "a second claim on the same (rule, message) must lose" + ); + assert!( + engine.claim_filter_fire("rule-2", "key-1").await.unwrap(), + "a different rule still gets its own claim" + ); + } + + #[tokio::test] + async fn bumping_the_identity_schema_rebuilds_placements_and_claims_too() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("nuncio.db"); + let secrets = crate::vault::SecretManager::mock(); + + { + let engine = DatabaseEngine::connect_file(&db_path, &secrets) + .await + .unwrap(); + sqlx::query( + "INSERT INTO placements (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES ('a', 'INBOX', '1', '1', 'k', 0)", + ) + .execute(engine.pool()) + .await + .unwrap(); + engine.claim_filter_fire("rule-1", "k").await.unwrap(); + // Pretend this file was written by the previous identity generation. + sqlx::query("PRAGMA user_version = 1") + .execute(engine.pool()) + .await + .unwrap(); + engine.close().await; + } + + let engine = DatabaseEngine::connect_file(&db_path, &secrets) + .await + .unwrap(); + let (placements,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM placements") + .fetch_one(engine.pool()) + .await + .unwrap(); + let (claims,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM filter_fired") + .fetch_one(engine.pool()) + .await + .unwrap(); assert_eq!( - engine.get_message("legacy-a").await.unwrap().remote_id, - "legacy-a" + placements, 0, + "placements are derived from the server and must be rebuilt" ); assert_eq!( - engine.get_message("legacy-b").await.unwrap().remote_id, - "legacy-b" + claims, 0, + "fire-once claims name message keys that no longer resolve" ); - - // The unique identity index is now present, and re-running is idempotent. - let index: Option<(String,)> = sqlx::query_as( - "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_messages_identity'", - ) - .fetch_optional(engine.pool()) - .await - .unwrap(); - assert!(index.is_some(), "the unique identity index must be built"); - engine - .ensure_messages_identity_columns() - .await - .expect("re-running an already-complete migration is a no-op"); + engine.close().await; } #[tokio::test] -- 2.54.0 From da7b67ba01fc202e08d8a98113d6bba6b57bf21f Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 14:58:19 -0500 Subject: [PATCH 65/95] feat(filter): evaluate rules against a placement, not a message A message can occupy several folders at once, so FOLDER and ACCOUNT conditions have no single answer at the message level. PlacedEmail pairs a message with the one mailbox occupancy under evaluation; evaluate_condition, evaluate, evaluate_with_timeout, and preview now take a PlacedEmail and read Folder/Account off its Placement, while content fields still read the Email. Callers fan out over a message's placements and evaluate each independently. --- crates/nuncio-filter/src/engine.rs | 382 ++++++++++++++++++++------ crates/nuncio-filter/src/validator.rs | 4 + 2 files changed, 302 insertions(+), 84 deletions(-) diff --git a/crates/nuncio-filter/src/engine.rs b/crates/nuncio-filter/src/engine.rs index 569823c..192d2bd 100644 --- a/crates/nuncio-filter/src/engine.rs +++ b/crates/nuncio-filter/src/engine.rs @@ -6,13 +6,27 @@ use crate::ast::{ }; use arc_swap::ArcSwap; use hmac::{Hmac, Mac}; -use nuncio_core::model::Email; +use nuncio_core::model::{Email, Placement}; use regex::Regex; use sha2::Sha256; use std::sync::Arc; use std::time::{Duration, Instant}; use tracing::{debug, info, warn}; +/// A message together with the one mailbox occupancy being evaluated. +/// +/// `WHERE FOLDER = 'INBOX'` has no answer for a message that sits in three +/// folders at once, so evaluation is defined against a single placement and the +/// caller fans out over however many the message has. Borrowing both halves +/// keeps that fan-out free of clones. +#[derive(Debug, Clone, Copy)] +pub struct PlacedEmail<'a> { + /// Identity and content -- the same for every placement. + pub email: &'a Email, + /// The occupancy this evaluation is about. + pub placement: &'a Placement, +} + /// Pre-compiled single filter rule optimizing regex and predicate execution. #[derive(Clone)] pub struct CompiledFilter { @@ -55,31 +69,40 @@ impl CompiledFilter { } } - /// Evaluate email message against compiled rule condition tree. + /// Evaluate one placement against the compiled rule condition tree. /// /// This checks only the WHERE-clause conditions; it does NOT apply the /// rule's account scope. Callers must gate on /// [`FilterRule::matches_account`] first (as every `FilterEngine` entry /// point does), or a rule scoped to one account will fire on messages of /// another. - pub fn evaluate_condition(&self, email: &Email) -> bool { - Self::eval_node(&self.rule.conditions, email, &self.compiled_regexes) + pub fn evaluate_condition(&self, placed: PlacedEmail<'_>) -> bool { + Self::eval_node(&self.rule.conditions, placed, &self.compiled_regexes) } - fn eval_node(node: &ConditionNode, email: &Email, regexes: &[(String, Regex)]) -> bool { + fn eval_node( + node: &ConditionNode, + placed: PlacedEmail<'_>, + regexes: &[(String, Regex)], + ) -> bool { match node { - ConditionNode::Leaf(leaf) => Self::eval_leaf(leaf, email, regexes), + ConditionNode::Leaf(leaf) => Self::eval_leaf(leaf, placed, regexes), ConditionNode::And(children) => { - children.iter().all(|c| Self::eval_node(c, email, regexes)) + children.iter().all(|c| Self::eval_node(c, placed, regexes)) } ConditionNode::Or(children) => { - children.iter().any(|c| Self::eval_node(c, email, regexes)) + children.iter().any(|c| Self::eval_node(c, placed, regexes)) } - ConditionNode::Not(inner) => !Self::eval_node(inner, email, regexes), + ConditionNode::Not(inner) => !Self::eval_node(inner, placed, regexes), } } - fn eval_leaf(leaf: &ConditionLeaf, email: &Email, regexes: &[(String, Regex)]) -> bool { + fn eval_leaf( + leaf: &ConditionLeaf, + placed: PlacedEmail<'_>, + regexes: &[(String, Regex)], + ) -> bool { + let email = placed.email; match &leaf.field { FilterField::Subject => { Self::eval_string_op(&email.subject, &leaf.operator, &leaf.value, regexes) @@ -98,12 +121,18 @@ impl CompiledFilter { .unwrap_or(""); Self::eval_string_op(body, &leaf.operator, &leaf.value, regexes) } - FilterField::Folder => { - Self::eval_string_op(&email.folder_id, &leaf.operator, &leaf.value, regexes) - } - FilterField::Account => { - Self::eval_string_op(&email.account_id, &leaf.operator, &leaf.value, regexes) - } + FilterField::Folder => Self::eval_string_op( + &placed.placement.folder_id, + &leaf.operator, + &leaf.value, + regexes, + ), + FilterField::Account => Self::eval_string_op( + &placed.placement.account_id, + &leaf.operator, + &leaf.value, + regexes, + ), FilterField::HasAttachment => { let has = !email.attachments.is_empty(); if let FilterValue::Boolean(b) = leaf.value { @@ -277,18 +306,26 @@ impl FilterEngine { Ok(()) } - /// Evaluate email message returning matching rule actions. + /// Evaluate one placement of a message, returning the matching rules' actions. + /// + /// Defined per placement, not per message: `FOLDER` and `ACCOUNT` are + /// properties of where the message sits, and a message can sit in several + /// places at once. Callers holding a multi-placement message evaluate each + /// placement and are responsible for not acting twice on one message -- see + /// the fire-once claim in the store. /// /// A rule only ever fires for the account it was created against /// (`FilterRule::target_account`, set via `ON ACCOUNT` or `*` for all /// accounts) — condition matching alone is not account-scoped, so this /// check must happen before `evaluate_condition` runs. - pub fn evaluate(&self, email: &Email) -> Vec<(FilterRule, Vec<RuleAction>)> { + pub fn evaluate(&self, placed: PlacedEmail<'_>) -> Vec<(FilterRule, Vec<RuleAction>)> { let guard = self.cache.load(); let mut results = Vec::new(); for filter in &guard.filters { - if filter.rule.matches_account(&email.account_id) && filter.evaluate_condition(email) { + if filter.rule.matches_account(&placed.placement.account_id) + && filter.evaluate_condition(placed) + { debug!( rule_id = %filter.rule.id, rule_name = %filter.rule.name, @@ -315,21 +352,25 @@ impl FilterEngine { /// the remaining rules rather than aborting the whole batch. pub async fn evaluate_with_timeout( &self, - email: &Email, + placed: PlacedEmail<'_>, timeout_duration: Duration, ) -> Vec<(FilterRule, Vec<RuleAction>)> { let guard = self.cache.load(); let mut results = Vec::new(); for filter in &guard.filters { - if !filter.rule.matches_account(&email.account_id) { + if !filter.rule.matches_account(&placed.placement.account_id) { continue; } let filter_for_task = filter.clone(); - let email_for_task = email.clone(); + let email_for_task = placed.email.clone(); + let placement_for_task = placed.placement.clone(); let eval_task = tokio::task::spawn_blocking(move || { - filter_for_task.evaluate_condition(&email_for_task) + filter_for_task.evaluate_condition(PlacedEmail { + email: &email_for_task, + placement: &placement_for_task, + }) }); match tokio::time::timeout(timeout_duration, eval_task).await { @@ -370,7 +411,7 @@ impl FilterEngine { /// Rules whose account scope does not match the message are reported as /// `SKIPPED (account scope mismatch)` rather than omitted, so the trace /// distinguishes "condition did not match" from "rule does not apply here". - pub fn preview(&self, email: &Email) -> FilterPreviewResult { + pub fn preview(&self, placed: PlacedEmail<'_>) -> FilterPreviewResult { let start = Instant::now(); let guard = self.cache.load(); let mut matched_rule_id = None; @@ -380,7 +421,7 @@ impl FilterEngine { let mut matched = false; for filter in &guard.filters { - if !filter.rule.matches_account(&email.account_id) { + if !filter.rule.matches_account(&placed.placement.account_id) { traces.push(format!( "Rule '{}' (priority {}): SKIPPED (account scope mismatch)", filter.rule.name, filter.rule.priority @@ -388,7 +429,7 @@ impl FilterEngine { continue; } - let is_match = filter.evaluate_condition(email); + let is_match = filter.evaluate_condition(placed); traces.push(format!( "Rule '{}' (priority {}): {}", filter.rule.name, @@ -406,7 +447,7 @@ impl FilterEngine { let elapsed = start.elapsed().as_micros() as u64; FilterPreviewResult { - message_id: email.id.clone(), + message_id: placed.email.id.clone(), matched, matched_rule_id, matched_rule_name, @@ -480,24 +521,31 @@ mod tests { fn exit(&self, _span: &span::Id) {} } - fn test_email(account_id: &str, subject: &str, folder_id: &str) -> Email { - Email { + /// Build an `Email`/`Placement` pair that shares one account between the + /// message and the mailbox it is placed in, matching how a message and + /// its own placement always agree on account. + fn test_placed_email(account_id: &str, subject: &str, folder_id: &str) -> (Email, Placement) { + let email = Email { id: "msg-1".to_string(), account_id: account_id.to_string(), - folder_id: folder_id.to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: false, body_plain: Some("Hello".to_string()), body_html: None, attachments: Vec::new(), message_id: None, content_hash: None, - } + }; + let placement = Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: "1".to_string(), + read: false, + }; + (email, placement) } #[test] @@ -506,29 +554,19 @@ mod tests { let rule = crate::parser::NsqlParser::parse_rule("Urgent", 1, nsql).unwrap(); let engine = FilterEngine::new(vec![rule]).unwrap(); - let email = Email { - id: "msg-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: "Urgent Meeting".to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1700000000, - read: false, - body_plain: Some("Hello".to_string()), - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, - }; + let (email, placement) = test_placed_email("acct-1", "Urgent Meeting", "inbox"); - let results = engine.evaluate(&email); + let results = engine.evaluate(PlacedEmail { + email: &email, + placement: &placement, + }); assert_eq!(results.len(), 1); assert_eq!(results[0].1.len(), 2); - let preview = engine.preview(&email); + let preview = engine.preview(PlacedEmail { + email: &email, + placement: &placement, + }); assert!(preview.matched); assert_eq!(preview.matched_rule_name, Some("Urgent".to_string())); } @@ -548,14 +586,27 @@ mod tests { // Condition matches the subject, but the message belongs to a // different account than the rule was created for. - let email_b = test_email("acct-b", "Weekly Report", "inbox"); + let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox"); assert!( - engine.evaluate(&email_b).is_empty(), + engine + .evaluate(PlacedEmail { + email: &email_b, + placement: &placement_b + }) + .is_empty(), "rule scoped to acct-a must not match a message from acct-b" ); - let email_a = test_email("acct-a", "Weekly Report", "inbox"); - assert_eq!(engine.evaluate(&email_a).len(), 1); + let (email_a, placement_a) = test_placed_email("acct-a", "Weekly Report", "inbox"); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email_a, + placement: &placement_a + }) + .len(), + 1 + ); } #[tokio::test] @@ -564,15 +615,27 @@ mod tests { let rule = crate::parser::NsqlParser::parse_rule("Scoped To A", 1, nsql).unwrap(); let engine = FilterEngine::new(vec![rule]).unwrap(); - let email_b = test_email("acct-b", "Weekly Report", "inbox"); + let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox"); let results = engine - .evaluate_with_timeout(&email_b, Duration::from_secs(1)) + .evaluate_with_timeout( + PlacedEmail { + email: &email_b, + placement: &placement_b, + }, + Duration::from_secs(1), + ) .await; assert!(results.is_empty()); - let email_a = test_email("acct-a", "Weekly Report", "inbox"); + let (email_a, placement_a) = test_placed_email("acct-a", "Weekly Report", "inbox"); let results = engine - .evaluate_with_timeout(&email_a, Duration::from_secs(1)) + .evaluate_with_timeout( + PlacedEmail { + email: &email_a, + placement: &placement_a, + }, + Duration::from_secs(1), + ) .await; assert_eq!(results.len(), 1); } @@ -583,8 +646,11 @@ mod tests { let rule = crate::parser::NsqlParser::parse_rule("Scoped To A", 1, nsql).unwrap(); let engine = FilterEngine::new(vec![rule]).unwrap(); - let email_b = test_email("acct-b", "Weekly Report", "inbox"); - let preview = engine.preview(&email_b); + let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox"); + let preview = engine.preview(PlacedEmail { + email: &email_b, + placement: &placement_b, + }); assert!(!preview.matched); assert!(preview .condition_traces @@ -598,8 +664,16 @@ mod tests { let rule = crate::parser::NsqlParser::parse_rule("Global", 1, nsql).unwrap(); let engine = FilterEngine::new(vec![rule]).unwrap(); - let email = test_email("any-account", "Weekly Report", "inbox"); - assert_eq!(engine.evaluate(&email).len(), 1); + let (email, placement) = test_placed_email("any-account", "Weekly Report", "inbox"); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &placement + }) + .len(), + 1 + ); } #[test] @@ -609,20 +683,30 @@ mod tests { let engine = FilterEngine::new(vec![rule]).unwrap(); // Same folder_id, different account_id: a rule matching on the - // `account` condition field must key off email.account_id, never - // fall through to comparing folder_id. - let email_a = test_email("acct-a", "Anything", "shared-folder"); + // `account` condition field must key off the placement's account_id, + // never fall through to comparing folder_id. + let (email_a, placement_a) = test_placed_email("acct-a", "Anything", "shared-folder"); assert_eq!( - engine.evaluate(&email_a).len(), + engine + .evaluate(PlacedEmail { + email: &email_a, + placement: &placement_a + }) + .len(), 1, - "rule on account = 'acct-a' must match a message with account_id 'acct-a'" + "rule on account = 'acct-a' must match a placement with account_id 'acct-a'" ); - let email_b = test_email("acct-b", "Anything", "shared-folder"); + let (email_b, placement_b) = test_placed_email("acct-b", "Anything", "shared-folder"); assert!( - engine.evaluate(&email_b).is_empty(), - "rule on account = 'acct-a' must not match a message with account_id 'acct-b', \ - even when it shares the same folder_id as an acct-a message" + engine + .evaluate(PlacedEmail { + email: &email_b, + placement: &placement_b + }) + .is_empty(), + "rule on account = 'acct-a' must not match a placement with account_id 'acct-b', \ + even when it shares the same folder_id as an acct-a placement" ); } @@ -632,11 +716,24 @@ mod tests { let rule = crate::parser::NsqlParser::parse_rule("Not In Spam Or Trash", 1, nsql).unwrap(); let engine = FilterEngine::new(vec![rule]).unwrap(); - let inbox_email = test_email("acct-1", "Anything", "inbox"); - assert_eq!(engine.evaluate(&inbox_email).len(), 1); + let (inbox_email, inbox_placement) = test_placed_email("acct-1", "Anything", "inbox"); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &inbox_email, + placement: &inbox_placement + }) + .len(), + 1 + ); - let spam_email = test_email("acct-1", "Anything", "spam"); - assert!(engine.evaluate(&spam_email).is_empty()); + let (spam_email, spam_placement) = test_placed_email("acct-1", "Anything", "spam"); + assert!(engine + .evaluate(PlacedEmail { + email: &spam_email, + placement: &spam_placement + }) + .is_empty()); } #[test] @@ -661,8 +758,16 @@ mod tests { }; let engine = FilterEngine::new(vec![rule]).unwrap(); - let email = test_email("acct-1", "Anything", "inbox"); - assert_eq!(engine.evaluate(&email).len(), 1); + let (email, placement) = test_placed_email("acct-1", "Anything", "inbox"); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &placement + }) + .len(), + 1 + ); } #[test] @@ -689,8 +794,16 @@ mod tests { }; let engine = FilterEngine::new(vec![rule]).unwrap(); - let email = test_email("acct-1", "Anything", "inbox"); - assert_eq!(engine.evaluate(&email).len(), 1); + let (email, placement) = test_placed_email("acct-1", "Anything", "inbox"); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &placement + }) + .len(), + 1 + ); } #[tokio::test] @@ -725,11 +838,17 @@ mod tests { }; let engine = FilterEngine::new(vec![rule]).unwrap(); - let mut email = test_email("acct-1", "Anything", "inbox"); + let (mut email, placement) = test_placed_email("acct-1", "Anything", "inbox"); email.body_plain = Some("A".repeat(60_000_000)); let results = engine - .evaluate_with_timeout(&email, Duration::from_millis(1)) + .evaluate_with_timeout( + PlacedEmail { + email: &email, + placement: &placement, + }, + Duration::from_millis(1), + ) .await; assert!( @@ -746,4 +865,99 @@ mod tests { "expected a WARN log carrying the rule id for the timed-out evaluation, got: {captured:?}" ); } + + fn sample_email(key: &str) -> Email { + Email { + id: key.to_string(), + account_id: "acct-1".to_string(), + subject: "Anything".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1700000000, + body_plain: Some("Hello".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + } + } + + fn sample_placement(folder_id: &str, remote_id: &str) -> Placement { + Placement { + account_id: "acct-1".to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: remote_id.to_string(), + read: false, + } + } + + fn rule_where_folder_is(folder: &str) -> FilterRule { + let nsql = format!("WHERE folder = '{folder}' ACTION MARK READ"); + crate::parser::NsqlParser::parse_rule("Folder Filter", 1, &nsql).unwrap() + } + + fn rule_where_account_is(account: &str) -> FilterRule { + let nsql = format!("WHERE account = '{account}' ACTION MARK READ"); + crate::parser::NsqlParser::parse_rule("Account Filter", 1, &nsql).unwrap() + } + + #[test] + fn a_folder_condition_matches_only_the_placement_it_is_evaluated_against() { + let engine = FilterEngine::new(vec![rule_where_folder_is("INBOX")]).unwrap(); + let email = sample_email("key-1"); + let inbox = sample_placement("INBOX", "5"); + let archive = sample_placement("Archive", "9"); + + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &inbox + }) + .len(), + 1, + "the INBOX placement matches" + ); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &archive + }) + .len(), + 0, + "the same message's Archive placement does not" + ); + } + + #[test] + fn an_account_condition_reads_the_placement_account() { + let engine = FilterEngine::new(vec![rule_where_account_is("acct-1")]).unwrap(); + let email = sample_email("key-1"); + let mine = sample_placement("INBOX", "5"); + let theirs = Placement { + account_id: "acct-2".into(), + ..sample_placement("INBOX", "5") + }; + + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &mine + }) + .len(), + 1 + ); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &theirs + }) + .len(), + 0 + ); + } } diff --git a/crates/nuncio-filter/src/validator.rs b/crates/nuncio-filter/src/validator.rs index 37a058d..8351c78 100644 --- a/crates/nuncio-filter/src/validator.rs +++ b/crates/nuncio-filter/src/validator.rs @@ -92,6 +92,10 @@ impl NsqlValidator { "header field conditions are not yet supported: message headers are not indexed" .to_string(), )), + // `Folder` and `Account` are validated the same as the + // content fields here: only the value's shape (string or + // string list) matters at this pass. At evaluation time they + // read a placement's coordinates, not the message's content. FilterField::Subject | FilterField::From | FilterField::To -- 2.54.0 From 031961ebc16b7f7e4fbc72decacbad345a44fac9 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 15:05:14 -0500 Subject: [PATCH 66/95] feat(store): write message identity and placement separately A message's content is now written once, keyed on its folder-independent message key, while each mailbox it occupies gets its own placement row. The message row is first-write-wins so a later sighting claiming a known key cannot rewrite a body the user already trusts, and the plaintext FTS row is written only on the insert that actually created the message. The placement is upserted, because its read flag genuinely changes. --- crates/nuncio-store/src/db.rs | 314 +++++++++++++++++++++++++++++++--- 1 file changed, 289 insertions(+), 25 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index e323432..a75667a 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -241,6 +241,21 @@ pub struct WormChainReport { pub first_broken_seq: Option<u64>, } +/// What a [`DatabaseEngine::save_email_at`] call actually changed. +/// +/// The two flags answer different questions and a caller usually needs both: +/// `message_is_new` gates work that should happen once per message (indexing, +/// side-effecting filter actions), `placement_is_new` gates work that should +/// happen once per mailbox occupancy (per-folder counters, per-placement +/// evaluation). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SaveOutcome { + /// The message identity had not been stored before this call. + pub message_is_new: bool, + /// This mailbox occupancy had not been stored before this call. + pub placement_is_new: bool, +} + /// Serialize a [`nuncio_core::TlsMode`] to its stable on-disk `accounts` /// column form. Kept as a bare snake_case token (matching the column's /// `'implicit_tls'` default) rather than JSON, so a row inserted by the @@ -842,7 +857,7 @@ impl DatabaseEngine { -- ciphertext and a trigger-based mirror would index that ciphertext verbatim -- (defeating search entirely). Instead `messages_fts` is populated explicitly from -- the plaintext body in application code, at the moment of encryption in - -- `DatabaseEngine::save_email` (see there) and via `backfill_message_fts` below for + -- `DatabaseEngine::save_email_at` (see there) and via `backfill_message_fts` below for -- any pre-existing rows. This means the trigram index now contains -- plaintext-derived body text: the FTS index itself is NOT encrypted, so message -- body content is recoverable from `messages_fts` by anyone with filesystem access @@ -860,8 +875,8 @@ impl DatabaseEngine { ); -- Subject/sender are never encrypted in `messages`, so a delete-only trigger is - -- sufficient here: it just keeps the FTS index free of orphaned rows. Insertion and - -- update of `messages_fts` content happens explicitly in `upsert_message`, never via an + -- sufficient here: it just keeps the FTS index free of orphaned rows. Insertion of + -- `messages_fts` content happens explicitly in `save_email_at`, never via an -- AFTER INSERT/UPDATE trigger, because such a trigger would only ever see the -- ciphertext body column. -- @@ -1158,7 +1173,7 @@ impl DatabaseEngine { /// Backfill `messages_fts` for any `messages` row that does not yet have a matching FTS /// entry -- e.g. rows written before the FTS5 index existed, or written directly by a - /// process that bypassed `save_email`'s explicit index population. Run automatically as + /// process that bypassed `save_email_at`'s explicit index population. Run automatically as /// part of [`DatabaseEngine::migrate`] (idempotent: a fully-indexed database performs no /// work). The stored `body_plain` column holds AES-256-GCM ciphertext, so each candidate /// row is decrypted with this engine's storage key before being written into the @@ -1446,7 +1461,19 @@ impl DatabaseEngine { Ok(()) } - /// Save an [`nuncio_core::model::Email`] to SQLite (INSERT OR REPLACE). + /// Persist a message's identity and content, and record that it occupies + /// `placement`, in one transaction. + /// + /// The message row is **first-write-wins**: a second sighting of the same + /// key -- which is the normal case for a message that also exists in another + /// folder -- leaves the stored subject and body untouched. That is what + /// keeps a message whose identity rests on `Message-ID` + content hash from + /// being rewritten by a later fetch claiming the same key, and it is why the + /// FTS row is written only on the insert that actually created the message. + /// + /// The placement row is upserted rather than ignored, because its read flag + /// is genuinely mutable: `\Seen` changes in the mailbox and the store must + /// follow it. /// /// The message body is encrypted (AES-256-GCM) before being written to the `messages` /// table, but the *plaintext* body is also indexed into the standalone `messages_fts` @@ -1456,7 +1483,12 @@ impl DatabaseEngine { /// `CREATE VIRTUAL TABLE` statement in [`DatabaseEngine::migrate`] -- the FTS index itself /// is not encrypted, so this intentionally trades some body confidentiality for working /// search. - pub async fn save_email(&self, email: &nuncio_core::model::Email) -> Result<(), DatabaseError> { + pub async fn save_email_at( + &self, + email: &nuncio_core::model::Email, + source: nuncio_core::model::IdentitySource, + placement: &nuncio_core::model::Placement, + ) -> Result<SaveOutcome, DatabaseError> { // Encryption failures MUST surface before any SQL runs: a swallowed error here would // otherwise leave a row with an empty/placeholder body column, indistinguishable from // a genuinely empty body on read-back. @@ -1473,55 +1505,121 @@ impl DatabaseEngine { let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; - sqlx::query( + // Ask before writing. SQLite reports one row affected for both arms of + // an upsert, so after the fact there is no way to tell an inserted + // placement from an updated one -- and the caller needs that distinction + // to decide whether this is a first arrival worth filtering. + let (already_placed,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&placement.account_id) + .bind(&placement.folder_id) + .bind(&placement.uid_validity) + .bind(&placement.remote_id) + .fetch_one(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + let placement_is_new = already_placed == 0; + + let inserted = sqlx::query( r#" - INSERT OR REPLACE INTO messages - (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag, body_plain, body_html, message_id, content_hash) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO messages + (message_key, account_id, subject, sender, recipient, received_at, + body_plain, body_html, message_id, content_hash, identity_source) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (message_key) DO NOTHING "#, ) .bind(&email.id) .bind(&email.account_id) - .bind(&email.folder_id) - .bind(&email.remote_id) - .bind(&email.uid_validity) .bind(&email.subject) .bind(&email.sender) .bind(&email.recipient) .bind(email.received_at) - .bind(if email.read { 1i64 } else { 0i64 }) .bind(&enc_plain) .bind(&enc_html) .bind(&email.message_id) .bind(&email.content_hash) + .bind(source.as_str()) .execute(&mut *tx) .await .map_err(DatabaseError::Query)?; - // Re-indexing: delete any prior FTS row for this message id, then insert the fresh - // plaintext-derived row. FTS5 has no natural "INSERT OR REPLACE" semantics for a - // standalone (non-external-content) table, so this is done explicitly rather than via - // trigger. - sqlx::query("DELETE FROM messages_fts WHERE id = ?") + let message_is_new = inserted.rows_affected() == 1; + + // Index only on the insert that created the message. Re-indexing on + // every placement would either duplicate the row (FTS5 standalone tables + // have no upsert) or rewrite it with a body the first-write-wins rule + // just rejected. + if message_is_new { + sqlx::query( + "INSERT INTO messages_fts (message_key, subject, sender, body_plain) VALUES (?, ?, ?, ?)", + ) .bind(&email.id) + .bind(&email.subject) + .bind(&email.sender) + .bind(email.body_plain.as_deref().unwrap_or("")) .execute(&mut *tx) .await .map_err(DatabaseError::Query)?; + } sqlx::query( - "INSERT INTO messages_fts (id, subject, sender, body_plain) VALUES (?, ?, ?, ?)", + r#" + INSERT INTO placements + (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (account_id, folder_id, uidvalidity, uid) + DO UPDATE SET message_key = excluded.message_key, read_flag = excluded.read_flag + "#, ) + .bind(&placement.account_id) + .bind(&placement.folder_id) + .bind(&placement.uid_validity) + .bind(&placement.remote_id) .bind(&email.id) - .bind(&email.subject) - .bind(&email.sender) - .bind(email.body_plain.as_deref().unwrap_or("")) + .bind(if placement.read { 1i64 } else { 0i64 }) .execute(&mut *tx) .await .map_err(DatabaseError::Query)?; tx.commit().await.map_err(DatabaseError::Query)?; - Ok(()) + Ok(SaveOutcome { + message_is_new, + placement_is_new, + }) + } + + /// Every mailbox this message currently occupies. + pub async fn placements_of( + &self, + message_key: &str, + ) -> Result<Vec<nuncio_core::model::Placement>, DatabaseError> { + let rows: Vec<(String, String, String, String, i64)> = sqlx::query_as( + "SELECT account_id, folder_id, uidvalidity, uid, read_flag + FROM placements WHERE message_key = ? ORDER BY folder_id ASC", + ) + .bind(message_key) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map( + |(account_id, folder_id, uid_validity, remote_id, read_flag)| { + nuncio_core::model::Placement { + account_id, + folder_id, + uid_validity, + remote_id, + read: read_flag != 0, + } + }, + ) + .collect()) } /// Query synced email messages for a specific folder. @@ -1815,7 +1913,7 @@ impl DatabaseEngine { /// Save a [`nuncio_core::model::CalendarEvent`] to SQLite (INSERT OR REPLACE). /// - /// Unlike [`Self::save_email`], calendar event summary/location are never encrypted at + /// Unlike [`Self::save_email_at`], calendar event summary/location are never encrypted at /// rest (see the confidentiality note above the `events_fts` `CREATE VIRTUAL TABLE` /// statement in [`Self::migrate`]), so the `events_ai`/`events_ad`/`events_au` triggers /// created there keep `events_fts` in sync automatically -- there is no separate manual @@ -4718,6 +4816,172 @@ mod tests { engine.close().await; } + /// An `(Email, Placement)` pair for `account_id = "acct-1"`, + /// `uid_validity = "42"` -- the shape a sync hands the store once identity + /// and occupancy are separate. + fn sample_message_and_placement( + key: &str, + folder: &str, + uid: &str, + ) -> (nuncio_core::model::Email, nuncio_core::model::Placement) { + ( + nuncio_core::model::Email { + id: key.into(), + account_id: "acct-1".into(), + subject: "Subject".into(), + sender: "a@nuncio.mx".into(), + recipient: "b@nuncio.mx".into(), + received_at: 1_000, + body_plain: Some("body".into()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::Placement { + account_id: "acct-1".into(), + folder_id: folder.into(), + uid_validity: "42".into(), + remote_id: uid.into(), + read: false, + }, + ) + } + + #[tokio::test] + async fn a_second_placement_does_not_overwrite_the_stored_body() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let email = nuncio_core::model::Email { + id: "key-1".into(), + account_id: "acct-1".into(), + subject: "Original".into(), + sender: "a@nuncio.mx".into(), + recipient: "b@nuncio.mx".into(), + received_at: 1_000, + body_plain: Some("the trusted body".into()), + body_html: None, + attachments: Vec::new(), + message_id: Some("a@b.example".into()), + content_hash: Some("1111".into()), + }; + let inbox = nuncio_core::model::Placement { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + read: false, + }; + let outcome = engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::MessageIdContent, + &inbox, + ) + .await + .unwrap(); + assert!(outcome.message_is_new && outcome.placement_is_new); + + // The same key arriving from another folder, carrying a different body. + let impostor = nuncio_core::model::Email { + subject: "Replaced".into(), + body_plain: Some("attacker body".into()), + ..email.clone() + }; + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + let outcome = engine + .save_email_at( + &impostor, + nuncio_core::model::IdentitySource::MessageIdContent, + &archive, + ) + .await + .unwrap(); + assert!(!outcome.message_is_new, "identity already existed"); + assert!(outcome.placement_is_new, "but the Archive placement is new"); + + let stored = engine.get_message("key-1").await.unwrap(); + assert_eq!(stored.subject, "Original", "first write must win"); + assert_eq!(stored.body_plain.as_deref(), Some("the trusted body")); + } + + #[tokio::test] + async fn re_placing_the_same_message_leaves_exactly_one_fts_row() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let (rows,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!( + rows, 1, + "the FTS index holds one row per message, not per placement" + ); + } + + #[tokio::test] + async fn read_state_is_tracked_per_placement() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + read: true, + ..inbox.clone() + }; + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let placements = engine.placements_of("key-1").await.unwrap(); + let inbox_read = placements + .iter() + .find(|p| p.folder_id == "INBOX") + .unwrap() + .read; + let archive_read = placements + .iter() + .find(|p| p.folder_id == "Archive") + .unwrap() + .read; + assert!(!inbox_read); + assert!( + archive_read, + "IMAP \\Seen is per-mailbox, so the flags differ" + ); + } + #[tokio::test] async fn message_provenance_round_trips_through_every_read_path() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); -- 2.54.0 From 882d2f12a251500f9edba07fdb689b852876c280 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 15:12:00 -0500 Subject: [PATCH 67/95] feat(store): read messages through their placements Reads now separate what a message is from where it sits: get_message returns identity and content alone, list_messages joins through placements and hands back the placement it found the message via, and folders are derived by grouping placements per account -- folder ids are not globally unique, so grouping across accounts would report one merged INBOX holding two accounts' mail. Read state is updated per placement because IMAP \Seen is per-mailbox. An FTS hit now names a message key, not a placement, since the index holds one row per message. --- crates/nuncio-store/src/db.rs | 1432 ++++++++++++++++++----------- crates/nuncio-store/src/search.rs | 92 +- 2 files changed, 954 insertions(+), 570 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index a75667a..d0c0128 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -256,6 +256,64 @@ pub struct SaveOutcome { pub placement_is_new: bool, } +/// The four coordinates that address one mailbox occupancy. +/// +/// `Ord` is derived because the sync path sorts and dedups batches of these +/// before deleting; `Hash`/`Eq` because it is also the key of the +/// already-present set used to classify a fetched chunk. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct PlacementKey { + /// Account owning the mailbox. + pub account_id: String, + /// Mailbox folder identifier. + pub folder_id: String, + /// The UIDVALIDITY generation `remote_id` was captured under. + pub uid_validity: String, + /// Protocol-native id addressing the message in this mailbox. + pub remote_id: String, +} + +/// A message paired with the mailbox occupancy it was read through. +/// +/// The two travel together out of every folder-scoped read: the caller almost +/// always needs the per-mailbox facts (the read flag, the addressing UID) as +/// well as the message, and re-deriving them would cost a query per row. +pub type PlacedMessage = (nuncio_core::model::Email, nuncio_core::model::Placement); + +/// Column order of the identity-only `messages` projection shared by every +/// whole-store read path: key, account, subject, sender, recipient, +/// received_at, body_plain, body_html, message_id, content_hash. +type MessageRow = ( + String, + String, + String, + String, + String, + i64, + Option<String>, + Option<String>, + Option<String>, + Option<String>, +); + +/// [`MessageRow`] widened with the joined placement's `uidvalidity`, `uid` and +/// `read_flag`, in that order. +type PlacedMessageRow = ( + String, + String, + String, + String, + String, + i64, + Option<String>, + Option<String>, + Option<String>, + Option<String>, + String, + String, + i64, +); + /// Serialize a [`nuncio_core::TlsMode`] to its stable on-disk `accounts` /// column form. Kept as a bare snake_case token (matching the column's /// `'implicit_tls'` default) rather than JSON, so a row inserted by the @@ -482,11 +540,16 @@ impl DatabaseEngine { } /// Total unread messages across every folder, read live from the store. + /// + /// Counted over placements, not messages: `\Seen` is per-mailbox, so a + /// message sitting unread in two folders is genuinely two unread items to + /// clear. pub async fn count_unread_messages(&self) -> Result<u64, DatabaseError> { - let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages WHERE read_flag = 0") - .fetch_one(&self.pool) - .await - .map_err(DatabaseError::Query)?; + let (count,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM placements WHERE read_flag = 0") + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; Ok(count.max(0) as u64) } @@ -1622,241 +1685,198 @@ impl DatabaseEngine { .collect()) } - /// Query synced email messages for a specific folder. - #[allow(clippy::type_complexity)] + /// Messages occupying one folder of one account, newest first, paired with + /// the placement they were found through. + /// + /// The placement travels with the message because the caller almost always + /// needs the per-mailbox facts -- the read flag and the addressing UID -- + /// and re-deriving them would mean a second query per row. pub async fn list_messages( &self, + account_id: &str, folder_id: &str, limit: usize, - ) -> Result<Vec<nuncio_core::model::Email>, DatabaseError> { - let rows: Vec<( - String, - String, - String, - String, - String, - String, - i64, - i64, - Option<String>, - Option<String>, - String, - String, - Option<String>, - Option<String>, - )> = sqlx::query_as( + ) -> Result<Vec<PlacedMessage>, DatabaseError> { + let rows: Vec<PlacedMessageRow> = sqlx::query_as( r#" - SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash - FROM messages - WHERE folder_id = ? - ORDER BY received_at DESC + SELECT m.message_key, m.account_id, m.subject, m.sender, m.recipient, + m.received_at, m.body_plain, m.body_html, m.message_id, m.content_hash, + p.uidvalidity, p.uid, p.read_flag + FROM messages m + JOIN placements p ON p.message_key = m.message_key + WHERE p.account_id = ? AND p.folder_id = ? + ORDER BY m.received_at DESC LIMIT ? "#, ) + .bind(account_id) .bind(folder_id) .bind(limit as i64) .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; - rows.into_iter() - .map( - |( - id, - account_id, - folder_id, - subject, - sender, - recipient, - received_at, - read_flag, - body_plain, - body_html, - remote_id, - uid_validity, - message_id, - content_hash, - )| { - let dec_plain = body_plain - .map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest( - &self.storage_key, - &p, - ) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - let dec_html = body_html - .map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest( - &self.storage_key, - &h, - ) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - Ok(nuncio_core::model::Email { - id, - account_id, - folder_id, - remote_id, - uid_validity, - subject, - sender, - recipient, - received_at, - read: read_flag != 0, - body_plain: dec_plain, - body_html: dec_html, - attachments: Vec::new(), - message_id, - content_hash, - }) + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let dec_plain = row + .6 + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = row + .7 + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + + out.push(( + nuncio_core::model::Email { + id: row.0, + account_id: row.1, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, + body_plain: dec_plain, + body_html: dec_html, + attachments: Vec::new(), + message_id: row.8, + content_hash: row.9, }, - ) - .collect::<Result<Vec<_>, DatabaseError>>() + nuncio_core::model::Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: row.10, + remote_id: row.11, + read: row.12 != 0, + }, + )); + } + Ok(out) } - /// Keyset-paginated listing of a folder's messages, newest first - /// (`received_at DESC, id DESC`). `after` is the `(received_at, id)` of - /// the last message of the previous page; `None` starts from the newest. + /// Keyset-paginated listing of one folder's messages, newest first + /// (`received_at DESC, message_key DESC`), each paired with the placement + /// it was found through. `after` is the `(received_at, message_key)` of the + /// last message of the previous page; `None` starts from the newest. /// Fetches `page_size + 1` rows to detect a following page: returns at - /// most `page_size` messages plus the `(received_at, id)` cursor of the - /// last returned message when more remain (else `None`). - #[allow(clippy::type_complexity)] + /// most `page_size` pairs plus the cursor of the last returned message when + /// more remain (else `None`). + /// + /// The cursor is a *message* coordinate rather than a placement one because + /// the ordering is by arrival time, which is a property of the message; the + /// join is one-to-one within a single folder, so no message can appear twice + /// on a page. pub async fn list_messages_page( &self, + account_id: &str, folder_id: &str, after: Option<(i64, String)>, page_size: usize, - ) -> Result<(Vec<nuncio_core::model::Email>, Option<(i64, String)>), DatabaseError> { + ) -> Result<(Vec<PlacedMessage>, Option<(i64, String)>), DatabaseError> { let fetch = page_size.saturating_add(1); let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages WHERE folder_id = ", + "SELECT m.message_key, m.account_id, m.subject, m.sender, m.recipient, \ + m.received_at, m.body_plain, m.body_html, m.message_id, m.content_hash, \ + p.uidvalidity, p.uid, p.read_flag \ + FROM messages m JOIN placements p ON p.message_key = m.message_key \ + WHERE p.account_id = ", ); + builder.push_bind(account_id.to_string()); + builder.push(" AND p.folder_id = "); builder.push_bind(folder_id.to_string()); - if let Some((ts, id)) = &after { - builder.push(" AND (received_at < "); + if let Some((ts, key)) = &after { + builder.push(" AND (m.received_at < "); builder.push_bind(*ts); - builder.push(" OR (received_at = "); + builder.push(" OR (m.received_at = "); builder.push_bind(*ts); - builder.push(" AND id < "); - builder.push_bind(id.clone()); + builder.push(" AND m.message_key < "); + builder.push_bind(key.clone()); builder.push("))"); } - builder.push(" ORDER BY received_at DESC, id DESC LIMIT "); + builder.push(" ORDER BY m.received_at DESC, m.message_key DESC LIMIT "); builder.push_bind(fetch as i64); let rows = builder - .build_query_as::<( - String, - String, - String, - String, - String, - String, - i64, - i64, - Option<String>, - Option<String>, - String, - String, - Option<String>, - Option<String>, - )>() + .build_query_as::<PlacedMessageRow>() .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; let has_more = rows.len() > page_size; - let mut emails = rows - .into_iter() - .take(page_size) - .map(|r| { - let dec_plain = - r.8.map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - let dec_html = - r.9.map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - Ok(nuncio_core::model::Email { - id: r.0, - account_id: r.1, - folder_id: r.2, - remote_id: r.10, - uid_validity: r.11, - subject: r.3, - sender: r.4, - recipient: r.5, - received_at: r.6, - read: r.7 != 0, + let mut placed = Vec::with_capacity(rows.len().min(page_size)); + for row in rows.into_iter().take(page_size) { + let dec_plain = row + .6 + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = row + .7 + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + placed.push(( + nuncio_core::model::Email { + id: row.0, + account_id: row.1, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), - message_id: r.12, - content_hash: r.13, - }) - }) - .collect::<Result<Vec<_>, DatabaseError>>()?; + message_id: row.8, + content_hash: row.9, + }, + nuncio_core::model::Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: row.10, + remote_id: row.11, + read: row.12 != 0, + }, + )); + } let next = if has_more { - emails - .last() - .map(|e| (e.received_at, e.id.clone())) - .filter(|_| !emails.is_empty()) + placed.last().map(|(e, _)| (e.received_at, e.id.clone())) } else { None }; - emails.shrink_to_fit(); - Ok((emails, next)) + placed.shrink_to_fit(); + Ok((placed, next)) } - /// Retrieve a single message by ID. - #[allow(clippy::type_complexity)] + /// Retrieve a message's identity and content by its key. + /// + /// Returns the message alone. Where it sits, and whether it is read there, + /// are per-mailbox facts -- ask [`Self::placements_of`] for those. pub async fn get_message( &self, - message_id: &str, + message_key: &str, ) -> Result<nuncio_core::model::Email, DatabaseError> { - let row: ( - String, - String, - String, - String, - String, - String, - i64, - i64, - Option<String>, - Option<String>, - String, - String, - Option<String>, - Option<String>, - ) = sqlx::query_as( + let row: MessageRow = sqlx::query_as( r#" - SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash + SELECT message_key, account_id, subject, sender, recipient, received_at, + body_plain, body_html, message_id, content_hash FROM messages - WHERE id = ? + WHERE message_key = ? "#, ) - .bind(message_id) + .bind(message_key) .fetch_one(&self.pool) .await .map_err(DatabaseError::Query)?; let dec_plain = row - .8 + .6 .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) .transpose() .map_err(|e| DatabaseError::Decryption(e.to_string()))?; let dec_html = row - .9 + .7 .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) .transpose() .map_err(|e| DatabaseError::Decryption(e.to_string()))?; @@ -1864,41 +1884,123 @@ impl DatabaseEngine { Ok(nuncio_core::model::Email { id: row.0, account_id: row.1, - folder_id: row.2, - remote_id: row.10, - uid_validity: row.11, - subject: row.3, - sender: row.4, - recipient: row.5, - received_at: row.6, - read: row.7 != 0, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), - message_id: row.12, - content_hash: row.13, + message_id: row.8, + content_hash: row.9, }) } - /// Return the subset of `ids` already present in the `messages` table, as a single - /// `SELECT id FROM messages WHERE id IN (...)` query. + /// Every placement currently stored for one folder of one account. + /// + /// The local half of a UID-set diff. This returns placements rather than + /// message keys because what a server stops reporting is an occupancy, not a + /// message: the same mail may still exist in another folder and must survive + /// there. + pub async fn placements_in_folder( + &self, + account_id: &str, + folder_id: &str, + ) -> Result<Vec<PlacementKey>, DatabaseError> { + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT uidvalidity, uid FROM placements WHERE account_id = ? AND folder_id = ?", + ) + .bind(account_id) + .bind(folder_id) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map(|(uid_validity, remote_id)| PlacementKey { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity, + remote_id, + }) + .collect()) + } + + /// The subset of `keys` already stored, in one round trip. + /// + /// SQLite supports row-value `IN` lists, so the whole batch is one query + /// rather than one existence check per placement. An empty slice + /// short-circuits, since `IN ()` is not valid SQL. + pub async fn existing_placements( + &self, + keys: &[PlacementKey], + ) -> Result<std::collections::HashSet<PlacementKey>, DatabaseError> { + if keys.is_empty() { + return Ok(std::collections::HashSet::new()); + } + + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT account_id, folder_id, uidvalidity, uid FROM placements + WHERE (account_id, folder_id, uidvalidity, uid) IN (", + ); + for (index, key) in keys.iter().enumerate() { + if index > 0 { + builder.push(", "); + } + builder.push("("); + builder.push_bind(key.account_id.clone()); + builder.push(", "); + builder.push_bind(key.folder_id.clone()); + builder.push(", "); + builder.push_bind(key.uid_validity.clone()); + builder.push(", "); + builder.push_bind(key.remote_id.clone()); + builder.push(")"); + } + builder.push(")"); + + let rows: Vec<(String, String, String, String)> = builder + .build_query_as() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map( + |(account_id, folder_id, uid_validity, remote_id)| PlacementKey { + account_id, + folder_id, + uid_validity, + remote_id, + }, + ) + .collect()) + } + + /// Return the subset of `keys` whose message identity is already present in the + /// `messages` table, as a single `SELECT message_key FROM messages WHERE message_key + /// IN (...)` query. /// /// Lets a caller classify a whole fetched batch as new-vs-seen with one round trip instead - /// of one existence lookup per id. Passing an empty slice short-circuits to an empty set - /// without querying at all, since `IN ()` is not valid SQL. + /// of one existence lookup per key. Answers a strictly different question from + /// [`Self::existing_placements`]: a message already known from another folder is *not* new + /// here, but its arrival in this folder still is. Passing an empty slice short-circuits to + /// an empty set without querying at all, since `IN ()` is not valid SQL. pub async fn existing_message_ids( &self, - ids: &[String], + keys: &[String], ) -> Result<std::collections::HashSet<String>, DatabaseError> { - if ids.is_empty() { + if keys.is_empty() { return Ok(std::collections::HashSet::new()); } let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = - sqlx::QueryBuilder::new("SELECT id FROM messages WHERE id IN ("); + sqlx::QueryBuilder::new("SELECT message_key FROM messages WHERE message_key IN ("); let mut separated = builder.separated(", "); - for id in ids { - separated.push_bind(id); + for key in keys { + separated.push_bind(key); } builder.push(")"); @@ -1908,7 +2010,7 @@ impl DatabaseEngine { .await .map_err(DatabaseError::Query)?; - Ok(rows.into_iter().map(|(id,)| id).collect()) + Ok(rows.into_iter().map(|(key,)| key).collect()) } /// Save a [`nuncio_core::model::CalendarEvent`] to SQLite (INSERT OR REPLACE). @@ -2249,23 +2351,30 @@ impl DatabaseEngine { Ok(contact_from_row(&row)) } - /// Update a single message's read/unread flag in place. + /// Update one placement's read flag in place. /// - /// Returns `DatabaseError::Query(sqlx::Error::RowNotFound)` if no message - /// with `message_id` exists, so callers can distinguish "flag flipped" - /// from "message never existed" rather than silently succeeding on a - /// no-op update. - pub async fn set_message_read( + /// Scoped to a single mailbox occupancy because IMAP `\Seen` is per-mailbox: + /// a message read in `INBOX` is not thereby read in `Archive`. Returns + /// `DatabaseError::Query(sqlx::Error::RowNotFound)` when no such placement + /// exists, so callers can still distinguish "flag flipped" from "never + /// there" rather than silently succeeding on a no-op update. + pub async fn set_placement_read( &self, - message_id: &str, + key: &PlacementKey, read: bool, ) -> Result<(), DatabaseError> { - let result = sqlx::query("UPDATE messages SET read_flag = ? WHERE id = ?") - .bind(if read { 1i64 } else { 0i64 }) - .bind(message_id) - .execute(&self.pool) - .await - .map_err(DatabaseError::Query)?; + let result = sqlx::query( + "UPDATE placements SET read_flag = ? + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(if read { 1i64 } else { 0i64 }) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; if result.rows_affected() == 0 { return Err(DatabaseError::Query(sqlx::Error::RowNotFound)); @@ -2273,15 +2382,28 @@ impl DatabaseEngine { Ok(()) } - /// Query available folders with message counts. - pub async fn list_folders(&self) -> Result<Vec<nuncio_core::model::Folder>, DatabaseError> { + /// Query one account's folders with message counts. + /// + /// Folders are derived from the placements that reference them rather than + /// stored in their own table, so a folder exists exactly as long as it holds + /// mail. Scoping by account is required, not cosmetic: folder ids are not + /// globally unique -- every account has an `INBOX` -- and grouping across + /// accounts would report one merged folder holding both accounts' mail. + pub async fn list_folders( + &self, + account_id: &str, + ) -> Result<Vec<nuncio_core::model::Folder>, DatabaseError> { let rows: Vec<(String, i64, i64)> = sqlx::query_as( r#" - SELECT folder_id, COUNT(*) as total, SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread - FROM messages + SELECT folder_id, COUNT(*) as total, + SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread + FROM placements + WHERE account_id = ? GROUP BY folder_id + ORDER BY folder_id ASC "#, ) + .bind(account_id) .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; @@ -2297,21 +2419,24 @@ impl DatabaseEngine { .collect()) } - /// Keyset-paginated listing of folders with message counts, folder id - /// ascending. `after` is the id of the last folder of the previous page. - /// Fetches `page_size + 1` groups to detect a following page and returns - /// the id of the last returned folder as the cursor when more remain. + /// Keyset-paginated listing of one account's folders with message counts, + /// folder id ascending. `after` is the id of the last folder of the previous + /// page. Fetches `page_size + 1` groups to detect a following page and + /// returns the id of the last returned folder as the cursor when more + /// remain. Account-scoped for the same reason as [`Self::list_folders`]. pub async fn list_folders_page( &self, + account_id: &str, after: Option<String>, page_size: usize, ) -> Result<(Vec<nuncio_core::model::Folder>, Option<String>), DatabaseError> { let fetch = page_size.saturating_add(1); let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT folder_id, COUNT(*) as total, SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread FROM messages ", + "SELECT folder_id, COUNT(*) as total, SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread FROM placements WHERE account_id = ", ); + builder.push_bind(account_id.to_string()); if let Some(id) = &after { - builder.push("WHERE folder_id > "); + builder.push(" AND folder_id > "); builder.push_bind(id.clone()); } builder.push(" GROUP BY folder_id ORDER BY folder_id ASC LIMIT "); @@ -2723,165 +2848,114 @@ impl DatabaseEngine { Ok(true) } - /// Fetch a page of messages using Keyset Chunking (`WHERE id > ? ORDER BY id ASC LIMIT ?`). + /// Fetch a page of messages using Keyset Chunking + /// (`WHERE message_key > ? ORDER BY message_key ASC LIMIT ?`). + /// + /// Walks message identities, not placements: a chunked pass over the store + /// wants each message exactly once, however many folders it occupies. pub async fn get_message_chunk( &self, - last_id: &str, + last_key: &str, limit: usize, ) -> Result<Vec<nuncio_core::model::Email>, DatabaseError> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages " + "SELECT message_key, account_id, subject, sender, recipient, received_at, \ + body_plain, body_html, message_id, content_hash FROM messages ", ); - if !last_id.is_empty() { - builder.push("WHERE id > "); - builder.push_bind(last_id); + if !last_key.is_empty() { + builder.push("WHERE message_key > "); + builder.push_bind(last_key); } - builder.push(" ORDER BY id ASC LIMIT "); + builder.push(" ORDER BY message_key ASC LIMIT "); builder.push_bind(limit as i64); - let query = builder.build_query_as::<( - String, - String, - String, - String, - String, - String, - i64, - i64, - Option<String>, - Option<String>, - String, - String, - Option<String>, - Option<String>, - )>(); + let query = builder.build_query_as::<MessageRow>(); let rows = query .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; - rows.into_iter() - .map(|r| { - let dec_plain = - r.8.map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - let dec_html = - r.9.map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - Ok(nuncio_core::model::Email { - id: r.0, - account_id: r.1, - folder_id: r.2, - remote_id: r.10, - uid_validity: r.11, - subject: r.3, - sender: r.4, - recipient: r.5, - received_at: r.6, - read: r.7 != 0, - body_plain: dec_plain, - body_html: dec_html, - attachments: Vec::new(), - message_id: r.12, - content_hash: r.13, - }) - }) - .collect::<Result<Vec<_>, DatabaseError>>() + rows.into_iter().map(|r| self.email_from_row(r)).collect() } /// Query messages across the WHOLE store for export purposes, /// optionally narrowed to a single account or a /// single folder. Passing `None` for both returns every message in the - /// store. Ordered by `id` ascending, mirroring [`Self::get_message_chunk`]'s - /// deterministic keyset ordering. - #[allow(clippy::type_complexity)] + /// store. Ordered by `message_key` ascending, mirroring + /// [`Self::get_message_chunk`]'s deterministic keyset ordering. + /// + /// The folder filter is an `EXISTS` over `placements` rather than a join: + /// a message occupying two folders must still export as one message, and a + /// join would emit it once per matching placement. pub async fn list_messages_for_export( &self, account_id: Option<&str>, folder_id: Option<&str>, ) -> Result<Vec<nuncio_core::model::Email>, DatabaseError> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages" + "SELECT m.message_key, m.account_id, m.subject, m.sender, m.recipient, \ + m.received_at, m.body_plain, m.body_html, m.message_id, m.content_hash \ + FROM messages m", ); let mut has_filter = false; if let Some(account_id) = account_id { - builder.push(" WHERE account_id = "); + builder.push(" WHERE m.account_id = "); builder.push_bind(account_id); has_filter = true; } if let Some(folder_id) = folder_id { - builder.push(if has_filter { - " AND folder_id = " - } else { - " WHERE folder_id = " - }); + builder.push(if has_filter { " AND " } else { " WHERE " }); + builder.push("EXISTS (SELECT 1 FROM placements p WHERE p.message_key = m.message_key AND p.folder_id = "); builder.push_bind(folder_id); + builder.push(")"); } - builder.push(" ORDER BY id ASC"); + builder.push(" ORDER BY m.message_key ASC"); - let query = builder.build_query_as::<( - String, - String, - String, - String, - String, - String, - i64, - i64, - Option<String>, - Option<String>, - String, - String, - Option<String>, - Option<String>, - )>(); + let query = builder.build_query_as::<MessageRow>(); let rows = query .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; - rows.into_iter() - .map(|r| { - let dec_plain = - r.8.map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - let dec_html = - r.9.map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - Ok(nuncio_core::model::Email { - id: r.0, - account_id: r.1, - folder_id: r.2, - remote_id: r.10, - uid_validity: r.11, - subject: r.3, - sender: r.4, - recipient: r.5, - received_at: r.6, - read: r.7 != 0, - body_plain: dec_plain, - body_html: dec_html, - attachments: Vec::new(), - message_id: r.12, - content_hash: r.13, - }) - }) - .collect::<Result<Vec<_>, DatabaseError>>() + rows.into_iter().map(|r| self.email_from_row(r)).collect() + } + + /// Rebuild an [`nuncio_core::model::Email`] from the identity-column row + /// shape shared by the whole-store read paths, decrypting the body columns + /// with this engine's storage key. + /// + /// Decryption failures propagate as [`DatabaseError::Decryption`] rather + /// than degrading to an empty body: a body that will not authenticate has + /// been tampered with or corrupted, and silently returning nothing would + /// hide exactly the event the AEAD exists to detect. + fn email_from_row(&self, row: MessageRow) -> Result<nuncio_core::model::Email, DatabaseError> { + let dec_plain = row + .6 + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = row + .7 + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + + Ok(nuncio_core::model::Email { + id: row.0, + account_id: row.1, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, + body_plain: dec_plain, + body_html: dec_html, + attachments: Vec::new(), + message_id: row.8, + content_hash: row.9, + }) } /// Append a new immutable WORM audit record to the log ledger, signed with the WORM @@ -3354,17 +3428,18 @@ mod tests { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); sqlx::query( - "INSERT INTO messages (id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain) - VALUES ('msg-1', 'acct-1', 'inbox', 'Hello', 'alice@nuncio.mx', 'bob@nuncio.mx', 1700000000, 0, 'Hi Bob')", + "INSERT INTO messages (message_key, account_id, subject, sender, recipient, received_at, body_plain) + VALUES ('msg-1', 'acct-1', 'Hello', 'alice@nuncio.mx', 'bob@nuncio.mx', 1700000000, 'Hi Bob')", ) .execute(engine.pool()) .await .unwrap(); - let subject: (String,) = sqlx::query_as("SELECT subject FROM messages WHERE id = 'msg-1'") - .fetch_one(engine.pool()) - .await - .unwrap(); + let subject: (String,) = + sqlx::query_as("SELECT subject FROM messages WHERE message_key = 'msg-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); assert_eq!(subject.0, "Hello"); } @@ -3376,23 +3451,30 @@ mod tests { let email = nuncio_core::model::Email { id: "msg-db-100".to_string(), account_id: "acct-1".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "100".to_string(), - uid_validity: "1".to_string(), subject: "Database Sync Test".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: false, body_plain: Some("Plaintext content".to_string()), body_html: Some("<p>HTML content</p>".to_string()), attachments: Vec::new(), message_id: None, content_hash: None, }; + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "1".to_string(), + remote_id: "100".to_string(), + read: false, + }; engine - .save_email(&email) + .save_email_at( + &email, + nuncio_core::model::IdentitySource::Surrogate, + &placement, + ) .await .expect("save email succeeds"); @@ -3401,16 +3483,23 @@ mod tests { .await .expect("get message succeeds"); assert_eq!(fetched.subject, "Database Sync Test"); - assert!(!fetched.read); + assert!( + !engine.placements_of("msg-db-100").await.unwrap()[0].read, + "read state now lives on the placement, not the message" + ); let msgs = engine - .list_messages("INBOX", 10) + .list_messages("acct-1", "INBOX", 10) .await .expect("list messages succeeds"); assert_eq!(msgs.len(), 1); - assert_eq!(msgs[0].id, "msg-db-100"); + assert_eq!(msgs[0].0.id, "msg-db-100"); + assert_eq!(msgs[0].1.remote_id, "100"); - let folders = engine.list_folders().await.expect("list folders succeeds"); + let folders = engine + .list_folders("acct-1") + .await + .expect("list folders succeeds"); assert_eq!(folders.len(), 1); assert_eq!(folders[0].id, "INBOX"); assert_eq!(folders[0].unread_messages, 1); @@ -3427,38 +3516,46 @@ mod tests { let email = nuncio_core::model::Email { id: "msg-db-corrupt".to_string(), account_id: "acct-1".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "corrupt".to_string(), - uid_validity: "1".to_string(), subject: "Corrupted At Rest".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: false, body_plain: Some("Sensitive plaintext body".to_string()), body_html: None, attachments: Vec::new(), message_id: None, content_hash: None, }; + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "1".to_string(), + remote_id: "corrupt".to_string(), + read: false, + }; engine - .save_email(&email) + .save_email_at( + &email, + nuncio_core::model::IdentitySource::Surrogate, + &placement, + ) .await .expect("save email succeeds"); // Flip a byte in the stored (still valid-hex) ciphertext to simulate tampering or // storage bit-rot. - let (stored,): (String,) = sqlx::query_as("SELECT body_plain FROM messages WHERE id = ?") - .bind("msg-db-corrupt") - .fetch_one(&engine.pool) - .await - .expect("fetch stored ciphertext"); + let (stored,): (String,) = + sqlx::query_as("SELECT body_plain FROM messages WHERE message_key = ?") + .bind("msg-db-corrupt") + .fetch_one(&engine.pool) + .await + .expect("fetch stored ciphertext"); let mut bytes = hex::decode(&stored).expect("stored ciphertext is valid hex"); let last_idx = bytes.len() - 1; bytes[last_idx] ^= 0xFF; let tampered = hex::encode(bytes); - sqlx::query("UPDATE messages SET body_plain = ? WHERE id = ?") + sqlx::query("UPDATE messages SET body_plain = ? WHERE message_key = ?") .bind(&tampered) .bind("msg-db-corrupt") .execute(&engine.pool) @@ -3472,10 +3569,19 @@ mod tests { assert!(matches!(err, DatabaseError::Decryption(_))); let err = engine - .list_messages("INBOX", 10) + .list_messages("acct-1", "INBOX", 10) .await .expect_err("list_messages must fail closed on corrupted ciphertext"); assert!(matches!(err, DatabaseError::Decryption(_))); + + // The export path reads the same ciphertext column and must fail closed too -- + // an export that silently dropped an unauthenticated body would write a file + // the user could not tell apart from a genuinely empty message. + let err = engine + .list_messages_for_export(None, None) + .await + .expect_err("export must fail closed on corrupted ciphertext"); + assert!(matches!(err, DatabaseError::Decryption(_))); } fn sample_calendar_event(id: &str, account_id: &str) -> nuncio_core::model::CalendarEvent { @@ -3722,57 +3828,69 @@ mod tests { )); } - /// `set_message_read` flips the persisted - /// `read_flag` in place (both directions), and reports - /// `sqlx::Error::RowNotFound` for a message ID that was never saved, - /// rather than silently succeeding on a no-op update. + /// `set_placement_read` flips the persisted `read_flag` in place (both + /// directions), and reports `sqlx::Error::RowNotFound` for a placement that + /// was never saved, rather than silently succeeding on a no-op update. #[tokio::test] - async fn set_message_read_flips_flag_and_reports_not_found_for_missing_message() { + async fn set_placement_read_flips_flag_and_reports_not_found_for_missing_placement() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); let email = nuncio_core::model::Email { id: "msg-mark-1".to_string(), account_id: "acct-1".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "mark-1".to_string(), - uid_validity: "1".to_string(), subject: "Mark Read Test".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: false, body_plain: None, body_html: None, attachments: Vec::new(), message_id: None, content_hash: None, }; - engine.save_email(&email).await.expect("save email"); + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "1".to_string(), + remote_id: "mark-1".to_string(), + read: false, + }; + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::Surrogate, + &placement, + ) + .await + .expect("save email"); + + let key = PlacementKey { + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "1".to_string(), + remote_id: "mark-1".to_string(), + }; engine - .set_message_read("msg-mark-1", true) + .set_placement_read(&key, true) .await .expect("mark read succeeds"); - let fetched = engine - .get_message("msg-mark-1") - .await - .expect("get message succeeds"); - assert!(fetched.read); + assert!(engine.placements_of("msg-mark-1").await.unwrap()[0].read); engine - .set_message_read("msg-mark-1", false) + .set_placement_read(&key, false) .await .expect("mark unread succeeds"); - let fetched = engine - .get_message("msg-mark-1") - .await - .expect("get message succeeds"); - assert!(!fetched.read); + assert!(!engine.placements_of("msg-mark-1").await.unwrap()[0].read); + let missing = PlacementKey { + remote_id: "never-placed".to_string(), + ..key.clone() + }; let err = engine - .set_message_read("msg-does-not-exist", true) + .set_placement_read(&missing, true) .await - .expect_err("marking an unknown message must fail"); + .expect_err("marking an unknown placement must fail"); assert!(matches!( err, DatabaseError::Query(sqlx::Error::RowNotFound) @@ -4427,21 +4545,31 @@ mod tests { let email = nuncio_core::model::Email { id: format!("msg-{i:03}"), account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: format!("{i}"), - uid_validity: "1".to_string(), subject: format!("Subject {i}"), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000 + i, - read: false, body_plain: Some("Hello".to_string()), body_html: None, attachments: Vec::new(), message_id: None, content_hash: None, }; - engine.save_email(&email).await.unwrap(); + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: format!("{i}"), + read: false, + }; + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::Surrogate, + &placement, + ) + .await + .unwrap(); } let chunk1 = engine.get_message_chunk("", 3).await.unwrap(); @@ -4539,42 +4667,62 @@ mod tests { ); } - /// Build a message exactly as a real sync would: an opaque surrogate id - /// hashed over its addressing coordinates, with the protocol-native id and - /// UIDVALIDITY scope carried in their own columns. + /// Build a message and its placement exactly as a real sync would when the + /// server offered no stable identity: an opaque surrogate key hashed over + /// the addressing coordinates, and a placement carrying those coordinates. fn synced_email( account_id: &str, folder_id: &str, uid_validity: &str, remote_id: &str, - ) -> nuncio_core::model::Email { - nuncio_core::model::Email { - id: nuncio_core::model::Email::surrogate_id( - account_id, - folder_id, - uid_validity, - remote_id, - ), - account_id: account_id.to_string(), - folder_id: folder_id.to_string(), - remote_id: remote_id.to_string(), - uid_validity: uid_validity.to_string(), - subject: format!("{folder_id}/{remote_id}"), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some("body".to_string()), - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, - } + ) -> (nuncio_core::model::Email, nuncio_core::model::Placement) { + ( + nuncio_core::model::Email { + id: nuncio_core::model::Email::surrogate_id( + account_id, + folder_id, + uid_validity, + remote_id, + ), + account_id: account_id.to_string(), + subject: format!("{folder_id}/{remote_id}"), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some("body".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: uid_validity.to_string(), + remote_id: remote_id.to_string(), + read: false, + }, + ) + } + + /// Persist a `synced_email` pair under the surrogate identity tier. + async fn save_synced( + engine: &DatabaseEngine, + pair: &(nuncio_core::model::Email, nuncio_core::model::Placement), + ) -> SaveOutcome { + engine + .save_email_at( + &pair.0, + nuncio_core::model::IdentitySource::Surrogate, + &pair.1, + ) + .await + .expect("save synced message") } /// The C1 regression: the same protocol UID reused across folders or /// accounts must persist as DISTINCT rows (never silently overwrite), and a - /// re-sync of the SAME message must upsert the one row it already owns. + /// re-sync of the SAME message must not duplicate the row it already owns. #[tokio::test] async fn message_identity_prevents_cross_folder_and_cross_account_collision() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); @@ -4585,41 +4733,77 @@ mod tests { let sent = synced_email("acct-1", "Sent", "7", "5"); let other_account = synced_email("acct-2", "INBOX", "99", "5"); - // Every surrogate id is distinct, so none can overwrite another. - assert_ne!(inbox.id, sent.id); - assert_ne!(inbox.id, other_account.id); - assert_ne!(sent.id, other_account.id); + // Every surrogate key is distinct, so none can overwrite another. + assert_ne!(inbox.0.id, sent.0.id); + assert_ne!(inbox.0.id, other_account.0.id); + assert_ne!(sent.0.id, other_account.0.id); - engine.save_email(&inbox).await.expect("save inbox"); - engine.save_email(&sent).await.expect("save sent"); - engine - .save_email(&other_account) - .await - .expect("save other account"); + save_synced(&engine, &inbox).await; + save_synced(&engine, &sent).await; + save_synced(&engine, &other_account).await; - // All three coexist as separate rows, addressed by their own folder. - assert_eq!(engine.list_messages("INBOX", 10).await.unwrap().len(), 2); - assert_eq!(engine.list_messages("Sent", 10).await.unwrap().len(), 1); - - // Re-syncing the SAME message recomputes the SAME id, so the upsert - // updates the one row rather than duplicating it. - let inbox_resynced = synced_email("acct-1", "INBOX", "42", "5"); - assert_eq!(inbox_resynced.id, inbox.id); - engine - .save_email(&inbox_resynced) - .await - .expect("re-sync upserts"); + // All three coexist as separate messages, each reachable only through + // its own account's folder -- the second account's INBOX is not folded + // into the first's. assert_eq!( - engine.list_messages("INBOX", 10).await.unwrap().len(), - 2, - "a re-sync must upsert, not duplicate" + engine + .list_messages("acct-1", "INBOX", 10) + .await + .unwrap() + .len(), + 1 + ); + assert_eq!( + engine + .list_messages("acct-1", "Sent", 10) + .await + .unwrap() + .len(), + 1 + ); + assert_eq!( + engine + .list_messages("acct-2", "INBOX", 10) + .await + .unwrap() + .len(), + 1 + ); + let (messages,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(messages, 3); + + // Re-syncing the SAME message recomputes the SAME key, so it lands on + // the row it already owns rather than duplicating it. + let inbox_resynced = synced_email("acct-1", "INBOX", "42", "5"); + assert_eq!(inbox_resynced.0.id, inbox.0.id); + let outcome = save_synced(&engine, &inbox_resynced).await; + assert!( + !outcome.message_is_new, + "a re-sync must not create a message" + ); + assert!( + !outcome.placement_is_new, + "nor a second placement in the same mailbox" + ); + assert_eq!( + engine + .list_messages("acct-1", "INBOX", 10) + .await + .unwrap() + .len(), + 1, + "a re-sync must not duplicate" ); - // The recovered row round-trips its protocol addressing columns. - let fetched = engine.get_message(&inbox.id).await.expect("get message"); - assert_eq!(fetched.remote_id, "5"); - assert_eq!(fetched.uid_validity, "42"); - assert_eq!(fetched.folder_id, "INBOX"); + // The placement round-trips the protocol addressing coordinates. + let placements = engine.placements_of(&inbox.0.id).await.expect("placements"); + assert_eq!(placements.len(), 1); + assert_eq!(placements[0].remote_id, "5"); + assert_eq!(placements[0].uid_validity, "42"); + assert_eq!(placements[0].folder_id, "INBOX"); } /// A crash between `ADD COLUMN remote_id` and its backfill can commit an @@ -4672,10 +4856,7 @@ mod tests { }), }; engine.save_account(&account).await.expect("save account"); - engine - .save_email(&synced_email("acct-reset", "INBOX", "42", "7")) - .await - .expect("save email"); + save_synced(&engine, &synced_email("acct-reset", "INBOX", "42", "7")).await; engine .save_folder_sync_state("acct-reset", "INBOX", "v1:uidnext:42:8") .await @@ -4982,32 +5163,249 @@ mod tests { ); } + #[tokio::test] + async fn folders_are_derived_from_placements_and_scoped_per_account() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + for (account, folder, uid) in [ + ("acct-1", "INBOX", "1"), + ("acct-1", "Archive", "2"), + ("acct-2", "INBOX", "1"), + ] { + let (mut email, mut placement) = + sample_message_and_placement(&format!("key-{account}-{folder}"), folder, uid); + email.account_id = account.into(); + placement.account_id = account.into(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &placement, + ) + .await + .unwrap(); + } + + let folders = engine.list_folders("acct-1").await.unwrap(); + let names: Vec<&str> = folders.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(names, vec!["Archive", "INBOX"]); + assert!( + !folders.iter().any(|f| f.total_messages > 1), + "acct-2's INBOX must not be folded into acct-1's" + ); + } + + #[tokio::test] + async fn one_message_in_two_folders_counts_once_in_each() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let folders = engine.list_folders("acct-1").await.unwrap(); + assert_eq!(folders.len(), 2); + for folder in &folders { + assert_eq!( + folder.total_messages, 1, + "{} should hold the message once", + folder.id + ); + } + let (messages,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(messages, 1, "but it is still one message"); + } + + #[tokio::test] + async fn existing_placements_reports_the_occupancy_not_the_message() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + + let in_inbox = PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + }; + let in_archive = PlacementKey { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..in_inbox.clone() + }; + + let found = engine + .existing_placements(&[in_inbox.clone(), in_archive.clone()]) + .await + .unwrap(); + assert!(found.contains(&in_inbox)); + assert!( + !found.contains(&in_archive), + "the same message arriving in a new folder is a new placement and must be seen as such" + ); + + // The message identity, by contrast, IS already known -- the two + // questions must not be conflated. + let known = engine + .existing_message_ids(&["key-1".to_string()]) + .await + .unwrap(); + assert!(known.contains("key-1")); + } + + #[tokio::test] + async fn existing_placements_with_empty_input_returns_empty_set_without_querying() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let found = engine.existing_placements(&[]).await.unwrap(); + + assert!(found.is_empty()); + } + + #[tokio::test] + async fn placements_in_folder_reports_only_that_mailbox() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let local = engine + .placements_in_folder("acct-1", "INBOX") + .await + .unwrap(); + assert_eq!( + local, + vec![PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + }], + "the local half of a UID-set diff is scoped to the folder being synced" + ); + } + + #[tokio::test] + async fn marking_read_in_one_folder_leaves_the_other_unread() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let inbox_key = PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + }; + engine.set_placement_read(&inbox_key, true).await.unwrap(); + + let placements = engine.placements_of("key-1").await.unwrap(); + assert!( + placements + .iter() + .find(|p| p.folder_id == "INBOX") + .unwrap() + .read + ); + assert!( + !placements + .iter() + .find(|p| p.folder_id == "Archive") + .unwrap() + .read + ); + } + #[tokio::test] async fn message_provenance_round_trips_through_every_read_path() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - let mut email = synced_email("acct-1", "INBOX", "42", "7"); - email.message_id = Some("abc123@mail.nuncio.mx".to_string()); - email.content_hash = Some(nuncio_core::model::Email::content_hash_of(b"raw octets")); - engine.save_email(&email).await.expect("save"); + let mut pair = synced_email("acct-1", "INBOX", "42", "7"); + pair.0.message_id = Some("abc123@mail.nuncio.mx".to_string()); + pair.0.content_hash = Some(nuncio_core::model::Email::content_hash_of(b"raw octets")); + save_synced(&engine, &pair).await; - let by_id = engine.get_message(&email.id).await.expect("get_message"); - assert_eq!(by_id.message_id, email.message_id); - assert_eq!(by_id.content_hash, email.content_hash); + let by_key = engine.get_message(&pair.0.id).await.expect("get_message"); + assert_eq!(by_key.message_id, pair.0.message_id); + assert_eq!(by_key.content_hash, pair.0.content_hash); - let listed = engine.list_messages("INBOX", 10).await.expect("list"); + let listed = engine + .list_messages("acct-1", "INBOX", 10) + .await + .expect("list"); let found = listed .iter() - .find(|m| m.id == email.id) + .find(|(m, _)| m.id == pair.0.id) .expect("the saved message is listed"); - assert_eq!(found.message_id, email.message_id); - assert_eq!(found.content_hash, email.content_hash); + assert_eq!(found.0.message_id, pair.0.message_id); + assert_eq!(found.0.content_hash, pair.0.content_hash); + + let exported = engine + .list_messages_for_export(None, None) + .await + .expect("export"); + let found = exported + .iter() + .find(|m| m.id == pair.0.id) + .expect("the saved message is exportable"); + assert_eq!(found.message_id, pair.0.message_id); + assert_eq!(found.content_hash, pair.0.content_hash); // A message with no Message-ID stays None rather than becoming "". let plain = synced_email("acct-1", "INBOX", "42", "8"); - assert_eq!(plain.message_id, None); - engine.save_email(&plain).await.expect("save"); - let fetched = engine.get_message(&plain.id).await.expect("get_message"); + assert_eq!(plain.0.message_id, None); + save_synced(&engine, &plain).await; + let fetched = engine.get_message(&plain.0.id).await.expect("get_message"); assert_eq!(fetched.message_id, None); assert_eq!(fetched.content_hash, None); } @@ -5021,8 +5419,8 @@ mod tests { // a row written before capture existed reads back as "not captured". sqlx::query( "INSERT INTO messages \ - (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag) \ - VALUES ('legacy-1', 'acct-1', 'INBOX', '9', '42', 'subj', 'a@nuncio.mx', 'b@nuncio.mx', 0, 0)", + (message_key, account_id, subject, sender, recipient, received_at) \ + VALUES ('legacy-1', 'acct-1', 'subj', 'a@nuncio.mx', 'b@nuncio.mx', 0)", ) .execute(engine.pool()) .await @@ -5049,51 +5447,41 @@ mod tests { } } - fn export_test_email(id: &str, account_id: &str, folder_id: &str) -> nuncio_core::model::Email { - nuncio_core::model::Email { + /// Save one exportable message into `folder_id` of `account_id`. + async fn save_export_message( + engine: &DatabaseEngine, + id: &str, + account_id: &str, + folder_id: &str, + ) { + let email = nuncio_core::model::Email { id: id.to_string(), account_id: account_id.to_string(), - folder_id: folder_id.to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), subject: format!("Subject {id}"), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some(format!("Body {id}")), body_html: None, attachments: Vec::new(), message_id: None, content_hash: None, - } - } - - #[tokio::test] - async fn existing_message_ids_returns_only_the_ids_already_persisted() { - let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + }; + let placement = nuncio_core::model::Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + }; engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) + .save_email_at( + &email, + nuncio_core::model::IdentitySource::Surrogate, + &placement, + ) .await .unwrap(); - engine - .save_email(&export_test_email("msg-2", "acct-a", "inbox")) - .await - .unwrap(); - - let found = engine - .existing_message_ids(&[ - "msg-1".to_string(), - "msg-2".to_string(), - "msg-3-never-saved".to_string(), - ]) - .await - .unwrap(); - - assert_eq!(found.len(), 2); - assert!(found.contains("msg-1")); - assert!(found.contains("msg-2")); - assert!(!found.contains("msg-3-never-saved")); } #[tokio::test] @@ -5108,14 +5496,8 @@ mod tests { #[tokio::test] async fn list_messages_for_export_with_no_filters_returns_every_message() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) - .await - .unwrap(); - engine - .save_email(&export_test_email("msg-2", "acct-b", "archive")) - .await - .unwrap(); + save_export_message(&engine, "msg-1", "acct-a", "inbox").await; + save_export_message(&engine, "msg-2", "acct-b", "archive").await; let all = engine.list_messages_for_export(None, None).await.unwrap(); assert_eq!(all.len(), 2); @@ -5126,14 +5508,8 @@ mod tests { #[tokio::test] async fn list_messages_for_export_filters_by_account_id() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) - .await - .unwrap(); - engine - .save_email(&export_test_email("msg-2", "acct-b", "inbox")) - .await - .unwrap(); + save_export_message(&engine, "msg-1", "acct-a", "inbox").await; + save_export_message(&engine, "msg-2", "acct-b", "inbox").await; let scoped = engine .list_messages_for_export(Some("acct-a"), None) @@ -5146,14 +5522,8 @@ mod tests { #[tokio::test] async fn list_messages_for_export_filters_by_folder_id() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) - .await - .unwrap(); - engine - .save_email(&export_test_email("msg-2", "acct-a", "archive")) - .await - .unwrap(); + save_export_message(&engine, "msg-1", "acct-a", "inbox").await; + save_export_message(&engine, "msg-2", "acct-a", "archive").await; let scoped = engine .list_messages_for_export(None, Some("archive")) @@ -5166,18 +5536,9 @@ mod tests { #[tokio::test] async fn list_messages_for_export_filters_by_account_and_folder_together() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) - .await - .unwrap(); - engine - .save_email(&export_test_email("msg-2", "acct-a", "archive")) - .await - .unwrap(); - engine - .save_email(&export_test_email("msg-3", "acct-b", "inbox")) - .await - .unwrap(); + save_export_message(&engine, "msg-1", "acct-a", "inbox").await; + save_export_message(&engine, "msg-2", "acct-a", "archive").await; + save_export_message(&engine, "msg-3", "acct-b", "inbox").await; let scoped = engine .list_messages_for_export(Some("acct-a"), Some("inbox")) @@ -5194,10 +5555,7 @@ mod tests { #[tokio::test] async fn export_messages_to_file_fails_when_worm_audit_write_fails() { let (engine, dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) - .await - .unwrap(); + save_export_message(&engine, "msg-1", "acct-a", "inbox").await; let messages = engine.list_messages_for_export(None, None).await.unwrap(); engine.close().await; diff --git a/crates/nuncio-store/src/search.rs b/crates/nuncio-store/src/search.rs index 36c6ac6..50ebf5f 100644 --- a/crates/nuncio-store/src/search.rs +++ b/crates/nuncio-store/src/search.rs @@ -6,7 +6,10 @@ use serde::{Deserialize, Serialize}; /// Search hit result for full-text query matches. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SearchHit { - /// Entity ID matching the query. + /// Entity ID matching the query. For a message hit this is the + /// folder-independent message key: the index holds one row per message, so + /// a hit names a message and never one of its mailbox placements. Resolve + /// it through [`DatabaseEngine::get_message`]. pub id: String, /// Matching title or subject. pub title: String, @@ -50,7 +53,7 @@ impl<'a> SearchEngine<'a> { let rows: Vec<(String, String, String)> = sqlx::query_as( r#" - SELECT id, subject, snippet(messages_fts, 3, '<b>', '</b>', '...', 10) as snippet + SELECT message_key, subject, snippet(messages_fts, 3, '<b>', '</b>', '...', 10) as snippet FROM messages_fts WHERE messages_fts MATCH ? ORDER BY rank @@ -94,7 +97,7 @@ impl<'a> SearchEngine<'a> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( "SELECT id, title, snippet, rk FROM ( \ - SELECT id, subject AS title, \ + SELECT message_key AS id, subject AS title, \ snippet(messages_fts, 3, '<b>', '</b>', '...', 10) AS snippet, rank AS rk \ FROM messages_fts WHERE messages_fts MATCH ", ); @@ -255,14 +258,10 @@ mod tests { nuncio_core::model::Email { id: id.to_string(), account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some(body_plain.to_string()), body_html: None, attachments: Vec::new(), @@ -271,7 +270,27 @@ mod tests { } } - /// The crux test: proves body search works over the REAL `save_email` write path (no + fn sample_placement(uid: &str) -> nuncio_core::model::Placement { + nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: uid.to_string(), + read: false, + } + } + + async fn save(db: &DatabaseEngine, email: &nuncio_core::model::Email) { + db.save_email_at( + email, + nuncio_core::model::IdentitySource::Surrogate, + &sample_placement(&email.id), + ) + .await + .unwrap(); + } + + /// The crux test: proves body search works over the REAL `save_email_at` write path (no /// raw-SQL plaintext injection bypassing encryption), while the stored body column stays /// ciphertext. This is what the old `fts5_message_search_and_triggers` test failed to /// prove -- it inserted plaintext directly via raw SQL, which trivially "worked" but never @@ -290,7 +309,7 @@ mod tests { "Quarterly Financial Meeting", "Let's discuss the annual budget revenue forecast", ); - db.save_email(&email).await.unwrap(); + save(&db, &email).await; // Body search matches a plaintext body term -- proves the FTS index holds real, // decrypted-at-write-time plaintext trigrams, not the AES-256-GCM ciphertext that is @@ -307,39 +326,44 @@ mod tests { // The body column at rest must remain encrypted ciphertext -- never the plaintext // search term, and never equal to the original plaintext body. let (stored_body,): (String,) = - sqlx::query_as("SELECT body_plain FROM messages WHERE id = 'msg-1'") + sqlx::query_as("SELECT body_plain FROM messages WHERE message_key = 'msg-1'") .fetch_one(db.pool()) .await .unwrap(); assert_ne!(stored_body, email.body_plain.clone().unwrap()); assert!(!stored_body.to_lowercase().contains("revenue")); - // Re-saving via save_email (an update, not a fresh insert) replaces the FTS row rather - // than duplicating or leaving a stale entry: the old term no longer matches, the new - // one does. - let mut updated = email.clone(); - updated.body_plain = Some("Updated strategy review notes".to_string()); - db.save_email(&updated).await.unwrap(); + // Re-saving the same key with a different body must leave the index alone: the message + // row is first-write-wins, so a second write that cannot change the stored body must + // not change what search returns for it either. Anything else would have search + // disagree with the store -- either finding a body that was never persisted, or holding + // two rows for one message. + let mut rewritten = email.clone(); + rewritten.body_plain = Some("Updated strategy review notes".to_string()); + save(&db, &rewritten).await; - assert!(search.search_messages("revenue").await.unwrap().is_empty()); - let new_hits = search.search_messages("strategy").await.unwrap(); - assert_eq!(new_hits.len(), 1); - assert_eq!(new_hits[0].id, "msg-1"); + let unchanged = search.search_messages("revenue").await.unwrap(); + assert_eq!(unchanged.len(), 1, "the stored body is still searchable"); + assert_eq!(unchanged[0].id, "msg-1"); + assert!( + search.search_messages("strategy").await.unwrap().is_empty(), + "a body the store rejected must never become searchable" + ); - // Delete message trigger still cleans up the FTS row (subject/sender/id only -- no + // Delete message trigger still cleans up the FTS row (subject/sender/key only -- no // ciphertext involved). - sqlx::query("DELETE FROM messages WHERE id = 'msg-1'") + sqlx::query("DELETE FROM messages WHERE message_key = 'msg-1'") .execute(db.pool()) .await .unwrap(); - let deleted_hits = search.search_messages("strategy").await.unwrap(); + let deleted_hits = search.search_messages("revenue").await.unwrap(); assert!(deleted_hits.is_empty()); } /// Proves the backfill half of the fix: a message saved through the real write path, /// whose FTS entry is then lost (simulating either a row written before the FTS5 index - /// existed, or one written by a process that bypassed `save_email`'s explicit indexing), + /// existed, or one written by a process that bypassed `save_email_at`'s explicit indexing), /// is NOT searchable until a migration pass runs -- at which point `backfill_message_fts` /// decrypts the ciphertext body column with this engine's real storage key and repopulates /// the plaintext-derived trigram index, with no search call ever required to trigger it. @@ -351,11 +375,11 @@ mod tests { "Legacy Roadmap Notes", "Confidential quarterly roadmap details", ); - db.save_email(&email).await.unwrap(); + save(&db, &email).await; // Simulate pre-existing data whose FTS entry is missing (e.g. written before the FTS5 // index existed) by dropping its messages_fts row directly. - sqlx::query("DELETE FROM messages_fts WHERE id = 'msg-legacy-1'") + sqlx::query("DELETE FROM messages_fts WHERE message_key = 'msg-legacy-1'") .execute(db.pool()) .await .unwrap(); @@ -385,13 +409,15 @@ mod tests { async fn search_messages_page_returns_every_hit_once_no_gaps() { let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); for i in 0..5 { - db.save_email(&sample_email( - &format!("msg-{i}"), - "Budget Planning Meeting", - "annual budget revenue forecast discussion", - )) - .await - .unwrap(); + save( + &db, + &sample_email( + &format!("msg-{i}"), + "Budget Planning Meeting", + "annual budget revenue forecast discussion", + ), + ) + .await; } let search = SearchEngine::new(&db); -- 2.54.0 From d4815ef768c840a09292811233858a71a8862ac0 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 15:15:05 -0500 Subject: [PATCH 68/95] feat(store): reap a message only when its last placement goes A server that stops reporting a UID has said the message left that mailbox, not that it ceased to exist -- a move reports exactly this in the source folder. So a placement is dropped unconditionally and the message only when its last placement goes. That reap is load-bearing for confidentiality: messages_fts holds decrypted bodies and is cleared by the messages_ad trigger on messages, so deleting the message row is the only thing that clears the index. Skip it and a deleted message leaves its plaintext body readable to anyone with filesystem access; invert it and search breaks for mail that still exists. Both directions are pinned by tests. --- crates/nuncio-store/src/db.rs | 180 ++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index d0c0128..b1bb71c 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -314,6 +314,15 @@ type PlacedMessageRow = ( i64, ); +/// What a [`DatabaseEngine::delete_placements`] call removed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DeleteOutcome { + /// Mailbox occupancies removed. + pub placements_removed: u64, + /// Messages that lost their last placement and were removed with it. + pub messages_reaped: u64, +} + /// Serialize a [`nuncio_core::TlsMode`] to its stable on-disk `accounts` /// column form. Kept as a bare snake_case token (matching the column's /// `'implicit_tls'` default) rather than JSON, so a row inserted by the @@ -1655,6 +1664,96 @@ impl DatabaseEngine { }) } + /// Remove mailbox occupancies, reaping any message left with none. + /// + /// A server that stops reporting a UID has told us the message left *that + /// mailbox*, not that it ceased to exist -- a move reports exactly this in + /// the source folder while the message continues in the destination. So the + /// placement goes unconditionally and the message goes only when its last + /// placement does. + /// + /// The reap is what keeps the plaintext FTS index honest. `messages_fts` + /// holds decrypted bodies keyed on `message_key`, reaped by the + /// `messages_ad` trigger on `messages`. Deleting the message row is + /// therefore the only thing that clears the index: skip the reap and every + /// fully-deleted message leaves its body readable to anyone with filesystem + /// access to the database. + pub async fn delete_placements( + &self, + keys: &[PlacementKey], + ) -> Result<DeleteOutcome, DatabaseError> { + if keys.is_empty() { + return Ok(DeleteOutcome { + placements_removed: 0, + messages_reaped: 0, + }); + } + + let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + let mut placements_removed = 0u64; + let mut touched: Vec<String> = Vec::with_capacity(keys.len()); + + for key in keys { + // Remember which message each placement pointed at before removing + // it -- afterwards there is nothing left to join through. + let owner: Option<(String,)> = sqlx::query_as( + "SELECT message_key FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .fetch_optional(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + let Some((message_key,)) = owner else { + continue; + }; + + let result = sqlx::query( + "DELETE FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + placements_removed += result.rows_affected(); + touched.push(message_key); + } + + touched.sort_unstable(); + touched.dedup(); + + let mut messages_reaped = 0u64; + for message_key in touched { + // Reap only where nothing is left pointing at the message. The + // `messages_ad` trigger clears the FTS row as a consequence. + let result = sqlx::query( + "DELETE FROM messages WHERE message_key = ? + AND NOT EXISTS (SELECT 1 FROM placements WHERE message_key = ?)", + ) + .bind(&message_key) + .bind(&message_key) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + messages_reaped += result.rows_affected(); + } + + tx.commit().await.map_err(DatabaseError::Query)?; + Ok(DeleteOutcome { + placements_removed, + messages_reaped, + }) + } + /// Every mailbox this message currently occupies. pub async fn placements_of( &self, @@ -5163,6 +5262,87 @@ mod tests { ); } + #[tokio::test] + async fn dropping_one_placement_keeps_the_body_searchable_from_the_other() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let inbox_key = PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + }; + let outcome = engine.delete_placements(&[inbox_key]).await.unwrap(); + assert_eq!(outcome.placements_removed, 1); + assert_eq!( + outcome.messages_reaped, 0, + "the message still lives in Archive" + ); + + engine + .get_message("key-1") + .await + .expect("the message must survive"); + let (fts,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(fts, 1, "search must still find a message that still exists"); + } + + #[tokio::test] + async fn dropping_the_last_placement_reaps_the_message_and_its_plaintext_index() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + + let inbox_key = PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + }; + let outcome = engine.delete_placements(&[inbox_key]).await.unwrap(); + assert_eq!(outcome.messages_reaped, 1); + + engine + .get_message("key-1") + .await + .expect_err("the message is gone"); + let (fts,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!( + fts, 0, + "an orphaned FTS row would leave the plaintext body readable to anyone \ + with filesystem access after the message itself was deleted" + ); + } + #[tokio::test] async fn folders_are_derived_from_placements_and_scoped_per_account() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); -- 2.54.0 From eb506bad53200dd357d3c6329e214c019c875119 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 15:31:06 -0500 Subject: [PATCH 69/95] refactor(core): rename IdentitySource::from_str to from_token The old name tripped clippy::should_implement_trait, which fails the workspace lint gate and therefore CI. Implementing std::str::FromStr instead would force a Result and a new error type when the only failure is "not one of four known tokens" -- Option already says that. The round-trip test now also pins that an unrecognised token returns None. --- crates/nuncio-core/src/model.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index 2fbffb0..de3bd43 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -93,7 +93,11 @@ impl IdentitySource { /// Parse a token written by [`Self::as_str`]. Returns `None` for anything /// else so an unrecognised value fails loudly rather than defaulting to a /// tier that would overstate how trustworthy the key is. - pub fn from_str(s: &str) -> Option<Self> { + /// + /// Deliberately not `std::str::FromStr`: that trait forces a `Result` and + /// therefore an error type, when the only failure here is "not one of four + /// known tokens" and `None` already says exactly that. + pub fn from_token(s: &str) -> Option<Self> { match s { "emailid" => Some(Self::EmailId), "gmsgid" => Some(Self::GmailMsgId), @@ -693,7 +697,12 @@ mod tests { IdentitySource::MessageIdContent, IdentitySource::Surrogate, ] { - assert_eq!(IdentitySource::from_str(source.as_str()), Some(source)); + assert_eq!(IdentitySource::from_token(source.as_str()), Some(source)); } + assert_eq!( + IdentitySource::from_token("not_a_tier"), + None, + "an unrecognised token must fail loudly, never default to a tier" + ); } } -- 2.54.0 From adef7fc1b650c89892f5499f78179bcad5899daa Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 15:31:06 -0500 Subject: [PATCH 70/95] fix(store): harden placement queries, paging, and write locking Five defects found in review: - existing_placements never pinned that a never-saved key is absent, and its negative case was vacuous (no Archive row existed to over-report). Over-reporting makes genuinely new mail look already-seen and be skipped by the sync classifier. Both are now real assertions, backed by a count so a dropped IN filter fails. - save_email_at and delete_placements both read then write in one deferred transaction. In WAL mode that aborts with SQLITE_BUSY, which busy_timeout does not retry; twenty racing saves reproducibly failed. Both now use BEGIN IMMEDIATE, matching the WORM and ledger appenders, with a concurrency regression test. - list_messages_page claimed the folder join was one-to-one. It is not: the placements key permits one message in one folder twice, and the message-only cursor made the second row unreachable across a page boundary. The keyset now carries the full placement address. - list_messages_for_export matched a folder id without its account, reintroducing the cross-account collision this model removes. --- crates/nuncio-store/src/db.rs | 257 ++++++++++++++++-- .../2026-08-11-message-placement-split.md | 3 +- 2 files changed, 238 insertions(+), 22 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index b1bb71c..6bd8115 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -280,6 +280,14 @@ pub struct PlacementKey { /// well as the message, and re-deriving them would cost a query per row. pub type PlacedMessage = (nuncio_core::model::Email, nuncio_core::model::Placement); +/// Keyset cursor for [`DatabaseEngine::list_messages_page`]: the +/// `(received_at, message_key, uidvalidity, uid)` of the last row of a page. +/// +/// It addresses a placement, not a message, because a message can occupy one +/// folder more than once -- see that method for why a message-only cursor +/// silently drops rows. +pub type MessagePageCursor = (i64, String, String, String); + /// Column order of the identity-only `messages` projection shared by every /// whole-store read path: key, account, subject, sender, recipient, /// received_at, body_plain, body_html, message_id, content_hash. @@ -1575,7 +1583,17 @@ impl DatabaseEngine { .map(|h| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, h)) .transpose()?; - let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + // `BEGIN IMMEDIATE` takes the write lock before the first statement runs. This + // transaction reads (the placement existence check) and then writes what it read, and in + // WAL mode a deferred transaction that upgrades to a writer after another connection has + // committed aborts with SQLITE_BUSY_SNAPSHOT -- which `busy_timeout` does not retry, + // because there is no lock to wait for, only a stale snapshot. Acquiring the lock up + // front makes a concurrent save block instead of failing. + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(DatabaseError::Query)?; // Ask before writing. SQLite reports one row affected for both arms of // an upsert, so after the fact there is no way to tell an inserted @@ -1689,7 +1707,15 @@ impl DatabaseEngine { }); } - let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + // `BEGIN IMMEDIATE` for the same reason as in [`Self::save_email_at`]: this reads each + // placement's owning message key and then deletes based on what it read, and a deferred + // WAL transaction that upgrades to a writer mid-flight aborts with SQLITE_BUSY_SNAPSHOT + // rather than waiting out `busy_timeout`. + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(DatabaseError::Query)?; let mut placements_removed = 0u64; let mut touched: Vec<String> = Vec::with_capacity(keys.len()); @@ -1854,25 +1880,32 @@ impl DatabaseEngine { Ok(out) } - /// Keyset-paginated listing of one folder's messages, newest first - /// (`received_at DESC, message_key DESC`), each paired with the placement - /// it was found through. `after` is the `(received_at, message_key)` of the - /// last message of the previous page; `None` starts from the newest. - /// Fetches `page_size + 1` rows to detect a following page: returns at - /// most `page_size` pairs plus the cursor of the last returned message when - /// more remain (else `None`). + /// Keyset-paginated listing of one folder's placements, newest first + /// (`received_at DESC, message_key DESC, uidvalidity DESC, uid DESC`), each + /// paired with the message it holds. `after` is the cursor of the last row + /// of the previous page; `None` starts from the newest. Fetches + /// `page_size + 1` rows to detect a following page: returns at most + /// `page_size` pairs plus the cursor of the last returned row when more + /// remain (else `None`). /// - /// The cursor is a *message* coordinate rather than a placement one because - /// the ordering is by arrival time, which is a property of the message; the - /// join is one-to-one within a single folder, so no message can appear twice - /// on a page. + /// The cursor carries the full placement address, not just the message + /// coordinates, because the join is **not** one-to-one within a folder: the + /// placements primary key is `(account, folder, uidvalidity, uid)`, so one + /// message legitimately occupies a single mailbox twice -- two `COPY`s of + /// the same mail into one folder land under different UIDs. With only + /// `(received_at, message_key)` in the keyset the ordering is not total, and + /// two such rows straddling a page boundary make the second unreachable: + /// the next page resumes strictly after the message key and skips its twin. + /// `uidvalidity`/`uid` are compared as the text they are stored as, which is + /// an arbitrary but total order -- and matching the `ORDER BY` exactly is + /// all a keyset needs. pub async fn list_messages_page( &self, account_id: &str, folder_id: &str, - after: Option<(i64, String)>, + after: Option<MessagePageCursor>, page_size: usize, - ) -> Result<(Vec<PlacedMessage>, Option<(i64, String)>), DatabaseError> { + ) -> Result<(Vec<PlacedMessage>, Option<MessagePageCursor>), DatabaseError> { let fetch = page_size.saturating_add(1); let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( "SELECT m.message_key, m.account_id, m.subject, m.sender, m.recipient, \ @@ -1884,16 +1917,33 @@ impl DatabaseEngine { builder.push_bind(account_id.to_string()); builder.push(" AND p.folder_id = "); builder.push_bind(folder_id.to_string()); - if let Some((ts, key)) = &after { + if let Some((ts, key, uid_validity, remote_id)) = &after { builder.push(" AND (m.received_at < "); builder.push_bind(*ts); builder.push(" OR (m.received_at = "); builder.push_bind(*ts); builder.push(" AND m.message_key < "); builder.push_bind(key.clone()); + builder.push(") OR (m.received_at = "); + builder.push_bind(*ts); + builder.push(" AND m.message_key = "); + builder.push_bind(key.clone()); + builder.push(" AND p.uidvalidity < "); + builder.push_bind(uid_validity.clone()); + builder.push(") OR (m.received_at = "); + builder.push_bind(*ts); + builder.push(" AND m.message_key = "); + builder.push_bind(key.clone()); + builder.push(" AND p.uidvalidity = "); + builder.push_bind(uid_validity.clone()); + builder.push(" AND p.uid < "); + builder.push_bind(remote_id.clone()); builder.push("))"); } - builder.push(" ORDER BY m.received_at DESC, m.message_key DESC LIMIT "); + builder.push( + " ORDER BY m.received_at DESC, m.message_key DESC, \ + p.uidvalidity DESC, p.uid DESC LIMIT ", + ); builder.push_bind(fetch as i64); let rows = builder @@ -1940,7 +1990,14 @@ impl DatabaseEngine { } let next = if has_more { - placed.last().map(|(e, _)| (e.received_at, e.id.clone())) + placed.last().map(|(e, p)| { + ( + e.received_at, + e.id.clone(), + p.uid_validity.clone(), + p.remote_id.clone(), + ) + }) } else { None }; @@ -3006,7 +3063,15 @@ impl DatabaseEngine { } if let Some(folder_id) = folder_id { builder.push(if has_filter { " AND " } else { " WHERE " }); - builder.push("EXISTS (SELECT 1 FROM placements p WHERE p.message_key = m.message_key AND p.folder_id = "); + // Scoped to the message's own account, not just the folder name: folder ids are not + // globally unique, so an unscoped `p.folder_id` match would export another account's + // INBOX alongside this one -- the very collision this model exists to remove. + builder.push( + "EXISTS (SELECT 1 FROM placements p \ + WHERE p.message_key = m.message_key \ + AND p.account_id = m.account_id \ + AND p.folder_id = ", + ); builder.push_bind(folder_id); builder.push(")"); } @@ -5421,6 +5486,20 @@ mod tests { .await .unwrap(); + // A *different* message really does occupy Archive. Without this the + // "Archive is not present" assertion below would hold even if the + // `IN (...)` filter were dropped entirely, since the table would have + // no Archive row to over-report in the first place. + let (other, other_in_archive) = sample_message_and_placement("key-2", "Archive", "77"); + engine + .save_email_at( + &other, + nuncio_core::model::IdentitySource::EmailId, + &other_in_archive, + ) + .await + .unwrap(); + let in_inbox = PlacementKey { account_id: "acct-1".into(), folder_id: "INBOX".into(), @@ -5432,9 +5511,14 @@ mod tests { remote_id: "9".into(), ..in_inbox.clone() }; + let never_placed = PlacementKey { + folder_id: "Sent".into(), + remote_id: "404".into(), + ..in_inbox.clone() + }; let found = engine - .existing_placements(&[in_inbox.clone(), in_archive.clone()]) + .existing_placements(&[in_inbox.clone(), in_archive.clone(), never_placed.clone()]) .await .unwrap(); assert!(found.contains(&in_inbox)); @@ -5442,14 +5526,145 @@ mod tests { !found.contains(&in_archive), "the same message arriving in a new folder is a new placement and must be seen as such" ); + assert!( + !found.contains(&never_placed), + "over-reporting would make genuinely new mail look already-seen and be skipped" + ); + assert_eq!(found.len(), 1, "and nothing else may be reported either"); // The message identity, by contrast, IS already known -- the two // questions must not be conflated. let known = engine - .existing_message_ids(&["key-1".to_string()]) + .existing_message_ids(&["key-1".to_string(), "key-3-never-saved".to_string()]) .await .unwrap(); assert!(known.contains("key-1")); + assert!( + !known.contains("key-3-never-saved"), + "a key that was never persisted must never be reported as known" + ); + assert_eq!(known.len(), 1, "and the query must not echo its own input"); + } + + /// `save_email_at` reads (the placement existence check) and then writes what + /// it read, so it must serialize rather than error under concurrency: in WAL + /// mode a deferred transaction upgrading to a writer aborts with + /// SQLITE_BUSY_SNAPSHOT, which `busy_timeout` does not retry. Twenty racing + /// saves of the SAME message and placement must therefore all succeed, and + /// exactly one of them must claim each of the two "is new" flags -- if two + /// callers both saw a first arrival, a fire-once filter action would run + /// twice. + #[tokio::test] + async fn concurrent_saves_of_one_placement_serialize_and_elect_one_winner() { + const CONCURRENT_SAVES: usize = 20; + + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let engine = std::sync::Arc::new(engine); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + + let mut handles = Vec::with_capacity(CONCURRENT_SAVES); + for _ in 0..CONCURRENT_SAVES { + let engine = std::sync::Arc::clone(&engine); + let email = email.clone(); + let inbox = inbox.clone(); + handles.push(tokio::spawn(async move { + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + })); + } + + let mut new_messages = 0; + let mut new_placements = 0; + for handle in handles { + let outcome = handle.await.expect("save task must not panic").expect( + "a concurrent save must never fail outright -- it must serialize, not error", + ); + new_messages += usize::from(outcome.message_is_new); + new_placements += usize::from(outcome.placement_is_new); + } + assert_eq!(new_messages, 1, "exactly one caller created the message"); + assert_eq!( + new_placements, 1, + "exactly one caller created the placement" + ); + + let (messages,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages") + .fetch_one(engine.pool()) + .await + .unwrap(); + let (fts,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages_fts") + .fetch_one(engine.pool()) + .await + .unwrap(); + let (placements,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM placements") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!((messages, fts, placements), (1, 1, 1)); + } + + /// Paging a folder must yield every placement in it EXACTLY once, with no + /// dupes and no gaps -- including the case the keyset exists to survive: one + /// message occupying a single folder more than once (two `COPY`s of the same + /// mail land under different UIDs). Every row here also shares one + /// `received_at`, so the tiebreaker columns carry the whole ordering. + #[tokio::test] + async fn list_messages_page_returns_every_placement_once_no_gaps() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let mut expected: Vec<(String, String)> = Vec::new(); + for (key, uid) in [ + ("key-a", "1"), + ("key-a", "2"), + ("key-b", "3"), + ("key-b", "4"), + ("key-c", "5"), + ] { + let (email, placement) = sample_message_and_placement(key, "INBOX", uid); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &placement, + ) + .await + .unwrap(); + expected.push((key.to_string(), uid.to_string())); + } + + let mut seen: Vec<(String, String)> = Vec::new(); + let mut after: Option<MessagePageCursor> = None; + let mut pages = 0; + loop { + let (rows, next) = engine + .list_messages_page("acct-1", "INBOX", after.clone(), 2) + .await + .unwrap(); + pages += 1; + assert!(rows.len() <= 2, "page must not exceed page_size"); + seen.extend( + rows.iter() + .map(|(e, p)| (e.id.clone(), p.remote_id.clone())), + ); + match next { + Some(cursor) => after = Some(cursor), + None => break, + } + assert!(pages < 100, "pagination must terminate"); + } + + let mut deduped = seen.clone(); + deduped.sort(); + deduped.dedup(); + assert_eq!(deduped.len(), seen.len(), "no placement returned twice"); + + expected.sort(); + assert_eq!( + deduped, expected, + "every placement in the folder must be reachable across pages" + ); + assert!(pages >= 3, "5 rows at page_size 2 must span multiple pages"); } #[tokio::test] diff --git a/docs/superpowers/plans/2026-08-11-message-placement-split.md b/docs/superpowers/plans/2026-08-11-message-placement-split.md index b2ab37a..f859f0c 100644 --- a/docs/superpowers/plans/2026-08-11-message-placement-split.md +++ b/docs/superpowers/plans/2026-08-11-message-placement-split.md @@ -12,7 +12,8 @@ - **Branch:** `story/message-placement-split`, already created off `dev` with `story/message-id-capture` (#386) and `story/schema-version-and-token-scheme` (#387) merged in. Baseline gate is green on it (627+ tests). Do not rebase onto the #401–#403 chain. - **Gate.** Removing `Email`'s folder scalars breaks four crates until Task 9 lands, so the gate is staged: - - **Tasks 1–8:** `cargo fmt --all -- --check` **and** `cargo test -p <crate under change>` must pass. The workspace will not compile end-to-end during these tasks; that is expected and is not a reason to stop or to widen the task. + - **Tasks 1–8:** all three of `cargo fmt --all -- --check`, `cargo clippy -p <crate under change> --all-targets -- -D warnings`, and `cargo test -p <crate under change>` must pass. The workspace will not compile end-to-end during these tasks; that is expected and is not a reason to stop or to widen the task. + **Clippy is not optional at the per-crate stage.** `cargo check-all` is the workspace-wide clippy gate and cannot run mid-refactor, so a per-crate clippy run is the only thing standing between a lint failure and the end of the plan. Omitting it once already let `should_implement_trait` reach the branch undetected. - **Task 9 onward, and the PR head:** the full gate — `cargo fmt --all -- --check`, then `cargo check-all`, then `cargo test-all`. - Warnings are hard errors throughout; `unwrap_used` / `expect_used` / `panic` / `todo` are `deny` outside tests. Never silence a lint to get a commit through. - `cargo test-all` can exceed a 10-minute tool timeout on this workspace; allow ~540s. -- 2.54.0 From bc72571f7a98390182200606c2944f70b54d30d4 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Tue, 11 Aug 2026 15:43:08 -0500 Subject: [PATCH 71/95] feat(mail): derive message identity and emit placements The IMAP and JMAP backends no longer mint a folder-scoped surrogate id for every fetched message. They derive the account-scoped message key by the identity precedence and report it as a PlacedMessage: the Email (identity and content), the tier that produced the key, and the Placement the fetch found it in. Read state moves onto the placement, where IMAP \Seen already lives. IMAP does not request EMAILID (RFC 8474) or X-GM-MSGID yet, so identity rests on the Message-ID/content pair where a Message-ID was captured and on the folder-scoped surrogate otherwise; adding those FETCH items later lifts stored rows to a stronger tier on the next resync. A JMAP Email id is server-assigned and account-stable, the same guarantee EMAILID gives, so JMAP enters at the top tier instead. MimeParserAdapter::parse_mime drops its id and folder_id arguments: identity cannot be settled until the parsed Message-ID and content hash are in hand alongside the transport's coordinates, so the caller assigns it. --- crates/nuncio-mail/src/backend.rs | 30 +- crates/nuncio-mail/src/imap.rs | 394 ++++++++++++++---- crates/nuncio-mail/src/jmap.rs | 92 ++-- crates/nuncio-mail/src/lib.rs | 3 +- crates/nuncio-mail/src/mock.rs | 66 +-- crates/nuncio-mail/src/parser.rs | 45 +- crates/nuncio-mail/src/smtp.rs | 4 - .../nuncio-mail/tests/wiremock_jmap_test.rs | 37 +- 8 files changed, 493 insertions(+), 178 deletions(-) diff --git a/crates/nuncio-mail/src/backend.rs b/crates/nuncio-mail/src/backend.rs index cb999bd..ef3078e 100644 --- a/crates/nuncio-mail/src/backend.rs +++ b/crates/nuncio-mail/src/backend.rs @@ -1,10 +1,28 @@ //! Protocol-agnostic async mail backend trait definitions. use async_trait::async_trait; -use nuncio_core::model::{Attachment, Email, Folder}; +use nuncio_core::model::{Attachment, Email, Folder, IdentitySource, Placement}; use crate::parser::MailError; +/// One message as a backend surfaced it: identity, content, and the mailbox +/// occupancy it was found in. +/// +/// Backends emit this rather than a bare [`Email`] because a fetch observes two +/// separate facts at once: *which message this is* -- folder-independent, and +/// the same in every mailbox that holds a copy -- and *where this pass found +/// it*, which is per-mailbox and changes under a move. Collapsing them into one +/// record is what made a moved message read as a new one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlacedMessage { + /// Identity and content. `email.id` is the derived message key. + pub email: Email, + /// Which precedence tier produced `email.id`. + pub source: IdentitySource, + /// The occupancy this fetch found it in. + pub placement: Placement, +} + /// The remote change a [`RemoteMutationSpec`] applies to one addressed message. /// /// This is the closed vocabulary of genuinely-remote filter actions that the @@ -72,12 +90,18 @@ pub trait MailBackend: Send + Sync { async fn sync_folders(&self) -> Result<Vec<Folder>, MailError>; /// Synchronize message envelopes for a specific folder since a checkpoint state. - /// Returns the updated email list and the new server state checkpoint. + /// Returns the messages with the occupancy each was found in, and the new + /// server state checkpoint. + /// + /// Every message is reported as a [`PlacedMessage`] so the caller can tell + /// a message it already stores in another folder from one it has never + /// seen: the derived key is the same in both folders, only the placement + /// differs. async fn sync_messages( &self, folder_id: &str, since_state: Option<&str>, - ) -> Result<(Vec<Email>, String), MailError>; + ) -> Result<(Vec<PlacedMessage>, String), MailError>; /// Apply a remote mutation (move/copy/flag/delete) to a single message /// against the real server. MUST return `Ok(())` only when the server diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index 420723b..77a3fcc 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -1,7 +1,7 @@ //! IMAP4rev1 dual-socket connection engine and IDLE push stream manager. use async_trait::async_trait; -use nuncio_core::model::{Email, Folder}; +use nuncio_core::model::{Email, Folder, MessageIdentity, Placement, RemoteIdentity}; use nuncio_core::TlsMode; use std::fmt::Debug; use std::pin::Pin; @@ -52,7 +52,7 @@ const MAX_MESSAGES_PER_FETCH_BATCH: usize = 200; /// would ever accept is pointless to transfer only to reject. const MAX_MESSAGE_BODY_BYTES: u32 = 25 * 1024 * 1024; -use crate::backend::{MailBackend, RemoteMutationKind, RemoteMutationSpec}; +use crate::backend::{MailBackend, PlacedMessage, RemoteMutationKind, RemoteMutationSpec}; use crate::parser::{MailError, MimeParserAdapter}; /// Recover a raw IMAP UID from a [`RemoteMutationSpec::remote_id`] (a decimal @@ -703,7 +703,8 @@ impl ImapEngine { } /// Fetch and parse the messages in a folder over an active IMAP session, - /// returning the fetched messages and the new sync checkpoint. + /// returning each message with the occupancy it was found in, plus the new + /// sync checkpoint. /// /// `since_state` is the checkpoint returned by a previous sync of this /// folder, encoded as `"{uidvalidity}:{uidnext}"` (see [`CHECKPOINT_DELIM`]). @@ -719,7 +720,7 @@ impl ImapEngine { folder_id: &str, since_state: Option<&str>, session: &mut async_imap::Session<S>, - ) -> Result<(Vec<Email>, String), MailError> + ) -> Result<(Vec<PlacedMessage>, String), MailError> where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { @@ -758,7 +759,7 @@ impl ImapEngine { since_state: Option<&str>, session: &mut async_imap::Session<S>, batch_size: usize, - ) -> Result<(Vec<Email>, String), MailError> + ) -> Result<(Vec<PlacedMessage>, String), MailError> where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { @@ -910,38 +911,88 @@ impl ImapEngine { Ok((emails, new_checkpoint)) } - /// Build an [`Email`] from a full-body (or envelope-only) FETCH item. When - /// the item carries a `BODY[]` it is parsed as a full MIME message; when it - /// does not (a server that returned only the ENVELOPE), a minimal record is - /// synthesised from the envelope so the message is still surfaced honestly. + /// Build the identity and placement for one fetched message. + /// + /// `EMAILID` and `X-GM-MSGID` are only present when the server advertised + /// `OBJECTID` / `X-GM-EXT-1` and the FETCH asked for them; when neither is + /// available the pair of `Message-ID` and content hash carries the identity, + /// and only if both are missing does this fall back to the folder-scoped + /// surrogate -- which is the one tier that does not survive a move. + /// + /// The server-supplied hints arrive as one [`RemoteIdentity`] rather than as + /// four loose arguments: they are already a single value at every call site, + /// and spreading them out buys no clarity while pushing this function past + /// the argument-count lint. + fn place( + &self, + folder_id: &str, + uid_validity: &str, + remote_id: &str, + remote: RemoteIdentity<'_>, + read: bool, + ) -> (MessageIdentity, Placement) { + let identity = + Email::derive_message_key(&self.account_id, remote, folder_id, uid_validity, remote_id); + let placement = Placement { + account_id: self.account_id.clone(), + folder_id: folder_id.to_string(), + uid_validity: uid_validity.to_string(), + remote_id: remote_id.to_string(), + read, + }; + (identity, placement) + } + + /// Build a [`PlacedMessage`] from a full-body (or envelope-only) FETCH item. + /// When the item carries a `BODY[]` it is parsed as a full MIME message; + /// when it does not (a server that returned only the ENVELOPE), a minimal + /// record is synthesised from the envelope so the message is still surfaced + /// honestly. fn build_email_from_fetch( &self, folder_id: &str, server_uid_validity: Option<u32>, fetch_data: &async_imap::types::Fetch, - ) -> Result<Email, MailError> { + ) -> Result<PlacedMessage, MailError> { let uid_num = fetch_data.uid.unwrap_or(0); // The message is addressed on the wire by its UID within the folder's - // UIDVALIDITY scope; the persisted id is an opaque surrogate hashed over - // both plus the account and folder, so the same UID in another - // folder/account can never collide. + // UIDVALIDITY scope. That pair, with the folder and account, is the + // *placement*; the message's own identity is derived separately so that + // moving it between folders does not make it a different message. let remote_id = uid_num.to_string(); let uid_validity = server_uid_validity .map(|v| v.to_string()) .unwrap_or_default(); - let email_id = Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id); let is_read = fetch_data .flags() .any(|flag| matches!(flag, async_imap::types::Flag::Seen)); + // EMAILID (RFC 8474) and X-GM-MSGID are not requested by the current + // FETCH, so identity rests on the Message-ID/content pair where the + // message carried a Message-ID and on the folder-scoped surrogate + // otherwise. Adding those FETCH items lifts existing rows to a stronger + // tier on the next resync at no cost to this code path. if let Some(raw_bytes) = fetch_data.body() { - let mut email = - MimeParserAdapter::parse_mime(&email_id, &self.account_id, folder_id, raw_bytes)?; - email.remote_id = remote_id; - email.uid_validity = uid_validity; - email.read = is_read; - return Ok(email); + let mut email = MimeParserAdapter::parse_mime(&self.account_id, raw_bytes)?; + let (identity, placement) = self.place( + folder_id, + &uid_validity, + &remote_id, + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: email.message_id.as_deref(), + content_hash: email.content_hash.as_deref(), + }, + is_read, + ); + email.id = identity.key; + return Ok(PlacedMessage { + email, + source: identity.source, + placement, + }); } let subject = fetch_data @@ -973,26 +1024,43 @@ impl ImapEngine { .and_then(|env| env.message_id.as_ref()) .and_then(|raw| Email::normalize_message_id(&String::from_utf8_lossy(raw))); - Ok(Email { - id: email_id, - account_id: self.account_id.clone(), - folder_id: folder_id.to_string(), - remote_id, - uid_validity, - subject, - sender, - recipient, - received_at, - read: is_read, - body_plain: None, - body_html: None, - attachments: Vec::new(), - message_id, - content_hash: None, + // With no octets there is no content hash, and the `Message-ID` alone + // is never enough to key on, so this path lands on the surrogate tier. + // That is the honest answer: nothing was fetched that could prove two + // folders hold the same message. + let (identity, placement) = self.place( + folder_id, + &uid_validity, + &remote_id, + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: message_id.as_deref(), + content_hash: None, + }, + is_read, + ); + + Ok(PlacedMessage { + email: Email { + id: identity.key, + account_id: self.account_id.clone(), + subject, + sender, + recipient, + received_at, + body_plain: None, + body_html: None, + attachments: Vec::new(), + message_id, + content_hash: None, + }, + source: identity.source, + placement, }) } - /// Build an [`Email`] for a message whose body exceeded + /// Build a [`PlacedMessage`] for a message whose body exceeded /// [`MAX_MESSAGE_BODY_BYTES`]: its headers (`BODY[HEADER]`) are parsed for /// honest metadata (subject/sender/date) but the oversized body is left /// unfetched, so the record carries an EMPTY body rather than a fabricated @@ -1003,26 +1071,20 @@ impl ImapEngine { folder_id: &str, server_uid_validity: Option<u32>, fetch_data: &async_imap::types::Fetch, - ) -> Email { + ) -> PlacedMessage { let uid_num = fetch_data.uid.unwrap_or(0); let remote_id = uid_num.to_string(); let uid_validity = server_uid_validity .map(|v| v.to_string()) .unwrap_or_default(); - let email_id = Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id); let is_read = fetch_data .flags() .any(|flag| matches!(flag, async_imap::types::Flag::Seen)); if let Some(header_bytes) = fetch_data.header() { - if let Ok(mut email) = - MimeParserAdapter::parse_mime(&email_id, &self.account_id, folder_id, header_bytes) - { + if let Ok(mut email) = MimeParserAdapter::parse_mime(&self.account_id, header_bytes) { // A headers-only parse yields no body, but be explicit that the // oversized body was intentionally not fetched. - email.remote_id = remote_id; - email.uid_validity = uid_validity; - email.read = is_read; email.body_plain = None; email.body_html = None; email.attachments = Vec::new(); @@ -1030,9 +1092,29 @@ impl ImapEngine { // `content_hash` is defined over a message's FULL octets. // Keeping that value would publish a hash that silently means // something else, so drop it; the `Message-ID` it recovered - // from those headers is still correct and is kept. + // from those headers is still correct and is kept. Dropping it + // also costs this message the content tier -- a `Message-ID` + // never keys on its own -- so an oversized message identifies + // by its folder-scoped surrogate until the octets are fetched. email.content_hash = None; - return email; + let (identity, placement) = self.place( + folder_id, + &uid_validity, + &remote_id, + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: email.message_id.as_deref(), + content_hash: None, + }, + is_read, + ); + email.id = identity.key; + return PlacedMessage { + email, + source: identity.source, + placement, + }; } } @@ -1040,22 +1122,31 @@ impl ImapEngine { // `build_email_from_fetch` cannot error on this path (no body to parse), // but if it ever did, surface a minimal honest record rather than panic. self.build_email_from_fetch(folder_id, server_uid_validity, fetch_data) - .unwrap_or_else(|_| Email { - id: email_id, - account_id: self.account_id.clone(), - folder_id: folder_id.to_string(), - remote_id, - uid_validity, - subject: "No Subject".to_string(), - sender: "unknown@nuncio.mx".to_string(), - recipient: "me@nuncio.mx".to_string(), - received_at: 0, - read: is_read, - body_plain: None, - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, + .unwrap_or_else(|_| { + let (identity, placement) = self.place( + folder_id, + &uid_validity, + &remote_id, + RemoteIdentity::default(), + is_read, + ); + PlacedMessage { + email: Email { + id: identity.key, + account_id: self.account_id.clone(), + subject: "No Subject".to_string(), + sender: "unknown@nuncio.mx".to_string(), + recipient: "me@nuncio.mx".to_string(), + received_at: 0, + body_plain: None, + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + source: identity.source, + placement, + } }) } @@ -1067,7 +1158,7 @@ impl ImapEngine { &self, folder_id: &str, since_state: Option<&str>, - ) -> Result<(Vec<Email>, String), MailError> { + ) -> Result<(Vec<PlacedMessage>, String), MailError> { let (username, password) = match (&self.username, &self.password) { (Some(u), Some(p)) => (u.as_str(), p.as_str()), _ => { @@ -1479,7 +1570,7 @@ impl MailBackend for ImapEngine { &self, folder_id: &str, since_state: Option<&str>, - ) -> Result<(Vec<Email>, String), MailError> { + ) -> Result<(Vec<PlacedMessage>, String), MailError> { self.sync_folder_messages(folder_id, since_state).await } @@ -1491,6 +1582,7 @@ impl MailBackend for ImapEngine { #[cfg(test)] mod tests { use super::*; + use nuncio_core::model::IdentitySource; use tracing_test::traced_test; #[tokio::test] @@ -1814,7 +1906,7 @@ mod tests { uidnext: u32, since_state: Option<String>, batch_size: usize, - ) -> (Vec<Email>, String, Vec<String>) { + ) -> (Vec<PlacedMessage>, String, Vec<String>) { let (client_io, server_io) = tokio::io::duplex(65536); let fetch_cmds = Arc::new(std::sync::Mutex::new(Vec::<String>::new())); let server_cmds = fetch_cmds.clone(); @@ -1975,14 +2067,14 @@ mod tests { // body -- never a fabricated one. let oversized = emails .iter() - .find(|e| e.remote_id == "2") + .find(|e| e.placement.remote_id == "2") .expect("oversized message must be present"); assert!( - oversized.body_plain.is_none() && oversized.body_html.is_none(), + oversized.email.body_plain.is_none() && oversized.email.body_html.is_none(), "oversized message must carry no body, got: {oversized:?}" ); assert!( - oversized.subject.contains('2'), + oversized.email.subject.contains('2'), "oversized message must keep honest header metadata, got: {oversized:?}" ); @@ -1990,10 +2082,10 @@ mod tests { for uid in ["1", "3"] { let email = emails .iter() - .find(|e| e.remote_id == uid) + .find(|e| e.placement.remote_id == uid) .expect("normal message present"); assert!( - email.body_plain.is_some(), + email.email.body_plain.is_some(), "normal message {uid} must have a body, got: {email:?}" ); } @@ -2800,4 +2892,164 @@ mod tests { ); Ok(()) } + + /// Drive a folder sync over a scripted server that serves exactly ONE + /// message: `uid`, with `raw` as its full RFC822 octets and `seen` + /// controlling its `\Seen` flag. Holding the octets fixed while varying the + /// folder, UIDVALIDITY and UID is the shape of "the same message, found in + /// two mailboxes" -- which is what the identity/placement split exists for. + async fn run_single_message_sync( + folder_id: &str, + uidvalidity: u32, + uid: u32, + seen: bool, + raw: &str, + ) -> PlacedMessage { + let (client_io, server_io) = tokio::io::duplex(65536); + let raw = raw.to_string(); + let uidnext = uid.saturating_add(1); + let flags = if seen { "\\Seen" } else { "" }; + + let server = tokio::spawn(async move { + let mut io = server_io; + while let Some(line) = read_scripted_line(&mut io).await { + let tag = line.split_whitespace().next().unwrap_or("").to_string(); + let upper = line.to_ascii_uppercase(); + if upper.contains("UID FETCH") { + let size = raw.len(); + let resp = if upper.contains("BODY.PEEK[]") { + format!( + "* 1 FETCH (UID {uid} RFC822.SIZE {size} FLAGS ({flags}) \ + BODY[] {{{size}}}\r\n{raw})\r\n{tag} OK FETCH completed\r\n" + ) + } else { + // Enumeration pass: UID + size only, no body. + format!( + "* 1 FETCH (UID {uid} RFC822.SIZE {size})\r\n\ + {tag} OK FETCH completed\r\n" + ) + }; + let _ = io.write_all(resp.as_bytes()).await; + } else if upper.contains("LOGIN") { + let _ = io + .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) + .await; + } else if upper.contains("SELECT") { + let resp = format!( + "* FLAGS (\\Seen)\r\n* 1 EXISTS\r\n* 0 RECENT\r\n\ + * OK [UIDVALIDITY {uidvalidity}] ok\r\n* OK [UIDNEXT {uidnext}] ok\r\n\ + {tag} OK [READ-WRITE] SELECT done\r\n" + ); + let _ = io.write_all(resp.as_bytes()).await; + } else if upper.contains("LOGOUT") { + let _ = io + .write_all(format!("* BYE\r\n{tag} OK LOGOUT\r\n").as_bytes()) + .await; + break; + } + } + }); + + let client = async_imap::Client::new(client_io); + let mut session = client + .login("user", "pass") + .await + .map_err(|(e, _)| e) + .expect("scripted login succeeds"); + let engine = ImapEngine::new("acct-1", "example.test", 993); + let (placed, _checkpoint) = engine + .sync_folder_messages_batched(folder_id, None, &mut session, 8) + .await + .expect("scripted sync succeeds"); + drop(session); + let _ = server.await; + + placed + .into_iter() + .next() + .expect("the scripted message must be surfaced") + } + + #[tokio::test] + async fn the_imap_backend_gives_one_identity_to_a_message_found_in_two_folders() { + // Byte-identical octets under two different UIDs in two different + // mailboxes -- an IMAP COPY, or a Gmail label. Identity comes from the + // Message-ID/content pair, so the derived key is ONE value while the + // placements stay distinct. Before the split this was two messages. + let raw = "Message-ID: <a@b.example>\r\nSubject: Hi\r\n\ + From: alice@b.example\r\n\r\nbody\r\n"; + let inbox = run_single_message_sync("INBOX", 42, 5, true, raw).await; + let archive = run_single_message_sync("Archive", 77, 9, false, raw).await; + + assert_eq!(inbox.source, IdentitySource::MessageIdContent); + assert_eq!(archive.source, IdentitySource::MessageIdContent); + assert_eq!( + inbox.email.id, archive.email.id, + "the same octets in two folders must derive one message key" + ); + + assert_eq!(inbox.placement.account_id, "acct-1"); + assert_eq!(inbox.placement.folder_id, "INBOX"); + assert_eq!(inbox.placement.uid_validity, "42"); + assert_eq!(inbox.placement.remote_id, "5"); + assert_eq!(archive.placement.folder_id, "Archive"); + assert_eq!(archive.placement.uid_validity, "77"); + assert_eq!(archive.placement.remote_id, "9"); + + // IMAP `\Seen` is per-mailbox (RFC 3501 section 2.3.2), so read state + // belongs to the occupancy, not to the message. + assert!(inbox.placement.read); + assert!(!archive.placement.read); + } + + #[tokio::test] + async fn a_message_with_no_message_id_stays_on_the_folder_scoped_surrogate() { + // RFC 5322 section 3.6.4 makes Message-ID a SHOULD, so a message may + // carry none. Without it there is no content tier, and the backend must + // report two honest folder-scoped keys rather than claim a stability + // the server gave it no basis for. + let raw = "Subject: No id\r\nFrom: alice@b.example\r\n\r\nbody\r\n"; + let inbox = run_single_message_sync("INBOX", 42, 5, false, raw).await; + let archive = run_single_message_sync("Archive", 77, 9, false, raw).await; + + assert_eq!(inbox.source, IdentitySource::Surrogate); + assert_ne!( + inbox.email.id, archive.email.id, + "a surrogate is folder-scoped and must not be claimed to survive a move" + ); + } + + #[tokio::test] + async fn a_message_fetched_from_two_folders_keeps_one_identity() { + // Both folders return the same Message-ID and the same octets, so the + // content tier must produce one key from two different UIDs. + let raw = b"Message-ID: <a@b.example>\r\nSubject: Hi\r\n\r\nbody"; + let hash = Email::content_hash_of(raw); + let inbox = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: Some(&hash), + }, + "INBOX", + "42", + "5", + ); + let archive = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: Some(&hash), + }, + "Archive", + "77", + "9", + ); + assert_eq!(inbox.key, archive.key); + assert_eq!(inbox.source, IdentitySource::MessageIdContent); + } } diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index 187f257..2abaca0 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -1,19 +1,19 @@ //! JMAP (RFC 8620 / RFC 8621) protocol engine, session discovery, and differential update parser. use async_trait::async_trait; -use nuncio_core::model::{Email, Folder}; +use nuncio_core::model::{Email, Folder, Placement, RemoteIdentity}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use crate::backend::{MailBackend, RemoteMutationKind, RemoteMutationSpec}; +use crate::backend::{MailBackend, PlacedMessage, RemoteMutationKind, RemoteMutationSpec}; use crate::parser::MailError; /// Stand-in for the UIDVALIDITY scope on JMAP, which has no such concept: a -/// JMAP Email object id is already globally stable within its account (RFC -/// 8621), so a fixed sentinel keeps the surrogate-id computation uniform with -/// IMAP without implying a UIDVALIDITY guard. The IMAP UIDVALIDITY guard treats -/// this non-numeric value as "no validity to compare" and is never engaged for -/// a JMAP mutation, which addresses messages by object id. +/// JMAP Email object id is already stable within its account (RFC 8621), so a +/// fixed sentinel keeps a placement key the same shape on both protocols +/// without implying a UIDVALIDITY guard. The IMAP UIDVALIDITY guard treats this +/// non-numeric value as "no validity to compare" and is never engaged for a +/// JMAP mutation, which addresses messages by object id. const JMAP_UID_VALIDITY_SENTINEL: &str = "jmap"; /// JMAP Session Object (RFC 8620 Section 2). @@ -368,11 +368,12 @@ impl JmapEngine { } } - /// Parse raw JMAP `Email/get` JSON response payload into domain [`Email`] list and new state string. + /// Parse a raw JMAP `Email/get` JSON response payload into a list of + /// [`PlacedMessage`] and the new state string. pub fn parse_email_get_response( &self, raw_json: &str, - ) -> Result<(Vec<Email>, String), MailError> { + ) -> Result<(Vec<PlacedMessage>, String), MailError> { let val: Value = serde_json::from_str(raw_json) .map_err(|e| MailError::ParseFailed(format!("invalid JMAP JSON response: {e}")))?; let payload = Self::extract_method_response_payload(&val)?; @@ -398,39 +399,58 @@ impl JmapEngine { .unwrap_or_else(|| "me@nuncio.mx".to_string()); let folder_id = "inbox".to_string(); - // The JMAP object id is the protocol-native id; the persisted - // id is an opaque surrogate hashed over it plus the account, - // folder, and (sentinel) scope so it stays uniform with IMAP. let remote_id = item.id; - let id = Email::surrogate_id( + let message_id = item + .message_id + .as_ref() + .and_then(|ids| ids.first()) + .and_then(|raw| Email::normalize_message_id(raw)); + + // A JMAP Email id is server-assigned and stable across mailboxes + // within the account, which is the same guarantee RFC 8474 + // EMAILID gives, so it enters at the top tier rather than + // through the surrogate. + let identity = Email::derive_message_key( &self.account_id, + RemoteIdentity { + email_id: Some(&remote_id), + gm_msgid: None, + message_id: message_id.as_deref(), + // `Email/get` returns a `bodySnippet`, not the + // message's octets, so there is nothing to hash here + // without an extra download per message. + content_hash: None, + }, &folder_id, JMAP_UID_VALIDITY_SENTINEL, &remote_id, ); - Email { - id, + let placement = Placement { account_id: self.account_id.clone(), folder_id, - remote_id, + // JMAP has no UIDVALIDITY; the sentinel keeps the placement + // key the same shape as IMAP's without implying a guard. uid_validity: JMAP_UID_VALIDITY_SENTINEL.to_string(), - subject: item.subject.unwrap_or_else(|| "No Subject".to_string()), - sender, - recipient, - received_at: item.received_at.unwrap_or(0), + remote_id: remote_id.clone(), read: !item.is_unread.unwrap_or(false), - body_plain: item.body_snippet, - body_html: None, - attachments: Vec::new(), - message_id: item - .message_id - .as_ref() - .and_then(|ids| ids.first()) - .and_then(|raw| Email::normalize_message_id(raw)), - // `Email/get` returns a `bodySnippet`, not the message's - // octets, so there is nothing to hash here without an - // extra download per message. - content_hash: None, + }; + + PlacedMessage { + email: Email { + id: identity.key, + account_id: self.account_id.clone(), + subject: item.subject.unwrap_or_else(|| "No Subject".to_string()), + sender, + recipient, + received_at: item.received_at.unwrap_or(0), + body_plain: item.body_snippet, + body_html: None, + attachments: Vec::new(), + message_id, + content_hash: None, + }, + source: identity.source, + placement, } }) .collect(); @@ -578,7 +598,7 @@ impl MailBackend for JmapEngine { &self, folder_id: &str, _since_state: Option<&str>, - ) -> Result<(Vec<Email>, String), MailError> { + ) -> Result<(Vec<PlacedMessage>, String), MailError> { if !self.has_credentials() { return Err(MailError::AuthError( "JMAP message sync requires credentials".to_string(), @@ -735,16 +755,16 @@ mod tests { let (emails, _state) = engine.parse_email_get_response(raw).expect("parse get"); assert_eq!( - emails[0].message_id.as_deref(), + emails[0].email.message_id.as_deref(), Some("Abc.123@mail.nuncio.mx") ); assert_eq!( - emails[1].message_id, None, + emails[1].email.message_id, None, "an absent messageId must stay None, not become empty" ); // `Email/get` returns a snippet, never the octets, so there is nothing // honest to hash on this path. - assert_eq!(emails[0].content_hash, None); + assert_eq!(emails[0].email.content_hash, None); } #[test] diff --git a/crates/nuncio-mail/src/lib.rs b/crates/nuncio-mail/src/lib.rs index cc8bfc9..fd02f1b 100644 --- a/crates/nuncio-mail/src/lib.rs +++ b/crates/nuncio-mail/src/lib.rs @@ -11,7 +11,8 @@ pub mod parser; pub mod smtp; pub use backend::{ - MailBackend, MessageSender, OutboundMessage, RemoteMutationKind, RemoteMutationSpec, + MailBackend, MessageSender, OutboundMessage, PlacedMessage, RemoteMutationKind, + RemoteMutationSpec, }; pub use imap::{IdleSocketState, ImapDualSocketManager, ImapEngine}; pub use jmap::JmapEngine; diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index 706f6ba..3216ad1 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -1,17 +1,19 @@ //! Deterministic mock mail backend for offline testing and integration verification. use async_trait::async_trait; -use nuncio_core::model::{Email, Folder}; +use nuncio_core::model::Folder; use std::sync::{Arc, Mutex}; -use crate::backend::{MailBackend, MessageSender, OutboundMessage, RemoteMutationSpec}; +use crate::backend::{ + MailBackend, MessageSender, OutboundMessage, PlacedMessage, RemoteMutationSpec, +}; use crate::parser::MailError; /// Thread-safe mock mail backend for offline testing. #[derive(Debug, Clone, Default)] pub struct MockMailBackend { folders: Arc<Mutex<Vec<Folder>>>, - messages: Arc<Mutex<Vec<Email>>>, + messages: Arc<Mutex<Vec<PlacedMessage>>>, should_fail: Arc<Mutex<bool>>, /// Every `since_state` argument this mock's `sync_messages` was called /// with, in call order, so tests can prove a caller threads the returned @@ -68,10 +70,13 @@ impl MockMailBackend { } } - /// Add a mock email message to the storage. - pub fn add_message(&self, email: Email) { + /// Add a mock message to the storage, as the occupancy a backend would + /// have found it in. Taking a [`PlacedMessage`] rather than a bare `Email` + /// lets a test stage the same message key in two folders and prove the + /// caller treats it as one message in two places. + pub fn add_message(&self, message: PlacedMessage) { if let Ok(mut guard) = self.messages.lock() { - guard.push(email); + guard.push(message); } } @@ -110,7 +115,7 @@ impl MailBackend for MockMailBackend { &self, folder_id: &str, since_state: Option<&str>, - ) -> Result<(Vec<Email>, String), MailError> { + ) -> Result<(Vec<PlacedMessage>, String), MailError> { // Record the checkpoint arg before any early return, so even a failing // call is visible to a test asserting on how the caller resumes. if let Ok(mut calls) = self.since_state_calls.lock() { @@ -145,7 +150,7 @@ impl MailBackend for MockMailBackend { .map_err(|e| MailError::ParseFailed(e.to_string()))?; let matches = messages .iter() - .filter(|m| m.folder_id == folder_id) + .filter(|m| m.placement.folder_id == folder_id) .cloned() .collect(); Ok((matches, returned_state)) @@ -229,6 +234,7 @@ impl MessageSender for MockMessageSender { #[cfg(test)] mod tests { use super::*; + use nuncio_core::model::{Email, IdentitySource, Placement}; #[tokio::test] async fn mock_backend_folder_and_message_operations() { @@ -240,24 +246,30 @@ mod tests { unread_messages: 1, }); - let email = Email { - id: "msg-mock-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: "Mock Test".to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1700000000, - read: false, - body_plain: Some("Mock body".to_string()), - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, + let placed = PlacedMessage { + email: Email { + id: "msg-mock-1".to_string(), + account_id: "acct-1".to_string(), + subject: "Mock Test".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1700000000, + body_plain: Some("Mock body".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + source: IdentitySource::Surrogate, + placement: Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: "1".to_string(), + read: false, + }, }; - mock.add_message(email.clone()); + mock.add_message(placed.clone()); let folders = mock.sync_folders().await.expect("sync folders succeeds"); assert_eq!(folders.len(), 1); @@ -267,6 +279,10 @@ mod tests { .await .expect("sync messages succeeds"); assert_eq!(msgs.len(), 1); + assert_eq!( + msgs[0], placed, + "the mock must hand back exactly the placement it was staged with" + ); assert_eq!(state, "mock-state-token-100"); // Test error simulation diff --git a/crates/nuncio-mail/src/parser.rs b/crates/nuncio-mail/src/parser.rs index e2c051e..25e7ef3 100644 --- a/crates/nuncio-mail/src/parser.rs +++ b/crates/nuncio-mail/src/parser.rs @@ -61,12 +61,15 @@ impl MimeParserAdapter { pub const MAX_PAYLOAD_BYTES: usize = 25 * 1024 * 1024; /// Parse a raw RFC 5322 byte slice into an [`Email`] domain entity. - pub fn parse_mime( - id: &str, - account_id: &str, - folder_id: &str, - raw_bytes: &[u8], - ) -> Result<Email, MailError> { + /// + /// The returned `Email` carries an **empty** `id`, and it is the caller's + /// job to fill it from [`Email::derive_message_key`]. The derivation needs + /// inputs this parser cannot see -- the transport's addressing coordinates + /// and any server-assigned identity (RFC 8474 `EMAILID`, `X-GM-MSGID`) -- + /// alongside the `Message-ID` and content hash recovered here, so the id + /// can only be settled once both halves are in hand. Minting a provisional + /// value here would risk one reaching the store as though it were derived. + pub fn parse_mime(account_id: &str, raw_bytes: &[u8]) -> Result<Email, MailError> { if raw_bytes.len() > Self::MAX_PAYLOAD_BYTES { return Err(MailError::ParseFailed( "MIME payload exceeds maximum allowed limit of 25MB".to_string(), @@ -129,18 +132,14 @@ impl MimeParserAdapter { } Ok(Email { - id: id.to_string(), + // Assigned by the caller from the derived message key; see the + // method doc for why it cannot be settled here. + id: String::new(), account_id: account_id.to_string(), - folder_id: folder_id.to_string(), - // The MIME body carries no protocol addressing; the caller (the - // IMAP sync) fills these from the transport before persisting. - remote_id: String::new(), - uid_validity: String::new(), subject, sender, recipient, received_at, - read: false, body_plain, body_html, attachments, @@ -163,9 +162,12 @@ mod tests { \r\n\ Hello Bob, this is a plain text email."; - let email = - MimeParserAdapter::parse_mime("msg-1", "acct-1", "inbox", raw).expect("parse succeeds"); - assert_eq!(email.id, "msg-1"); + let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds"); + assert_eq!( + email.id, "", + "the parser must not mint an id; the caller derives it" + ); + assert_eq!(email.account_id, "acct-1"); assert_eq!(email.subject, "Test Subject"); assert_eq!(email.sender, "alice@nuncio.mx"); assert_eq!(email.recipient, "bob@nuncio.mx"); @@ -184,8 +186,7 @@ mod tests { \r\n\ body"; - let email = - MimeParserAdapter::parse_mime("msg-1", "acct-1", "inbox", raw).expect("parse succeeds"); + let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds"); assert_eq!(email.message_id.as_deref(), Some("Abc.123@mail.nuncio.mx")); assert_eq!( email.content_hash, @@ -202,8 +203,7 @@ mod tests { Subject: No id\r\n\ \r\n\ body"; - let email = - MimeParserAdapter::parse_mime("msg-2", "acct-1", "inbox", raw).expect("parse succeeds"); + let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds"); assert_eq!(email.message_id, None); assert!(email.content_hash.is_some()); } @@ -211,8 +211,7 @@ mod tests { #[test] fn parse_invalid_mime_returns_error() { let raw = b""; - let err = MimeParserAdapter::parse_mime("msg-2", "acct-1", "inbox", raw) - .expect_err("should fail"); + let err = MimeParserAdapter::parse_mime("acct-1", raw).expect_err("should fail"); assert_eq!( err, MailError::ParseFailed("invalid RFC 5322 MIME structure".to_string()) @@ -226,7 +225,7 @@ mod tests { #[test] fn parse_oversized_mime_returns_error() { let oversized = vec![0u8; MimeParserAdapter::MAX_PAYLOAD_BYTES + 1]; - let err = MimeParserAdapter::parse_mime("msg-huge", "acct-1", "inbox", &oversized) + let err = MimeParserAdapter::parse_mime("acct-1", &oversized) .expect_err("should fail for oversized payload"); assert!(err .to_string() diff --git a/crates/nuncio-mail/src/smtp.rs b/crates/nuncio-mail/src/smtp.rs index 25e7a80..673d963 100644 --- a/crates/nuncio-mail/src/smtp.rs +++ b/crates/nuncio-mail/src/smtp.rs @@ -357,14 +357,10 @@ mod tests { Email { id: "msg-1".to_string(), account_id: "acct-1".to_string(), - folder_id: "outbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), subject: "Status Update".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: true, body_plain: Some("Plaintext status update".to_string()), body_html: None, attachments: Vec::new(), diff --git a/crates/nuncio-mail/tests/wiremock_jmap_test.rs b/crates/nuncio-mail/tests/wiremock_jmap_test.rs index f809590..46b9415 100644 --- a/crates/nuncio-mail/tests/wiremock_jmap_test.rs +++ b/crates/nuncio-mail/tests/wiremock_jmap_test.rs @@ -2,6 +2,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +use nuncio_core::model::IdentitySource; use nuncio_mail::{JmapEngine, MailBackend, RemoteMutationKind, RemoteMutationSpec}; use serde_json::json; use wiremock::matchers::{body_partial_json, method, path}; @@ -84,13 +85,16 @@ async fn wiremock_jmap_session_discovery_and_email_get_sync() { .expect("parse email get"); assert_eq!(emails.len(), 1); - // The JMAP object id is preserved as the protocol-native `remote_id`, while - // the persisted id is an opaque 64-char surrogate hash (never the raw id). - assert_eq!(emails[0].remote_id, "msg-wm-100"); - assert_ne!(emails[0].id, "msg-wm-100"); - assert_eq!(emails[0].id.len(), 64); - assert_eq!(emails[0].subject, "WireMock JMAP E2E Test"); - assert_eq!(emails[0].sender, "sender@nuncio.mx"); + // The JMAP object id is preserved as the placement's protocol-native + // `remote_id`, while the message's own id is an opaque 64-char derived key + // (never the raw id). The object id is account-stable, so it enters the + // precedence at the EMAILID tier rather than as a folder-scoped surrogate. + assert_eq!(emails[0].placement.remote_id, "msg-wm-100"); + assert_eq!(emails[0].source, IdentitySource::EmailId); + assert_ne!(emails[0].email.id, "msg-wm-100"); + assert_eq!(emails[0].email.id.len(), 64); + assert_eq!(emails[0].email.subject, "WireMock JMAP E2E Test"); + assert_eq!(emails[0].email.sender, "sender@nuncio.mx"); assert_eq!(new_state, "sync-state-500"); } @@ -204,15 +208,18 @@ async fn jmap_engine_sync_folders_and_messages_through_mail_backend_trait() { .await .expect("sync_messages succeeds"); assert_eq!(emails.len(), 1); - assert_eq!(emails[0].remote_id, "msg-wm-200"); - assert_ne!(emails[0].id, "msg-wm-200"); - assert_eq!(emails[0].id.len(), 64); - assert_eq!(emails[0].subject, "Trait-Level JMAP Sync"); - assert_eq!(emails[0].sender, "sender@nuncio.mx"); - assert_eq!(emails[0].received_at, 1700002000); - assert!(!emails[0].read); + assert_eq!(emails[0].placement.remote_id, "msg-wm-200"); + assert_eq!(emails[0].placement.folder_id, "inbox"); + assert_eq!(emails[0].source, IdentitySource::EmailId); + assert_ne!(emails[0].email.id, "msg-wm-200"); + assert_eq!(emails[0].email.id.len(), 64); + assert_eq!(emails[0].email.subject, "Trait-Level JMAP Sync"); + assert_eq!(emails[0].email.sender, "sender@nuncio.mx"); + assert_eq!(emails[0].email.received_at, 1700002000); + // Read state is a property of the occupancy, not of the message. + assert!(!emails[0].placement.read); assert_eq!( - emails[0].body_plain, + emails[0].email.body_plain, Some("Fetched through the MailBackend trait.".to_string()) ); assert_eq!(new_state, "sync-state-900"); -- 2.54.0 From 1d113ef11a4b132e61407d395b9d04faa39d5768 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 09:07:18 -0500 Subject: [PATCH 72/95] fix(mail): report no absence when a mailbox omits UIDVALIDITY A SELECT that omits [UIDVALIDITY] left every placement key this pass built carrying an empty generation. None of them match placements stored under a real one, and under the `present` contract -- delete any placement absent from this list -- that reads as "the folder holds nothing", so the caller would delete the whole mailbox. The pass now declines to report absence at all, mirroring the checkpoint path, which already refuses to write a validity-less value, and warns so the reason is visible. Upserts are still surfaced. [UIDVALIDITY] is REQUIRED on SELECT (RFC 3501 section 6.3.1), so this is a guard against a broken server rather than an expected path -- but the impact is a wiped mailbox. Also pin the identity tier on the two branches that fetch no octets. The headers-only branch drops its hash because it covers only the header section; publishing it would key the message on the wrong octets and mint a second key once the body arrived, resurrecting the message as a duplicate. The fixtures now carry a Message-ID so those branches reaching the surrogate is a real assertion rather than an accident of the fixture, and the full-body path is asserted to reach the content tier for contrast. Give PlacementKey and Placement::key() direct tests: all four coordinates reproduced, the read flag excluded, each coordinate distinguishing, usable as a set key and sortable, and a serde round trip. --- crates/nuncio-core/src/model.rs | 91 +++++++++++++ crates/nuncio-mail/src/imap.rs | 231 ++++++++++++++++++++++++++------ 2 files changed, 283 insertions(+), 39 deletions(-) diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index 33a9f8f..508ef43 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -725,6 +725,97 @@ mod tests { assert_ne!(as_emailid.key, as_gmsgid.key); } + fn sample_placement() -> Placement { + Placement { + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "42".to_string(), + remote_id: "5".to_string(), + read: true, + } + } + + #[test] + fn placement_key_carries_every_addressing_coordinate_and_drops_the_read_flag() { + // The key must reproduce all four coordinates exactly. A key that + // dropped or reordered one would address a different occupancy, and a + // sync diffing reported-against-stored would delete live mail. + let placement = sample_placement(); + let key = placement.key(); + assert_eq!(key.account_id, "acct-1"); + assert_eq!(key.folder_id, "INBOX"); + assert_eq!(key.uid_validity, "42"); + assert_eq!(key.remote_id, "5"); + + // The read flag is the one field a key must NOT carry: it describes an + // occupancy rather than naming it, so including it would make the same + // occupancy compare unequal to itself after the user opened the message. + let mut unread = placement.clone(); + unread.read = false; + assert_ne!(placement, unread); + assert_eq!(placement.key(), unread.key()); + } + + #[test] + fn placement_keys_distinguish_every_coordinate_that_can_differ() { + let base = sample_placement().key(); + for mutated in [ + PlacementKey { + account_id: "acct-2".to_string(), + ..base.clone() + }, + PlacementKey { + folder_id: "Archive".to_string(), + ..base.clone() + }, + PlacementKey { + uid_validity: "43".to_string(), + ..base.clone() + }, + PlacementKey { + remote_id: "6".to_string(), + ..base.clone() + }, + ] { + assert_ne!( + base, mutated, + "changing one coordinate must yield a different occupancy" + ); + } + + // Usable as a set key and sortable, which is what the sync path needs + // to diff a reported UID set against what the store holds. + let mut set = std::collections::HashSet::new(); + assert!(set.insert(base.clone())); + assert!(!set.insert(base.clone()), "an equal key must not re-insert"); + let mut keys = [ + PlacementKey { + remote_id: "6".to_string(), + ..base.clone() + }, + base.clone(), + ]; + keys.sort(); + assert_eq!(keys[0], base); + } + + #[test] + fn placement_key_round_trips_through_serde() { + let key = sample_placement().key(); + let json = serde_json::to_string(&key).unwrap(); + let parsed: PlacementKey = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, key); + } + + #[test] + fn placement_round_trips_through_serde_including_its_read_flag() { + let placement = sample_placement(); + let json = serde_json::to_string(&placement).unwrap(); + let parsed: Placement = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, placement); + assert!(parsed.read); + } + #[test] fn identity_source_round_trips_through_its_stored_string() { for source in [ diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index e840f18..bbb6fe1 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -972,18 +972,44 @@ impl ImapEngine { .unwrap_or_else(|| "full-resync-required".to_string()), }; - // Both are occupancies, not messages: a UID that vanished from this - // folder says nothing about copies of the same message elsewhere. - let removals: Vec<PlacementKey> = plan - .vanished_uids - .iter() - .map(|uid| self.placement_key_for(folder_id, server_uid_validity, *uid)) - .collect(); - let present = plan.present_uids.as_ref().map(|uids| { - uids.iter() - .map(|uid| self.placement_key_for(folder_id, server_uid_validity, *uid)) - .collect() - }); + // Absence can only be reported against a known UIDVALIDITY generation. + // + // `[UIDVALIDITY]` is REQUIRED on a successful SELECT (RFC 3501 section + // 6.3.1), so this should not happen -- but a server that omits it would + // otherwise produce keys carrying an empty generation, and NONE of them + // would match placements stored under a real one. Under the documented + // contract ("delete any placement absent from `present`") that reads as + // "the folder holds nothing", and the caller deletes the whole mailbox. + // Saying nothing is the only safe answer, and it mirrors the checkpoint + // above, which likewise refuses to write a validity-less value. + let (removals, present) = match server_uid_validity { + Some(_) => { + // Both are occupancies, not messages: a UID that vanished from + // this folder says nothing about copies of the same message + // elsewhere. + let removals: Vec<PlacementKey> = plan + .vanished_uids + .iter() + .map(|uid| self.placement_key_for(folder_id, server_uid_validity, *uid)) + .collect(); + let present = plan.present_uids.as_ref().map(|uids| { + uids.iter() + .map(|uid| self.placement_key_for(folder_id, server_uid_validity, *uid)) + .collect() + }); + (removals, present) + } + None => { + tracing::warn!( + folder_id, + vanished = plan.vanished_uids.len(), + enumerated = plan.present_uids.as_ref().map_or(0, Vec::len), + "mailbox reported no UIDVALIDITY, so this pass cannot key an occupancy and \ + reports no absence; upserts are still surfaced" + ); + (Vec::new(), None) + } + }; tracing::info!( account_id = %self.account_id, @@ -991,7 +1017,7 @@ impl ImapEngine { rung = ?plan.rung, fetched = emails.len(), removed = removals.len(), - enumerated = plan.present_uids.as_ref().map_or(0, Vec::len), + enumerated = present.as_ref().map_or(0, Vec::len), oversized = oversized_uids.len(), skipped = skipped_no_uid, "imap folder sync complete" @@ -2324,17 +2350,25 @@ mod tests { } /// The RFC822 body of a scripted fixture message. + /// + /// Carries a `Message-ID` so the full-body path reaches the content tier. + /// That is what makes the headers-only path's tier meaningful to assert: + /// the same message identifies differently there, and only because the + /// header-section hash is deliberately discarded. fn fixture_body(uid: u32) -> String { format!( "Subject: Message {uid}\r\nFrom: sender{uid}@example.test\r\n\ + Message-ID: <msg{uid}@example.test>\r\n\ To: me@example.test\r\n\r\nBody of message {uid}.\r\n" ) } - /// The header block (no body) of a scripted fixture message. + /// The header block (no body) of a scripted fixture message. Same headers + /// as [`fixture_body`], including the `Message-ID`. fn fixture_header(uid: u32) -> String { format!( "Subject: Message {uid}\r\nFrom: sender{uid}@example.test\r\n\ + Message-ID: <msg{uid}@example.test>\r\n\ To: me@example.test\r\n\r\n" ) } @@ -2368,13 +2402,17 @@ mod tests { /// and records every `UID FETCH` command line the client issued so a test /// can assert on the exact batching. Returns the synced emails, the new /// checkpoint, and the captured `UID FETCH` command lines. + /// + /// `uidvalidity` is an `Option` so a test can script the one thing RFC 3501 + /// section 6.3.1 says a SELECT must never do -- omit `[UIDVALIDITY]` -- and + /// check the sync stays safe when a server does it anyway. async fn run_batched_sync( messages: Vec<(u32, u32)>, - uidvalidity: u32, + uidvalidity: Option<u32>, uidnext: u32, since_state: Option<String>, batch_size: usize, - ) -> (Vec<PlacedMessage>, String, Vec<String>) { + ) -> (FolderChanges, Vec<String>) { let (client_io, server_io) = tokio::io::duplex(65536); let fetch_cmds = Arc::new(std::sync::Mutex::new(Vec::<String>::new())); let server_cmds = fetch_cmds.clone(); @@ -2436,9 +2474,13 @@ mod tests { .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) .await; } else if upper.contains("SELECT") { + let validity_line = match uidvalidity { + Some(v) => format!("* OK [UIDVALIDITY {v}] ok\r\n"), + None => String::new(), + }; let resp = format!( "* FLAGS (\\Seen)\r\n* {exists} EXISTS\r\n* 0 RECENT\r\n\ - * OK [UIDVALIDITY {uidvalidity}] ok\r\n* OK [UIDNEXT {uidnext}] ok\r\n\ + {validity_line}* OK [UIDNEXT {uidnext}] ok\r\n\ {tag} OK [READ-WRITE] SELECT done\r\n" ); let _ = io.write_all(resp.as_bytes()).await; @@ -2462,12 +2504,11 @@ mod tests { .sync_folder_messages_batched("INBOX", since_state.as_deref(), &mut session, batch_size) .await .expect("batched sync succeeds"); - let (emails, checkpoint) = (changes.upserts, changes.next_state); drop(session); let _ = server.await; let cmds = fetch_cmds.lock().expect("lock captured commands").clone(); - (emails, checkpoint, cmds) + (changes, cmds) } /// The captured `UID FETCH` command lines that pulled FULL bodies (a @@ -2489,10 +2530,11 @@ mod tests { // `1:*` body fetch. The enumeration discovers the UIDs; the bodies are // then fetched in explicit UID-set chunks of at most the batch size. let messages: Vec<(u32, u32)> = (1..=5).map(|u| (u, 100)).collect(); - let (emails, checkpoint, cmds) = run_batched_sync(messages, 1, 6, None, 2).await; + let (changes, cmds) = run_batched_sync(messages, Some(1), 6, None, 2).await; + let emails = &changes.upserts; assert_eq!(emails.len(), 5, "every message must be synced"); - assert_eq!(checkpoint, "v1:uidnext:1:6"); + assert_eq!(changes.next_state, "v1:uidnext:1:6"); let body_cmds = body_fetch_commands(&cmds); assert_eq!( @@ -2522,7 +2564,8 @@ mod tests { // normal messages get full bodies. let huge = MAX_MESSAGE_BODY_BYTES + 1; let messages: Vec<(u32, u32)> = vec![(1, 100), (2, huge), (3, 100)]; - let (emails, _checkpoint, cmds) = run_batched_sync(messages, 1, 4, None, 200).await; + let (changes, cmds) = run_batched_sync(messages, Some(1), 4, None, 200).await; + let emails = &changes.upserts; assert_eq!(emails.len(), 3, "no message may be silently dropped"); @@ -2558,6 +2601,26 @@ mod tests { "oversized message must keep honest header metadata, got: {oversized:?}" ); + // The header section it WAS given hashes to something, but that hash + // covers only the headers, not the message. Publishing it would key the + // message on the wrong octets -- and then the same message would key + // differently once its body was finally fetched, resurrecting it as a + // duplicate. So the hash is dropped, which costs this message the + // content tier even though it carried a Message-ID. + assert!( + oversized.email.message_id.is_some(), + "the headers-only parse must still recover the Message-ID" + ); + assert_eq!( + oversized.email.content_hash, None, + "a hash over only the header section must never be published" + ); + assert_eq!( + oversized.source, + IdentitySource::Surrogate, + "without octets to hash, a Message-ID alone must not reach the content tier" + ); + // The normal messages DID get full bodies. for uid in ["1", "3"] { let email = emails @@ -2568,6 +2631,13 @@ mod tests { email.email.body_plain.is_some(), "normal message {uid} must have a body, got: {email:?}" ); + // The contrast that gives the assertion above its force: the same + // headers, plus the octets, DO reach the content tier. + assert_eq!( + email.source, + IdentitySource::MessageIdContent, + "a full-body fetch with a Message-ID must reach the content tier" + ); } } @@ -2578,24 +2648,24 @@ mod tests { // checkpoint enumerates only the narrowed `{uidnext}:*` range (so a // fire-once-per-new-message consumer never re-sees old mail). let messages: Vec<(u32, u32)> = (1..=3).map(|u| (u, 100)).collect(); - let (first, checkpoint, first_cmds) = - run_batched_sync(messages.clone(), 1, 4, None, 2).await; - assert_eq!(first.len(), 3); + let (first, first_cmds) = run_batched_sync(messages.clone(), Some(1), 4, None, 2).await; + let checkpoint = first.next_state.clone(); + assert_eq!(first.upserts.len(), 3); assert_eq!(checkpoint, "v1:uidnext:1:4"); assert!( first_cmds.iter().any(|c| c.contains("1:*")), "first sync enumerates the full range, got: {first_cmds:?}" ); - let (second, checkpoint2, second_cmds) = - run_batched_sync(messages, 1, 4, Some(checkpoint.clone()), 2).await; + let (second, second_cmds) = + run_batched_sync(messages, Some(1), 4, Some(checkpoint.clone()), 2).await; assert_eq!( - second.len(), + second.upserts.len(), 0, "no new mail arrived, so an incremental sync returns nothing" ); assert_eq!( - checkpoint2, "v1:uidnext:1:4", + second.next_state, "v1:uidnext:1:4", "checkpoint stays stable across an empty pass" ); let second_bodies = body_fetch_ranges(&second_cmds); @@ -3395,15 +3465,19 @@ mod tests { /// controlling its `\Seen` flag. Holding the octets fixed while varying the /// folder, UIDVALIDITY and UID is the shape of "the same message, found in /// two mailboxes" -- which is what the identity/placement split exists for. + /// + /// Passing `raw = None` scripts the envelope-only path instead: the server + /// answers the body fetch with an `ENVELOPE` and no `BODY[]`, which is what + /// a server that will not hand over octets looks like. async fn run_single_message_sync( folder_id: &str, uidvalidity: u32, uid: u32, seen: bool, - raw: &str, + raw: Option<&str>, ) -> PlacedMessage { let (client_io, server_io) = tokio::io::duplex(65536); - let raw = raw.to_string(); + let raw = raw.map(str::to_string); let uidnext = uid.saturating_add(1); let flags = if seen { "\\Seen" } else { "" }; @@ -3413,16 +3487,31 @@ mod tests { let tag = line.split_whitespace().next().unwrap_or("").to_string(); let upper = line.to_ascii_uppercase(); if upper.contains("UID FETCH") { - let size = raw.len(); - let resp = if upper.contains("BODY.PEEK[]") { + let size = raw.as_ref().map_or(64, String::len); + let resp = if !upper.contains("BODY.PEEK[]") { + // Enumeration pass: UID + size only, no body. + format!( + "* 1 FETCH (UID {uid} RFC822.SIZE {size})\r\n\ + {tag} OK FETCH completed\r\n" + ) + } else if let Some(raw) = raw.as_ref() { format!( "* 1 FETCH (UID {uid} RFC822.SIZE {size} FLAGS ({flags}) \ BODY[] {{{size}}}\r\n{raw})\r\n{tag} OK FETCH completed\r\n" ) } else { - // Enumeration pass: UID + size only, no body. + // A server that answers with metadata but no octets. + // ENVELOPE still carries `message-id` (RFC 3501 + // section 7.4.2), so identity capture is not skipped -- + // but there is nothing to hash. format!( - "* 1 FETCH (UID {uid} RFC822.SIZE {size})\r\n\ + "* 1 FETCH (UID {uid} RFC822.SIZE {size} FLAGS ({flags}) \ + ENVELOPE (\"Mon, 1 Jan 2024 00:00:00 +0000\" \"Env subject\" \ + ((\"A\" NIL \"alice\" \"b.example\")) \ + ((\"A\" NIL \"alice\" \"b.example\")) \ + ((\"A\" NIL \"alice\" \"b.example\")) \ + ((\"B\" NIL \"bob\" \"b.example\")) \ + NIL NIL NIL \"<env@b.example>\"))\r\n\ {tag} OK FETCH completed\r\n" ) }; @@ -3487,8 +3576,8 @@ mod tests { // placements stay distinct. Before the split this was two messages. let raw = "Message-ID: <a@b.example>\r\nSubject: Hi\r\n\ From: alice@b.example\r\n\r\nbody\r\n"; - let inbox = run_single_message_sync("INBOX", 42, 5, true, raw).await; - let archive = run_single_message_sync("Archive", 77, 9, false, raw).await; + let inbox = run_single_message_sync("INBOX", 42, 5, true, Some(raw)).await; + let archive = run_single_message_sync("Archive", 77, 9, false, Some(raw)).await; assert_eq!(inbox.source, IdentitySource::MessageIdContent); assert_eq!(archive.source, IdentitySource::MessageIdContent); @@ -3518,8 +3607,8 @@ mod tests { // report two honest folder-scoped keys rather than claim a stability // the server gave it no basis for. let raw = "Subject: No id\r\nFrom: alice@b.example\r\n\r\nbody\r\n"; - let inbox = run_single_message_sync("INBOX", 42, 5, false, raw).await; - let archive = run_single_message_sync("Archive", 77, 9, false, raw).await; + let inbox = run_single_message_sync("INBOX", 42, 5, false, Some(raw)).await; + let archive = run_single_message_sync("Archive", 77, 9, false, Some(raw)).await; assert_eq!(inbox.source, IdentitySource::Surrogate); assert_ne!( @@ -3528,6 +3617,70 @@ mod tests { ); } + #[tokio::test] + async fn an_envelope_only_fetch_keeps_its_message_id_but_stays_on_the_surrogate() { + // A server that returns metadata and no octets. ENVELOPE carries + // `message-id` (RFC 3501 section 7.4.2) so the header is still + // captured, but there is nothing to hash -- and a Message-ID alone is + // public and forgeable, so it must never key storage by itself. + let placed = run_single_message_sync("INBOX", 42, 5, false, None).await; + + assert_eq!(placed.email.subject, "Env subject"); + assert_eq!(placed.email.message_id.as_deref(), Some("env@b.example")); + assert_eq!( + placed.email.content_hash, None, + "no octets were fetched, so there is nothing honest to hash" + ); + assert_eq!( + placed.source, + IdentitySource::Surrogate, + "an envelope-only fetch must not reach the content tier" + ); + assert_eq!(placed.placement.remote_id, "5"); + assert_eq!(placed.placement.uid_validity, "42"); + } + + #[traced_test] + #[tokio::test] + async fn a_select_without_uidvalidity_reports_no_absence_at_all() { + // RFC 3501 section 6.3.1 makes `[UIDVALIDITY]` REQUIRED on SELECT, so a + // server omitting it is broken -- but the failure mode is a wiped + // mailbox, not a bad sync. Every placement key this pass could build + // would carry an empty generation and match nothing the store holds, + // and under the `present` contract that reads as "the folder is empty". + // The pass must therefore decline to speak about absence, while still + // surfacing what it fetched. + let messages: Vec<(u32, u32)> = (1..=2).map(|u| (u, 100)).collect(); + let (changes, _cmds) = run_batched_sync(messages, None, 3, None, 200).await; + + assert_eq!( + changes.upserts.len(), + 2, + "the messages themselves are still surfaced" + ); + assert_eq!( + changes.present, None, + "a pass that cannot key an occupancy must not claim to know what is present" + ); + assert!( + changes.removals.is_empty(), + "nor may it report a removal, got: {:?}", + changes.removals + ); + // And it must say why, rather than failing silently. + assert!(logs_contain("mailbox reported no UIDVALIDITY")); + + // The same pass against a compliant server DOES report presence, so the + // assertions above are about the missing UIDVALIDITY and nothing else. + let messages: Vec<(u32, u32)> = (1..=2).map(|u| (u, 100)).collect(); + let (compliant, _cmds) = run_batched_sync(messages, Some(7), 3, None, 200).await; + let present = compliant + .present + .expect("a compliant SELECT must let the pass report presence"); + assert_eq!(present.len(), 2); + assert!(present.iter().all(|k| k.uid_validity == "7")); + } + #[tokio::test] async fn a_message_fetched_from_two_folders_keeps_one_identity() { // Both folders return the same Message-ID and the same octets, so the -- 2.54.0 From 1260dd0a0983996a5f07b77668f64b8024558a6b Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 09:36:16 -0500 Subject: [PATCH 73/95] feat(sync): classify arrivals per placement and claim fires once Message identity is folder-independent, so a message already stored from INBOX has a known key when it turns up in Archive. Classifying "is this new?" by message key answered "seen" for a mailbox nothing had ever filtered, and the arrival silently fired no rule at all -- no error, no log. fetch_and_persist now classifies on PlacementKey via existing_placements, persists through save_email_at, and reconciles absence with placements_in_folder/delete_placements. Evaluation is therefore per placement, so acting has to be gated separately: apply_filter_actions takes the placement it evaluates against and must win claim_filter_fire, keyed on message identity, before any action runs. MOVE and FLAG converge under repetition; FORWARD and CALL WEBHOOK land on third parties and do not. The claim fails closed -- an unverifiable claim skips the actions rather than licensing them. MARK READ now names the occupancy it was evaluated against, since IMAP \Seen is per-mailbox. The daemon's gRPC and outbox surfaces are ported only as far as compilation and honest behaviour require, and say so in place: the outbox recovers its remote addressing from a placement and warns when the row cannot say which one the rule matched, and the Mail read RPCs merge across configured accounts pending an account on the wire. --- crates/nuncio-mail/src/mock.rs | 101 ++- crates/nunciod/src/grpc.rs | 580 ++++++++++++------ crates/nunciod/src/outbox.rs | 84 ++- crates/nunciod/src/sync.rs | 580 +++++++++++++----- .../nunciod/tests/filters_triage_e2e_test.rs | 73 ++- .../nunciod/tests/no_secrets_canary_test.rs | 37 +- .../nunciod/tests/outbox_executor_e2e_test.rs | 42 +- crates/nunciod/tests/spine_e2e_test.rs | 46 +- 8 files changed, 1124 insertions(+), 419 deletions(-) diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index 47fcd63..44cce17 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -1,7 +1,7 @@ //! Deterministic mock mail backend for offline testing and integration verification. use async_trait::async_trait; -use nuncio_core::model::Folder; +use nuncio_core::model::{Email, Folder, IdentitySource, Placement}; use std::sync::{Arc, Mutex}; use crate::backend::{ @@ -81,6 +81,71 @@ impl MockMailBackend { } } + /// Account every message staged by [`Self::shared_message_in`] and + /// [`Self::with_same_message_in`] belongs to. + pub const SHARED_ACCOUNT_ID: &'static str = "acct-1"; + + /// The one derived message key every occupancy staged by + /// [`Self::shared_message_in`] carries, whatever folder it sits in. + pub const SHARED_MESSAGE_KEY: &'static str = "msg-shared-identity"; + + /// One occupancy of the single shared message, in `folder_id` under + /// `remote_id`. + /// + /// Identity is byte-identical across every folder -- that is the whole + /// point: a caller cannot tell a first arrival in a second mailbox from a + /// message it already stores unless it compares occupancies rather than + /// message keys. + pub fn shared_message_in(folder_id: &str, remote_id: &str) -> PlacedMessage { + PlacedMessage { + email: Email { + id: Self::SHARED_MESSAGE_KEY.to_string(), + account_id: Self::SHARED_ACCOUNT_ID.to_string(), + subject: "Shared across folders".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some("one message, several mailboxes".to_string()), + body_html: None, + message_id: Some("shared@nuncio.mx".to_string()), + content_hash: None, + attachments: Vec::new(), + }, + // EmailId, not Surrogate: a folder-independent key is exactly what + // a server-assigned identity buys, and it is the tier under which + // one message legitimately reports from several mailboxes. + source: IdentitySource::EmailId, + placement: Placement { + account_id: Self::SHARED_ACCOUNT_ID.to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: remote_id.to_string(), + read: false, + }, + } + } + + /// A backend serving `folders`, each holding the same message under its own + /// `remote_id` -- one identity, one occupancy per folder. + /// + /// Stages the folders as well as the messages, so a caller that enumerates + /// folders and then syncs each one sees every occupancy. + pub fn with_same_message_in(folders: &[&str]) -> Self { + let backend = Self::new(); + for (index, folder_id) in folders.iter().enumerate() { + backend.add_folder(Folder { + id: (*folder_id).to_string(), + name: (*folder_id).to_string(), + total_messages: 1, + unread_messages: 1, + }); + // A distinct UID per folder: UIDs are folder-scoped, so the same + // mail in two mailboxes is addressed by two different ones. + backend.add_message(Self::shared_message_in(folder_id, &(index + 1).to_string())); + } + backend + } + /// Every [`RemoteMutationSpec`] this mock's `apply_mutation` was called /// with, in call order. A failing configuration (`set_should_fail(true)`) /// records nothing, so a test can prove a failed op is never counted as @@ -250,7 +315,6 @@ impl MessageSender for MockMessageSender { #[cfg(test)] mod tests { use super::*; - use nuncio_core::model::{Email, IdentitySource, Placement}; #[tokio::test] async fn mock_backend_folder_and_message_operations() { @@ -307,6 +371,39 @@ mod tests { assert!(mock.sync_messages("inbox", None).await.is_err()); } + /// The staging helper is only useful if it really produces one identity in + /// two places: same message key, different occupancies. If it ever drifted + /// into two keys, every test built on it would pass for the wrong reason. + #[tokio::test] + async fn with_same_message_in_stages_one_identity_across_every_folder() { + let mock = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]); + + let folders = mock.sync_folders().await.expect("sync folders succeeds"); + let folder_ids: Vec<&str> = folders.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(folder_ids, vec!["INBOX", "Archive"]); + + let inbox = mock + .sync_changes("INBOX", None) + .await + .expect("inbox pass succeeds"); + let archive = mock + .sync_changes("Archive", None) + .await + .expect("archive pass succeeds"); + assert_eq!(inbox.upserts.len(), 1); + assert_eq!(archive.upserts.len(), 1); + + assert_eq!( + inbox.upserts[0].email.id, archive.upserts[0].email.id, + "both occupancies must carry the same derived message key" + ); + assert_ne!( + inbox.upserts[0].placement.key(), + archive.upserts[0].placement.key(), + "the occupancies themselves must differ, or there is nothing to classify" + ); + } + fn sample_outbound_message() -> OutboundMessage { OutboundMessage { from: "alice@nuncio.mx".to_string(), diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 0cb2ee9..612e88d 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -963,23 +963,28 @@ fn map_attachment_from_proto(attachment: AttachmentProto) -> nuncio_core::model: } } -/// Maps a `nuncio_core::model::Email` onto its wire-format `nuncio.v1.Message` -/// representation. The store (`DatabaseEngine::get_message` / -/// `list_messages`) already returns `body_plain`/`body_html` decrypted, so -/// there is no further decryption to do here -- just field-for-field -/// mapping. -fn map_email_to_proto(email: nuncio_core::model::Email) -> MessageProto { +/// Maps a message and the mailbox occupancy it was read through onto the +/// wire-format `nuncio.v1.Message` representation. The store +/// (`DatabaseEngine::get_message` / `list_messages`) already returns +/// `body_plain`/`body_html` decrypted, so there is no further decryption to do +/// here -- just field-for-field mapping. +/// +/// `folder_id` and `read` come from the placement: both are per-mailbox facts +/// now, and the wire message still carries exactly one of each, so it describes +/// one occupancy of the message rather than the message as a whole. +fn map_email_to_proto(placed: nuncio_store::db::MessageWithPlacement) -> MessageProto { + let (email, placement) = placed; MessageProto { id: email.id, account_id: email.account_id, - folder_id: email.folder_id, + folder_id: placement.folder_id, subject: email.subject, sender: email.sender, recipient: email.recipient, received_at: Some(nuncio_proto::time::timestamp_from_unix_secs( email.received_at, )), - read: email.read, + read: placement.read, body_plain: email.body_plain, body_html: email.body_html, attachments: email @@ -1633,6 +1638,145 @@ struct MailGrpcService { sync_dispatcher: Arc<SyncDispatcher>, } +/// Account-merging read helpers. +/// +/// The store's folder and message reads are account-scoped, because folder ids +/// are not globally unique -- every account has an `INBOX`, and grouping across +/// accounts would report one merged folder holding two accounts' mail. The +/// `nuncio.v1.Mail` requests carry no account field, so these reproduce the +/// pre-split cross-account behaviour by asking each account and merging. They +/// are a compatibility shim, not the destination: the honest fix is an account +/// on the wire, which lands with the placement-aware surface. +impl MailGrpcService { + /// One page of folders, merged across every configured account. + /// + /// Counts for a folder id that several accounts share are summed, matching + /// what the old account-blind `GROUP BY folder_id` returned. + async fn merged_folders_page( + &self, + after: Option<String>, + page_size: usize, + ) -> Result<(Vec<nuncio_core::model::Folder>, Option<String>), nuncio_store::db::DatabaseError> + { + let accounts = self.db.list_accounts().await?; + let mut merged: std::collections::BTreeMap<String, nuncio_core::model::Folder> = + std::collections::BTreeMap::new(); + // One more than the page from each account, for the same reason the + // store itself over-fetches by one: the merge cannot tell "this is the + // last page" from "this account was truncated at exactly page_size" + // unless at least one extra row can show up. + let probe = page_size.saturating_add(1); + for account in &accounts { + // Each account is asked from the same cursor; the merge below + // re-truncates, so no account's page is starved by another's. + let (folders, _) = self + .db + .list_folders_page(&account.id, after.clone(), probe) + .await?; + for folder in folders { + merged + .entry(folder.id.clone()) + .and_modify(|existing| { + existing.total_messages += folder.total_messages; + existing.unread_messages += folder.unread_messages; + }) + .or_insert(folder); + } + } + + let mut folders: Vec<nuncio_core::model::Folder> = merged.into_values().collect(); + let has_more = folders.len() > page_size; + folders.truncate(page_size); + let next = has_more + .then(|| folders.last().map(|f| f.id.clone())) + .flatten(); + Ok((folders, next)) + } + + /// One page of a folder's messages, merged across every configured account, + /// in the store's own `received_at DESC, message_key DESC, uidvalidity + /// DESC, uid DESC` order. + async fn merged_messages_page( + &self, + folder_id: &str, + after: Option<nuncio_store::db::MessagePageCursor>, + page_size: usize, + ) -> Result< + ( + Vec<nuncio_store::db::MessageWithPlacement>, + Option<nuncio_store::db::MessagePageCursor>, + ), + nuncio_store::db::DatabaseError, + > { + let accounts = self.db.list_accounts().await?; + let mut rows: Vec<nuncio_store::db::MessageWithPlacement> = Vec::new(); + // One more than the page from each account -- see `merged_folders_page` + // for why the extra row is what makes "is there another page?" + // answerable after the merge. + let probe = page_size.saturating_add(1); + for account in &accounts { + let (page, _) = self + .db + .list_messages_page(&account.id, folder_id, after.clone(), probe) + .await?; + rows.extend(page); + } + + // Re-impose the store's ordering over the concatenated per-account + // pages, so the cursor this returns is a position in the merged + // sequence rather than in whichever account happened to be last. + rows.sort_by(|(a_mail, a_place), (b_mail, b_place)| { + b_mail + .received_at + .cmp(&a_mail.received_at) + .then_with(|| b_mail.id.cmp(&a_mail.id)) + .then_with(|| b_place.uid_validity.cmp(&a_place.uid_validity)) + .then_with(|| b_place.remote_id.cmp(&a_place.remote_id)) + }); + + let has_more = rows.len() > page_size; + rows.truncate(page_size); + let next = has_more + .then(|| { + rows.last().map(|(email, placement)| { + ( + email.received_at, + email.id.clone(), + placement.uid_validity.clone(), + placement.remote_id.clone(), + ) + }) + }) + .flatten(); + Ok((rows, next)) + } + + /// The mailbox occupancy a message-scoped RPC should speak for. + /// + /// `GetMessage`/`MarkRead` name a message, but `folder_id` and the read + /// flag are per-mailbox now, so one occupancy has to be chosen. This takes + /// the store's first in its deterministic order. A message with no + /// occupancy at all is not "unread in no folder" -- it is gone -- and is + /// reported as not found. + async fn primary_placement( + &self, + message_key: &str, + ) -> Result<nuncio_core::model::Placement, Status> { + let placements = self.db.placements_of(message_key).await.map_err(|e| { + Status::internal(format!( + "failed to read placements for '{message_key}': {e}" + )) + })?; + placements.into_iter().next().ok_or_else(|| { + errors::status_with_metadata( + ErrorReason::MessageNotFound, + format!("message '{message_key}' occupies no mailbox"), + [("message_id".to_string(), message_key.to_string())], + ) + }) + } +} + #[tonic::async_trait] impl Mail for MailGrpcService { async fn list_folders( @@ -1646,9 +1790,13 @@ impl Mail for MailGrpcService { _ => None, })?; + // The store scopes folders by account -- folder ids are not globally + // unique, every account has an INBOX -- while this RPC has no account + // field yet. Merging every account's page by folder id reproduces the + // pre-split behaviour exactly; narrowing it properly needs a wire + // change and belongs with the placement-aware surface. let (folders, next) = self - .db - .list_folders_page(after, page_size) + .merged_folders_page(after, page_size) .await .map_err(|e| Status::internal(format!("failed to list folders: {e}")))?; @@ -1672,19 +1820,31 @@ impl Mail for MailGrpcService { return Err(Status::invalid_argument("folder_id is required")); } let page_size = pagination::clamp_page_size(req.page_size); + // Four cursor fields, not two: the store's keyset addresses a + // placement (`received_at, message_key, uidvalidity, uid`), because one + // message can occupy a single folder twice and a message-only cursor + // makes the second copy unreachable across a page boundary. let after = pagination::parse_token(&req.page_token, |f| match f { - [CursorField::I(ts), CursorField::S(id)] => Some((*ts, id.clone())), + [CursorField::I(ts), CursorField::S(key), CursorField::S(uidvalidity), CursorField::S(uid)] => { + Some((*ts, key.clone(), uidvalidity.clone(), uid.clone())) + } _ => None, })?; let (messages, next) = self - .db - .list_messages_page(&req.folder_id, after, page_size) + .merged_messages_page(&req.folder_id, after, page_size) .await .map_err(|e| Status::internal(format!("failed to list messages: {e}")))?; let next_page_token = next - .map(|(ts, id)| pagination::encode_cursor(&[CursorField::I(ts), CursorField::S(id)])) + .map(|(ts, key, uidvalidity, uid)| { + pagination::encode_cursor(&[ + CursorField::I(ts), + CursorField::S(key), + CursorField::S(uidvalidity), + CursorField::S(uid), + ]) + }) .unwrap_or_default(); let messages = messages.into_iter().map(map_email_to_proto).collect(); @@ -1711,8 +1871,10 @@ impl Mail for MailGrpcService { ) })?; + let placement = self.primary_placement(&req.message_id).await?; + Ok(Response::new(GetMessageResponse { - message: Some(map_email_to_proto(email)), + message: Some(map_email_to_proto((email, placement))), })) } @@ -1733,8 +1895,13 @@ impl Mail for MailGrpcService { return Err(Status::invalid_argument("message_id is required")); } + // `\Seen` is per-mailbox, and this request names only a message, so it + // is applied to the occupancy `GetMessage` would report -- keeping the + // two RPCs consistent with each other. Naming a placement on the wire + // is what makes this well-defined, and lands with that surface. + let placement = self.primary_placement(&req.message_id).await?; self.db - .set_message_read(&req.message_id, req.read) + .set_placement_read(&placement.key(), req.read) .await .map_err(|e| { errors::status_with_metadata( @@ -1977,15 +2144,10 @@ fn synthetic_preview_email(id: &str) -> nuncio_core::model::Email { nuncio_core::model::Email { id: id.to_string(), account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - // Synthetic preview only -- never persisted or mutated remotely. - remote_id: id.to_string(), - uid_validity: String::new(), subject: "Test Subject".to_string(), sender: "test@nuncio.mx".to_string(), recipient: "me@nuncio.mx".to_string(), received_at, - read: false, body_plain: Some("Sample body text".to_string()), body_html: None, attachments: Vec::new(), @@ -1994,6 +2156,19 @@ fn synthetic_preview_email(id: &str) -> nuncio_core::model::Email { } } +/// The occupancy a synthetic preview message is pretended to sit in, so +/// `WHERE FOLDER = ...` and `WHERE ACCOUNT = ...` have something to answer +/// against. Synthetic only -- never persisted, never mutated remotely. +fn synthetic_preview_placement(id: &str) -> nuncio_core::model::Placement { + nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + } +} + /// `nuncio.v1.Filters` gRPC service implementation backed by the daemon's /// live [`DatabaseEngine`] filter-rule CRUD and the daemon's live /// [`FilterEngine`] -- the authenticated gRPC replacement for the existing @@ -2175,17 +2350,39 @@ impl Filters for FiltersGrpcService { let preview_engine = FilterEngine::new(vec![rule]) .map_err(|e| Status::internal(format!("failed to build preview engine: {e}")))?; - let email = match &req.message_id { + // A preview needs a placement as well as a message, because the rule + // language can ask where the message sits. A stored message is + // previewed against a real occupancy; a message that does not resolve + // falls back to the synthetic pair, exactly as before. + let (email, placement) = match &req.message_id { Some(message_id) if !message_id.is_empty() => { match self.db.get_message(message_id).await { - Ok(email) => email, - Err(_) => synthetic_preview_email(message_id), + Ok(email) => { + let placement = self + .db + .placements_of(message_id) + .await + .ok() + .and_then(|placements| placements.into_iter().next()) + .unwrap_or_else(|| synthetic_preview_placement(message_id)); + (email, placement) + } + Err(_) => ( + synthetic_preview_email(message_id), + synthetic_preview_placement(message_id), + ), } } - _ => synthetic_preview_email("msg-test"), + _ => ( + synthetic_preview_email("msg-test"), + synthetic_preview_placement("msg-test"), + ), }; - let preview = preview_engine.preview(&email); + let preview = preview_engine.preview(nuncio_filter::PlacedEmail { + email: &email, + placement: &placement, + }); Ok(Response::new(map_preview_result_to_proto(preview))) } @@ -2421,8 +2618,28 @@ impl Filters for FiltersGrpcService { let is_last_page = batch.len() < chunk_size; for email in &batch { - let actions_for_email = - crate::sync::apply_filter_actions(&db, &engine, email).await; + // Rules can ask where a message is, so a retroactive + // pass evaluates each mailbox the message occupies -- + // the same fan-out live sync does. The fire-once claim + // inside `apply_filter_actions` keeps a message in two + // folders from acting twice. + let placements = match db.placements_of(&email.id).await { + Ok(placements) => placements, + Err(e) => { + let _ = tx + .send(Err(Status::internal(format!( + "failed to read placements during triage: {e}" + )))) + .await; + return; + } + }; + let mut actions_for_email = 0usize; + for placement in &placements { + actions_for_email += + crate::sync::apply_filter_actions(&db, &engine, email, placement) + .await; + } scanned += 1; applied += actions_for_email as u64; if actions_for_email > 0 { @@ -3296,22 +3513,25 @@ mod tests { let email = nuncio_core::model::Email { id: "msg-1".to_string(), account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), subject: "s".to_string(), sender: "a@b.com".to_string(), recipient: "c@d.com".to_string(), received_at: original_secs, - read: false, body_plain: None, body_html: None, attachments: Vec::new(), message_id: None, content_hash: None, - remote_id: "remote-1".to_string(), + }; + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), uid_validity: "1".to_string(), + remote_id: "remote-1".to_string(), + read: false, }; - let proto = map_email_to_proto(email); + let proto = map_email_to_proto((email, placement)); let ts = proto.received_at.expect("received_at is always populated"); assert_eq!( nuncio_proto::time::timestamp_to_unix_secs(&ts), @@ -4482,29 +4702,72 @@ mod tests { SearchMessagesRequest, SendMessageRequest, SyncRequest, }; - fn sample_email( + /// A message and the one mailbox occupancy the tests seed it into. + fn sample_placed( id: &str, folder_id: &str, subject: &str, body: &str, - ) -> nuncio_core::model::Email { - nuncio_core::model::Email { - id: id.to_string(), - account_id: "acct-mail-1".to_string(), - folder_id: folder_id.to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), - subject: subject.to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some(body.to_string()), - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, + ) -> (nuncio_core::model::Email, nuncio_core::model::Placement) { + ( + nuncio_core::model::Email { + id: id.to_string(), + account_id: "acct-mail-1".to_string(), + subject: subject.to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some(body.to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::Placement { + account_id: "acct-mail-1".to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + }, + ) + } + + /// Persist a message into its occupancy the way a sync pass would, first + /// making sure the owning account row exists. + /// + /// The account row is not decoration: the `Mail` read RPCs carry no account + /// field and so merge across the configured accounts, which means mail + /// belonging to no account is unreachable through them. That is also true + /// in production -- mail only ever arrives for an account the daemon holds + /// a configuration for -- so seeding one keeps these tests on the real path. + async fn seed_message( + db: &DatabaseEngine, + email: &nuncio_core::model::Email, + placement: &nuncio_core::model::Placement, + ) { + let existing = db.list_accounts().await.expect("list accounts"); + if !existing.iter().any(|a| a.id == placement.account_id) { + db.save_account(&nuncio_core::AccountConfig { + id: placement.account_id.clone(), + name: placement.account_id.clone(), + email_address: format!("{}@nuncio.mx", placement.account_id), + keyring_secret_key: format!("nuncio/{}", placement.account_id), + sync_interval_secs: 60, + transport: nuncio_core::Transport::Jmap(nuncio_core::JmapTransport { + endpoint_host: "jmap.nuncio.mx".to_string(), + }), + }) + .await + .expect("save the owning account"); } + db.save_email_at( + email, + nuncio_core::model::IdentitySource::Surrogate, + placement, + ) + .await + .expect("seed message"); } /// Paging a folder larger than `page_size` across multiple pages via @@ -4528,9 +4791,9 @@ mod tests { ("m7", 1_700_000_030), ]; for (id, ts) in seed { - let mut email = sample_email(id, "inbox", "subject", "body"); + let (mut email, placement) = sample_placed(id, "inbox", "subject", "body"); email.received_at = ts; - db.save_email(&email).await.expect("seed message"); + seed_message(&db, &email, &placement).await; } let event_bus = Arc::new(EventBus::new()); @@ -4805,22 +5068,16 @@ mod tests { let (db, _dir) = DatabaseEngine::connect_ephemeral() .await .expect("connect ephemeral test db"); - db.save_email(&sample_email( + let (email_1, placement_1) = sample_placed( "msg-mail-1", "inbox", "Quarterly Roadmap", "Let's discuss the annual revenue forecast", - )) - .await - .expect("seed message 1"); - db.save_email(&sample_email( - "msg-mail-2", - "inbox", - "Lunch Plans", - "Sandwiches at noon", - )) - .await - .expect("seed message 2"); + ); + seed_message(&db, &email_1, &placement_1).await; + let (email_2, placement_2) = + sample_placed("msg-mail-2", "inbox", "Lunch Plans", "Sandwiches at noon"); + seed_message(&db, &email_2, &placement_2).await; let event_bus = Arc::new(EventBus::new()); let mut events = event_bus.subscribe_events(); @@ -5169,12 +5426,17 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock_backend.add_message(sample_email( + let (injected_email, injected_placement) = sample_placed( "msg-sync-injected-1", "inbox", "Injected Sync Subject", "Injected sync body", - )); + ); + mock_backend.add_message(nuncio_mail::PlacedMessage { + email: injected_email, + source: nuncio_core::model::IdentitySource::Surrogate, + placement: injected_placement, + }); let overrides = MailEngineOverrides { mail_backend: Some(Arc::new(mock_backend)), @@ -5535,6 +5797,42 @@ mod tests { /// not just re-reading it back over `ListRules`); `ListRules` reflects /// the persisted rule; `DeleteRule` removes it AND reloads the engine /// back to empty. + /// Evaluate `engine` against a fixed sample message in a fixed INBOX + /// occupancy. + /// + /// Evaluation needs both halves now, because rules can ask where a message + /// sits; these tests only care whether the live rule set matched at all, so + /// the occupancy is a constant. + fn evaluate_sample_subject( + engine: &FilterEngine, + subject: &str, + ) -> Vec<(nuncio_filter::FilterRule, Vec<nuncio_filter::RuleAction>)> { + let email = nuncio_core::model::Email { + id: "msg-filters-1".to_string(), + account_id: "acct-1".to_string(), + subject: subject.to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some("body".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }; + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: "1".to_string(), + read: false, + }; + engine.evaluate(nuncio_filter::PlacedEmail { + email: &email, + placement: &placement, + }) + } + #[tokio::test] async fn create_list_and_delete_rule_round_trip_and_keep_the_live_engine_in_sync() { let (addr, _handle, _db, filter_engine, _dir) = @@ -5544,28 +5842,8 @@ mod tests { .await .expect("client connects"); - let sample_email = |subject: &str| nuncio_core::model::Email { - id: "msg-filters-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: subject.to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some("body".to_string()), - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, - }; - // Before creating anything, the live engine matches nothing. - assert!(filter_engine - .evaluate(&sample_email("Urgent Meeting")) - .is_empty()); + assert!(evaluate_sample_subject(&filter_engine, "Urgent Meeting").is_empty()); let create_response = client .create_rule(authed_bearer_request(CreateRuleRequest { @@ -5587,7 +5865,7 @@ mod tests { // The live `FilterEngine`'s `ArcSwap` rule set was reloaded -- // proven by evaluating the SAME instance the server holds, not a // fresh one. - let matches = filter_engine.evaluate(&sample_email("Urgent Meeting")); + let matches = evaluate_sample_subject(&filter_engine, "Urgent Meeting"); assert_eq!(matches.len(), 1); assert_eq!(matches[0].0.id, rule_id); @@ -5620,9 +5898,7 @@ mod tests { .expect("list_rules succeeds") .into_inner(); assert!(list_after_delete.rules.is_empty()); - assert!(filter_engine - .evaluate(&sample_email("Urgent Meeting")) - .is_empty()); + assert!(evaluate_sample_subject(&filter_engine, "Urgent Meeting").is_empty()); } #[tokio::test] @@ -5656,25 +5932,7 @@ mod tests { list_response.rules.is_empty(), "an invalid rule must never be persisted" ); - assert!(filter_engine - .evaluate(&nuncio_core::model::Email { - id: "msg-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: "anything".to_string(), - sender: "a@b.com".to_string(), - recipient: "c@d.com".to_string(), - received_at: 0, - read: false, - body_plain: None, - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, - }) - .is_empty()); + assert!(evaluate_sample_subject(&filter_engine, "anything").is_empty()); } /// Two `CreateRule` calls with the SAME name AND the SAME NSQL text @@ -5809,23 +6067,29 @@ mod tests { async fn preview_rule_dry_runs_against_a_stored_message_without_persisting() { let (addr, _handle, db, _engine, _dir) = spawn_filters_test_server("correct-token").await; - db.save_email(&nuncio_core::model::Email { - id: "msg-preview-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "preview-1".to_string(), - uid_validity: "1".to_string(), - subject: "Urgent Board Meeting".to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some("Please review the attached deck".to_string()), - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, - }) + db.save_email_at( + &nuncio_core::model::Email { + id: "msg-preview-1".to_string(), + account_id: "acct-1".to_string(), + subject: "Urgent Board Meeting".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some("Please review the attached deck".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::IdentitySource::Surrogate, + &nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: "preview-1".to_string(), + read: false, + }, + ) .await .expect("seed message"); @@ -5985,23 +6249,7 @@ mod tests { assert_eq!(list_response.rules[0].name, "Renamed"); // The live `FilterEngine` was reloaded with the updated rule. - let matches = filter_engine.evaluate(&nuncio_core::model::Email { - id: "msg-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: "Urgent Meeting".to_string(), - sender: "a@b.com".to_string(), - recipient: "c@d.com".to_string(), - received_at: 0, - read: false, - body_plain: None, - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, - }); + let matches = evaluate_sample_subject(&filter_engine, "Urgent Meeting"); assert_eq!(matches.len(), 1); assert_eq!(matches[0].0.id, rule_id); } @@ -6168,23 +6416,7 @@ mod tests { assert_eq!(list_response.rules[0].nsql_text, SAMPLE_RULE_NSQL); // The live engine was reloaded to include the imported rule. - let matches = filter_engine.evaluate(&nuncio_core::model::Email { - id: "msg-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: "Urgent Meeting".to_string(), - sender: "a@b.com".to_string(), - recipient: "c@d.com".to_string(), - received_at: 0, - read: false, - body_plain: None, - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, - }); + let matches = evaluate_sample_subject(&filter_engine, "Urgent Meeting"); assert_eq!(matches.len(), 1); } @@ -6290,22 +6522,20 @@ mod tests { let (db, _dir) = DatabaseEngine::connect_ephemeral() .await .expect("connect ephemeral test db"); - db.save_email(&sample_email( + let (export_1, export_1_at) = sample_placed( "msg-export-1", "inbox", "Export Subject One", "Export body one", - )) - .await - .expect("seed message 1"); - db.save_email(&sample_email( + ); + seed_message(&db, &export_1, &export_1_at).await; + let (export_2, export_2_at) = sample_placed( "msg-export-2", "archive", "Export Subject Two", "Export body two", - )) - .await - .expect("seed message 2"); + ); + seed_message(&db, &export_2, &export_2_at).await; let event_bus = Arc::new(EventBus::new()); let secrets = Arc::new(SecretManager::mock()); @@ -6355,12 +6585,16 @@ mod tests { let (db, _dir) = DatabaseEngine::connect_ephemeral() .await .expect("connect ephemeral test db"); - let mut msg_a = sample_email("msg-acct-a", "inbox", "From Account A", "Body A"); + let (mut msg_a, mut at_a) = + sample_placed("msg-acct-a", "inbox", "From Account A", "Body A"); msg_a.account_id = "acct-a".to_string(); - db.save_email(&msg_a).await.expect("seed account a message"); - let mut msg_b = sample_email("msg-acct-b", "inbox", "From Account B", "Body B"); + at_a.account_id = "acct-a".to_string(); + seed_message(&db, &msg_a, &at_a).await; + let (mut msg_b, mut at_b) = + sample_placed("msg-acct-b", "inbox", "From Account B", "Body B"); msg_b.account_id = "acct-b".to_string(); - db.save_email(&msg_b).await.expect("seed account b message"); + at_b.account_id = "acct-b".to_string(); + seed_message(&db, &msg_b, &at_b).await; let event_bus = Arc::new(EventBus::new()); let secrets = Arc::new(SecretManager::mock()); diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 40428b0..f5ef168 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -272,7 +272,7 @@ async fn execute_one( match mutation.mutation_type.as_str() { "MOVE" | "COPY" | "FLAG" | "UNFLAG" | "DELETE" => { - dispatch_mailbox_mutation(env, mutation, &email, &payload).await + dispatch_mailbox_mutation(db, env, mutation, &email, &payload).await } "FORWARD" => dispatch_forward(db, env, &email, &payload).await, "WEBHOOK" => dispatch_webhook(env, mutation, &email, &payload).await, @@ -281,10 +281,20 @@ async fn execute_one( } /// Execute a mailbox-affecting mutation (move/copy/flag/unflag/delete) against -/// the account's mail backend, recovering the remote addressing (protocol id + -/// UIDVALIDITY scope) from the stored message row so the backend can enforce -/// its UIDVALIDITY guard. +/// the account's mail backend, recovering the remote addressing (folder, +/// protocol id, UIDVALIDITY scope) from the message's stored placement so the +/// backend can enforce its UIDVALIDITY guard. +/// +/// The addressing now comes from a placement rather than the message row, +/// because those coordinates are properties of a mailbox occupancy and a +/// message can hold several. The outbox row records only the message key, so +/// which occupancy the rule was evaluated against is not recoverable here: this +/// addresses the first placement in the store's deterministic order and says so +/// in the log when there is more than one. Carrying the originating placement on +/// the outbox row is the real fix and belongs with the placement-aware wire +/// surface, not here. async fn dispatch_mailbox_mutation( + db: &DatabaseEngine, env: &dyn RemoteExecutionEnv, mutation: &PendingRemoteMutation, email: &nuncio_core::model::Email, @@ -313,20 +323,44 @@ async fn dispatch_mailbox_mutation( other => return Disposition::Permanent(format!("unexpected mailbox mutation '{other}'")), }; - let backend = match env.mail_backend(&email.account_id).await { + // Recover the remote addressing from a stored placement rather than + // parsing it out of the opaque message key. The backend still + // SELECT-verifies the UIDVALIDITY before mutating. + let placements = match db.placements_of(&email.id).await { + Ok(placements) => placements, + Err(e) => { + return Disposition::Retry(format!("failed to resolve the message's placements: {e}")) + } + }; + let Some(placement) = placements.first() else { + // No occupancy means no way to address the message on any server, and + // no retry can conjure one back. + return Disposition::Permanent(format!( + "message '{}' occupies no mailbox to address", + email.id + )); + }; + if placements.len() > 1 { + tracing::warn!( + mutation_id = %mutation.id, + message_id = %email.id, + folder_id = %placement.folder_id, + placement_count = placements.len(), + "outbox: message occupies several mailboxes and the row does not record which one \ + the rule matched; addressing the first" + ); + } + + let backend = match env.mail_backend(&placement.account_id).await { Ok(backend) => backend, Err(e) => return Disposition::Retry(format!("failed to build mail backend: {e}")), }; - // Recover the remote addressing from the stored message row -- the - // protocol-native id and the UIDVALIDITY scope it was captured under -- - // rather than parsing it out of the opaque surrogate id. The backend still - // SELECT-verifies the UIDVALIDITY before mutating. let spec = RemoteMutationSpec { message_id: email.id.clone(), - remote_id: email.remote_id.clone(), - folder_id: email.folder_id.clone(), - uid_validity: email.uid_validity.clone(), + remote_id: placement.remote_id.clone(), + folder_id: placement.folder_id.clone(), + uid_validity: placement.uid_validity.clone(), kind, }; @@ -563,7 +597,7 @@ mod tests { use super::*; use crate::lifecycle::{ShutdownController, ShutdownSignal}; use crate::test_tracing::with_recorder; - use nuncio_core::model::Email; + use nuncio_core::model::{Email, IdentitySource, Placement}; use nuncio_core::EventBus; use nuncio_filter::MutationPayload; use nuncio_mail::MockMailBackend; @@ -617,14 +651,10 @@ mod tests { Email { id: "msg-outbox-log".to_string(), account_id: "acct-outbox-log".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "77".to_string(), - uid_validity: "9".to_string(), subject: "Confidential outbox subject".to_string(), sender: "alice@example.com".to_string(), recipient: "owner@nuncio.mx".to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some("secret outbox body".to_string()), body_html: None, attachments: Vec::new(), @@ -633,6 +663,18 @@ mod tests { } } + /// The one mailbox occupancy `sample_email` is stored in -- the addressing + /// the outbox recovers to build its `RemoteMutationSpec`. + fn sample_placement() -> Placement { + Placement { + account_id: "acct-outbox-log".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "9".to_string(), + remote_id: "77".to_string(), + read: false, + } + } + fn flag_mutation() -> PendingRemoteMutation { PendingRemoteMutation { id: "mut-1".to_string(), @@ -668,7 +710,13 @@ mod tests { db.save_folder_sync_state("acct-outbox-log", "INBOX", "9:100") .await .expect("save folder checkpoint"); - db.save_email(&sample_email()).await.expect("save email"); + db.save_email_at( + &sample_email(), + IdentitySource::Surrogate, + &sample_placement(), + ) + .await + .expect("save email"); db.save_pending_mutation(&flag_mutation()) .await .expect("save pending mutation"); diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index bdbdfee..a5998b3 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -7,7 +7,8 @@ //! account it connects a real [`MailBackend`] (a real [`ImapEngine`] for an //! IMAP/SMTP transport, built from [`AccountConfig`] plus the //! account's keyring-stored password), fetches folders and messages, and -//! persists every message through [`DatabaseEngine::save_email`]. +//! persists every message, together with the mailbox occupancy it was found +//! in, through [`DatabaseEngine::save_email_at`]. //! //! # Testability //! @@ -20,6 +21,7 @@ //! all-accounts sync fans these per-account runs out through //! [`crate::sync_dispatcher::sync_all_configured`], so concurrency and the //! per-account timeout are bounded in one place rather than serialized here. +use nuncio_core::model::PlacementKey; use nuncio_core::{AccountConfig, CoreCommand, CoreEvent, EventBus, Transport}; use nuncio_filter::{FilterEngine, OutboxManager, RuleAction}; use nuncio_mail::{ImapEngine, JmapEngine, MailBackend, MailError}; @@ -68,24 +70,32 @@ fn remote_action_tag(action: &RuleAction) -> Option<(&'static str, Option<String } } -/// Evaluate `email` against `filter_engine` and route every matched rule's -/// actions to their real, persisted effect. +/// Evaluate `email` **as it sits in `placement`** against `filter_engine` and +/// route every matched rule's actions to their real, persisted effect. /// -/// Filters fire EXACTLY ONCE per newly-arrived message: callers (see -/// [`fetch_and_persist`]) must only invoke this for a message that was -/// genuinely new to the store on this sync pass, never for one that already -/// existed. Sync is not yet incremental -- every sync re-fetches and -/// re-persists messages the backend still reports -- so calling this -/// unconditionally on every persisted message would re-fire a rule's actions -/// on every single re-sync of the same mailbox: a fresh -/// `PendingRemoteMutation` and a fresh `FilterExecutionLog` entry per cycle -/// for a message that arrived once, growing the outbox and the audit ledger -/// without bound and (once a real mutation transport lands) duplicating the -/// real remote operation. This function itself has no way to tell "new" from -/// "already seen" -- that determination is the caller's responsibility. +/// Evaluation is per placement because the rule language can ask where a +/// message is: `WHERE FOLDER = 'INBOX'` has no answer for a message that +/// occupies three mailboxes, so the caller fans out over the occupancies and +/// each call answers for exactly one. /// -/// `MarkRead`/`MarkUnread` apply immediately to the stored message via -/// [`DatabaseEngine::set_message_read`]. Every other action enqueues a real +/// Acting, in contrast, is once per (rule, message). Every matched rule must +/// win [`DatabaseEngine::claim_filter_fire`] before any of its actions run, and +/// the claim is keyed on message identity, not on the placement. `MOVE` and +/// `FLAG` converge under repetition -- ten moves leave one net effect -- but +/// `FORWARD` and `CALL WEBHOOK` land on third parties and have no shared state +/// to converge against: one message in ten folders would page an on-call +/// engineer ten times. The claim fails closed, so an unverifiable claim skips +/// the actions rather than licensing them. +/// +/// The claim also subsumes the older "only call me for a new message" contract +/// for re-arrivals of the same message. Callers should still gate on a genuinely +/// new occupancy (see [`fetch_and_persist`]) -- the claim stops repeat *actions*, +/// not the wasted evaluation of a mailbox nothing changed in. +/// +/// `MarkRead`/`MarkUnread` apply immediately to the occupancy that was +/// evaluated, via [`DatabaseEngine::set_placement_read`]: IMAP `\Seen` is +/// per-mailbox, so "mark this read" is only well-defined once it names a +/// mailbox. Every other action enqueues a real /// [`nuncio_filter::PendingRemoteMutation`] through the existing outbox /// (`OutboxManager::create_mutation` + `DatabaseEngine::save_pending_mutation`) /// for the background outbox worker to pick up -- this function never @@ -104,40 +114,69 @@ fn remote_action_tag(action: &RuleAction) -> Option<(&'static str, Option<String /// /// Standalone and reusable by design: this is the exact evaluate-and-route /// logic other sync paths (e.g. a bulk retroactive triage over the whole -/// store) need, so it takes only a `DatabaseEngine`/`FilterEngine`/`Email` -/// and has no dependency on the live inbound-sync call chain. +/// store) need, so it takes only a `DatabaseEngine`/`FilterEngine`/`Email`/ +/// `Placement` and has no dependency on the live inbound-sync call chain. pub async fn apply_filter_actions( db: &nuncio_store::db::DatabaseEngine, filter_engine: &FilterEngine, email: &nuncio_core::model::Email, + placement: &nuncio_core::model::Placement, ) -> usize { let mut applied = 0usize; + let placed = nuncio_filter::PlacedEmail { email, placement }; - for (rule, actions) in filter_engine.evaluate(email) { + for (rule, actions) in filter_engine.evaluate(placed) { + // Claim before acting, never after. The claim is keyed on message + // identity, so the same mail reaching a second mailbox -- an ordinary + // MOVE, or a Gmail label -- evaluates again but acts only once. + match db.claim_filter_fire(&rule.id, &email.id).await { + Ok(true) => {} + Ok(false) => { + tracing::debug!( + rule_id = %rule.id, + message_key = %email.id, + folder_id = %placement.folder_id, + "rule already fired for this message; skipping actions for this placement" + ); + continue; + } + Err(e) => { + // Fail closed. An unverifiable claim must not become a licence + // to act: a lost claim check on a FORWARD is a duplicate mail + // the recipient cannot un-receive. + tracing::error!( + rule_id = %rule.id, + message_key = %email.id, + error = %e, + "could not claim the filter fire; skipping actions" + ); + continue; + } + } + + let placement_key = placement.key(); for action in actions { let immediate_ok = match &action { - RuleAction::MarkRead => match db.set_message_read(&email.id, true).await { - Ok(()) => true, - Err(e) => { - tracing::warn!( - "filter rule '{}' MARK READ failed for message '{}': {e}", - rule.id, - email.id - ); - false + // One write with a boolean rather than two near-identical + // arms. Both name the occupancy that was evaluated, because + // `\Seen` is per-mailbox: a message read in INBOX is not + // thereby read in Archive. + RuleAction::MarkRead | RuleAction::MarkUnread => { + let read = matches!(action, RuleAction::MarkRead); + let label = if read { "MARK READ" } else { "MARK UNREAD" }; + match db.set_placement_read(&placement_key, read).await { + Ok(()) => true, + Err(e) => { + tracing::warn!( + "filter rule '{}' {label} failed for message '{}' in folder '{}': {e}", + rule.id, + email.id, + placement.folder_id + ); + false + } } - }, - RuleAction::MarkUnread => match db.set_message_read(&email.id, false).await { - Ok(()) => true, - Err(e) => { - tracing::warn!( - "filter rule '{}' MARK UNREAD failed for message '{}': {e}", - rule.id, - email.id - ); - false - } - }, + } RuleAction::MoveTo(_) | RuleAction::CopyTo(_) | RuleAction::Flag @@ -197,31 +236,42 @@ pub async fn apply_filter_actions( applied } -/// Fetch every folder and every message in every folder from `backend`, -/// persisting each message via [`DatabaseEngine::save_email`] and, for -/// messages genuinely new to the store, evaluating it against -/// `filter_engine` via [`apply_filter_actions`]. +/// Fetch every folder from `backend`, persist each reported occupancy via +/// [`DatabaseEngine::save_email_at`], evaluate the genuinely-new occupancies +/// against `filter_engine` through [`apply_filter_actions`], and delete the +/// occupancies the server no longer reports. /// -/// Sync is not yet incremental (a re-sync re-fetches every message the -/// backend still reports, not just genuinely new ones), so new-vs-seen must -/// be determined BEFORE `save_email` persists (or re-persists) anything -- -/// `save_email` is `INSERT OR REPLACE` and would otherwise erase the -/// distinction between "arriving for the first time" and "seen again". Doing -/// that with one existence lookup per message (`DatabaseEngine::get_message`) -/// is an N+1 query pattern over a fetched chunk; instead, this classifies an -/// entire folder's chunk with a single batched -/// `DatabaseEngine::existing_message_ids` lookup up front, then folds in a -/// per-batch "already seen this pass" set while iterating so a chunk that -/// itself contains a repeated id (e.g. a backend surfacing the same message -/// twice in one fetch) still fires at most once -- exactly as if each -/// message's existence had been checked one at a time immediately before its -/// own `save_email`. +/// # Why classification is per placement, not per message /// -/// Returns the total number of messages processed (a message id "processed" -/// more than once, e.g. by a backend that returns the same id from multiple -/// folders, is counted once per occurrence -- `save_email` itself is -/// `INSERT OR REPLACE`, so storage stays deduplicated by message id -/// regardless). +/// Message identity is folder-independent: a message already stored from +/// `INBOX` has the same key when it turns up in `Archive`. Asking "have I seen +/// this *message*?" therefore answers `yes` for a mailbox this daemon has never +/// filtered, and the arrival silently fires nothing -- no error, no log. The +/// question that matches the intent is "have I seen this *occupancy*?", so this +/// classifies on [`PlacementKey`] via +/// [`DatabaseEngine::existing_placements`]. +/// +/// Classification happens BEFORE the write, because `save_email_at` upserts and +/// would otherwise erase the distinction between a first arrival and a +/// re-report. One batched lookup covers the whole chunk rather than an N+1 of +/// per-row existence checks, and a per-pass "already seen" set folds in on top +/// so a backend that surfaces the same occupancy twice in one fetch still +/// classifies it as new exactly once. +/// +/// Firing once per *arrival* is not the same as acting once per *message*; the +/// second guarantee lives in the claim inside [`apply_filter_actions`]. +/// +/// # Absence +/// +/// [`nuncio_mail::FolderChanges::present`] is honoured as written: `None` means +/// the pass was incremental and cannot speak to absence, and treating it as +/// "nothing is present" would delete the whole folder. Only `Some(list)` -- +/// which `nuncio-mail` withholds when a SELECT omitted UIDVALIDITY, since the +/// keys it could build would name no stored row -- licenses deleting the stored +/// placements missing from it. +/// +/// Returns the total number of occupancies processed: one message reported from +/// two folders counts twice, because two mailboxes really did report it. /// /// Contains no event-bus or credential logic -- pure fetch-and-persist, so /// it is the smallest unit tests can exercise directly with a @@ -248,25 +298,29 @@ async fn fetch_and_persist( let changes = backend .sync_changes(&folder.id, last_state.as_deref()) .await?; - let emails = changes.upserts; - let fetched = emails.len(); + let placed = changes.upserts; + let fetched = placed.len(); let new_state = changes.next_state; // One round trip classifies the whole chunk instead of one lookup per - // message. A lookup failure that is not "not found" would be a - // genuine DB error; `existing_message_ids` surfaces those as `Err` - // rather than silently treating the batch as "not new", so callers - // never persist and skip-filter on an unverified assumption. - let chunk_ids: Vec<String> = emails.iter().map(|e| e.id.clone()).collect(); - let already_present = db.existing_message_ids(&chunk_ids).await?; + // occupancy. This asks about *placements*, not messages: identity is + // folder-independent, so a message already stored from another folder + // is still a first arrival here, and testing the message key instead + // would classify it as seen and skip filtering it entirely. A lookup + // failure that is not "not found" surfaces as `Err` rather than + // silently reading as "not new", so nothing is persisted and + // skip-filtered on an unverified assumption. + let chunk_keys: Vec<PlacementKey> = placed.iter().map(|p| p.placement.key()).collect(); + let already_present = db.existing_placements(&chunk_keys).await?; let mut seen_this_pass = std::collections::HashSet::new(); - for email in emails { - let is_new = - !already_present.contains(&email.id) && seen_this_pass.insert(email.id.clone()); - db.save_email(&email).await?; + for message in placed { + let key = message.placement.key(); + let is_new = !already_present.contains(&key) && seen_this_pass.insert(key); + db.save_email_at(&message.email, message.source, &message.placement) + .await?; if is_new { - apply_filter_actions(db, filter_engine, &email).await; + apply_filter_actions(db, filter_engine, &message.email, &message.placement).await; } synced += 1; } @@ -274,8 +328,12 @@ async fn fetch_and_persist( // could only ever add: a message moved or deleted by any other client // -- another daemon, a phone, webmail -- stayed in the local store // forever, and every account accumulated ghosts. + // + // What a folder stops mentioning is an occupancy, never a message: the + // same mail may still sit in another mailbox, and it goes only when its + // last placement does -- which `delete_placements` decides, not this. if let Some(acct) = account_id { - let mut gone: Vec<String> = changes.removals; + let mut gone: Vec<PlacementKey> = changes.removals; // `present` is the folder's complete contents when the pass was // able to enumerate them. `None` and `Some(vec![])` are NOT the @@ -283,20 +341,22 @@ async fn fetch_and_persist( // absence, and treating it as "nothing is present" would delete // the folder. if let Some(present) = changes.present { - let present: std::collections::HashSet<String> = present.into_iter().collect(); - let stored = db.message_ids_in_folder(acct, &folder.id).await?; - gone.extend(stored.into_iter().filter(|id| !present.contains(id))); + let present: std::collections::HashSet<PlacementKey> = + present.into_iter().collect(); + let stored = db.placements_in_folder(acct, &folder.id).await?; + gone.extend(stored.into_iter().filter(|key| !present.contains(key))); } gone.sort_unstable(); gone.dedup(); if !gone.is_empty() { - let deleted = db.delete_messages(&gone).await?; + let outcome = db.delete_placements(&gone).await?; tracing::info!( account_id = %acct, folder_id = %folder.id, - deleted, - "removed messages that are no longer on the server" + placements_removed = outcome.placements_removed, + messages_reaped = outcome.messages_reaped, + "removed placements that are no longer on the server" ); } } @@ -446,9 +506,9 @@ pub async fn run_account_sync( #[cfg(test)] mod tests { use super::*; - use nuncio_core::model::{Email, Folder}; + use nuncio_core::model::{Email, Folder, IdentitySource, Placement}; use nuncio_core::{CoreEvent, EngineStatus, TlsMode}; - use nuncio_mail::MockMailBackend; + use nuncio_mail::{MockMailBackend, PlacedMessage}; use nuncio_store::db::DatabaseEngine; use serde_json::json; use wiremock::matchers::{body_partial_json, method, path}; @@ -568,26 +628,58 @@ mod tests { } } - fn mock_email(id: &str, folder_id: &str, subject: &str) -> Email { - Email { - id: id.to_string(), - account_id: "acct-mock-1".to_string(), - folder_id: folder_id.to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), - subject: subject.to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some(format!("body for {id}")), - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, + /// One mock message as a backend would surface it: identity plus the + /// occupancy it was found in. `id` doubles as the folder-scoped UID, which + /// keeps distinct ids in one folder distinct occupancies. + fn mock_placed(id: &str, folder_id: &str, subject: &str) -> PlacedMessage { + PlacedMessage { + email: Email { + id: id.to_string(), + account_id: "acct-mock-1".to_string(), + subject: subject.to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some(format!("body for {id}")), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + source: IdentitySource::Surrogate, + placement: Placement { + account_id: "acct-mock-1".to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + }, } } + /// The read flag of a message's one occupancy, for the single-placement + /// mock fixtures. Fails loudly rather than defaulting if the message turns + /// out to sit in no mailbox at all -- "unread" and "not there" are + /// different answers and must not be conflated by a test helper. + async fn read_flag_of(db: &DatabaseEngine, message_key: &str) -> bool { + let placements = db + .placements_of(message_key) + .await + .expect("read the message's placements"); + assert_eq!( + placements.len(), + 1, + "fixture expects exactly one occupancy of '{message_key}'" + ); + placements[0].read + } + + /// The message key every [`MockMailBackend::with_same_message_in`] + /// occupancy carries. + fn shared_key() -> &'static str { + MockMailBackend::SHARED_MESSAGE_KEY + } + #[tokio::test] async fn sync_with_backend_persists_mock_messages_and_emits_events() { let (db, _dir) = DatabaseEngine::connect_ephemeral() @@ -603,8 +695,8 @@ mod tests { total_messages: 2, unread_messages: 2, }); - mock.add_message(mock_email("m1", "inbox", "Hello")); - mock.add_message(mock_email("m2", "inbox", "World")); + mock.add_message(mock_placed("m1", "inbox", "Hello")); + mock.add_message(mock_placed("m2", "inbox", "World")); let filter_engine = empty_filter_engine(); let synced = sync_with_backend( @@ -619,11 +711,14 @@ mod tests { assert_eq!(synced, 2); let persisted = db - .list_messages("inbox", 10) + .list_messages("acct-mock-1", "inbox", 10) .await .expect("list persisted messages"); assert_eq!(persisted.len(), 2); - let subjects: Vec<String> = persisted.iter().map(|e| e.subject.clone()).collect(); + let subjects: Vec<String> = persisted + .iter() + .map(|(email, _placement)| email.subject.clone()) + .collect(); assert!(subjects.contains(&"Hello".to_string())); assert!(subjects.contains(&"World".to_string())); @@ -817,12 +912,13 @@ mod tests { // folder, and its protocol-native JMAP object id round-trips in // `remote_id`. let persisted = db - .list_messages("inbox", 10) + .list_messages("acct-jmap-real-1", "inbox", 10) .await .expect("jmap message persisted"); assert_eq!(persisted.len(), 1); - assert_eq!(persisted[0].subject, "Welcome to JMAP Sync"); - assert_eq!(persisted[0].remote_id, "jmap-msg-1"); + let (email, placement) = &persisted[0]; + assert_eq!(email.subject, "Welcome to JMAP Sync"); + assert_eq!(placement.remote_id, "jmap-msg-1"); assert_eq!( events.recv().await.expect("start event"), @@ -929,17 +1025,16 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock.add_message(mock_email("m-urgent", "inbox", "Urgent: server down")); + mock.add_message(mock_placed("m-urgent", "inbox", "Urgent: server down")); let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) .await .expect("sync succeeds"); assert_eq!(synced, 1); - let stored = db.get_message("m-urgent").await.expect("message persisted"); assert!( - stored.read, - "MARK READ action must genuinely flip the stored read flag" + read_flag_of(&db, "m-urgent").await, + "MARK READ action must genuinely flip the occupancy's read flag" ); let logs = db @@ -981,7 +1076,7 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock.add_message(mock_email("m-archive", "inbox", "Please Archive Me")); + mock.add_message(mock_placed("m-archive", "inbox", "Please Archive Me")); let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) .await @@ -1006,11 +1101,7 @@ mod tests { assert_eq!(logs[0].message_id, "m-archive"); // The remote action must not have applied a local read-flag change. - let stored = db - .get_message("m-archive") - .await - .expect("message persisted"); - assert!(!stored.read); + assert!(!read_flag_of(&db, "m-archive").await); } #[tokio::test] @@ -1035,15 +1126,17 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock.add_message(mock_email("m-plain", "inbox", "Just a normal update")); + mock.add_message(mock_placed("m-plain", "inbox", "Just a normal update")); let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) .await .expect("sync succeeds"); assert_eq!(synced, 1); - let stored = db.get_message("m-plain").await.expect("message persisted"); - assert!(!stored.read, "non-matching message must stay untouched"); + assert!( + !read_flag_of(&db, "m-plain").await, + "non-matching message must stay untouched" + ); let logs = db .list_filter_execution_logs(10) @@ -1094,7 +1187,9 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock.add_message(mock_email("m-repeat", "inbox", "Urgent: server down")); + let repeat = mock_placed("m-repeat", "inbox", "Urgent: server down"); + let repeat_key = repeat.placement.key(); + mock.add_message(repeat); // First sync: the message is genuinely new, so both rules must fire. let synced_first = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) @@ -1102,11 +1197,7 @@ mod tests { .expect("first sync succeeds"); assert_eq!(synced_first, 1); - let stored_after_first = db - .get_message("m-repeat") - .await - .expect("message persisted after first sync"); - assert!(stored_after_first.read); + assert!(read_flag_of(&db, "m-repeat").await); let logs_after_first = db .list_filter_execution_logs(10) @@ -1123,7 +1214,7 @@ mod tests { // Manually flip the message back to unread, exactly as a user // reading and then un-reading it would -- proves the second sync // doesn't re-flip it back to read via a repeat MARK READ fire. - db.set_message_read("m-repeat", false) + db.set_placement_read(&repeat_key, false) .await .expect("manually mark unread"); @@ -1134,12 +1225,8 @@ mod tests { .expect("second sync succeeds"); assert_eq!(synced_second, 1); - let stored_after_second = db - .get_message("m-repeat") - .await - .expect("message persisted after second sync"); assert!( - !stored_after_second.read, + !read_flag_of(&db, "m-repeat").await, "a re-sync of an already-seen message must not re-fire MARK READ" ); @@ -1166,10 +1253,10 @@ mod tests { #[tokio::test] async fn sync_with_backend_fires_filters_only_for_new_messages_in_a_mixed_batch() { - // A single fetched chunk containing a mix of an already-seen message - // (persisted by a prior sync) and genuinely new ones must fire filter - // actions ONLY for the new ones, via the batched - // `existing_message_ids` classification -- not a per-message lookup. + // A single fetched chunk containing a mix of an already-stored + // occupancy (persisted by a prior sync) and genuinely new ones must + // fire filter actions ONLY for the new ones, via the batched + // `existing_placements` classification -- not a per-row lookup. let (db, _dir) = DatabaseEngine::connect_ephemeral() .await .expect("ephemeral db"); @@ -1183,16 +1270,13 @@ mod tests { .expect("parse rule"); let filter_engine = FilterEngine::new(vec![mark_read_rule]).expect("compile rule"); - // Pre-seed one message directly, simulating it having landed in a + // Pre-seed one occupancy directly, simulating it having landed in a // prior sync pass. It must NOT re-fire even though this pass's // backend still reports it (sync is not yet incremental). - db.save_email(&mock_email( - "m-already-seen", - "inbox", - "Urgent: known issue", - )) - .await - .expect("pre-seed existing message"); + let seeded = mock_placed("m-already-seen", "inbox", "Urgent: known issue"); + db.save_email_at(&seeded.email, seeded.source, &seeded.placement) + .await + .expect("pre-seed existing occupancy"); let mock = MockMailBackend::new(); mock.add_folder(Folder { @@ -1202,9 +1286,13 @@ mod tests { unread_messages: 3, }); // Mixed chunk: one already-present id, two genuinely new ids. - mock.add_message(mock_email("m-already-seen", "inbox", "Urgent: known issue")); - mock.add_message(mock_email("m-new-1", "inbox", "Urgent: brand new")); - mock.add_message(mock_email("m-new-2", "inbox", "Urgent: also new")); + mock.add_message(mock_placed( + "m-already-seen", + "inbox", + "Urgent: known issue", + )); + mock.add_message(mock_placed("m-new-1", "inbox", "Urgent: brand new")); + mock.add_message(mock_placed("m-new-2", "inbox", "Urgent: also new")); let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) .await @@ -1226,16 +1314,12 @@ mod tests { assert!(fired_ids.contains("m-new-2")); assert!(!fired_ids.contains("m-already-seen")); - // The pre-seeded message must still be persisted (INSERT OR REPLACE - // semantics are unaffected by the batched new/seen classification) - // but its read flag must be untouched by this pass's MARK READ rule. - let already_seen = db - .get_message("m-already-seen") - .await - .expect("still persisted"); + // The pre-seeded occupancy must still be persisted (the upsert is + // unaffected by the batched new/seen classification) but its read flag + // must be untouched by this pass's MARK READ rule. assert!( - !already_seen.read, - "a message that already existed before this sync must not re-fire MARK READ" + !read_flag_of(&db, "m-already-seen").await, + "an occupancy that already existed before this sync must not re-fire MARK READ" ); } @@ -1258,7 +1342,7 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock.add_message(mock_email("m-inc-1", "inbox", "First")); + mock.add_message(mock_placed("m-inc-1", "inbox", "First")); let filter_engine = empty_filter_engine(); @@ -1303,4 +1387,180 @@ mod tests { vec![None, Some("uidnext-4242".to_string())], ); } + + /// A rule that matches the one message the shared-identity mock stages. + fn rule_matching_the_shared_message() -> nuncio_filter::FilterRule { + nuncio_filter::NsqlParser::parse_rule( + "Archive Shared", + 1, + "WHERE subject CONTAINS 'Shared' ACTION MOVE TO 'Archive'", + ) + .expect("parse rule") + } + + #[tokio::test] + async fn a_message_arriving_in_a_second_folder_still_gets_evaluated() { + // The regression this guards: with identity now folder-independent, a + // message already stored from INBOX has a known message_key, so a + // message-scoped "already present" test would classify its arrival in + // Archive as old and skip filtering it entirely -- silently, with no + // error and nothing in the log. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let event_bus = EventBus::new(); + + let rule = rule_matching_the_shared_message(); + let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule"); + + // Already stored, from INBOX. The message key is therefore known + // before this sync begins -- but nothing has ever filtered Archive. + let in_inbox = MockMailBackend::shared_message_in("INBOX", "1"); + db.save_email_at(&in_inbox.email, in_inbox.source, &in_inbox.placement) + .await + .expect("pre-seed the INBOX occupancy"); + + let backend = MockMailBackend::with_same_message_in(&["Archive"]); + let processed = fetch_and_persist( + &db, + &event_bus, + &backend, + &filter_engine, + Some(MockMailBackend::SHARED_ACCOUNT_ID), + ) + .await + .expect("sync succeeds"); + + assert_eq!(processed, 1, "the Archive occupancy is processed"); + let placements = db + .placements_of(shared_key()) + .await + .expect("read the message's placements"); + assert_eq!( + placements.len(), + 2, + "one message, now occupying both mailboxes" + ); + + assert!( + db.has_filter_fired(&rule.id, shared_key()) + .await + .expect("read the claim ledger"), + "an arrival in a second folder must be evaluated, not skipped as already-seen" + ); + let pending = db + .list_pending_mutations(10) + .await + .expect("list pending mutations"); + assert_eq!( + pending.len(), + 1, + "with a real outbox mutation behind the fire, not just a ledger row" + ); + assert_eq!(pending[0].message_id, shared_key()); + } + + #[tokio::test] + async fn a_rule_fires_once_for_a_message_that_lands_in_two_folders() { + // MOVE and FLAG converge under repetition; FORWARD and CALL WEBHOOK do + // not. Two occupancies of one message must not page an on-call + // engineer twice. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let event_bus = EventBus::new(); + + let rule = rule_matching_the_shared_message(); + let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule"); + + let backend = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]); + let processed = fetch_and_persist( + &db, + &event_bus, + &backend, + &filter_engine, + Some(MockMailBackend::SHARED_ACCOUNT_ID), + ) + .await + .expect("sync succeeds"); + + assert_eq!(processed, 2, "both occupancies are processed"); + let placements = db + .placements_of(shared_key()) + .await + .expect("read the message's placements"); + assert_eq!(placements.len(), 2); + + assert_eq!( + db.filter_fire_count().await.expect("read the claim ledger"), + 1, + "one claim, therefore one set of actions" + ); + assert!(db + .has_filter_fired(&rule.id, shared_key()) + .await + .expect("read the claim ledger")); + + let pending = db + .list_pending_mutations(10) + .await + .expect("list pending mutations"); + assert_eq!( + pending.len(), + 1, + "one outbox mutation for the message, not one per folder" + ); + let logs = db + .list_filter_execution_logs(10) + .await + .expect("list execution logs"); + assert_eq!(logs.len(), 1, "and one audit entry, not one per folder"); + } + + #[tokio::test] + async fn an_unverifiable_claim_skips_the_actions_rather_than_licensing_them() { + // Fail closed. If the claim itself errors, the engine cannot know + // whether this rule has already acted on this message -- and a lost + // claim check on a FORWARD is a duplicate mail the recipient cannot + // un-receive. Dropping the ledger table makes the claim error for real + // while every other write still works, so a mutation appearing here + // would be a genuine "acted without a claim", not a dead store. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + + let rule = rule_matching_the_shared_message(); + let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule"); + + let placed = MockMailBackend::shared_message_in("INBOX", "1"); + db.save_email_at(&placed.email, placed.source, &placed.placement) + .await + .expect("store the occupancy"); + + sqlx::query("DROP TABLE filter_fired") + .execute(db.pool()) + .await + .expect("drop the claim ledger"); + + let applied = + apply_filter_actions(&db, &filter_engine, &placed.email, &placed.placement).await; + assert_eq!(applied, 0, "an unclaimed rule must apply nothing"); + + let pending = db + .list_pending_mutations(10) + .await + .expect("list pending mutations"); + assert!( + pending.is_empty(), + "no outbox mutation may be enqueued off an unverifiable claim" + ); + let logs = db + .list_filter_execution_logs(10) + .await + .expect("list execution logs"); + assert!( + logs.is_empty(), + "and nothing may be recorded as having fired" + ); + } } diff --git a/crates/nunciod/tests/filters_triage_e2e_test.rs b/crates/nunciod/tests/filters_triage_e2e_test.rs index 9fa2b36..57680d0 100644 --- a/crates/nunciod/tests/filters_triage_e2e_test.rs +++ b/crates/nunciod/tests/filters_triage_e2e_test.rs @@ -3,7 +3,7 @@ //! Boots a real `nunciod` daemon gRPC server (`nunciod::grpc:: //! serve_on_listener`) on an ephemeral loopback port, backed by a real //! temp file-backed [`DatabaseEngine`] seeded directly via -//! [`DatabaseEngine::save_email`] (standing in for messages that were synced +//! [`DatabaseEngine::save_email_at`] (standing in for messages that were synced //! before the active rule set existed) and a live [`FilterEngine`] carrying //! a real persisted rule. //! @@ -25,7 +25,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use nuncio_core::model::Email; +use nuncio_core::model::{Email, IdentitySource, Placement}; use nuncio_core::EventBus; use nuncio_filter::{FilterEngine, NsqlParser}; use nuncio_proto::v1::filters_client::FiltersClient; @@ -42,14 +42,10 @@ fn sample_email(id: &str, subject: &str) -> Email { Email { id: id.to_string(), account_id: ACCOUNT_ID.to_string(), - folder_id: "inbox".to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "triage-e2e@nuncio.mx".to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some("body".to_string()), body_html: None, attachments: Vec::new(), @@ -58,6 +54,40 @@ fn sample_email(id: &str, subject: &str) -> Email { } } +/// The INBOX occupancy a seeded message sits in. `id` doubles as the UID, so +/// distinct messages occupy distinct rows. +fn sample_placement(id: &str) -> Placement { + Placement { + account_id: ACCOUNT_ID.to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + } +} + +/// Seed a message and its occupancy the way a sync pass would. +async fn seed(db: &DatabaseEngine, id: &str, subject: &str) { + db.save_email_at( + &sample_email(id, subject), + IdentitySource::Surrogate, + &sample_placement(id), + ) + .await + .unwrap_or_else(|e| panic!("seed message '{id}': {e}")); +} + +/// The read flag of a seeded message's single occupancy. +async fn read_flag_of(db: &DatabaseEngine, id: &str) -> bool { + let placements = db.placements_of(id).await.expect("read placements"); + assert_eq!( + placements.len(), + 1, + "'{id}' should occupy exactly one folder" + ); + placements[0].read +} + /// Boots the real daemon gRPC server (bearer-token authenticated) on an /// ephemeral loopback port, backed by a real temp file-backed /// `DatabaseEngine` and the given live `FilterEngine`. Returns the server @@ -145,15 +175,9 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun // Seed messages directly into the store, standing in for mail that was // synced BEFORE these rules existed -- exactly the scenario `Triage` is // for. Two match (one per rule), one does not. - db.save_email(&sample_email("m-urgent", "Urgent: server down")) - .await - .expect("seed urgent message"); - db.save_email(&sample_email("m-archive", "Please Archive Me")) - .await - .expect("seed archive message"); - db.save_email(&sample_email("m-plain", "Just saying hi")) - .await - .expect("seed non-matching message"); + seed(&db, "m-urgent", "Urgent: server down").await; + seed(&db, "m-archive", "Please Archive Me").await; + seed(&db, "m-plain", "Just saying hi").await; let mut client = FiltersClient::connect(format!("http://{addr}")) .await @@ -190,8 +214,10 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun // Real side effects, not a fabricated count: the MARK READ action // actually flipped the stored read flag... - let urgent = db.get_message("m-urgent").await.expect("get urgent"); - assert!(urgent.read, "MARK READ must genuinely flip the read flag"); + assert!( + read_flag_of(&db, "m-urgent").await, + "MARK READ must genuinely flip the occupancy's read flag" + ); // ...and the MOVE TO action actually queued a real outbox mutation. let pending = db @@ -205,8 +231,10 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun ); assert_eq!(pending[0].message_id, "m-archive"); - let plain = db.get_message("m-plain").await.expect("get plain"); - assert!(!plain.read, "a non-matching message must be left untouched"); + assert!( + !read_flag_of(&db, "m-plain").await, + "a non-matching message must be left untouched" + ); } #[tokio::test] @@ -221,12 +249,7 @@ async fn triage_streams_multiple_chunks_for_a_store_bigger_than_one_chunk() { // updates before the final `done: true` -- proof this is a real // multi-chunk stream, not one batch disguised as one. for i in 0..5 { - db.save_email(&sample_email( - &format!("m-{i:02}"), - &format!("Message number {i}"), - )) - .await - .expect("seed message"); + seed(&db, &format!("m-{i:02}"), &format!("Message number {i}")).await; } let mut client = FiltersClient::connect(format!("http://{addr}")) diff --git a/crates/nunciod/tests/no_secrets_canary_test.rs b/crates/nunciod/tests/no_secrets_canary_test.rs index e4150a9..413a78c 100644 --- a/crates/nunciod/tests/no_secrets_canary_test.rs +++ b/crates/nunciod/tests/no_secrets_canary_test.rs @@ -27,10 +27,10 @@ use std::sync::{Arc, Mutex}; use nuncio_cal::{CalDavAccountConfig, CalDavClient, CalendarBackend}; use nuncio_contacts::{CardDavAccountConfig, CardDavClient, ContactsBackend}; -use nuncio_core::model::{Email, Folder}; +use nuncio_core::model::{Email, Folder, IdentitySource, Placement}; use nuncio_core::EventBus; use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher}; -use nuncio_mail::{JmapEngine, MailBackend, MockMailBackend, MockMessageSender}; +use nuncio_mail::{JmapEngine, MailBackend, MockMailBackend, MockMessageSender, PlacedMessage}; use nuncio_proto::v1::{ AccountConfig, AddAccountRequest, GetStatusRequest, MarkReadRequest, SendMessageRequest, SyncRequest, TlsMode, @@ -139,14 +139,10 @@ fn sentinel_inbound_email() -> Email { Email { id: "msg-canary-1".to_string(), account_id: ACCOUNT_ID.to_string(), - folder_id: "inbox".to_string(), - remote_id: "msg-canary-1".to_string(), - uid_validity: "1".to_string(), subject: SENTINEL_SUBJECT.to_string(), sender: SENTINEL_SENDER.to_string(), recipient: SENTINEL_RECIPIENT.to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some(SENTINEL_BODY.to_string()), body_html: None, attachments: Vec::new(), @@ -155,6 +151,26 @@ fn sentinel_inbound_email() -> Email { } } +/// The INBOX occupancy the sentinel message is found in. +fn sentinel_placement() -> Placement { + Placement { + account_id: ACCOUNT_ID.to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: "msg-canary-1".to_string(), + read: false, + } +} + +/// The sentinel message as a backend would surface it. +fn sentinel_placed_message() -> PlacedMessage { + PlacedMessage { + email: sentinel_inbound_email(), + source: IdentitySource::Surrogate, + placement: sentinel_placement(), + } +} + /// Boots the real daemon gRPC server (all services, bearer-authenticated) on an /// ephemeral loopback port, backed by a real temp file DB and the given mail /// overrides + filter engine. @@ -214,7 +230,7 @@ async fn drive_daemon() { total_messages: 1, unread_messages: 1, }); - mock_backend.add_message(sentinel_inbound_email()); + mock_backend.add_message(sentinel_placed_message()); let mock_sender = MockMessageSender::new(); let overrides = MailEngineOverrides { mail_backend: Some(Arc::new(mock_backend)), @@ -309,7 +325,12 @@ async fn drive_filter_and_webhook() { ) .expect("parse rule"); let engine = FilterEngine::new(vec![rule]).expect("compile rule"); - let matches = engine.evaluate(&sentinel_inbound_email()); + let sentinel = sentinel_inbound_email(); + let placement = sentinel_placement(); + let matches = engine.evaluate(nuncio_filter::PlacedEmail { + email: &sentinel, + placement: &placement, + }); assert_eq!(matches.len(), 1, "rule must fire on the sentinel message"); let mock_server = MockServer::start().await; diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index 933f84e..f7babcf 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -12,7 +12,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use async_trait::async_trait; -use nuncio_core::model::Email; +use nuncio_core::model::{Email, IdentitySource, Placement}; use nuncio_core::EventBus; use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher, WebhookError}; use nuncio_mail::{ @@ -116,14 +116,10 @@ fn sample_email(subject: &str) -> Email { Email { id: "surrogate-inbox-42".to_string(), account_id: ACCOUNT_ID.to_string(), - folder_id: FOLDER_ID.to_string(), - remote_id: "42".to_string(), - uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "owner@nuncio.mx".to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some("original body".to_string()), body_html: None, attachments: Vec::new(), @@ -132,6 +128,19 @@ fn sample_email(subject: &str) -> Email { } } +/// The mailbox occupancy the seeded message sits in. The outbox recovers its +/// remote addressing from here now -- the UID and the UIDVALIDITY scope it was +/// captured under are properties of the occupancy, not of the message. +fn sample_placement(remote_id: &str) -> Placement { + Placement { + account_id: ACCOUNT_ID.to_string(), + folder_id: FOLDER_ID.to_string(), + uid_validity: "1".to_string(), + remote_id: remote_id.to_string(), + read: false, + } +} + /// Seed the account, source-folder checkpoint, and a matching message, then run /// the given rule's actions through `apply_filter_actions` so the outbox is /// populated exactly as a real sync would populate it. @@ -144,11 +153,14 @@ async fn seed_and_enqueue(db: &DatabaseEngine, rule_nsql: &str, subject: &str) { .expect("save folder checkpoint"); let email = sample_email(subject); - db.save_email(&email).await.expect("save email"); + let placement = sample_placement("42"); + db.save_email_at(&email, IdentitySource::Surrogate, &placement) + .await + .expect("save email"); let rule = NsqlParser::parse_rule("Outbox Rule", 1, rule_nsql).expect("parse rule"); let engine = FilterEngine::new(vec![rule]).expect("compile rule"); - apply_filter_actions(db, &engine, &email).await; + apply_filter_actions(db, &engine, &email, &placement).await; let pending = db.list_pending_mutations(10).await.expect("list pending"); assert_eq!( @@ -175,14 +187,16 @@ async fn seed_and_enqueue_distinct( .await .expect("save folder checkpoint"); - // `remote_id`/`uid_validity` (not just `id`) are part of the store's - // `messages` identity UNIQUE index -- leaving them at `sample_email`'s - // shared default would make this INSERT OR REPLACE the OTHER seeded - // message sharing that same remote identity out from under it. + // The occupancy needs its own UID as well as its own message key: the + // placements primary key is (account, folder, uidvalidity, uid), so + // leaving `remote_id` at the shared default would repoint the OTHER + // seeded message's occupancy at this message. let mut email = sample_email(subject); email.id = message_id.to_string(); - email.remote_id = message_id.to_string(); - db.save_email(&email).await.expect("save email"); + let placement = sample_placement(message_id); + db.save_email_at(&email, IdentitySource::Surrogate, &placement) + .await + .expect("save email"); let before = db .list_pending_mutations(100) @@ -192,7 +206,7 @@ async fn seed_and_enqueue_distinct( let rule = NsqlParser::parse_rule("Outbox Rule", 1, rule_nsql).expect("parse rule"); let engine = FilterEngine::new(vec![rule]).expect("compile rule"); - apply_filter_actions(db, &engine, &email).await; + apply_filter_actions(db, &engine, &email, &placement).await; let after = db .list_pending_mutations(100) diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index 93e28e1..81f798e 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -31,10 +31,10 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use nuncio_core::model::{Email, Folder}; +use nuncio_core::model::{Email, Folder, IdentitySource, Placement}; use nuncio_core::EventBus; use nuncio_filter::FilterEngine; -use nuncio_mail::{MockMailBackend, MockMessageSender}; +use nuncio_mail::{MockMailBackend, MockMessageSender, PlacedMessage}; use nuncio_proto::v1::event::Kind; use nuncio_proto::v1::{ AccountConfig, AddAccountRequest, GetMessageRequest, ListAccountsRequest, ListMessagesRequest, @@ -73,23 +73,31 @@ fn sample_account_config() -> AccountConfig { } } -fn mock_inbound_email(id: &str, subject: &str, body: &str) -> Email { - Email { - id: id.to_string(), - account_id: ACCOUNT_ID.to_string(), - folder_id: "inbox".to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), - subject: subject.to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: ACCOUNT_EMAIL.to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some(body.to_string()), - body_html: None, - attachments: Vec::new(), - message_id: None, - content_hash: None, +/// One inbound message as a backend surfaces it: identity, the tier the key +/// came from, and the INBOX occupancy this pass found it in. +fn mock_inbound_email(id: &str, subject: &str, body: &str) -> PlacedMessage { + PlacedMessage { + email: Email { + id: id.to_string(), + account_id: ACCOUNT_ID.to_string(), + subject: subject.to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: ACCOUNT_EMAIL.to_string(), + received_at: 1_700_000_000, + body_plain: Some(body.to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + source: IdentitySource::Surrogate, + placement: Placement { + account_id: ACCOUNT_ID.to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + }, } } -- 2.54.0 From 4f1dc296f135de9c8197ee86b6a687e1fca2985b Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 09:58:22 -0500 Subject: [PATCH 74/95] fix(outbox): refuse an ambiguously-addressed mailbox mutation An outbox row records only a message key, so for a message occupying several mailboxes there is no way to know which copy the rule matched. Addressing whichever placement sorted first meant a rule that matched the INBOX copy could MOVE or DELETE the Archive one -- against mail the user never targeted, with no undo, and Gmail labels make multi-placement ordinary rather than exotic. Unresolvable addressing is no more a licence to act than an unverifiable claim: it now fails permanently, naming the competing mailboxes. Single-placement mutations are unaffected. Also from review: PreviewRule no longer substitutes an invented inbox occupancy when a stored message's placements cannot be read, which let WHERE FOLDER match a location the message does not have; explicit VANISHED removals no longer need ambient account context, since a PlacementKey names its own account; and both Mail read RPCs now state that they list across configured accounts only, so removing an account hides its cached mail by design rather than by accident. Covers the cross-account merge shims, which nothing exercised: summed counts for a shared folder id, merged ordering across page boundaries, and a page boundary that only shows up when one account owns the folder. --- crates/nuncio-mail/src/mock.rs | 31 ++- crates/nunciod/src/grpc.rs | 417 ++++++++++++++++++++++++++++++--- crates/nunciod/src/outbox.rs | 173 ++++++++++++-- crates/nunciod/src/sync.rs | 103 ++++++-- 4 files changed, 633 insertions(+), 91 deletions(-) diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index 44cce17..ebcb10b 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -1,7 +1,7 @@ //! Deterministic mock mail backend for offline testing and integration verification. use async_trait::async_trait; -use nuncio_core::model::{Email, Folder, IdentitySource, Placement}; +use nuncio_core::model::{Email, Folder, IdentitySource, Placement, PlacementKey}; use std::sync::{Arc, Mutex}; use crate::backend::{ @@ -26,6 +26,10 @@ pub struct MockMailBackend { /// so a daemon E2E can prove the outbox executor genuinely invoked the /// backend op (not a fabricated completion). applied_mutations: Arc<Mutex<Vec<RemoteMutationSpec>>>, + /// Occupancies this mock reports as explicitly gone, the way QRESYNC + /// `VANISHED` does. Handed back on the pass for the folder each key names, + /// so a test can prove a caller acts on a server-confirmed removal. + removals: Arc<Mutex<Vec<PlacementKey>>>, } impl MockMailBackend { @@ -57,6 +61,15 @@ impl MockMailBackend { .unwrap_or_default() } + /// Stage occupancies this mock reports as explicitly gone, as a QRESYNC + /// `VANISHED` would. Each key is handed back on the pass for the folder it + /// names, so staging a removal for `Archive` does not leak into `INBOX`. + pub fn set_removals(&self, removals: Vec<PlacementKey>) { + if let Ok(mut guard) = self.removals.lock() { + *guard = removals; + } + } + /// Configure the mock to simulate network failure errors. pub fn set_should_fail(&self, fail: bool) { if let Ok(mut flag) = self.should_fail.lock() { @@ -203,11 +216,25 @@ impl MailBackend for MockMailBackend { .map_err(|e| MailError::ParseFailed(e.to_string()))? .clone(); + // Staged removals are scoped to the folder each key names, and are + // reported on both the full and the incremental pass -- a `VANISHED` + // report is exactly the kind of thing an incremental resync carries. + let removals: Vec<PlacementKey> = self + .removals + .lock() + .map_err(|e| MailError::ParseFailed(e.to_string()))? + .iter() + .filter(|key| key.folder_id == folder_id) + .cloned() + .collect(); + // Simulate a since_state-aware backend: with a prior checkpoint there // is nothing new to hand back, so the incremental fetch is genuinely // narrower (empty) rather than a full re-report of every message. + // `present` stays `None`: an incremental pass cannot speak to absence. if since_state.is_some() { return Ok(FolderChanges { + removals, next_state: returned_state, ..Default::default() }); @@ -228,7 +255,7 @@ impl MailBackend for MockMailBackend { let present = matches.iter().map(|m| m.placement.key()).collect(); Ok(FolderChanges { upserts: matches, - removals: Vec::new(), + removals, present: Some(present), next_state: returned_state, }) diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 612e88d..c8b4d12 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -1750,35 +1750,51 @@ impl MailGrpcService { .flatten(); Ok((rows, next)) } +} - /// The mailbox occupancy a message-scoped RPC should speak for. - /// - /// `GetMessage`/`MarkRead` name a message, but `folder_id` and the read - /// flag are per-mailbox now, so one occupancy has to be chosen. This takes - /// the store's first in its deterministic order. A message with no - /// occupancy at all is not "unread in no folder" -- it is gone -- and is - /// reported as not found. - async fn primary_placement( - &self, - message_key: &str, - ) -> Result<nuncio_core::model::Placement, Status> { - let placements = self.db.placements_of(message_key).await.map_err(|e| { - Status::internal(format!( - "failed to read placements for '{message_key}': {e}" - )) - })?; - placements.into_iter().next().ok_or_else(|| { - errors::status_with_metadata( - ErrorReason::MessageNotFound, - format!("message '{message_key}' occupies no mailbox"), - [("message_id".to_string(), message_key.to_string())], - ) - }) - } +/// The mailbox occupancy a message-scoped RPC should speak for. +/// +/// `GetMessage`/`MarkRead`/`PreviewRule` name a message, but `folder_id` and the +/// read flag are per-mailbox now, so one occupancy has to be chosen. This takes +/// the store's first in its deterministic order, so every message-scoped RPC +/// picks the same one and they stay consistent with each other. +/// +/// The two failure modes are deliberately distinct and neither is papered over: +/// a read error surfaces as `internal`, and a message with no occupancy is +/// reported as not found. A message that occupies no mailbox is not "unread in +/// no folder" -- it is gone -- and inventing a location for it would let a +/// `WHERE FOLDER = …` preview match somewhere the message has never been. +/// +/// A free function rather than a method because `Mail` and `Filters` are +/// separate services that both need it. +async fn primary_placement( + db: &DatabaseEngine, + message_key: &str, +) -> Result<nuncio_core::model::Placement, Status> { + let placements = db.placements_of(message_key).await.map_err(|e| { + Status::internal(format!( + "failed to read placements for '{message_key}': {e}" + )) + })?; + placements.into_iter().next().ok_or_else(|| { + errors::status_with_metadata( + ErrorReason::MessageNotFound, + format!("message '{message_key}' occupies no mailbox"), + [("message_id".to_string(), message_key.to_string())], + ) + }) } #[tonic::async_trait] impl Mail for MailGrpcService { + /// Lists folders across every **configured** account, merged by folder id. + /// + /// Mail whose owning account no longer has a configuration row is not + /// listed, even though its rows survive in the store. That is deliberate: + /// removing an account should stop its cached mail appearing, and the + /// account list is what defines "whose mail this daemon serves". The rows + /// are still there for an explicit `Export`, and are reachable again if the + /// account is re-added. async fn list_folders( &self, request: Request<ListFoldersRequest>, @@ -1811,6 +1827,13 @@ impl Mail for MailGrpcService { })) } + /// Lists one folder's messages across every **configured** account, merged + /// into a single newest-first sequence. + /// + /// As with [`Self::list_folders`], mail whose owning account no longer has + /// a configuration row is not listed: the account list defines whose mail + /// this daemon serves, so removing an account hides its cached mail rather + /// than leaving it visible under a folder nobody owns. async fn list_messages( &self, request: Request<ListMessagesRequest>, @@ -1871,7 +1894,7 @@ impl Mail for MailGrpcService { ) })?; - let placement = self.primary_placement(&req.message_id).await?; + let placement = primary_placement(&self.db, &req.message_id).await?; Ok(Response::new(GetMessageResponse { message: Some(map_email_to_proto((email, placement))), @@ -1899,7 +1922,7 @@ impl Mail for MailGrpcService { // is applied to the occupancy `GetMessage` would report -- keeping the // two RPCs consistent with each other. Naming a placement on the wire // is what makes this well-defined, and lands with that surface. - let placement = self.primary_placement(&req.message_id).await?; + let placement = primary_placement(&self.db, &req.message_id).await?; self.db .set_placement_read(&placement.key(), req.read) .await @@ -2351,22 +2374,24 @@ impl Filters for FiltersGrpcService { .map_err(|e| Status::internal(format!("failed to build preview engine: {e}")))?; // A preview needs a placement as well as a message, because the rule - // language can ask where the message sits. A stored message is - // previewed against a real occupancy; a message that does not resolve - // falls back to the synthetic pair, exactly as before. + // language can ask where the message sits. + // + // For a message that IS stored, the occupancy must be the real one. + // Substituting an invented `inbox` placement -- on a read error, or for + // a message with none -- would let `WHERE FOLDER = 'inbox'` report a + // match against a location the message does not have, which is the one + // thing a preview must never do. So a read error surfaces, and a stored + // message with no occupancy is reported as not found rather than + // previewed somewhere imaginary; `primary_placement` does both. + // + // A message id that resolves to nothing at all still falls back to the + // fully synthetic pair, exactly as before: that is the documented + // "preview against a sample" path, and it invents a message as well as + // a placement, so it claims nothing about stored mail. let (email, placement) = match &req.message_id { Some(message_id) if !message_id.is_empty() => { match self.db.get_message(message_id).await { - Ok(email) => { - let placement = self - .db - .placements_of(message_id) - .await - .ok() - .and_then(|placements| placements.into_iter().next()) - .unwrap_or_else(|| synthetic_preview_placement(message_id)); - (email, placement) - } + Ok(email) => (email, primary_placement(&self.db, message_id).await?), Err(_) => ( synthetic_preview_email(message_id), synthetic_preview_placement(message_id), @@ -4847,6 +4872,253 @@ mod tests { assert!(pages >= 4, "a 7-item set at page_size 2 must span >1 page"); } + // ---- Cross-account merge shims ------------------------------------- + // + // `ListFolders`/`ListMessages` carry no account field, so the handlers ask + // every configured account and merge. These exercise that merge across two + // accounts, which nothing else does: the keyset test above seeds one + // account, and the export scoping test bypasses the Mail service entirely. + + /// Seed one message into `account_id`'s copy of `folder_id`, at + /// `received_at`, creating the account row if it is not there yet. + async fn seed_in_account( + db: &DatabaseEngine, + account_id: &str, + id: &str, + folder_id: &str, + received_at: i64, + ) { + let (mut email, mut placement) = sample_placed(id, folder_id, "subject", "body"); + email.account_id = account_id.to_string(); + email.received_at = received_at; + placement.account_id = account_id.to_string(); + seed_message(db, &email, &placement).await; + } + + /// Boot a Mail server over `db` and return a connected client. + async fn mail_client_over(db: DatabaseEngine) -> MailClient<tonic::transport::Channel> { + let (addr, _handle) = spawn_test_server_with( + Arc::new(EventBus::new()), + Arc::new(db), + Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")), + Arc::new(SecretManager::mock()), + "correct-token", + ) + .await; + // The handle is dropped: `spawn_test_server_with` detaches the server + // task, and every test here finishes its calls before returning. + MailClient::connect(format!("http://{addr}")) + .await + .expect("client connects") + } + + /// Two accounts both have an `INBOX`. Folder ids are not globally unique, + /// so the merge has to fold them into ONE wire folder whose counts are the + /// sum -- which is what the account-blind `GROUP BY folder_id` used to + /// return before the store became account-scoped. + #[tokio::test] + async fn list_folders_sums_counts_for_a_folder_id_two_accounts_share() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + seed_in_account(&db, "acct-a", "m-a1", "inbox", 1_700_000_001).await; + seed_in_account(&db, "acct-a", "m-a2", "inbox", 1_700_000_002).await; + seed_in_account(&db, "acct-b", "m-b1", "inbox", 1_700_000_003).await; + // A folder only one of the two accounts has must survive the merge + // intact rather than being lost or double-counted. + seed_in_account(&db, "acct-b", "m-b2", "archive", 1_700_000_004).await; + + let mut client = mail_client_over(db).await; + let resp = client + .list_folders(authed_bearer_request(ListFoldersRequest { + page_size: 0, + page_token: String::new(), + })) + .await + .expect("list_folders succeeds") + .into_inner(); + + let mut folders = resp.folders; + folders.sort_by(|a, b| a.id.cmp(&b.id)); + assert_eq!( + folders.iter().map(|f| f.id.as_str()).collect::<Vec<_>>(), + vec!["archive", "inbox"], + "one wire folder per distinct id, however many accounts hold it" + ); + let inbox = &folders[1]; + assert_eq!(inbox.total_messages, 3, "2 from acct-a + 1 from acct-b"); + assert_eq!(inbox.unread_messages, 3); + assert_eq!(folders[0].total_messages, 1); + } + + /// Paging a folder whose messages come from two accounts must return every + /// message exactly once, newest-first, with pages that straddle the account + /// boundary. Timestamps interleave so no page can be satisfied from a + /// single account -- if the merge ever paged one account at a time, the + /// order would come out wrong even though the set did not. + #[tokio::test] + async fn list_messages_merges_two_accounts_across_page_boundaries_without_gaps() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + seed_in_account(&db, "acct-a", "m-a1", "inbox", 1_700_000_001).await; + seed_in_account(&db, "acct-b", "m-b2", "inbox", 1_700_000_002).await; + seed_in_account(&db, "acct-a", "m-a3", "inbox", 1_700_000_003).await; + seed_in_account(&db, "acct-b", "m-b4", "inbox", 1_700_000_004).await; + seed_in_account(&db, "acct-a", "m-a5", "inbox", 1_700_000_005).await; + + let mut client = mail_client_over(db).await; + + let mut seen = Vec::new(); + let mut page_sizes = Vec::new(); + let mut page_token = String::new(); + let mut pages = 0; + loop { + let resp = client + .list_messages(authed_bearer_request(ListMessagesRequest { + folder_id: "inbox".to_string(), + page_size: 2, + page_token: page_token.clone(), + })) + .await + .expect("list_messages page succeeds") + .into_inner(); + pages += 1; + assert!( + resp.messages.len() <= 2, + "a merged page must still respect page_size" + ); + page_sizes.push(resp.messages.len()); + seen.extend(resp.messages.iter().map(|m| m.id.clone())); + if resp.next_page_token.is_empty() { + break; + } + page_token = resp.next_page_token; + assert!(pages < 100, "pagination must terminate"); + } + + assert_eq!( + seen, + vec!["m-a5", "m-b4", "m-a3", "m-b2", "m-a1"], + "the merged listing must be newest-first across BOTH accounts, once each" + ); + assert_eq!( + page_sizes, + vec![2, 2, 1], + "and it must genuinely span pages that straddle the account boundary" + ); + } + + /// A configured account holding nothing in this folder must not truncate + /// the listing. + /// + /// This is the exact shape the merge gets wrong if it asks each account for + /// only `page_size` rows: one account returns a full page, the other + /// returns none, the merged total lands on `page_size` exactly, and "is + /// there more?" reads as no -- so pages 2 and 3 are silently unreachable. + /// The interleaved test above cannot see it, because there both accounts + /// contribute and the total always overshoots. + #[tokio::test] + async fn a_folder_owned_by_one_account_still_pages_past_the_first_page() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + for (index, id) in ["m-1", "m-2", "m-3", "m-4", "m-5"].iter().enumerate() { + seed_in_account(&db, "acct-a", id, "inbox", 1_700_000_000 + index as i64).await; + } + // Configured, but with nothing in `inbox` -- an ordinary second account + // whose mail lives elsewhere. + seed_in_account(&db, "acct-b", "m-elsewhere", "archive", 1_700_000_009).await; + + let mut client = mail_client_over(db).await; + + let mut seen = Vec::new(); + let mut page_token = String::new(); + let mut pages = 0; + loop { + let resp = client + .list_messages(authed_bearer_request(ListMessagesRequest { + folder_id: "inbox".to_string(), + page_size: 2, + page_token: page_token.clone(), + })) + .await + .expect("list_messages page succeeds") + .into_inner(); + pages += 1; + seen.extend(resp.messages.iter().map(|m| m.id.clone())); + if resp.next_page_token.is_empty() { + break; + } + page_token = resp.next_page_token; + assert!(pages < 100, "pagination must terminate"); + } + + assert_eq!( + seen, + vec!["m-5", "m-4", "m-3", "m-2", "m-1"], + "an empty second account must not cut the listing short" + ); + } + + /// Removing an account hides its cached mail, even though the rows survive + /// in the store for an explicit export. This is intended behaviour, not an + /// accident of the merge -- it is the rule both RPCs' doc comments state, + /// and it is why seeding a message in these tests also persists an account. + #[tokio::test] + async fn mail_for_a_deleted_account_is_no_longer_listed() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + seed_in_account(&db, "acct-keep", "m-keep", "inbox", 1_700_000_001).await; + seed_in_account(&db, "acct-drop", "m-drop", "inbox", 1_700_000_002).await; + + db.delete_account("acct-drop") + .await + .expect("delete the account"); + // The mail itself is deliberately NOT deleted with the account, so the + // test is about visibility, not about a cascade. + assert!( + !db.placements_of("m-drop") + .await + .expect("read placements") + .is_empty(), + "the orphaned mail must still be in the store, just unlisted" + ); + + let mut client = mail_client_over(db).await; + let messages = client + .list_messages(authed_bearer_request(ListMessagesRequest { + folder_id: "inbox".to_string(), + page_size: 50, + page_token: String::new(), + })) + .await + .expect("list_messages succeeds") + .into_inner() + .messages; + assert_eq!( + messages.iter().map(|m| m.id.as_str()).collect::<Vec<_>>(), + vec!["m-keep"], + "mail belonging to a removed account must not be listed" + ); + + let folders = client + .list_folders(authed_bearer_request(ListFoldersRequest { + page_size: 50, + page_token: String::new(), + })) + .await + .expect("list_folders succeeds") + .into_inner() + .folders; + assert_eq!(folders.len(), 1); + assert_eq!( + folders[0].total_messages, 1, + "the removed account's mail must not be counted either" + ); + } + /// A `page_token` the server cannot decode is rejected with a typed /// `VALIDATION_FAILED` `ErrorInfo` mapped to `InvalidArgument` -- never a /// panic and never a silently-empty first page. @@ -6139,6 +6411,73 @@ mod tests { ); } + /// A preview of a STORED message must never invent the occupancy it + /// evaluates against. + /// + /// `WHERE FOLDER = …` is answered from the placement, so substituting a + /// synthetic `inbox` one when the real placements cannot be read would make + /// the preview report a match against a folder the message is not in -- + /// stated with full confidence, about real stored mail. Preview's whole job + /// is to say truthfully what a rule would do, so an unreadable placement + /// has to surface as an error. + /// + /// Dropping the `placements` table makes `placements_of` fail for real + /// while `get_message` still succeeds, which is exactly the split that + /// produced the fabrication. + #[tokio::test] + async fn preview_rule_surfaces_a_placement_read_error_instead_of_inventing_a_folder() { + let (addr, _handle, db, _engine, _dir) = spawn_filters_test_server("correct-token").await; + + db.save_email_at( + &nuncio_core::model::Email { + id: "msg-preview-archive".to_string(), + account_id: "acct-1".to_string(), + subject: "Urgent Board Meeting".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some("Please review the attached deck".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::IdentitySource::Surrogate, + &nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "archive".to_string(), + uid_validity: "1".to_string(), + remote_id: "preview-archive".to_string(), + read: false, + }, + ) + .await + .expect("seed message"); + + sqlx::query("DROP TABLE placements") + .execute(db.pool()) + .await + .expect("drop the placements table"); + + let mut client = FiltersClient::connect(format!("http://{addr}")) + .await + .expect("client connects"); + + let err = client + .preview_rule(authed_bearer_request(PreviewRuleRequest { + nsql: "WHERE folder = 'inbox' ACTION DELETE".to_string(), + message_id: Some("msg-preview-archive".to_string()), + })) + .await + .expect_err("an unreadable placement must not be papered over"); + assert_eq!(err.code(), Code::Internal); + assert!( + err.message().contains("placements"), + "the error must name what could not be read: {}", + err.message() + ); + } + /// When no `message_id` is given (or it does not resolve to a stored /// message), `PreviewRule` falls back to a fixed synthetic sample /// message rather than failing the RPC. diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index f5ef168..896f42f 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -285,14 +285,24 @@ async fn execute_one( /// protocol id, UIDVALIDITY scope) from the message's stored placement so the /// backend can enforce its UIDVALIDITY guard. /// -/// The addressing now comes from a placement rather than the message row, -/// because those coordinates are properties of a mailbox occupancy and a -/// message can hold several. The outbox row records only the message key, so -/// which occupancy the rule was evaluated against is not recoverable here: this -/// addresses the first placement in the store's deterministic order and says so -/// in the log when there is more than one. Carrying the originating placement on -/// the outbox row is the real fix and belongs with the placement-aware wire -/// surface, not here. +/// The addressing comes from a placement rather than the message row, because +/// those coordinates are properties of a mailbox occupancy and a message can +/// hold several. The outbox row records only the message key, so which occupancy +/// the rule matched is not recoverable here -- and when the message occupies +/// more than one mailbox, this **refuses** rather than picking one. +/// +/// Guessing is not the conservative option. A rule that matched the `INBOX` +/// copy would `MOVE` or `DELETE` whichever copy happened to sort first, against +/// mail the user never targeted and with no undo; Gmail labels make +/// multi-placement ordinary rather than exotic. Unresolvable *addressing* is no +/// more a licence to act than an unverifiable claim is in +/// [`crate::sync::apply_filter_actions`]: refusing is inert and visible, +/// guessing is destructive and irreversible. +/// +/// The refusal is permanent, not retryable: nothing about a later attempt makes +/// the outbox row remember a placement it never recorded. Carrying the +/// originating placement on the row is the real fix, and belongs with the +/// placement-aware wire surface. async fn dispatch_mailbox_mutation( db: &DatabaseEngine, env: &dyn RemoteExecutionEnv, @@ -332,24 +342,36 @@ async fn dispatch_mailbox_mutation( return Disposition::Retry(format!("failed to resolve the message's placements: {e}")) } }; - let Some(placement) = placements.first() else { - // No occupancy means no way to address the message on any server, and - // no retry can conjure one back. - return Disposition::Permanent(format!( - "message '{}' occupies no mailbox to address", - email.id - )); + let placement = match placements.as_slice() { + [only] => only, + [] => { + // No occupancy means no way to address the message on any server, + // and no retry can conjure one back. + return Disposition::Permanent(format!( + "message '{}' occupies no mailbox to address", + email.id + )); + } + several => { + let folders: Vec<&str> = several.iter().map(|p| p.folder_id.as_str()).collect(); + tracing::warn!( + mutation_id = %mutation.id, + op = %mutation.mutation_type, + message_id = %email.id, + placement_count = several.len(), + "outbox: refusing an ambiguously-addressed mutation" + ); + return Disposition::Permanent(format!( + "message '{}' occupies {} mailboxes ({}) and the outbox row does not record \ + which one rule '{}' matched; refusing to guess a {} target", + email.id, + several.len(), + folders.join(", "), + mutation.rule_id, + mutation.mutation_type + )); + } }; - if placements.len() > 1 { - tracing::warn!( - mutation_id = %mutation.id, - message_id = %email.id, - folder_id = %placement.folder_id, - placement_count = placements.len(), - "outbox: message occupies several mailboxes and the row does not record which one \ - the rule matched; addressing the first" - ); - } let backend = match env.mail_backend(&placement.account_id).await { Ok(backend) => backend, @@ -760,4 +782,105 @@ mod tests { ); } } + + /// A second occupancy of `sample_email`, in a different mailbox -- an + /// ordinary Gmail label, or a `COPY` another client made. + fn second_placement() -> Placement { + Placement { + account_id: "acct-outbox-log".to_string(), + folder_id: "Archive".to_string(), + uid_validity: "9".to_string(), + remote_id: "78".to_string(), + read: false, + } + } + + /// Seed the message into every given occupancy and dispatch one FLAG + /// mutation against it. Returns the honest disposition and every spec the + /// backend was actually asked to apply. + /// + /// Calls [`dispatch_mailbox_mutation`] rather than draining through + /// [`execute_pending_mutations`] deliberately. The drain's own + /// "attempting"/"completed" `tracing` callsites are what + /// `drain_logs_attempt_and_completion_without_leaking_message_content` + /// asserts on, and `tracing` caches callsite interest globally: a + /// concurrent test hitting those callsites with no recorder installed + /// caches them as uninteresting and makes that test fail. Addressing is + /// decided here anyway, so the narrower call is also the tighter assertion. + async fn dispatch_with_placements( + placements: &[Placement], + ) -> (Disposition, Vec<RemoteMutationSpec>) { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + for placement in placements { + db.save_email_at(&sample_email(), IdentitySource::Surrogate, placement) + .await + .expect("store the occupancy"); + } + + let backend = MockMailBackend::new(); + let env = SucceedingEnv { + backend: backend.clone(), + }; + let mutation = flag_mutation(); + let payload = + serde_json::from_str::<MutationPayload>(&mutation.payload).expect("parse payload"); + let disposition = + dispatch_mailbox_mutation(&db, &env, &mutation, &sample_email(), &payload).await; + (disposition, backend.applied_mutations()) + } + + /// The unambiguous case still works: exactly one occupancy, so the row's + /// target is fully determined and the mutation is addressed from it. + #[tokio::test] + async fn a_single_placement_mutation_is_addressed_from_that_placement() { + let (disposition, applied) = dispatch_with_placements(&[sample_placement()]).await; + + assert!( + matches!(disposition, Disposition::Completed), + "a fully-addressed mutation must go through" + ); + assert_eq!(applied.len(), 1, "the backend really was asked to act"); + assert_eq!(applied[0].folder_id, "INBOX"); + assert_eq!(applied[0].remote_id, "77"); + assert_eq!(applied[0].uid_validity, "9"); + assert_eq!(applied[0].message_id, "msg-outbox-log"); + } + + /// The ambiguous case refuses. The outbox row names only a message, so for + /// a message in two mailboxes there is no way to know which copy the rule + /// matched -- and a guessed MOVE or DELETE lands on mail the user never + /// targeted, with no undo. + #[tokio::test] + async fn a_multi_placement_mutation_is_refused_rather_than_guessed() { + let (disposition, applied) = + dispatch_with_placements(&[sample_placement(), second_placement()]).await; + + assert!( + applied.is_empty(), + "an ambiguously-addressed mutation must never reach the server" + ); + match disposition { + // Permanent, not Retry: no later attempt makes the outbox row + // remember a placement it never recorded. + Disposition::Permanent(reason) => { + assert!( + reason.contains("INBOX") && reason.contains("Archive"), + "the refusal must name the competing mailboxes: {reason}" + ); + assert!( + reason.contains("refusing"), + "and say plainly that it refused: {reason}" + ); + } + Disposition::Completed => panic!("expected a permanent refusal, got Completed"), + Disposition::Conflict(observed) => { + panic!("expected a permanent refusal, got Conflict({observed})") + } + Disposition::Retry(reason) => { + panic!("expected a permanent refusal, got Retry({reason})") + } + } + } } diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index a5998b3..13b1309 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -87,6 +87,13 @@ fn remote_action_tag(action: &RuleAction) -> Option<(&'static str, Option<String /// engineer ten times. The claim fails closed, so an unverifiable claim skips /// the actions rather than licensing them. /// +/// The claim is spent the moment it is won, before the actions are known to +/// have succeeded. A rule whose enqueue then fails is therefore not retried on +/// a later pass. That is the deliberate trade: re-firing FORWARD and CALL +/// WEBHOOK on a partial failure would re-deliver to a third party, and a +/// duplicate mail cannot be un-received, while a dropped one is visible in the +/// warning below and recoverable by hand. +/// /// The claim also subsumes the older "only call me for a new message" contract /// for re-arrivals of the same message. Callers should still gate on a genuinely /// new occupancy (see [`fetch_and_persist`]) -- the claim stops repeat *actions*, @@ -332,33 +339,35 @@ async fn fetch_and_persist( // What a folder stops mentioning is an occupancy, never a message: the // same mail may still sit in another mailbox, and it goes only when its // last placement does -- which `delete_placements` decides, not this. - if let Some(acct) = account_id { - let mut gone: Vec<PlacementKey> = changes.removals; + // + // A `PlacementKey` names its own account, so an explicitly-reported + // removal is actionable with no ambient account context at all. Only + // the `present` diff needs one, because its other half is a lookup + // keyed by (account, folder). + let mut gone: Vec<PlacementKey> = changes.removals; - // `present` is the folder's complete contents when the pass was - // able to enumerate them. `None` and `Some(vec![])` are NOT the - // same: `None` means the pass was incremental and cannot speak to - // absence, and treating it as "nothing is present" would delete - // the folder. - if let Some(present) = changes.present { - let present: std::collections::HashSet<PlacementKey> = - present.into_iter().collect(); - let stored = db.placements_in_folder(acct, &folder.id).await?; - gone.extend(stored.into_iter().filter(|key| !present.contains(key))); - } + // `present` is the folder's complete contents when the pass was + // able to enumerate them. `None` and `Some(vec![])` are NOT the + // same: `None` means the pass was incremental and cannot speak to + // absence, and treating it as "nothing is present" would delete + // the folder. + if let (Some(acct), Some(present)) = (account_id, changes.present) { + let present: std::collections::HashSet<PlacementKey> = present.into_iter().collect(); + let stored = db.placements_in_folder(acct, &folder.id).await?; + gone.extend(stored.into_iter().filter(|key| !present.contains(key))); + } - gone.sort_unstable(); - gone.dedup(); - if !gone.is_empty() { - let outcome = db.delete_placements(&gone).await?; - tracing::info!( - account_id = %acct, - folder_id = %folder.id, - placements_removed = outcome.placements_removed, - messages_reaped = outcome.messages_reaped, - "removed placements that are no longer on the server" - ); - } + gone.sort_unstable(); + gone.dedup(); + if !gone.is_empty() { + let outcome = db.delete_placements(&gone).await?; + tracing::info!( + account_id = ?account_id, + folder_id = %folder.id, + placements_removed = outcome.placements_removed, + messages_reaped = outcome.messages_reaped, + "removed placements that are no longer on the server" + ); } // Persist the returned checkpoint only AFTER this folder's messages @@ -1563,4 +1572,48 @@ mod tests { "and nothing may be recorded as having fired" ); } + + #[tokio::test] + async fn an_account_less_pass_still_applies_server_confirmed_removals() { + // A `PlacementKey` names its own account, so an explicitly-reported + // removal needs no ambient account context. Gating the whole removals + // block on `account_id.is_some()` would silently discard a VANISHED + // report on a mock-driven `SyncAll`, leaving a ghost the server has + // already told us is gone. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let event_bus = EventBus::new(); + let filter_engine = empty_filter_engine(); + + let staged = mock_placed("m-vanished", "inbox", "Deleted elsewhere"); + db.save_email_at(&staged.email, staged.source, &staged.placement) + .await + .expect("store the occupancy a prior pass left behind"); + + let mock = MockMailBackend::new(); + mock.add_folder(Folder { + id: "inbox".to_string(), + name: "Inbox".to_string(), + total_messages: 0, + unread_messages: 0, + }); + mock.set_removals(vec![staged.placement.key()]); + + // `None` account: the pass has nowhere to key a checkpoint and cannot + // run the `present` diff, but the removal is still actionable. + let synced = fetch_and_persist(&db, &event_bus, &mock, &filter_engine, None) + .await + .expect("sync succeeds"); + assert_eq!(synced, 0, "nothing new arrived"); + + let placements = db + .placements_of("m-vanished") + .await + .expect("read the message's placements"); + assert!( + placements.is_empty(), + "the server-confirmed removal must be applied without an account id" + ); + } } -- 2.54.0 From dbe7acdb72db4c115c5335bee9a2dbcd8e2192b4 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 10:12:56 -0500 Subject: [PATCH 75/95] docs(proto): describe content-addressed ids and placement-scoped folders Message.id is no longer a folder-scoped surrogate: it is content-addressed from the strongest identity the server offered, so it survives a move. Message.folder_id names the one occupancy the message was reached through, since a message can sit in several folders at once. Triage's contract is stated too -- it claims each (rule, message) pair once and for all, so a second run over the same rules matches nothing. Refs #394 --- .../nuncio-proto/proto/nuncio/v1/nuncio.proto | 21 +++++++++++++++---- crates/nunciod/src/grpc.rs | 6 +++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto index 41302bf..abe84f8 100644 --- a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto +++ b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto @@ -519,10 +519,11 @@ message Attachment { // (rather than `Email`) to avoid colliding with common generated-code // identifiers in hand-written Rust using this module. message Message { - // Unique message identifier. OPAQUE and server-assigned: a deterministic - // surrogate derived from the message's stable protocol identity, never a - // raw upstream protocol id (e.g. an IMAP UID, which is only stable within - // a single IMAP session and is NOT a durable cross-session identifier). + // Unique message identifier. OPAQUE and server-assigned: content-addressed + // from the message's strongest available identity, so it is independent of + // which folder the message currently occupies and survives a move. Never a + // raw upstream protocol id (e.g. an IMAP UID, which is only stable within a + // single IMAP session and is NOT a durable cross-session identifier). // Clients MUST treat this as an opaque token -- pass it back verbatim in // later requests, but never parse, construct, or assume any structure in // its value. @@ -530,6 +531,11 @@ message Message { // Account identifier owning the message. string account_id = 2; // Mailbox folder identifier (e.g. "inbox"). + // + // A message can occupy several folders at once. This names the one the + // message was reached through: the requested folder for a listing, and the + // lowest-sorting folder id for a lookup by id. A field carrying the complete + // placement set is not yet part of this contract. string folder_id = 3; // Subject line. string subject = 4; @@ -777,6 +783,13 @@ service Filters { // applied retroactively to mail that was synced before those rules // existed. Streams cumulative progress as it works, ending with a final // `done: true` message once the whole store has been scanned. + // + // Each (rule, message) pair fires at most once, ever -- the same claim + // ledger live sync uses, keyed by rule id and message id. This is not a + // repeatable rescan: running Triage again with the same rules finds + // nothing left to claim and reports `matched_count: 0`. Only messages + // stored since the last run, or rules created since then, produce further + // matches; editing an existing rule's contents does not reset its claims. rpc Triage(TriageRequest) returns (stream TriageProgress); } diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index c8b4d12..1a9b8be 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -4798,9 +4798,9 @@ mod tests { /// Paging a folder larger than `page_size` across multiple pages via /// `next_page_token` must return every message EXACTLY ONCE, with no /// duplicates and no gaps, and in the stable newest-first keyset order - /// (`received_at DESC, id DESC`). Some messages deliberately share a - /// `received_at` so the `(received_at, id)` tiebreaker is exercised across - /// a page boundary. + /// (`received_at DESC, message_key DESC, uidvalidity DESC, uid DESC`). + /// Some messages deliberately share a `received_at` so the tiebreaker + /// fields are exercised across a page boundary. #[tokio::test] async fn list_messages_keyset_pagination_returns_every_message_once_no_gaps() { let (db, _dir) = DatabaseEngine::connect_ephemeral() -- 2.54.0 From 1be51907ab360c666cca02c40618b7781eb1adab Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 10:21:11 -0500 Subject: [PATCH 76/95] docs(proto): stop overstating when a message id survives a move The surrogate tier hashes the folder, so a message with no usable Message-ID -- including any fetched headers-only -- gets a folder-scoped id that changes when it moves. Saying ids survive a move unconditionally was false for exactly the messages most likely to have one. --- crates/nuncio-proto/proto/nuncio/v1/nuncio.proto | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto index abe84f8..2abaec2 100644 --- a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto +++ b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto @@ -520,10 +520,18 @@ message Attachment { // identifiers in hand-written Rust using this module. message Message { // Unique message identifier. OPAQUE and server-assigned: content-addressed - // from the message's strongest available identity, so it is independent of - // which folder the message currently occupies and survives a move. Never a - // raw upstream protocol id (e.g. an IMAP UID, which is only stable within a + // from the strongest identity the message's server offered. Never a raw + // upstream protocol id (e.g. an IMAP UID, which is only stable within a // single IMAP session and is NOT a durable cross-session identifier). + // + // Stability across a move depends on what the server offered. Where a + // server-assigned object id or a usable Message-ID was available, the id is + // folder-independent and survives a move between folders. Where none was -- + // a message carrying no Message-ID, or one fetched headers-only -- the id + // falls back to a derivation that includes the folder, and moving such a + // message yields a different id. Clients MUST NOT assume an id outlives a + // move. + // // Clients MUST treat this as an opaque token -- pass it back verbatim in // later requests, but never parse, construct, or assume any structure in // its value. -- 2.54.0 From 7fadc04662f89a9971212bdd02709b64de927bae Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 10:27:43 -0500 Subject: [PATCH 77/95] docs(roadmap): refresh WS counts and record the ADR 0002 split WS-B, WS-C and WS-E grew when ADR 0002 broke the multi-engine sync model into #386-#397, and the 2026-08-07 snapshot never caught up. Counts now match the live milestones. Nothing is marked closed: the identity split and its chain are green on a branch, and the numbers move when it merges. Refs #394 --- docs/ROADMAP.md | 18 ++++++++++++++---- docs/roadmap/index.html | 6 ++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index c564ae6..a9bf4be 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -228,19 +228,29 @@ The engine becomes production-solid. ### M6.5, WS and OBS — the work between M6 and M7 Opened after M5, when the reshape needed before an honest `v1` freeze proved -larger than M6. Status as of 2026-08-07: +larger than M6. Status as of 2026-08-12: | Track | Closed / Open | Scope | | :--- | :---: | :--- | | **M6.5** Feature completeness & de-fabrication | 3 / 8 | Remove the remaining fabricated or silently-wrong paths (Contacts sync wiring #255, DAV XML namespaces #257, TLS-mode fallback #271, NL scheduler #258) | | **WS-A** Contract hardening (pre-freeze reshape) | 6 / 3 | Typed errors, `Timestamp`/`Duration`, keyset pagination, id conventions, enum hygiene, account transport `oneof` — **gates M7** | -| **WS-B** Mail model & mutations | 1 / 9 | Message identity, drafts, folder management, attachments on receive | -| **WS-C** Sync, push & lifecycle | 4 / 4 | Autonomous scheduler, IMAP IDLE, resumable event stream, graceful shutdown | +| **WS-B** Mail model & mutations | 1 / 14 | Message identity, drafts, folder management, attachments on receive | +| **WS-C** Sync, push & lifecycle | 4 / 9 | Autonomous scheduler, IMAP IDLE, resumable event stream, graceful shutdown | | **WS-D** Security to 9/10 | 4 / 8 | Whole-DB SQLCipher (#299), UDS / named-pipe IPC (#305), signed updates (#300), HTML sanitization (#302) | -| **WS-E** Calendar/Contacts write & Search | 2 / 10 | DAV write-back with conditional PUT/DELETE, recurrence editing, structured Search service | +| **WS-E** Calendar/Contacts write & Search | 2 / 12 | DAV write-back with conditional PUT/DELETE, recurrence editing, structured Search service | | **WS-F** Ops, usability & maintainability | 5 / 10 | CLI polish, `System.Backup`, and the `grpc.rs` (#333) / `DatabaseEngine` (#334) decompositions | | **OBS-1 … OBS-9** Observability | 9 / 0 | **COMPLETE.** Tracing subscriber, RPC spans + request-id, domain/lifecycle coverage, protocol instrumentation, `GetStatus`, CLI verbosity, redaction canary | +WS-B and WS-C grew when [ADR 0002](adr/0002-multi-engine-sync-model.md) decided +the convergent multi-engine sync model and broke it into #386–#397. Those are +in flight, not closed: the message/placement identity split, Message-ID and +content-hash capture, `user_version` gating, the raw IMAP command layer, the +four-rung change enumeration ladder, and three-state verified mutations are all +implemented and green on a branch, and the counts above move when it merges. +Still open behind them: the mutation RPCs and durable conflict surface (#395), +DAV href/ETag identity and `sync-collection` (#392, #393), the scheduler and +rate limiting (#396), and the ≥3-engine convergence harness (#397). + The OBS stories (#357–#365) were merged without a milestone assignment; the epic is complete regardless, and `docs/LOGGING.md` is its reference. diff --git a/docs/roadmap/index.html b/docs/roadmap/index.html index ac57958..2aa1164 100644 --- a/docs/roadmap/index.html +++ b/docs/roadmap/index.html @@ -280,11 +280,13 @@ <p class="cbody">Opened after M5, when the reshape needed before an honest <code>v1</code> freeze proved larger than M6 alone. <b>M6.5</b> de-fabrication (3/11) removes the remaining silently-wrong paths: Contacts sync wiring (#255), namespace-aware DAV XML (#257), the TLS-mode fallback (#271), the NL scheduler (#258). - <b>WS-A … WS-F</b> (22/66) is the pre-freeze reshape: contract hardening (WS-A, <b>gates M7</b>), mail model + <b>WS-A … WS-F</b> (22/78) is the pre-freeze reshape: contract hardening (WS-A, <b>gates M7</b>), mail model & mutations (WS-B), sync/push/lifecycle (WS-C), security to 9/10 (WS-D — SQLCipher #299, UDS & named-pipe IPC #305, signed updates #300), calendar/contacts write-back & search (WS-E), and ops, usability & maintainability (WS-F — including the <code>grpc.rs</code> #333 and <code>DatabaseEngine</code> - #334 decompositions). <b>OBS-1 … OBS-9 is complete</b> (9/9, #357–#365): layered tracing subscriber, per-RPC + #334 decompositions). WS-B and WS-C grew when <b>ADR 0002</b> decided the convergent multi-engine sync + model and broke it into #386–#397; the message/placement identity split, change enumeration and verified + mutations are implemented and green on a branch, not yet merged. <b>OBS-1 … OBS-9 is complete</b> (9/9, #357–#365): layered tracing subscriber, per-RPC spans with request-id correlation, domain and lifecycle coverage, protocol instrumentation, a real <code>System.GetStatus</code>, CLI <code>-v</code>/<code>-vv</code> with typed errors, and a <code>Redacted<T></code> policy with a no-secrets-in-logs canary.</p> -- 2.54.0 From 79f357864aea66606aafd74ea614ea093f73ca8d Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 10:45:28 -0500 Subject: [PATCH 78/95] fix(store): chunk IN-list lookups below SQLite's bind cap existing_placements binds four parameters per key into one row-value IN (...) list, and its caller hands it an entire folder: FolderChanges accumulates every batch into one Vec, so the Full rung -- an ordinary first sync, and the unrecognised-checkpoint fallback -- passes the whole mailbox. SQLITE_MAX_VARIABLE_NUMBER is 32766, so past 8191 placements the statement fails, the folder sync aborts before writing a checkpoint, and every retry fails identically. A first sync of any large mailbox was permanently broken. Split the batch internally at 4000 keys and union the results, so the limit is unreachable from outside rather than one forgetful caller away. existing_message_ids had the same shape at a 32766 ceiling and is chunked the same way; it stays because it is public API drawing the identity-vs-occupancy distinction, not because it has callers. Also correct three comments that documented traps as free: - The IMAP batching doc claimed the working set stays bounded regardless of mailbox size. Batching bounds each request, not memory; the parsed messages accumulate across batches. - Adding the EMAILID/X-GM-MSGID FETCH items is not free: it repoints every stored message's key, orphaning the old messages row and stranding its decrypted body in messages_fts. It needs the repoint path to reap first, or it leaks plaintext. - IMAP never requests the tier-1/2 identifiers, so an IMAP-derived key and a JMAP-derived key for the same message differ. Recorded on derive_message_key and on the IMAP place() helper. De-link ADR 0002 in the roadmap: the file exists on no merge target, so the markdown now matches the rendered page, which already used plain bold. --- crates/nuncio-core/src/model.rs | 12 ++ crates/nuncio-mail/src/imap.rs | 33 ++++- crates/nuncio-store/src/db.rs | 218 ++++++++++++++++++++++++-------- docs/ROADMAP.md | 2 +- 4 files changed, 208 insertions(+), 57 deletions(-) diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index 508ef43..0df4a26 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -349,6 +349,18 @@ impl Email { /// /// The surrogate tier is folder-scoped and therefore *not* stable across a /// move; it is the honest answer when the server offered nothing better. + /// + /// # Keys do not yet converge across engine types + /// + /// Precedence makes a key reproducible from the same inputs, not from the + /// same *message*. The IMAP engine does not currently request `EMAILID` or + /// `X-GM-MSGID`, so it can only ever reach the `Message-ID`+content tier or + /// the surrogate, while JMAP supplies a server id and enters at the top + /// tier. Two engines syncing one account therefore produce disjoint key + /// spaces: the same message is two identities, one per engine. This is a + /// known gap in the convergent multi-engine model, not a property to rely + /// on -- do not assume a key derived by one engine addresses the row + /// another engine wrote. pub fn derive_message_key( account_id: &str, remote: RemoteIdentity<'_>, diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index bbb6fe1..fe93558 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -822,15 +822,21 @@ impl ImapEngine { /// exercise the multi-batch path with a small window instead of a huge /// fixture. /// - /// The sync runs in two phases so the transient working set stays bounded - /// regardless of mailbox size: + /// The sync runs in two phases: /// 1. A cheap enumeration ([`build_enumerate_command_query`]) reads every /// in-range message's UID and `RFC822.SIZE` with NO body. Messages at or /// below [`MAX_MESSAGE_BODY_BYTES`] are queued for a full-body fetch; /// oversized ones are queued for a headers-only fetch and WARN-logged. /// 2. Each queue is drained in explicit UID-set batches of at most - /// `batch_size`, so the full bodies held in memory at once never exceed - /// one batch. + /// `batch_size`, so a single `UID FETCH` request never asks the server + /// for more than that many bodies at a time. + /// + /// Batching bounds the size of each *request*, not the memory this function + /// holds: the parsed messages accumulate into one `Vec` across every batch, + /// so the returned [`FolderChanges`] carries the whole in-range set and the + /// peak working set still scales with the mailbox. That is the shape a + /// caller receiving `FolderChanges` depends on, so anything downstream must + /// be sized for a whole folder rather than a batch. #[tracing::instrument( skip(self, session), fields(account_id = %self.account_id, folder_id = %folder_id), @@ -1246,6 +1252,13 @@ impl ImapEngine { /// four loose arguments: they are already a single value at every call site, /// and spreading them out buys no clarity while pushing this function past /// the argument-count lint. + /// + /// In practice every IMAP call site passes `email_id: None` and + /// `gm_msgid: None` today, because the FETCH never asks for them, so this + /// engine reaches only the `Message-ID`+content tier or the surrogate. The + /// JMAP engine enters at the top tier, which means an IMAP-derived key and + /// a JMAP-derived key for the *same* message will not match. Syncing one + /// account over both engines does not converge on a shared identity yet. fn place( &self, folder_id: &str, @@ -1294,8 +1307,16 @@ impl ImapEngine { // EMAILID (RFC 8474) and X-GM-MSGID are not requested by the current // FETCH, so identity rests on the Message-ID/content pair where the // message carried a Message-ID and on the folder-scoped surrogate - // otherwise. Adding those FETCH items lifts existing rows to a stronger - // tier on the next resync at no cost to this code path. + // otherwise. + // + // Adding those FETCH items is not a local change. Every already-stored + // message would derive a DIFFERENT key on its next resync, so each + // re-reported placement repoints to the new key -- and the old message + // row, now referenced by nothing, is never reaped. Its decrypted body + // stays in the plaintext FTS index forever, which is exactly the leak + // the reap on placement removal exists to prevent. The FETCH items can + // only be requested once the repoint path reaps the message it + // abandons; until then, requesting them leaks plaintext. if let Some(raw_bytes) = fetch_data.body() { let mut email = MimeParserAdapter::parse_mime(&self.account_id, raw_bytes)?; let (identity, placement) = self.place( diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 4df914f..32e99b7 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -328,6 +328,20 @@ pub struct DeleteOutcome { pub messages_reaped: u64, } +/// How many keys one generated `IN (...)` list may carry. +/// +/// SQLite caps a statement's bind parameters at `SQLITE_MAX_VARIABLE_NUMBER`, +/// 32766 in the bundled build. The widest key built here spends four +/// parameters per entry (account, folder, uidvalidity, uid), so 4000 keys is +/// 16000 parameters -- comfortably under the cap with room for the limit to +/// halve, or for a future key to grow a fifth column, without silently +/// becoming a runtime error again. +/// +/// The cap is worth this margin because overrunning it is not a slow query but +/// a hard failure of the whole statement: the enclosing folder sync aborts, no +/// checkpoint is written, and every retry fails in exactly the same place. +const MAX_KEYS_PER_IN_LIST: usize = 4_000; + /// Serialize a [`nuncio_core::TlsMode`] to its stable on-disk `accounts` /// column form. Kept as a bare snake_case token (matching the column's /// `'implicit_tls'` default) rather than JSON, so a row inserted by the @@ -2080,90 +2094,103 @@ impl DatabaseEngine { .collect()) } - /// The subset of `keys` already stored, in one round trip. + /// The subset of `keys` already stored. /// - /// SQLite supports row-value `IN` lists, so the whole batch is one query + /// SQLite supports row-value `IN` lists, so a whole batch is one query /// rather than one existence check per placement. An empty slice /// short-circuits, since `IN ()` is not valid SQL. + /// + /// The batch is split internally at [`MAX_KEYS_PER_IN_LIST`] and the + /// results unioned, so callers may pass an arbitrarily large set: an entire + /// mailbox's worth of placements is exactly what a first sync produces, and + /// pushing the split onto callers means the ceiling is only ever one + /// forgetful caller away from aborting every retry of that sync + /// identically. Chunking here makes the limit unreachable from outside. pub async fn existing_placements( &self, keys: &[PlacementKey], ) -> Result<std::collections::HashSet<PlacementKey>, DatabaseError> { - if keys.is_empty() { - return Ok(std::collections::HashSet::new()); - } + let mut found = std::collections::HashSet::new(); - let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT account_id, folder_id, uidvalidity, uid FROM placements - WHERE (account_id, folder_id, uidvalidity, uid) IN (", - ); - for (index, key) in keys.iter().enumerate() { - if index > 0 { + for chunk in keys.chunks(MAX_KEYS_PER_IN_LIST) { + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT account_id, folder_id, uidvalidity, uid FROM placements + WHERE (account_id, folder_id, uidvalidity, uid) IN (", + ); + for (index, key) in chunk.iter().enumerate() { + if index > 0 { + builder.push(", "); + } + builder.push("("); + builder.push_bind(key.account_id.clone()); builder.push(", "); + builder.push_bind(key.folder_id.clone()); + builder.push(", "); + builder.push_bind(key.uid_validity.clone()); + builder.push(", "); + builder.push_bind(key.remote_id.clone()); + builder.push(")"); } - builder.push("("); - builder.push_bind(key.account_id.clone()); - builder.push(", "); - builder.push_bind(key.folder_id.clone()); - builder.push(", "); - builder.push_bind(key.uid_validity.clone()); - builder.push(", "); - builder.push_bind(key.remote_id.clone()); builder.push(")"); - } - builder.push(")"); - let rows: Vec<(String, String, String, String)> = builder - .build_query_as() - .fetch_all(&self.pool) - .await - .map_err(DatabaseError::Query)?; + let rows: Vec<(String, String, String, String)> = builder + .build_query_as() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; - Ok(rows - .into_iter() - .map( + found.extend(rows.into_iter().map( |(account_id, folder_id, uid_validity, remote_id)| PlacementKey { account_id, folder_id, uid_validity, remote_id, }, - ) - .collect()) + )); + } + + Ok(found) } /// Return the subset of `keys` whose message identity is already present in the - /// `messages` table, as a single `SELECT message_key FROM messages WHERE message_key - /// IN (...)` query. + /// `messages` table, via `SELECT message_key FROM messages WHERE message_key + /// IN (...)`. /// /// Lets a caller classify a whole fetched batch as new-vs-seen with one round trip instead /// of one existence lookup per key. Answers a strictly different question from /// [`Self::existing_placements`]: a message already known from another folder is *not* new /// here, but its arrival in this folder still is. Passing an empty slice short-circuits to /// an empty set without querying at all, since `IN ()` is not valid SQL. + /// + /// Chunked at [`MAX_KEYS_PER_IN_LIST`] for the same reason as + /// [`Self::existing_placements`]: the bind-parameter ceiling is a property + /// of the statement, so the only place it can be enforced once and for all + /// is where the statement is built. pub async fn existing_message_ids( &self, keys: &[String], ) -> Result<std::collections::HashSet<String>, DatabaseError> { - if keys.is_empty() { - return Ok(std::collections::HashSet::new()); + let mut found = std::collections::HashSet::new(); + + for chunk in keys.chunks(MAX_KEYS_PER_IN_LIST) { + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = + sqlx::QueryBuilder::new("SELECT message_key FROM messages WHERE message_key IN ("); + let mut separated = builder.separated(", "); + for key in chunk { + separated.push_bind(key); + } + builder.push(")"); + + let rows: Vec<(String,)> = builder + .build_query_as() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + found.extend(rows.into_iter().map(|(key,)| key)); } - let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = - sqlx::QueryBuilder::new("SELECT message_key FROM messages WHERE message_key IN ("); - let mut separated = builder.separated(", "); - for key in keys { - separated.push_bind(key); - } - builder.push(")"); - - let rows: Vec<(String,)> = builder - .build_query_as() - .fetch_all(&self.pool) - .await - .map_err(DatabaseError::Query)?; - - Ok(rows.into_iter().map(|(key,)| key).collect()) + Ok(found) } /// Save a [`nuncio_core::model::CalendarEvent`] to SQLite (INSERT OR REPLACE). @@ -5543,6 +5570,97 @@ mod tests { assert_eq!(known.len(), 1, "and the query must not echo its own input"); } + /// A first sync of a real mailbox hands `existing_placements` the entire + /// folder in one call. Each key spends four bind parameters, and SQLite + /// caps a statement at 32766 of them, so an unchunked `IN (...)` list dies + /// somewhere past 8191 keys -- and dies the same way on every retry, because + /// the folder sync aborts before writing a checkpoint. 10000 keys is + /// comfortably over that ceiling, and 6000 of them are really placed, so + /// this also fails if chunking loses or double-counts a chunk's results + /// rather than unioning them. + #[tokio::test] + async fn existing_placements_handles_a_batch_past_the_bind_parameter_ceiling() { + const PLACED: usize = 6_000; + const QUERIED: usize = 10_000; + + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Insert the placement rows directly: this test is about the read + // path's statement size, and 6000 encrypt-and-commit round trips + // through `save_email_at` would test nothing extra at real cost. + let mut tx = engine.pool().begin().await.unwrap(); + for uid in 0..PLACED { + sqlx::query( + "INSERT INTO placements + (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES ('acct-1', 'INBOX', '42', ?, ?, 0)", + ) + .bind(uid.to_string()) + .bind(format!("key-{uid}")) + .execute(&mut *tx) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + + let keys: Vec<PlacementKey> = (0..QUERIED) + .map(|uid| PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: uid.to_string(), + }) + .collect(); + + let found = engine.existing_placements(&keys).await.unwrap(); + + assert_eq!( + found.len(), + PLACED, + "every stored placement in the batch must be reported exactly once" + ); + assert!( + found.contains(&keys[0]) && found.contains(&keys[PLACED - 1]), + "hits at both ends of the stored range must survive the split" + ); + assert!( + !found.contains(&keys[PLACED]), + "and a key that was never placed must not be invented by the union" + ); + } + + /// The same ceiling in the one-parameter-per-key shape. 32766 parameters is + /// far away here, but the chunk boundary is not, and a batch that straddles + /// several chunks must still come back as one answer. + #[tokio::test] + async fn existing_message_ids_handles_a_batch_spanning_several_chunks() { + const QUERIED: usize = 10_000; + // Deliberately placed in different chunks of the split. + const STORED: [usize; 3] = [0, 5_000, 9_999]; + + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + for (nth, index) in STORED.iter().enumerate() { + let (email, placement) = + sample_message_and_placement(&format!("key-{index}"), "INBOX", &nth.to_string()); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &placement, + ) + .await + .unwrap(); + } + + let keys: Vec<String> = (0..QUERIED).map(|n| format!("key-{n}")).collect(); + let found = engine.existing_message_ids(&keys).await.unwrap(); + + assert_eq!(found.len(), STORED.len(), "no chunk's result may be lost"); + for index in STORED { + assert!(found.contains(&format!("key-{index}"))); + } + } + /// `save_email_at` reads (the placement existence check) and then writes what /// it read, so it must serialize rather than error under concurrency: in WAL /// mode a deferred transaction upgrading to a writer aborts with diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a9bf4be..5dda291 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -241,7 +241,7 @@ larger than M6. Status as of 2026-08-12: | **WS-F** Ops, usability & maintainability | 5 / 10 | CLI polish, `System.Backup`, and the `grpc.rs` (#333) / `DatabaseEngine` (#334) decompositions | | **OBS-1 … OBS-9** Observability | 9 / 0 | **COMPLETE.** Tracing subscriber, RPC spans + request-id, domain/lifecycle coverage, protocol instrumentation, `GetStatus`, CLI verbosity, redaction canary | -WS-B and WS-C grew when [ADR 0002](adr/0002-multi-engine-sync-model.md) decided +WS-B and WS-C grew when **ADR 0002** decided the convergent multi-engine sync model and broke it into #386–#397. Those are in flight, not closed: the message/placement identity split, Message-ID and content-hash capture, `user_version` gating, the raw IMAP command layer, the -- 2.54.0 From 4f6a024d3b9983536cca723825cc16809ac095ec Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 10:46:08 -0500 Subject: [PATCH 79/95] docs(roadmap): reconcile the prose WS total with the table --- docs/ROADMAP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 5dda291..c23b358 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -104,7 +104,7 @@ part of the original M1–M7 numbering: `tls_mode_from_db` silent `ImplicitTls` fallback (#271), the `NaturalLanguageScheduler` that returns wrong times (#258), plus CLI and audit cleanups. -- **WS-A … WS-F — the pre-freeze reshape** (22 closed / 44 open). Six tracks +- **WS-A … WS-F — the pre-freeze reshape** (22 closed / 56 open). Six tracks that must land before `nuncio.v1` can be frozen: **WS-A** contract hardening (typed errors, `Timestamp`/`Duration` unification, keyset pagination, id conventions, enum hygiene, account transport `oneof`); **WS-B** mail model and -- 2.54.0 From 99a8d1a96fb972b8b220cbb4c13995bff955e2ce Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 11:47:34 -0500 Subject: [PATCH 80/95] fix(test): capture logs through one global subscriber, not a thread-local tracing resolves a callsite's Interest once per process and caches it. A callsite first executed on a thread with no subscriber caches never and is skipped forever, before any thread-local subscriber is consulted. Under the parallel harness, whichever thread reached a callsite first decided whether any other test could capture it, so every log-capture assertion in this crate was a coin flip -- reproducible at --test-threads=2, which is what CI runners give us. Install the capture layer once as the global default and route events to a per-thread recorder instead. Rebuild the interest cache once at install so callsites already resolved against the no-op dispatcher are recoverable. Measured on nunciod: ~90% of runs red at --test-threads=2 before, 0/12 after. --- crates/nunciod/src/test_tracing.rs | 133 ++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 33 deletions(-) diff --git a/crates/nunciod/src/test_tracing.rs b/crates/nunciod/src/test_tracing.rs index 9c68c9b..57eabd5 100644 --- a/crates/nunciod/src/test_tracing.rs +++ b/crates/nunciod/src/test_tracing.rs @@ -1,15 +1,21 @@ //! Shared in-test `tracing` capture helper. //! -//! Installs a thread-local subscriber that records every span (with its -//! fields) and every event (with its level, fields, and message) into an -//! in-memory buffer, so tests can assert on what the code logged -- including -//! the negative assertion that a secret never appears in any recorded field. -//! Deterministic and fully offline: no global subscriber, no I/O. +//! Records every span (with its fields) and every event (with its level, +//! fields, and message) into an in-memory buffer, so tests can assert on what +//! the code logged -- including the negative assertion that a secret never +//! appears in any recorded field. Fully offline: no I/O. +//! +//! The subscriber is installed **once, globally**, and events are routed to a +//! per-thread recorder. That split is deliberate and load-bearing: a +//! thread-local subscriber cannot work here, because `tracing` caches each +//! callsite's `Interest` process-wide on first use. See +//! [`install_global_capture`]. #![cfg(test)] +use std::cell::RefCell; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use tracing::field::{Field, Visit}; use tracing::span::Attributes; @@ -88,46 +94,107 @@ impl Visit for FieldVisitor { } } -struct CaptureLayer(Arc<Recorder>); +thread_local! { + /// The recorder collecting telemetry for the test running on THIS thread, + /// or `None` on any thread not currently inside [`with_recorder`]. + /// + /// Routing per thread is what keeps the single global subscriber from + /// letting concurrent tests read each other's telemetry. + static ACTIVE_RECORDER: RefCell<Option<Arc<Recorder>>> = const { RefCell::new(None) }; +} + +/// Run `f` against the recorder installed for this thread, if any. +fn with_active<R>(f: impl FnOnce(&Recorder) -> R) -> Option<R> { + ACTIVE_RECORDER.with(|slot| slot.borrow().as_ref().map(|rec| f(rec))) +} + +struct CaptureLayer; impl<S> tracing_subscriber::Layer<S> for CaptureLayer where S: Subscriber + for<'a> LookupSpan<'a>, { fn on_new_span(&self, attrs: &Attributes<'_>, _id: &Id, _ctx: LayerContext<'_, S>) { - let mut visitor = FieldVisitor(HashMap::new()); - attrs.record(&mut visitor); - self.0 - .spans - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(RecordedSpan { - name: attrs.metadata().name().to_string(), - fields: visitor.0, - }); + with_active(|rec| { + let mut visitor = FieldVisitor(HashMap::new()); + attrs.record(&mut visitor); + rec.spans + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedSpan { + name: attrs.metadata().name().to_string(), + fields: visitor.0, + }); + }); } fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) { - let mut visitor = FieldVisitor(HashMap::new()); - event.record(&mut visitor); - self.0 - .events - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(RecordedEvent { - level: *event.metadata().level(), - fields: visitor.0, - }); + with_active(|rec| { + let mut visitor = FieldVisitor(HashMap::new()); + event.record(&mut visitor); + rec.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedEvent { + level: *event.metadata().level(), + fields: visitor.0, + }); + }); } } -/// Runs `f` with a thread-local capturing subscriber installed and returns the -/// recorder alongside `f`'s result. Because the subscriber is thread-local, -/// any async work driven inside `f` must run on the calling thread (e.g. a -/// current-thread runtime's `block_on`) for its telemetry to be captured. +/// Install the capturing subscriber as the process-wide default, exactly once. +/// +/// It MUST be global rather than thread-local, and this is the whole reason the +/// helper is shaped this way. `tracing` resolves each callsite's `Interest` +/// **once for the process** and caches it; a callsite first executed on a thread +/// with no subscriber caches `Interest::never()` and is then skipped forever -- +/// before any later thread-local subscriber is ever consulted. Under a parallel +/// test harness, whichever thread reaches a callsite first therefore decides +/// whether any other test can ever capture it, which made every log-capture +/// assertion in this crate a coin flip. +/// +/// A single global subscriber removes the race: every callsite registers against +/// a real subscriber, so interest is always enabled, and per-thread routing +/// (`ACTIVE_RECORDER`) decides what is actually recorded. +fn install_global_capture() { + static INSTALLED: OnceLock<()> = OnceLock::new(); + INSTALLED.get_or_init(|| { + let subscriber = tracing_subscriber::registry().with(CaptureLayer); + // A global default may already be set by another component of the same + // test binary; capturing still works through the thread-local routing + // above, so a lost race here is not fatal. + let _ = tracing::subscriber::set_global_default(subscriber); + // Callsites executed before this point resolved their interest against + // the no-op dispatcher and cached "never". Recompute so they can be + // captured from here on. Done once, at install -- not per test. + tracing::callsite::rebuild_interest_cache(); + }); +} + +/// Resets this thread's recorder slot on drop, so a panicking test cannot leak +/// its recorder into whatever the harness runs next on the same thread. +struct RecorderGuard; + +impl Drop for RecorderGuard { + fn drop(&mut self) { + ACTIVE_RECORDER.with(|slot| *slot.borrow_mut() = None); + } +} + +/// Runs `f` with telemetry from THIS thread captured, returning the recorder +/// alongside `f`'s result. +/// +/// Only this thread's telemetry is recorded, so any async work driven inside +/// `f` must run on the calling thread (e.g. a current-thread runtime's +/// `block_on`) to be captured -- unchanged from the previous thread-local +/// implementation. What changed is that the subscriber itself is now global; +/// see [`install_global_capture`] for why that is load-bearing. pub(crate) fn with_recorder<T>(f: impl FnOnce() -> T) -> (Arc<Recorder>, T) { + install_global_capture(); let recorder = Arc::new(Recorder::default()); - let subscriber = tracing_subscriber::registry().with(CaptureLayer(recorder.clone())); - let value = tracing::subscriber::with_default(subscriber, f); + ACTIVE_RECORDER.with(|slot| *slot.borrow_mut() = Some(recorder.clone())); + let _guard = RecorderGuard; + let value = f(); (recorder, value) } -- 2.54.0 From 1cd1bcc13c764432c9343530684024b18e77d418 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 12:26:46 -0500 Subject: [PATCH 81/95] fix(test): capture logs through a global subscriber in 4 more crates Ports the nunciod fix (99a8d1a) to nuncio-cal, nuncio-contacts, nuncio-store, and nuncio-filter: install the tracing capture subscriber once as the process default and route events to a per-thread buffer, instead of a thread-local subscriber that a racing callsite's cached Interest::never() can permanently defeat under a parallel test harness. nuncio-cal and nuncio-contacts get the same structured Recorder shape as nunciod. nuncio-store gains a formatted-text test_tracing helper (its assertions match on rendered level tags and substrings) used by vault.rs; the two migration-log CapturedLogs sites in db.rs are left untouched because those tests are being deleted separately. nuncio-filter keeps its dependency-free hand-rolled Subscriber but makes it global and per-thread-routed, replacing the local CapturingSubscriber in engine.rs and webhook.rs. --- crates/nuncio-cal/src/test_tracing.rs | 133 ++++++++++++++----- crates/nuncio-contacts/src/test_tracing.rs | 133 ++++++++++++++----- crates/nuncio-filter/src/engine.rs | 53 +------- crates/nuncio-filter/src/lib.rs | 2 + crates/nuncio-filter/src/test_tracing.rs | 146 +++++++++++++++++++++ crates/nuncio-filter/src/webhook.rs | 63 +-------- crates/nuncio-store/src/lib.rs | 2 + crates/nuncio-store/src/test_tracing.rs | 143 ++++++++++++++++++++ crates/nuncio-store/src/vault.rs | 38 +----- 9 files changed, 501 insertions(+), 212 deletions(-) create mode 100644 crates/nuncio-filter/src/test_tracing.rs create mode 100644 crates/nuncio-store/src/test_tracing.rs diff --git a/crates/nuncio-cal/src/test_tracing.rs b/crates/nuncio-cal/src/test_tracing.rs index 178898c..cdb5e73 100644 --- a/crates/nuncio-cal/src/test_tracing.rs +++ b/crates/nuncio-cal/src/test_tracing.rs @@ -1,15 +1,21 @@ //! Shared in-test `tracing` capture helper. //! -//! Installs a thread-local subscriber that records every span (with its -//! fields) and every event (with its level, fields, and message) into an -//! in-memory buffer, so tests can assert on what the code logged -- including -//! the negative assertion that a secret never appears in any recorded field. -//! Deterministic and fully offline: no global subscriber, no I/O. +//! Records every span (with its fields) and every event (with its level, +//! fields, and message) into an in-memory buffer, so tests can assert on what +//! the code logged -- including the negative assertion that a secret never +//! appears in any recorded field. Fully offline: no I/O. +//! +//! The subscriber is installed **once, globally**, and events are routed to a +//! per-thread recorder. That split is deliberate and load-bearing: a +//! thread-local subscriber cannot work here, because `tracing` caches each +//! callsite's `Interest` process-wide on first use. See +//! [`install_global_capture`]. #![cfg(test)] +use std::cell::RefCell; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use tracing::field::{Field, Visit}; use tracing::span::Attributes; @@ -89,46 +95,107 @@ impl Visit for FieldVisitor { } } -struct CaptureLayer(Arc<Recorder>); +thread_local! { + /// The recorder collecting telemetry for the test running on THIS thread, + /// or `None` on any thread not currently inside [`with_recorder`]. + /// + /// Routing per thread is what keeps the single global subscriber from + /// letting concurrent tests read each other's telemetry. + static ACTIVE_RECORDER: RefCell<Option<Arc<Recorder>>> = const { RefCell::new(None) }; +} + +/// Run `f` against the recorder installed for this thread, if any. +fn with_active<R>(f: impl FnOnce(&Recorder) -> R) -> Option<R> { + ACTIVE_RECORDER.with(|slot| slot.borrow().as_ref().map(|rec| f(rec))) +} + +struct CaptureLayer; impl<S> tracing_subscriber::Layer<S> for CaptureLayer where S: Subscriber + for<'a> LookupSpan<'a>, { fn on_new_span(&self, attrs: &Attributes<'_>, _id: &Id, _ctx: LayerContext<'_, S>) { - let mut visitor = FieldVisitor(HashMap::new()); - attrs.record(&mut visitor); - self.0 - .spans - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(RecordedSpan { - name: attrs.metadata().name().to_string(), - fields: visitor.0, - }); + with_active(|rec| { + let mut visitor = FieldVisitor(HashMap::new()); + attrs.record(&mut visitor); + rec.spans + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedSpan { + name: attrs.metadata().name().to_string(), + fields: visitor.0, + }); + }); } fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) { - let mut visitor = FieldVisitor(HashMap::new()); - event.record(&mut visitor); - self.0 - .events - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(RecordedEvent { - level: *event.metadata().level(), - fields: visitor.0, - }); + with_active(|rec| { + let mut visitor = FieldVisitor(HashMap::new()); + event.record(&mut visitor); + rec.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedEvent { + level: *event.metadata().level(), + fields: visitor.0, + }); + }); } } -/// Runs `f` with a thread-local capturing subscriber installed and returns the -/// recorder alongside `f`'s result. Because the subscriber is thread-local, -/// any async work driven inside `f` must run on the calling thread (e.g. a -/// current-thread runtime's `block_on`) for its telemetry to be captured. +/// Install the capturing subscriber as the process-wide default, exactly once. +/// +/// It MUST be global rather than thread-local, and this is the whole reason the +/// helper is shaped this way. `tracing` resolves each callsite's `Interest` +/// **once for the process** and caches it; a callsite first executed on a thread +/// with no subscriber caches `Interest::never()` and is then skipped forever -- +/// before any later thread-local subscriber is ever consulted. Under a parallel +/// test harness, whichever thread reaches a callsite first therefore decides +/// whether any other test can ever capture it, which made every log-capture +/// assertion in this crate a coin flip. +/// +/// A single global subscriber removes the race: every callsite registers against +/// a real subscriber, so interest is always enabled, and per-thread routing +/// (`ACTIVE_RECORDER`) decides what is actually recorded. +fn install_global_capture() { + static INSTALLED: OnceLock<()> = OnceLock::new(); + INSTALLED.get_or_init(|| { + let subscriber = tracing_subscriber::registry().with(CaptureLayer); + // A global default may already be set by another component of the same + // test binary; capturing still works through the thread-local routing + // above, so a lost race here is not fatal. + let _ = tracing::subscriber::set_global_default(subscriber); + // Callsites executed before this point resolved their interest against + // the no-op dispatcher and cached "never". Recompute so they can be + // captured from here on. Done once, at install -- not per test. + tracing::callsite::rebuild_interest_cache(); + }); +} + +/// Resets this thread's recorder slot on drop, so a panicking test cannot leak +/// its recorder into whatever the harness runs next on the same thread. +struct RecorderGuard; + +impl Drop for RecorderGuard { + fn drop(&mut self) { + ACTIVE_RECORDER.with(|slot| *slot.borrow_mut() = None); + } +} + +/// Runs `f` with telemetry from THIS thread captured, returning the recorder +/// alongside `f`'s result. +/// +/// Only this thread's telemetry is recorded, so any async work driven inside +/// `f` must run on the calling thread (e.g. a current-thread runtime's +/// `block_on`) to be captured -- unchanged from the previous thread-local +/// implementation. What changed is that the subscriber itself is now global; +/// see [`install_global_capture`] for why that is load-bearing. pub(crate) fn with_recorder<T>(f: impl FnOnce() -> T) -> (Arc<Recorder>, T) { + install_global_capture(); let recorder = Arc::new(Recorder::default()); - let subscriber = tracing_subscriber::registry().with(CaptureLayer(recorder.clone())); - let value = tracing::subscriber::with_default(subscriber, f); + ACTIVE_RECORDER.with(|slot| *slot.borrow_mut() = Some(recorder.clone())); + let _guard = RecorderGuard; + let value = f(); (recorder, value) } diff --git a/crates/nuncio-contacts/src/test_tracing.rs b/crates/nuncio-contacts/src/test_tracing.rs index 7e7c4bf..745b352 100644 --- a/crates/nuncio-contacts/src/test_tracing.rs +++ b/crates/nuncio-contacts/src/test_tracing.rs @@ -1,15 +1,21 @@ //! Shared in-test `tracing` capture helper. //! -//! Installs a thread-local subscriber that records every span (with its -//! fields) and every event (with its level, fields, and message) into an -//! in-memory buffer, so tests can assert on what the code logged -- including -//! the negative assertion that a secret never appears in any recorded field. -//! Deterministic and fully offline: no global subscriber, no I/O. +//! Records every span (with its fields) and every event (with its level, +//! fields, and message) into an in-memory buffer, so tests can assert on what +//! the code logged -- including the negative assertion that a secret never +//! appears in any recorded field. Fully offline: no I/O. +//! +//! The subscriber is installed **once, globally**, and events are routed to a +//! per-thread recorder. That split is deliberate and load-bearing: a +//! thread-local subscriber cannot work here, because `tracing` caches each +//! callsite's `Interest` process-wide on first use. See +//! [`install_global_capture`]. #![cfg(test)] +use std::cell::RefCell; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use tracing::field::{Field, Visit}; use tracing::span::Attributes; @@ -90,46 +96,107 @@ impl Visit for FieldVisitor { } } -struct CaptureLayer(Arc<Recorder>); +thread_local! { + /// The recorder collecting telemetry for the test running on THIS thread, + /// or `None` on any thread not currently inside [`with_recorder`]. + /// + /// Routing per thread is what keeps the single global subscriber from + /// letting concurrent tests read each other's telemetry. + static ACTIVE_RECORDER: RefCell<Option<Arc<Recorder>>> = const { RefCell::new(None) }; +} + +/// Run `f` against the recorder installed for this thread, if any. +fn with_active<R>(f: impl FnOnce(&Recorder) -> R) -> Option<R> { + ACTIVE_RECORDER.with(|slot| slot.borrow().as_ref().map(|rec| f(rec))) +} + +struct CaptureLayer; impl<S> tracing_subscriber::Layer<S> for CaptureLayer where S: Subscriber + for<'a> LookupSpan<'a>, { fn on_new_span(&self, attrs: &Attributes<'_>, _id: &Id, _ctx: LayerContext<'_, S>) { - let mut visitor = FieldVisitor(HashMap::new()); - attrs.record(&mut visitor); - self.0 - .spans - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(RecordedSpan { - name: attrs.metadata().name().to_string(), - fields: visitor.0, - }); + with_active(|rec| { + let mut visitor = FieldVisitor(HashMap::new()); + attrs.record(&mut visitor); + rec.spans + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedSpan { + name: attrs.metadata().name().to_string(), + fields: visitor.0, + }); + }); } fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) { - let mut visitor = FieldVisitor(HashMap::new()); - event.record(&mut visitor); - self.0 - .events - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(RecordedEvent { - level: *event.metadata().level(), - fields: visitor.0, - }); + with_active(|rec| { + let mut visitor = FieldVisitor(HashMap::new()); + event.record(&mut visitor); + rec.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(RecordedEvent { + level: *event.metadata().level(), + fields: visitor.0, + }); + }); } } -/// Runs `f` with a thread-local capturing subscriber installed and returns the -/// recorder alongside `f`'s result. Because the subscriber is thread-local, -/// any async work driven inside `f` must run on the calling thread (e.g. a -/// current-thread runtime's `block_on`) for its telemetry to be captured. +/// Install the capturing subscriber as the process-wide default, exactly once. +/// +/// It MUST be global rather than thread-local, and this is the whole reason the +/// helper is shaped this way. `tracing` resolves each callsite's `Interest` +/// **once for the process** and caches it; a callsite first executed on a thread +/// with no subscriber caches `Interest::never()` and is then skipped forever -- +/// before any later thread-local subscriber is ever consulted. Under a parallel +/// test harness, whichever thread reaches a callsite first therefore decides +/// whether any other test can ever capture it, which made every log-capture +/// assertion in this crate a coin flip. +/// +/// A single global subscriber removes the race: every callsite registers against +/// a real subscriber, so interest is always enabled, and per-thread routing +/// (`ACTIVE_RECORDER`) decides what is actually recorded. +fn install_global_capture() { + static INSTALLED: OnceLock<()> = OnceLock::new(); + INSTALLED.get_or_init(|| { + let subscriber = tracing_subscriber::registry().with(CaptureLayer); + // A global default may already be set by another component of the same + // test binary; capturing still works through the thread-local routing + // above, so a lost race here is not fatal. + let _ = tracing::subscriber::set_global_default(subscriber); + // Callsites executed before this point resolved their interest against + // the no-op dispatcher and cached "never". Recompute so they can be + // captured from here on. Done once, at install -- not per test. + tracing::callsite::rebuild_interest_cache(); + }); +} + +/// Resets this thread's recorder slot on drop, so a panicking test cannot leak +/// its recorder into whatever the harness runs next on the same thread. +struct RecorderGuard; + +impl Drop for RecorderGuard { + fn drop(&mut self) { + ACTIVE_RECORDER.with(|slot| *slot.borrow_mut() = None); + } +} + +/// Runs `f` with telemetry from THIS thread captured, returning the recorder +/// alongside `f`'s result. +/// +/// Only this thread's telemetry is recorded, so any async work driven inside +/// `f` must run on the calling thread (e.g. a current-thread runtime's +/// `block_on`) to be captured -- unchanged from the previous thread-local +/// implementation. What changed is that the subscriber itself is now global; +/// see [`install_global_capture`] for why that is load-bearing. pub(crate) fn with_recorder<T>(f: impl FnOnce() -> T) -> (Arc<Recorder>, T) { + install_global_capture(); let recorder = Arc::new(Recorder::default()); - let subscriber = tracing_subscriber::registry().with(CaptureLayer(recorder.clone())); - let value = tracing::subscriber::with_default(subscriber, f); + ACTIVE_RECORDER.with(|slot| *slot.borrow_mut() = Some(recorder.clone())); + let _guard = RecorderGuard; + let value = f(); (recorder, value) } diff --git a/crates/nuncio-filter/src/engine.rs b/crates/nuncio-filter/src/engine.rs index d5d1230..72605aa 100644 --- a/crates/nuncio-filter/src/engine.rs +++ b/crates/nuncio-filter/src/engine.rs @@ -434,51 +434,6 @@ impl FilterEngine { mod tests { use super::*; use nuncio_core::model::Email; - use std::sync::Mutex; - use tracing::field::{Field, Visit}; - use tracing::span; - - /// Minimal `tracing::Subscriber` that records a formatted line per event - /// so tests can assert on emitted level + fields without pulling in - /// `tracing-subscriber`'s registry machinery. - struct CapturingSubscriber { - events: Arc<Mutex<Vec<String>>>, - } - - struct LineVisitor<'a>(&'a mut String); - - impl Visit for LineVisitor<'_> { - fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { - self.0.push_str(&format!(" {}={:?}", field.name(), value)); - } - } - - impl tracing::Subscriber for CapturingSubscriber { - fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool { - true - } - - fn new_span(&self, _span: &span::Attributes<'_>) -> span::Id { - span::Id::from_u64(1) - } - - fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {} - - fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {} - - fn event(&self, event: &tracing::Event<'_>) { - let mut line = format!("{}", event.metadata().level()); - let mut visitor = LineVisitor(&mut line); - event.record(&mut visitor); - if let Ok(mut events) = self.events.lock() { - events.push(line); - } - } - - fn enter(&self, _span: &span::Id) {} - - fn exit(&self, _span: &span::Id) {} - } fn test_email(account_id: &str, subject: &str, folder_id: &str) -> Email { Email { @@ -697,11 +652,7 @@ mod tests { // real blocking-pool thread while the timer races it on the async // task, so the timer reliably wins for any evaluation slower than a // few microseconds. - let events: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(Vec::new())); - let subscriber = CapturingSubscriber { - events: events.clone(), - }; - let _guard = tracing::subscriber::set_default(subscriber); + let logs = crate::test_tracing::capture_logs(); let rule = FilterRule { id: "slow-rule".to_string(), @@ -734,7 +685,7 @@ mod tests { not fabricated as a match" ); - let captured = events.lock().unwrap(); + let captured = logs.lines(); assert!( captured.iter().any(|line| line.contains("WARN") && line.contains("timed out") diff --git a/crates/nuncio-filter/src/lib.rs b/crates/nuncio-filter/src/lib.rs index 7056052..d96fdf3 100644 --- a/crates/nuncio-filter/src/lib.rs +++ b/crates/nuncio-filter/src/lib.rs @@ -8,6 +8,8 @@ pub mod codegen; pub mod engine; pub mod outbox; pub mod parser; +#[cfg(test)] +mod test_tracing; pub mod validator; pub mod webhook; diff --git a/crates/nuncio-filter/src/test_tracing.rs b/crates/nuncio-filter/src/test_tracing.rs new file mode 100644 index 0000000..bfe547e --- /dev/null +++ b/crates/nuncio-filter/src/test_tracing.rs @@ -0,0 +1,146 @@ +//! Shared in-test `tracing` capture helper. +//! +//! Records one formatted line per event (level plus `Debug`-rendered fields, +//! including the message) into an in-memory buffer, so tests can assert that +//! a level tag and specific fields appear on the SAME logged event. This +//! crate has no dependency on `tracing-subscriber`, so the capture is a +//! minimal, hand-rolled `tracing::Subscriber` rather than a `Layer` -- that +//! keeps this helper free of a new dependency. Fully offline: no I/O. +//! +//! The subscriber is installed **once, globally**, and events are routed to a +//! per-thread line buffer. That split is deliberate and load-bearing: a +//! thread-local subscriber cannot work here, because `tracing` caches each +//! callsite's `Interest` process-wide on first use. See +//! [`install_global_capture`]. + +#![cfg(test)] + +use std::cell::RefCell; +use std::sync::{Arc, Mutex, OnceLock}; + +use tracing::field::{Field, Visit}; +use tracing::span; + +thread_local! { + /// The line buffer collecting telemetry for the test running on THIS + /// thread, or `None` on any thread not currently inside [`capture_logs`]. + /// + /// Routing per thread is what keeps the single global subscriber from + /// letting concurrent tests read each other's telemetry. + static ACTIVE_LINES: RefCell<Option<Arc<Mutex<Vec<String>>>>> = const { RefCell::new(None) }; +} + +struct LineVisitor<'a>(&'a mut String); + +impl Visit for LineVisitor<'_> { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } +} + +/// Minimal `tracing::Subscriber` that records a formatted line per event +/// (level plus fields) without pulling in `tracing-subscriber`'s registry +/// machinery. Spans are not tracked (`new_span` returns a constant id) since +/// no test in this crate asserts on span structure, only on event lines. +struct CaptureSubscriber; + +impl tracing::Subscriber for CaptureSubscriber { + fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool { + true + } + + fn new_span(&self, _span: &span::Attributes<'_>) -> span::Id { + span::Id::from_u64(1) + } + + fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {} + + fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {} + + fn event(&self, event: &tracing::Event<'_>) { + let mut line = format!("{}", event.metadata().level()); + let mut visitor = LineVisitor(&mut line); + event.record(&mut visitor); + ACTIVE_LINES.with(|slot| { + if let Some(lines) = slot.borrow().as_ref() { + lines.lock().unwrap_or_else(|e| e.into_inner()).push(line); + } + }); + } + + fn enter(&self, _span: &span::Id) {} + + fn exit(&self, _span: &span::Id) {} +} + +/// Install the capturing subscriber as the process-wide default, exactly once. +/// +/// It MUST be global rather than thread-local, and this is the whole reason the +/// helper is shaped this way. `tracing` resolves each callsite's `Interest` +/// **once for the process** and caches it; a callsite first executed on a thread +/// with no subscriber caches `Interest::never()` and is then skipped forever -- +/// before any later thread-local subscriber is ever consulted. Under a parallel +/// test harness, whichever thread reaches a callsite first therefore decides +/// whether any other test can ever capture it, which made every log-capture +/// assertion in this crate a coin flip. +/// +/// A single global subscriber removes the race: every callsite registers against +/// a real subscriber, so interest is always enabled, and per-thread routing +/// (`ACTIVE_LINES`) decides what is actually recorded. +fn install_global_capture() { + static INSTALLED: OnceLock<()> = OnceLock::new(); + INSTALLED.get_or_init(|| { + // A global default may already be set by another component of the same + // test binary; capturing still works through the thread-local routing + // above, so a lost race here is not fatal. + let _ = tracing::subscriber::set_global_default(CaptureSubscriber); + // Callsites executed before this point resolved their interest against + // the no-op dispatcher and cached "never". Recompute so they can be + // captured from here on. Done once, at install -- not per test. + tracing::callsite::rebuild_interest_cache(); + }); +} + +/// Resets this thread's line buffer slot on drop, so a panicking test cannot +/// leak its buffer into whatever the harness runs next on the same thread. +struct LinesGuard; + +impl Drop for LinesGuard { + fn drop(&mut self) { + ACTIVE_LINES.with(|slot| *slot.borrow_mut() = None); + } +} + +/// A capture in progress on the current thread. Held across the code under +/// test -- including any `.await` points, as long as the test stays on a +/// single OS thread (e.g. the default current-thread `#[tokio::test]` +/// runtime) -- then queried for the lines captured so far. +pub(crate) struct CapturedLogs { + lines: Arc<Mutex<Vec<String>>>, + _guard: LinesGuard, +} + +impl CapturedLogs { + /// A snapshot of every formatted event line captured on this thread so + /// far: level plus `Debug`-rendered fields (including `message`). + pub fn lines(&self) -> Vec<String> { + self.lines.lock().unwrap_or_else(|e| e.into_inner()).clone() + } +} + +/// Start capturing this thread's `tracing` events as formatted lines. +/// +/// Only this thread's events are captured, so any async work driven while the +/// returned [`CapturedLogs`] is alive must run on the calling thread (e.g. a +/// current-thread runtime's `block_on`) to be captured -- the subscriber +/// itself is global, but the destination buffer is per-thread; see +/// [`install_global_capture`] for why that split is load-bearing. +pub(crate) fn capture_logs() -> CapturedLogs { + install_global_capture(); + let lines = Arc::new(Mutex::new(Vec::new())); + ACTIVE_LINES.with(|slot| *slot.borrow_mut() = Some(lines.clone())); + CapturedLogs { + lines, + _guard: LinesGuard, + } +} diff --git a/crates/nuncio-filter/src/webhook.rs b/crates/nuncio-filter/src/webhook.rs index 123f82e..0a18320 100644 --- a/crates/nuncio-filter/src/webhook.rs +++ b/crates/nuncio-filter/src/webhook.rs @@ -240,54 +240,9 @@ impl WebhookDispatcher { #[cfg(test)] mod tests { use super::*; - use std::sync::Mutex; - use tracing::field::{Field, Visit}; - use tracing::span; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; - /// Minimal `tracing::Subscriber` that records a formatted line per event - /// so tests can assert on emitted level + fields without pulling in - /// `tracing-subscriber`'s registry machinery. - struct CapturingSubscriber { - events: std::sync::Arc<Mutex<Vec<String>>>, - } - - struct LineVisitor<'a>(&'a mut String); - - impl Visit for LineVisitor<'_> { - fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { - self.0.push_str(&format!(" {}={:?}", field.name(), value)); - } - } - - impl tracing::Subscriber for CapturingSubscriber { - fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool { - true - } - - fn new_span(&self, _span: &span::Attributes<'_>) -> span::Id { - span::Id::from_u64(1) - } - - fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {} - - fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {} - - fn event(&self, event: &tracing::Event<'_>) { - let mut line = format!("{}", event.metadata().level()); - let mut visitor = LineVisitor(&mut line); - event.record(&mut visitor); - if let Ok(mut events) = self.events.lock() { - events.push(line); - } - } - - fn enter(&self, _span: &span::Id) {} - - fn exit(&self, _span: &span::Id) {} - } - #[tokio::test] async fn test_pinned_client_connects_to_checked_ip_not_dns() { // `.invalid` is reserved by RFC 2606 to never resolve: any real DNS @@ -393,12 +348,7 @@ mod tests { #[tokio::test] async fn test_blocked_webhook_logs_warn_with_host_and_reason() { - let events: std::sync::Arc<Mutex<Vec<String>>> = - std::sync::Arc::new(Mutex::new(Vec::new())); - let subscriber = CapturingSubscriber { - events: events.clone(), - }; - let _guard = tracing::subscriber::set_default(subscriber); + let logs = crate::test_tracing::capture_logs(); let dispatcher = WebhookDispatcher::new("secret_key_123"); let result = dispatcher @@ -413,7 +363,7 @@ mod tests { assert!(matches!(result, Err(WebhookError::SecurityViolation(_)))); - let captured = events.lock().unwrap(); + let captured = logs.lines(); assert!( captured.iter().any(|line| line.contains("WARN") && line.contains("blocked") @@ -435,12 +385,7 @@ mod tests { .mount(&mock_server) .await; - let events: std::sync::Arc<Mutex<Vec<String>>> = - std::sync::Arc::new(Mutex::new(Vec::new())); - let subscriber = CapturingSubscriber { - events: events.clone(), - }; - let _guard = tracing::subscriber::set_default(subscriber); + let logs = crate::test_tracing::capture_logs(); let dispatcher = WebhookDispatcher::new("secret_key_123"); let opts = ValidationOptions { @@ -455,7 +400,7 @@ mod tests { .expect("dispatch to the mock server must succeed"); assert_eq!(status, 200); - let captured = events.lock().unwrap(); + let captured = logs.lines(); assert!( captured .iter() diff --git a/crates/nuncio-store/src/lib.rs b/crates/nuncio-store/src/lib.rs index 5783c30..ad82973 100644 --- a/crates/nuncio-store/src/lib.rs +++ b/crates/nuncio-store/src/lib.rs @@ -7,6 +7,8 @@ pub mod cipher; pub mod db; pub mod recovery; pub mod search; +#[cfg(test)] +mod test_tracing; pub mod vault; pub use cipher::{CipherError, PayloadCipher}; diff --git a/crates/nuncio-store/src/test_tracing.rs b/crates/nuncio-store/src/test_tracing.rs new file mode 100644 index 0000000..64175ca --- /dev/null +++ b/crates/nuncio-store/src/test_tracing.rs @@ -0,0 +1,143 @@ +//! Shared in-test `tracing` capture helper. +//! +//! Captures the process's formatted `tracing` output (the same text a real +//! log line would carry, level tag and all) into an in-memory buffer, so +//! tests can assert on substrings such as a table/column name or an `"INFO"`/ +//! `"WARN"` level tag. Fully offline: no I/O beyond the in-memory buffer. +//! +//! The subscriber is installed **once, globally**, and each thread's output is +//! routed to that thread's own buffer. That split is deliberate and +//! load-bearing: a thread-local subscriber cannot work here, because +//! `tracing` caches each callsite's `Interest` process-wide on first use. See +//! [`install_global_capture`]. + +#![cfg(test)] + +use std::cell::RefCell; +use std::io; +use std::sync::{Arc, Mutex, OnceLock}; + +use tracing_subscriber::fmt::MakeWriter; + +thread_local! { + /// The buffer collecting formatted log output for the test running on + /// THIS thread, or `None` on any thread not currently inside + /// [`capture_logs`]. + /// + /// Routing per thread is what keeps the single global subscriber from + /// letting concurrent tests read each other's telemetry. + static ACTIVE_BUFFER: RefCell<Option<Arc<Mutex<Vec<u8>>>>> = const { RefCell::new(None) }; +} + +/// A `Write`/`MakeWriter` that appends into whichever buffer is active for +/// the calling thread, discarding the write if no test on this thread is +/// currently capturing. +#[derive(Clone, Default)] +struct ThreadRoutedWriter; + +impl io::Write for ThreadRoutedWriter { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + ACTIVE_BUFFER.with(|slot| { + if let Some(buffer) = slot.borrow().as_ref() { + buffer + .lock() + .unwrap_or_else(|e| e.into_inner()) + .extend_from_slice(buf); + } + }); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl<'a> MakeWriter<'a> for ThreadRoutedWriter { + type Writer = ThreadRoutedWriter; + fn make_writer(&'a self) -> Self::Writer { + ThreadRoutedWriter + } +} + +/// Install the capturing subscriber as the process-wide default, exactly once. +/// +/// It MUST be global rather than thread-local, and this is the whole reason the +/// helper is shaped this way. `tracing` resolves each callsite's `Interest` +/// **once for the process** and caches it; a callsite first executed on a thread +/// with no subscriber caches `Interest::never()` and is then skipped forever -- +/// before any later thread-local subscriber is ever consulted. Under a parallel +/// test harness, whichever thread reaches a callsite first therefore decides +/// whether any other test can ever capture it, which made every log-capture +/// assertion in this crate a coin flip. +/// +/// A single global subscriber removes the race: every callsite registers against +/// a real subscriber, so interest is always enabled, and per-thread routing +/// (`ACTIVE_BUFFER`) decides what is actually recorded. +fn install_global_capture() { + static INSTALLED: OnceLock<()> = OnceLock::new(); + INSTALLED.get_or_init(|| { + let subscriber = tracing_subscriber::fmt() + .with_writer(ThreadRoutedWriter) + .with_ansi(false) + .finish(); + // A global default may already be set by another component of the same + // test binary; capturing still works through the thread-local routing + // above, so a lost race here is not fatal. + let _ = tracing::subscriber::set_global_default(subscriber); + // Callsites executed before this point resolved their interest against + // the no-op dispatcher and cached "never". Recompute so they can be + // captured from here on. Done once, at install -- not per test. + tracing::callsite::rebuild_interest_cache(); + }); +} + +/// Resets this thread's buffer slot on drop, so a panicking test cannot leak +/// its buffer into whatever the harness runs next on the same thread. +struct BufferGuard; + +impl Drop for BufferGuard { + fn drop(&mut self) { + ACTIVE_BUFFER.with(|slot| *slot.borrow_mut() = None); + } +} + +/// A capture in progress on the current thread. Held across the code under +/// test -- including any `.await` points, as long as the test stays on a +/// single OS thread (e.g. the default current-thread `#[tokio::test]` +/// runtime) -- then queried for the formatted text captured so far. +pub(crate) struct CapturedLogs { + buffer: Arc<Mutex<Vec<u8>>>, + _guard: BufferGuard, +} + +impl CapturedLogs { + /// A snapshot of everything captured on this thread so far, as the + /// formatted text a real log line would carry (level tag included). + pub fn text(&self) -> String { + let bytes = self + .buffer + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + String::from_utf8(bytes) + .unwrap_or_else(|e| String::from_utf8_lossy(&e.into_bytes()).into_owned()) + } +} + +/// Start capturing this thread's formatted `tracing` output. +/// +/// Only this thread's output is captured, so any async work driven while the +/// returned [`CapturedLogs`] is alive must run on the calling thread (e.g. a +/// current-thread runtime's `block_on`) to be captured -- the subscriber +/// itself is global, but the destination buffer is per-thread; see +/// [`install_global_capture`] for why that split is load-bearing. +pub(crate) fn capture_logs() -> CapturedLogs { + install_global_capture(); + let buffer = Arc::new(Mutex::new(Vec::new())); + ACTIVE_BUFFER.with(|slot| *slot.borrow_mut() = Some(buffer.clone())); + CapturedLogs { + buffer, + _guard: BufferGuard, + } +} diff --git a/crates/nuncio-store/src/vault.rs b/crates/nuncio-store/src/vault.rs index 26ec80c..a3fb2a7 100644 --- a/crates/nuncio-store/src/vault.rs +++ b/crates/nuncio-store/src/vault.rs @@ -365,47 +365,13 @@ mod tests { #[test] fn get_or_create_key_bytes_logs_minting_without_leaking_the_key() { - use std::sync::{Arc, Mutex}; - use tracing_subscriber::fmt::MakeWriter; - - #[derive(Clone, Default)] - struct CapturedLogs(Arc<Mutex<Vec<u8>>>); - - impl std::io::Write for CapturedLogs { - fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { - self.0 - .lock() - .expect("log buffer lock") - .extend_from_slice(buf); - Ok(buf.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - - impl<'a> MakeWriter<'a> for CapturedLogs { - type Writer = CapturedLogs; - fn make_writer(&'a self) -> Self::Writer { - self.clone() - } - } - let manager = SecretManager::mock(); - let logs = CapturedLogs::default(); - let subscriber = tracing_subscriber::fmt() - .with_writer(logs.clone()) - .with_ansi(false) - .finish(); - - let guard = tracing::subscriber::set_default(subscriber); + let logs = crate::test_tracing::capture_logs(); let key = manager .get_or_create_key_bytes(WEBHOOK_SIGNING_KEY_ACCOUNT, 32) .expect("key provisioned from vault"); - drop(guard); - let captured = String::from_utf8(logs.0.lock().expect("log buffer lock").clone()) - .expect("captured log is valid utf8"); + let captured = logs.text(); assert!( captured.contains(WEBHOOK_SIGNING_KEY_ACCOUNT), "expected the key's purpose to be named in the INFO log, got: {captured}" -- 2.54.0 From 15226b62a146e52b33bd31c4440ac2fe3093f6d5 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 12:32:20 -0500 Subject: [PATCH 82/95] refactor(store): create the schema, never migrate it Four ALTER TABLE helpers upgraded a pre-existing on-disk schema: smtp columns, TLS-mode columns, the DAV collection_url, and the messages identity columns. Every column they add is already declared in the CREATE TABLE IF NOT EXISTS in the same migrate(), so on any database this engine actually creates they were unconditional no-ops. Nothing is released and every store is a local development database, so there is no older schema to meet. Delete the helpers and the three tests that existed only to exercise them. backfill_message_fts stays: salvage restores rows straight into messages, bypassing save_email's index write, so a recovered database really does have content the trigram index has never seen. Its doc no longer claims the legacy justification. Removes 495 lines, and with them the migrate_logs flake outright. --- crates/nuncio-store/src/db.rs | 520 +--------------------------------- 1 file changed, 10 insertions(+), 510 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index c22ba55..50b6e37 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -747,239 +747,22 @@ impl DatabaseEngine { .await .map_err(DatabaseError::Query)?; - self.ensure_accounts_smtp_columns().await?; - self.ensure_accounts_tls_mode_columns().await?; - self.ensure_accounts_dav_columns().await?; - self.ensure_messages_identity_columns().await?; + // The statements above create the schema in full, so there is nothing to + // upgrade: every column this engine reads is declared in its own + // `CREATE TABLE IF NOT EXISTS`. Databases are created, never migrated. self.backfill_message_fts().await?; Ok(()) } - /// Additive, backfill-safe migration that adds the `remote_id` / - /// `uid_validity` message-identity columns to a pre-existing `messages` - /// table that predated the opaque surrogate id, and installs the - /// `UNIQUE(account_id, folder_id, uid_validity, remote_id)` index that is - /// the real correctness guard against a shared protocol id (e.g. an IMAP - /// UID reused across folders/accounts) overwriting an unrelated message. - /// - /// A fresh database already gets both columns from `CREATE TABLE IF NOT - /// EXISTS messages` above, so on a fresh database the `PRAGMA table_info` - /// check finds them present and the backfill matches no rows -- only the - /// index is created. For a pre-existing database file the columns are added - /// with an empty-string default and every legacy row is backfilled to - /// `remote_id = id` (the old primary key, which was globally unique) so the - /// UNIQUE index can be built without collisions. SQLite has no `ADD COLUMN - /// IF NOT EXISTS`, so column presence is checked explicitly first, making - /// this safe to run on every daemon startup. - /// - /// The whole step runs in a single transaction, and the backfill is - /// unconditional (idempotent `WHERE remote_id = ''`) rather than gated on - /// having just added the column. Both properties make a partial run - /// self-healing: a crash mid-migration rolls back atomically, and even a - /// committed intermediate state (column present but a legacy row still - /// carrying the empty default) is completed by the next startup's backfill - /// before the index is built -- so the daemon can never wedge on an index - /// collision it could have avoided. - async fn ensure_messages_identity_columns(&self) -> Result<(), DatabaseError> { - let existing_columns: Vec<String> = sqlx::query("PRAGMA table_info(messages)") - .fetch_all(&self.pool) - .await - .map_err(DatabaseError::Query)? - .iter() - .map(|row| row.get::<String, _>("name")) - .collect(); - - let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; - - if !existing_columns.iter().any(|c| c == "remote_id") { - sqlx::query("ALTER TABLE messages ADD COLUMN remote_id TEXT NOT NULL DEFAULT ''") - .execute(&mut *tx) - .await - .map_err(DatabaseError::Query)?; - tracing::info!( - table = "messages", - column = "remote_id", - "applying schema migration" - ); - } - if !existing_columns.iter().any(|c| c == "uid_validity") { - sqlx::query("ALTER TABLE messages ADD COLUMN uid_validity TEXT NOT NULL DEFAULT ''") - .execute(&mut *tx) - .await - .map_err(DatabaseError::Query)?; - tracing::info!( - table = "messages", - column = "uid_validity", - "applying schema migration" - ); - } - - // Seed any legacy row still carrying the empty-string default with a - // unique, non-empty remote id (the old primary key) so the UNIQUE - // identity index is satisfiable. Unconditional so a partial prior run - // is always completed before the index is (re)built; a no-op on a fresh - // or already-migrated database. - sqlx::query("UPDATE messages SET remote_id = id WHERE remote_id = ''") - .execute(&mut *tx) - .await - .map_err(DatabaseError::Query)?; - - sqlx::query( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_identity \ - ON messages (account_id, folder_id, uid_validity, remote_id)", - ) - .execute(&mut *tx) - .await - .map_err(DatabaseError::Query)?; - - tx.commit().await.map_err(DatabaseError::Query)?; - - Ok(()) - } - - /// Additive, backfill-safe migration that adds the `imap_tls_mode` / - /// `smtp_tls_mode` columns to a pre-existing `accounts` table that - /// predates persisted per-protocol TLS transport modes. - /// - /// A fresh database already gets these columns from `CREATE TABLE IF NOT - /// EXISTS accounts` above, so on a fresh database this is a no-op. For a - /// pre-existing database file the columns are added with a - /// `'implicit_tls'` default, so every existing row keeps loading - /// successfully and retains the historical behavior (implicit TLS) it was - /// created under. SQLite has no `ADD COLUMN IF NOT EXISTS`, so column - /// presence is checked explicitly via `PRAGMA table_info` first, making - /// this safe to run on every daemon startup. - async fn ensure_accounts_tls_mode_columns(&self) -> Result<(), DatabaseError> { - let existing_columns: Vec<String> = sqlx::query("PRAGMA table_info(accounts)") - .fetch_all(&self.pool) - .await - .map_err(DatabaseError::Query)? - .iter() - .map(|row| row.get::<String, _>("name")) - .collect(); - - if !existing_columns.iter().any(|c| c == "imap_tls_mode") { - sqlx::query( - "ALTER TABLE accounts ADD COLUMN imap_tls_mode TEXT NOT NULL DEFAULT 'implicit_tls'", - ) - .execute(&self.pool) - .await - .map_err(DatabaseError::Query)?; - tracing::info!( - table = "accounts", - column = "imap_tls_mode", - "applying schema migration" - ); - } - if !existing_columns.iter().any(|c| c == "smtp_tls_mode") { - sqlx::query( - "ALTER TABLE accounts ADD COLUMN smtp_tls_mode TEXT NOT NULL DEFAULT 'implicit_tls'", - ) - .execute(&self.pool) - .await - .map_err(DatabaseError::Query)?; - tracing::info!( - table = "accounts", - column = "smtp_tls_mode", - "applying schema migration" - ); - } - - Ok(()) - } - - /// Additive, backfill-safe migration that adds the - /// `smtp_host` / `smtp_port` columns to a pre-existing `accounts` table - /// that predates the SMTP account-endpoint feature. - /// - /// A fresh database already gets these columns from `CREATE TABLE IF NOT - /// EXISTS accounts` above, so on a fresh database this is a no-op (the - /// `PRAGMA table_info` check below finds both columns already present). - /// For a pre-existing database file, `CREATE TABLE IF NOT EXISTS` is - /// itself a no-op (the table already exists), so this is what actually - /// adds the new columns -- as `NULL`-able columns, so every existing row - /// keeps loading successfully (see [`Self::list_accounts`], which falls - /// back to `server_host`/`server_port` for any row where `smtp_host` / - /// `smtp_port` is still `NULL`). SQLite has no `ADD COLUMN IF NOT - /// EXISTS`, so column presence is checked explicitly via `PRAGMA - /// table_info` first, making this safe to run on every daemon startup. - async fn ensure_accounts_smtp_columns(&self) -> Result<(), DatabaseError> { - let existing_columns: Vec<String> = sqlx::query("PRAGMA table_info(accounts)") - .fetch_all(&self.pool) - .await - .map_err(DatabaseError::Query)? - .iter() - .map(|row| row.get::<String, _>("name")) - .collect(); - - if !existing_columns.iter().any(|c| c == "smtp_host") { - sqlx::query("ALTER TABLE accounts ADD COLUMN smtp_host TEXT") - .execute(&self.pool) - .await - .map_err(DatabaseError::Query)?; - tracing::info!( - table = "accounts", - column = "smtp_host", - "applying schema migration" - ); - } - if !existing_columns.iter().any(|c| c == "smtp_port") { - sqlx::query("ALTER TABLE accounts ADD COLUMN smtp_port INTEGER") - .execute(&self.pool) - .await - .map_err(DatabaseError::Query)?; - tracing::info!( - table = "accounts", - column = "smtp_port", - "applying schema migration" - ); - } - - Ok(()) - } - - /// Additive, backfill-safe migration that adds the `collection_url` - /// column to a pre-existing `accounts` table that predates DAV-protocol - /// (CalDAV/CardDAV) accounts. - /// - /// A fresh database already gets this column from `CREATE TABLE IF NOT - /// EXISTS accounts` above, so on a fresh database this is a no-op. For a - /// pre-existing database file the column is added as a `NULL`-able column, - /// so every existing (mail) row keeps loading successfully with an empty - /// collection URL (see [`Self::list_accounts`], which maps a `NULL` - /// `collection_url` to an empty string). SQLite has no `ADD COLUMN IF NOT - /// EXISTS`, so column presence is checked explicitly via `PRAGMA - /// table_info` first, making this safe to run on every daemon startup. - async fn ensure_accounts_dav_columns(&self) -> Result<(), DatabaseError> { - let existing_columns: Vec<String> = sqlx::query("PRAGMA table_info(accounts)") - .fetch_all(&self.pool) - .await - .map_err(DatabaseError::Query)? - .iter() - .map(|row| row.get::<String, _>("name")) - .collect(); - - if !existing_columns.iter().any(|c| c == "collection_url") { - sqlx::query("ALTER TABLE accounts ADD COLUMN collection_url TEXT") - .execute(&self.pool) - .await - .map_err(DatabaseError::Query)?; - tracing::info!( - table = "accounts", - column = "collection_url", - "applying schema migration" - ); - } - - Ok(()) - } - /// Backfill `messages_fts` for any `messages` row that does not yet have a matching FTS - /// entry -- e.g. rows written before the FTS5 index existed, or written directly by a - /// process that bypassed `save_email`'s explicit index population. Run automatically as - /// part of [`DatabaseEngine::migrate`] (idempotent: a fully-indexed database performs no - /// work). The stored `body_plain` column holds AES-256-GCM ciphertext, so each candidate + /// entry -- rows written directly by a process that bypassed `save_email`'s explicit + /// index population. That is not a legacy concern: `SqliteRecoveryEngine::salvage` + /// restores `messages` rows straight into the table, so a database that has been + /// through recovery has content the trigram index has never seen, and search would + /// silently miss it. Run automatically as part of [`DatabaseEngine::migrate`] + /// (idempotent: a fully-indexed database performs no work). The stored `body_plain` + /// column holds AES-256-GCM ciphertext, so each candidate /// row is decrypted with this engine's storage key before being written into the /// plaintext-derived trigram index -- see the confidentiality tradeoff documented above /// `messages_fts`'s `CREATE VIRTUAL TABLE` statement in [`DatabaseEngine::migrate`]. @@ -3756,230 +3539,6 @@ mod tests { ); } - /// Proves the additive `smtp_host`/`smtp_port` - /// migration is backfill-safe. Simulates a database created BEFORE this - /// feature existed (an `accounts` table with no `smtp_host`/`smtp_port` - /// columns at all, populated via a direct `INSERT` bypassing - /// `save_account`), then opens it through `DatabaseEngine::connect_file` - /// (which runs `migrate()`, including `ensure_accounts_smtp_columns`) - /// and confirms: (1) opening an old-schema database never errors, (2) - /// the pre-existing row still loads, falling back to `server_host`/ - /// `server_port` for the missing SMTP endpoint, and (3) re-running - /// `migrate()` a second time (simulating a daemon restart) is a no-op - /// that does not error or duplicate columns. - #[tokio::test] - async fn migrate_backfills_smtp_columns_for_a_pre_existing_accounts_table() { - let dir = tempfile::tempdir().expect("tempdir"); - let db_path = dir.path().join("pre_smtp_migration.db"); - let secrets = crate::vault::SecretManager::mock(); - - // Step 1: create an OLD-schema `accounts` table (no smtp_host/smtp_port - // columns) directly, bypassing `DatabaseEngine::migrate` entirely, and - // insert one pre-existing row -- exactly what a database created - // before the `smtp_host`/`smtp_port` columns existed would look like on disk. - { - let url = format!("sqlite://{}", db_path.to_string_lossy()); - let options = sqlx::sqlite::SqliteConnectOptions::from_str(&url) - .expect("valid sqlite url") - .create_if_missing(true); - let pool = sqlx::SqlitePool::connect_with(options) - .await - .expect("connect to fresh old-schema db"); - - sqlx::query( - r#" - CREATE TABLE accounts ( - id TEXT PRIMARY KEY NOT NULL, - name TEXT NOT NULL, - email_address TEXT NOT NULL, - protocol TEXT NOT NULL, - server_host TEXT NOT NULL, - server_port INTEGER NOT NULL, - use_tls INTEGER NOT NULL, - keyring_secret_key TEXT NOT NULL, - sync_interval_secs INTEGER NOT NULL - ) - "#, - ) - .execute(&pool) - .await - .expect("create old-schema accounts table"); - - sqlx::query( - r#" - INSERT INTO accounts - (id, name, email_address, protocol, server_host, server_port, use_tls, keyring_secret_key, sync_interval_secs) - VALUES ('acct-pre-migration', 'Pre-Migration Account', 'pre@nuncio.mx', '"imap-smtp"', 'imap.nuncio.mx', 993, 1, 'nuncio/acct-pre-migration', 60) - "#, - ) - .execute(&pool) - .await - .expect("insert pre-existing row"); - - pool.close().await; - } - - // Step 2: open through the real production path -- this is what - // actually runs `migrate()` / `ensure_accounts_smtp_columns`. - let engine = DatabaseEngine::connect_file(&db_path, &secrets) - .await - .expect("opening an old-schema database must never error"); - - let accounts = engine - .list_accounts() - .await - .expect("list accounts succeeds on a migrated old-schema table"); - assert_eq!(accounts.len(), 1); - let acct = &accounts[0]; - assert_eq!(acct.id, "acct-pre-migration"); - // Falls back to the IMAP/JMAP endpoint since smtp_host/smtp_port - // were NULL for this pre-existing row. - let acct_t = acct.imap_smtp().expect("imap-smtp transport"); - assert_eq!(acct_t.smtp_host, "imap.nuncio.mx"); - assert_eq!(acct_t.smtp_port, 993); - - // Step 3: re-running migrate() (e.g. a second daemon startup against - // the same file) must be a no-op, not an error. - engine - .migrate() - .await - .expect("re-running migrate on an already-migrated table must not error"); - - // A newly saved account (going through the real `save_account` path) - // now gets its own genuine smtp_host/smtp_port persisted, proving - // the migrated columns are fully writable/readable going forward. - let new_acct = nuncio_core::AccountConfig { - id: "acct-post-migration".to_string(), - name: "Post Migration Account".to_string(), - email_address: "post@nuncio.mx".to_string(), - keyring_secret_key: "nuncio/acct-post-migration".to_string(), - sync_interval_secs: 60, - transport: nuncio_core::Transport::ImapSmtp(nuncio_core::ImapSmtpTransport { - imap_host: "imap.nuncio.mx".to_string(), - imap_port: 993, - imap_tls_mode: nuncio_core::TlsMode::ImplicitTls, - smtp_host: "smtp.nuncio.mx".to_string(), - smtp_port: 587, - smtp_tls_mode: nuncio_core::TlsMode::StartTls, - }), - }; - engine - .save_account(&new_acct) - .await - .expect("save account succeeds after migration"); - let accounts = engine.list_accounts().await.expect("list accounts"); - let post = accounts - .iter() - .find(|a| a.id == "acct-post-migration") - .expect("newly saved account present"); - let post_t = post.imap_smtp().expect("imap-smtp transport"); - assert_eq!(post_t.smtp_host, "smtp.nuncio.mx"); - assert_eq!(post_t.smtp_port, 587); - - engine.close().await; - } - - /// Applying an additive column migration against a pre-existing (old-schema) - /// database must be visible in telemetry -- an `INFO` log naming the table and - /// column -- rather than only observable indirectly via the resulting schema. - #[tokio::test] - async fn migrate_logs_when_an_additive_column_migration_actually_runs() { - use std::sync::{Arc, Mutex}; - use tracing_subscriber::fmt::MakeWriter; - - #[derive(Clone, Default)] - struct CapturedLogs(Arc<Mutex<Vec<u8>>>); - - impl std::io::Write for CapturedLogs { - fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { - self.0 - .lock() - .expect("log buffer lock") - .extend_from_slice(buf); - Ok(buf.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - - impl<'a> MakeWriter<'a> for CapturedLogs { - type Writer = CapturedLogs; - fn make_writer(&'a self) -> Self::Writer { - self.clone() - } - } - - let dir = tempfile::tempdir().expect("tempdir"); - let db_path = dir.path().join("pre_dav_migration.db"); - let secrets = crate::vault::SecretManager::mock(); - - // An old-schema `accounts` table predating `collection_url`. - { - let url = format!("sqlite://{}", db_path.to_string_lossy()); - let options = sqlx::sqlite::SqliteConnectOptions::from_str(&url) - .expect("valid sqlite url") - .create_if_missing(true); - let pool = sqlx::SqlitePool::connect_with(options) - .await - .expect("connect to fresh old-schema db"); - - sqlx::query( - r#" - CREATE TABLE accounts ( - id TEXT PRIMARY KEY NOT NULL, - name TEXT NOT NULL, - email_address TEXT NOT NULL, - protocol TEXT NOT NULL, - server_host TEXT NOT NULL, - server_port INTEGER NOT NULL, - use_tls INTEGER NOT NULL, - keyring_secret_key TEXT NOT NULL, - sync_interval_secs INTEGER NOT NULL, - smtp_host TEXT, - smtp_port INTEGER, - imap_tls_mode TEXT NOT NULL DEFAULT 'implicit_tls', - smtp_tls_mode TEXT NOT NULL DEFAULT 'implicit_tls' - ) - "#, - ) - .execute(&pool) - .await - .expect("create old-schema accounts table"); - - pool.close().await; - } - - let logs = CapturedLogs::default(); - let subscriber = tracing_subscriber::fmt() - .with_writer(logs.clone()) - .with_ansi(false) - .finish(); - - // `set_default` returns a guard that stays active across `.await` - // points as long as this test stays on the single `#[tokio::test]` - // current-thread executor, which is the default runtime flavor and - // exactly what is used here. - let guard = tracing::subscriber::set_default(subscriber); - let engine = DatabaseEngine::connect_file(&db_path, &secrets) - .await - .expect("opening an old-schema database must never error"); - drop(guard); - - let captured = String::from_utf8(logs.0.lock().expect("log buffer lock").clone()) - .expect("captured log is valid utf8"); - assert!( - captured.contains("collection_url"), - "expected the migrated column to be named in the INFO log, got: {captured}" - ); - assert!( - captured.to_ascii_uppercase().contains("INFO"), - "expected an INFO-level log for the applied migration, got: {captured}" - ); - - engine.close().await; - } - #[test] fn database_error_display() { let err = DatabaseError::PoolCreation("failed".to_string()); @@ -4309,65 +3868,6 @@ mod tests { assert_eq!(fetched.folder_id, "INBOX"); } - /// A crash between `ADD COLUMN remote_id` and its backfill can commit an - /// intermediate state -- the column present but a legacy row still carrying - /// the empty-string default -- with the UNIQUE index not yet built. The - /// migration must complete that backfill on the next open and build the - /// index without a collision, rather than wedging the daemon. - #[tokio::test] - async fn messages_identity_migration_completes_from_a_partial_run() { - let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - - // Reproduce the partial state: drop the identity index, then insert two - // rows in the same folder that both still hold the empty default. With - // the index gone this is permitted, and the two would collide on - // (account, folder, uid_validity='', remote_id='') if it were rebuilt now. - sqlx::query("DROP INDEX IF EXISTS idx_messages_identity") - .execute(engine.pool()) - .await - .unwrap(); - for id in ["legacy-a", "legacy-b"] { - sqlx::query( - "INSERT INTO messages \ - (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag) \ - VALUES (?, 'acct-1', 'INBOX', '', '', 'subj', 'a@nuncio.mx', 'b@nuncio.mx', 0, 0)", - ) - .bind(id) - .execute(engine.pool()) - .await - .unwrap(); - } - - // Re-running the migration must finish the backfill and build the index. - engine - .ensure_messages_identity_columns() - .await - .expect("a partial migration must complete on the next open"); - - // Both legacy rows survive with a unique, non-empty backfilled remote id. - assert_eq!( - engine.get_message("legacy-a").await.unwrap().remote_id, - "legacy-a" - ); - assert_eq!( - engine.get_message("legacy-b").await.unwrap().remote_id, - "legacy-b" - ); - - // The unique identity index is now present, and re-running is idempotent. - let index: Option<(String,)> = sqlx::query_as( - "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_messages_identity'", - ) - .fetch_optional(engine.pool()) - .await - .unwrap(); - assert!(index.is_some(), "the unique identity index must be built"); - engine - .ensure_messages_identity_columns() - .await - .expect("re-running an already-complete migration is a no-op"); - } - fn export_test_email(id: &str, account_id: &str, folder_id: &str) -> nuncio_core::model::Email { nuncio_core::model::Email { id: id.to_string(), -- 2.54.0 From bbebd576d8c444fd90e565a183eb67bb33bbdd8f Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Wed, 12 Aug 2026 13:10:12 -0500 Subject: [PATCH 83/95] fix(test): time out outbox mutations in real time, not on a paused clock The two hung-mutation tests paused the Tokio clock so auto-advance would jump the 30s per-item timeout. That cannot be made reliable: the executor does real SQLite I/O on a blocking pool inside the same window, and auto-advance fires whenever the runtime looks idle. It also fires sqlx's own pool-acquire timer, which defaults to the same 30 seconds -- turning a healthy list_pending_mutations into PoolTimedOut, whose error arm returns an all-zero summary. That is exactly the shape CI reported: completed 0 and failed 0 both as expected, retried 0 instead of 1. Make the per-item timeout an overridable method on RemoteExecutionEnv and give the hanging test env 250ms, so the tests wait it out in real time and no test pauses the clock. The hang is infinite, so a loaded runner makes these slower, never wrong. 50 runs at --test-threads=2 under 16-way CPU load: 0 failures. --- crates/nunciod/src/outbox.rs | 21 +++++++- .../nunciod/tests/outbox_executor_e2e_test.rs | 49 +++++++------------ 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 7de867e..5c00417 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -107,6 +107,22 @@ pub trait RemoteExecutionEnv: Send + Sync { subject: &str, sender: &str, ) -> Result<u16, WebhookError>; + + /// How long a single mutation may run before it is abandoned as a + /// retryable timeout. Production keeps [`PER_ITEM_EXECUTION_TIMEOUT`]. + /// + /// Overridable so a test can pick a duration it can actually wait out in + /// real time. The alternative -- pausing the Tokio clock and letting + /// auto-advance jump the 30 seconds -- cannot be made reliable here, + /// because this executor performs real SQLite I/O on a blocking pool + /// within the same window. Auto-advance fires whenever the runtime looks + /// idle, and a paused clock also fires sqlx's own pool-acquire timeout + /// (30s by default -- the same duration), which turns a healthy query into + /// `PoolTimedOut` and makes this function return an all-zero summary. That + /// is precisely the shape the flaky CI failures took. + fn per_item_timeout(&self) -> Duration { + PER_ITEM_EXECUTION_TIMEOUT + } } /// The disposition of a single mutation attempt. @@ -174,12 +190,13 @@ pub async fn execute_pending_mutations( ); break; } - result = tokio::time::timeout(PER_ITEM_EXECUTION_TIMEOUT, execute_one(db, env, &item)) => result, + result = tokio::time::timeout(env.per_item_timeout(), execute_one(db, env, &item)) => result, }; let disposition = match outcome { Ok(disposition) => disposition, Err(_elapsed) => Disposition::Retry(format!( - "mutation execution exceeded the {PER_ITEM_EXECUTION_TIMEOUT:?} timeout" + "mutation execution exceeded the {:?} timeout", + env.per_item_timeout() )), }; let next_retry = item.retry_count + 1; diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index 5fccdcd..6a9c42e 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -421,19 +421,30 @@ impl HangEnv { #[async_trait] impl RemoteExecutionEnv for HangEnv { + /// Short enough to wait out in real time, so these tests never pause the + /// Tokio clock. A paused clock auto-advances whenever the runtime looks + /// idle, and this executor does real SQLite I/O on a blocking pool in the + /// same window -- the advance also fires sqlx's 30s pool-acquire timer, + /// turning a healthy query into `PoolTimedOut` and the pass into an + /// all-zero summary. Generous enough to survive a loaded CI runner: the + /// hang it bounds is infinite, so a slow machine only makes the test + /// slower, never wrong. + fn per_item_timeout(&self) -> std::time::Duration { + std::time::Duration::from_millis(250) + } + async fn mail_backend( &self, account_id: &str, ) -> Result<Box<dyn MailBackend>, nunciod::outbox::OutboxExecuteError> { let call = self.calls.fetch_add(1, Ordering::SeqCst); if call < self.hang_calls { - // A `tokio::time`-tracked sleep far longer than any per-item - // timeout under test -- NOT `std::future::pending`, which never - // registers with the time driver and so behaves unpredictably - // once `tokio::time::pause`'s auto-advance-on-idle interacts with - // real (non-timer) I/O elsewhere in the same test, like the - // SQLite pool's own connection bookkeeping. A real (virtual) - // sleep is what the timeout under test is meant to race against. + // Stands in for a remote operation that never answers. An hour is + // simply "longer than any timeout under test"; the executor + // abandons the attempt and drops this future, so nothing actually + // waits. A sleep rather than `std::future::pending` keeps the + // future registered with the time driver, which is the shape a + // real stalled network call has. tokio::time::sleep(std::time::Duration::from_secs(3600)).await; } self.inner.mail_backend(account_id).await @@ -489,23 +500,9 @@ async fn a_hung_mutation_times_out_and_is_retried_never_completed() { ) .await; - // Paused only AFTER setup, and this pass touches the database exactly - // ONCE (a single `get_message`) before the hang -- mixing - // `tokio::time::pause`'s auto-advance-on-idle with real (non-timer) - // background I/O is only safe when there is a single, already-in-flight - // real operation for the auto-advance to race past; see the sibling - // two-pass test below for why a SECOND real DB operation in the same - // paused pass is deliberately avoided. - tokio::time::pause(); let env = HangEnv::new(usize::MAX); let (_shutdown_ctrl, mut shutdown) = no_shutdown(); let summary = execute_pending_mutations(&db, &env, 10, &mut shutdown).await; - // Resume real time before any further DB access: `tokio::time::pause`'s - // auto-advance-on-idle only plays safely with the ONE real DB operation - // this pass performs (the `get_message` before the hang); a second real - // query made while still paused is what made the sibling two-pass test - // flaky before it was split across a resume point. - tokio::time::resume(); // The hung item is NEVER marked completed -- it is a retryable timeout, // exactly like any other transient remote failure. @@ -533,13 +530,6 @@ async fn a_timed_out_item_does_not_block_a_later_item_in_a_subsequent_pass() { let (db, _dir) = DatabaseEngine::connect_ephemeral().await.expect("db"); seed_two_move_mutations(&db).await; - // First pass: bound to the FIRST (created-earliest) mutation only, so - // this pass's real DB work is a single `get_message` call before the - // hang, same as the sibling single-item timeout test. Real threaded - // SQLite I/O racing `tokio::time::pause`'s auto-advance is only - // exercised once per paused pass here -- see that test's comment for - // why a second real DB call in the SAME paused pass is unreliable. - tokio::time::pause(); let env = HangEnv::new(usize::MAX); let (_shutdown_ctrl, mut shutdown) = no_shutdown(); let first_pass = execute_pending_mutations(&db, &env, 1, &mut shutdown).await; @@ -551,7 +541,6 @@ async fn a_timed_out_item_does_not_block_a_later_item_in_a_subsequent_pass() { // tick: the item that timed out in the first pass did not jam the queue // -- both it (now against a healthy backend) and the item queued behind // it are processed and genuinely complete. - tokio::time::resume(); let normal_env = MockEnv::new(); let (_shutdown_ctrl2, mut shutdown2) = no_shutdown(); let second_pass = execute_pending_mutations(&db, &normal_env, 10, &mut shutdown2).await; @@ -586,7 +575,6 @@ async fn shutdown_signal_interrupts_a_hung_item_instead_of_waiting_out_the_timeo // Paused only after setup -- see the comment on the sibling timeout test // for why pausing from the start would spuriously fail the DB pool. - tokio::time::pause(); // Hangs on every call -- the shutdown signal, not the timeout, must be // what unblocks this drain pass. @@ -598,7 +586,6 @@ async fn shutdown_signal_interrupts_a_hung_item_instead_of_waiting_out_the_timeo }); let summary = execute_pending_mutations(&db, &env, 10, &mut shutdown).await; - tokio::time::resume(); // The drain pass stopped on shutdown, well before the per-item timeout // would have elapsed -- the hung item is untouched: not completed, not -- 2.54.0 From 62e1b40998c2536cdf000bd11e2890f919c0cfbb Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 17:37:19 -0500 Subject: [PATCH 84/95] fix(store): reap the message a repointed placement strands Refs #406 --- crates/nuncio-store/src/db.rs | 114 ++++++++++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 4 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 9aa7253..23f64b8 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -1446,18 +1446,22 @@ impl DatabaseEngine { // an upsert, so after the fact there is no way to tell an inserted // placement from an updated one -- and the caller needs that distinction // to decide whether this is a first arrival worth filtering. - let (already_placed,): (i64,) = sqlx::query_as( - "SELECT COUNT(*) FROM placements + // The occupant's key is read, not just its existence, because the upsert below may + // repoint this slot at a different message and the previous key is unrecoverable + // afterwards. + let occupant: Option<(String,)> = sqlx::query_as( + "SELECT message_key FROM placements WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", ) .bind(&placement.account_id) .bind(&placement.folder_id) .bind(&placement.uid_validity) .bind(&placement.remote_id) - .fetch_one(&mut *tx) + .fetch_optional(&mut *tx) .await .map_err(DatabaseError::Query)?; - let placement_is_new = already_placed == 0; + let placement_is_new = occupant.is_none(); + let displaced_key = occupant.map(|(key,)| key).filter(|key| *key != email.id); let inserted = sqlx::query( r#" @@ -1521,6 +1525,24 @@ impl DatabaseEngine { .await .map_err(DatabaseError::Query)?; + // Repointing a slot at a different message drops the displaced message's placement + // without it ever passing through [`Self::delete_placements`]. If that was its last + // one, nothing else will ever delete it -- and its decrypted body would stay readable + // in `messages_fts` forever, since only the `messages_ad` trigger clears that index. + // Reap it here on exactly the terms `delete_placements` uses: gone only when nothing + // points at it any more. + if let Some(displaced_key) = displaced_key { + sqlx::query( + "DELETE FROM messages WHERE message_key = ? + AND NOT EXISTS (SELECT 1 FROM placements WHERE message_key = ?)", + ) + .bind(&displaced_key) + .bind(&displaced_key) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + } + tx.commit().await.map_err(DatabaseError::Query)?; Ok(SaveOutcome { @@ -5055,6 +5077,90 @@ mod tests { ); } + #[tokio::test] + async fn repointing_a_slot_reaps_the_displaced_message_and_its_plaintext_index() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (first, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine + .save_email_at(&first, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + + // The same mailbox slot re-saved under a different identity. The displaced + // message never passes through `delete_placements`, so this write is its only + // chance to be reaped. + let (second, _) = sample_message_and_placement("key-2", "INBOX", "5"); + engine + .save_email_at(&second, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + + engine + .get_message("key-1") + .await + .expect_err("the displaced message has no placements left"); + let (fts,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!( + fts, 0, + "a repoint that strands a message must not leave its decrypted body \ + readable in the FTS index" + ); + + engine + .get_message("key-2") + .await + .expect("the message that took the slot must survive"); + let placements = engine.placements_of("key-2").await.unwrap(); + assert_eq!(placements.len(), 1, "the slot holds exactly one occupant"); + } + + #[tokio::test] + async fn repointing_a_slot_keeps_a_displaced_message_that_still_lives_elsewhere() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (first, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at(&first, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &first, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let (second, _) = sample_message_and_placement("key-2", "INBOX", "5"); + engine + .save_email_at(&second, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + + engine + .get_message("key-1") + .await + .expect("the displaced message still lives in Archive"); + let (fts,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(fts, 1, "search must still find a message that still exists"); + let placements = engine.placements_of("key-1").await.unwrap(); + assert_eq!(placements.len(), 1); + assert_eq!(placements[0].folder_id, "Archive"); + } + #[tokio::test] async fn folders_are_derived_from_placements_and_scoped_per_account() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); -- 2.54.0 From 1a94716abbdc072c07630985b09db275e5be5ea7 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 17:37:44 -0500 Subject: [PATCH 85/95] docs(store): state the lifetime of a filter fire claim Refs #409 --- crates/nuncio-store/src/db.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 23f64b8..24ced58 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -727,6 +727,21 @@ impl DatabaseEngine { /// fired the rule again. `INSERT ... ON CONFLICT DO NOTHING` makes winning /// the claim a single atomic statement, so two passes racing over the same /// message cannot both act. + /// + /// A claim outlives the message it was made against. Nothing reaps + /// `filter_fired` when the last placement goes, so a message deleted and + /// later re-delivered byte for byte will not fire its rules a second time. + /// That is deliberate: reaping alongside the message would hand a remote + /// party a way to re-trigger third-party side effects at will -- a + /// mailing-list message deleted and re-sent would `FORWARD` again -- and + /// under content-addressed identity an identical redelivery genuinely *is* + /// the same message, so declining to re-fire is the honest reading. The + /// failure this leaves open ("a rule that should have fired again did not") + /// is the safe direction for actions that reach outside the mailbox. + /// + /// The cost is a table that only grows, bounded by rules x messages ever + /// seen at roughly 150 bytes a row, and cleared only when an identity + /// schema bump resets `user_version` and rebuilds the derived tables. pub async fn claim_filter_fire( &self, rule_id: &str, -- 2.54.0 From fdbd1a51ca2b404574c42289fce4eda35080c702 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 17:40:02 -0500 Subject: [PATCH 86/95] chore(mail): rename folder sync to match what it returns Refs #410 --- crates/nuncio-core/src/export.rs | 13 ++++++--- crates/nuncio-mail/src/imap.rs | 46 +++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/crates/nuncio-core/src/export.rs b/crates/nuncio-core/src/export.rs index 79a71e4..83fd163 100644 --- a/crates/nuncio-core/src/export.rs +++ b/crates/nuncio-core/src/export.rs @@ -87,10 +87,15 @@ impl ExportEngine { writer.write_all(from_line.as_bytes())?; total_bytes += from_line.len() as u64; - // Headers. No X-Nuncio-Folder header: the folder is a property of - // a placement now, not of the message, so `Email` has none to - // name here. Restoring it would mean threading the relevant - // `Placement` into this function's signature. + // Headers. `X-Nuncio-Folder` is permanently gone, not pending: a + // message's folder is a property of a placement, and a message may + // hold several at once (the same message filed in INBOX and in + // Archive is one message, not two). A single header could therefore + // only ever name one of them, which would be a wrong answer dressed + // as an authoritative one -- and MBOX has no way to spell "these + // several". Callers that need occupancy read it from the store, + // where it is complete. `X-Nuncio-Account` stays because an account + // *is* single-valued on the message. let headers = format!( "Message-ID: <{}>\r\nFrom: {}\r\nTo: {}\r\nSubject: {}\r\nDate: {}\r\nX-Nuncio-Account: {}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n", email.id, email.sender, email.recipient, email.subject, date_str, email.account_id diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index fe93558..57c5922 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -798,7 +798,7 @@ impl ImapEngine { /// returned checkpoint is derived from real server state -- the mailbox's /// UIDVALIDITY plus its `UIDNEXT` (or one past the highest UID actually /// seen) after selecting the folder -- never from a synthetic counter. - pub async fn sync_folder_messages_with_session<S>( + pub async fn sync_folder_changes_with_session<S>( &self, folder_id: &str, since_state: Option<&str>, @@ -807,7 +807,7 @@ impl ImapEngine { where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { - self.sync_folder_messages_batched( + self.sync_folder_changes_batched( folder_id, since_state, session, @@ -842,7 +842,7 @@ impl ImapEngine { fields(account_id = %self.account_id, folder_id = %folder_id), err )] - async fn sync_folder_messages_batched<S>( + async fn sync_folder_changes_batched<S>( &self, folder_id: &str, since_state: Option<&str>, @@ -923,6 +923,14 @@ impl ImapEngine { drive_fetch_stream(&mut fetch_stream, folder_id, emails.len()).await? }; for item in &items { + // The enumeration pass rejected UID-less items; a response to a + // `UID FETCH` must carry one (RFC 3501 section 6.4.8), so this + // only fires for a non-conforming server. Recording it anyway + // would write a placement at the meaningless UID 0. + if item.uid.filter(|u| *u >= 1).is_none() { + skipped_no_uid += 1; + continue; + } emails.push(self.build_email_from_fetch(folder_id, server_uid_validity, item)?); } } @@ -945,6 +953,12 @@ impl ImapEngine { drive_fetch_stream(&mut fetch_stream, folder_id, emails.len()).await? }; for item in &items { + // Same guard as the body pass: an item with no UID cannot be + // placed, and UID 0 is not an address. + if item.uid.filter(|u| *u >= 1).is_none() { + skipped_no_uid += 1; + continue; + } emails.push(self.build_headers_only_email(folder_id, server_uid_validity, item)); } } @@ -1497,8 +1511,8 @@ impl ImapEngine { /// Fetch a folder's messages over a freshly authenticated session, /// honoring `since_state` for an incremental fetch and returning the new /// sync checkpoint alongside the messages. See - /// [`Self::sync_folder_messages_with_session`] for the checkpoint model. - pub async fn sync_folder_messages( + /// [`Self::sync_folder_changes_with_session`] for the checkpoint model. + pub async fn sync_folder_changes( &self, folder_id: &str, since_state: Option<&str>, @@ -1517,7 +1531,7 @@ impl ImapEngine { .connect_session(username, password) .await?; let result = self - .sync_folder_messages_with_session(folder_id, since_state, &mut session) + .sync_folder_changes_with_session(folder_id, since_state, &mut session) .await; let _ = session.logout().await; result @@ -1971,7 +1985,7 @@ impl MailBackend for ImapEngine { folder_id: &str, since_state: Option<&str>, ) -> Result<FolderChanges, MailError> { - self.sync_folder_messages(folder_id, since_state).await + self.sync_folder_changes(folder_id, since_state).await } async fn apply_mutation( @@ -2327,7 +2341,7 @@ mod tests { // First sync: no checkpoint -> full history; returns // "{uidvalidity}:{uidnext}" from real server state. let first_pass = engine - .sync_folder_messages_with_session("INBOX", None, &mut session) + .sync_folder_changes_with_session("INBOX", None, &mut session) .await .expect("first sync succeeds"); let (first, checkpoint) = (first_pass.upserts, first_pass.next_state); @@ -2337,7 +2351,7 @@ mod tests { // Second sync: feed back the checkpoint; UIDVALIDITY still matches // (1), so the fetch is narrowed. let __changes = engine - .sync_folder_messages_with_session("INBOX", Some(&checkpoint), &mut session) + .sync_folder_changes_with_session("INBOX", Some(&checkpoint), &mut session) .await .expect("second sync succeeds"); let (_second, checkpoint2) = (__changes.upserts, __changes.next_state); @@ -2416,7 +2430,7 @@ mod tests { } } - /// Drive `sync_folder_messages_batched` over a scripted, in-memory IMAP + /// Drive `sync_folder_changes_batched` over a scripted, in-memory IMAP /// server that serves a fixed set of `(uid, size)` messages. The server /// answers the enumeration pass (UID + RFC822.SIZE), full-body batches /// (`BODY[]`), and headers-only batches (`BODY[HEADER]`) from the fixture, @@ -2522,7 +2536,7 @@ mod tests { .expect("scripted login succeeds"); let engine = ImapEngine::new("acct-1", "example.test", 993); let changes = engine - .sync_folder_messages_batched("INBOX", since_state.as_deref(), &mut session, batch_size) + .sync_folder_changes_batched("INBOX", since_state.as_deref(), &mut session, batch_size) .await .expect("batched sync succeeds"); drop(session); @@ -2779,7 +2793,7 @@ mod tests { // First sync under UIDVALIDITY 1 stores "1:105". let __changes = engine - .sync_folder_messages_with_session("INBOX", None, &mut session) + .sync_folder_changes_with_session("INBOX", None, &mut session) .await .expect("first sync succeeds"); let (_first, checkpoint) = (__changes.upserts, __changes.next_state); @@ -2789,7 +2803,7 @@ mod tests { // is stale, so the fetch MUST be a full `1:*`, and the checkpoint is // rewritten under the new UIDVALIDITY. let __changes = engine - .sync_folder_messages_with_session("INBOX", Some(&checkpoint), &mut session) + .sync_folder_changes_with_session("INBOX", Some(&checkpoint), &mut session) .await .expect("second sync succeeds"); let (_second, checkpoint2) = (__changes.upserts, __changes.next_state); @@ -2873,7 +2887,7 @@ mod tests { // elapsed time despite the 60s virtual timeout expiring. let started = std::time::Instant::now(); let err = engine - .sync_folder_messages_with_session("INBOX", None, &mut session) + .sync_folder_changes_with_session("INBOX", None, &mut session) .await .expect_err("a never-responding FETCH must surface an error"); assert!( @@ -3428,7 +3442,7 @@ mod tests { let engine = ImapEngine::new("acct-completion", "example.test", 993); let __changes = engine - .sync_folder_messages_with_session("INBOX", None, &mut session) + .sync_folder_changes_with_session("INBOX", None, &mut session) .await .expect("sync succeeds"); let (emails, _checkpoint) = (__changes.upserts, __changes.next_state); @@ -3576,7 +3590,7 @@ mod tests { .expect("scripted login succeeds"); let engine = ImapEngine::new("acct-1", "example.test", 993); let changes = engine - .sync_folder_messages_batched(folder_id, None, &mut session, 8) + .sync_folder_changes_batched(folder_id, None, &mut session, 8) .await .expect("scripted sync succeeds"); drop(session); -- 2.54.0 From b4292b00b03bdf45f59f5852b240f1706856fa90 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 17:49:46 -0500 Subject: [PATCH 87/95] feat(jmap): enumerate changes so deleted mail stops haunting the store JMAP's folder sync could only ever add. It returned `removals: []` and `present: None` unconditionally, so mail deleted or moved by any other client -- another daemon, a phone, webmail -- stayed in the local store forever and every JMAP account accumulated ghosts indefinitely. `sync_changes` now resumes through `Email/changes` (RFC 8621 s4.2 over the RFC 8620 s5.2 core method): - `created`/`updated` become upserts, with bodies fetched by `Email/get` - `destroyed` becomes placement removals - `hasMoreChanges` is followed to convergence, bounded so a server that never advances its `newState` cannot pin the sync task - `cannotCalculateChanges` falls back to a full enumeration, the one pass entitled to report `present: Some(...)` because it is the only one that genuinely looked at the whole mailbox An ordinary incremental pass still reports `present: None`: it never enumerated the folder, and claiming otherwise would delete every message that simply did not change. `Email/changes` is account-wide, so a folder pass sees ids belonging to other mailboxes. `Email/get` now requests `mailboxIds`, which both keeps those ids from being placed here and reconciles away a message another client moved out of this folder. Checkpoints are scheme-tagged (`v1:jmapstate:{state}`) like the IMAP ones, since a JMAP state is an opaque token with no self-describing shape. An untagged or unknown token re-enumerates rather than being handed back to a server that never issued it. Placements now carry the mailbox that was actually synced. Removals are addressed by that same key, so a placement stored under any other folder id could never be reconciled away again. Refs #408 --- crates/nuncio-mail/src/jmap.rs | 468 ++++++++++++++++-- crates/nuncio-mail/src/lib.rs | 2 +- .../nuncio-mail/tests/wiremock_jmap_test.rs | 393 ++++++++++++++- crates/nunciod/src/sync.rs | 6 +- 4 files changed, 813 insertions(+), 56 deletions(-) diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index 83d5227..9f307de 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -1,7 +1,7 @@ //! JMAP (RFC 8620 / RFC 8621) protocol engine, session discovery, and differential update parser. use async_trait::async_trait; -use nuncio_core::model::{Email, Folder, Placement, RemoteIdentity}; +use nuncio_core::model::{Email, Folder, Placement, PlacementKey, RemoteIdentity}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -19,6 +19,47 @@ use crate::parser::MailError; /// JMAP mutation, which addresses messages by object id. const JMAP_UID_VALIDITY_SENTINEL: &str = "jmap"; +/// Scheme tag prefixing every folder checkpoint this engine writes: +/// `"v1:jmapstate:{state}"`. +/// +/// A JMAP state string is an opaque server token (RFC 8620 section 1.6.2) with +/// no self-describing shape, so a stored checkpoint that carries no scheme +/// cannot be told apart from one written by a different protocol engine or a +/// different meaning of the same field. Tagging makes an unrecognised token +/// resolve to a full re-enumeration instead of being handed to `Email/changes` +/// as a `sinceState` it never came from -- over-fetching is recoverable, +/// resuming from a token the server reads differently is not. +const JMAP_STATE_SCHEME: &str = "v1:jmapstate"; + +/// Separator between the scheme tag and the opaque state it wraps. The state +/// itself may contain further separators, so only the first one is structural. +const JMAP_STATE_DELIM: char = ':'; + +/// Ceiling on `Email/changes` pages followed in one pass. +/// +/// `hasMoreChanges` is a server-driven loop, and a server that answers `true` +/// forever -- or returns a `newState` that never advances -- would otherwise +/// pin the sync task in place. The bound is far above any real paging depth, so +/// hitting it means the server is misbehaving and the pass says so rather than +/// spinning. +const MAX_CHANGES_PAGES: usize = 256; + +/// Wrap an opaque JMAP state token in this build's checkpoint scheme. +fn encode_jmap_checkpoint(state: &str) -> String { + format!("{JMAP_STATE_SCHEME}{JMAP_STATE_DELIM}{state}") +} + +/// Recover the opaque JMAP state from a stored checkpoint, or `None` when the +/// token carries no scheme this build recognises. Only the scheme's own +/// delimiter is consumed; the remainder is the server's token verbatim. +fn parse_jmap_checkpoint(raw: &str) -> Option<&str> { + let state = raw + .trim() + .strip_prefix(JMAP_STATE_SCHEME)? + .strip_prefix(JMAP_STATE_DELIM)?; + (!state.is_empty()).then_some(state) +} + /// JMAP Session Object (RFC 8620 Section 2). #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -54,18 +95,44 @@ pub struct JmapEmailGetResponse { pub list: Vec<JmapEmail>, } -/// JMAP `Email/changes` response payload wrapper. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// One page of a JMAP `Email/changes` response (RFC 8621 section 4.2, over the +/// RFC 8620 section 5.2 core method). +#[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct JmapEmailChangesResponse { /// Old state token queried against. + #[serde(default)] pub old_state: String, - /// New state token after differential sync. + /// New state token after this page of changes. pub new_state: String, - /// List of created or updated email IDs. + /// Ids of objects created since `oldState`. + #[serde(default)] + pub created: Vec<String>, + /// Ids of objects updated since `oldState`. + #[serde(default)] pub updated: Vec<String>, - /// List of destroyed/deleted email IDs. + /// Ids of objects destroyed since `oldState`. + #[serde(default)] pub destroyed: Vec<String>, + /// Whether the server truncated this page. When `true`, `newState` is an + /// intermediate checkpoint and the caller must ask again from it. + #[serde(default)] + pub has_more_changes: bool, +} + +/// What an `Email/changes` call established. +/// +/// The `cannotCalculateChanges` arm is a first-class outcome rather than an +/// error: RFC 8620 section 5.2 defines it as the server declining to compute a +/// delta from the given state (the state is too old, or the server never kept +/// enough history), and the prescribed response is to re-enumerate rather than +/// to fail the sync. +#[derive(Debug, Clone)] +pub enum EmailChangesPage { + /// The server answered with a page of changes. + Changes(JmapEmailChangesResponse), + /// The server cannot compute a delta from the supplied `sinceState`. + CannotCalculateChanges, } /// JMAP `Email/query` response payload wrapper (RFC 8620 Section 5.5), narrowed @@ -99,6 +166,16 @@ pub struct JmapEmail { /// than one `Message-ID` header. The first entry is the one used. #[serde(default)] pub message_id: Option<Vec<String>>, + /// Mailboxes this message currently belongs to, as `{mailboxId: true}` + /// (RFC 8621 section 4.1.1). JMAP models mailbox membership as a set, so + /// this is the only thing that says which folders a message is placed in -- + /// and, for a message the server reports as changed, whether it is still + /// placed in the folder being synced at all. + /// + /// `None` means the property was not requested or not returned, which is + /// "unknown membership", never "belongs to nothing". + #[serde(default)] + pub mailbox_ids: Option<std::collections::HashMap<String, bool>>, } /// JMAP email address object. @@ -191,7 +268,10 @@ impl JmapEngine { { "accountId": account_id, "ids": ids, - "properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet", "messageId"] + // `mailboxIds` is requested because a changed message's + // folder membership is what decides whether this pass + // places it or reconciles it away. + "properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet", "messageId", "mailboxIds"] }, "c1" ] @@ -376,6 +456,24 @@ impl JmapEngine { pub fn parse_email_get_response( &self, raw_json: &str, + ) -> Result<(Vec<PlacedMessage>, String), MailError> { + self.parse_email_get_response_in_folder(raw_json, "inbox") + } + + /// Parse a raw JMAP `Email/get` JSON response payload into the occupancies + /// it establishes for `folder_id`, plus the account's Email state string. + /// + /// A returned message whose `mailboxIds` is present and does NOT name + /// `folder_id` is dropped rather than placed: `Email/changes` is + /// account-wide (RFC 8621 section 4.2 offers no mailbox filter), so a + /// folder pass sees ids that belong to other mailboxes, and placing them + /// here would invent an occupancy the server never reported. Membership + /// that the server did not return at all is left alone -- unknown is not + /// absent. + fn parse_email_get_response_in_folder( + &self, + raw_json: &str, + folder_id: &str, ) -> Result<(Vec<PlacedMessage>, String), MailError> { let val: Value = serde_json::from_str(raw_json) .map_err(|e| MailError::ParseFailed(format!("invalid JMAP JSON response: {e}")))?; @@ -386,6 +484,10 @@ impl JmapEngine { let emails = resp .list .into_iter() + .filter(|item| match &item.mailbox_ids { + Some(ids) => ids.get(folder_id).copied().unwrap_or(false), + None => true, + }) .map(|item| { let sender = item .from @@ -401,7 +503,7 @@ impl JmapEngine { .map(|a| a.email.clone()) .unwrap_or_else(|| "me@nuncio.mx".to_string()); - let folder_id = "inbox".to_string(); + let folder_id = folder_id.to_string(); let remote_id = item.id; let message_id = item .message_id @@ -461,18 +563,31 @@ impl JmapEngine { Ok((emails, resp.state)) } - /// Parse raw JMAP `Email/changes` JSON response payload. - pub fn parse_email_changes_response( - &self, - raw_json: &str, - ) -> Result<(Vec<String>, Vec<String>, String), MailError> { + /// Parse one raw JMAP `Email/changes` JSON response payload. + /// + /// A method-level error carries a `type` (RFC 8620 section 3.6.2), which a + /// successful `Email/changes` payload never does, so the discriminator + /// works whether the caller hands over the full `methodResponses` envelope + /// or a bare payload. `cannotCalculateChanges` is returned as an outcome so + /// the caller can fall back to a full enumeration; any other error type is + /// a genuine failure and surfaces as one. + pub fn parse_email_changes_response(raw_json: &str) -> Result<EmailChangesPage, MailError> { let val: Value = serde_json::from_str(raw_json) .map_err(|e| MailError::ParseFailed(format!("invalid JMAP JSON response: {e}")))?; let payload = Self::extract_method_response_payload(&val)?; + + if let Some(error_type) = payload.get("type").and_then(|v| v.as_str()) { + if error_type == "cannotCalculateChanges" { + return Ok(EmailChangesPage::CannotCalculateChanges); + } + return Err(MailError::OperationFailed(format!( + "JMAP Email/changes failed with error type '{error_type}'" + ))); + } + let resp: JmapEmailChangesResponse = serde_json::from_value(payload) .map_err(|e| MailError::ParseFailed(format!("invalid Email/changes payload: {e}")))?; - - Ok((resp.updated, resp.destroyed, resp.new_state)) + Ok(EmailChangesPage::Changes(resp)) } /// Parse raw JMAP `Email/query` JSON response payload into a matched id list. @@ -556,6 +671,202 @@ impl JmapEngine { .cloned() .unwrap_or_else(|| self.account_id.clone()) } + + /// Address one occupancy of `remote_id` in `folder_id`. JMAP has no + /// UIDVALIDITY, so the sentinel stands in for that half of the key. + fn placement_key(&self, folder_id: &str, remote_id: &str) -> PlacementKey { + PlacementKey { + account_id: self.account_id.clone(), + folder_id: folder_id.to_string(), + uid_validity: JMAP_UID_VALIDITY_SENTINEL.to_string(), + remote_id: remote_id.to_string(), + } + } + + /// Follow `Email/changes` from `since_state` until the server stops setting + /// `hasMoreChanges`, accumulating every page. + /// + /// Returns `None` when the server answered `cannotCalculateChanges`, which + /// is the caller's cue to re-enumerate instead of failing. Stopping at the + /// first page would silently drop every change past the server's page + /// limit, so the loop is the contract, not an optimisation. + async fn collect_email_changes( + &self, + api_url: &str, + jmap_account_id: &str, + since_state: &str, + ) -> Result<Option<EmailChangeSet>, MailError> { + let mut accumulated = EmailChangeSet { + changed: Vec::new(), + destroyed: Vec::new(), + new_state: since_state.to_string(), + }; + let mut seen_changed: std::collections::HashSet<String> = std::collections::HashSet::new(); + let mut seen_destroyed: std::collections::HashSet<String> = + std::collections::HashSet::new(); + let mut cursor = since_state.to_string(); + + for _ in 0..MAX_CHANGES_PAGES { + let request = Self::build_email_changes_request(jmap_account_id, &cursor); + let raw = self.post_jmap(api_url, &request).await?; + let page = match Self::parse_email_changes_response(&raw)? { + EmailChangesPage::CannotCalculateChanges => return Ok(None), + EmailChangesPage::Changes(page) => page, + }; + + for id in page.created.into_iter().chain(page.updated) { + if seen_changed.insert(id.clone()) { + accumulated.changed.push(id); + } + } + for id in page.destroyed { + if seen_destroyed.insert(id.clone()) { + accumulated.destroyed.push(id); + } + } + accumulated.new_state = page.new_state.clone(); + + if !page.has_more_changes { + // A message created and then destroyed within the same window + // appears in both lists; it is gone, so absence wins. + accumulated + .changed + .retain(|id| !seen_destroyed.contains(id)); + return Ok(Some(accumulated)); + } + if page.new_state == cursor { + return Err(MailError::OperationFailed( + "JMAP Email/changes reported more changes without advancing newState" + .to_string(), + )); + } + cursor = page.new_state; + } + + Err(MailError::OperationFailed(format!( + "JMAP Email/changes did not converge within {MAX_CHANGES_PAGES} pages" + ))) + } + + /// Enumerate a folder in full: every message it currently holds, and a + /// `present` set that says so. + /// + /// This is the only pass entitled to report presence, because it is the + /// only one that genuinely looked at the whole mailbox. + async fn full_folder_enumeration( + &self, + api_url: &str, + jmap_account_id: &str, + folder_id: &str, + ) -> Result<FolderChanges, MailError> { + let query_request = Self::build_email_query_request(jmap_account_id, folder_id); + let query_raw = self.post_jmap(api_url, &query_request).await?; + let ids = Self::parse_email_query_response(&query_raw)?; + + let get_request = Self::build_email_get_request(jmap_account_id, Some(ids)); + let get_raw = self.post_jmap(api_url, &get_request).await?; + let (emails, state) = self.parse_email_get_response_in_folder(&get_raw, folder_id)?; + + let present: Vec<PlacementKey> = emails.iter().map(|m| m.placement.key()).collect(); + + tracing::info!( + account_id = %self.account_id, + folder_id, + fetched = emails.len(), + "jmap full folder enumeration complete" + ); + + Ok(FolderChanges { + upserts: emails, + // Everything absent is expressed through `present`; a full pass has + // no separate report of removals to make. + removals: Vec::new(), + present: Some(present), + next_state: encode_jmap_checkpoint(&state), + }) + } + + /// Enumerate one folder's changes since `since_state` via `Email/changes`. + /// + /// Returns `None` when the server cannot compute the delta, so the caller + /// can fall back to a full enumeration. + async fn incremental_folder_pass( + &self, + api_url: &str, + jmap_account_id: &str, + folder_id: &str, + since_state: &str, + ) -> Result<Option<FolderChanges>, MailError> { + let Some(changes) = self + .collect_email_changes(api_url, jmap_account_id, since_state) + .await? + else { + return Ok(None); + }; + + let upserts = if changes.changed.is_empty() { + Vec::new() + } else { + let get_request = + Self::build_email_get_request(jmap_account_id, Some(changes.changed.clone())); + let get_raw = self.post_jmap(api_url, &get_request).await?; + let (emails, _state) = self.parse_email_get_response_in_folder(&get_raw, folder_id)?; + emails + }; + + // Destruction is account-wide, so a destroyed id takes its occupancy + // here with it. A changed id that came back placed somewhere other than + // this folder -- or did not come back at all, having been destroyed + // between the two calls -- has left this folder, and its occupancy goes + // too. The message itself survives as long as another placement does; + // that is the caller's decision, not this one's. + let placed: std::collections::HashSet<&str> = upserts + .iter() + .map(|m| m.placement.remote_id.as_str()) + .collect(); + let removals: Vec<PlacementKey> = changes + .destroyed + .iter() + .map(String::as_str) + .chain( + changes + .changed + .iter() + .map(String::as_str) + .filter(|id| !placed.contains(id)), + ) + .map(|id| self.placement_key(folder_id, id)) + .collect(); + + tracing::info!( + account_id = %self.account_id, + folder_id, + fetched = upserts.len(), + removed = removals.len(), + "jmap incremental folder sync complete" + ); + + Ok(Some(FolderChanges { + upserts, + removals, + // An `Email/changes` pass sees only what moved. It never enumerates + // the mailbox, so it cannot speak to absence beyond what the server + // explicitly reported, and claiming otherwise would delete every + // message that simply did not change. + present: None, + next_state: encode_jmap_checkpoint(&changes.new_state), + })) + } +} + +/// Every `Email/changes` page for one pass, collapsed. +struct EmailChangeSet { + /// Created and updated ids, deduplicated, in first-seen order. + changed: Vec<String>, + /// Destroyed ids, deduplicated, in first-seen order. + destroyed: Vec<String>, + /// The final `newState`, once the server stopped paging. + new_state: String, } #[async_trait] @@ -593,14 +904,14 @@ impl MailBackend for JmapEngine { } #[tracing::instrument( - skip(self, _since_state), + skip(self, since_state), fields(account_id = %self.account_id, folder_id = %folder_id), err )] async fn sync_changes( &self, folder_id: &str, - _since_state: Option<&str>, + since_state: Option<&str>, ) -> Result<FolderChanges, MailError> { if !self.has_credentials() { return Err(MailError::AuthError( @@ -610,31 +921,25 @@ impl MailBackend for JmapEngine { let session = self.discover_session().await?; let account_id = self.resolve_account_id(&session); - let query_request = Self::build_email_query_request(&account_id, folder_id); - let query_raw = self.post_jmap(&session.api_url, &query_request).await?; - let ids = Self::parse_email_query_response(&query_raw)?; + // A checkpoint this build wrote resumes differentially. Anything else + // -- absent, or carrying no scheme this build recognises -- enumerates + // the folder in full, which is always safe. + if let Some(state) = since_state.and_then(parse_jmap_checkpoint) { + if let Some(changes) = self + .incremental_folder_pass(&session.api_url, &account_id, folder_id, state) + .await? + { + return Ok(changes); + } + tracing::info!( + account_id = %self.account_id, + folder_id, + "jmap server cannot calculate changes from the stored state; re-enumerating" + ); + } - let get_request = Self::build_email_get_request(&account_id, Some(ids)); - let get_raw = self.post_jmap(&session.api_url, &get_request).await?; - let (emails, state) = self.parse_email_get_response(&get_raw)?; - - tracing::info!( - account_id = %self.account_id, - folder_id, - fetched = emails.len(), - "jmap message sync complete" - ); - - Ok(FolderChanges { - upserts: emails, - removals: Vec::new(), - // `Email/query` was filtered to one mailbox, but this engine does - // not yet use `Email/changes`, so it cannot claim to have seen the - // complete set. Reporting `None` keeps the caller from deleting - // anything on the strength of a partial view. - present: None, - next_state: state, - }) + self.full_folder_enumeration(&session.api_url, &account_id, folder_id) + .await } #[tracing::instrument(skip(self), fields(account_id = %self.account_id), err)] @@ -743,19 +1048,84 @@ mod tests { #[test] fn parse_email_changes_response_valid_payload() { - let engine = JmapEngine::new("acct-1"); let raw = r#"{ "oldState": "s-1", "newState": "s-2", + "created": ["msg-12"], "updated": ["msg-10", "msg-11"], - "destroyed": ["msg-5"] + "destroyed": ["msg-5"], + "hasMoreChanges": true }"#; - let (updated, destroyed, new_state) = engine - .parse_email_changes_response(raw) - .expect("parse changes"); - assert_eq!(updated.len(), 2); - assert_eq!(destroyed.len(), 1); - assert_eq!(new_state, "s-2"); + let EmailChangesPage::Changes(page) = + JmapEngine::parse_email_changes_response(raw).expect("parse changes") + else { + panic!("a well-formed page must not read as cannotCalculateChanges"); + }; + assert_eq!(page.created, vec!["msg-12".to_string()]); + assert_eq!(page.updated.len(), 2); + assert_eq!(page.destroyed.len(), 1); + assert_eq!(page.new_state, "s-2"); + assert!(page.has_more_changes); + } + + /// `cannotCalculateChanges` is an outcome, not a failure: RFC 8620 s5.2 + /// defines it as "re-enumerate", so it must not surface as an error that + /// aborts the sync. Any other method error still must. + #[test] + fn parse_email_changes_response_distinguishes_cannot_calculate_from_failure() { + let raw = r#"{"methodResponses":[["error",{"type":"cannotCalculateChanges"},"c1"]]}"#; + assert!(matches!( + JmapEngine::parse_email_changes_response(raw), + Ok(EmailChangesPage::CannotCalculateChanges) + )); + + let failed = r#"{"methodResponses":[["error",{"type":"serverFail"},"c1"]]}"#; + assert!(JmapEngine::parse_email_changes_response(failed).is_err()); + } + + #[test] + fn jmap_checkpoints_round_trip_through_their_scheme() { + // A JMAP state is opaque and may itself contain the delimiter, so only + // the scheme's own separator is structural. + let encoded = encode_jmap_checkpoint("state:with:colons"); + assert_eq!(encoded, "v1:jmapstate:state:with:colons"); + assert_eq!(parse_jmap_checkpoint(&encoded), Some("state:with:colons")); + } + + #[test] + fn an_untagged_or_unknown_scheme_checkpoint_is_not_interpreted() { + // A bare token written before tagging existed, or one from another + // scheme, must resolve to a full re-enumeration rather than being + // handed to the server as a sinceState it never issued. + assert_eq!(parse_jmap_checkpoint("sync-state-900"), None); + assert_eq!(parse_jmap_checkpoint("v1:uidnext:42:105"), None); + assert_eq!(parse_jmap_checkpoint("v1:jmapstate:"), None); + } + + #[test] + fn parse_email_get_response_drops_messages_placed_in_another_mailbox() { + // `Email/changes` is account-wide, so a folder pass sees ids that + // belong elsewhere. Membership the server did return is authoritative; + // membership it never returned is unknown, not absent. + let raw = r#"{ + "methodResponses": [ + ["Email/get", {"state": "s-1", "list": [ + {"id": "m-here", "mailboxIds": {"mb-inbox": true}}, + {"id": "m-elsewhere", "mailboxIds": {"mb-archive": true}}, + {"id": "m-unknown"} + ]}, "c1"] + ] + }"#; + let engine = JmapEngine::new("acct-1"); + let (emails, _state) = engine + .parse_email_get_response_in_folder(raw, "mb-inbox") + .expect("parse get"); + let ids: Vec<&str> = emails + .iter() + .map(|m| m.placement.remote_id.as_str()) + .collect(); + assert_eq!(ids, vec!["m-here", "m-unknown"]); + assert_eq!(emails[0].placement.folder_id, "mb-inbox"); } #[test] diff --git a/crates/nuncio-mail/src/lib.rs b/crates/nuncio-mail/src/lib.rs index 6155b1d..b2cb9ef 100644 --- a/crates/nuncio-mail/src/lib.rs +++ b/crates/nuncio-mail/src/lib.rs @@ -22,7 +22,7 @@ pub use backend::{ RemoteMutationKind, RemoteMutationSpec, }; pub use imap::{IdleSocketState, ImapDualSocketManager, ImapEngine}; -pub use jmap::JmapEngine; +pub use jmap::{EmailChangesPage, JmapEngine}; pub use mock::{MockMailBackend, MockMessageSender}; pub use parser::{MailError, MimeParserAdapter}; pub use smtp::SmtpTransportEngine; diff --git a/crates/nuncio-mail/tests/wiremock_jmap_test.rs b/crates/nuncio-mail/tests/wiremock_jmap_test.rs index 46b9415..d330f76 100644 --- a/crates/nuncio-mail/tests/wiremock_jmap_test.rs +++ b/crates/nuncio-mail/tests/wiremock_jmap_test.rs @@ -2,7 +2,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use nuncio_core::model::IdentitySource; +use nuncio_core::model::{IdentitySource, PlacementKey}; use nuncio_mail::{JmapEngine, MailBackend, RemoteMutationKind, RemoteMutationSpec}; use serde_json::json; use wiremock::matchers::{body_partial_json, method, path}; @@ -209,7 +209,10 @@ async fn jmap_engine_sync_folders_and_messages_through_mail_backend_trait() { .expect("sync_messages succeeds"); assert_eq!(emails.len(), 1); assert_eq!(emails[0].placement.remote_id, "msg-wm-200"); - assert_eq!(emails[0].placement.folder_id, "inbox"); + // The occupancy is labelled with the mailbox that was actually synced. + // Removals are addressed by the same key, so a placement stored under any + // other folder id could never be reconciled away again. + assert_eq!(emails[0].placement.folder_id, "mb-inbox"); assert_eq!(emails[0].source, IdentitySource::EmailId); assert_ne!(emails[0].email.id, "msg-wm-200"); assert_eq!(emails[0].email.id.len(), 64); @@ -222,7 +225,9 @@ async fn jmap_engine_sync_folders_and_messages_through_mail_backend_trait() { emails[0].email.body_plain, Some("Fetched through the MailBackend trait.".to_string()) ); - assert_eq!(new_state, "sync-state-900"); + // The checkpoint is scheme-tagged, so a later build can tell what the + // opaque server token inside it actually means. + assert_eq!(new_state, "v1:jmapstate:sync-state-900"); } /// A session-discovery failure (non-2xx status) must surface as a genuine `MailError`, never a @@ -392,3 +397,385 @@ async fn jmap_apply_mutation_surfaces_server_rejection_as_error() { .expect_err("a server-rejected mutation must not be reported as success"); assert!(err.to_string().contains("jmap-msg-1")); } + +// --- Email/changes differential sync (RFC 8621 s4.2 over RFC 8620 s5.2) --- + +/// Mount one `Email/changes` page, matched on the exact `sinceState` the engine +/// must send. A pass that resumes from the wrong token matches no mock and the +/// request fails, so the matcher is itself the round-trip assertion. +async fn mount_email_changes_page( + mock_server: &MockServer, + since_state: &str, + body: serde_json::Value, +) { + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(json!({ + "methodCalls": [["Email/changes", { "sinceState": since_state }, "c1"]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "methodResponses": [["Email/changes", body, "c1"]] + }))) + .mount(mock_server) + .await; +} + +/// Mount an `Email/get` answering with `list` verbatim and the given state. +async fn mount_email_get(mock_server: &MockServer, state: &str, list: serde_json::Value) { + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(json!({ + "methodCalls": [["Email/get", {}, "c1"]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "methodResponses": [["Email/get", {"state": state, "list": list}, "c1"]] + }))) + .mount(mock_server) + .await; +} + +fn message(id: &str, mailbox_id: &str) -> serde_json::Value { + json!({ + "id": id, + "subject": format!("Message {id}"), + "from": [{ "email": "sender@nuncio.mx" }], + "to": [{ "email": "james.maes@kof22.com" }], + "receivedAt": 1700003000i64, + "isUnread": true, + "mailboxIds": { mailbox_id: true } + }) +} + +fn placement_key(folder_id: &str, remote_id: &str) -> PlacementKey { + PlacementKey { + account_id: "acct-100".to_string(), + folder_id: folder_id.to_string(), + uid_validity: "jmap".to_string(), + remote_id: remote_id.to_string(), + } +} + +/// The whole point of the differential pass: ids the server reports as +/// `destroyed` come back as placement removals, addressed exactly as the +/// occupancies this engine stores are -- so the sync path deletes those +/// placements and nothing else. An incremental pass still reports +/// `present: None`, because it never enumerated the folder. +#[tokio::test] +async fn jmap_sync_changes_turns_destroyed_ids_into_placement_removals() { + let mock_server = MockServer::start().await; + mount_session(&mock_server).await; + mount_email_changes_page( + &mock_server, + "srv-state-1", + json!({ + "oldState": "srv-state-1", + "newState": "srv-state-2", + "created": ["m-new"], + "updated": [], + "destroyed": ["m-gone-1", "m-gone-2"], + "hasMoreChanges": false + }), + ) + .await; + mount_email_get( + &mock_server, + "srv-state-2", + json!([message("m-new", "mb-inbox")]), + ) + .await; + + let engine = jmap_engine(&mock_server); + let changes = engine + .sync_changes("mb-inbox", Some("v1:jmapstate:srv-state-1")) + .await + .expect("incremental sync succeeds"); + + assert_eq!(changes.upserts.len(), 1); + assert_eq!(changes.upserts[0].placement.remote_id, "m-new"); + assert_eq!(changes.upserts[0].placement.folder_id, "mb-inbox"); + assert_eq!(changes.upserts[0].source, IdentitySource::EmailId); + + // Exactly the destroyed ids, keyed the same way the stored placement is. + assert_eq!( + changes.removals, + vec![ + placement_key("mb-inbox", "m-gone-1"), + placement_key("mb-inbox", "m-gone-2"), + ] + ); + // A message the pass placed must never also be reconciled away. + assert!(!changes.removals.iter().any(|key| key.remote_id == "m-new")); + + // An incremental pass cannot speak to absence beyond what it was told. + assert!( + changes.present.is_none(), + "an incremental pass must not claim to have enumerated the folder" + ); + assert_eq!(changes.next_state, "v1:jmapstate:srv-state-2"); +} + +/// `hasMoreChanges: true` means the server truncated the page. Stopping there +/// silently drops every change past the first page, so the pass must follow the +/// intermediate `newState` until the server stops paging, and accumulate. +#[tokio::test] +async fn jmap_sync_changes_follows_has_more_changes_to_convergence() { + let mock_server = MockServer::start().await; + mount_session(&mock_server).await; + mount_email_changes_page( + &mock_server, + "page-1", + json!({ + "oldState": "page-1", + "newState": "page-2", + "created": ["m-page1"], + "destroyed": ["m-gone-page1"], + "hasMoreChanges": true + }), + ) + .await; + mount_email_changes_page( + &mock_server, + "page-2", + json!({ + "oldState": "page-2", + "newState": "page-3", + "updated": ["m-page2"], + "destroyed": ["m-gone-page2"], + "hasMoreChanges": false + }), + ) + .await; + mount_email_get( + &mock_server, + "page-3", + json!([ + message("m-page1", "mb-inbox"), + message("m-page2", "mb-inbox") + ]), + ) + .await; + + let engine = jmap_engine(&mock_server); + let changes = engine + .sync_changes("mb-inbox", Some("v1:jmapstate:page-1")) + .await + .expect("paged incremental sync succeeds"); + + let upserted: Vec<&str> = changes + .upserts + .iter() + .map(|m| m.placement.remote_id.as_str()) + .collect(); + assert_eq!( + upserted, + vec!["m-page1", "m-page2"], + "both pages' changed ids must be fetched, not just the first" + ); + assert_eq!( + changes.removals, + vec![ + placement_key("mb-inbox", "m-gone-page1"), + placement_key("mb-inbox", "m-gone-page2"), + ], + "removals must accumulate across pages" + ); + // The last page's state is the resume point; an intermediate one would + // replay the tail of the change log forever. + assert_eq!(changes.next_state, "v1:jmapstate:page-3"); +} + +/// `cannotCalculateChanges` (RFC 8620 s5.2) is the server declining to compute +/// a delta. The pass must re-enumerate the folder rather than fail -- and only +/// then may it report `present`, because only then has it genuinely seen +/// everything the folder holds. +#[tokio::test] +async fn jmap_sync_changes_falls_back_to_full_enumeration_and_only_then_reports_presence() { + let mock_server = MockServer::start().await; + mount_session(&mock_server).await; + + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(json!({ + "methodCalls": [["Email/changes", {}, "c1"]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "methodResponses": [["error", {"type": "cannotCalculateChanges"}, "c1"]] + }))) + .mount(&mock_server) + .await; + + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(json!({ + "methodCalls": [["Email/query", {"filter": {"inMailboxes": ["mb-inbox"]}}, "c1"]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "methodResponses": [["Email/query", {"ids": ["m-a", "m-b"]}, "c1"]] + }))) + .mount(&mock_server) + .await; + + mount_email_get( + &mock_server, + "srv-full-9", + json!([message("m-a", "mb-inbox"), message("m-b", "mb-inbox")]), + ) + .await; + + let engine = jmap_engine(&mock_server); + let changes = engine + .sync_changes("mb-inbox", Some("v1:jmapstate:far-too-old")) + .await + .expect("a stale state must re-enumerate, not fail the sync"); + + assert_eq!(changes.upserts.len(), 2); + assert_eq!( + changes.present, + Some(vec![ + placement_key("mb-inbox", "m-a"), + placement_key("mb-inbox", "m-b"), + ]), + "a genuine full enumeration is the one pass entitled to report presence" + ); + assert!( + changes.removals.is_empty(), + "a full pass expresses absence through `present`, not through removals" + ); + assert_eq!(changes.next_state, "v1:jmapstate:srv-full-9"); +} + +/// The `newState` a pass returns is the token the next pass must resume from. +/// The second pass's `Email/changes` mock only matches that exact state, so a +/// token that failed to round-trip through `next_state` would match nothing. +#[tokio::test] +async fn jmap_sync_changes_resumes_from_the_state_the_previous_pass_returned() { + let mock_server = MockServer::start().await; + mount_session(&mock_server).await; + mount_email_changes_page( + &mock_server, + "round-1", + json!({ + "oldState": "round-1", + "newState": "round-2", + "destroyed": ["m-gone"], + "hasMoreChanges": false + }), + ) + .await; + mount_email_changes_page( + &mock_server, + "round-2", + json!({ + "oldState": "round-2", + "newState": "round-3", + "destroyed": ["m-gone-later"], + "hasMoreChanges": false + }), + ) + .await; + + let engine = jmap_engine(&mock_server); + let first = engine + .sync_changes("mb-inbox", Some("v1:jmapstate:round-1")) + .await + .expect("first pass succeeds"); + assert_eq!(first.next_state, "v1:jmapstate:round-2"); + + let second = engine + .sync_changes("mb-inbox", Some(&first.next_state)) + .await + .expect("the returned checkpoint must be accepted verbatim on the next pass"); + assert_eq!( + second.removals, + vec![placement_key("mb-inbox", "m-gone-later")] + ); + assert_eq!(second.next_state, "v1:jmapstate:round-3"); +} + +/// A checkpoint carrying no scheme this build recognises -- a bare token +/// written before tagging existed -- must never be handed to the server as a +/// `sinceState`. It resolves to a full enumeration instead: over-fetching is +/// recoverable, resuming from a token the server reads differently is not. +#[tokio::test] +async fn jmap_sync_changes_re_enumerates_when_the_stored_checkpoint_is_untagged() { + let mock_server = MockServer::start().await; + mount_session(&mock_server).await; + + // No `Email/changes` mock is mounted on purpose: such a call would 404 and + // fail the sync, which is what proves none was attempted. + Mock::given(method("POST")) + .and(path("/jmap/api")) + .and(body_partial_json(json!({ + "methodCalls": [["Email/query", {}, "c1"]] + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "methodResponses": [["Email/query", {"ids": ["m-a"]}, "c1"]] + }))) + .mount(&mock_server) + .await; + mount_email_get( + &mock_server, + "srv-full-1", + json!([message("m-a", "mb-inbox")]), + ) + .await; + + let engine = jmap_engine(&mock_server); + let changes = engine + .sync_changes("mb-inbox", Some("sync-state-900")) + .await + .expect("an unrecognised checkpoint must re-enumerate"); + assert_eq!( + changes.present, + Some(vec![placement_key("mb-inbox", "m-a")]) + ); + assert_eq!(changes.next_state, "v1:jmapstate:srv-full-1"); +} + +/// `Email/changes` is account-wide (RFC 8621 s4.2 has no mailbox filter), so a +/// folder pass sees ids that now live in other mailboxes. A message another +/// client moved out has left this folder: its occupancy here is reconciled +/// away, and it is NOT placed here on the strength of having merely changed. +#[tokio::test] +async fn jmap_sync_changes_reconciles_a_message_moved_out_of_the_folder() { + let mock_server = MockServer::start().await; + mount_session(&mock_server).await; + mount_email_changes_page( + &mock_server, + "srv-state-1", + json!({ + "oldState": "srv-state-1", + "newState": "srv-state-2", + "updated": ["m-moved", "m-stayed"], + "hasMoreChanges": false + }), + ) + .await; + mount_email_get( + &mock_server, + "srv-state-2", + json!([ + message("m-moved", "mb-archive"), + message("m-stayed", "mb-inbox") + ]), + ) + .await; + + let engine = jmap_engine(&mock_server); + let changes = engine + .sync_changes("mb-inbox", Some("v1:jmapstate:srv-state-1")) + .await + .expect("incremental sync succeeds"); + + let upserted: Vec<&str> = changes + .upserts + .iter() + .map(|m| m.placement.remote_id.as_str()) + .collect(); + assert_eq!(upserted, vec!["m-stayed"]); + assert_eq!( + changes.removals, + vec![placement_key("mb-inbox", "m-moved")], + "a message whose mailboxes no longer include this folder must lose its occupancy here" + ); +} diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index 3b9dbe4..f30e8b6 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -940,10 +940,10 @@ mod tests { assert_eq!(synced, 1); // The message persists under an opaque surrogate id; it is recovered by - // folder, and its protocol-native JMAP object id round-trips in - // `remote_id`. + // the mailbox it was actually synced from, and its protocol-native JMAP + // object id round-trips in `remote_id`. let persisted = db - .list_messages("acct-jmap-real-1", "inbox", 10) + .list_messages("acct-jmap-real-1", "jmap-inbox", 10) .await .expect("jmap message persisted"); assert_eq!(persisted.len(), 1); -- 2.54.0 From b6987746ab9ae55cc5cbf01651fe7bac331bed56 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 18:08:52 -0500 Subject: [PATCH 88/95] docs: chart the shortest route to a hand-drivable daemon The engine is ahead of its contract: it executes verified mutations but the Mail service has no RPC to ask for one, so a human cannot move, delete or flag anything. By this repo's definition of done the placement work has an engine and nothing else. Records the three steps that buy manual usability, what works and what does not if you boot it today, the 81-issue backlog behind them, and the two test traps (tracing callsite Interest, tokio::time::pause over real I/O) that are invisible at default parallelism. --- docs/PATH-TO-MANUAL-USE.md | 167 +++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 docs/PATH-TO-MANUAL-USE.md diff --git a/docs/PATH-TO-MANUAL-USE.md b/docs/PATH-TO-MANUAL-USE.md new file mode 100644 index 0000000..072f535 --- /dev/null +++ b/docs/PATH-TO-MANUAL-USE.md @@ -0,0 +1,167 @@ +# Path to manual use + +> **Purpose:** the shortest honest route from where the engine is today to a +> daemon a human can drive by hand and test end to end. Written 2026-08-14, +> after the ADR 0002 sync model landed. Complements [`ROADMAP.md`](ROADMAP.md) +> (the full plan) and [`HANDOFF.md`](HANDOFF.md) (machine-to-machine state). + +## Where we actually are + +`dev` is green and carries the whole of [ADR 0002](adr/0002-multi-engine-sync-model.md): +message/placement identity split, four-rung IMAP change enumeration, three-state +verified mutations, per-account filter ownership, JMAP `Email/changes`. + +**The engine is ahead of its contract.** That is the one fact that matters here. + +``` +$ grep -c 'rpc \(MoveMessage\|DeleteMessage\|FlagMessage\)' \ + crates/nuncio-proto/proto/nuncio/v1/nuncio.proto +0 +``` + +The daemon can move, delete and flag mail — the outbox executes verified +mutations with three honest outcomes — but **nothing can ask it to**. By this +repo's own definition of done (*engine + proto + CLI command + offline E2E*), +the placement work is not finished: it has an engine and nothing else. + +## What that means if you boot it today + +Works: add an account, sync, list folders, list and read messages, search, +calendar and contacts sync, export, audit. + +Does not: **any mutation**. You cannot move, delete or flag a message from the +CLI, because the RPC does not exist. + +Two consequences of the same gap: + +- **Filters cannot be switched on.** `filters_enabled` is per-account and + **default off** (deliberately — one engine owns side-effecting actions, see + ADR 0002 §2). There is no proto field and no CLI command, so the only way to + enable it is editing the `accounts` row in SQLite by hand. +- **Placements are invisible.** `Message.folder_id` names the one occupancy the + message was reached through. A client cannot see that a message sits in three + folders, which is the thing the split exists to model. + +### Before you run it + +```bash +git checkout dev && git pull +rm -f ~/.nuncio/nuncio_main.db* # see below +cargo run -p nunciod # gRPC on 127.0.0.1:9420 +cargo run -p nuncio-cli -- system status +``` + +Schema migrations were removed: databases are **created, never migrated**. A +store from before that change errors on read rather than upgrading. Deleting it +is correct — accounts and credentials live in the OS keyring and survive; cached +mail re-syncs. Expect this again whenever `IDENTITY_SCHEMA_VERSION` bumps. + +## The critical path + +Three steps, in order. Everything else in the roadmap is behind them. + +### 1. #407 — cross-engine identity *(in flight)* + +IMAP requests `EMAILID` (RFC 8474, where the server advertises `OBJECTID`) and +`X-GM-MSGID` (where it advertises `X-GM-EXT-1`), so it reaches identity tier 1 +instead of only tiers 3–4. Until this lands, an IMAP engine and a JMAP engine +mint **disjoint key spaces** for the same mailbox and ADR 0002's convergence +claim is not true across protocols. + +Bumps `IDENTITY_SCHEMA_VERSION` (2 → 3): the store rebuilds and re-syncs rather +than re-keying every row incrementally through the placement repoint arm. + +### 2. #395 — the keystone + +The single thing standing between "engine works" and "a human can drive it": + +- **Mutation RPCs on `Mail`** — Move / Delete / Flag / Unflag, addressed by + message key **plus the placement to act on**, since a message can occupy + several mailboxes and `MOVE`/`DELETE` against the wrong one is destructive. +- **`Conflict` as a typed error**, not a response field, so clients that predate + it take the error path instead of silently misreading success. Conflicts are + durable and queryable (ADR 0002 §5), so this needs list/resolve RPCs too. +- **`Message` carries its placement set** — the field the split has been waiting + for. Today's `folder_id` becomes one view of it. +- **`filters_enabled` on the wire** so filter ownership is settable without + hand-editing SQLite. +- DAV conditional write-back. + +This takes the repo's one proto-touching-story slot (currently free) and gates +the M7 API freeze. + +### 3. Manual-readiness pass + +Drive every RPC from `nuncio-cli` against a live daemon, fix what is missing or +broken, and write the exercise guide. The open WS-F CLI issues (#327, #328, +#329, #332) largely fall out of this. + +## Scope, honestly + +**81 open issues** as of 2026-08-14 — that is the roadmap through M7, not a task +list: + +| open | milestone | +| ---: | --- | +| 15 | WS-B Mail model & mutations | +| 12 | WS-E Calendar/Contacts write & Search | +| 10 | WS-F Ops, usability & maintainability | +| 9 | WS-C Sync, push & lifecycle | +| 8 | WS-D Security to 9/10 · M6.5 de-fabrication · (no milestone) | +| 5 | M7 API freeze | +| 3 | M6 Security hardening · WS-A Contract hardening | + +The three steps above are what buy manual usability. The rest is the product. + +## Known-open, deliberately + +Recorded so nobody re-derives them: + +- **`recovery::tests::test_stage_2_backup_creation_and_stage_3_table_salvage`** + failed once in CI and has never reproduced (0/40 locally under artificial CPU + load, quiet across every run since). Genuinely undiagnosed — not fixed, not + understood. If it recurs, get a reproduction before attempting a fix. +- **#339** — `parse_email_get_response` keeps a fixed-folder signature. The + *production* path was corrected while implementing JMAP `Email/changes` + (removals key on folder, so the hardcoded `"inbox"` made every JMAP removal + inert); the public helper still reads as the issue describes. +- **`export_mbox` no longer emits `X-Nuncio-Folder`**, permanently. A message can + hold several placements; one header would name one arbitrarily and MBOX cannot + express the set. +- **A filter fire claim outlives its message.** `filter_fired` is never reaped, so + a deleted-and-re-arriving message never re-fires. Deliberate: reaping would + silently re-enable duplicate `FORWARD` / `CALL WEBHOOK`. +- **`db.rs` has one pre-existing `set_default`** log-capture test. Everything else + uses the process-global subscriber. Harmless today, worth folding into a + cleanup — see below. + +## Two traps that cost real time here + +Both are written up because they are invisible locally and expensive to +rediscover. + +1. **`tracing` caches callsite `Interest` process-globally; `set_default` is + per-thread.** A callsite first executed by a thread with no subscriber caches + `never` and is skipped forever, before any thread-local subscriber is + consulted. Reproduces at `--test-threads=2` (CI's core count), invisible at + higher parallelism. Every log-capture helper must install **one global** + subscriber and route per thread. Never reintroduce `set_default` / + `with_default`. +2. **`tokio::time::pause()` cannot wrap real blocking I/O.** Auto-advance fires + whenever the runtime *looks* idle — including while a `sqlx` query sits on the + blocking pool — and it also trips sqlx's pool-acquire timer (default 30s, + frequently the same duration as the timeout under test), turning a healthy + query into `PoolTimedOut` and the result into misleading zeros. Make timeouts + injectable and wait them out in real time instead. + +## Picking this up cold + +1. Read [ADR 0002](adr/0002-multi-engine-sync-model.md) — it is the design, and + its **Pre-release posture** section is load-bearing: nothing is released, + stores are disposable, behaviour may change without a deprecation path. +2. `cargo fmt --all -- --check`, `cargo check-all`, `cargo test-all` — the gate. + Also run `RUST_TEST_THREADS=2 cargo test-all` before trusting a green run; + default parallelism hides the flake class above. +3. `gh pr list` — PRs stack (`A → B → C`), and merging into `dev` does **not** + auto-close issues, because GitHub only honours `Closes #N` on merge into the + default branch. Close them by hand. -- 2.54.0 From 5402dfb0383f248736bb9a6a8658cf11b66292b9 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 18:36:40 -0500 Subject: [PATCH 89/95] feat(mail): make messages hand-drivable over the wire The engine already executed verified remote mutations, but the Mail service had no way to ask for one, so a human could not move, delete or flag a message at all. Add MoveMessage, DeleteMessage and FlagMessage, each addressing a specific mailbox occupancy rather than a bare message key: identity is content-addressed and folder-independent, so one message can sit in several mailboxes, and aiming a MOVE or DELETE at the wrong occupancy is destructive and has no undo. The outbox already refused an ambiguously-addressed mutation permanently; the queued row now carries the occupancy its intent was formed against, which turns that refusal into something a caller can avoid instead of a dead end. A named occupancy is honoured only after confirming the message still holds it. Conflicts become a typed error and durable state rather than a hopeful field. Append ERROR_REASON_CONFLICT (13) and surface it through the existing ErrorInfo status-details mechanism, so a client predating it takes the error path instead of misreading success; a conflict is raised only on the server's positive evidence, never on an unverifiable outcome. Because the event stream is a lossy broadcast with no resumption, the conflict is also written to a queryable table, with ListConflicts and ResolveConflict over it. Message now carries its complete placement set. folder_id keeps its meaning -- the occupancy the message was reached through -- so this is purely additive; its doc comment now points at the new field instead of saying no such field exists. filters_enabled reaches the wire. It was per-account and settable only by hand-editing the accounts row, which made filters untestable by hand; it is now on AccountConfig and flippable through UpdateAccount, still defaulting to off because filters take irreversible remote actions. CLI verbs land alongside: mail move/delete/flag/unflag, mail conflicts/resolve, and account filters. Refs #395 --- crates/nuncio-cli/src/args.rs | 80 ++++ crates/nuncio-cli/src/runner.rs | 433 ++++++++++++++++++ crates/nuncio-core/src/model.rs | 32 ++ crates/nuncio-filter/src/outbox.rs | 30 +- .../proto/nuncio/v1/descriptor.bin | Bin 16326 -> 18496 bytes .../nuncio-proto/proto/nuncio/v1/nuncio.proto | 177 ++++++- crates/nuncio-proto/src/errors.rs | 5 + crates/nuncio-store/src/db.rs | 170 +++++++ crates/nunciod/src/grpc.rs | 341 +++++++++++++- crates/nunciod/src/outbox.rs | 77 +++- .../tests/accounts_lifecycle_e2e_test.rs | 1 + .../tests/caldav_production_e2e_test.rs | 1 + .../nunciod/tests/no_secrets_canary_test.rs | 1 + crates/nunciod/tests/spine_e2e_test.rs | 1 + 14 files changed, 1320 insertions(+), 29 deletions(-) diff --git a/crates/nuncio-cli/src/args.rs b/crates/nuncio-cli/src/args.rs index b4059af..8e8bdf4 100644 --- a/crates/nuncio-cli/src/args.rs +++ b/crates/nuncio-cli/src/args.rs @@ -250,6 +250,26 @@ pub enum AccountSubcommand { #[arg(short, long, help = "Unique account identifier")] id: String, }, + /// Turn the filter engine on or off for one account. + Filters { + /// Unique account identifier. + #[arg(short, long, help = "Unique account identifier")] + id: String, + /// Run filters against this account's incoming mail. + #[arg( + long, + help = "Enable filters for this account", + conflicts_with = "disable" + )] + enable: bool, + /// Stop running filters against this account's incoming mail. + #[arg( + long, + help = "Disable filters for this account", + conflicts_with = "enable" + )] + disable: bool, + }, /// Edit an existing account profile configuration. Edit { /// Unique account identifier. @@ -343,6 +363,66 @@ pub enum MailSubcommand { #[arg(long, help = "Mark the message as unread", conflicts_with = "read")] unread: bool, }, + /// Move one occupancy of a message into another folder. + Move { + /// Unique message identifier. + #[arg(short, long, help = "Unique message identifier")] + id: String, + /// Destination folder identifier. + #[arg(short, long, help = "Destination folder identifier")] + to: String, + /// Which mailbox occupancy to act on, as + /// `account/folder/uidvalidity/uid`. Required when the message occupies + /// more than one mailbox; `mail read` lists them. + #[arg(long, help = "Occupancy as account/folder/uidvalidity/uid")] + placement: Option<String>, + }, + /// Delete one occupancy of a message. Other occupancies survive. + Delete { + /// Unique message identifier. + #[arg(short, long, help = "Unique message identifier")] + id: String, + /// Which mailbox occupancy to act on, as + /// `account/folder/uidvalidity/uid`. + #[arg(long, help = "Occupancy as account/folder/uidvalidity/uid")] + placement: Option<String>, + }, + /// Flag one occupancy of a message. + Flag { + /// Unique message identifier. + #[arg(short, long, help = "Unique message identifier")] + id: String, + /// Which mailbox occupancy to act on, as + /// `account/folder/uidvalidity/uid`. + #[arg(long, help = "Occupancy as account/folder/uidvalidity/uid")] + placement: Option<String>, + }, + /// Remove the flag from one occupancy of a message. + Unflag { + /// Unique message identifier. + #[arg(short, long, help = "Unique message identifier")] + id: String, + /// Which mailbox occupancy to act on, as + /// `account/folder/uidvalidity/uid`. + #[arg(long, help = "Occupancy as account/folder/uidvalidity/uid")] + placement: Option<String>, + }, + /// List mutations the server refused because another client changed the + /// message first. + Conflicts { + /// Also list conflicts that have already been resolved. + #[arg(long, help = "Include already-resolved conflicts")] + all: bool, + }, + /// Mark a conflict handled, recording how it was resolved. + Resolve { + /// Conflict identifier, as reported by `mail conflicts`. + #[arg(short, long, help = "Conflict identifier")] + id: String, + /// Note recording how the conflict was resolved. + #[arg(short, long, help = "How the conflict was resolved")] + note: String, + }, /// Export mailbox messages to a portable file format: MBOX, an EML /// zip archive, JSON, or JSON Lines. Export { diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index cf063c7..1d8ae7e 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -425,6 +425,14 @@ impl HeadlessRunner { .await } AccountSubcommand::Show { id } => self.handle_account_show(id, json_mode).await, + AccountSubcommand::Filters { + id, + enable, + disable, + } => { + self.handle_account_filters(id, *enable, *disable, json_mode) + .await + } AccountSubcommand::Edit { id, email, @@ -468,6 +476,28 @@ impl HeadlessRunner { MailSubcommand::Mark { id, read, unread } => { self.handle_mark_read(id, *read, *unread, json_mode).await } + MailSubcommand::Move { id, to, placement } => { + self.handle_mail_mutation("move", id, placement.as_ref(), Some(to), json_mode) + .await + } + MailSubcommand::Delete { id, placement } => { + self.handle_mail_mutation("delete", id, placement.as_ref(), None, json_mode) + .await + } + MailSubcommand::Flag { id, placement } => { + self.handle_mail_mutation("flag", id, placement.as_ref(), None, json_mode) + .await + } + MailSubcommand::Unflag { id, placement } => { + self.handle_mail_mutation("unflag", id, placement.as_ref(), None, json_mode) + .await + } + MailSubcommand::Conflicts { all } => { + self.handle_list_conflicts(*all, json_mode).await + } + MailSubcommand::Resolve { id, note } => { + self.handle_resolve_conflict(id, note, json_mode).await + } MailSubcommand::Export { format, out, @@ -1079,6 +1109,245 @@ impl HeadlessRunner { } } + /// Parse an `account/folder/uidvalidity/uid` occupancy selector. + /// + /// All four coordinates are required and none may be empty: a partial + /// selector addresses nothing, and silently completing it would aim a + /// destructive mutation at a mailbox the user did not name. + fn parse_placement(spec: &str) -> Result<nuncio_proto::v1::PlacementRef, String> { + let parts: Vec<&str> = spec.split('/').collect(); + match parts.as_slice() { + [account_id, folder_id, uidvalidity, uid] + if !account_id.is_empty() + && !folder_id.is_empty() + && !uidvalidity.is_empty() + && !uid.is_empty() => + { + Ok(nuncio_proto::v1::PlacementRef { + account_id: (*account_id).to_string(), + folder_id: (*folder_id).to_string(), + uidvalidity: (*uidvalidity).to_string(), + uid: (*uid).to_string(), + }) + } + _ => Err(format!( + "placement '{spec}' is not account/folder/uidvalidity/uid" + )), + } + } + + /// `mail move`/`delete`/`flag`/`unflag`: queue a placement-addressed remote + /// mutation via the daemon and report the outbox id it was queued as. + /// + /// The reported outcome is "queued", never "done": the daemon's outbox is + /// what actually talks to the server, and only it can say whether the + /// mutation applied, conflicted, or could not be verified. + async fn handle_mail_mutation( + &self, + verb: &str, + id: &str, + placement: Option<&String>, + destination: Option<&String>, + json_mode: bool, + ) -> String { + let placement = match placement.map(|p| Self::parse_placement(p)) { + Some(Ok(placement)) => Some(placement), + Some(Err(e)) => return Self::render_error(&e, json_mode), + None => None, + }; + + let mut client = match self.connect_mail_client().await { + Ok(client) => client, + Err(e) => return Self::render_error(&e, json_mode), + }; + + let queued = match verb { + "move" => { + let Some(to) = destination else { + return Self::render_error("a destination folder is required", json_mode); + }; + client + .move_message(nuncio_proto::v1::MoveMessageRequest { + message_id: id.to_string(), + placement, + destination_folder_id: to.clone(), + }) + .await + .map(|r| r.into_inner().mutation_id) + } + "delete" => client + .delete_message(nuncio_proto::v1::DeleteMessageRequest { + message_id: id.to_string(), + placement, + }) + .await + .map(|r| r.into_inner().mutation_id), + _ => client + .flag_message(nuncio_proto::v1::FlagMessageRequest { + message_id: id.to_string(), + placement, + flagged: verb == "flag", + }) + .await + .map(|r| r.into_inner().mutation_id), + }; + + match queued { + Ok(mutation_id) => { + if json_mode { + format_json(&json!({ + "status": "queued", + "operation": verb, + "id": id, + "mutation_id": mutation_id, + })) + } else { + format!("Queued {verb} of '{id}' as mutation '{mutation_id}'") + } + } + Err(status) => Self::render_status_error(verb, &status, json_mode), + } + } + + /// `mail conflicts`: list mutations the server refused because another + /// client changed the message first. + async fn handle_list_conflicts(&self, include_resolved: bool, json_mode: bool) -> String { + let mut client = match self.connect_mail_client().await { + Ok(client) => client, + Err(e) => return Self::render_error(&e, json_mode), + }; + match client + .list_conflicts(nuncio_proto::v1::ListConflictsRequest { + include_resolved, + limit: 100, + }) + .await + { + Ok(response) => { + let conflicts = response.into_inner().conflicts; + if json_mode { + let rows: Vec<_> = conflicts + .iter() + .map(|c| { + json!({ + "id": c.id, + "mutation_id": c.mutation_id, + "message_id": c.message_id, + "operation": c.mutation_type, + "observed": c.observed, + "resolved": c.resolved_at.is_some(), + }) + }) + .collect(); + format_json(&json!({ "conflicts": rows })) + } else if conflicts.is_empty() { + "No conflicts recorded".to_string() + } else { + let mut out = String::new(); + for c in &conflicts { + let state = if c.resolved_at.is_some() { + "resolved" + } else { + "open" + }; + out.push_str(&format!( + "{} [{}] {} on '{}': {}\n", + c.id, state, c.mutation_type, c.message_id, c.observed + )); + } + out.trim_end().to_string() + } + } + Err(status) => Self::render_status_error("list_conflicts", &status, json_mode), + } + } + + /// `mail resolve`: record a human's decision about a conflict. + async fn handle_resolve_conflict(&self, id: &str, note: &str, json_mode: bool) -> String { + let mut client = match self.connect_mail_client().await { + Ok(client) => client, + Err(e) => return Self::render_error(&e, json_mode), + }; + match client + .resolve_conflict(nuncio_proto::v1::ResolveConflictRequest { + conflict_id: id.to_string(), + resolution: note.to_string(), + }) + .await + { + Ok(_) => { + if json_mode { + format_json(&json!({ "status": "resolved", "id": id })) + } else { + format!("Conflict '{id}' resolved") + } + } + Err(status) => Self::render_status_error("resolve_conflict", &status, json_mode), + } + } + + /// `account filters`: flip one account's `filters_enabled` and persist it + /// through the daemon's `UpdateAccount` RPC. + /// + /// The account is read back first so the update carries its real current + /// configuration; sending a locally-invented config would silently reset + /// every field the CLI did not know about. + async fn handle_account_filters( + &self, + id: &str, + enable: bool, + disable: bool, + json_mode: bool, + ) -> String { + if enable == disable { + return Self::render_error( + "exactly one of --enable or --disable must be specified", + json_mode, + ); + } + + let mut client = match self.connect_accounts_client().await { + Ok(client) => client, + Err(e) => return Self::render_error(&e, json_mode), + }; + + let accounts = match client + .list_accounts(nuncio_proto::v1::ListAccountsRequest {}) + .await + { + Ok(response) => response.into_inner().accounts, + Err(status) => return Self::render_status_error("list_accounts", &status, json_mode), + }; + let Some(mut config) = accounts.into_iter().find(|a| a.id == id) else { + return Self::render_error(&format!("account '{id}' not found"), json_mode); + }; + config.filters_enabled = enable; + + match client + .update_account(nuncio_proto::v1::UpdateAccountRequest { + config: Some(config), + password: None, + }) + .await + { + Ok(_) => { + if json_mode { + format_json(&json!({ + "status": "updated", + "id": id, + "filters_enabled": enable, + })) + } else { + format!( + "Filters {} for account '{id}'", + if enable { "enabled" } else { "disabled" } + ) + } + } + Err(status) => Self::render_status_error("update_account", &status, json_mode), + } + } + /// `mail export`: a real thin gRPC client of the running `nunciod` /// daemon's `nuncio.v1.Export` API. The /// daemon writes the export to `out` on ITS OWN host filesystem (the @@ -1826,6 +2095,9 @@ impl HeadlessRunner { keyring_secret_key: keyring_key.clone(), sync_interval: Some(nuncio_proto::time::duration_from_secs(300)), transport: Some(transport), + // Filters take irreversible remote actions, so adding an account + // never opts it in; `account filters` does, deliberately. + filters_enabled: false, }; let mut client = match self.connect_accounts_client().await { @@ -3147,6 +3419,7 @@ mod tests { email_address: "stub@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-stub-1".to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(300)), + filters_enabled: false, transport: Some(nuncio_proto::v1::account_config::Transport::ImapSmtp( nuncio_proto::v1::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), @@ -3371,6 +3644,13 @@ mod tests { body_plain: Some("Stub body text".to_string()), body_html: None, attachments: Vec::new(), + placements: vec![nuncio_proto::v1::Placement { + account_id: "acct-stub-1".to_string(), + folder_id: "inbox".to_string(), + uidvalidity: "1".to_string(), + uid: "42".to_string(), + read: false, + }], } } @@ -3383,10 +3663,77 @@ mod tests { last_mark_read: Arc<Mutex<Option<MarkReadRequest>>>, last_send_message: Arc<Mutex<Option<SendMessageRequest>>>, last_sync: Arc<Mutex<Option<SyncRequest>>>, + last_move: Arc<Mutex<Option<nuncio_proto::v1::MoveMessageRequest>>>, + last_delete: Arc<Mutex<Option<nuncio_proto::v1::DeleteMessageRequest>>>, + last_flag: Arc<Mutex<Option<nuncio_proto::v1::FlagMessageRequest>>>, } #[tonic::async_trait] impl MailService for StubMail { + async fn move_message( + &self, + request: tonic::Request<nuncio_proto::v1::MoveMessageRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::MoveMessageResponse>, tonic::Status> + { + *self.last_move.lock().unwrap_or_else(|e| e.into_inner()) = + Some(request.into_inner()); + Ok(tonic::Response::new( + nuncio_proto::v1::MoveMessageResponse { + mutation_id: "mut-stub-1".to_string(), + }, + )) + } + + async fn delete_message( + &self, + request: tonic::Request<nuncio_proto::v1::DeleteMessageRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::DeleteMessageResponse>, tonic::Status> + { + *self.last_delete.lock().unwrap_or_else(|e| e.into_inner()) = + Some(request.into_inner()); + Ok(tonic::Response::new( + nuncio_proto::v1::DeleteMessageResponse { + mutation_id: "mut-stub-2".to_string(), + }, + )) + } + + async fn flag_message( + &self, + request: tonic::Request<nuncio_proto::v1::FlagMessageRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::FlagMessageResponse>, tonic::Status> + { + *self.last_flag.lock().unwrap_or_else(|e| e.into_inner()) = + Some(request.into_inner()); + Ok(tonic::Response::new( + nuncio_proto::v1::FlagMessageResponse { + mutation_id: "mut-stub-3".to_string(), + }, + )) + } + + async fn list_conflicts( + &self, + _request: tonic::Request<nuncio_proto::v1::ListConflictsRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::ListConflictsResponse>, tonic::Status> + { + Ok(tonic::Response::new( + nuncio_proto::v1::ListConflictsResponse { + conflicts: Vec::new(), + }, + )) + } + + async fn resolve_conflict( + &self, + _request: tonic::Request<nuncio_proto::v1::ResolveConflictRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::ResolveConflictResponse>, tonic::Status> + { + Ok(tonic::Response::new( + nuncio_proto::v1::ResolveConflictResponse {}, + )) + } + async fn list_folders( &self, _request: tonic::Request<ListFoldersRequest>, @@ -3717,6 +4064,46 @@ mod tests { #[tonic::async_trait] impl MailService for StubMailAlwaysErrors { + async fn move_message( + &self, + _request: tonic::Request<nuncio_proto::v1::MoveMessageRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::MoveMessageResponse>, tonic::Status> + { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + + async fn delete_message( + &self, + _request: tonic::Request<nuncio_proto::v1::DeleteMessageRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::DeleteMessageResponse>, tonic::Status> + { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + + async fn flag_message( + &self, + _request: tonic::Request<nuncio_proto::v1::FlagMessageRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::FlagMessageResponse>, tonic::Status> + { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + + async fn list_conflicts( + &self, + _request: tonic::Request<nuncio_proto::v1::ListConflictsRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::ListConflictsResponse>, tonic::Status> + { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + + async fn resolve_conflict( + &self, + _request: tonic::Request<nuncio_proto::v1::ResolveConflictRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::ResolveConflictResponse>, tonic::Status> + { + Err(tonic::Status::internal("mailbox index is corrupt")) + } + async fn list_folders( &self, _request: tonic::Request<ListFoldersRequest>, @@ -3820,6 +4207,52 @@ mod tests { #[tonic::async_trait] impl MailService for StubMailTypedError { + /// The one typed-error path this stub exists to exercise on the + /// mutation surface: a genuine, server-evidenced conflict. + async fn move_message( + &self, + _request: tonic::Request<nuncio_proto::v1::MoveMessageRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::MoveMessageResponse>, tonic::Status> + { + Err(nuncio_proto::errors::status_with_metadata( + ErrorReason::Conflict, + "message was moved by another client", + [("message_id".to_string(), "msg-stub-1".to_string())], + )) + } + + async fn delete_message( + &self, + _request: tonic::Request<nuncio_proto::v1::DeleteMessageRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::DeleteMessageResponse>, tonic::Status> + { + Err(tonic::Status::internal("unused in this test")) + } + + async fn flag_message( + &self, + _request: tonic::Request<nuncio_proto::v1::FlagMessageRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::FlagMessageResponse>, tonic::Status> + { + Err(tonic::Status::internal("unused in this test")) + } + + async fn list_conflicts( + &self, + _request: tonic::Request<nuncio_proto::v1::ListConflictsRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::ListConflictsResponse>, tonic::Status> + { + Err(tonic::Status::internal("unused in this test")) + } + + async fn resolve_conflict( + &self, + _request: tonic::Request<nuncio_proto::v1::ResolveConflictRequest>, + ) -> Result<tonic::Response<nuncio_proto::v1::ResolveConflictResponse>, tonic::Status> + { + Err(tonic::Status::internal("unused in this test")) + } + async fn list_folders( &self, _request: tonic::Request<ListFoldersRequest>, diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index 0df4a26..3cf92fe 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -183,6 +183,38 @@ pub struct PlacementKey { pub remote_id: String, } +/// A mutation the server refused because another client had already changed the +/// message, recorded as durable state rather than as a passing notification. +/// +/// A conflict is raised only on positive evidence from the server; an +/// unverifiable outcome is never one. Because the intent was formed against +/// state that no longer exists, a human has to decide what happens next, and +/// that decision may come long after the event. The event stream is a lossy +/// broadcast -- it drops on lag and carries no sequence number to resume from -- +/// so it can only ever be an optimisation over a queryable table, never the +/// record itself. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MutationConflict { + /// Stable identifier of this conflict record. + pub id: String, + /// The outbox mutation that conflicted. + pub mutation_id: String, + /// Content-addressed key of the message the mutation targeted. + pub message_id: String, + /// The operation that was refused (`MOVE`, `DELETE`, `FLAG`, ...). + pub mutation_type: String, + /// The occupancy the mutation was aimed at, when the row recorded one. + pub placement: Option<PlacementKey>, + /// The server's positive evidence that another client changed the message. + pub observed: String, + /// Unix timestamp (seconds) when the conflict was detected. + pub detected_at: i64, + /// Unix timestamp (seconds) when a human resolved it, if they have. + pub resolved_at: Option<i64>, + /// Free-form note recording how the conflict was resolved. + pub resolution: Option<String>, +} + impl Placement { /// The addressing coordinates of this occupancy, dropping the read flag. pub fn key(&self) -> PlacementKey { diff --git a/crates/nuncio-filter/src/outbox.rs b/crates/nuncio-filter/src/outbox.rs index bb9b191..ada4a71 100644 --- a/crates/nuncio-filter/src/outbox.rs +++ b/crates/nuncio-filter/src/outbox.rs @@ -1,6 +1,7 @@ //! Transactional Outbox Pattern for Remote IMAP/JMAP Mutations. use crate::ast::PendingRemoteMutation; +use nuncio_core::model::PlacementKey; use serde::{Deserialize, Serialize}; /// Payload details stored for pending remote mutation. @@ -10,6 +11,17 @@ pub struct MutationPayload { pub action_type: String, /// Target parameter (e.g. folder name, target email address). pub target: Option<String>, + /// The mailbox occupancy this mutation was formed against. + /// + /// Message identity is content-addressed and folder-independent, so a single + /// message can occupy several mailboxes at once (Gmail labels make that + /// ordinary). `MOVE` and `DELETE` act on *one* occupancy, and acting on the + /// wrong one is destructive and irreversible, so the caller that formed the + /// intent records which occupancy it meant. `None` preserves the older + /// rows' meaning -- "unrecorded" -- which the executor may resolve only when + /// the message occupies exactly one mailbox. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub placement: Option<PlacementKey>, } /// Transactional outbox helper managing retry backoffs and mutation payloads. @@ -31,17 +43,33 @@ impl OutboxManager { base + jitter } - /// Construct a new `PendingRemoteMutation` record. + /// Construct a new `PendingRemoteMutation` record whose target occupancy is + /// left unrecorded. The executor may still address it when the message + /// occupies exactly one mailbox, and refuses when it occupies several. pub fn create_mutation( rule_id: impl Into<String>, message_id: impl Into<String>, action_type: impl Into<String>, target: Option<String>, + ) -> PendingRemoteMutation { + Self::create_mutation_at(rule_id, message_id, action_type, target, None) + } + + /// Construct a new `PendingRemoteMutation` record that names the mailbox + /// occupancy it was formed against, so a message occupying several mailboxes + /// is still unambiguously addressable. + pub fn create_mutation_at( + rule_id: impl Into<String>, + message_id: impl Into<String>, + action_type: impl Into<String>, + target: Option<String>, + placement: Option<PlacementKey>, ) -> PendingRemoteMutation { let action_str = action_type.into(); let payload_struct = MutationPayload { action_type: action_str.clone(), target, + placement, }; let payload_json = serde_json::to_string(&payload_struct).unwrap_or_default(); let now = chrono::Utc::now().timestamp(); diff --git a/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin b/crates/nuncio-proto/proto/nuncio/v1/descriptor.bin index d555779ca64587964e879d5272f5d2b66c622cc0..94f9059fe4a2e80ae2dcb72296afc4dc0d87de46 100644 GIT binary patch delta 1926 zcmb_dJ8#oa6y_#@CdWw{-6RlFQPNbX?Lbu`RG=)RFSSx1aH#|=L~eXjS#s?X-$W`P zyf82zf#iXW1tGy!<c)`b1u?P1pXg73>u2mxhN^7$Jbcb~zVq-m<I8ijhhp)MPmvXm zy$1BjdYcx}1RjIImaH(^)*Dn6Hx(+Ca0QJgN!C*@xY}2cO`z|VfV=1dz6h|Rh)udp zRi>BHmH4<&D@>{#wJB?p-3yb~eIs%G%!m2W^fLr0-(dd8RSY}?sg|ZlwB3-UQjBL! zk)*3wmT(1xjx2RWMV4f?TZ%j8unOlO&OJ&A$6`M*hD4i+Qt7x;f%C3Dg0!EN5!%9M z0K{ff>!{|>T=B+#$6&q8@5hqw5a3m<OU<vv^9G)QR6v(gr%*d}AX~Jxbd2<TpCE*5 zP?V_7WL0FcrZ#$fxrF3H#zlwAWob4I&<GM%>(h|8v8+;E7q=*(`i`dR6rTlQyTjbD zoJQo(2)=4rsv1z3rwV09uc8mpWhlCV4*@C5tLK<raAFq~aqH;BmSCX818wnvx2a4L zU+e?R*|QTK0Cmde${ydp>D!A#Zq*}rPE%WoY}P08!+o^7V;|6N&W}`$QOE?L+3pVY zMm+5+H@Gf3RQA9hVyMcH)HZe6?(*VKIDHRfd=&-+i8sB;%nx{?;6ABTXZRWjZK`WZ z*SuREGnNnmSG0}xsE&fG=-885O4)IjVxE(u<4#d>XS0xQ_(2*Xo6UgW%8q~a78F)w zow@8anNH}v4qpa50Xaifn@UHb!RJl~l4Q`+^D*a0se}t~T#>hB<}6-@xI&dcu`jZ> z6;~kDbehG}@sRdq8w%}e&``s3B0cx&9XRQw^%47kdDH!<p<(e={Mrpixbud>Va8?R zG@OctM$CGA@_rfpvVKo}K|}a76c-3tCk?Vtzq!8Fm|I_4Tv?vm$Yre0wfBkGy8-LL zrD=3-1%%Ksz-Pj>1xLc%pD7w4H=f$P83Zzs&BbR!-$yK7<>ku19lV$vQE0&eL|OYs z(la=adSdU4aY&I)Mj%Bj2SMs{@F3lW!6@4dkMKHX{Qp~@(GOe=)!OG`y}3L*@CP3L BW)A=W delta 73 zcmV-P0Ji_YkO9VjumJ)B!+Wy?0;LK8l(VV~VF9Bw4g>*`vo#WS0h8eq9<vb@?gFz; f9dQ}~zq7zMzyY(HIl2S_lLWI@J~jcfD?t?p!LJz^ diff --git a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto index 2abaec2..bba6752 100644 --- a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto +++ b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto @@ -45,6 +45,14 @@ enum ErrorReason { ERROR_REASON_UNSUPPORTED = 11; // An unexpected internal error the client cannot act on specifically. ERROR_REASON_INTERNAL = 12; + // Another client changed the target first and the server said so. + // + // Raised only on positive evidence; an outcome the server left unverifiable + // is never reported as a conflict. Retrying is not the remedy -- the intent + // was formed against state that no longer exists, so re-issuing it would + // either do nothing or overwrite a change the caller never saw. The conflict + // is also recorded durably and is retrievable via `Mail.ListConflicts`. + ERROR_REASON_CONFLICT = 13; } // ErrorInfo is the structured error payload the daemon attaches to a failed @@ -378,6 +386,11 @@ message AccountConfig { JmapTransport jmap = 16; DavTransport dav = 17; } + // Whether the filter engine runs against this account's incoming mail. + // + // Defaults to false: filters take irreversible remote actions, so an account + // does not get them by being added, only by being told to. + bool filters_enabled = 18; } // AddAccountRequest carries the full account configuration plus the raw @@ -478,6 +491,24 @@ service Mail { rpc GetMessage(GetMessageRequest) returns (GetMessageResponse); // MarkRead updates a message's read/unread flag. rpc MarkRead(MarkReadRequest) returns (MarkReadResponse); + // MoveMessage enqueues a verified remote move of one occupancy of a message + // into another folder. Returns as soon as the intent is durably queued; the + // outbox executes it against the server and only records success on positive + // proof. + rpc MoveMessage(MoveMessageRequest) returns (MoveMessageResponse); + // DeleteMessage enqueues a verified remote delete of one occupancy of a + // message. Other occupancies of the same message are untouched. + rpc DeleteMessage(DeleteMessageRequest) returns (DeleteMessageResponse); + // FlagMessage enqueues a verified remote flag/unflag of one occupancy of a + // message. + rpc FlagMessage(FlagMessageRequest) returns (FlagMessageResponse); + // ListConflicts returns mutations the server refused because another client + // had already changed the message. Conflicts are durable state, not a + // notification: the event stream is a lossy broadcast, so this table is the + // source of truth a client reconciles against. + rpc ListConflicts(ListConflictsRequest) returns (ListConflictsResponse); + // ResolveConflict marks a conflict handled, recording the human's decision. + rpc ResolveConflict(ResolveConflictRequest) returns (ResolveConflictResponse); // SearchMessages runs a full-text (FTS5) search over cached message // subjects, senders, and bodies, returning a keyset-paginated page of hits // ordered by relevance rank. @@ -542,8 +573,8 @@ message Message { // // A message can occupy several folders at once. This names the one the // message was reached through: the requested folder for a listing, and the - // lowest-sorting folder id for a lookup by id. A field carrying the complete - // placement set is not yet part of this contract. + // lowest-sorting folder id for a lookup by id. See `placements` for the + // complete set of mailboxes this message occupies. string folder_id = 3; // Subject line. string subject = 4; @@ -561,6 +592,48 @@ message Message { optional string body_html = 10; // List of attached files. repeated Attachment attachments = 11; + // Every mailbox this message occupies, folder id ascending. + // + // Message identity is content-addressed and folder-independent, so one + // message can sit in several mailboxes at once -- Gmail labels make that + // ordinary rather than exotic. Mutations that act on a single occupancy + // (`MoveMessage`, `DeleteMessage`) need one of these to address, and a client + // that only ever saw `folder_id` cannot tell a one-mailbox message from a + // ten-mailbox one. + repeated Placement placements = 12; +} + +// A single mailbox occupancy of a message: where one copy of it lives, and how +// to address that copy on the server that holds it. +message Placement { + // Account owning the mailbox. + string account_id = 1; + // Mailbox folder identifier. + string folder_id = 2; + // The UIDVALIDITY generation `uid` was captured under. Addressing is only + // valid within this generation; the server rejects a stale one rather than + // acting on whatever now bears that uid. + string uidvalidity = 3; + // Protocol-native id addressing the message in this mailbox. + string uid = 4; + // Read/unread flag of this occupancy specifically. + bool read = 5; +} + +// Names one occupancy for a mutation to act on. +// +// All four coordinates are required. This is deliberately not "the folder" -- +// UIDVALIDITY scopes the uid, and dropping it would let a mutation formed +// before a mailbox was recreated land on an unrelated message. +message PlacementRef { + // Account owning the mailbox. + string account_id = 1; + // Mailbox folder identifier. + string folder_id = 2; + // The UIDVALIDITY generation `uid` was captured under. + string uidvalidity = 3; + // Protocol-native id addressing the message in this mailbox. + string uid = 4; } // Mirrors `nuncio_core::model::Folder`. @@ -647,6 +720,106 @@ message MarkReadRequest { // response (as opposed to a `Status` error) is confirmation enough. message MarkReadResponse {} +// Request to move one occupancy of a message into another folder. +message MoveMessageRequest { + // Content-addressed key of the message to move. + string message_id = 1; + // Which occupancy to move. Optional only when the message occupies exactly + // one mailbox; with several, omitting it is refused rather than guessed, + // because moving the wrong copy is destructive and has no undo. + optional PlacementRef placement = 2; + // Destination folder id, within the same account as `placement`. + string destination_folder_id = 3; +} + +// Result of enqueuing a move. +message MoveMessageResponse { + // Identifier of the queued outbox mutation, for correlating with conflicts. + string mutation_id = 1; +} + +// Request to delete one occupancy of a message. +message DeleteMessageRequest { + // Content-addressed key of the message to delete. + string message_id = 1; + // Which occupancy to delete. Optional only when the message occupies exactly + // one mailbox. Deleting removes that copy; other occupancies survive. + optional PlacementRef placement = 2; +} + +// Result of enqueuing a delete. +message DeleteMessageResponse { + // Identifier of the queued outbox mutation. + string mutation_id = 1; +} + +// Request to flag or unflag one occupancy of a message. +message FlagMessageRequest { + // Content-addressed key of the message to flag. + string message_id = 1; + // Which occupancy to flag. Optional only when the message occupies exactly + // one mailbox. + optional PlacementRef placement = 2; + // Desired end state: true flags, false unflags. + bool flagged = 3; +} + +// Result of enqueuing a flag change. +message FlagMessageResponse { + // Identifier of the queued outbox mutation. + string mutation_id = 1; +} + +// A mutation the server refused because another client changed the message +// first, recorded durably so the decision can be made later. +message MutationConflict { + // Stable identifier of this conflict record. + string id = 1; + // The outbox mutation that conflicted. + string mutation_id = 2; + // Content-addressed key of the message the mutation targeted. + string message_id = 3; + // The operation that was refused (e.g. "MOVE", "DELETE", "FLAG"). + string mutation_type = 4; + // The occupancy the mutation was aimed at, when the queued intent named one. + optional PlacementRef placement = 5; + // The server's positive evidence that another client changed the message. + // A conflict is never inferred from an unverifiable outcome. + string observed = 6; + // When the conflict was detected. + google.protobuf.Timestamp detected_at = 7; + // When a human resolved it, if they have. + optional google.protobuf.Timestamp resolved_at = 8; + // Free-form note recording how it was resolved. + optional string resolution = 9; +} + +// Request to list recorded mutation conflicts. +message ListConflictsRequest { + // Include conflicts a human has already resolved. Resolved rows are kept as + // the audit trail of what was decided, so they are opt-in. + bool include_resolved = 1; + // Maximum number of conflicts to return; clamped to the server's page limit. + uint32 limit = 2; +} + +// A page of mutation conflicts, newest first. +message ListConflictsResponse { + // The conflicts. + repeated MutationConflict conflicts = 1; +} + +// Request to mark a conflict handled. +message ResolveConflictRequest { + // Identifier of the conflict to resolve. + string conflict_id = 1; + // Note recording how it was resolved. + string resolution = 2; +} + +// Result of resolving a conflict. +message ResolveConflictResponse {} + // SearchMessagesRequest. Pages hits using the shared keyset convention (see // `ListMessagesRequest`); the cursor here encodes `(rank, id)` so paging // preserves the relevance ordering while resuming strictly after the last diff --git a/crates/nuncio-proto/src/errors.rs b/crates/nuncio-proto/src/errors.rs index c13f1c6..7438b0a 100644 --- a/crates/nuncio-proto/src/errors.rs +++ b/crates/nuncio-proto/src/errors.rs @@ -37,6 +37,11 @@ impl ErrorReason { ErrorReason::BackendUnreachable => Code::Unavailable, ErrorReason::AuthRequired => Code::Unauthenticated, ErrorReason::Unsupported => Code::Unimplemented, + // `Aborted` is the canonical code for a concurrency conflict, and + // unlike `FailedPrecondition` it tells a client the call may be + // retried only after re-reading the current state -- which is + // exactly the remedy here. + ErrorReason::Conflict => Code::Aborted, } } } diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 24ced58..946c8f5 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -11,6 +11,64 @@ use zeroize::Zeroizing; /// Raw row shape for a pending remote mutation record fetched from SQLite. type PendingMutationRow = (String, String, String, String, String, String, i64, i64); +/// `mutation_conflicts` row: id, mutation_id, message_id, mutation_type, then the +/// four nullable placement coordinates, observed evidence, and resolution state. +type ConflictRow = ( + String, + String, + String, + String, + Option<String>, + Option<String>, + Option<String>, + Option<String>, + String, + i64, + Option<i64>, + Option<String>, +); + +/// Rebuild a conflict from its row, reassembling the placement only when all +/// four coordinates are present -- a partial placement addresses nothing. +fn conflict_from_row(row: ConflictRow) -> nuncio_core::model::MutationConflict { + let ( + id, + mutation_id, + message_id, + mutation_type, + account_id, + folder_id, + uid_validity, + remote_id, + observed, + detected_at, + resolved_at, + resolution, + ) = row; + let placement = match (account_id, folder_id, uid_validity, remote_id) { + (Some(account_id), Some(folder_id), Some(uid_validity), Some(remote_id)) => { + Some(nuncio_core::model::PlacementKey { + account_id, + folder_id, + uid_validity, + remote_id, + }) + } + _ => None, + }; + nuncio_core::model::MutationConflict { + id, + mutation_id, + message_id, + mutation_type, + placement, + observed, + detected_at, + resolved_at, + resolution, + } +} + /// Database errors emitted by `nuncio-store`. #[derive(Error, Debug)] pub enum DatabaseError { @@ -919,6 +977,21 @@ impl DatabaseEngine { created_at INTEGER NOT NULL ); + CREATE TABLE IF NOT EXISTS mutation_conflicts ( + id TEXT PRIMARY KEY NOT NULL, + mutation_id TEXT NOT NULL, + message_id TEXT NOT NULL, + mutation_type TEXT NOT NULL, + account_id TEXT, + folder_id TEXT, + uidvalidity TEXT, + uid TEXT, + observed TEXT NOT NULL, + detected_at INTEGER NOT NULL, + resolved_at INTEGER, + resolution TEXT + ); + CREATE TABLE IF NOT EXISTS filter_execution_logs ( id INTEGER PRIMARY KEY AUTOINCREMENT, rule_id TEXT NOT NULL, @@ -2670,6 +2743,103 @@ impl DatabaseEngine { Ok(()) } + /// Record a mutation conflict as durable, queryable state. + /// + /// `INSERT OR REPLACE` keyed on the conflict id makes a re-record of the + /// same conflict idempotent, so a drain pass that is retried after a crash + /// cannot multiply one refusal into several rows a human has to dismiss. + pub async fn record_conflict( + &self, + conflict: &nuncio_core::model::MutationConflict, + ) -> Result<(), DatabaseError> { + let placement = conflict.placement.as_ref(); + sqlx::query( + r#" + INSERT OR REPLACE INTO mutation_conflicts + (id, mutation_id, message_id, mutation_type, account_id, folder_id, + uidvalidity, uid, observed, detected_at, resolved_at, resolution) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + "#, + ) + .bind(&conflict.id) + .bind(&conflict.mutation_id) + .bind(&conflict.message_id) + .bind(&conflict.mutation_type) + .bind(placement.map(|p| p.account_id.clone())) + .bind(placement.map(|p| p.folder_id.clone())) + .bind(placement.map(|p| p.uid_validity.clone())) + .bind(placement.map(|p| p.remote_id.clone())) + .bind(&conflict.observed) + .bind(conflict.detected_at) + .bind(conflict.resolved_at) + .bind(conflict.resolution.as_deref()) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + Ok(()) + } + + /// List recorded mutation conflicts, newest first. + /// + /// Resolved conflicts stay in the table -- they are the audit trail of what + /// a human decided -- so they are only returned when explicitly asked for. + pub async fn list_conflicts( + &self, + include_resolved: bool, + limit: usize, + ) -> Result<Vec<nuncio_core::model::MutationConflict>, DatabaseError> { + let sql = if include_resolved { + r#" + SELECT id, mutation_id, message_id, mutation_type, account_id, folder_id, + uidvalidity, uid, observed, detected_at, resolved_at, resolution + FROM mutation_conflicts + ORDER BY detected_at DESC, id ASC + LIMIT ? + "# + } else { + r#" + SELECT id, mutation_id, message_id, mutation_type, account_id, folder_id, + uidvalidity, uid, observed, detected_at, resolved_at, resolution + FROM mutation_conflicts + WHERE resolved_at IS NULL + ORDER BY detected_at DESC, id ASC + LIMIT ? + "# + }; + let rows: Vec<ConflictRow> = sqlx::query_as(sql) + .bind(limit as i64) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows.into_iter().map(conflict_from_row).collect()) + } + + /// Mark a conflict resolved. Returns `false` when no such unresolved + /// conflict exists, so a caller can tell "already handled" from "applied" + /// instead of reporting a success that never happened. + pub async fn resolve_conflict( + &self, + id: &str, + resolution: &str, + resolved_at: i64, + ) -> Result<bool, DatabaseError> { + let result = sqlx::query( + r#" + UPDATE mutation_conflicts + SET resolved_at = ?, resolution = ? + WHERE id = ? AND resolved_at IS NULL + "#, + ) + .bind(resolved_at) + .bind(resolution) + .bind(id) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + Ok(result.rows_affected() > 0) + } + /// Save a [`nuncio_filter::PendingRemoteMutation`] outbox record. pub async fn save_pending_mutation( &self, diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index ce0986a..317df20 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -34,27 +34,30 @@ use nuncio_proto::v1::{ CalendarSyncRequest, CalendarSyncResponse, Contact as ContactProto, ContactEmail as ContactEmailProto, ContactPhone as ContactPhoneProto, ContactsSyncRequest, ContactsSyncResponse, CreateContactRequest, CreateContactResponse, CreateRuleRequest, - CreateRuleResponse, DatabaseRecovered, DavTransport as DavTransportProto, DeleteRuleRequest, - DeleteRuleResponse, Event, EventError, ExportFormat as ExportFormatProto, ExportRequest, - ExportResponse, ExportRulesRequest, ExportRulesResponse, FilterExecuted, - FilterExecutionLog as FilterExecutionLogProto, FilterRule as FilterRuleProto, - Folder as FolderProto, GetContactRequest, GetContactResponse, GetEventRequest, - GetEventResponse, GetExecutionLogsRequest, GetExecutionLogsResponse, GetMessageRequest, - GetMessageResponse, GetStatusRequest, GetStatusResponse, - ImapSmtpTransport as ImapSmtpTransportProto, ImportRulesRequest, ImportRulesResponse, - JmapTransport as JmapTransportProto, ListAccountsRequest, ListAccountsResponse, + CreateRuleResponse, DatabaseRecovered, DavTransport as DavTransportProto, DeleteMessageRequest, + DeleteMessageResponse, DeleteRuleRequest, DeleteRuleResponse, Event, EventError, + ExportFormat as ExportFormatProto, ExportRequest, ExportResponse, ExportRulesRequest, + ExportRulesResponse, FilterExecuted, FilterExecutionLog as FilterExecutionLogProto, + FilterRule as FilterRuleProto, FlagMessageRequest, FlagMessageResponse, Folder as FolderProto, + GetContactRequest, GetContactResponse, GetEventRequest, GetEventResponse, + GetExecutionLogsRequest, GetExecutionLogsResponse, GetMessageRequest, GetMessageResponse, + GetStatusRequest, GetStatusResponse, ImapSmtpTransport as ImapSmtpTransportProto, + ImportRulesRequest, ImportRulesResponse, JmapTransport as JmapTransportProto, + ListAccountsRequest, ListAccountsResponse, ListConflictsRequest, ListConflictsResponse, ListContactsRequest, ListContactsResponse, ListEventsRequest, ListEventsResponse, ListFoldersRequest, ListFoldersResponse, ListMessagesRequest, ListMessagesResponse, ListRecordsRequest, ListRecordsResponse, ListRulesRequest, ListRulesResponse, MarkReadRequest, MarkReadResponse, Message as MessageProto, MessageFlagsChanged, MessageSearchHit, - PreviewRuleRequest, PreviewRuleResponse, RemoveAccountRequest, RemoveAccountResponse, - RuleExportFormat as RuleExportFormatProto, SearchMessagesRequest, SearchMessagesResponse, - SendMessageRequest, SendMessageResponse, ShuttingDown, SubscribeRequest, SyncCompleted, - SyncRequest, SyncResponse, SyncStarted, SyncState as SyncStateProto, - TestAccountConnectionRequest, TestAccountConnectionResponse, TlsMode as TlsModeProto, - TriageProgress, TriageRequest, UpdateAccountRequest, UpdateAccountResponse, UpdateAvailable, - UpdateRuleRequest, UpdateRuleResponse, ValidateRuleRequest, ValidateRuleResponse, - VerifyChainRequest, VerifyChainResponse, + MoveMessageRequest, MoveMessageResponse, MutationConflict as MutationConflictProto, + Placement as PlacementProto, PlacementRef as PlacementRefProto, PreviewRuleRequest, + PreviewRuleResponse, RemoveAccountRequest, RemoveAccountResponse, ResolveConflictRequest, + ResolveConflictResponse, RuleExportFormat as RuleExportFormatProto, SearchMessagesRequest, + SearchMessagesResponse, SendMessageRequest, SendMessageResponse, ShuttingDown, + SubscribeRequest, SyncCompleted, SyncRequest, SyncResponse, SyncStarted, + SyncState as SyncStateProto, TestAccountConnectionRequest, TestAccountConnectionResponse, + TlsMode as TlsModeProto, TriageProgress, TriageRequest, UpdateAccountRequest, + UpdateAccountResponse, UpdateAvailable, UpdateRuleRequest, UpdateRuleResponse, + ValidateRuleRequest, ValidateRuleResponse, VerifyChainRequest, VerifyChainResponse, }; use nuncio_store::db::DatabaseEngine; use nuncio_store::search::SearchEngine; @@ -429,6 +432,7 @@ fn map_account_config_to_proto(config: nuncio_core::AccountConfig) -> AccountCon config.sync_interval_secs, )), transport: Some(transport), + filters_enabled: config.filters_enabled, } } @@ -490,7 +494,7 @@ fn map_account_config_from_proto( email_address: config.email_address, keyring_secret_key: config.keyring_secret_key, sync_interval_secs: nuncio_proto::time::duration_to_secs(&sync_interval), - filters_enabled: false, + filters_enabled: config.filters_enabled, transport, }) } @@ -993,6 +997,54 @@ fn map_email_to_proto(placed: nuncio_store::db::MessageWithPlacement) -> Message .into_iter() .map(map_attachment_to_proto) .collect(), + // Filled in by `fill_placements`, which needs the store. + placements: Vec::new(), + } +} + +/// Maps one stored occupancy onto its wire-format `nuncio.v1.Placement`. +fn map_placement_to_proto(placement: nuncio_core::model::Placement) -> PlacementProto { + PlacementProto { + account_id: placement.account_id, + folder_id: placement.folder_id, + uidvalidity: placement.uid_validity, + uid: placement.remote_id, + read: placement.read, + } +} + +/// Maps a wire-format `nuncio.v1.PlacementRef` onto the store's key type. +fn map_placement_ref_from_proto(placement: PlacementRefProto) -> nuncio_core::model::PlacementKey { + nuncio_core::model::PlacementKey { + account_id: placement.account_id, + folder_id: placement.folder_id, + uid_validity: placement.uidvalidity, + remote_id: placement.uid, + } +} + +/// Fills each message's complete placement set from the store. +/// +/// This costs one query per message rather than one per page. That is the +/// honest price of the field: occupancy is per-message state and a page can mix +/// messages that sit in one mailbox with messages that sit in ten. A failure to +/// read one message's placements leaves that message's set empty rather than +/// failing the whole listing, and is logged -- a partial page is more useful +/// than none, and the field is additive so a client that ignores it is +/// unaffected. +async fn fill_placements(db: &DatabaseEngine, messages: &mut [MessageProto]) { + for message in messages.iter_mut() { + match db.placements_of(&message.id).await { + Ok(placements) => { + message.placements = placements.into_iter().map(map_placement_to_proto).collect(); + } + Err(e) => { + tracing::warn!( + message_id = %message.id, + "Mail: failed to read the placement set: {e}" + ); + } + } } } @@ -1786,6 +1838,145 @@ async fn primary_placement( }) } +/// Rule id recorded on outbox rows a human queued directly rather than a filter. +/// +/// The outbox row's `rule_id` explains *why* a mutation exists; for a +/// hand-driven one the honest answer is "a person asked", not a rule that does +/// not exist. +const MANUAL_ORIGIN: &str = "manual"; + +/// Seconds since the Unix epoch, saturating at 0 before it. +fn unix_now_secs() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or_default() +} + +/// Resolve which occupancy a mutation should act on. +/// +/// A named placement is honoured only after confirming the message still +/// occupies it, so a client working from a stale listing cannot aim a `MOVE` or +/// `DELETE` at a mailbox the message has left. An unnamed placement is inferred +/// only when the message occupies exactly one mailbox; with several, this is a +/// typed validation failure that lists the candidates, because picking one +/// would act on mail the caller never selected and there is no undo. +async fn resolve_target_placement( + db: &DatabaseEngine, + message_id: &str, + named: Option<PlacementRefProto>, +) -> Result<nuncio_core::model::PlacementKey, Status> { + if message_id.is_empty() { + return Err(Status::invalid_argument("message_id is required")); + } + let placements = db.placements_of(message_id).await.map_err(|e| { + Status::internal(format!("failed to read placements for '{message_id}': {e}")) + })?; + + if let Some(named) = named { + let key = map_placement_ref_from_proto(named); + return if placements.iter().any(|p| p.key() == key) { + Ok(key) + } else { + Err(errors::status_with_metadata( + ErrorReason::MessageNotFound, + format!( + "message '{message_id}' does not occupy {}/{} (uidvalidity {}, uid {})", + key.account_id, key.folder_id, key.uid_validity, key.remote_id + ), + [("message_id".to_string(), message_id.to_string())], + )) + }; + } + + match placements.as_slice() { + [only] => Ok(only.key()), + [] => Err(errors::status_with_metadata( + ErrorReason::MessageNotFound, + format!("message '{message_id}' occupies no mailbox"), + [("message_id".to_string(), message_id.to_string())], + )), + several => { + let folders: Vec<&str> = several.iter().map(|p| p.folder_id.as_str()).collect(); + Err(errors::status_with_metadata( + ErrorReason::ValidationFailed, + format!( + "message '{message_id}' occupies {} mailboxes ({}); name one in `placement` \ + rather than leaving the target to be guessed", + several.len(), + folders.join(", ") + ), + [ + ("message_id".to_string(), message_id.to_string()), + ("placement_count".to_string(), several.len().to_string()), + ], + )) + } + } +} + +/// Maps a stored conflict onto its wire-format `nuncio.v1.MutationConflict`. +fn map_conflict_to_proto(conflict: nuncio_core::model::MutationConflict) -> MutationConflictProto { + MutationConflictProto { + id: conflict.id, + mutation_id: conflict.mutation_id, + message_id: conflict.message_id, + mutation_type: conflict.mutation_type, + placement: conflict.placement.map(|p| PlacementRefProto { + account_id: p.account_id, + folder_id: p.folder_id, + uidvalidity: p.uid_validity, + uid: p.remote_id, + }), + observed: conflict.observed, + detected_at: Some(nuncio_proto::time::timestamp_from_unix_secs( + conflict.detected_at, + )), + resolved_at: conflict + .resolved_at + .map(nuncio_proto::time::timestamp_from_unix_secs), + resolution: conflict.resolution, + } +} + +impl MailGrpcService { + /// Durably queue a placement-addressed remote mutation on the outbox. + /// + /// Every mutation RPC funnels through here rather than touching a backend + /// directly: the outbox is what makes an intent survive a crash, and what + /// turns the server's answer into one of three honest outcomes instead of a + /// hopeful assumption. The RPC returns once the intent is durable, which is + /// the only thing it can truthfully promise. + async fn enqueue_mutation( + &self, + message_id: &str, + op: &str, + target: Option<String>, + placement: nuncio_core::model::PlacementKey, + ) -> Result<String, Status> { + let mutation = nuncio_filter::OutboxManager::create_mutation_at( + MANUAL_ORIGIN, + message_id, + op, + target, + Some(placement.clone()), + ); + self.db + .save_pending_mutation(&mutation) + .await + .map_err(|e| Status::internal(format!("failed to queue {op} mutation: {e}")))?; + tracing::info!( + mutation_id = %mutation.id, + op = %op, + message_id = %message_id, + account_id = %placement.account_id, + folder_id = %placement.folder_id, + "Mail: queued placement-addressed mutation" + ); + Ok(mutation.id) + } +} + #[tonic::async_trait] impl Mail for MailGrpcService { /// Lists folders across every **configured** account, merged by folder id. @@ -1870,7 +2061,9 @@ impl Mail for MailGrpcService { ]) }) .unwrap_or_default(); - let messages = messages.into_iter().map(map_email_to_proto).collect(); + let mut messages: Vec<MessageProto> = + messages.into_iter().map(map_email_to_proto).collect(); + fill_placements(&self.db, &mut messages).await; Ok(Response::new(ListMessagesResponse { messages, @@ -1897,8 +2090,23 @@ impl Mail for MailGrpcService { let placement = primary_placement(&self.db, &req.message_id).await?; + let mut message = map_email_to_proto((email, placement)); + message.placements = self + .db + .placements_of(&req.message_id) + .await + .map_err(|e| { + Status::internal(format!( + "failed to read placements for '{}': {e}", + req.message_id + )) + })? + .into_iter() + .map(map_placement_to_proto) + .collect(); + Ok(Response::new(GetMessageResponse { - message: Some(map_email_to_proto((email, placement))), + message: Some(message), })) } @@ -1949,6 +2157,97 @@ impl Mail for MailGrpcService { Ok(Response::new(MarkReadResponse {})) } + async fn move_message( + &self, + request: Request<MoveMessageRequest>, + ) -> Result<Response<MoveMessageResponse>, Status> { + let req = request.into_inner(); + if req.destination_folder_id.is_empty() { + return Err(Status::invalid_argument( + "destination_folder_id is required", + )); + } + let placement = resolve_target_placement(&self.db, &req.message_id, req.placement).await?; + let mutation_id = self + .enqueue_mutation( + &req.message_id, + "MOVE", + Some(req.destination_folder_id.clone()), + placement, + ) + .await?; + Ok(Response::new(MoveMessageResponse { mutation_id })) + } + + async fn delete_message( + &self, + request: Request<DeleteMessageRequest>, + ) -> Result<Response<DeleteMessageResponse>, Status> { + let req = request.into_inner(); + let placement = resolve_target_placement(&self.db, &req.message_id, req.placement).await?; + let mutation_id = self + .enqueue_mutation(&req.message_id, "DELETE", None, placement) + .await?; + Ok(Response::new(DeleteMessageResponse { mutation_id })) + } + + async fn flag_message( + &self, + request: Request<FlagMessageRequest>, + ) -> Result<Response<FlagMessageResponse>, Status> { + let req = request.into_inner(); + let placement = resolve_target_placement(&self.db, &req.message_id, req.placement).await?; + let op = if req.flagged { "FLAG" } else { "UNFLAG" }; + let mutation_id = self + .enqueue_mutation(&req.message_id, op, None, placement) + .await?; + Ok(Response::new(FlagMessageResponse { mutation_id })) + } + + async fn list_conflicts( + &self, + request: Request<ListConflictsRequest>, + ) -> Result<Response<ListConflictsResponse>, Status> { + let req = request.into_inner(); + let limit = pagination::clamp_page_size(req.limit); + let conflicts = self + .db + .list_conflicts(req.include_resolved, limit) + .await + .map_err(|e| Status::internal(format!("failed to list conflicts: {e}")))?; + Ok(Response::new(ListConflictsResponse { + conflicts: conflicts.into_iter().map(map_conflict_to_proto).collect(), + })) + } + + async fn resolve_conflict( + &self, + request: Request<ResolveConflictRequest>, + ) -> Result<Response<ResolveConflictResponse>, Status> { + let req = request.into_inner(); + if req.conflict_id.is_empty() { + return Err(Status::invalid_argument("conflict_id is required")); + } + let resolved_at = unix_now_secs(); + let applied = self + .db + .resolve_conflict(&req.conflict_id, &req.resolution, resolved_at) + .await + .map_err(|e| Status::internal(format!("failed to resolve conflict: {e}")))?; + if !applied { + // Either it never existed or someone already resolved it. Reporting + // success would tell the caller their decision was recorded when it + // was not. + return Err(errors::status_with_metadata( + ErrorReason::MessageNotFound, + format!("no unresolved conflict '{}' to resolve", req.conflict_id), + [("conflict_id".to_string(), req.conflict_id.clone())], + )); + } + tracing::info!(conflict_id = %req.conflict_id, "Mail: conflict resolved"); + Ok(Response::new(ResolveConflictResponse {})) + } + async fn search_messages( &self, request: Request<SearchMessagesRequest>, @@ -4298,6 +4597,7 @@ mod tests { email_address: format!("{id}@nuncio.mx"), keyring_secret_key: keyring_secret_key.to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), + filters_enabled: false, transport: Some(TransportProto::ImapSmtp(ImapSmtpTransportProto { imap_host: "imap.nuncio.mx".to_string(), imap_port: 993, @@ -4317,6 +4617,7 @@ mod tests { email_address: "no-transport@nuncio.mx".to_string(), keyring_secret_key: "nuncio/acct-no-transport".to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), + filters_enabled: false, transport: None, }; let err = map_account_config_from_proto(config) diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 05ed581..826de22 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -222,6 +222,7 @@ pub async fn execute_pending_mutations( message_id = %item.message_id, "outbox: mutation conflicted with another client: {observed}" ); + record_conflict(db, &item, &observed).await; ("conflicted", item.retry_count, |s| s.failed += 1) } Disposition::Permanent(reason) => { @@ -260,6 +261,35 @@ pub async fn execute_pending_mutations( /// Execute a single mutation and report its honest disposition. Performs no /// status writes itself -- the caller persists the outcome. +/// Persist a conflict so it outlives the log line and the event stream. +/// +/// The conflict id is derived from the mutation id, which makes re-recording +/// the same refusal idempotent. A failure to persist is logged and swallowed: +/// the mutation's own status write is what stops it being re-attempted, and +/// losing the audit row must not take the drain pass down with it. +async fn record_conflict(db: &DatabaseEngine, item: &PendingRemoteMutation, observed: &str) { + let placement = serde_json::from_str::<MutationPayload>(&item.payload) + .ok() + .and_then(|p| p.placement); + let conflict = nuncio_core::model::MutationConflict { + id: format!("conflict-{}", item.id), + mutation_id: item.id.clone(), + message_id: item.message_id.clone(), + mutation_type: item.mutation_type.clone(), + placement, + observed: observed.to_string(), + detected_at: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or_default(), + resolved_at: None, + resolution: None, + }; + if let Err(e) = db.record_conflict(&conflict).await { + tracing::warn!(mutation_id = %item.id, "outbox: failed to record conflict: {e}"); + } +} + async fn execute_one( db: &DatabaseEngine, env: &dyn RemoteExecutionEnv, @@ -304,9 +334,12 @@ async fn execute_one( /// /// The addressing comes from a placement rather than the message row, because /// those coordinates are properties of a mailbox occupancy and a message can -/// hold several. The outbox row records only the message key, so which occupancy -/// the rule matched is not recoverable here -- and when the message occupies -/// more than one mailbox, this **refuses** rather than picking one. +/// hold several. When the row records the occupancy its intent was formed +/// against, that occupancy is used verbatim -- after confirming the message +/// still occupies it, so a mutation cannot be aimed at a mailbox the message has +/// since left. Otherwise the occupancy is inferred, and inference is only +/// allowed when it is unambiguous: with more than one placement this **refuses** +/// rather than picking one. /// /// Guessing is not the conservative option. A rule that matched the `INBOX` /// copy would `MOVE` or `DELETE` whichever copy happened to sort first, against @@ -317,9 +350,7 @@ async fn execute_one( /// guessing is destructive and irreversible. /// /// The refusal is permanent, not retryable: nothing about a later attempt makes -/// the outbox row remember a placement it never recorded. Carrying the -/// originating placement on the row is the real fix, and belongs with the -/// placement-aware wire surface. +/// a row remember a placement it never recorded. async fn dispatch_mailbox_mutation( db: &DatabaseEngine, env: &dyn RemoteExecutionEnv, @@ -359,6 +390,27 @@ async fn dispatch_mailbox_mutation( return Disposition::Retry(format!("failed to resolve the message's placements: {e}")) } }; + if let Some(named) = &payload.placement { + return match placements.iter().find(|p| &p.key() == named) { + Some(placement) => { + let placement = placement.clone(); + apply_mailbox_mutation(env, mutation, email, &placement, kind).await + } + // The named occupancy is gone. Re-addressing to a different one + // would act on mail the caller never selected, so this is terminal. + None => Disposition::Permanent(format!( + "message '{}' no longer occupies {}/{} (uidvalidity {}, uid {}); \ + refusing to re-aim a {} at a different mailbox", + email.id, + named.account_id, + named.folder_id, + named.uid_validity, + named.remote_id, + mutation.mutation_type + )), + }; + } + let placement = match placements.as_slice() { [only] => only, [] => { @@ -390,6 +442,18 @@ async fn dispatch_mailbox_mutation( } }; + apply_mailbox_mutation(env, mutation, email, placement, kind).await +} + +/// Issue an already-addressed mailbox mutation against the account's backend and +/// translate the backend's three honest outcomes into a disposition. +async fn apply_mailbox_mutation( + env: &dyn RemoteExecutionEnv, + mutation: &PendingRemoteMutation, + email: &nuncio_core::model::Email, + placement: &nuncio_core::model::Placement, + kind: RemoteMutationKind, +) -> Disposition { let backend = match env.mail_backend(&placement.account_id).await { Ok(backend) => backend, Err(e) => return Disposition::Retry(format!("failed to build mail backend: {e}")), @@ -723,6 +787,7 @@ mod tests { payload: serde_json::to_string(&MutationPayload { action_type: "FLAG".to_string(), target: None, + placement: None, }) .expect("serialize payload"), status: "pending".to_string(), diff --git a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs index 020bae1..268e905 100644 --- a/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs +++ b/crates/nunciod/tests/accounts_lifecycle_e2e_test.rs @@ -90,6 +90,7 @@ fn sample_account_config() -> AccountConfig { email_address: ACCOUNT_EMAIL.to_string(), keyring_secret_key: KEYRING_KEY.to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), + filters_enabled: false, transport: Some(nuncio_proto::v1::account_config::Transport::ImapSmtp( nuncio_proto::v1::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), diff --git a/crates/nunciod/tests/caldav_production_e2e_test.rs b/crates/nunciod/tests/caldav_production_e2e_test.rs index 8576be5..7272b18 100644 --- a/crates/nunciod/tests/caldav_production_e2e_test.rs +++ b/crates/nunciod/tests/caldav_production_e2e_test.rs @@ -159,6 +159,7 @@ async fn caldav_production_sync_builds_real_client_and_persists_events() { email_address: ACCOUNT_EMAIL.to_string(), keyring_secret_key: KEYRING_KEY.to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(300)), + filters_enabled: false, transport: Some(nuncio_proto::v1::account_config::Transport::Dav( nuncio_proto::v1::DavTransport { collection_url: collection_url.clone(), diff --git a/crates/nunciod/tests/no_secrets_canary_test.rs b/crates/nunciod/tests/no_secrets_canary_test.rs index 413a78c..92bf3b2 100644 --- a/crates/nunciod/tests/no_secrets_canary_test.rs +++ b/crates/nunciod/tests/no_secrets_canary_test.rs @@ -122,6 +122,7 @@ fn sample_proto_config() -> AccountConfig { email_address: ACCOUNT_EMAIL.to_string(), keyring_secret_key: KEYRING_KEY.to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), + filters_enabled: false, transport: Some(nuncio_proto::v1::account_config::Transport::ImapSmtp( nuncio_proto::v1::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index 81f798e..398c213 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -60,6 +60,7 @@ fn sample_account_config() -> AccountConfig { email_address: ACCOUNT_EMAIL.to_string(), keyring_secret_key: KEYRING_KEY.to_string(), sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), + filters_enabled: false, transport: Some(nuncio_proto::v1::account_config::Transport::ImapSmtp( nuncio_proto::v1::ImapSmtpTransport { imap_host: "imap.nuncio.mx".to_string(), -- 2.54.0 From 93f00cdd0bf6e84e935805ea195dcae6532b828f Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 18:41:10 -0500 Subject: [PATCH 90/95] feat(imap): key messages by their server object id IMAP hardcoded no object id into the message key, so it could only ever reach the Message-ID+content tier or the folder-scoped surrogate, while JMAP entered at the top tier from the server's Email id. One account synced through both protocols therefore minted disjoint key spaces. RFC 8474 EMAILID is the same identifier JMAP exposes, so requesting it makes the two keys equal. Negotiate per capability: EMAILID only where the server advertises OBJECTID, X-GM-MSGID only where it advertises X-GM-EXT-1. A server offering neither behaves exactly as before. EMAILID cannot go through the typed client: imap-proto's msg_att parser is a closed alt with no catch-all, and async-imap turns the resulting nom error into a fatal io::Error that closes the session. The raw layer reads it off the transport beneath ImapStream instead, so an attribute the crate cannot parse can never reach its decoder. Lifting stored rows to a stronger tier changes every message_key, so IDENTITY_SCHEMA_VERSION moves to 3 and the existing reconciliation rebuilds the cached store rather than repointing rows one at a time. --- crates/nuncio-core/src/model.rs | 31 +- crates/nuncio-mail/src/imap.rs | 200 ++++++++--- crates/nuncio-mail/src/imap_raw.rs | 327 +++++++++++++++++- crates/nuncio-mail/src/test_server.rs | 246 +++++++++++-- .../nuncio-mail/tests/imap_identity_test.rs | 231 +++++++++++++ .../nuncio-mail/tests/imap_raw_wire_test.rs | 85 ++++- crates/nuncio-store/src/db.rs | 12 +- docs/adr/0002-multi-engine-sync-model.md | 24 +- 8 files changed, 1055 insertions(+), 101 deletions(-) create mode 100644 crates/nuncio-mail/tests/imap_identity_test.rs diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index 0df4a26..1b76353 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -350,17 +350,30 @@ impl Email { /// The surrogate tier is folder-scoped and therefore *not* stable across a /// move; it is the honest answer when the server offered nothing better. /// - /// # Keys do not yet converge across engine types + /// # What convergence across engine types does and does not follow from /// /// Precedence makes a key reproducible from the same inputs, not from the - /// same *message*. The IMAP engine does not currently request `EMAILID` or - /// `X-GM-MSGID`, so it can only ever reach the `Message-ID`+content tier or - /// the surrogate, while JMAP supplies a server id and enters at the top - /// tier. Two engines syncing one account therefore produce disjoint key - /// spaces: the same message is two identities, one per engine. This is a - /// known gap in the convergent multi-engine model, not a property to rely - /// on -- do not assume a key derived by one engine addresses the row - /// another engine wrote. + /// same *message*. Two engines converge exactly when they enter at the same + /// tier with the same value. + /// + /// They do when the server publishes a stable object id and both engines + /// ask for it. The IMAP engine requests RFC 8474 `EMAILID` from an + /// `OBJECTID` server and `X-GM-MSGID` from an `X-GM-EXT-1` one, and JMAP's + /// Email object id is the same identifier RFC 8474 exposes to IMAP -- so + /// one message synced through both protocols yields one key. That is the + /// whole point of the OBJECTID extension. + /// + /// They do not when the server offers neither. IMAP then falls to the + /// `Message-ID`+content tier or the surrogate, both derived from what the + /// message or the mailbox happened to contain, while a JMAP engine against + /// the same account still enters at the top tier from a server-assigned id. + /// Those key spaces are disjoint, and no comparison across them means + /// anything. The surrogate additionally is not stable across a *move*, even + /// within one engine. + /// + /// Object ids are also only comparable within one account on one server: + /// RFC 8474 section 4 scopes uniqueness that way, which is why every tier + /// here is account-scoped. pub fn derive_message_key( account_id: &str, remote: RemoteIdentity<'_>, diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index 57c5922..d7c9130 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -175,6 +175,13 @@ pub(crate) struct MailboxSyncCapabilities { /// `NOMODSEQ` has no mod-sequences to compare against no matter what the /// server advertises. pub mailbox_has_modseq: bool, + /// Server advertises `OBJECTID` (RFC 8474), so `EMAILID` may be asked for. + /// Asking a server that did not advertise it is not merely rude: an + /// unrecognised FETCH item is answered `BAD`, costing a round trip per + /// batch to learn nothing. + pub objectid: bool, + /// Server advertises `X-GM-EXT-1`, so `X-GM-MSGID` may be asked for. + pub gmail_extensions: bool, } /// One folder-sync pass, resolved before any body is fetched. @@ -182,6 +189,11 @@ pub(crate) struct MailboxSyncCapabilities { pub(crate) struct FolderSyncPlan { /// The mechanism this pass negotiated. pub rung: SyncRung, + /// What the negotiation found the server able to do. Carried past the + /// planning stage because the identity extensions decide what the *fetch* + /// asks for, and a fetch that asks for an unadvertised item is answered + /// `BAD`. + pub caps: MailboxSyncCapabilities, pub uid_validity: Option<u32>, pub uid_next: Option<u32>, pub highest_modseq: Option<u64>, @@ -326,6 +338,24 @@ pub fn build_fetch_command_query() -> &'static str { "(FLAGS INTERNALDATE RFC822.SIZE ENVELOPE BODY.PEEK[])" } +/// Add `X-GM-MSGID` to a FETCH item list when, and only when, the server +/// advertised `X-GM-EXT-1`. +/// +/// Gmail's message id is the second identity tier, and unlike RFC 8474's +/// `EMAILID` it can be asked for through the typed client: `imap-proto` parses +/// the attribute and `Fetch::gmail_msg_id` reads it back. So it rides along on +/// the fetch the sync already issues and costs no extra round trip. +/// +/// A server that advertises nothing gets a query byte-identical to the one it +/// got before this existed. That is the point: capability negotiation must not +/// change what an unextended server sees. +fn with_gmail_msgid(items: &str, advertised: bool) -> String { + match items.strip_suffix(')').filter(|_| advertised) { + Some(head) => format!("{head} X-GM-MSGID)"), + None => items.to_string(), + } +} + /// The `UID FETCH` query for the cheap enumeration pass: UID plus `RFC822.SIZE` /// only, deliberately WITHOUT a `BODY.PEEK[]`. This lets the sync discover every /// message's UID and size (to decide per-message whether the body is safe to @@ -906,15 +936,24 @@ impl ImapEngine { drop(enumeration); let mut emails = Vec::new(); + let body_query = with_gmail_msgid(build_fetch_command_query(), plan.caps.gmail_extensions); + let header_query = with_gmail_msgid( + build_headers_only_command_query(), + plan.caps.gmail_extensions, + ); // Phase 2a: normal messages -- full bodies, in bounded UID-set batches. for chunk in body_uids.chunks(batch_size) { let set = join_uid_batch(chunk); + // Read the batch's object ids first. It has to be its own command: + // `EMAILID` cannot ride on the body fetch, because that response + // goes through a parser which treats the item as a fatal error. + // Issuing it here is also what keeps the raw layer's precondition + // true -- the last thing on this connection was a tagged + // completion, so the client holds no buffered bytes. + let object_ids = self.fetch_object_ids(session, plan.caps, &set).await?; let items = { - let mut fetch_stream = session - .uid_fetch(&set, build_fetch_command_query()) - .await - .map_err(|e| { + let mut fetch_stream = session.uid_fetch(&set, &body_query).await.map_err(|e| { MailError::ImapError(format!( "UID FETCH failed for folder '{}': {}", folder_id, e @@ -927,11 +966,16 @@ impl ImapEngine { // `UID FETCH` must carry one (RFC 3501 section 6.4.8), so this // only fires for a non-conforming server. Recording it anyway // would write a placement at the meaningless UID 0. - if item.uid.filter(|u| *u >= 1).is_none() { + let Some(uid) = item.uid.filter(|u| *u >= 1) else { skipped_no_uid += 1; continue; - } - emails.push(self.build_email_from_fetch(folder_id, server_uid_validity, item)?); + }; + emails.push(self.build_email_from_fetch( + folder_id, + server_uid_validity, + item, + object_ids.get(&uid).map(String::as_str), + )?); } } @@ -940,11 +984,13 @@ impl ImapEngine { // buffer an unbounded number of header fetches at once. for chunk in oversized_uids.chunks(batch_size) { let set = join_uid_batch(chunk); + // An oversized message needs the object id most: its body is never + // fetched, so there is no content hash and the alternative is the + // folder-scoped surrogate. + let object_ids = self.fetch_object_ids(session, plan.caps, &set).await?; let items = { - let mut fetch_stream = session - .uid_fetch(&set, build_headers_only_command_query()) - .await - .map_err(|e| { + let mut fetch_stream = + session.uid_fetch(&set, &header_query).await.map_err(|e| { MailError::ImapError(format!( "UID FETCH (headers) failed for folder '{}': {}", folder_id, e @@ -955,11 +1001,16 @@ impl ImapEngine { for item in &items { // Same guard as the body pass: an item with no UID cannot be // placed, and UID 0 is not an address. - if item.uid.filter(|u| *u >= 1).is_none() { + let Some(uid) = item.uid.filter(|u| *u >= 1) else { skipped_no_uid += 1; continue; - } - emails.push(self.build_headers_only_email(folder_id, server_uid_validity, item)); + }; + emails.push(self.build_headers_only_email( + folder_id, + server_uid_validity, + item, + object_ids.get(&uid).map(String::as_str), + )); } } @@ -1067,12 +1118,19 @@ impl ImapEngine { where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { - let (has_qresync, has_condstore) = { + let (has_qresync, has_condstore, has_objectid, has_gmail) = { let caps = session .capabilities() .await .map_err(|e| MailError::ImapError(format!("CAPABILITY query failed: {e}")))?; - (caps.has_str("QRESYNC"), caps.has_str("CONDSTORE")) + ( + caps.has_str("QRESYNC"), + caps.has_str("CONDSTORE"), + // RFC 8474 names the capability OBJECTID; EMAILID is the FETCH + // item it brings, and asking for it otherwise earns a BAD. + caps.has_str("OBJECTID"), + caps.has_str("X-GM-EXT-1"), + ) }; let checkpoint = since_state.and_then(parse_checkpoint); @@ -1127,6 +1185,8 @@ impl ImapEngine { qresync_enabled, condstore: has_condstore, mailbox_has_modseq: selected.highest_modseq.is_some(), + objectid: has_objectid, + gmail_extensions: has_gmail, }; let mut rung = choose_rung(checkpoint, selected.uid_validity, caps); // The QRESYNC SELECT was issued optimistically; if the server answered @@ -1144,6 +1204,7 @@ impl ImapEngine { let mut plan = FolderSyncPlan { rung, + caps, uid_validity: selected.uid_validity, uid_next: selected.uid_next, highest_modseq: selected.highest_modseq, @@ -1202,6 +1263,29 @@ impl ImapEngine { Ok(plan) } + /// The batch's `uid -> EMAILID` map, or an empty one when the server never + /// advertised RFC 8474 `OBJECTID`. + /// + /// The capability gate is the whole reason this is a separate method: a + /// server without `OBJECTID` must see exactly the traffic it saw before + /// object ids existed -- no extra command, no extra round trip -- and a + /// server that has it must be asked through the raw layer rather than the + /// typed client, which cannot parse the reply without dying. + async fn fetch_object_ids<S>( + &self, + session: &mut async_imap::Session<S>, + caps: MailboxSyncCapabilities, + uid_set: &str, + ) -> Result<std::collections::BTreeMap<u32, String>, MailError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, + { + if !caps.objectid { + return Ok(std::collections::BTreeMap::new()); + } + crate::imap_raw::uid_fetch_email_ids(session, uid_set).await + } + /// List the folder's UIDs, optionally narrowed to those changed since /// `changed_since`. Requests no bodies and no flags, so the cost is one /// short line per message rather than a download. @@ -1267,12 +1351,15 @@ impl ImapEngine { /// and spreading them out buys no clarity while pushing this function past /// the argument-count lint. /// - /// In practice every IMAP call site passes `email_id: None` and - /// `gm_msgid: None` today, because the FETCH never asks for them, so this - /// engine reaches only the `Message-ID`+content tier or the surrogate. The - /// JMAP engine enters at the top tier, which means an IMAP-derived key and - /// a JMAP-derived key for the *same* message will not match. Syncing one - /// account over both engines does not converge on a shared identity yet. + /// Both server-assigned tiers are reachable from here. `EMAILID` comes from + /// a raw `UID FETCH` issued only against an `OBJECTID` server + /// ([`crate::imap_raw::uid_fetch_email_ids`]) and `X-GM-MSGID` from the + /// typed fetch against an `X-GM-EXT-1` server, so an IMAP-derived key and a + /// JMAP-derived key for the same message on a server that publishes object + /// ids through both protocols are the same key -- which is what RFC 8474 + /// exists to make possible. Against a server that advertises neither, this + /// engine still lands on the `Message-ID`+content tier or the surrogate, + /// and those do not converge with a JMAP engine's server-assigned ids. fn place( &self, folder_id: &str, @@ -1298,11 +1385,16 @@ impl ImapEngine { /// when it does not (a server that returned only the ENVELOPE), a minimal /// record is synthesised from the envelope so the message is still surfaced /// honestly. + /// + /// `email_id` is this UID's RFC 8474 object id when the folder pass read + /// one; `X-GM-MSGID` is read from the item itself, since the typed parser + /// understands that attribute and the raw layer never needs to see it. fn build_email_from_fetch( &self, folder_id: &str, server_uid_validity: Option<u32>, fetch_data: &async_imap::types::Fetch, + email_id: Option<&str>, ) -> Result<PlacedMessage, MailError> { let uid_num = fetch_data.uid.unwrap_or(0); // The message is addressed on the wire by its UID within the folder's @@ -1318,19 +1410,11 @@ impl ImapEngine { .flags() .any(|flag| matches!(flag, async_imap::types::Flag::Seen)); - // EMAILID (RFC 8474) and X-GM-MSGID are not requested by the current - // FETCH, so identity rests on the Message-ID/content pair where the - // message carried a Message-ID and on the folder-scoped surrogate - // otherwise. - // - // Adding those FETCH items is not a local change. Every already-stored - // message would derive a DIFFERENT key on its next resync, so each - // re-reported placement repoints to the new key -- and the old message - // row, now referenced by nothing, is never reaped. Its decrypted body - // stays in the plaintext FTS index forever, which is exactly the leak - // the reap on placement removal exists to prevent. The FETCH items can - // only be requested once the repoint path reaps the message it - // abandons; until then, requesting them leaks plaintext. + // Gmail's id is a 64-bit integer on the wire; it is rendered decimal + // here because identity inputs are hashed as strings and the rendering + // has to be the one every engine would produce from the same number. + let gm_msgid = fetch_data.gmail_msg_id().map(u64::to_string); + if let Some(raw_bytes) = fetch_data.body() { let mut email = MimeParserAdapter::parse_mime(&self.account_id, raw_bytes)?; let (identity, placement) = self.place( @@ -1338,8 +1422,8 @@ impl ImapEngine { &uid_validity, &remote_id, RemoteIdentity { - email_id: None, - gm_msgid: None, + email_id, + gm_msgid: gm_msgid.as_deref(), message_id: email.message_id.as_deref(), content_hash: email.content_hash.as_deref(), }, @@ -1383,16 +1467,16 @@ impl ImapEngine { .and_then(|raw| Email::normalize_message_id(&String::from_utf8_lossy(raw))); // With no octets there is no content hash, and the `Message-ID` alone - // is never enough to key on, so this path lands on the surrogate tier. - // That is the honest answer: nothing was fetched that could prove two - // folders hold the same message. + // is never enough to key on, so absent a server-assigned id this path + // lands on the surrogate tier. That is the honest answer: nothing was + // fetched that could prove two folders hold the same message. let (identity, placement) = self.place( folder_id, &uid_validity, &remote_id, RemoteIdentity { - email_id: None, - gm_msgid: None, + email_id, + gm_msgid: gm_msgid.as_deref(), message_id: message_id.as_deref(), content_hash: None, }, @@ -1429,6 +1513,7 @@ impl ImapEngine { folder_id: &str, server_uid_validity: Option<u32>, fetch_data: &async_imap::types::Fetch, + email_id: Option<&str>, ) -> PlacedMessage { let uid_num = fetch_data.uid.unwrap_or(0); let remote_id = uid_num.to_string(); @@ -1438,6 +1523,7 @@ impl ImapEngine { let is_read = fetch_data .flags() .any(|flag| matches!(flag, async_imap::types::Flag::Seen)); + let gm_msgid = fetch_data.gmail_msg_id().map(u64::to_string); if let Some(header_bytes) = fetch_data.header() { if let Ok(mut email) = MimeParserAdapter::parse_mime(&self.account_id, header_bytes) { @@ -1452,16 +1538,17 @@ impl ImapEngine { // something else, so drop it; the `Message-ID` it recovered // from those headers is still correct and is kept. Dropping it // also costs this message the content tier -- a `Message-ID` - // never keys on its own -- so an oversized message identifies - // by its folder-scoped surrogate until the octets are fetched. + // never keys on its own -- so absent a server-assigned id an + // oversized message identifies by its folder-scoped surrogate + // until the octets are fetched. email.content_hash = None; let (identity, placement) = self.place( folder_id, &uid_validity, &remote_id, RemoteIdentity { - email_id: None, - gm_msgid: None, + email_id, + gm_msgid: gm_msgid.as_deref(), message_id: email.message_id.as_deref(), content_hash: None, }, @@ -1479,13 +1566,17 @@ impl ImapEngine { // No usable header section: fall back to whatever the envelope offers. // `build_email_from_fetch` cannot error on this path (no body to parse), // but if it ever did, surface a minimal honest record rather than panic. - self.build_email_from_fetch(folder_id, server_uid_validity, fetch_data) + self.build_email_from_fetch(folder_id, server_uid_validity, fetch_data, email_id) .unwrap_or_else(|_| { let (identity, placement) = self.place( folder_id, &uid_validity, &remote_id, - RemoteIdentity::default(), + RemoteIdentity { + email_id, + gm_msgid: gm_msgid.as_deref(), + ..RemoteIdentity::default() + }, is_read, ); PlacedMessage { @@ -2141,6 +2232,10 @@ mod tests { qresync_enabled: true, condstore: true, mailbox_has_modseq: true, + // The identity extensions decide what a FETCH asks for, never + // which rung a pass runs on; they are set for completeness. + objectid: true, + gmail_extensions: true, } } @@ -2217,7 +2312,8 @@ mod tests { MailboxSyncCapabilities { qresync_enabled: false, condstore: true, - mailbox_has_modseq: true + mailbox_has_modseq: true, + ..MailboxSyncCapabilities::default() } ), SyncRung::Condstore @@ -2231,7 +2327,8 @@ mod tests { MailboxSyncCapabilities { qresync_enabled: false, condstore: false, - mailbox_has_modseq: false + mailbox_has_modseq: false, + ..MailboxSyncCapabilities::default() } ), SyncRung::UidSetDiff @@ -2247,7 +2344,8 @@ mod tests { MailboxSyncCapabilities { qresync_enabled: true, condstore: true, - mailbox_has_modseq: false + mailbox_has_modseq: false, + ..MailboxSyncCapabilities::default() } ), SyncRung::Full diff --git a/crates/nuncio-mail/src/imap_raw.rs b/crates/nuncio-mail/src/imap_raw.rs index cee5f64..b4eb29b 100644 --- a/crates/nuncio-mail/src/imap_raw.rs +++ b/crates/nuncio-mail/src/imap_raw.rs @@ -1,8 +1,8 @@ //! Raw IMAP command layer: the responses `async-imap`'s typed API discards. //! -//! The engine needs four things the typed client cannot express, and all four -//! decide whether a sync saw everything or whether a mutation actually -//! happened: +//! The engine needs five things the typed client cannot express, and they all +//! decide whether a sync saw everything, whether a mutation actually happened, +//! or whether two engines can agree on what a message *is*: //! //! | Needed | Why the typed API cannot give it | //! | --- | --- | @@ -10,6 +10,7 @@ //! | `SELECT (QRESYNC …)` | no such method, and `select()`'s parser drops `VANISHED` | //! | `COPYUID` | `uid_copy`/`uid_mv` return `Result<()>`; the tagged response *code* is discarded | //! | `[MODIFIED …]` | same discard, and `imap-proto` has no `ResponseCode` variant for it at all | +//! | `EMAILID` | `imap-proto` cannot *parse* it, and failing to parse kills the connection -- see [`uid_fetch_email_ids`] | //! //! ## Why this is hand-rolled rather than a fork //! @@ -32,13 +33,15 @@ //! against a server that emits the real syntax, and the alternative is having //! no conflict detection at all. +use std::collections::BTreeMap; use std::fmt::Debug; +use std::sync::atomic::{AtomicU64, Ordering}; use async_imap::Session; // Reached through `async-imap`'s re-export rather than a direct dependency, // so the parser types can never version-skew against the client using them. use async_imap::imap_proto::{Response, ResponseCode, Status, UidSetMember}; -use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use crate::parser::MailError; @@ -386,10 +389,326 @@ where Ok(exchange) } +/// Upper bound on the bytes one raw exchange will accumulate before giving up. +/// +/// The raw reader has no framing beyond "a line beginning with our tag", so a +/// server that answers with an endless stream and never completes the command +/// would otherwise grow this buffer without limit. One `EMAILID` line is a few +/// dozen bytes and a batch is capped at a few hundred messages, so this is +/// orders of magnitude above any legitimate reply. +const MAX_RAW_RESPONSE_BYTES: usize = 4 * 1024 * 1024; + +/// Tag prefix for commands this module writes straight to the transport. +/// +/// `async-imap` allocates its own tags from an `IdGenerator` that emits +/// `format!("A{:04}", n % 10_000)` -- always the letter `A` followed by four +/// decimal digits. A tag starting with `NX` therefore cannot collide with one +/// the typed client will ever issue, no matter how many commands a session +/// runs or how often the generator wraps. Colliding would be worse than +/// untidy: two commands sharing a tag makes each one's completion look like +/// the other's, and a reader would stop at the wrong response. +const RAW_TAG_PREFIX: &str = "NX"; + +/// Sequence behind [`RAW_TAG_PREFIX`]. Process-wide rather than per-session +/// because it costs nothing and makes a tag unique in a packet capture across +/// every connection, which is the only thing anyone reads these tags for. +static RAW_TAG_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +fn next_raw_tag() -> String { + let n = RAW_TAG_SEQUENCE.fetch_add(1, Ordering::Relaxed); + format!("{RAW_TAG_PREFIX}{n}") +} + +/// `UID FETCH <set> (UID EMAILID)` (RFC 8474 section 5.2), read straight off +/// the transport, returning `uid -> EMAILID`. +/// +/// # Why this bypasses the client entirely +/// +/// `imap-proto` 0.16.7 cannot parse `EMAILID`: its `msg_att` parser is a closed +/// `alt(...)` of known attributes with no catch-all, so an `EMAILID` item is a +/// hard nom error rather than an unknown-but-tolerated one. `async-imap`'s +/// decoder turns any non-`Incomplete` parse error into a fatal `io::Error` and +/// then latches `read_closed`, after which the stream yields `None` forever -- +/// the session is dead, not degraded. So `EMAILID` must never reach +/// `parse_response`, which rules out `Session::uid_fetch` **and** +/// [`run_collected`], since the latter reads through `Session::read_response` +/// and therefore through the same decoder. +/// +/// What is left is the transport underneath: `Session` exposes `&mut T` +/// (`AsMut`, and `Connection::get_mut` through `Deref`), so the command is +/// written and its reply scanned here, by hand, and no byte of it is offered +/// to nom. +/// +/// # The two conditions this is only correct under +/// +/// 1. **The client's read buffer must be empty.** `ImapStream` owns a buffer +/// that can hold bytes already pulled off the socket. Raw reads bypass it, +/// so anything sitting there is invisible to this scanner. Call this only +/// at a quiescent point -- immediately after a previous command's *tagged* +/// completion has been read -- and never interleaved with an in-flight +/// typed command or a live response stream. +/// 2. **The tag must not collide** with the client's own. See +/// [`RAW_TAG_PREFIX`]. +/// +/// # Degrading rather than failing +/// +/// A tagged `NO`/`BAD` yields an empty map, not an error. The object id is an +/// identity *upgrade*: without it a message still syncs under the +/// `Message-ID`+content tier or the surrogate, so a server that advertises +/// `OBJECTID` and then refuses the fetch must cost the sync precision, never +/// the mail. A stalled read is different and does surface, because a server +/// that stopped answering mid-command has left the session unusable. +pub async fn uid_fetch_email_ids<S>( + session: &mut Session<S>, + uid_set: &str, +) -> Result<BTreeMap<u32, String>, MailError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, +{ + if uid_set.is_empty() { + return Ok(BTreeMap::new()); + } + + let tag = next_raw_tag(); + let command = format!("{tag} UID FETCH {uid_set} (UID EMAILID)\r\n"); + + let stream: &mut S = session.as_mut(); + stream + .write_all(command.as_bytes()) + .await + .map_err(|e| MailError::ImapError(format!("failed to issue '{command:?}': {e}")))?; + stream + .flush() + .await + .map_err(|e| MailError::ImapError(format!("failed to flush '{command:?}': {e}")))?; + + let (lines, tagged) = read_until_tagged(stream, &tag).await?; + + if !tagged_is_ok(&tagged) { + tracing::warn!( + response = %tagged, + "server refused UID FETCH (UID EMAILID) despite advertising OBJECTID; \ + this pass identifies its messages by a weaker tier" + ); + return Ok(BTreeMap::new()); + } + + let mut ids = BTreeMap::new(); + for line in &lines { + if let Some((uid, email_id)) = parse_email_id_line(line) { + ids.insert(uid, email_id); + } + } + Ok(ids) +} + +/// Read complete CRLF-terminated lines until one begins with `tag`, returning +/// the lines before it and the tagged line itself. +/// +/// Line-oriented scanning is sound *for this command only*: an `EMAILID` +/// response carries no literal (`{n}`) section, so no line body can contain an +/// embedded CRLF. It must not be reused for a command whose reply can carry +/// literals -- a body fetch -- without adding literal handling. +async fn read_until_tagged<S>(stream: &mut S, tag: &str) -> Result<(Vec<String>, String), MailError> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let mut buffer: Vec<u8> = Vec::new(); + let mut chunk = [0u8; 4096]; + let mut scanned = 0usize; + let mut lines: Vec<String> = Vec::new(); + let tag_prefix = format!("{tag} "); + + loop { + let read = tokio::time::timeout(RESPONSE_READ_TIMEOUT, stream.read(&mut chunk)) + .await + .map_err(|_| { + MailError::FetchStalled(format!( + "no response to raw '{tag}' within {RESPONSE_READ_TIMEOUT:?}" + )) + })? + .map_err(|e| MailError::ImapError(format!("reading response to raw '{tag}': {e}")))?; + if read == 0 { + return Err(MailError::ImapError(format!( + "connection closed before raw '{tag}' completed" + ))); + } + buffer.extend_from_slice(&chunk[..read]); + if buffer.len() > MAX_RAW_RESPONSE_BYTES { + return Err(MailError::ImapError(format!( + "raw '{tag}' response exceeded {MAX_RAW_RESPONSE_BYTES} bytes without completing" + ))); + } + + // Consume every line that is now complete. `scanned` marks how far the + // buffer has already been split, so a reply arriving in many chunks is + // not re-scanned from the start each time. + while let Some(offset) = buffer[scanned..].iter().position(|b| *b == b'\n') { + let end = scanned + offset; + let line = String::from_utf8_lossy(&buffer[scanned..end]) + .trim_end_matches('\r') + .to_string(); + scanned = end + 1; + if line.starts_with(&tag_prefix) { + return Ok((lines, line)); + } + lines.push(line); + } + } +} + +/// Whether a tagged completion line reports `OK`. +fn tagged_is_ok(tagged: &str) -> bool { + tagged + .split_whitespace() + .nth(1) + .is_some_and(|status| status.eq_ignore_ascii_case("OK")) +} + +/// Pull `(uid, EMAILID)` out of one untagged `FETCH` line. +/// +/// Both wire spellings are accepted. RFC 8474 section 5.2 defines the response +/// as `"EMAILID" SP "(" objectid ")"`, but the bare `EMAILID <objectid>` form +/// is what a hand-rolled scanner meets often enough that rejecting it would +/// mean silently dropping to a weaker identity tier against a server that +/// answered the question correctly enough. +/// +/// Returns `None` for anything that is not an untagged `FETCH` carrying both a +/// UID and a non-empty object id -- there is nothing to key on without both. +fn parse_email_id_line(line: &str) -> Option<(u32, String)> { + let upper = line.to_ascii_uppercase(); + if !upper.starts_with("* ") || !upper.contains(" FETCH ") { + return None; + } + // `to_ascii_uppercase` is byte-length preserving, so offsets found in the + // uppercased copy index the original exactly. Item *names* are + // case-insensitive (RFC 3501 section 9); an object id is not, so the value + // is always taken from the original line. + let uid = find_item_value(&upper, line, "UID ")?.parse::<u32>().ok()?; + let email_id = find_item_value(&upper, line, "EMAILID ")?; + let email_id = email_id + .trim_start_matches('(') + .trim_end_matches(')') + .trim_matches('"') + .to_string(); + if email_id.is_empty() { + return None; + } + Some((uid, email_id)) +} + +/// The token following FETCH item `name` in `line`, located via its uppercased +/// twin `upper`. +/// +/// The name must sit on a token boundary, so `UID ` never matches the tail of +/// another attribute name. A parenthesised value is returned whole, brackets +/// included, for the caller to strip. +fn find_item_value(upper: &str, line: &str, name: &str) -> Option<String> { + let mut from = 0usize; + loop { + let at = from + upper[from..].find(name)?; + let boundary = at == 0 + || upper[..at] + .chars() + .next_back() + .is_some_and(|c| c == '(' || c == ' '); + if boundary { + let value_start = at + name.len(); + let rest = &line[value_start..]; + let end = if rest.starts_with('(') { + rest.find(')').map_or(rest.len(), |i| i + 1) + } else { + rest.find([' ', ')']).unwrap_or(rest.len()) + }; + let value = rest[..end].trim(); + if !value.is_empty() { + return Some(value.to_string()); + } + } + from = at + name.len(); + } +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn an_object_id_is_read_from_both_wire_spellings() { + // RFC 8474 5.2 parenthesises the id; the bare form appears in the wild. + // Both must key the same message, or which spelling a server chose + // would decide whether identity converges. + assert_eq!( + parse_email_id_line("* 1 FETCH (UID 7 EMAILID (M00000001))"), + Some((7, "M00000001".to_string())) + ); + assert_eq!( + parse_email_id_line("* 1 FETCH (UID 7 EMAILID M00000001)"), + Some((7, "M00000001".to_string())) + ); + // Item names are case-insensitive; the id itself is not. + assert_eq!( + parse_email_id_line("* 1 fetch (uid 7 emailid (Mixed-Case_01))"), + Some((7, "Mixed-Case_01".to_string())) + ); + // Order is not fixed, and other attributes may sit between them. + assert_eq!( + parse_email_id_line("* 3 FETCH (EMAILID (M9) FLAGS (\\Seen) UID 12)"), + Some((12, "M9".to_string())) + ); + } + + #[test] + fn a_line_missing_either_half_yields_nothing() { + // Half an answer is not a weaker identity, it is no identity: without + // the UID there is nothing to attach the id to, and without the id + // there is nothing to attach. + assert_eq!( + parse_email_id_line("* 1 FETCH (UID 7 FLAGS (\\Seen))"), + None + ); + assert_eq!(parse_email_id_line("* 1 FETCH (EMAILID (M1))"), None); + assert_eq!(parse_email_id_line("* 1 FETCH (UID 7 EMAILID ())"), None); + assert_eq!(parse_email_id_line("NX0 OK UID FETCH completed"), None); + assert_eq!(parse_email_id_line("* 4 EXPUNGE"), None); + } + + #[test] + fn an_item_name_is_only_matched_on_a_token_boundary() { + // RFC 8474 defines THREADID alongside EMAILID, so a line carrying both + // is ordinary. Every occurrence of `ID ` in it is the tail of a longer + // attribute name; without the boundary rule a search would read + // THREADID's value as the item's own and key the message on it. + let line = "* 1 FETCH (THREADID (T1) EMAILID (M1) UID 7)"; + let upper = line.to_ascii_uppercase(); + assert_eq!(find_item_value(&upper, line, "ID "), None); + assert_eq!( + find_item_value(&upper, line, "EMAILID "), + Some("(M1)".to_string()) + ); + assert_eq!(find_item_value(&upper, line, "UID "), Some("7".to_string())); + assert_eq!(parse_email_id_line(line), Some((7, "M1".to_string()))); + } + + #[test] + fn only_a_tagged_ok_counts_as_a_usable_answer() { + assert!(tagged_is_ok("NX7 OK UID FETCH completed")); + assert!(tagged_is_ok("NX7 ok uid fetch completed")); + assert!(!tagged_is_ok("NX7 NO server does not do that")); + assert!(!tagged_is_ok("NX7 BAD unknown item")); + assert!(!tagged_is_ok("NX7")); + } + + #[test] + fn raw_tags_can_never_collide_with_the_clients_own() { + // async-imap's IdGenerator emits `A` + four digits and wraps at 10_000, + // so a shared tag is only impossible if the prefix differs. + let tag = next_raw_tag(); + assert!(tag.starts_with(RAW_TAG_PREFIX)); + assert_ne!(&tag[..1], "A"); + assert_ne!(next_raw_tag(), tag, "each raw command gets its own tag"); + } + #[test] fn modified_response_code_is_recovered_from_the_tagged_text() { // imap-proto has no ResponseCode::Modified, and its opt(resp_text_code) diff --git a/crates/nuncio-mail/src/test_server.rs b/crates/nuncio-mail/src/test_server.rs index 9eaffea..357517d 100644 --- a/crates/nuncio-mail/src/test_server.rs +++ b/crates/nuncio-mail/src/test_server.rs @@ -48,21 +48,6 @@ pub enum ServerProfile { } impl ServerProfile { - /// The `CAPABILITY` atom list for this profile. `UIDPLUS` and `MOVE` are - /// advertised by all three because every profile we model supports them. - fn capability_line(self) -> String { - let mut caps = vec!["IMAP4rev1", "UIDPLUS", "MOVE", "ENABLE", "LIST-STATUS"]; - match self { - ServerProfile::Basic => {} - ServerProfile::Condstore => caps.push("CONDSTORE"), - ServerProfile::Qresync => { - caps.push("CONDSTORE"); - caps.push("QRESYNC"); - } - } - caps.join(" ") - } - fn supports_condstore(self) -> bool { matches!(self, ServerProfile::Condstore | ServerProfile::Qresync) } @@ -72,6 +57,57 @@ impl ServerProfile { } } +/// Which server-assigned message-id extensions this server publishes. +/// +/// Independent of [`ServerProfile`] because the two axes genuinely are +/// independent in the field: Gmail advertises `X-GM-EXT-1` with CONDSTORE and +/// no QRESYNC, Fastmail advertises `OBJECTID` with QRESYNC, and Exchange +/// advertises neither. Selectable per instance so a test can pin the +/// behaviour of a server that offers *nothing*, which is the case that must +/// stay byte-identical to how the engine behaved before object ids existed. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct IdentityExtensions { + /// RFC 8474 `OBJECTID`, so `EMAILID` is served when asked for. + pub objectid: bool, + /// Gmail's `X-GM-EXT-1`, so `X-GM-MSGID` is served when asked for. + pub gmail: bool, +} + +impl IdentityExtensions { + /// A server that publishes no stable message id at all. + pub fn none() -> Self { + Self::default() + } + + /// RFC 8474 only. + pub fn objectid() -> Self { + Self { + objectid: true, + gmail: false, + } + } + + /// Gmail's vendor extension only, which is what Gmail itself does. + pub fn gmail() -> Self { + Self { + objectid: false, + gmail: true, + } + } + + /// The `CAPABILITY` atoms these extensions contribute. + fn atoms(self) -> Vec<&'static str> { + let mut atoms = Vec::new(); + if self.objectid { + atoms.push("OBJECTID"); + } + if self.gmail { + atoms.push("X-GM-EXT-1"); + } + atoms + } +} + /// One message as the server holds it. #[derive(Debug, Clone)] pub struct MockMessage { @@ -80,6 +116,23 @@ pub struct MockMessage { pub flags: BTreeSet<String>, /// Full RFC822 octets served for `BODY[]`. pub body: String, + /// The server-assigned ids this message keeps wherever it sits. + pub object_ids: ObjectIds, +} + +/// The message-level ids a server assigns once and reports identically from +/// every mailbox holding the message. +/// +/// That invariant is the entire point of RFC 8474: a UID is per-mailbox, an +/// `EMAILID` is not, so a copy in a second folder reports the same one. A mock +/// that minted a fresh id per placement would model a server that does not +/// exist and would let a broken client pass. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObjectIds { + /// RFC 8474 `EMAILID`. + pub email_id: String, + /// Gmail `X-GM-MSGID`, a 64-bit integer on the wire. + pub gm_msgid: u64, } /// A mailbox: its UID space, its mod-sequence, and the tombstones needed to @@ -113,7 +166,7 @@ impl MockMailbox { self.highest_modseq } - fn append(&mut self, body: &str, flags: &[&str]) -> u32 { + fn append(&mut self, body: &str, flags: &[&str], object_ids: ObjectIds) -> u32 { let uid = self.uid_next; self.uid_next += 1; let modseq = self.next_modseq(); @@ -122,6 +175,7 @@ impl MockMailbox { modseq, flags: flags.iter().map(|f| (*f).to_string()).collect(), body: body.to_string(), + object_ids, }); uid } @@ -150,21 +204,57 @@ impl MockMailbox { #[derive(Debug)] pub struct ServerState { pub profile: ServerProfile, + /// Which server-assigned id extensions this instance publishes. + pub extensions: IdentityExtensions, pub mailboxes: BTreeMap<String, MockMailbox>, /// Every command line received, across all connections, in arrival order. /// Tests assert on this to prove *which* protocol path ran -- e.g. that a /// second sync narrowed its FETCH rather than re-fetching everything. pub commands: Vec<String>, next_uid_validity: u32, + next_object_id: u64, } impl ServerState { - fn new(profile: ServerProfile) -> Self { + fn new(profile: ServerProfile, extensions: IdentityExtensions) -> Self { Self { profile, + extensions, mailboxes: BTreeMap::new(), commands: Vec::new(), next_uid_validity: 1000, + next_object_id: 1, + } + } + + /// The `CAPABILITY` atom list. `UIDPLUS` and `MOVE` are advertised by every + /// profile because every server we model supports them; the sync-ladder and + /// identity atoms are what varies. + fn capability_line(&self) -> String { + let mut caps = vec!["IMAP4rev1", "UIDPLUS", "MOVE", "ENABLE", "LIST-STATUS"]; + match self.profile { + ServerProfile::Basic => {} + ServerProfile::Condstore => caps.push("CONDSTORE"), + ServerProfile::Qresync => { + caps.push("CONDSTORE"); + caps.push("QRESYNC"); + } + } + caps.extend(self.extensions.atoms()); + caps.join(" ") + } + + /// Mint the ids for a message the test did not name explicitly. + fn mint_object_ids(&mut self) -> ObjectIds { + let n = self.next_object_id; + self.next_object_id += 1; + ObjectIds { + // RFC 8474 section 4 restricts an objectid to 1*255 of + // ALPHA/DIGIT/`_`/`-`, so this shape is representative rather than + // decorative -- a scanner that mishandles it mishandles the real + // thing. + email_id: format!("M{n:08}"), + gm_msgid: 1_278_455_344_230_334_000 + n, } } } @@ -195,11 +285,20 @@ fn lock_state(state: &Arc<Mutex<ServerState>>) -> MutexGuard<'_, ServerState> { } impl MockImapServer { - /// Bind an ephemeral loopback port and start accepting connections. + /// Bind an ephemeral loopback port and start accepting connections, with + /// no server-assigned message-id extensions. pub async fn start(profile: ServerProfile) -> std::io::Result<Self> { + Self::start_with_identity(profile, IdentityExtensions::none()).await + } + + /// As [`Self::start`], additionally publishing `extensions`. + pub async fn start_with_identity( + profile: ServerProfile, + extensions: IdentityExtensions, + ) -> std::io::Result<Self> { let listener = TcpListener::bind("127.0.0.1:0").await?; let addr = listener.local_addr()?; - let state = Arc::new(Mutex::new(ServerState::new(profile))); + let state = Arc::new(Mutex::new(ServerState::new(profile, extensions))); let accept_state = Arc::clone(&state); let accept_task = tokio::spawn(async move { @@ -241,8 +340,26 @@ impl MockImapServer { .insert(name.to_string(), MockMailbox::new(uid_validity)); } - /// Append a message, creating the mailbox if absent. Returns its UID. + /// Append a message under freshly minted object ids, creating the mailbox + /// if absent. Returns its UID. pub fn append_message(&self, mailbox: &str, body: &str, flags: &[&str]) -> u32 { + let ids = lock_state(&self.state).mint_object_ids(); + self.append_message_with_object_ids(mailbox, body, flags, ids) + } + + /// Append a message under *given* object ids. + /// + /// This is how a test puts the same message in two mailboxes: a real + /// OBJECTID server reports one `EMAILID` for both copies, and only that + /// makes the cross-folder identity claim testable. Passing distinct ids to + /// two appends models two genuinely different messages. + pub fn append_message_with_object_ids( + &self, + mailbox: &str, + body: &str, + flags: &[&str], + object_ids: ObjectIds, + ) -> u32 { let mut state = lock_state(&self.state); let uid_validity = state.next_uid_validity; let entry = state.mailboxes.entry(mailbox.to_string()); @@ -251,13 +368,24 @@ impl MockImapServer { MockMailbox::new(uid_validity) }); let fresh = mailbox.uid_next == 1 && mailbox.messages.is_empty(); - let uid = mailbox.append(body, flags); + let uid = mailbox.append(body, flags, object_ids); if fresh { state.next_uid_validity += 1; } uid } + /// The object ids the server assigned to `uid` in `mailbox`. + pub fn object_ids(&self, mailbox: &str, uid: u32) -> Option<ObjectIds> { + let state = lock_state(&self.state); + state.mailboxes.get(mailbox).and_then(|mb| { + mb.messages + .iter() + .find(|m| m.uid == uid) + .map(|m| m.object_ids.clone()) + }) + } + /// Simulate another client removing a message -- the case a forward-only /// high-water-mark sync can never observe. pub fn expunge(&self, mailbox: &str, uid: u32) -> bool { @@ -331,7 +459,7 @@ async fn handle_connection( let st = lock_state(&state); format!( "* OK [CAPABILITY {}] Nuncio mock IMAP ready\r\n", - st.profile.capability_line() + st.capability_line() ) }; socket.write_all(greeting.as_bytes()).await?; @@ -381,11 +509,11 @@ fn dispatch(line: &str, session: &mut Session, st: &mut ServerState) -> Reply { match command.as_str() { "CAPABILITY" => Reply::open(format!( "* CAPABILITY {}\r\n{tag} OK CAPABILITY completed\r\n", - st.profile.capability_line() + st.capability_line() )), "LOGIN" => Reply::open(format!( "{tag} OK [CAPABILITY {}] Logged in\r\n", - st.profile.capability_line() + st.capability_line() )), "ENABLE" => handle_enable(&tag, &rest, session, st), "NOOP" => Reply::open(format!("{tag} OK NOOP completed\r\n")), @@ -499,7 +627,7 @@ fn handle_select( out.push_str(&format!("* VANISHED (EARLIER) {}\r\n", gone.join(","))); } for msg in mb.messages.iter().filter(|m| m.modseq > since) { - out.push_str(&fetch_line(mb, msg, false)); + out.push_str(&fetch_line(mb, msg, FetchItems::none())); } } } @@ -576,7 +704,39 @@ fn format_flags(flags: &BTreeSet<String>) -> String { flags.iter().cloned().collect::<Vec<_>>().join(" ") } -fn fetch_line(mb: &MockMailbox, msg: &MockMessage, include_body: bool) -> String { +/// Which optional items a `FETCH` reply should carry. +/// +/// A server returns what was asked for. That is not pedantry here: `EMAILID` +/// is unparseable by the client's own parser, so serving it unasked would kill +/// every connection that ran a typed fetch -- exactly the failure the raw +/// layer exists to route around. +#[derive(Debug, Clone, Copy, Default)] +struct FetchItems { + body: bool, + email_id: bool, + gm_msgid: bool, +} + +impl FetchItems { + /// Read the request off the command line, honouring what this server + /// actually advertised: an item the client asked for without the + /// capability is simply not served. + fn requested(upper: &str, extensions: IdentityExtensions) -> Self { + Self { + body: upper.contains("BODY[]") || upper.contains("RFC822"), + email_id: extensions.objectid && upper.contains("EMAILID"), + gm_msgid: extensions.gmail && upper.contains("X-GM-MSGID"), + } + } + + /// The shape a non-FETCH path (a QRESYNC `SELECT`, a non-silent `STORE`) + /// volunteers: flags and mod-sequence only. + fn none() -> Self { + Self::default() + } +} + +fn fetch_line(mb: &MockMailbox, msg: &MockMessage, items_wanted: FetchItems) -> String { let seq = mb.seq_of(msg.uid); let mut items = format!( "UID {} FLAGS ({}) MODSEQ ({})", @@ -584,7 +744,14 @@ fn fetch_line(mb: &MockMailbox, msg: &MockMessage, include_body: bool) -> String format_flags(&msg.flags), msg.modseq ); - if include_body { + if items_wanted.email_id { + // RFC 8474 section 5.2 parenthesises the value. + items.push_str(&format!(" EMAILID ({})", msg.object_ids.email_id)); + } + if items_wanted.gm_msgid { + items.push_str(&format!(" X-GM-MSGID {}", msg.object_ids.gm_msgid)); + } + if items_wanted.body { items.push_str(&format!(" BODY[] {{{}}}\r\n{}", msg.body.len(), msg.body)); } format!("* {seq} FETCH ({items})\r\n") @@ -598,6 +765,7 @@ fn handle_uid_fetch( st: &mut ServerState, ) -> Reply { let condstore = st.profile.supports_condstore(); + let extensions = st.extensions; let Some(mb) = st.mailboxes.get(selected) else { return Reply::open(format!("{tag} NO mailbox vanished\r\n")); }; @@ -606,7 +774,14 @@ fn handle_uid_fetch( }; let upper = line.to_ascii_uppercase(); - let include_body = upper.contains("BODY[]") || upper.contains("RFC822"); + // An unrecognised FETCH item is a syntax error (RFC 3501 section 6.4.5), so + // a server that never advertised OBJECTID rejects the whole command rather + // than quietly answering without the item. Modelling that is what makes the + // client's capability gate load-bearing instead of decorative. + if upper.contains("EMAILID") && !extensions.objectid { + return Reply::open(format!("{tag} BAD unknown FETCH item EMAILID\r\n")); + } + let items_wanted = FetchItems::requested(&upper, extensions); let changed_since = if condstore { parse_changed_since(&upper) } else { @@ -623,7 +798,7 @@ fn handle_uid_fetch( continue; } } - out.push_str(&fetch_line(mb, msg, include_body)); + out.push_str(&fetch_line(mb, msg, items_wanted)); } // RFC 7162 3.2.6: with CHANGEDSINCE ... VANISHED the server reports @@ -724,7 +899,7 @@ fn handle_uid_store( if !silent { for uid in &touched { if let Some(msg) = mb.messages.iter().find(|m| m.uid == *uid) { - out.push_str(&fetch_line(mb, msg, false)); + out.push_str(&fetch_line(mb, msg, FetchItems::none())); } } } @@ -784,11 +959,14 @@ fn handle_uid_copy( }; let present: Vec<u32> = mb.messages.iter().map(|m| m.uid).collect(); let uids = expand_uid_set(spec, &present, mb.uid_next); - let bodies: Vec<(u32, String, BTreeSet<String>)> = mb + // The copies keep the source's object ids. RFC 8474 section 4 makes + // EMAILID a property of the message, so a server that minted a new one + // per mailbox would be reporting two messages where there is one. + let bodies: Vec<(String, BTreeSet<String>, ObjectIds)> = mb .messages .iter() .filter(|m| uids.contains(&m.uid)) - .map(|m| (m.uid, m.body.clone(), m.flags.clone())) + .map(|m| (m.body.clone(), m.flags.clone(), m.object_ids.clone())) .collect(); (uids, mb.uid_validity, bodies) }; @@ -806,9 +984,9 @@ fn handle_uid_copy( let Some(dst) = st.mailboxes.get_mut(&dest) else { return Reply::open(format!("{tag} NO destination vanished\r\n")); }; - for (_, body, flags) in &bodies { + for (body, flags, object_ids) in &bodies { let refs: Vec<&str> = flags.iter().map(String::as_str).collect(); - dest_uids.push(dst.append(body, &refs)); + dest_uids.push(dst.append(body, &refs, object_ids.clone())); } dst.uid_validity }; diff --git a/crates/nuncio-mail/tests/imap_identity_test.rs b/crates/nuncio-mail/tests/imap_identity_test.rs new file mode 100644 index 0000000..06a2f5a --- /dev/null +++ b/crates/nuncio-mail/tests/imap_identity_test.rs @@ -0,0 +1,231 @@ +//! Offline tests for server-assigned message identity over IMAP. +//! +//! The claim under test is the one ADR 0002 rests on: a message has one +//! account-scoped key, and which protocol observed it does not change that key. +//! RFC 8474 exists precisely so a server implementing both IMAP and JMAP +//! reports the same object id through both, so the last test here compares an +//! IMAP-derived key against a JMAP-derived one and requires them to be equal. +//! +//! Everything runs against [`MockImapServer`] over loopback and against a JMAP +//! response parsed from a literal; no test here reaches a network. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use nuncio_core::model::IdentitySource; +use nuncio_mail::backend::FolderChanges; +use nuncio_mail::test_server::{IdentityExtensions, MockImapServer, ObjectIds, ServerProfile}; +use nuncio_mail::{JmapEngine, MailBackend}; + +/// Sync one folder through the real engine against the mock. +async fn sync(server: &MockImapServer, account_id: &str, folder: &str) -> FolderChanges { + let engine = nuncio_mail::ImapEngine::with_credentials( + account_id, + server.host(), + server.port(), + nuncio_core::TlsMode::Plain, + "user", + "pass", + ); + engine + .sync_changes(folder, None) + .await + .expect("folder sync succeeds") +} + +#[tokio::test] +async fn an_objectid_server_keys_messages_by_their_emailid() { + // Tier 1. Reaching it at all is the difference between an identity the + // server vouches for and one derived from whatever the message happened to + // contain. + let server = + MockImapServer::start_with_identity(ServerProfile::Qresync, IdentityExtensions::objectid()) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + let uid = server.append_message("INBOX", "Subject: one\r\n\r\nbody", &[]); + + let changes = sync(&server, "acct-objectid", "INBOX").await; + + assert_eq!(changes.upserts.len(), 1); + assert_eq!( + changes.upserts[0].source, + IdentitySource::EmailId, + "an OBJECTID server must be asked for EMAILID and the answer used" + ); + assert_eq!(changes.upserts[0].placement.remote_id, uid.to_string()); + assert!( + server + .commands_matching("EMAILID") + .iter() + .any(|c| c.to_ascii_uppercase().contains("UID FETCH")), + "the object id must come from a real UID FETCH, got {:?}", + server.commands() + ); +} + +#[tokio::test] +async fn the_same_message_in_a_second_folder_keys_identically() { + // The property a folder-scoped surrogate can never have, and the reason + // tier 1 is worth a separate round trip: two placements, one message. + let server = + MockImapServer::start_with_identity(ServerProfile::Qresync, IdentityExtensions::objectid()) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.create_mailbox("Archive"); + + let shared = ObjectIds { + email_id: "M00000042".to_string(), + gm_msgid: 1_278_455_344_230_334_865, + }; + let body = "Subject: filed\r\n\r\nbody"; + server.append_message_with_object_ids("INBOX", body, &[], shared.clone()); + server.append_message_with_object_ids("Archive", body, &[], shared); + + let inbox = sync(&server, "acct-shared", "INBOX").await; + let archive = sync(&server, "acct-shared", "Archive").await; + + assert_eq!(inbox.upserts.len(), 1); + assert_eq!(archive.upserts.len(), 1); + assert_eq!( + inbox.upserts[0].email.id, archive.upserts[0].email.id, + "one message in two folders must be one identity" + ); + assert_eq!(inbox.upserts[0].source, IdentitySource::EmailId); + assert_eq!(archive.upserts[0].source, IdentitySource::EmailId); + assert_ne!( + inbox.upserts[0].placement, archive.upserts[0].placement, + "the two occupancies are still distinct; only the message is shared" + ); +} + +#[tokio::test] +async fn a_gmail_shaped_server_keys_messages_by_x_gm_msgid() { + // Gmail advertises X-GM-EXT-1 and not OBJECTID, so tier 2 is the best it + // offers -- and it is reachable through the typed client, with no extra + // round trip, because `imap-proto` can parse that attribute. + let server = + MockImapServer::start_with_identity(ServerProfile::Condstore, IdentityExtensions::gmail()) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.append_message("INBOX", "Subject: gmail\r\n\r\nbody", &[]); + + let changes = sync(&server, "acct-gmail", "INBOX").await; + + assert_eq!(changes.upserts.len(), 1); + assert_eq!( + changes.upserts[0].source, + IdentitySource::GmailMsgId, + "X-GM-EXT-1 must lift identity to the Gmail message id" + ); + assert!( + server.commands_matching("EMAILID").is_empty(), + "a server without OBJECTID must never be asked for EMAILID, got {:?}", + server.commands() + ); +} + +#[tokio::test] +async fn a_server_advertising_neither_extension_is_left_exactly_as_it_was() { + // The regression that matters most: capability negotiation must be + // invisible to a server that has neither extension. No new command, and + // the same tier it reached before any of this existed. + for profile in [ + ServerProfile::Qresync, + ServerProfile::Condstore, + ServerProfile::Basic, + ] { + let server = MockImapServer::start(profile).await.expect("server starts"); + server.create_mailbox("INBOX"); + server.append_message("INBOX", "Subject: plain\r\n\r\nbody", &[]); + + let changes = sync(&server, "acct-plain", "INBOX").await; + + assert_eq!(changes.upserts.len(), 1, "{profile:?}"); + assert_eq!( + changes.upserts[0].source, + IdentitySource::Surrogate, + "{profile:?}: with no server id and no octets, the surrogate is the honest answer" + ); + assert!( + server.commands_matching("EMAILID").is_empty(), + "{profile:?}: no EMAILID command may be issued, got {:?}", + server.commands() + ); + assert!( + server.commands_matching("X-GM-MSGID").is_empty(), + "{profile:?}: no X-GM-MSGID item may be requested, got {:?}", + server.commands() + ); + } +} + +#[tokio::test] +async fn an_imap_emailid_and_a_jmap_object_id_produce_the_same_key() { + // The convergence assertion. One account, one message, one server-assigned + // id -- reported as RFC 8474 EMAILID over IMAP and as the Email object id + // over JMAP. If either engine derived its key from a different tier, or + // hashed under a different domain label, these two strings would differ. + const ACCOUNT: &str = "acct-converge"; + const OBJECT_ID: &str = "M00000001"; + + let server = + MockImapServer::start_with_identity(ServerProfile::Qresync, IdentityExtensions::objectid()) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + server.append_message_with_object_ids( + "INBOX", + "Subject: both\r\n\r\nbody", + &[], + ObjectIds { + email_id: OBJECT_ID.to_string(), + gm_msgid: 1, + }, + ); + let imap = sync(&server, ACCOUNT, "INBOX").await; + + let jmap_response = format!( + r#"{{"methodResponses":[["Email/get",{{"accountId":"{ACCOUNT}", + "state":"s-1","list":[{{"id":"{OBJECT_ID}","subject":"both", + "from":[{{"email":"a@nuncio.mx"}}],"to":[{{"email":"b@nuncio.mx"}}], + "receivedAt":1700001000,"isUnread":true}}]}},"c1"]]}}"# + ); + let (jmap, _state) = JmapEngine::new(ACCOUNT) + .parse_email_get_response(&jmap_response) + .expect("the JMAP response parses"); + + assert_eq!(imap.upserts.len(), 1); + assert_eq!(jmap.len(), 1); + assert_eq!( + imap.upserts[0].source, + IdentitySource::EmailId, + "the IMAP side must have entered at the object-id tier" + ); + assert_eq!( + jmap[0].source, + IdentitySource::EmailId, + "the JMAP side must have entered at the object-id tier" + ); + assert_eq!( + imap.upserts[0].email.id, jmap[0].email.id, + "two engines syncing one account must agree on what the message is" + ); + + // Equality alone would survive both engines moving to a different tier or + // a different domain label together, since they share one derivation. The + // golden pins the actual value: SHA-256 over the length-prefixed + // (tier, account, object id) triple. It is a tripwire, not a spec -- if it + // moves, every stored message_key has moved with it, and + // `IDENTITY_SCHEMA_VERSION` must move in the same commit. + const EXPECTED_KEY: &str = "ba2eb228c6b973c548837e32b84e95cf06b73664b3e350f5dfa30614b2e38bd0"; + assert_eq!( + imap.upserts[0].email.id, EXPECTED_KEY, + "the object-id tier's derivation changed; bump IDENTITY_SCHEMA_VERSION with it" + ); + + // And the two engines still describe *where* it sits in their own terms: + // convergence is a claim about identity, never about placement. + assert_ne!(imap.upserts[0].placement, jmap[0].placement); +} diff --git a/crates/nuncio-mail/tests/imap_raw_wire_test.rs b/crates/nuncio-mail/tests/imap_raw_wire_test.rs index 97ed5f2..8638380 100644 --- a/crates/nuncio-mail/tests/imap_raw_wire_test.rs +++ b/crates/nuncio-mail/tests/imap_raw_wire_test.rs @@ -13,7 +13,7 @@ use nuncio_core::model::PlacementKey; use nuncio_mail::imap_raw; -use nuncio_mail::test_server::{MockImapServer, ServerProfile}; +use nuncio_mail::test_server::{IdentityExtensions, MockImapServer, ObjectIds, ServerProfile}; /// Connect the real client to the mock server and authenticate. async fn login( @@ -301,6 +301,89 @@ async fn a_qresync_select_without_enable_is_refused() { ); } +#[tokio::test] +async fn object_ids_are_read_off_the_transport_and_the_session_survives_it() { + // The whole reason this bypasses the client: `imap-proto` has no parser arm + // for EMAILID, and `async-imap` turns a parse error into a fatal I/O error + // that latches the stream closed. So the test that matters is not only that + // the ids come back, but that the *typed* client still works afterwards -- + // proving the raw exchange left the connection and the client's read buffer + // in a state the next command can use. + let server = + MockImapServer::start_with_identity(ServerProfile::Qresync, IdentityExtensions::objectid()) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + let first = server.append_message_with_object_ids( + "INBOX", + "Subject: one\r\n\r\nx", + &[], + ObjectIds { + email_id: "M-first".to_string(), + gm_msgid: 11, + }, + ); + let second = server.append_message_with_object_ids( + "INBOX", + "Subject: two\r\n\r\nx", + &[], + ObjectIds { + email_id: "M-second".to_string(), + gm_msgid: 12, + }, + ); + + let mut session = login(&server).await; + imap_raw::run_collected(&mut session, "SELECT INBOX") + .await + .expect("select"); + + let ids = imap_raw::uid_fetch_email_ids(&mut session, &format!("{first},{second}")) + .await + .expect("the raw EMAILID fetch succeeds"); + + assert_eq!(ids.get(&first).map(String::as_str), Some("M-first")); + assert_eq!(ids.get(&second).map(String::as_str), Some("M-second")); + + // The session is still alive: a typed command after the raw exchange must + // still see the mailbox. + let after = imap_raw::run_collected(&mut session, "UID FETCH 1:* (UID)") + .await + .expect("the session survives the raw exchange"); + assert_eq!(after.fetched_uids, vec![first, second]); +} + +#[tokio::test] +async fn a_refused_object_id_fetch_costs_precision_not_the_sync() { + // A server that rejects the item leaves the message identifiable by a + // weaker tier. Erroring here would fail an entire folder sync over an + // identity *upgrade*, which is the wrong trade in both directions. + let server = MockImapServer::start(ServerProfile::Qresync) + .await + .expect("server starts"); + server.create_mailbox("INBOX"); + let uid = server.append_message("INBOX", "Subject: one\r\n\r\nx", &[]); + + let mut session = login(&server).await; + imap_raw::run_collected(&mut session, "SELECT INBOX") + .await + .expect("select"); + + let ids = imap_raw::uid_fetch_email_ids(&mut session, &uid.to_string()) + .await + .expect("a BAD answer is not a transport failure"); + assert!( + ids.is_empty(), + "nothing was learned, and that is the answer" + ); + + // And the connection is still usable, which a `BAD` must not change. + let after = imap_raw::run_collected(&mut session, "UID FETCH 1:* (UID)") + .await + .expect("the session survives a refusal"); + assert_eq!(after.fetched_uids, vec![uid]); +} + /// Drive a real folder sync against the mock and return what it observed. async fn sync_folder( server: &MockImapServer, diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 24ced58..6d8c5b0 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -628,7 +628,17 @@ impl DatabaseEngine { /// /// Bump this only for a change of that kind, and only together with the /// reset it implies (see [`DatabaseEngine::reconcile_schema_version`]). - pub const IDENTITY_SCHEMA_VERSION: i64 = 2; + /// + /// Generation 3 is where the IMAP engine began asking for RFC 8474 + /// `EMAILID` and `X-GM-MSGID`. Every message a capable server holds now + /// derives its key from a stronger identity tier, so no key stored under + /// generation 2 still names the message it named. Letting that happen a row + /// at a time -- as each resync repointed a placement -- would work, but the + /// repoint reap is a safety net against stranded plaintext, not a migration + /// mechanism, and it would leave the store in two identity models for as + /// long as any folder went unsynced. Rebuilding once is the honest form of + /// the same change. + pub const IDENTITY_SCHEMA_VERSION: i64 = 3; /// Reconcile the on-disk identity-schema generation with this binary's. /// diff --git a/docs/adr/0002-multi-engine-sync-model.md b/docs/adr/0002-multi-engine-sync-model.md index 2e45fa1..a874edd 100644 --- a/docs/adr/0002-multi-engine-sync-model.md +++ b/docs/adr/0002-multi-engine-sync-model.md @@ -175,6 +175,24 @@ sound without a sequence number and resumption. Precedence: `EMAILID` (RFC 8474 OBJECTID) → `X-GM-MSGID` → `(account_id, normalized Message-ID, content_hash)` → `(account, folder, uidvalidity, uid)`. +**Both engines negotiate the top two tiers, so keys converge exactly where the +server publishes an object id.** The IMAP engine requests `EMAILID` from a +server advertising `OBJECTID` and `X-GM-MSGID` from one advertising +`X-GM-EXT-1`; JMAP's Email object id is the same identifier RFC 8474 exposes to +IMAP. One message synced over both protocols therefore lands on one key, which +is what makes the multi-engine model's identity claim true across protocols and +not only within one. + +Be precise about where it stops. Against a server advertising neither +extension, IMAP reaches only the Message-ID+content tier or the folder-scoped +surrogate, while a JMAP engine on the same account still enters at the top tier +from a server-assigned id — those key spaces are disjoint and cross-engine +comparison means nothing there. Object ids are unique within one account on one +server (RFC 8474 §4), never globally, which is why every tier is +account-scoped. And `EMAILID` cannot be requested through the typed IMAP client +at all: `imap-proto` has no parser arm for the attribute and a parse failure is +fatal to the connection, so it is read through the raw command layer instead. + Bodies are keyed by a content hash of the full RFC822 octets, first-write-wins. **A Message-ID hash alone must not be the key**: Message-IDs are public, echoed in every `References:` header, and forgeable, so with `INSERT OR REPLACE` a @@ -238,7 +256,11 @@ work is chunked per folder so it yields to user actions. - **`async-imap` 0.11.3 cannot express any of this**: no `ENABLE`, its `SELECT` parser drops `VANISHED`, and `check_status_ok` discards the tagged response code where `COPYUID` and `[MODIFIED …]` live. A fork-versus-hand-rolled - decision gates the protocol work. + decision gates the protocol work. `EMAILID` is worse than discarded: + `imap-proto`'s `msg_att` is a closed alternation with no catch-all, and the + decoder turns the resulting parse error into a fatal I/O error that latches + the connection closed — so the object id must be read off the transport + underneath the client, never through it. ## Alternatives considered -- 2.54.0 From 619d37f3ab65be54e27873e0e272f6c1a23f830d Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 18:47:43 -0500 Subject: [PATCH 91/95] test(mail): cover placement-addressed mutations and conflicts offline Prove the new wire surface end to end without a network: a multi-mailbox message refuses an unaddressed move and accepts an addressed one; the queued outbox row really carries the occupancy the caller named; naming an occupancy the message does not hold is refused rather than re-aimed; reads report every occupancy while folder_id still names the one they arrived through; conflicts survive from record through resolution; and filters_enabled round-trips over UpdateAccount. Also pin the executor half: a mutation whose payload names a placement dispatches against exactly that occupancy instead of hitting the ambiguity refusal. Refs #395 --- crates/nunciod/src/outbox.rs | 46 +- .../nunciod/tests/mutation_rpcs_e2e_test.rs | 622 ++++++++++++++++++ 2 files changed, 667 insertions(+), 1 deletion(-) create mode 100644 crates/nunciod/tests/mutation_rpcs_e2e_test.rs diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 826de22..77fcaaf 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -891,6 +891,15 @@ mod tests { /// decided here anyway, so the narrower call is also the tighter assertion. async fn dispatch_with_placements( placements: &[Placement], + ) -> (Disposition, Vec<RemoteMutationSpec>) { + dispatch_with_placements_naming(placements, None).await + } + + /// As [`dispatch_with_placements`], but with the queued row's payload + /// naming `named` as the occupancy the intent was formed against. + async fn dispatch_with_placements_naming( + placements: &[Placement], + named: Option<nuncio_core::model::PlacementKey>, ) -> (Disposition, Vec<RemoteMutationSpec>) { let (db, _dir) = DatabaseEngine::connect_ephemeral() .await @@ -905,7 +914,15 @@ mod tests { let env = SucceedingEnv { backend: backend.clone(), }; - let mutation = flag_mutation(); + let mut mutation = flag_mutation(); + if named.is_some() { + mutation.payload = serde_json::to_string(&MutationPayload { + action_type: mutation.mutation_type.clone(), + target: None, + placement: named, + }) + .expect("serialize payload"); + } let payload = serde_json::from_str::<MutationPayload>(&mutation.payload).expect("parse payload"); let disposition = @@ -965,4 +982,31 @@ mod tests { } } } + + /// The same two-mailbox message stops being ambiguous once the queued row + /// records which occupancy the intent was formed against: there is nothing + /// left to guess, so the mutation is addressed at exactly that copy -- + /// including when it is not the one a single-placement inference would have + /// reached. + #[tokio::test] + async fn a_named_placement_addresses_that_occupancy_rather_than_being_refused() { + let (disposition, applied) = dispatch_with_placements_naming( + &[sample_placement(), second_placement()], + Some(second_placement().key()), + ) + .await; + + assert!( + matches!(disposition, Disposition::Completed), + "a row that names its occupancy is fully addressed, not ambiguous" + ); + assert_eq!(applied.len(), 1, "the backend really was asked to act"); + assert_eq!( + applied[0].folder_id, "Archive", + "the named occupancy is the one acted on, not the other copy" + ); + assert_eq!(applied[0].remote_id, "78"); + assert_eq!(applied[0].uid_validity, "9"); + assert_eq!(applied[0].message_id, "msg-outbox-log"); + } } diff --git a/crates/nunciod/tests/mutation_rpcs_e2e_test.rs b/crates/nunciod/tests/mutation_rpcs_e2e_test.rs new file mode 100644 index 0000000..671428e --- /dev/null +++ b/crates/nunciod/tests/mutation_rpcs_e2e_test.rs @@ -0,0 +1,622 @@ +//! Offline end-to-end tests for the hand-driven mail mutation surface: the +//! placement-addressed `MoveMessage`/`DeleteMessage`/`FlagMessage` RPCs, the +//! durable conflict ledger behind `ListConflicts`/`ResolveConflict`, the +//! complete placement set on `Message`, and the per-account filter switch. +//! +//! Each test boots a real `nunciod` daemon gRPC server +//! (`nunciod::grpc::serve_on_listener_with_overrides`) on an ephemeral loopback +//! port, backed by a real temp file-backed [`DatabaseEngine`] and a +//! [`SecretManager::mock`] vault (never the real OS keyring), with every engine +//! override left at its default. The store handle is shared with the server, so +//! the assertions can seed state a sync pass would have written and read back +//! the durable rows the RPCs produced -- which is the only thing enqueuing can +//! truthfully promise, since nothing here reaches a remote server. +//! +//! Fully offline: no real network, no real OS keyring, no live server. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use nuncio_core::model::{Email, IdentitySource, MutationConflict, Placement, PlacementKey}; +use nuncio_core::EventBus; +use nuncio_filter::{FilterEngine, MutationPayload}; +use nuncio_proto::errors; +use nuncio_proto::v1::{ + AccountConfig, AddAccountRequest, DeleteMessageRequest, ErrorReason, FlagMessageRequest, + GetMessageRequest, ListAccountsRequest, ListConflictsRequest, ListMessagesRequest, + MoveMessageRequest, PlacementRef, ResolveConflictRequest, UpdateAccountRequest, +}; +use nuncio_store::db::DatabaseEngine; +use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT}; +use nunciod::grpc::{ + serve_on_listener_with_overrides, AccountsEngineOverrides, CalendarEngineOverrides, + ContactsEngineOverrides, MailEngineOverrides, +}; +use std::sync::Arc; +use tokio::net::TcpListener; +use tonic::Code; + +const ACCOUNT: &str = "acct-mutation-e2e"; +/// One UIDVALIDITY generation for every seeded occupancy: these tests are about +/// which mailbox a mutation lands in, not about generation bumps. +const UIDVALIDITY: &str = "7"; + +/// Boots the real daemon gRPC server on an ephemeral loopback port with every +/// override at its default. +/// +/// Returns the dialable address, the minted bearer token, and the SAME store +/// handle the server is using, so a test can seed rows and inspect what an RPC +/// durably wrote. The `TempDir` guard is returned so the caller controls its +/// lifetime for the whole test (the spawned server keeps using the db). +async fn boot_daemon(db_name: &str) -> (String, String, Arc<DatabaseEngine>, tempfile::TempDir) { + let secrets = Arc::new(SecretManager::mock()); + let dir = tempfile::tempdir().expect("create temp dir for file-backed db"); + let db_path = dir.path().join(db_name); + let db = Arc::new( + DatabaseEngine::connect_file(&db_path, &secrets) + .await + .expect("open temp file-backed database"), + ); + + let token = hex::encode( + secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("mint gRPC bearer token from mock vault"), + ); + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("listener has local addr"); + + let event_bus = Arc::new(EventBus::new()); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + let server_db = db.clone(); + let server_token = token.clone(); + tokio::spawn(async move { + let _ = serve_on_listener_with_overrides( + listener, + event_bus, + server_db, + filter_engine, + secrets, + server_token, + MailEngineOverrides::default(), + CalendarEngineOverrides::default(), + ContactsEngineOverrides::default(), + AccountsEngineOverrides::default(), + ) + .await; + }); + + (addr.to_string(), token, db, dir) +} + +/// One seeded mailbox occupancy of the test account. +fn placement(folder_id: &str, uid: &str) -> Placement { + Placement { + account_id: ACCOUNT.to_string(), + folder_id: folder_id.to_string(), + uid_validity: UIDVALIDITY.to_string(), + remote_id: uid.to_string(), + read: false, + } +} + +/// The wire form a client uses to name that same occupancy. +fn placement_ref(folder_id: &str, uid: &str) -> PlacementRef { + PlacementRef { + account_id: ACCOUNT.to_string(), + folder_id: folder_id.to_string(), + uidvalidity: UIDVALIDITY.to_string(), + uid: uid.to_string(), + } +} + +/// The store-side key the queued outbox row is expected to carry. +fn placement_key(folder_id: &str, uid: &str) -> PlacementKey { + placement(folder_id, uid).key() +} + +/// The account row the read RPCs merge over. `Mail` reads carry no account +/// field, so mail belonging to no configured account is unreachable through +/// them -- seeding one keeps these tests on the production path. +async fn seed_account(db: &DatabaseEngine) { + db.save_account(&nuncio_core::AccountConfig { + id: ACCOUNT.to_string(), + name: "Mutation E2E".to_string(), + email_address: "mutation-e2e@nuncio.mx".to_string(), + keyring_secret_key: format!("nuncio/{ACCOUNT}"), + sync_interval_secs: 60, + filters_enabled: false, + transport: nuncio_core::Transport::Jmap(nuncio_core::JmapTransport { + endpoint_host: "jmap.nuncio.mx".to_string(), + }), + }) + .await + .expect("save the owning account"); +} + +/// Persist one message into each of the given occupancies, the way a sync pass +/// over several mailboxes would. +async fn seed_message(db: &DatabaseEngine, id: &str, placements: &[Placement]) { + let email = Email { + id: id.to_string(), + account_id: ACCOUNT.to_string(), + subject: format!("subject of {id}"), + sender: "alice@nuncio.mx".to_string(), + recipient: "owner@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some(format!("body of {id}")), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }; + for p in placements { + db.save_email_at(&email, IdentitySource::Surrogate, p) + .await + .expect("seed the occupancy"); + } +} + +/// The queued outbox row with this id, decoded into its operation and payload. +async fn queued_mutation(db: &DatabaseEngine, mutation_id: &str) -> (String, MutationPayload) { + let pending = db + .list_pending_mutations(50) + .await + .expect("read the outbox queue"); + let row = pending + .into_iter() + .find(|m| m.id == mutation_id) + .unwrap_or_else(|| panic!("no queued outbox row '{mutation_id}'")); + let payload = + serde_json::from_str::<MutationPayload>(&row.payload).expect("decode the queued payload"); + assert_eq!( + row.rule_id, "manual", + "a hand-driven mutation is attributed to a person, not to a rule" + ); + (row.mutation_type, payload) +} + +/// A message in two mailboxes cannot be moved by naming only the message: one +/// of the copies would be picked for the caller, and a misaimed MOVE is +/// destructive with no undo. Naming the occupancy makes the same call succeed, +/// and the queued row records which copy the intent was formed against. +#[tokio::test] +async fn moving_a_multi_placement_message_requires_the_occupancy_to_be_named() { + let (addr, token, db, _dir) = boot_daemon("mutation_move.db").await; + seed_account(&db).await; + seed_message( + &db, + "msg-multi", + &[placement("INBOX", "101"), placement("Archive", "202")], + ) + .await; + + let mut mail = nuncio_proto::client::connect_mail(&addr, &token) + .await + .expect("mail client connects"); + + // ---- unnamed: refused, and the refusal names the candidates ---- + let status = mail + .move_message(MoveMessageRequest { + message_id: "msg-multi".to_string(), + placement: None, + destination_folder_id: "Trash".to_string(), + }) + .await + .expect_err("an ambiguously-addressed move must be refused"); + assert_eq!(status.code(), Code::InvalidArgument); + assert_eq!( + errors::error_reason(&status), + Some(ErrorReason::ValidationFailed) + ); + assert!( + status.message().contains("INBOX") && status.message().contains("Archive"), + "the refusal must name the candidate mailboxes: {}", + status.message() + ); + assert!( + db.list_pending_mutations(50) + .await + .expect("read the outbox queue") + .is_empty(), + "a refused move must not leave a queued intent behind" + ); + + // ---- named: accepted, and durable ---- + let mutation_id = mail + .move_message(MoveMessageRequest { + message_id: "msg-multi".to_string(), + placement: Some(placement_ref("INBOX", "101")), + destination_folder_id: "Trash".to_string(), + }) + .await + .expect("naming the occupancy makes the move well-defined") + .into_inner() + .mutation_id; + assert!( + !mutation_id.is_empty(), + "the caller needs an id to correlate the queued intent with a conflict" + ); + + let (mutation_type, payload) = queued_mutation(&db, &mutation_id).await; + assert_eq!(mutation_type, "MOVE"); + assert_eq!(payload.action_type, "MOVE"); + assert_eq!(payload.target.as_deref(), Some("Trash")); + assert_eq!( + payload.placement, + Some(placement_key("INBOX", "101")), + "the queued row must remember the occupancy the intent was formed against" + ); +} + +/// Delete, flag and unflag queue the operation they were asked for, addressed +/// at a specific occupancy: named explicitly when there are several to choose +/// between, and inferred only when the message occupies exactly one mailbox. +#[tokio::test] +async fn delete_and_flag_queue_placement_addressed_rows() { + let (addr, token, db, _dir) = boot_daemon("mutation_delete_flag.db").await; + seed_account(&db).await; + seed_message( + &db, + "msg-multi", + &[placement("INBOX", "101"), placement("Archive", "202")], + ) + .await; + seed_message(&db, "msg-solo", &[placement("INBOX", "303")]).await; + + let mut mail = nuncio_proto::client::connect_mail(&addr, &token) + .await + .expect("mail client connects"); + + let delete_id = mail + .delete_message(DeleteMessageRequest { + message_id: "msg-multi".to_string(), + placement: Some(placement_ref("Archive", "202")), + }) + .await + .expect("delete of a named occupancy is well-defined") + .into_inner() + .mutation_id; + let (mutation_type, payload) = queued_mutation(&db, &delete_id).await; + assert_eq!(mutation_type, "DELETE"); + assert_eq!(payload.action_type, "DELETE"); + assert_eq!(payload.target, None); + assert_eq!(payload.placement, Some(placement_key("Archive", "202"))); + + let flag_id = mail + .flag_message(FlagMessageRequest { + message_id: "msg-multi".to_string(), + placement: Some(placement_ref("INBOX", "101")), + flagged: true, + }) + .await + .expect("flag of a named occupancy is well-defined") + .into_inner() + .mutation_id; + let (mutation_type, payload) = queued_mutation(&db, &flag_id).await; + assert_eq!(mutation_type, "FLAG"); + assert_eq!(payload.action_type, "FLAG"); + assert_eq!(payload.placement, Some(placement_key("INBOX", "101"))); + + // `flagged: false` is a distinct operation, not a FLAG with a parameter -- + // the executor maps the two onto opposite remote calls. + let unflag_id = mail + .flag_message(FlagMessageRequest { + message_id: "msg-solo".to_string(), + placement: None, + flagged: false, + }) + .await + .expect("a single-occupancy message needs no placement to be unambiguous") + .into_inner() + .mutation_id; + let (mutation_type, payload) = queued_mutation(&db, &unflag_id).await; + assert_eq!(mutation_type, "UNFLAG"); + assert_eq!(payload.action_type, "UNFLAG"); + assert_eq!( + payload.placement, + Some(placement_key("INBOX", "303")), + "the inferred occupancy is recorded too, so the executor never re-infers it" + ); +} + +/// Naming an occupancy the message does not hold is a not-found error, never a +/// silent re-aim at whichever copy does exist: the caller formed its intent +/// against a mailbox that is no longer true, and acting on a different one +/// would touch mail it never selected. +#[tokio::test] +async fn naming_an_occupancy_the_message_does_not_hold_is_refused() { + let (addr, token, db, _dir) = boot_daemon("mutation_stale_placement.db").await; + seed_account(&db).await; + seed_message(&db, "msg-solo", &[placement("INBOX", "303")]).await; + + let mut mail = nuncio_proto::client::connect_mail(&addr, &token) + .await + .expect("mail client connects"); + + let status = mail + .move_message(MoveMessageRequest { + message_id: "msg-solo".to_string(), + placement: Some(placement_ref("Spam", "999")), + destination_folder_id: "Trash".to_string(), + }) + .await + .expect_err("a stale occupancy must not be honoured"); + assert_eq!(status.code(), Code::NotFound); + assert_eq!( + errors::error_reason(&status), + Some(ErrorReason::MessageNotFound) + ); + assert!( + status.message().contains("Spam"), + "the error must name the occupancy that was asked for: {}", + status.message() + ); + + // A delete aimed at the wrong uid within a real folder is equally refused: + // the uid is part of the address, not a hint. + let status = mail + .delete_message(DeleteMessageRequest { + message_id: "msg-solo".to_string(), + placement: Some(placement_ref("INBOX", "304")), + }) + .await + .expect_err("a wrong uid must not be rounded to the right one"); + assert_eq!( + errors::error_reason(&status), + Some(ErrorReason::MessageNotFound) + ); + + assert!( + db.list_pending_mutations(50) + .await + .expect("read the outbox queue") + .is_empty(), + "a refused mutation must leave nothing queued" + ); +} + +/// Both read RPCs report the message's complete occupancy set, folder id +/// ascending, while `folder_id` keeps its older meaning: the occupancy the +/// message was reached through. A client that only ever saw `folder_id` cannot +/// tell a one-mailbox message from a two-mailbox one. +#[tokio::test] +async fn read_rpcs_report_every_occupancy_of_a_message() { + let (addr, token, db, _dir) = boot_daemon("mutation_placements_read.db").await; + seed_account(&db).await; + seed_message( + &db, + "msg-multi", + &[placement("INBOX", "101"), placement("Archive", "202")], + ) + .await; + + let mut mail = nuncio_proto::client::connect_mail(&addr, &token) + .await + .expect("mail client connects"); + + let fetched = mail + .get_message(GetMessageRequest { + message_id: "msg-multi".to_string(), + }) + .await + .expect("get_message succeeds") + .into_inner() + .message + .expect("a found message is always populated"); + let folders: Vec<&str> = fetched + .placements + .iter() + .map(|p| p.folder_id.as_str()) + .collect(); + assert_eq!(folders, vec!["Archive", "INBOX"]); + let uids: Vec<&str> = fetched.placements.iter().map(|p| p.uid.as_str()).collect(); + assert_eq!(uids, vec!["202", "101"]); + assert!(fetched + .placements + .iter() + .all(|p| p.account_id == ACCOUNT && p.uidvalidity == UIDVALIDITY)); + assert_eq!( + fetched.folder_id, "Archive", + "folder_id still names the single occupancy this read reached the message through" + ); + + // Reached through INBOX this time: the same complete set, a different + // `folder_id`. + let listed = mail + .list_messages(ListMessagesRequest { + folder_id: "INBOX".to_string(), + page_size: 10, + page_token: String::new(), + }) + .await + .expect("list_messages succeeds") + .into_inner() + .messages; + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].folder_id, "INBOX"); + let folders: Vec<&str> = listed[0] + .placements + .iter() + .map(|p| p.folder_id.as_str()) + .collect(); + assert_eq!(folders, vec!["Archive", "INBOX"]); +} + +/// A conflict is durable state, not a passing notification: it survives in a +/// queryable table, resolving it is recorded once and only once, and resolved +/// rows stay as the audit trail of what was decided. +#[tokio::test] +async fn conflicts_round_trip_from_record_through_resolution() { + let (addr, token, db, _dir) = boot_daemon("mutation_conflicts.db").await; + seed_account(&db).await; + seed_message(&db, "msg-solo", &[placement("INBOX", "303")]).await; + + db.record_conflict(&MutationConflict { + id: "conflict-1".to_string(), + mutation_id: "mut-conflicted".to_string(), + message_id: "msg-solo".to_string(), + mutation_type: "MOVE".to_string(), + placement: Some(placement_key("INBOX", "303")), + observed: "another client moved the message to Archive first".to_string(), + detected_at: 1_700_000_500, + resolved_at: None, + resolution: None, + }) + .await + .expect("record the conflict"); + + let mut mail = nuncio_proto::client::connect_mail(&addr, &token) + .await + .expect("mail client connects"); + + let conflicts = mail + .list_conflicts(ListConflictsRequest { + include_resolved: false, + limit: 10, + }) + .await + .expect("list_conflicts succeeds") + .into_inner() + .conflicts; + assert_eq!(conflicts.len(), 1); + assert_eq!(conflicts[0].id, "conflict-1"); + assert_eq!(conflicts[0].mutation_id, "mut-conflicted"); + assert_eq!(conflicts[0].mutation_type, "MOVE"); + assert_eq!( + conflicts[0].observed, + "another client moved the message to Archive first" + ); + assert_eq!( + conflicts[0].placement, + Some(placement_ref("INBOX", "303")), + "the conflict names the occupancy the refused mutation was aimed at" + ); + assert_eq!(conflicts[0].resolved_at, None); + + mail.resolve_conflict(ResolveConflictRequest { + conflict_id: "conflict-1".to_string(), + resolution: "kept the other client's move".to_string(), + }) + .await + .expect("resolve_conflict succeeds"); + + // Resolving it again is an honest error: reporting success would tell the + // caller their decision was recorded when the earlier one already stands. + let status = mail + .resolve_conflict(ResolveConflictRequest { + conflict_id: "conflict-1".to_string(), + resolution: "changed my mind".to_string(), + }) + .await + .expect_err("a conflict can only be resolved once"); + assert_eq!(status.code(), Code::NotFound); + assert_eq!( + errors::error_reason(&status), + Some(ErrorReason::MessageNotFound) + ); + + let unresolved = mail + .list_conflicts(ListConflictsRequest { + include_resolved: false, + limit: 10, + }) + .await + .expect("list_conflicts succeeds") + .into_inner() + .conflicts; + assert!( + unresolved.is_empty(), + "a handled conflict is no longer outstanding work" + ); + + let all = mail + .list_conflicts(ListConflictsRequest { + include_resolved: true, + limit: 10, + }) + .await + .expect("list_conflicts succeeds") + .into_inner() + .conflicts; + assert_eq!(all.len(), 1, "the audit trail survives the resolution"); + assert_eq!( + all[0].resolution.as_deref(), + Some("kept the other client's move") + ); + assert!( + all[0].resolved_at.is_some(), + "a resolved conflict records when the decision was made" + ); +} + +/// `filters_enabled` is settable over the wire rather than only by editing the +/// accounts row by hand. It still defaults to off, because filters take +/// irreversible remote actions and an account should not acquire them merely by +/// being added. +#[tokio::test] +async fn filters_enabled_round_trips_over_the_wire() { + let (addr, token, _db, _dir) = boot_daemon("mutation_filters_flag.db").await; + + let mut accounts = nuncio_proto::client::connect_accounts(&addr, &token) + .await + .expect("accounts client connects"); + + let config = AccountConfig { + id: ACCOUNT.to_string(), + name: "Mutation E2E".to_string(), + email_address: "mutation-e2e@nuncio.mx".to_string(), + keyring_secret_key: format!("nuncio/{ACCOUNT}"), + sync_interval: Some(nuncio_proto::time::duration_from_secs(60)), + filters_enabled: false, + transport: Some(nuncio_proto::v1::account_config::Transport::Jmap( + nuncio_proto::v1::JmapTransport { + endpoint_host: "jmap.nuncio.mx".to_string(), + }, + )), + }; + accounts + .add_account(AddAccountRequest { + config: Some(config.clone()), + password: "mutation-e2e-secret".to_string(), + }) + .await + .expect("add_account succeeds"); + + let listed = accounts + .list_accounts(ListAccountsRequest {}) + .await + .expect("list_accounts succeeds") + .into_inner() + .accounts; + assert_eq!(listed.len(), 1); + assert!( + !listed[0].filters_enabled, + "an account does not get filters by being added, only by being told to" + ); + + accounts + .update_account(UpdateAccountRequest { + config: Some(AccountConfig { + filters_enabled: true, + ..config + }), + password: None, + }) + .await + .expect("update_account succeeds"); + + let relisted = accounts + .list_accounts(ListAccountsRequest {}) + .await + .expect("list_accounts succeeds") + .into_inner() + .accounts; + assert_eq!(relisted.len(), 1); + assert!( + relisted[0].filters_enabled, + "the flip must be persisted and reported back, not silently discarded" + ); +} -- 2.54.0 From 9cc9292912ede2d95389af6eb2698c5a9359122c Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 19:40:16 -0500 Subject: [PATCH 92/95] perf(grpc): batch placement reads for ListMessages Populating `Message.placements` cost one query per message, so a page of fifty messages cost fifty-one round trips against the store. Add a batched store read that fetches a whole set of message keys in one statement per chunk and groups the result by key, and use it from the listing path. The batch chunks at the existing bind-parameter ceiling and orders by folder id within each key, so the first placement remains the message's canonical mailbox exactly as the per-message read had it. --- crates/nuncio-store/src/db.rs | 142 ++++++++++++++++++++++++++++++++++ crates/nunciod/src/grpc.rs | 34 ++++---- 2 files changed, 159 insertions(+), 17 deletions(-) diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 919a2ab..12ef57e 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -1777,6 +1777,71 @@ impl DatabaseEngine { .collect()) } + /// Every mailbox each of `message_keys` currently occupies, grouped by key. + /// + /// The batched form of [`Self::placements_of`], for callers holding a whole + /// page of messages: one query per chunk instead of one per message. Keys + /// absent from the result occupy no mailbox; callers that need an entry for + /// every requested key should treat a missing entry as an empty set. + /// + /// Each key's placements are ordered by folder id ascending, exactly as + /// [`Self::placements_of`] orders them -- callers take the first placement + /// as the message's canonical mailbox, so the order is part of the + /// contract, not an incidental property of the query plan. + /// + /// Chunked at [`MAX_KEYS_PER_IN_LIST`] for the same reason as + /// [`Self::existing_placements`]: the bind-parameter ceiling belongs to the + /// statement, so it can only be enforced where the statement is built. + /// Duplicate keys are collapsed first, so no key's placements can be + /// gathered from two chunks and appended twice. + pub async fn placements_of_batch( + &self, + message_keys: &[String], + ) -> Result<std::collections::HashMap<String, Vec<nuncio_core::model::Placement>>, DatabaseError> + { + let mut seen = std::collections::HashSet::new(); + let unique: Vec<&String> = message_keys + .iter() + .filter(|key| seen.insert(key.as_str())) + .collect(); + + let mut grouped: std::collections::HashMap<String, Vec<nuncio_core::model::Placement>> = + std::collections::HashMap::new(); + + for chunk in unique.chunks(MAX_KEYS_PER_IN_LIST) { + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT message_key, account_id, folder_id, uidvalidity, uid, read_flag + FROM placements WHERE message_key IN (", + ); + let mut separated = builder.separated(", "); + for key in chunk { + separated.push_bind(*key); + } + builder.push(") ORDER BY message_key ASC, folder_id ASC"); + + let rows: Vec<(String, String, String, String, String, i64)> = builder + .build_query_as() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + for (message_key, account_id, folder_id, uid_validity, remote_id, read_flag) in rows { + grouped + .entry(message_key) + .or_default() + .push(nuncio_core::model::Placement { + account_id, + folder_id, + uid_validity, + remote_id, + read: read_flag != 0, + }); + } + } + + Ok(grouped) + } + /// Messages occupying one folder of one account, newest first, paired with /// the placement they were found through. /// @@ -5494,6 +5559,83 @@ mod tests { assert_eq!(known.len(), 1, "and the query must not echo its own input"); } + #[tokio::test] + async fn placements_of_batch_matches_per_message_placements_of_including_order() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Folders are saved in an order that is neither insertion-sorted nor + // reverse-sorted, so a batch query that forgot to order by folder id + // would come back visibly wrong rather than accidentally right. + for (key, folder, uid) in [ + ("key-a", "Zeta", "1"), + ("key-a", "Alpha", "2"), + ("key-a", "Mid", "3"), + ("key-b", "Sent", "4"), + ("key-b", "Archive", "5"), + // Placed but never asked for: proves the `IN (...)` filter is real. + ("key-unrequested", "INBOX", "6"), + ] { + let (email, placement) = sample_message_and_placement(key, folder, uid); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &placement, + ) + .await + .unwrap(); + } + + // "key-a" appears twice: a caller may hand over a page with repeats, + // and a key gathered from two chunks must not be appended twice. + let requested = [ + "key-a".to_string(), + "key-b".to_string(), + "key-a".to_string(), + "key-never-saved".to_string(), + ]; + let grouped = engine.placements_of_batch(&requested).await.unwrap(); + + for key in ["key-a", "key-b"] { + let one_at_a_time = engine.placements_of(key).await.unwrap(); + assert!( + one_at_a_time.len() > 1 || key == "key-b", + "the fixture must exercise more than one placement per message" + ); + assert_eq!( + grouped.get(key), + Some(&one_at_a_time), + "the batched read of '{key}' must equal the per-message read, element for element" + ); + + let folders: Vec<&str> = one_at_a_time + .iter() + .map(|placement| placement.folder_id.as_str()) + .collect(); + let mut sorted = folders.clone(); + sorted.sort_unstable(); + assert_eq!( + folders, sorted, + "the first placement of '{key}' is taken as its canonical mailbox, so folder id order is load-bearing" + ); + } + + assert_eq!( + grouped.get("key-a").map(Vec::len), + Some(3), + "a repeated key must not have its placements gathered twice" + ); + assert!( + !grouped.contains_key("key-never-saved"), + "a key that occupies no mailbox must be absent, not invented" + ); + assert!( + !grouped.contains_key("key-unrequested"), + "the batch must answer only for the keys it was asked about" + ); + assert_eq!(grouped.len(), 2, "and for nothing else"); + } + /// A first sync of a real mailbox hands `existing_placements` the entire /// folder in one call. Each key spends four bind parameters, and SQLite /// caps a statement at 32766 of them, so an unchunked `IN (...)` list dies diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 317df20..481c2a8 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -1025,25 +1025,25 @@ fn map_placement_ref_from_proto(placement: PlacementRefProto) -> nuncio_core::mo /// Fills each message's complete placement set from the store. /// -/// This costs one query per message rather than one per page. That is the -/// honest price of the field: occupancy is per-message state and a page can mix -/// messages that sit in one mailbox with messages that sit in ten. A failure to -/// read one message's placements leaves that message's set empty rather than -/// failing the whole listing, and is logged -- a partial page is more useful -/// than none, and the field is additive so a client that ignores it is -/// unaffected. +/// One batched query for the whole page rather than one per message: occupancy +/// is per-message state, but reading it per message makes a page of fifty cost +/// fifty-one round trips, which shows up directly as page latency. A failure to +/// read the batch leaves every set empty rather than failing the whole listing, +/// and is logged -- a page without placements is more useful than none, and the +/// field is additive so a client that ignores it is unaffected. async fn fill_placements(db: &DatabaseEngine, messages: &mut [MessageProto]) { + let keys: Vec<String> = messages.iter().map(|message| message.id.clone()).collect(); + let mut grouped = match db.placements_of_batch(&keys).await { + Ok(grouped) => grouped, + Err(e) => { + tracing::warn!("Mail: failed to read the placement sets for this page: {e}"); + return; + } + }; + for message in messages.iter_mut() { - match db.placements_of(&message.id).await { - Ok(placements) => { - message.placements = placements.into_iter().map(map_placement_to_proto).collect(); - } - Err(e) => { - tracing::warn!( - message_id = %message.id, - "Mail: failed to read the placement set: {e}" - ); - } + if let Some(placements) = grouped.remove(&message.id) { + message.placements = placements.into_iter().map(map_placement_to_proto).collect(); } } } -- 2.54.0 From 4bbb23be3d6a8c487f8549a45c7e5d9b06280753 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 19:41:55 -0500 Subject: [PATCH 93/95] fix(jmap): parse Email/get for the folder being synced `parse_email_get_response` labelled every placement it produced with a fixed `"inbox"`, so a caller outside the sync loop got occupancies keyed by a mailbox the account may never sync -- and a placement keyed by a folder that is never enumerated can never be matched again. Take the folder id as a parameter and drop the private in-folder variant the sync loop had been calling to work around the fixed label. Callers now say which mailbox they are parsing for. --- crates/nuncio-mail/src/jmap.rs | 28 +++++++++---------- .../nuncio-mail/tests/imap_identity_test.rs | 2 +- .../nuncio-mail/tests/wiremock_jmap_test.rs | 2 +- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index 9f307de..a29bf62 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -451,18 +451,14 @@ impl JmapEngine { } } - /// Parse a raw JMAP `Email/get` JSON response payload into a list of - /// [`PlacedMessage`] and the new state string. - pub fn parse_email_get_response( - &self, - raw_json: &str, - ) -> Result<(Vec<PlacedMessage>, String), MailError> { - self.parse_email_get_response_in_folder(raw_json, "inbox") - } - /// Parse a raw JMAP `Email/get` JSON response payload into the occupancies /// it establishes for `folder_id`, plus the account's Email state string. /// + /// The caller supplies the mailbox being parsed for because the placements + /// this produces are keyed by it: a placement labelled with a folder the + /// account does not sync can never be matched again, so a fixed label would + /// silently strand every message it produced. + /// /// A returned message whose `mailboxIds` is present and does NOT name /// `folder_id` is dropped rather than placed: `Email/changes` is /// account-wide (RFC 8621 section 4.2 offers no mailbox filter), so a @@ -470,7 +466,7 @@ impl JmapEngine { /// here would invent an occupancy the server never reported. Membership /// that the server did not return at all is left alone -- unknown is not /// absent. - fn parse_email_get_response_in_folder( + pub fn parse_email_get_response( &self, raw_json: &str, folder_id: &str, @@ -765,7 +761,7 @@ impl JmapEngine { let get_request = Self::build_email_get_request(jmap_account_id, Some(ids)); let get_raw = self.post_jmap(api_url, &get_request).await?; - let (emails, state) = self.parse_email_get_response_in_folder(&get_raw, folder_id)?; + let (emails, state) = self.parse_email_get_response(&get_raw, folder_id)?; let present: Vec<PlacementKey> = emails.iter().map(|m| m.placement.key()).collect(); @@ -810,7 +806,7 @@ impl JmapEngine { let get_request = Self::build_email_get_request(jmap_account_id, Some(changes.changed.clone())); let get_raw = self.post_jmap(api_url, &get_request).await?; - let (emails, _state) = self.parse_email_get_response_in_folder(&get_raw, folder_id)?; + let (emails, _state) = self.parse_email_get_response(&get_raw, folder_id)?; emails }; @@ -1118,7 +1114,7 @@ mod tests { }"#; let engine = JmapEngine::new("acct-1"); let (emails, _state) = engine - .parse_email_get_response_in_folder(raw, "mb-inbox") + .parse_email_get_response(raw, "mb-inbox") .expect("parse get"); let ids: Vec<&str> = emails .iter() @@ -1142,7 +1138,9 @@ mod tests { ] }"#; let engine = JmapEngine::new("acct-1"); - let (emails, _state) = engine.parse_email_get_response(raw).expect("parse get"); + let (emails, _state) = engine + .parse_email_get_response(raw, "mb-inbox") + .expect("parse get"); assert_eq!( emails[0].email.message_id.as_deref(), @@ -1160,7 +1158,7 @@ mod tests { #[test] fn parse_invalid_jmap_json_fails() { let engine = JmapEngine::new("acct-1"); - let res = engine.parse_email_get_response("{ invalid json }"); + let res = engine.parse_email_get_response("{ invalid json }", "mb-inbox"); assert!(res.is_err()); } diff --git a/crates/nuncio-mail/tests/imap_identity_test.rs b/crates/nuncio-mail/tests/imap_identity_test.rs index 06a2f5a..38ec07a 100644 --- a/crates/nuncio-mail/tests/imap_identity_test.rs +++ b/crates/nuncio-mail/tests/imap_identity_test.rs @@ -193,7 +193,7 @@ async fn an_imap_emailid_and_a_jmap_object_id_produce_the_same_key() { "receivedAt":1700001000,"isUnread":true}}]}},"c1"]]}}"# ); let (jmap, _state) = JmapEngine::new(ACCOUNT) - .parse_email_get_response(&jmap_response) + .parse_email_get_response(&jmap_response, "mb-inbox") .expect("the JMAP response parses"); assert_eq!(imap.upserts.len(), 1); diff --git a/crates/nuncio-mail/tests/wiremock_jmap_test.rs b/crates/nuncio-mail/tests/wiremock_jmap_test.rs index d330f76..7033966 100644 --- a/crates/nuncio-mail/tests/wiremock_jmap_test.rs +++ b/crates/nuncio-mail/tests/wiremock_jmap_test.rs @@ -81,7 +81,7 @@ async fn wiremock_jmap_session_discovery_and_email_get_sync() { let engine = JmapEngine::new("acct-100"); let (emails, new_state) = engine - .parse_email_get_response(&resp.text().await.unwrap()) + .parse_email_get_response(&resp.text().await.unwrap(), "mb-inbox") .expect("parse email get"); assert_eq!(emails.len(), 1); -- 2.54.0 From 4732d6e7865e3a2b2263ad920df3a0b2b9630afc Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 19:46:23 -0500 Subject: [PATCH 94/95] fix(cli): let the CLI take its gRPC token from the environment nunciod mints the gRPC bearer token into the OS keyring, so the keychain item's ACL trusts nunciod. nuncio-cli is a different binary, so reading that item raises an interactive consent prompt. Nothing can answer that prompt from CI, ssh, or a background shell, and the blocking keyring read never returns -- the CLI hung with no output and no error. Add an opt-in NUNCIO_GRPC_TOKEN override: when set and non-empty the CLI uses it verbatim and never touches the keyring. The keyring remains the default and is otherwise unchanged. The keyring read itself can no longer stall forever either. It now runs on a detached OS thread behind a bounded timeout that reports what is blocking and how to bypass it. A detached thread rather than spawn_blocking, because Tokio joins its blocking pool during runtime shutdown, which would just move the hang to process exit. Collapse the eight copies of the token-resolution block onto that one resolver so no call site can drift. Refs #419 --- crates/nuncio-cli/src/lib.rs | 2 +- crates/nuncio-cli/src/runner.rs | 273 ++++++++++++++---- .../nunciod/tests/headless_cli_token_test.rs | 249 ++++++++++++++++ docs/PATH-TO-MANUAL-USE.md | 28 ++ 4 files changed, 501 insertions(+), 51 deletions(-) create mode 100644 crates/nunciod/tests/headless_cli_token_test.rs diff --git a/crates/nuncio-cli/src/lib.rs b/crates/nuncio-cli/src/lib.rs index 3b35cac..ab4b8a6 100644 --- a/crates/nuncio-cli/src/lib.rs +++ b/crates/nuncio-cli/src/lib.rs @@ -12,4 +12,4 @@ pub use args::{ SystemSubcommand, }; pub use output::JsonResponse; -pub use runner::HeadlessRunner; +pub use runner::{HeadlessRunner, GRPC_TOKEN_ENV}; diff --git a/crates/nuncio-cli/src/runner.rs b/crates/nuncio-cli/src/runner.rs index 1d8ae7e..5bb57d5 100644 --- a/crates/nuncio-cli/src/runner.rs +++ b/crates/nuncio-cli/src/runner.rs @@ -5,6 +5,7 @@ use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT}; use nuncio_store::{DatabaseEngine, DatabaseError}; use serde_json::json; use std::sync::Arc; +use std::time::Duration; use thiserror::Error; use crate::args::{ @@ -14,6 +15,40 @@ use crate::args::{ use crate::output::{format_json, format_json_error, format_json_error_with_info}; +/// Environment variable that supplies the gRPC bearer token directly, bypassing +/// the OS keyring entirely. +/// +/// The keyring remains the default and only this client-side lookup honours the +/// override: `nunciod` always mints and owns the authoritative token, so the +/// daemon must never accept one from its environment. +/// +/// The override exists because the keyring item's ACL is bound to the process +/// that created it (`nunciod`). A *different* binary reading the same item — the +/// CLI — makes the platform keychain ask the logged-in user for consent, which +/// nothing can answer without an interactive desktop session. +pub const GRPC_TOKEN_ENV: &str = "NUNCIO_GRPC_TOKEN"; + +/// Upper bound on how long a keyring read may block before the CLI gives up and +/// reports an actionable error. Generous enough that a human can answer a +/// keychain consent prompt, short enough that an unattended run always +/// terminates instead of stalling forever. +const KEYRING_READ_TIMEOUT: Duration = Duration::from_secs(20); + +/// Normalizes a raw environment value into a usable token override. +/// +/// Surrounding whitespace is stripped (shell heredocs and `env` files routinely +/// add a trailing newline) and an empty value is treated as "not set", so +/// `NUNCIO_GRPC_TOKEN=` cannot silently authenticate with an empty token. +fn normalize_token_override(raw: Option<String>) -> Option<String> { + raw.map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +/// Reads the gRPC bearer token override from the process environment. +fn grpc_token_from_env() -> Option<String> { + normalize_token_override(std::env::var(GRPC_TOKEN_ENV).ok()) +} + /// Parses a `--imap-mode`/`--smtp-mode` CLI string into `nuncio_core::TlsMode`. /// Rejects anything else rather than silently defaulting, since silently /// falling back to e.g. `Plain` for a typo'd mode would be a serious @@ -326,7 +361,9 @@ pub enum RunnerError { /// `nuncio.v1.Accounts` APIs, authenticated by a bearer token read from an /// injected [`SecretManager`] — production code uses /// [`SecretManager::production`] (the real OS keyring), while tests inject -/// [`SecretManager::mock`] so no test ever touches the real vault. +/// [`SecretManager::mock`] so no test ever touches the real vault. Setting +/// [`GRPC_TOKEN_ENV`] overrides that lookup entirely, which is how the CLI is +/// driven from a non-interactive session. /// /// `account add` persists through the daemon so the account (and its /// password, stored ONLY in the daemon's OS keyring vault) survives past @@ -337,6 +374,10 @@ pub struct HeadlessRunner { db: DatabaseEngine, secrets: Arc<SecretManager>, grpc_addr: String, + // When present, the bearer token to send instead of consulting `secrets`. + // Resolved once at construction so a single CLI invocation cannot change + // which credential it authenticates with halfway through. + token_override: Option<String>, // Kept only to hold the ephemeral database's backing directory open for // the runner's lifetime: dropping it would unlink the directory out from // under `db` while the pool may still need to open new connections. @@ -347,10 +388,13 @@ impl HeadlessRunner { /// Initialize a new `HeadlessRunner` with an ephemeral database, the /// real OS keyring vault ([`SecretManager::production`]), and the gRPC /// daemon address resolved from [`nuncio_proto::grpc_addr_from_env`]. + /// [`GRPC_TOKEN_ENV`], when set, supplies the bearer token instead of the + /// keyring. pub async fn ephemeral() -> Result<Self, RunnerError> { - Self::ephemeral_with( + Self::ephemeral_with_token_override( Arc::new(SecretManager::production()), nuncio_proto::grpc_addr_from_env(), + grpc_token_from_env(), ) .await } @@ -362,9 +406,28 @@ impl HeadlessRunner { /// `system status`: pass a [`SecretManager::mock`]-backed instance /// (never the real OS keyring) and the address of a test-local gRPC /// server. + /// The ambient [`GRPC_TOKEN_ENV`] override is deliberately NOT consulted + /// here, so a variable that happens to be exported in a developer's shell + /// can never change what an injected-vault test authenticates with; tests + /// that want the override pass it explicitly to + /// [`Self::ephemeral_with_token_override`]. pub async fn ephemeral_with( secrets: Arc<SecretManager>, grpc_addr: String, + ) -> Result<Self, RunnerError> { + Self::ephemeral_with_token_override(secrets, grpc_addr, None).await + } + + /// Initialize a new `HeadlessRunner` with an ephemeral database, an + /// explicit secret vault + gRPC daemon address, and an explicit bearer + /// token override. + /// + /// `token_override`, when `Some`, is used verbatim as the `authorization: + /// Bearer` credential and the vault is never consulted for it. + pub async fn ephemeral_with_token_override( + secrets: Arc<SecretManager>, + grpc_addr: String, + token_override: Option<String>, ) -> Result<Self, RunnerError> { let (db, db_dir) = DatabaseEngine::connect_ephemeral() .await @@ -376,9 +439,59 @@ impl HeadlessRunner { secrets, _db_dir: db_dir, grpc_addr, + token_override: normalize_token_override(token_override), }) } + /// Resolves the gRPC bearer token, preferring an explicit override over the + /// OS keyring. + /// + /// The keyring read runs on a detached OS thread behind a bounded timeout + /// rather than inline. On macOS a keychain item whose ACL does not trust + /// this binary makes the read block on a GUI consent prompt that never + /// arrives in a headless session, and that call cannot be cancelled — so + /// the timeout must be able to abandon the thread. A plain + /// `std::thread::spawn` is used instead of `spawn_blocking` precisely + /// because Tokio joins its blocking pool during runtime shutdown, which + /// would reintroduce the hang at process exit; a detached thread is reaped + /// when the process ends. + /// + /// The returned error never carries token material — only the vault's own + /// failure description, which is keyed by account name, not secret value. + async fn resolve_grpc_token(&self) -> Result<String, String> { + if let Some(token) = &self.token_override { + return Ok(token.clone()); + } + + let secrets = Arc::clone(&self.secrets); + let (tx, rx) = tokio::sync::oneshot::channel(); + std::thread::spawn(move || { + let resolved = secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .map(hex::encode) + .map_err(|e| e.to_string()); + // A closed receiver means the timeout already fired; nothing to do. + let _ = tx.send(resolved); + }); + + match tokio::time::timeout(KEYRING_READ_TIMEOUT, rx).await { + Ok(Ok(Ok(token))) => Ok(token), + Ok(Ok(Err(e))) => Err(format!("failed to read gRPC bearer token from vault: {e}")), + Ok(Err(_)) => { + Err("gRPC bearer token lookup ended without producing a result".to_string()) + } + Err(_) => Err(format!( + "timed out after {}s reading the gRPC bearer token from the OS keyring: the \ + keyring is most likely waiting on an interactive access-consent prompt that \ + cannot be answered in this session (the token belongs to nunciod, so another \ + binary reading it needs consent). Run this command from a logged-in desktop \ + session, or set {GRPC_TOKEN_ENV} to the daemon's bearer token to bypass the \ + keyring entirely", + KEYRING_READ_TIMEOUT.as_secs() + )), + } + } + /// Access the underlying `EventBus`. #[allow(dead_code)] pub fn event_bus(&self) -> &EventBus { @@ -1412,17 +1525,13 @@ impl HeadlessRunner { } } - /// Resolves the gRPC bearer token from the injected vault and dials the + /// Resolves the gRPC bearer token (see [`Self::resolve_grpc_token`]) and dials the /// `nuncio.v1.Export` service at `self.grpc_addr`, shared by /// [`Self::handle_mail_export`]. async fn connect_export_client( &self, ) -> Result<nuncio_proto::client::AuthenticatedExportClient, String> { - let token_bytes = self - .secrets - .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) - .map_err(|e| format!("failed to read gRPC bearer token from vault: {e}"))?; - let token = hex::encode(token_bytes); + let token = self.resolve_grpc_token().await?; nuncio_proto::client::connect_export(&self.grpc_addr, &token) .await @@ -1524,17 +1633,13 @@ impl HeadlessRunner { } } - /// Resolves the gRPC bearer token from the injected vault and dials the + /// Resolves the gRPC bearer token (see [`Self::resolve_grpc_token`]) and dials the /// `nuncio.v1.Audit` service at `self.grpc_addr`, shared by /// [`Self::handle_audit_list`] / [`Self::handle_audit_verify`]. async fn connect_audit_client( &self, ) -> Result<nuncio_proto::client::AuthenticatedAuditClient, String> { - let token_bytes = self - .secrets - .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) - .map_err(|e| format!("failed to read gRPC bearer token from vault: {e}"))?; - let token = hex::encode(token_bytes); + let token = self.resolve_grpc_token().await?; nuncio_proto::client::connect_audit(&self.grpc_addr, &token) .await @@ -1994,17 +2099,13 @@ impl HeadlessRunner { } } - /// Resolves the gRPC bearer token from the injected vault and dials the + /// Resolves the gRPC bearer token (see [`Self::resolve_grpc_token`]) and dials the /// `nuncio.v1.Filters` service at `self.grpc_addr`, shared by every /// `filter` handler above. async fn connect_filters_client( &self, ) -> Result<nuncio_proto::client::AuthenticatedFiltersClient, String> { - let token_bytes = self - .secrets - .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) - .map_err(|e| format!("failed to read gRPC bearer token from vault: {e}"))?; - let token = hex::encode(token_bytes); + let token = self.resolve_grpc_token().await?; nuncio_proto::client::connect_filters(&self.grpc_addr, &token) .await @@ -2459,51 +2560,39 @@ impl HeadlessRunner { } } - /// Resolves the gRPC bearer token from the injected vault and dials the + /// Resolves the gRPC bearer token (see [`Self::resolve_grpc_token`]) and dials the /// `nuncio.v1.Accounts` service at `self.grpc_addr`, shared by /// [`Self::handle_add_account`] and [`Self::handle_accounts_list`]. async fn connect_accounts_client( &self, ) -> Result<nuncio_proto::client::AuthenticatedAccountsClient, String> { - let token_bytes = self - .secrets - .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) - .map_err(|e| format!("failed to read gRPC bearer token from vault: {e}"))?; - let token = hex::encode(token_bytes); + let token = self.resolve_grpc_token().await?; nuncio_proto::client::connect_accounts(&self.grpc_addr, &token) .await .map_err(|e| format!("nunciod daemon unreachable at {}: {e}", self.grpc_addr)) } - /// Resolves the gRPC bearer token from the injected vault and dials the + /// Resolves the gRPC bearer token (see [`Self::resolve_grpc_token`]) and dials the /// `nuncio.v1.Mail` service at `self.grpc_addr`, shared by every /// `mail`/`folder` read-path handler above. async fn connect_mail_client( &self, ) -> Result<nuncio_proto::client::AuthenticatedMailClient, String> { - let token_bytes = self - .secrets - .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) - .map_err(|e| format!("failed to read gRPC bearer token from vault: {e}"))?; - let token = hex::encode(token_bytes); + let token = self.resolve_grpc_token().await?; nuncio_proto::client::connect_mail(&self.grpc_addr, &token) .await .map_err(|e| format!("nunciod daemon unreachable at {}: {e}", self.grpc_addr)) } - /// Resolves the gRPC bearer token from the injected vault and dials the + /// Resolves the gRPC bearer token (see [`Self::resolve_grpc_token`]) and dials the /// `nuncio.v1.Calendar` service at `self.grpc_addr`, shared by /// [`Self::handle_cal_list`] and [`Self::handle_cal_sync`]. async fn connect_calendar_client( &self, ) -> Result<nuncio_proto::client::AuthenticatedCalendarClient, String> { - let token_bytes = self - .secrets - .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) - .map_err(|e| format!("failed to read gRPC bearer token from vault: {e}"))?; - let token = hex::encode(token_bytes); + let token = self.resolve_grpc_token().await?; nuncio_proto::client::connect_calendar(&self.grpc_addr, &token) .await @@ -2612,17 +2701,13 @@ impl HeadlessRunner { } } - /// Resolves the gRPC bearer token from the injected vault and dials the + /// Resolves the gRPC bearer token (see [`Self::resolve_grpc_token`]) and dials the /// `nuncio.v1.Contacts` service at `self.grpc_addr`, shared by every /// `contact` handler above. async fn connect_contacts_client( &self, ) -> Result<nuncio_proto::client::AuthenticatedContactsClient, String> { - let token_bytes = self - .secrets - .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) - .map_err(|e| format!("failed to read gRPC bearer token from vault: {e}"))?; - let token = hex::encode(token_bytes); + let token = self.resolve_grpc_token().await?; nuncio_proto::client::connect_contacts(&self.grpc_addr, &token) .await @@ -2991,16 +3076,12 @@ impl HeadlessRunner { } } - /// Reads the gRPC bearer token from the injected vault, connects to the + /// Resolves the gRPC bearer token (see [`Self::resolve_grpc_token`]), connects to the /// daemon over gRPC, and calls `GetStatus`. Returns the full live health /// response on success, or a human-readable error string describing /// exactly what failed (vault, connection, or the RPC itself). async fn query_daemon_status(&self) -> Result<nuncio_proto::v1::GetStatusResponse, String> { - let token_bytes = self - .secrets - .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) - .map_err(|e| format!("failed to read gRPC bearer token from vault: {e}"))?; - let token = hex::encode(token_bytes); + let token = self.resolve_grpc_token().await?; let mut client = nuncio_proto::client::connect_system(&self.grpc_addr, &token) .await @@ -3039,6 +3120,98 @@ fn sync_state_label(state: i32) -> &'static str { mod tests { use super::*; + #[test] + fn normalize_token_override_trims_and_rejects_blank_values() { + assert_eq!( + normalize_token_override(Some(" deadbeef\n".to_string())), + Some("deadbeef".to_string()) + ); + assert_eq!(normalize_token_override(Some(String::new())), None); + assert_eq!(normalize_token_override(Some(" ".to_string())), None); + assert_eq!(normalize_token_override(None), None); + } + + /// The env override must be picked up when set and must leave the runner on + /// the keyring path when it is unset or blank. + #[test] + fn grpc_token_from_env_reads_the_override_only_when_it_is_non_empty() { + std::env::remove_var(GRPC_TOKEN_ENV); + assert_eq!(grpc_token_from_env(), None); + + std::env::set_var(GRPC_TOKEN_ENV, ""); + assert_eq!(grpc_token_from_env(), None); + + std::env::set_var(GRPC_TOKEN_ENV, " c0ffee "); + assert_eq!(grpc_token_from_env(), Some("c0ffee".to_string())); + + std::env::remove_var(GRPC_TOKEN_ENV); + } + + /// An injected override must be used verbatim and must never reach the + /// vault -- proven here by giving the runner a vault whose token is + /// necessarily different from the override. + #[tokio::test] + async fn resolve_grpc_token_prefers_the_override_over_the_vault() { + let secrets = Arc::new(SecretManager::mock()); + let vault_token = hex::encode( + secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("mock vault mints a token"), + ); + + let runner = HeadlessRunner::ephemeral_with_token_override( + Arc::clone(&secrets), + "127.0.0.1:1".to_string(), + Some(" override-token ".to_string()), + ) + .await + .expect("runner init"); + + let resolved = runner + .resolve_grpc_token() + .await + .expect("override resolves without touching the vault"); + assert_eq!(resolved, "override-token"); + assert_ne!(resolved, vault_token); + } + + /// With no override the vault remains the source of the token. + #[tokio::test] + async fn resolve_grpc_token_falls_back_to_the_vault_without_an_override() { + let secrets = Arc::new(SecretManager::mock()); + let vault_token = hex::encode( + secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("mock vault mints a token"), + ); + + let runner = + HeadlessRunner::ephemeral_with(Arc::clone(&secrets), "127.0.0.1:1".to_string()) + .await + .expect("runner init"); + + assert_eq!( + runner + .resolve_grpc_token() + .await + .expect("vault resolves the token"), + vault_token + ); + } + + /// `ephemeral_with` must ignore an ambient override so an exported variable + /// in a developer's shell cannot silently change what tests authenticate + /// with. + #[tokio::test] + async fn ephemeral_with_ignores_an_ambient_env_override() { + let secrets = Arc::new(SecretManager::mock()); + let runner = + HeadlessRunner::ephemeral_with(Arc::clone(&secrets), "127.0.0.1:1".to_string()) + .await + .expect("runner init"); + assert_eq!(runner.token_override, None); + } + #[test] fn parse_tls_mode_accepts_all_valid_modes_and_rejects_garbage() { assert_eq!( diff --git a/crates/nunciod/tests/headless_cli_token_test.rs b/crates/nunciod/tests/headless_cli_token_test.rs new file mode 100644 index 0000000..d1b3e6e --- /dev/null +++ b/crates/nunciod/tests/headless_cli_token_test.rs @@ -0,0 +1,249 @@ +//! Headless bearer-token resolution for the reference CLI. +//! +//! `nunciod` mints the gRPC bearer token into the OS keyring, so the keyring +//! item's access control list trusts the *daemon* binary. Any other binary +//! reading that item -- the CLI -- is a different program, and platform +//! keychains answer that by asking the logged-in user for consent. Nothing can +//! answer that prompt from CI, an ssh session, or a background shell, so the +//! read blocks forever. `NUNCIO_GRPC_TOKEN` is the explicit, opt-in escape +//! hatch; the keyring stays the default. +//! +//! These tests boot the real daemon gRPC server on an ephemeral loopback port +//! (mock vault, no real keychain, no live network) and drive the real +//! `HeadlessRunner` against it to prove: +//! +//! - an override authenticates and the vault is never consulted for it; +//! - with no override the keyring path is unchanged and still authenticates; +//! - a wrong override fails honestly and the token never reaches the output +//! or the telemetry. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::sync::{Arc, Mutex, OnceLock}; + +use nuncio_cli::{Commands, HeadlessRunner, SystemSubcommand}; +use nuncio_core::EventBus; +use nuncio_filter::FilterEngine; +use nuncio_store::db::DatabaseEngine; +use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT}; +use tokio::net::TcpListener; +use tracing::field::{Field, Visit}; +use tracing::span::Attributes; +use tracing::{Event, Id, Subscriber}; +use tracing_subscriber::layer::Context as LayerContext; +use tracing_subscriber::prelude::*; +use tracing_subscriber::registry::LookupSpan; + +/// A bearer token that is unmistakable if it ever leaks into output or logs. +const SENTINEL_TOKEN: &str = "SENTINELheadlessCliBearerTokenVvWw001"; + +/// Boots the real daemon gRPC server on an ephemeral loopback port, +/// authenticated by `token`. Returns the address; the temp dir is kept alive +/// by the returned guard. +async fn boot_daemon(secrets: Arc<SecretManager>, token: String) -> std::net::SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("listener has local addr"); + let (db, db_dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")); + tokio::spawn(async move { + // The temp dir must outlive the server, so it is moved into the task. + let _db_dir = db_dir; + let _ = nunciod::grpc::serve_on_listener( + listener, + Arc::new(EventBus::new()), + Arc::new(db), + filter_engine, + secrets, + token, + ) + .await; + }); + addr +} + +/// Runs `system status` through the reference CLI against `addr`. +async fn run_status(runner: &HeadlessRunner) -> String { + runner + .execute_command( + &Commands::System { + action: SystemSubcommand::Status, + }, + true, + ) + .await +} + +/// An explicitly supplied token must authenticate on its own, with the runner's +/// vault holding a *different* token -- which is only possible if the override +/// short-circuits the keyring entirely. +#[tokio::test] +async fn env_override_authenticates_without_consulting_the_vault() { + let daemon_secrets = Arc::new(SecretManager::mock()); + let token = hex::encode( + daemon_secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("mock vault mints gRPC bearer token"), + ); + let addr = boot_daemon(Arc::clone(&daemon_secrets), token.clone()).await; + + // A vault the CLI could only fail with: an independent mock mints an + // unrelated token. + let unrelated_vault = Arc::new(SecretManager::mock()); + let unrelated_token = hex::encode( + unrelated_vault + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("mock vault mints gRPC bearer token"), + ); + assert_ne!(unrelated_token, token); + + let runner = HeadlessRunner::ephemeral_with_token_override( + unrelated_vault, + addr.to_string(), + Some(token), + ) + .await + .expect("cli headless runner initializes"); + + let out = run_status(&runner).await; + assert!( + out.contains(r#""status":"ok""#), + "override should authenticate, got: {out}" + ); +} + +/// With no override the CLI must behave exactly as before: read the token from +/// the injected vault and authenticate with it. +#[tokio::test] +async fn keyring_path_is_unchanged_when_no_override_is_set() { + let secrets = Arc::new(SecretManager::mock()); + let token = hex::encode( + secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("mock vault mints gRPC bearer token"), + ); + let addr = boot_daemon(Arc::clone(&secrets), token).await; + + let runner = HeadlessRunner::ephemeral_with(secrets, addr.to_string()) + .await + .expect("cli headless runner initializes"); + + let out = run_status(&runner).await; + assert!( + out.contains(r#""status":"ok""#), + "vault-resolved token should authenticate, got: {out}" + ); +} + +// ---- Telemetry capture: the token may never appear in logs. ---- + +#[derive(Default)] +struct Sink(Mutex<Vec<String>>); + +impl Sink { + fn push_all(&self, values: Vec<String>) { + if let Ok(mut guard) = self.0.lock() { + guard.extend(values); + } + } + + fn snapshot(&self) -> Vec<String> { + self.0.lock().map(|g| g.clone()).unwrap_or_default() + } +} + +struct FieldVisitor(Vec<String>); + +impl Visit for FieldVisitor { + fn record_str(&mut self, field: &Field, value: &str) { + self.0.push(format!("{}={}", field.name(), value)); + } + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0.push(format!("{}={:?}", field.name(), value)); + } +} + +struct CaptureLayer(Arc<Sink>); + +impl<S> tracing_subscriber::Layer<S> for CaptureLayer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_new_span(&self, attrs: &Attributes<'_>, _id: &Id, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(vec![format!("span={}", attrs.metadata().name())]); + attrs.record(&mut visitor); + self.0.push_all(visitor.0); + } + + fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) { + let mut visitor = FieldVisitor(vec![format!("event={}", event.metadata().target())]); + event.record(&mut visitor); + self.0.push_all(visitor.0); + } +} + +/// One process-wide capturing subscriber, installed once: a global subscriber +/// can only be set once per test binary and every test here shares it. +fn capture_sink() -> Arc<Sink> { + static SINK: OnceLock<Arc<Sink>> = OnceLock::new(); + Arc::clone(SINK.get_or_init(|| { + let sink = Arc::new(Sink::default()); + let subscriber = tracing_subscriber::registry().with(CaptureLayer(Arc::clone(&sink))); + let _ = tracing::subscriber::set_global_default(subscriber); + sink + })) +} + +/// A wrong override must fail honestly rather than hang or fabricate a status, +/// and the token must never reach the command output or the telemetry. +#[tokio::test] +async fn rejected_override_never_echoes_the_token_to_output_or_logs() { + let sink = capture_sink(); + + let daemon_secrets = Arc::new(SecretManager::mock()); + let token = hex::encode( + daemon_secrets + .get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32) + .expect("mock vault mints gRPC bearer token"), + ); + let addr = boot_daemon(Arc::clone(&daemon_secrets), token).await; + + let runner = HeadlessRunner::ephemeral_with_token_override( + Arc::new(SecretManager::mock()), + addr.to_string(), + Some(SENTINEL_TOKEN.to_string()), + ) + .await + .expect("cli headless runner initializes"); + + let out = run_status(&runner).await; + assert!( + out.contains(r#""status":"error""#), + "a wrong token must be rejected, got: {out}" + ); + assert!( + !out.contains(SENTINEL_TOKEN), + "the bearer token must never be echoed in command output: {out}" + ); + + // Proves the capture is live, so the absence assertion below cannot pass + // by capturing nothing at all. + tracing::info!(marker = "headless-cli-token-capture-live", "capture probe"); + + let captured = sink.snapshot(); + assert!( + captured + .iter() + .any(|line| line.contains("headless-cli-token-capture-live")), + "the capturing subscriber must be recording telemetry" + ); + for line in &captured { + assert!( + !line.contains(SENTINEL_TOKEN), + "the bearer token leaked into telemetry: {line}" + ); + } +} diff --git a/docs/PATH-TO-MANUAL-USE.md b/docs/PATH-TO-MANUAL-USE.md index 072f535..fb2c599 100644 --- a/docs/PATH-TO-MANUAL-USE.md +++ b/docs/PATH-TO-MANUAL-USE.md @@ -51,6 +51,34 @@ cargo run -p nunciod # gRPC on 127.0.0.1:9420 cargo run -p nuncio-cli -- system status ``` +### Driving the CLI headlessly + +`nunciod` mints the gRPC bearer token into the OS keyring, so the keychain +item's ACL trusts **`nunciod`**. `nuncio-cli` is a different binary, so reading +the same item makes macOS ask the logged-in user for consent (*"nuncio-cli +wants to access key … in your keychain"*). Interactively you click Allow. In CI, +over ssh, or in a background shell there is nobody to click it — the read +blocks, and before this was fixed the CLI hung with no output at all. + +Set `NUNCIO_GRPC_TOKEN` to bypass the keyring for the CLI only: + +```bash +NUNCIO_GRPC_TOKEN=<token> cargo run -p nuncio-cli -- system status +``` + +When it is set and non-empty the CLI uses it verbatim and never touches the +keyring. When it is unset, behaviour is unchanged: the token still comes from +the keyring, now behind a bounded timeout that reports an actionable error +instead of stalling forever. The token is the daemon's hex-encoded +`grpc-bearer-token` keychain item; read it once from a session that can answer +the prompt (`security find-generic-password -s mx.nuncio.vault -a +grpc-bearer-token -w`) and pass it through your own secret store. It is a +credential: never commit it, never log it, never echo it into a shell history +file. + +The daemon itself is unaffected — it *creates* the item rather than reading one +another binary owns, so its own startup raises no consent prompt. + Schema migrations were removed: databases are **created, never migrated**. A store from before that change errors on read rather than upgrading. Deleting it is correct — accounts and credentials live in the OS keyring and survive; cached -- 2.54.0 From 9daec8a7316afcc6d322de3429963a77ec59ed01 Mon Sep 17 00:00:00 2001 From: James Maes <james@kof22.com> Date: Fri, 14 Aug 2026 20:05:13 -0500 Subject: [PATCH 95/95] fix(test): close the store pool before salvaging its database test_stage_2_backup_creation_and_stage_3_table_salvage built its fixture engine inside a block and let it drop rather than closing it. A dropped sqlx pool tears its connections down asynchronously, so SQLite's close-time WAL handling -- which operates on -wal/-shm by path -- could land after salvage had unlinked those paths and created a fresh database at the same location. That pairs the new, empty main file with a -wal describing the old one, and replaying it reads past end-of-file: SQLITE_IOERR_SHORT_READ (522), surfaced as the observed Query(Database(SqliteError { code: 522, message: "disk I/O error" })). close() awaits the teardown, so the window ceases to exist rather than merely narrowing. This mirrors what the production caller (DatabaseEngine::close_and_salvage) already does and what the sibling stage-1 detection test already did. Record the precondition on SqliteRecoveryEngine::salvage so the next caller does not repeat it. --- crates/nuncio-store/src/recovery.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/nuncio-store/src/recovery.rs b/crates/nuncio-store/src/recovery.rs index 324a8a6..0f267e7 100644 --- a/crates/nuncio-store/src/recovery.rs +++ b/crates/nuncio-store/src/recovery.rs @@ -179,6 +179,18 @@ impl SqliteRecoveryEngine { /// Salvage readable accounts and filter rules from damaged database or backup file, /// re-creating a fresh SQLite file at `target_db_path`. Cryptographic key material for /// the freshly re-created engine is provisioned from `secrets`. + /// + /// # Precondition + /// + /// The caller MUST have fully closed (via [`DatabaseEngine::close`], which awaits every + /// pooled connection's teardown) any engine still holding `target_db_path` before calling + /// this. Salvage unlinks `target_db_path` and its `-wal`/`-shm` companions and then creates + /// a brand-new database at that same path; a connection left open over the old file is torn + /// down asynchronously, and SQLite's close-time WAL handling operates on `-wal`/`-shm` *by + /// path*. A teardown landing after the unlink therefore collides with the fresh database's + /// own companions -- leaving the new, empty main file paired with a `-wal` describing the + /// old one, whose replay reads past end-of-file and fails with `SQLITE_IOERR_SHORT_READ`. + /// `DatabaseEngine::close_and_salvage` is the in-tree caller that honours this. pub async fn salvage( corrupted_db_path: &Path, target_db_path: &Path, @@ -530,6 +542,17 @@ mod tests { let nsql = "SELECT * FROM emails WHERE subject CONTAINS 'Spam' ACTION DELETE"; let rule = nuncio_filter::NsqlParser::parse_rule("Spam Filter", 1, nsql).unwrap(); engine.save_filter_rule(&rule).await.unwrap(); + + // Closing before salvage is the documented precondition of + // `SqliteRecoveryEngine::salvage`, and it is what the production caller + // (`DatabaseEngine::close_and_salvage`) does. Merely dropping the engine is not + // equivalent: the pool's teardown is deferred onto the runtime, so the old + // connection's close-time WAL handling can land *after* salvage has unlinked + // `-wal`/`-shm` and created a fresh database at this same path -- pairing the new, + // empty main file with a `-wal` describing the old one and failing the subsequent + // page read with `SQLITE_IOERR_SHORT_READ`. `close()` awaits that teardown, so the + // window does not exist at all rather than merely being narrow. + engine.close().await; } // Step 2: Perform salvage recovery -- 2.54.0