perf(grpc): batch ListMessages placements; fix JMAP parse folder #418

Merged
KofTwentyTwo merged 2 commits from perf/list-messages-placements into dev 2026-08-15 00:51:49 +00:00
5 changed files with 174 additions and 34 deletions

No files matched your search

+13 -15
View File
@@ -451,18 +451,14 @@ impl JmapEngine {
}
}
/// Parse a raw JMAP `Email/get` JSON response payload into a list of
/// [`PlacedMessage`] and the new state string.
pub fn parse_email_get_response(
&self,
raw_json: &str,
) -> Result<(Vec<PlacedMessage>, String), MailError> {
self.parse_email_get_response_in_folder(raw_json, "inbox")
}
/// Parse a raw JMAP `Email/get` JSON response payload into the occupancies
/// it establishes for `folder_id`, plus the account's Email state string.
///
/// The caller supplies the mailbox being parsed for because the placements
/// this produces are keyed by it: a placement labelled with a folder the
/// account does not sync can never be matched again, so a fixed label would
/// silently strand every message it produced.
///
/// A returned message whose `mailboxIds` is present and does NOT name
/// `folder_id` is dropped rather than placed: `Email/changes` is
/// account-wide (RFC 8621 section 4.2 offers no mailbox filter), so a
@@ -470,7 +466,7 @@ impl JmapEngine {
/// here would invent an occupancy the server never reported. Membership
/// that the server did not return at all is left alone -- unknown is not
/// absent.
fn parse_email_get_response_in_folder(
pub fn parse_email_get_response(
&self,
raw_json: &str,
folder_id: &str,
@@ -765,7 +761,7 @@ impl JmapEngine {
let get_request = Self::build_email_get_request(jmap_account_id, Some(ids));
let get_raw = self.post_jmap(api_url, &get_request).await?;
let (emails, state) = self.parse_email_get_response_in_folder(&get_raw, folder_id)?;
let (emails, state) = self.parse_email_get_response(&get_raw, folder_id)?;
let present: Vec<PlacementKey> = emails.iter().map(|m| m.placement.key()).collect();
@@ -810,7 +806,7 @@ impl JmapEngine {
let get_request =
Self::build_email_get_request(jmap_account_id, Some(changes.changed.clone()));
let get_raw = self.post_jmap(api_url, &get_request).await?;
let (emails, _state) = self.parse_email_get_response_in_folder(&get_raw, folder_id)?;
let (emails, _state) = self.parse_email_get_response(&get_raw, folder_id)?;
emails
};
@@ -1118,7 +1114,7 @@ mod tests {
}"#;
let engine = JmapEngine::new("acct-1");
let (emails, _state) = engine
.parse_email_get_response_in_folder(raw, "mb-inbox")
.parse_email_get_response(raw, "mb-inbox")
.expect("parse get");
let ids: Vec<&str> = emails
.iter()
@@ -1142,7 +1138,9 @@ mod tests {
]
}"#;
let engine = JmapEngine::new("acct-1");
let (emails, _state) = engine.parse_email_get_response(raw).expect("parse get");
let (emails, _state) = engine
.parse_email_get_response(raw, "mb-inbox")
.expect("parse get");
assert_eq!(
emails[0].email.message_id.as_deref(),
@@ -1160,7 +1158,7 @@ mod tests {
#[test]
fn parse_invalid_jmap_json_fails() {
let engine = JmapEngine::new("acct-1");
let res = engine.parse_email_get_response("{ invalid json }");
let res = engine.parse_email_get_response("{ invalid json }", "mb-inbox");
assert!(res.is_err());
}
@@ -193,7 +193,7 @@ async fn an_imap_emailid_and_a_jmap_object_id_produce_the_same_key() {
"receivedAt":1700001000,"isUnread":true}}]}},"c1"]]}}"#
);
let (jmap, _state) = JmapEngine::new(ACCOUNT)
.parse_email_get_response(&jmap_response)
.parse_email_get_response(&jmap_response, "mb-inbox")
.expect("the JMAP response parses");
assert_eq!(imap.upserts.len(), 1);
@@ -81,7 +81,7 @@ async fn wiremock_jmap_session_discovery_and_email_get_sync() {
let engine = JmapEngine::new("acct-100");
let (emails, new_state) = engine
.parse_email_get_response(&resp.text().await.unwrap())
.parse_email_get_response(&resp.text().await.unwrap(), "mb-inbox")
.expect("parse email get");
assert_eq!(emails.len(), 1);
+142
View File
@@ -1777,6 +1777,71 @@ impl DatabaseEngine {
.collect())
}
/// Every mailbox each of `message_keys` currently occupies, grouped by key.
///
/// The batched form of [`Self::placements_of`], for callers holding a whole
/// page of messages: one query per chunk instead of one per message. Keys
/// absent from the result occupy no mailbox; callers that need an entry for
/// every requested key should treat a missing entry as an empty set.
///
/// Each key's placements are ordered by folder id ascending, exactly as
/// [`Self::placements_of`] orders them -- callers take the first placement
/// as the message's canonical mailbox, so the order is part of the
/// contract, not an incidental property of the query plan.
///
/// Chunked at [`MAX_KEYS_PER_IN_LIST`] for the same reason as
/// [`Self::existing_placements`]: the bind-parameter ceiling belongs to the
/// statement, so it can only be enforced where the statement is built.
/// Duplicate keys are collapsed first, so no key's placements can be
/// gathered from two chunks and appended twice.
pub async fn placements_of_batch(
&self,
message_keys: &[String],
) -> Result<std::collections::HashMap<String, Vec<nuncio_core::model::Placement>>, DatabaseError>
{
let mut seen = std::collections::HashSet::new();
let unique: Vec<&String> = message_keys
.iter()
.filter(|key| seen.insert(key.as_str()))
.collect();
let mut grouped: std::collections::HashMap<String, Vec<nuncio_core::model::Placement>> =
std::collections::HashMap::new();
for chunk in unique.chunks(MAX_KEYS_PER_IN_LIST) {
let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new(
"SELECT message_key, account_id, folder_id, uidvalidity, uid, read_flag
FROM placements WHERE message_key IN (",
);
let mut separated = builder.separated(", ");
for key in chunk {
separated.push_bind(*key);
}
builder.push(") ORDER BY message_key ASC, folder_id ASC");
let rows: Vec<(String, String, String, String, String, i64)> = builder
.build_query_as()
.fetch_all(&self.pool)
.await
.map_err(DatabaseError::Query)?;
for (message_key, account_id, folder_id, uid_validity, remote_id, read_flag) in rows {
grouped
.entry(message_key)
.or_default()
.push(nuncio_core::model::Placement {
account_id,
folder_id,
uid_validity,
remote_id,
read: read_flag != 0,
});
}
}
Ok(grouped)
}
/// Messages occupying one folder of one account, newest first, paired with
/// the placement they were found through.
///
@@ -5494,6 +5559,83 @@ mod tests {
assert_eq!(known.len(), 1, "and the query must not echo its own input");
}
#[tokio::test]
async fn placements_of_batch_matches_per_message_placements_of_including_order() {
let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap();
// Folders are saved in an order that is neither insertion-sorted nor
// reverse-sorted, so a batch query that forgot to order by folder id
// would come back visibly wrong rather than accidentally right.
for (key, folder, uid) in [
("key-a", "Zeta", "1"),
("key-a", "Alpha", "2"),
("key-a", "Mid", "3"),
("key-b", "Sent", "4"),
("key-b", "Archive", "5"),
// Placed but never asked for: proves the `IN (...)` filter is real.
("key-unrequested", "INBOX", "6"),
] {
let (email, placement) = sample_message_and_placement(key, folder, uid);
engine
.save_email_at(
&email,
nuncio_core::model::IdentitySource::EmailId,
&placement,
)
.await
.unwrap();
}
// "key-a" appears twice: a caller may hand over a page with repeats,
// and a key gathered from two chunks must not be appended twice.
let requested = [
"key-a".to_string(),
"key-b".to_string(),
"key-a".to_string(),
"key-never-saved".to_string(),
];
let grouped = engine.placements_of_batch(&requested).await.unwrap();
for key in ["key-a", "key-b"] {
let one_at_a_time = engine.placements_of(key).await.unwrap();
assert!(
one_at_a_time.len() > 1 || key == "key-b",
"the fixture must exercise more than one placement per message"
);
assert_eq!(
grouped.get(key),
Some(&one_at_a_time),
"the batched read of '{key}' must equal the per-message read, element for element"
);
let folders: Vec<&str> = one_at_a_time
.iter()
.map(|placement| placement.folder_id.as_str())
.collect();
let mut sorted = folders.clone();
sorted.sort_unstable();
assert_eq!(
folders, sorted,
"the first placement of '{key}' is taken as its canonical mailbox, so folder id order is load-bearing"
);
}
assert_eq!(
grouped.get("key-a").map(Vec::len),
Some(3),
"a repeated key must not have its placements gathered twice"
);
assert!(
!grouped.contains_key("key-never-saved"),
"a key that occupies no mailbox must be absent, not invented"
);
assert!(
!grouped.contains_key("key-unrequested"),
"the batch must answer only for the keys it was asked about"
);
assert_eq!(grouped.len(), 2, "and for nothing else");
}
/// A first sync of a real mailbox hands `existing_placements` the entire
/// folder in one call. Each key spends four bind parameters, and SQLite
/// caps a statement at 32766 of them, so an unchunked `IN (...)` list dies
+17 -17
View File
@@ -1025,25 +1025,25 @@ fn map_placement_ref_from_proto(placement: PlacementRefProto) -> nuncio_core::mo
/// Fills each message's complete placement set from the store.
///
/// This costs one query per message rather than one per page. That is the
/// honest price of the field: occupancy is per-message state and a page can mix
/// messages that sit in one mailbox with messages that sit in ten. A failure to
/// read one message's placements leaves that message's set empty rather than
/// failing the whole listing, and is logged -- a partial page is more useful
/// than none, and the field is additive so a client that ignores it is
/// unaffected.
/// One batched query for the whole page rather than one per message: occupancy
/// is per-message state, but reading it per message makes a page of fifty cost
/// fifty-one round trips, which shows up directly as page latency. A failure to
/// read the batch leaves every set empty rather than failing the whole listing,
/// and is logged -- a page without placements is more useful than none, and the
/// field is additive so a client that ignores it is unaffected.
async fn fill_placements(db: &DatabaseEngine, messages: &mut [MessageProto]) {
let keys: Vec<String> = messages.iter().map(|message| message.id.clone()).collect();
let mut grouped = match db.placements_of_batch(&keys).await {
Ok(grouped) => grouped,
Err(e) => {
tracing::warn!("Mail: failed to read the placement sets for this page: {e}");
return;
}
};
for message in messages.iter_mut() {
match db.placements_of(&message.id).await {
Ok(placements) => {
message.placements = placements.into_iter().map(map_placement_to_proto).collect();
}
Err(e) => {
tracing::warn!(
message_id = %message.id,
"Mail: failed to read the placement set: {e}"
);
}
if let Some(placements) = grouped.remove(&message.id) {
message.placements = placements.into_iter().map(map_placement_to_proto).collect();
}
}
}