feat(filter): add streaming Filters.Triage RPC for retroactive rescan #226

Merged
KofTwentyTwo merged 1 commits from story/m4-filters-triage into dev 2026-07-30 00:54:30 +00:00
7 changed files with 592 additions and 5 deletions

No files matched your search

+12
View File
@@ -517,6 +517,18 @@ pub enum FilterSubcommand {
#[arg(short, long, default_value_t = 50, help = "Log limit")]
limit: usize,
},
/// Retroactively apply the current rule set to every stored message,
/// streaming cumulative progress as the scan runs.
Triage {
/// Restrict the scan's reporting to a single rule ID. Not yet
/// honored by the daemon (see `nuncio.v1.Filters.Triage`) -- the
/// scan always evaluates every active rule.
#[arg(short, long, help = "Rule identifier (not yet honored)")]
rule_id: Option<String>,
/// Messages fetched per streamed progress update.
#[arg(short, long, help = "Messages scanned per chunk")]
chunk_size: Option<u32>,
},
}
#[cfg(test)]
+146 -2
View File
@@ -512,6 +512,13 @@ impl HeadlessRunner {
FilterSubcommand::Logs { limit } => {
self.handle_filter_logs(*limit, json_mode).await
}
FilterSubcommand::Triage {
rule_id,
chunk_size,
} => {
self.handle_filter_triage(rule_id.clone(), *chunk_size, json_mode)
.await
}
},
Commands::Update { action } => match action {
UpdateSubcommand::Check => match nuncio_core::UpdateEngine::new() {
@@ -1484,6 +1491,93 @@ impl HeadlessRunner {
}
}
/// Drains the server-streaming `Filters.Triage` RPC, printing (or
/// accumulating, in `--json` mode) each `TriageProgress` update as it
/// arrives so the caller can watch a retroactive rescan of the whole
/// message store progress in real time rather than blocking silently
/// until it finishes.
async fn handle_filter_triage(
&self,
rule_id: Option<String>,
chunk_size: Option<u32>,
json_mode: bool,
) -> String {
let mut client = match self.connect_filters_client().await {
Ok(client) => client,
Err(e) => return Self::render_error(&e, json_mode),
};
let mut stream = match client
.triage(nuncio_proto::v1::TriageRequest {
rule_id,
chunk_size: chunk_size.unwrap_or(0),
})
.await
{
Ok(response) => response.into_inner(),
Err(status) => {
return Self::render_error(
&format!("nunciod daemon rejected triage: {status}"),
json_mode,
)
}
};
let mut updates = Vec::new();
loop {
match stream.message().await {
Ok(Some(progress)) => {
if !json_mode {
println!(
"scanned={} matched={} actions_applied={} last_message_id={} done={}",
progress.scanned_count,
progress.matched_count,
progress.actions_applied_count,
progress.last_message_id,
progress.done
);
}
let done = progress.done;
updates.push(progress);
if done {
break;
}
}
Ok(None) => break,
Err(status) => {
return Self::render_error(
&format!("nunciod daemon triage stream failed: {status}"),
json_mode,
)
}
}
}
if json_mode {
let updates_json: Vec<serde_json::Value> = updates
.iter()
.map(|p| {
json!({
"scanned_count": p.scanned_count,
"matched_count": p.matched_count,
"actions_applied_count": p.actions_applied_count,
"last_message_id": p.last_message_id,
"done": p.done,
})
})
.collect();
format_json(&json!({ "updates": updates_json }))
} else {
match updates.last() {
Some(last) => format!(
"Triage complete: scanned={} matched={} actions_applied={}",
last.scanned_count, last.matched_count, last.actions_applied_count
),
None => "Triage produced no progress updates.".to_string(),
}
}
}
/// Resolves the gRPC bearer token from the injected vault and dials the
/// `nuncio.v1.Filters` service at `self.grpc_addr`, shared by every
/// `filter` handler above.
@@ -3235,8 +3329,8 @@ mod tests {
ExportRulesRequest, ExportRulesResponse, FilterExecutionLog as FilterExecutionLogProto,
FilterRule as FilterRuleProto, GetExecutionLogsRequest, GetExecutionLogsResponse,
ImportRulesRequest, ImportRulesResponse, ListRulesRequest, ListRulesResponse,
PreviewRuleRequest, PreviewRuleResponse, UpdateRuleRequest, UpdateRuleResponse,
ValidateRuleRequest, ValidateRuleResponse,
PreviewRuleRequest, PreviewRuleResponse, TriageProgress, TriageRequest,
UpdateRuleRequest, UpdateRuleResponse, ValidateRuleRequest, ValidateRuleResponse,
};
use std::sync::Mutex;
@@ -3407,6 +3501,37 @@ mod tests {
}],
}))
}
type TriageStream = std::pin::Pin<
Box<
dyn tokio_stream::Stream<Item = Result<TriageProgress, tonic::Status>>
+ Send
+ 'static,
>,
>;
async fn triage(
&self,
_request: tonic::Request<TriageRequest>,
) -> Result<tonic::Response<Self::TriageStream>, tonic::Status> {
let updates = vec![
Ok(TriageProgress {
scanned_count: 2,
matched_count: 1,
actions_applied_count: 1,
last_message_id: "msg-stub-2".to_string(),
done: false,
}),
Ok(TriageProgress {
scanned_count: 3,
matched_count: 2,
actions_applied_count: 2,
last_message_id: "msg-stub-3".to_string(),
done: true,
}),
];
Ok(tonic::Response::new(Box::pin(tokio_stream::iter(updates))))
}
}
let stub = StubFilters::default();
@@ -3647,6 +3772,25 @@ mod tests {
.await;
assert!(logs_out.contains("msg-stub-1"));
assert!(logs_out.contains(r#""action_taken":"MARK READ""#));
// `filter triage`: proves the CLI drains the server-streaming
// `Triage` RPC (its first real streaming-RPC consumer) and reports
// the final cumulative progress rather than only the first chunk.
let triage_out = runner
.execute_command(
&Commands::Filter {
action: FilterSubcommand::Triage {
rule_id: None,
chunk_size: Some(2),
},
},
true,
)
.await;
assert!(triage_out.contains(r#""scanned_count":3"#));
assert!(triage_out.contains(r#""matched_count":2"#));
assert!(triage_out.contains(r#""actions_applied_count":2"#));
assert!(triage_out.contains(r#""done":true"#));
}
#[tokio::test]
Binary file not shown.
@@ -479,6 +479,13 @@ service Filters {
// log entries (hash-chain ledger), newest-eligible-first as stored,
// capped at `limit`.
rpc GetExecutionLogs(GetExecutionLogsRequest) returns (GetExecutionLogsResponse);
// Triage scans every stored message in bounded chunks, evaluating the
// daemon's live `FilterEngine` against each one and applying the same
// actions live sync would -- so the currently active rule set can be
// applied retroactively to mail that was synced before those rules
// existed. Streams cumulative progress as it works, ending with a final
// `done: true` message once the whole store has been scanned.
rpc Triage(TriageRequest) returns (stream TriageProgress);
}
// FilterRule mirrors `nuncio_filter::FilterRule`'s persisted/user-facing
@@ -675,6 +682,36 @@ message FilterExecutionLog {
string hash = 7;
}
// TriageRequest configures a retroactive rescan of the whole message store.
message TriageRequest {
// Scope the scan to a single rule's actions. Currently NOT honored --
// `Triage` always evaluates a message against the full live rule set,
// matching what a real (non-retroactive) sync run would do. Reserved for
// forward compatibility; a client that sets this today gets a whole-store
// scan against all active rules, not an error.
optional string rule_id = 1;
// Number of messages fetched per keyset page. 0 selects the server
// default (100).
uint32 chunk_size = 2;
}
// TriageProgress reports the real, cumulative state of an in-progress (or
// just-finished) `Triage` scan. Every field is the true running total as of
// the chunk just processed -- never an estimate or a fabricated count.
message TriageProgress {
// Total messages scanned so far.
uint64 scanned_count = 1;
// Total messages that had at least one action applied or enqueued.
uint64 matched_count = 2;
// Total individual actions applied or enqueued across all messages.
uint64 actions_applied_count = 3;
// ID of the last message scanned in the most recently processed chunk.
string last_message_id = 4;
// True only on the final message of the stream, once the whole store has
// been scanned.
bool done = 5;
}
// Export exposes the daemon's real mailbox export engine
// (`nuncio_core::export`, via `DatabaseEngine::export_messages_to_file`)
// over authenticated gRPC. The daemon and its caller run on the SAME host
+100 -3
View File
@@ -40,8 +40,9 @@ use nuncio_proto::v1::{
Message as MessageProto, MessageFlagsChanged, MessageSearchHit, PreviewRuleRequest,
PreviewRuleResponse, SearchMessagesRequest, SearchMessagesResponse, SendMessageRequest,
SendMessageResponse, ShuttingDown, SubscribeRequest, SyncCompleted, SyncRequest, SyncResponse,
SyncStarted, TlsMode as TlsModeProto, UpdateAvailable, UpdateRuleRequest, UpdateRuleResponse,
ValidateRuleRequest, ValidateRuleResponse, VerifyChainRequest, VerifyChainResponse,
SyncStarted, TlsMode as TlsModeProto, TriageProgress, TriageRequest, UpdateAvailable,
UpdateRuleRequest, UpdateRuleResponse, ValidateRuleRequest, ValidateRuleResponse,
VerifyChainRequest, VerifyChainResponse,
};
use nuncio_store::db::DatabaseEngine;
use nuncio_store::search::SearchEngine;
@@ -53,7 +54,7 @@ use subtle::ConstantTimeEq;
use thiserror::Error;
use tokio::net::TcpListener;
use tokio_stream::wrappers::errors::BroadcastStreamRecvError;
use tokio_stream::wrappers::{BroadcastStream, TcpListenerStream};
use tokio_stream::wrappers::{BroadcastStream, ReceiverStream, TcpListenerStream};
use tokio_stream::{Stream, StreamExt};
use tonic::transport::Server;
use tonic::{Request, Response, Status};
@@ -1472,6 +1473,102 @@ impl Filters for FiltersGrpcService {
Ok(Response::new(GetExecutionLogsResponse { logs }))
}
/// Server-streaming retroactive rescan of the whole message store.
/// Unlike `System::subscribe` (which wraps an already-live broadcast),
/// this RPC actively drives the scan: a spawned worker task walks
/// `db.get_message_chunk` in keyset pages, applies the shared
/// [`crate::sync::apply_filter_actions`] evaluate-and-route logic to
/// each message (the SAME side effects live sync produces), and pushes
/// cumulative progress onto an mpsc channel that this method wraps as
/// the returned stream.
///
/// `rule_id` is accepted for forward compatibility but NOT currently
/// honored: `apply_filter_actions` always evaluates against the whole
/// live rule set, matching what a real sync pass would do, rather than
/// silently narrowing to one rule.
///
/// A `get_message_chunk` failure sends a single `Err(Status::internal)`
/// on the stream and stops the scan -- never a fabricated `done: true`
/// on error. The receiver disconnecting (the caller dropped the call)
/// also stops the worker rather than continuing to scan for no one.
type TriageStream =
Pin<Box<dyn Stream<Item = Result<TriageProgress, Status>> + Send + 'static>>;
async fn triage(
&self,
request: Request<TriageRequest>,
) -> Result<Response<Self::TriageStream>, Status> {
let req = request.into_inner();
let chunk_size = if req.chunk_size == 0 {
100
} else {
req.chunk_size as usize
};
let (tx, rx) = tokio::sync::mpsc::channel::<Result<TriageProgress, Status>>(16);
let db = self.db.clone();
let engine = self.filter_engine.clone();
tokio::spawn(async move {
let mut last_id = String::new();
let mut scanned: u64 = 0;
let mut matched: u64 = 0;
let mut applied: u64 = 0;
loop {
let batch = match db.get_message_chunk(&last_id, chunk_size).await {
Ok(batch) => batch,
Err(e) => {
let _ = tx
.send(Err(Status::internal(format!(
"failed to read message chunk during triage: {e}"
))))
.await;
return;
}
};
let is_last_page = batch.len() < chunk_size;
for email in &batch {
let actions_for_email =
crate::sync::apply_filter_actions(&db, &engine, email).await;
scanned += 1;
applied += actions_for_email as u64;
if actions_for_email > 0 {
matched += 1;
}
}
if let Some(last) = batch.last() {
last_id = last.id.clone();
}
let done = is_last_page;
if tx
.send(Ok(TriageProgress {
scanned_count: scanned,
matched_count: matched,
actions_applied_count: applied,
last_message_id: last_id.clone(),
done,
}))
.await
.is_err()
{
// Receiver dropped: the caller disconnected, so there is
// no one left to report progress to.
return;
}
if is_last_page {
return;
}
}
});
Ok(Response::new(Box::pin(ReceiverStream::new(rx))))
}
}
/// Maps a wire-format `nuncio.v1.ExportFormat` enum value back onto
@@ -0,0 +1,267 @@
//! Offline end-to-end test for the `nuncio.v1.Filters.Triage` streaming RPC.
//!
//! Boots a real `nunciod` daemon gRPC server (`nunciod::grpc::
//! serve_on_listener`) on an ephemeral loopback port, backed by a real
//! temp file-backed [`DatabaseEngine`] seeded directly via
//! [`DatabaseEngine::save_email`] (standing in for messages that were synced
//! before the active rule set existed) and a live [`FilterEngine`] carrying
//! a real persisted rule.
//!
//! Drives ONLY the authenticated `nuncio-proto` `FiltersClient::triage` call
//! against the live server, proving:
//! a. the final cumulative `matched_count`/`actions_applied_count` are
//! correct against a store containing both matching and non-matching
//! messages;
//! b. matching messages show the REAL side effects `apply_filter_actions`
//! produces (the read flag flips for an immediate `MARK READ` action;
//! an outbox row is queued for a remote `MOVE TO` action) -- not a
//! fabricated count with no backing state change;
//! c. a store bigger than one `chunk_size` genuinely streams more than one
//! non-final `TriageProgress` message, proving this is a real
//! multi-chunk stream rather than one batch dressed up as a stream.
//!
//! Fully offline and deterministic: no real network, no real OS keyring
//! (`SecretManager::mock`), no fixed sleeps.
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use nuncio_core::model::Email;
use nuncio_core::EventBus;
use nuncio_filter::{FilterEngine, NsqlParser};
use nuncio_proto::v1::filters_client::FiltersClient;
use nuncio_proto::v1::TriageRequest;
use nuncio_store::db::DatabaseEngine;
use nuncio_store::vault::{SecretManager, GRPC_TOKEN_ACCOUNT};
use nunciod::grpc::serve_on_listener;
use std::sync::Arc;
use tokio::net::TcpListener;
const ACCOUNT_ID: &str = "acct-triage-e2e-1";
fn sample_email(id: &str, subject: &str) -> Email {
Email {
id: id.to_string(),
account_id: ACCOUNT_ID.to_string(),
folder_id: "inbox".to_string(),
subject: subject.to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "triage-e2e@nuncio.mx".to_string(),
received_at: 1_700_000_000,
read: false,
body_plain: Some("body".to_string()),
body_html: None,
attachments: Vec::new(),
}
}
/// Boots the real daemon gRPC server (bearer-token authenticated) on an
/// ephemeral loopback port, backed by a real temp file-backed
/// `DatabaseEngine` and the given live `FilterEngine`. Returns the server
/// address, the minted token, the `DatabaseEngine` handle (used to seed
/// messages before triggering `Triage`), and the `TempDir` guard, which the
/// caller must hold for as long as it talks to the server.
async fn boot_daemon(
secrets: Arc<SecretManager>,
filter_engine: Arc<FilterEngine>,
) -> (
std::net::SocketAddr,
String,
Arc<DatabaseEngine>,
tempfile::TempDir,
) {
let dir = tempfile::tempdir().expect("create temp dir for file-backed db");
let db_path = dir.path().join("triage_e2e.db");
let db = Arc::new(
DatabaseEngine::connect_file(&db_path, &secrets)
.await
.expect("open temp file-backed database"),
);
let token_bytes = secrets
.get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32)
.expect("mint gRPC bearer token from mock vault");
let token = hex::encode(token_bytes);
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind ephemeral loopback port");
let addr = listener.local_addr().expect("listener has local addr");
let event_bus = Arc::new(EventBus::new());
let server_db = db.clone();
let server_secrets = secrets.clone();
let server_token = token.clone();
tokio::spawn(async move {
let _ = serve_on_listener(
listener,
event_bus,
server_db,
filter_engine,
server_secrets,
server_token,
)
.await;
});
(addr, token, db, dir)
}
fn authed_request<T>(token: &str, message: T) -> tonic::Request<T> {
let mut request = tonic::Request::new(message);
request.metadata_mut().insert(
"authorization",
format!("Bearer {token}")
.parse()
.expect("valid ascii metadata value"),
);
request
}
#[tokio::test]
async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_counts() {
let secrets = Arc::new(SecretManager::mock());
let mark_read_rule = NsqlParser::parse_rule(
"Urgent Auto-Read",
1,
"WHERE subject CONTAINS 'Urgent' ACTION MARK READ",
)
.expect("parse mark-read rule");
let move_rule = NsqlParser::parse_rule(
"Archive Rule",
2,
"WHERE subject CONTAINS 'Archive Me' ACTION MOVE TO 'Archive'",
)
.expect("parse move rule");
let filter_engine =
Arc::new(FilterEngine::new(vec![mark_read_rule, move_rule]).expect("compile rules"));
let (addr, token, db, _dir) = boot_daemon(secrets, filter_engine).await;
// Seed messages directly into the store, standing in for mail that was
// synced BEFORE these rules existed -- exactly the scenario `Triage` is
// for. Two match (one per rule), one does not.
db.save_email(&sample_email("m-urgent", "Urgent: server down"))
.await
.expect("seed urgent message");
db.save_email(&sample_email("m-archive", "Please Archive Me"))
.await
.expect("seed archive message");
db.save_email(&sample_email("m-plain", "Just saying hi"))
.await
.expect("seed non-matching message");
let mut client = FiltersClient::connect(format!("http://{addr}"))
.await
.expect("client connects");
let mut stream = client
.triage(authed_request(
&token,
TriageRequest {
rule_id: None,
chunk_size: 100,
},
))
.await
.expect("triage call succeeds")
.into_inner();
let mut updates = Vec::new();
while let Some(progress) = stream.message().await.expect("stream message") {
updates.push(progress);
}
let last = updates.last().expect("at least one progress update");
assert!(last.done, "final update must be marked done");
assert_eq!(last.scanned_count, 3);
assert_eq!(
last.matched_count, 2,
"exactly the two messages with a rule match count as matched"
);
assert_eq!(
last.actions_applied_count, 2,
"one action applied per matching message"
);
// Real side effects, not a fabricated count: the MARK READ action
// actually flipped the stored read flag...
let urgent = db.get_message("m-urgent").await.expect("get urgent");
assert!(urgent.read, "MARK READ must genuinely flip the read flag");
// ...and the MOVE TO action actually queued a real outbox mutation.
let pending = db
.list_pending_mutations(10)
.await
.expect("list pending mutations");
assert_eq!(
pending.len(),
1,
"the remote MOVE action must enqueue exactly one outbox mutation"
);
assert_eq!(pending[0].message_id, "m-archive");
let plain = db.get_message("m-plain").await.expect("get plain");
assert!(!plain.read, "a non-matching message must be left untouched");
}
#[tokio::test]
async fn triage_streams_multiple_chunks_for_a_store_bigger_than_one_chunk() {
let secrets = Arc::new(SecretManager::mock());
let filter_engine = Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set"));
let (addr, token, db, _dir) = boot_daemon(secrets, filter_engine).await;
// Five messages against a chunk size of two forces three chunks
// (2 + 2 + 1), so the stream must carry at least two non-final progress
// updates before the final `done: true` -- proof this is a real
// multi-chunk stream, not one batch disguised as one.
for i in 0..5 {
db.save_email(&sample_email(
&format!("m-{i:02}"),
&format!("Message number {i}"),
))
.await
.expect("seed message");
}
let mut client = FiltersClient::connect(format!("http://{addr}"))
.await
.expect("client connects");
let mut stream = client
.triage(authed_request(
&token,
TriageRequest {
rule_id: None,
chunk_size: 2,
},
))
.await
.expect("triage call succeeds")
.into_inner();
let mut updates = Vec::new();
while let Some(progress) = stream.message().await.expect("stream message") {
updates.push(progress);
}
let non_final: Vec<_> = updates.iter().filter(|p| !p.done).collect();
assert!(
non_final.len() > 1,
"expected more than one non-final progress update for a multi-chunk scan, got {}",
non_final.len()
);
let last = updates.last().expect("at least one progress update");
assert!(last.done);
assert_eq!(last.scanned_count, 5);
// Cumulative counts must monotonically increase across the stream --
// never reset or double-count between chunks.
let mut prev_scanned = 0u64;
for update in &updates {
assert!(update.scanned_count >= prev_scanned);
prev_scanned = update.scanned_count;
}
}
+30
View File
@@ -187,3 +187,33 @@ async fn rejects_calls_to_new_filters_rpcs_without_a_bearer_token() {
.expect_err("call without any authorization header must be rejected");
assert_eq!(err.code(), Code::Unauthenticated);
}
/// Confirms the streaming `Filters.Triage` RPC is mounted behind the SAME
/// `BearerAuthInterceptor` as every other `Filters` RPC -- an unauthenticated
/// call must be rejected before the stream is ever established, exactly
/// like an unauthenticated unary call.
#[tokio::test]
async fn rejects_triage_calls_without_a_bearer_token() {
let event_bus = Arc::new(EventBus::new());
let secrets = SecretManager::mock();
let token = hex::encode(
secrets
.get_or_create_key_bytes(GRPC_TOKEN_ACCOUNT, 32)
.expect("token provisioned from mock vault"),
);
let (addr, _dir) = start_server(event_bus, token).await;
let mut client =
nuncio_proto::v1::filters_client::FiltersClient::connect(format!("http://{addr}"))
.await
.expect("client connects");
let err = client
.triage(nuncio_proto::v1::TriageRequest {
rule_id: None,
chunk_size: 0,
})
.await
.expect_err("call without any authorization header must be rejected");
assert_eq!(err.code(), Code::Unauthenticated);
}