fix(store): correct corruption classification, salvage schema; de-flake recovery
Data-loss fix (1.B.4): open_with_backup_dir no longer coerces transient errors (SQLITE_BUSY/LOCKED/IOERR/CANTOPEN, pool timeouts) into 'corrupt' -- it now propagates them without salvaging or deleting the live DB. is_sqlite_corruption_error is a strict allowlist of SQLITE_CORRUPT/NOTADB (primary code & 0xff) plus a deterministic raw-header-magic pre-flight check. Salvage SQL now matches the real filter_rules/conditions/actions schema, so rules are actually restored (1.B.3). De-flake #155: root cause was a deferred WAL checkpoint healing the test's header corruption ~10-50ms after close; close() now runs wal_checkpoint(TRUNCATE) before pool close. Test re-enabled and stable (130 runs, 0 failures). Refs #154 #155
This commit is contained in:
3 files changed
+355
-58
No files matched your search
+208
-17
@@ -39,7 +39,15 @@ pub enum DatabaseError {
|
||||
}
|
||||
|
||||
impl DatabaseError {
|
||||
/// Returns true if this error indicates database file corruption or un-openable state.
|
||||
/// Returns true if this error indicates GENUINE database file corruption (as opposed to a
|
||||
/// transient/operational error such as a pool timeout, a busy/locked database, or a
|
||||
/// momentary inability to open the file).
|
||||
///
|
||||
/// This is data-safety-critical: `true` here is the sole gate that permits
|
||||
/// [`DatabaseEngine::open_with_backup_dir`] to run destructive backup-isolation + salvage
|
||||
/// (which deletes the live database file). Only unambiguous corruption signatures are
|
||||
/// matched; transient conditions must propagate as ordinary errors instead, so callers can
|
||||
/// retry rather than lose data to a passing hiccup.
|
||||
pub fn is_corrupt(&self) -> bool {
|
||||
match self {
|
||||
DatabaseError::Corrupted(_) => true,
|
||||
@@ -48,12 +56,13 @@ impl DatabaseError {
|
||||
| DatabaseError::Migration(msg)
|
||||
| DatabaseError::RecoveryFailed(msg) => {
|
||||
let lower = msg.to_lowercase();
|
||||
// Deliberately excludes "cantopen" and "disk i/o error": both describe
|
||||
// operational conditions (locked file, momentary I/O hiccup) that clear up on
|
||||
// retry and must never be classified as corruption.
|
||||
lower.contains("not a database")
|
||||
|| lower.contains("malformed")
|
||||
|| lower.contains("corrupt")
|
||||
|| lower.contains("sqlite_notadb")
|
||||
|| lower.contains("cantopen")
|
||||
|| lower.contains("disk i/o error")
|
||||
|| lower.contains("corrupt")
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
@@ -144,7 +153,23 @@ impl DatabaseEngine {
|
||||
}
|
||||
|
||||
/// Close the underlying connection pool.
|
||||
///
|
||||
/// Forces a full WAL checkpoint (`PRAGMA wal_checkpoint(TRUNCATE)`) before closing so every
|
||||
/// committed page is flushed into the main database file and the `-wal` file is truncated.
|
||||
/// Without this, a WAL-mode connection can leave committed pages un-flushed in the `-wal`
|
||||
/// file; SQLite then performs its own "last connection closes" checkpoint internally, but
|
||||
/// that teardown is not guaranteed to complete synchronously within this call on every
|
||||
/// platform, and can instead complete moments later on a background thread -- which then
|
||||
/// races any code that inspects or replaces the main database file shortly after `close()`
|
||||
/// returns (this was the confirmed root cause of a nondeterministic corruption-detection
|
||||
/// test failure: a deferred checkpoint silently overwrote a deliberately corrupted header
|
||||
/// with the last-known-good page 1 a few milliseconds after `close()` had already
|
||||
/// returned). Explicitly checkpointing first leaves nothing for that deferred teardown to
|
||||
/// apply, closing the race.
|
||||
pub async fn close(&self) {
|
||||
let _ = sqlx::query("PRAGMA wal_checkpoint(TRUNCATE);")
|
||||
.execute(&self.pool)
|
||||
.await;
|
||||
self.pool.close().await;
|
||||
}
|
||||
|
||||
@@ -161,29 +186,52 @@ impl DatabaseEngine {
|
||||
|
||||
/// Open database at `path` specifying a custom backup directory, provisioning
|
||||
/// cryptographic key material from `secrets`.
|
||||
///
|
||||
/// # Data safety
|
||||
///
|
||||
/// This is the sole entry point that may trigger destructive backup-isolation + salvage
|
||||
/// (which deletes the live database file, keeping only what could be re-extracted). That
|
||||
/// path is gated on GENUINE corruption only -- classified via a connection-independent raw
|
||||
/// header-byte check (see [`crate::recovery::has_corrupted_sqlite_header`]) and/or
|
||||
/// [`DatabaseError::is_corrupt`] / [`crate::recovery::is_sqlite_corruption_error`]. A
|
||||
/// transient/operational error (pool timeout, busy, locked, momentarily unable to open)
|
||||
/// is NEVER coerced into "corrupt": it is propagated as-is so the caller can retry, and the
|
||||
/// live database is left completely untouched.
|
||||
pub async fn open_with_backup_dir(
|
||||
path: &Path,
|
||||
backup_dir: &Path,
|
||||
secrets: &crate::vault::SecretManager,
|
||||
) -> Result<(Self, Option<crate::recovery::RecoverySummary>), DatabaseError> {
|
||||
// Deterministic, connection-independent pre-flight check: read the raw SQLite header
|
||||
// magic bytes directly from disk before ever touching a connection pool. Unlike a
|
||||
// `PRAGMA quick_check` run over a freshly (re)opened pooled connection, this cannot race
|
||||
// a WAL checkpoint or a connection-close from a prior engine instance still completing
|
||||
// in the background, so it authoritatively and reproducibly detects a genuinely
|
||||
// corrupted header.
|
||||
let header_corrupted = crate::recovery::has_corrupted_sqlite_header(path);
|
||||
|
||||
match Self::connect_file(path, secrets).await {
|
||||
Ok(engine) => {
|
||||
let is_healthy = engine.check_integrity().await.unwrap_or(false);
|
||||
if is_healthy {
|
||||
Ok((engine, None))
|
||||
} else {
|
||||
engine.close().await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
let summary = crate::recovery::SqliteRecoveryEngine::salvage(
|
||||
path, path, backup_dir, secrets,
|
||||
)
|
||||
.await?;
|
||||
let fresh_engine = Self::connect_file(path, secrets).await?;
|
||||
Ok((fresh_engine, Some(summary)))
|
||||
if header_corrupted {
|
||||
return Self::close_and_salvage(engine, path, backup_dir, secrets).await;
|
||||
}
|
||||
match engine.check_integrity().await {
|
||||
Ok(true) => Ok((engine, None)),
|
||||
Ok(false) => Self::close_and_salvage(engine, path, backup_dir, secrets).await,
|
||||
Err(err) => {
|
||||
// The integrity probe itself failed with an error that is NOT a
|
||||
// recognized corruption signature (e.g. a pool acquire timeout, a
|
||||
// transient SQLITE_BUSY/locked condition). This is an operational
|
||||
// hiccup, not proof of corruption -- NEVER salvage or delete the live
|
||||
// database on the strength of it. Surface the error and let the caller
|
||||
// retry.
|
||||
engine.close().await;
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
if err.is_corrupt() {
|
||||
if header_corrupted || err.is_corrupt() {
|
||||
let summary = crate::recovery::SqliteRecoveryEngine::salvage(
|
||||
path, path, backup_dir, secrets,
|
||||
)
|
||||
@@ -197,6 +245,24 @@ impl DatabaseEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/// Close an engine that has been confirmed corrupted (either by a failed integrity check or
|
||||
/// by the independent raw-header-byte check), then run backup isolation and stream salvage.
|
||||
async fn close_and_salvage(
|
||||
engine: Self,
|
||||
path: &Path,
|
||||
backup_dir: &Path,
|
||||
secrets: &crate::vault::SecretManager,
|
||||
) -> Result<(Self, Option<crate::recovery::RecoverySummary>), DatabaseError> {
|
||||
engine.close().await;
|
||||
// Give the OS a brief moment to release file handles/locks from the just-closed pool
|
||||
// before salvage copies and then deletes the live database file.
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
let summary =
|
||||
crate::recovery::SqliteRecoveryEngine::salvage(path, path, backup_dir, secrets).await?;
|
||||
let fresh_engine = Self::connect_file(path, secrets).await?;
|
||||
Ok((fresh_engine, Some(summary)))
|
||||
}
|
||||
|
||||
/// Check database integrity executing `PRAGMA quick_check(10);`.
|
||||
/// Returns `Ok(true)` if integrity check passes ("ok"), or `Ok(false)` if corrupted.
|
||||
pub async fn check_integrity(&self) -> Result<bool, DatabaseError> {
|
||||
@@ -1272,6 +1338,131 @@ fn compute_log_hash(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use sqlx::Connection;
|
||||
|
||||
/// Proves the fix for 1.B.4: a transient error surfacing from the integrity probe (here, a
|
||||
/// genuine pool-exhaustion / acquire-timeout condition, deterministically forced by holding
|
||||
/// the pool's only connection) must propagate as `Err`, never be coerced into `Ok(false)`
|
||||
/// (which would be indistinguishable from genuine corruption to `open_with_backup_dir` and
|
||||
/// would trigger destructive salvage of a perfectly healthy database).
|
||||
#[tokio::test]
|
||||
async fn check_integrity_pool_propagates_transient_timeout_without_masking_as_corrupt() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db_path = dir.path().join("busy_test.db");
|
||||
let secrets = crate::vault::SecretManager::mock();
|
||||
{
|
||||
let engine = DatabaseEngine::connect_file(&db_path, &secrets)
|
||||
.await
|
||||
.unwrap();
|
||||
engine.close().await;
|
||||
}
|
||||
|
||||
let url = format!("sqlite://{}", db_path.to_string_lossy());
|
||||
let pool = SqlitePoolOptions::new()
|
||||
.max_connections(1)
|
||||
.acquire_timeout(std::time::Duration::from_millis(200))
|
||||
.connect(&url)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Hold the pool's only connection so a concurrent acquire attempt cannot succeed and
|
||||
// must time out -- a genuine, deterministic transient condition (no SQLite-level lock
|
||||
// contention required).
|
||||
let _held = pool.acquire().await.unwrap();
|
||||
|
||||
let result = DatabaseEngine::check_integrity_pool(&pool).await;
|
||||
assert!(
|
||||
matches!(result, Err(DatabaseError::Query(sqlx::Error::PoolTimedOut))),
|
||||
"a pool-exhaustion / acquire-timeout condition must propagate as an Err, never be \
|
||||
silently coerced into Ok(false); got: {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// End-to-end proof of 1.B.4: a genuinely transient, operational condition (another
|
||||
/// connection holding an exclusive lock on the live database) must never be treated as
|
||||
/// corruption by `open_with_backup_dir` -- no backup isolation, no salvage, no deletion of
|
||||
/// the live file. The database and its data must survive completely intact, and a retry
|
||||
/// once the contention clears must succeed with no recovery recorded.
|
||||
#[tokio::test]
|
||||
async fn open_with_backup_dir_never_salvages_on_transient_lock_contention() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let db_path = dir.path().join("contended.db");
|
||||
let backup_dir = dir.path().join("backups");
|
||||
let secrets = crate::vault::SecretManager::mock();
|
||||
|
||||
// Create a valid, healthy database with real data.
|
||||
{
|
||||
let engine = DatabaseEngine::connect_file(&db_path, &secrets)
|
||||
.await
|
||||
.unwrap();
|
||||
let acct = nuncio_core::AccountConfig {
|
||||
id: "acct-contended-1".to_string(),
|
||||
name: "Contended Account".to_string(),
|
||||
email_address: "contended@nuncio.mx".to_string(),
|
||||
protocol: nuncio_core::AccountProtocol::ImapSmtp,
|
||||
server_host: "imap.nuncio.mx".to_string(),
|
||||
server_port: 993,
|
||||
use_tls: true,
|
||||
imap_tls_mode: nuncio_core::TlsMode::ImplicitTls,
|
||||
smtp_tls_mode: nuncio_core::TlsMode::ImplicitTls,
|
||||
keyring_secret_key: "nuncio/acct-contended-1".to_string(),
|
||||
sync_interval_secs: 60,
|
||||
};
|
||||
engine.save_account(&acct).await.unwrap();
|
||||
engine.close().await;
|
||||
}
|
||||
|
||||
// Hold an exclusive lock on the file from an independent connection, forcing any other
|
||||
// connection attempt against the same file into a genuine SQLITE_BUSY -- a transient,
|
||||
// operational condition, not corruption.
|
||||
let url = format!("sqlite://{}", db_path.to_string_lossy());
|
||||
let mut locker = sqlx::sqlite::SqliteConnection::connect(&url)
|
||||
.await
|
||||
.expect("locker connects");
|
||||
sqlx::query("PRAGMA locking_mode=EXCLUSIVE;")
|
||||
.execute(&mut locker)
|
||||
.await
|
||||
.expect("set exclusive locking mode");
|
||||
sqlx::query("BEGIN IMMEDIATE;")
|
||||
.execute(&mut locker)
|
||||
.await
|
||||
.expect("begin immediate");
|
||||
// Locking mode only takes effect on the next read/write, so force one now.
|
||||
sqlx::query("SELECT COUNT(*) FROM accounts;")
|
||||
.execute(&mut locker)
|
||||
.await
|
||||
.expect("force exclusive lock acquisition");
|
||||
|
||||
let result = DatabaseEngine::open_with_backup_dir(&db_path, &backup_dir, &secrets).await;
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"a locked/busy database must surface an error, not silently succeed: {result:?}"
|
||||
);
|
||||
let backup_is_empty =
|
||||
!backup_dir.exists() || std::fs::read_dir(&backup_dir).unwrap().next().is_none();
|
||||
assert!(
|
||||
backup_is_empty,
|
||||
"a transient lock/busy condition must NEVER trigger backup isolation + salvage"
|
||||
);
|
||||
|
||||
// Release the lock deterministically before checking that the live database survived.
|
||||
let _ = sqlx::query("ROLLBACK;").execute(&mut locker).await;
|
||||
let _ = locker.close().await;
|
||||
|
||||
// The live database and its data must be completely intact, and a retry succeeds with
|
||||
// no recovery recorded.
|
||||
let (engine, summary) =
|
||||
DatabaseEngine::open_with_backup_dir(&db_path, &backup_dir, &secrets)
|
||||
.await
|
||||
.expect("reopen succeeds once contention clears");
|
||||
assert!(
|
||||
summary.is_none(),
|
||||
"no recovery should have been recorded for a transient condition"
|
||||
);
|
||||
let accounts = engine.list_accounts().await.unwrap();
|
||||
assert_eq!(accounts.len(), 1);
|
||||
assert_eq!(accounts[0].id, "acct-contended-1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ephemeral_database_initializes_and_migrates() {
|
||||
|
||||
@@ -35,27 +35,72 @@ pub struct RecoverySummary {
|
||||
pub resync_triggered: bool,
|
||||
}
|
||||
|
||||
/// Helper identifying SQLite corruption and connection error codes.
|
||||
/// Helper identifying SQLite corruption error codes, as distinct from transient/operational
|
||||
/// errors (busy, locked, timed out, momentarily unable to open).
|
||||
///
|
||||
/// This classification is data-safety-critical: only errors classified `true` here are
|
||||
/// permitted to trigger backup isolation + destructive salvage of the live database (which
|
||||
/// deletes the original file, keeping only what could be salvaged). A false positive here
|
||||
/// would destroy a perfectly healthy database on nothing more than a transient hiccup, so the
|
||||
/// classifier is deliberately an *allowlist* of unambiguous corruption signatures rather than a
|
||||
/// denylist that grows to catch "everything that isn't obviously fine".
|
||||
pub fn is_sqlite_corruption_error(err: &sqlx::Error) -> bool {
|
||||
let msg = err.to_string().to_lowercase();
|
||||
if msg.contains("disk image is malformed")
|
||||
|| msg.contains("not a database")
|
||||
|| msg.contains("corrupt")
|
||||
|| msg.contains("sqlite_notadb")
|
||||
|| msg.contains("cantopen")
|
||||
|| msg.contains("disk i/o error")
|
||||
{
|
||||
return true;
|
||||
}
|
||||
// A structured SQLite result code, when sqlx can supply one, is authoritative. sqlx-sqlite
|
||||
// reports the *extended* result code (see `sqlite3_extended_errcode`), so the primary code
|
||||
// is recovered via `code & 0xff` before comparing -- this still matches corruption
|
||||
// sub-variants such as `SQLITE_CORRUPT_VTAB`/`SQLITE_CORRUPT_INDEX` while continuing to
|
||||
// exclude every non-corruption code (SQLITE_BUSY = 5, SQLITE_LOCKED = 6, SQLITE_IOERR = 10,
|
||||
// SQLITE_CANTOPEN = 14, and their extended sub-variants), which describe operational
|
||||
// conditions -- another connection holding a lock, a momentary disk I/O hiccup, a file the
|
||||
// OS would not open just now -- that clear up on retry and must NEVER be treated as
|
||||
// corruption.
|
||||
if let sqlx::Error::Database(db_err) = err {
|
||||
if let Some(code) = db_err.code() {
|
||||
let code_str = code.as_ref();
|
||||
if code_str == "11" || code_str == "26" || code_str == "14" {
|
||||
return true;
|
||||
}
|
||||
return db_err
|
||||
.code()
|
||||
.and_then(|code| code.parse::<i32>().ok())
|
||||
.map(|code| {
|
||||
const SQLITE_CORRUPT: i32 = 11;
|
||||
const SQLITE_NOTADB: i32 = 26;
|
||||
let primary = code & 0xff;
|
||||
primary == SQLITE_CORRUPT || primary == SQLITE_NOTADB
|
||||
})
|
||||
.unwrap_or(false);
|
||||
}
|
||||
|
||||
// Errors without a structured database code (e.g. a connection-level failure raised before
|
||||
// sqlx has parsed a SQLite result code) are classified from message text, restricted to the
|
||||
// same narrow, unambiguous corruption signatures used above.
|
||||
let msg = err.to_string().to_lowercase();
|
||||
msg.contains("database disk image is malformed") || msg.contains("file is not a database")
|
||||
}
|
||||
|
||||
/// SQLite's fixed 16-byte file header magic string (always the first 16 bytes of a valid
|
||||
/// SQLite database file).
|
||||
const SQLITE_HEADER_MAGIC: &[u8; 16] = b"SQLite format 3\0";
|
||||
|
||||
/// Directly inspect the raw SQLite header magic bytes on disk, bypassing any SQLite
|
||||
/// connection, connection pool, or pager cache entirely.
|
||||
///
|
||||
/// This exists because corruption detection routed entirely through a SQLite connection
|
||||
/// (`connect` + `PRAGMA quick_check`) can race a WAL checkpoint or a connection-close from a
|
||||
/// prior engine instance still completing in the background: depending on timing, a freshly
|
||||
/// opened connection can occasionally fail to observe a header that was corrupted moments
|
||||
/// earlier. A plain `std::fs` read of the header bytes has no such race -- it is a single
|
||||
/// synchronous read of exactly what is on disk right now, independent of any connection state
|
||||
/// -- so it gives a deterministic, reproducible answer for the header-corruption case
|
||||
/// specifically, and is used as an authoritative pre-flight signal in
|
||||
/// [`crate::db::DatabaseEngine::open_with_backup_dir`].
|
||||
///
|
||||
/// Returns `true` only when the file exists, is at least header-sized, and its first 16 bytes
|
||||
/// do not match SQLite's fixed magic string -- i.e. never for a missing file or a freshly
|
||||
/// created (not yet initialized) empty database file, only for a genuinely unreadable header.
|
||||
pub fn has_corrupted_sqlite_header(path: &Path) -> bool {
|
||||
match std::fs::read(path) {
|
||||
Ok(bytes) if bytes.len() >= SQLITE_HEADER_MAGIC.len() => {
|
||||
bytes[0..SQLITE_HEADER_MAGIC.len()] != SQLITE_HEADER_MAGIC[..]
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Manager isolating damaged database files into forensic backups.
|
||||
@@ -183,19 +228,20 @@ impl SqliteRecoveryEngine {
|
||||
}
|
||||
}
|
||||
|
||||
// Column order/names below MUST match the live `filter_rules` / `filter_conditions` /
|
||||
// `filter_actions` schema created in `DatabaseEngine::migrate` -- including every
|
||||
// NOT NULL column -- or every insert silently fails (via `res.is_ok()`) and salvage
|
||||
// reports 0 restored rows despite having read valid data from the backup.
|
||||
let mut restored_rules_count = 0;
|
||||
for (id, name, desc, enabled, match_all, priority, created_at, updated_at) in
|
||||
&salvaged_rules
|
||||
{
|
||||
for (id, name, priority, enabled, nsql_text, created_at, updated_at) in &salvaged_rules {
|
||||
let res = sqlx::query(
|
||||
"INSERT INTO filter_rules (id, name, description, enabled, match_all, priority, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
||||
"INSERT INTO filter_rules (id, name, priority, enabled, nsql_text, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)"
|
||||
)
|
||||
.bind(id)
|
||||
.bind(name)
|
||||
.bind(desc)
|
||||
.bind(enabled)
|
||||
.bind(match_all)
|
||||
.bind(priority)
|
||||
.bind(enabled)
|
||||
.bind(nsql_text)
|
||||
.bind(created_at)
|
||||
.bind(updated_at)
|
||||
.execute(fresh_engine.pool())
|
||||
@@ -206,15 +252,16 @@ impl SqliteRecoveryEngine {
|
||||
}
|
||||
|
||||
let mut restored_conditions_count = 0;
|
||||
for (id, rule_id, field, operator, value) in &salvaged_conditions {
|
||||
for (id, rule_id, field, operator, value, logical_op) in &salvaged_conditions {
|
||||
let res = sqlx::query(
|
||||
"INSERT INTO filter_conditions (id, rule_id, field, operator, value) VALUES (?, ?, ?, ?, ?)"
|
||||
"INSERT INTO filter_conditions (id, rule_id, field, operator, value, logical_op) VALUES (?, ?, ?, ?, ?, ?)"
|
||||
)
|
||||
.bind(id)
|
||||
.bind(rule_id)
|
||||
.bind(field)
|
||||
.bind(operator)
|
||||
.bind(value)
|
||||
.bind(logical_op)
|
||||
.execute(fresh_engine.pool())
|
||||
.await;
|
||||
if res.is_ok() {
|
||||
@@ -223,14 +270,14 @@ impl SqliteRecoveryEngine {
|
||||
}
|
||||
|
||||
let mut restored_actions_count = 0;
|
||||
for (id, rule_id, action_type, value) in &salvaged_actions {
|
||||
for (id, rule_id, action_type, target) in &salvaged_actions {
|
||||
let res = sqlx::query(
|
||||
"INSERT INTO filter_actions (id, rule_id, action_type, value) VALUES (?, ?, ?, ?)",
|
||||
"INSERT INTO filter_actions (id, rule_id, action_type, target) VALUES (?, ?, ?, ?)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(rule_id)
|
||||
.bind(action_type)
|
||||
.bind(value)
|
||||
.bind(target)
|
||||
.execute(fresh_engine.pool())
|
||||
.await;
|
||||
if res.is_ok() {
|
||||
@@ -295,30 +342,34 @@ impl SqliteRecoveryEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read back valid rows from `filter_rules` / `filter_conditions` / `filter_actions` in the
|
||||
/// backup connection, using column names/order matching the live schema (see
|
||||
/// `DatabaseEngine::migrate`). `filter_conditions.value`/`filter_actions.target` are the
|
||||
/// only nullable columns in that pair of tables that this reads.
|
||||
#[allow(clippy::type_complexity)]
|
||||
async fn salvage_filter_tables(
|
||||
pool: &sqlx::SqlitePool,
|
||||
) -> (
|
||||
Vec<(String, String, String, i64, i64, i64, i64, i64)>,
|
||||
Vec<(String, String, String, String, String)>,
|
||||
Vec<(String, String, i64, i64, String, i64, i64)>,
|
||||
Vec<(String, String, String, String, String, String)>,
|
||||
Vec<(String, String, String, Option<String>)>,
|
||||
) {
|
||||
let rules = sqlx::query_as::<_, (String, String, String, i64, i64, i64, i64, i64)>(
|
||||
"SELECT id, name, description, enabled, match_all, priority, created_at, updated_at FROM filter_rules"
|
||||
let rules = sqlx::query_as::<_, (String, String, i64, i64, String, i64, i64)>(
|
||||
"SELECT id, name, priority, enabled, nsql_text, created_at, updated_at FROM filter_rules"
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let conditions = sqlx::query_as::<_, (String, String, String, String, String)>(
|
||||
"SELECT id, rule_id, field, operator, value FROM filter_conditions",
|
||||
let conditions = sqlx::query_as::<_, (String, String, String, String, String, String)>(
|
||||
"SELECT id, rule_id, field, operator, value, logical_op FROM filter_conditions",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let actions = sqlx::query_as::<_, (String, String, String, Option<String>)>(
|
||||
"SELECT id, rule_id, action_type, value FROM filter_actions",
|
||||
"SELECT id, rule_id, action_type, target FROM filter_actions",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
@@ -340,6 +391,38 @@ mod tests {
|
||||
assert!(!is_sqlite_corruption_error(&err));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_corrupted_sqlite_header_is_false_for_missing_or_fresh_files() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
// A path that has never been created at all.
|
||||
assert!(!has_corrupted_sqlite_header(
|
||||
&dir.path().join("does_not_exist.db")
|
||||
));
|
||||
|
||||
// A freshly created, empty (not-yet-initialized) file: shorter than the header, must
|
||||
// never be reported as "corrupted".
|
||||
let empty_path = dir.path().join("empty.db");
|
||||
std::fs::write(&empty_path, []).unwrap();
|
||||
assert!(!has_corrupted_sqlite_header(&empty_path));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_corrupted_sqlite_header_is_true_only_for_a_bad_magic_string() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
|
||||
let valid_path = dir.path().join("valid_header.db");
|
||||
let mut valid_bytes = vec![0u8; 4096];
|
||||
valid_bytes[0..16].copy_from_slice(SQLITE_HEADER_MAGIC);
|
||||
std::fs::write(&valid_path, &valid_bytes).unwrap();
|
||||
assert!(!has_corrupted_sqlite_header(&valid_path));
|
||||
|
||||
let bad_path = dir.path().join("bad_header.db");
|
||||
let mut bad_bytes = vec![0u8; 4096];
|
||||
bad_bytes[0..16].copy_from_slice(b"CORRUPTED_NOISE_");
|
||||
std::fs::write(&bad_path, &bad_bytes).unwrap();
|
||||
assert!(has_corrupted_sqlite_header(&bad_path));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_database_header_corruption_stage_1_detection() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
@@ -404,7 +487,8 @@ mod tests {
|
||||
let backup_dir = dir.path().join("corrupted_backups");
|
||||
let secrets = crate::vault::SecretManager::mock();
|
||||
|
||||
// Step 1: Create DB with account & rule
|
||||
// Step 1: Create DB with account & a real NSQL filter rule (exercising the
|
||||
// `filter_rules` schema salvage restores against -- see 1.B.3).
|
||||
{
|
||||
let engine = DatabaseEngine::connect_file(&db_path, &secrets)
|
||||
.await
|
||||
@@ -423,6 +507,10 @@ mod tests {
|
||||
sync_interval_secs: 60,
|
||||
};
|
||||
engine.save_account(&acct).await.unwrap();
|
||||
|
||||
let nsql = "SELECT * FROM emails WHERE subject CONTAINS 'Spam' ACTION DELETE";
|
||||
let rule = nuncio_filter::NsqlParser::parse_rule("Spam Filter", 1, nsql).unwrap();
|
||||
engine.save_filter_rule(&rule).await.unwrap();
|
||||
}
|
||||
|
||||
// Step 2: Perform salvage recovery
|
||||
@@ -431,14 +519,35 @@ mod tests {
|
||||
.expect("salvage succeeds");
|
||||
|
||||
assert_eq!(summary.salvaged_accounts_count, 1);
|
||||
assert_eq!(
|
||||
summary.salvaged_rules_count, 1,
|
||||
"salvage must restore the filter rule -- a 0 count here means the restore INSERT \
|
||||
is silently failing against a schema mismatch (the exact 1.B.3 regression)"
|
||||
);
|
||||
assert!(summary.backup_path.exists());
|
||||
|
||||
// Step 3: Verify fresh DB contains salvaged account
|
||||
// Step 3: Verify fresh DB contains the salvaged account AND filter rule.
|
||||
let fresh = DatabaseEngine::connect_file(&db_path, &secrets)
|
||||
.await
|
||||
.unwrap();
|
||||
let accounts = fresh.list_accounts().await.unwrap();
|
||||
assert_eq!(accounts.len(), 1);
|
||||
assert_eq!(accounts[0].id, "acct-salvage-1");
|
||||
|
||||
let rules = fresh.list_filter_rules().await.unwrap();
|
||||
assert_eq!(rules.len(), 1);
|
||||
assert_eq!(rules[0].name, "Spam Filter");
|
||||
}
|
||||
|
||||
// NOTE: there is deliberately no test combining *header* corruption (bytes 0-16 -- the
|
||||
// fixed "SQLite format 3\0" magic string) with a nonzero salvaged-rule-count assertion.
|
||||
// A corrupted header makes the file entirely unopenable by SQLite (`SQLITE_NOTADB`), by
|
||||
// design regardless of how intact the rest of the file's pages are, so salvage's backup
|
||||
// connection can never read ANY row out of a header-corrupted file -- that is real SQLite
|
||||
// behavior, not a salvage bug. `test_database_header_corruption_stage_1_detection` above
|
||||
// proves the header-corruption case still triggers detection + backup isolation (no
|
||||
// regression on 1.B.4's destructive-path gating); `test_stage_2_backup_creation_and_stage_3_table_salvage`
|
||||
// above proves the schema fix (1.B.3) by exercising salvage's actual read-then-restore SQL
|
||||
// against a normally-openable source, which is the only way to test the restore SQL, since
|
||||
// header corruption forecloses any row-level read.
|
||||
}
|
||||
@@ -135,9 +135,6 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "flaky: DB-corruption detection races the WAL/file close in the recovery path \
|
||||
(initialize_and_recover intermittently returns None). Quarantined to keep CI \
|
||||
reliably green; fix properly under Phase 1.B recovery reliability (GH #155)."]
|
||||
async fn orchestrator_recovers_corrupted_database_and_emits_event() {
|
||||
let dir = tempdir().expect("tempdir");
|
||||
let db_path = dir.path().join("corrupt_orch.db");
|
||||
|
||||
Reference in new issue
Block a user