Compare commits

...
Author SHA1 Message Date
james.maes ca1ef02161 docs(rebuild): record verified IMAP correction [skip ci] 2026-09-14 15:52:37 -05:00
james.maes bb70f95f93 fix(imap): reconcile changes during initial mailbox sync 2026-09-14 15:29:29 -05:00
james.maes b9ca641d5e docs(rebuild): record verified logging delivery [skip ci] 2026-09-14 14:01:04 -05:00
james.maes 779264348d feat(daemon): report startup progress and update Rustls 2026-09-14 13:25:23 -05:00
james.maes 92e93949b8 feat(cli): make help and terminal output usable 2026-09-14 12:13:58 -05:00
james.maes a5b71ba8a2 feat(daemon): show safe activity logs with configurable levels 2026-09-14 10:42:48 -05:00
james.maes 31cfba3691 docs(rebuild): align report with qualified artifacts [skip ci] 2026-09-13 20:03:53 -05:00
james.maes 3738d3f419 docs(rebuild): record stable installer delivery [skip ci] 2026-09-13 19:56:13 -05:00
james.maes f14b02a223 fix(installer): keep testing commands at a permanent path 2026-09-13 19:37:56 -05:00
james.maes 0c7083bf8e fix(installer): activate verified builds at stable paths 2026-09-13 19:29:12 -05:00
james.maes 7e927963c5 docs(rebuild): record account help delivery [skip ci] 2026-09-13 14:30:21 -05:00
james.maes 7508e25194 fix(cli): explain how to work on individual accounts 2026-09-13 14:07:22 -05:00
james.maes ac58e5dad8 docs(rebuild): prepare Google registration approval [skip ci] 2026-09-13 13:57:44 -05:00
james.maes 269c0ea51a docs(rebuild): correct selected candidate handoff [skip ci] 2026-09-13 12:52:09 -05:00
james.maes 093dba7b73 docs(rebuild): record guided setup delivery [skip ci] 2026-09-13 12:47:37 -05:00
james.maes 82a92a06ee test(sync): observe account progress during provider backoff 2026-09-13 12:25:09 -05:00
james.maes b2b3ed5732 test(sync): measure recovery after provider backoff expires 2026-09-13 12:12:24 -05:00
james.maes 78d4d9e2cf feat(cli): guide account setup with secure terminal prompts 2026-09-13 11:45:31 -05:00
james.maes c953ae8ef8 docs: record public bootstrap verification [skip ci] 2026-09-12 13:27:16 -05:00
james.maes 57c619e1f9 feat(installer): add verified curl-to-bash testing bootstrap 2026-09-12 13:06:22 -05:00
james.maes 13dbe549cb docs: clarify native keystore acceptance [skip ci] 2026-09-12 12:11:57 -05:00
james.maes c662e48d10 docs: record verified account testing delivery and future milestones 2026-09-12 11:47:02 -05:00
james.maes 164b021750 feat(accounts): complete lifecycle management and testing delivery 2026-09-12 11:10:47 -05:00
james.maes 1f47fe4e64 fix(ci): pin actions and replace the yanked lockfile dependency 2026-09-12 11:00:17 -05:00
james.maes 549a5983db ci(security): scan development code and complete Cargo lockfiles 2026-09-12 10:39:38 -05:00
james.maes 81a1bc6ce6 fix(deps): update libraries and remove vulnerable dependency paths 2026-09-12 10:38:16 -05:00
james.maes 7390e77e68 docs(rebuild): record verified delivery and pending live acceptance 2026-09-12 04:18:48 -05:00
james.maes 6ff9bb9e7b fix(rebuild): stabilize IMAP coverage across optional LIST hints 2026-09-12 03:47:03 -05:00
james.maes 275eb427cd test(rebuild): retain evidence for hosted verification failures 2026-09-12 02:38:32 -05:00
james.maes 6a324f9d1c perf(rebuild): batch search replacement during mail promotion 2026-09-12 02:01:54 -05:00
james.maes 1a0d954d90 fix(rebuild): isolate CI egress without disconnecting runners 2026-09-12 00:45:32 -05:00
james.maes 2e4d30ef94 test(rebuild): use production deadline for large transfers 2026-09-11 23:18:07 -05:00
james.maes 7f81b735b5 build(rebuild): verify isolated release candidates 2026-09-11 22:50:40 -05:00
james.maes 720081f661 fix(rebuild): support limited calendar writers safely 2026-09-11 20:50:14 -05:00
james.maes 0805a7dc4b feat(rebuild): bound provider work and expose resource status 2026-09-11 20:03:04 -05:00
james.maes a29812fade fix(rebuild): bound request admission and retain queued work 2026-09-11 19:29:31 -05:00
james.maes c1180f1dc8 test(rebuild): verify large mail workloads and process memory 2026-09-11 18:47:58 -05:00
james.maes 8811acdd14 test(rebuild): audit authorization and subprocess confidentiality 2026-09-11 18:37:56 -05:00
james.maes befc6bbbcf test(rebuild): verify convergence across three independent engines 2026-09-11 18:14:14 -05:00
james.maes 57dd648eb3 fix(rebuild): allow restore retries without restarting the daemon 2026-09-11 18:06:58 -05:00
james.maes 16de94f67e docs(rebuild): record checkpoint evidence and standing authorization 2026-09-11 17:48:59 -05:00
james.maes 2480bf94cd feat(rebuild): checkpoint Google and MailPlus engine and CLI
Add the isolated Rust rebuild with encrypted storage, authenticated gRPC,
Google mail/calendar, IMAP/SMTP, independent local providers, durable
operations, and daemon/CLI recovery tests. Preserve the original workspace.

The latest offline checkpoint passes formatting, both Clippy configurations,
126 engine tests, 4 recovery E2E tests, 13 IMAP E2E tests, and 2 production
isolation tests. Staged secret and whitespace scans pass; raw fixtures retain
their exact bytes. Recovery hardening, broader acceptance suites, packaging,
CI definitions, and separately authorized live acceptance remain unfinished.
2026-09-11 17:47:13 -05:00
504 changed files with 102182 additions and 1046 deletions

No files matched your search

+5 -2
View File
@@ -2,7 +2,10 @@ version: 2
updates:
# 1. Rust Workspace Crates (Cargo)
- package-ecosystem: "cargo"
directory: "/"
directories:
- "/"
- "/rebuild"
- "/rebuild/clients/smoke"
schedule:
interval: "daily"
time: "04:00"
@@ -26,6 +29,6 @@ updates:
open-pull-requests-limit: 5
labels:
- "dependencies"
- "actions"
- "github_actions"
commit-message:
prefix: "deps(actions)"
+7 -7
View File
@@ -2,9 +2,9 @@ name: CI Quality Gates & Build Pipeline
on:
push:
branches: [ main, dev, rc ]
branches: [ main, dev, develop, rc ]
pull_request:
branches: [ main, dev, rc ]
branches: [ main, dev, develop, rc ]
env:
CARGO_TERM_COLOR: always
@@ -32,7 +32,7 @@ jobs:
steps:
- uses: actions/checkout@v7
- name: Cache Cargo Dependencies
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
- name: Run Clippy Linter (all targets, matches local `cargo check-all`)
run: cargo check-all
@@ -42,7 +42,7 @@ jobs:
steps:
- uses: actions/checkout@v7
- name: Cache Cargo Dependencies
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
- name: Run Workspace Tests
run: cargo test-all
@@ -55,9 +55,9 @@ jobs:
- name: Install llvm-tools component
run: rustup component add llvm-tools-preview
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@cargo-llvm-cov
uses: taiki-e/install-action@212e92b595ceefa6daaad2cda95b921ed5da4cd4 # cargo-llvm-cov
- name: Cache Cargo Dependencies
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
# Coverage is informational only (matches the local `cargo cov` alias):
# no --fail-under-lines threshold, and the job is continue-on-error so a
# coverage regression or tooling hiccup never reds the pipeline. A
@@ -76,6 +76,6 @@ jobs:
steps:
- uses: actions/checkout@v7
- name: Cache Cargo Dependencies
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
- name: Build All Workspace Crates
run: cargo build --workspace --all-targets
+75 -8
View File
@@ -2,26 +2,36 @@ name: "CodeQL Security Analysis"
on:
push:
branches: [ "main" ]
branches: [main, dev, develop, rc, feature/nuncio-google-first-rebuild]
pull_request:
branches: [ "main" ]
workflow_dispatch:
schedule:
- cron: '30 1 * * 1' # Every Monday at 01:30 UTC
permissions:
actions: read
contents: read
security-events: write
concurrency:
group: security-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
analyze:
name: Analyze CodeQL (${{ matrix.language }})
runs-on: ubuntu-latest
timeout-minutes: 360
timeout-minutes: 90
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
include:
- language: rust
build-mode: none
- language: python
build-mode: none
- language: javascript-typescript
build-mode: none
- language: actions
@@ -29,15 +39,72 @@ jobs:
steps:
- name: Checkout Code
uses: actions/checkout@v4
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Prepare pinned Rust toolchain
if: matrix.language == 'rust'
run: rustup show active-toolchain
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
uses: github/codeql-action/init@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
queries: security-extended
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
uses: github/codeql-action/analyze@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4
with:
category: "/language:${{ matrix.language }}"
dependency-advisories:
name: Dependency advisories (${{ matrix.workspace }})
runs-on: ubuntu-latest
timeout-minutes: 30
env:
NUNCIO_ADVISORY_DB: ${{ github.workspace }}/rebuild/test-results/advisory-db
strategy:
fail-fast: false
matrix:
include:
- workspace: original
manifest: Cargo.toml
lockfile: Cargo.lock
- workspace: rebuild
manifest: rebuild/Cargo.toml
lockfile: rebuild/Cargo.lock
- workspace: client
manifest: rebuild/clients/smoke/Cargo.toml
lockfile: rebuild/clients/smoke/Cargo.lock
steps:
- name: Checkout source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Prepare advisory scanners and locked dependencies
env:
NUNCIO_MANIFEST: ${{ matrix.manifest }}
run: |
rustup show active-toolchain
cargo install cargo-deny --version 0.19.8 --locked
cargo install cargo-audit --version 0.22.2 --locked --no-default-features
cargo fetch --locked --manifest-path "$NUNCIO_MANIFEST"
cargo deny --manifest-path "$NUNCIO_MANIFEST" fetch -c rebuild/deny.toml db
- name: Reject vulnerable dependencies using the fetched advisory database
env:
NUNCIO_MANIFEST: ${{ matrix.manifest }}
run: cargo deny --manifest-path "$NUNCIO_MANIFEST" --locked --offline check -c rebuild/deny.toml advisories
- name: Reject advisories in the complete lockfile, including optional dependencies
shell: bash
env:
NUNCIO_LOCKFILE: ${{ matrix.lockfile }}
CARGO_NET_OFFLINE: 'true'
run: |
# Reuse cargo-deny's fetched RustSec checkout without hardcoding its directory hash.
databases=("$NUNCIO_ADVISORY_DB"/advisory-db-*)
test "${#databases[@]}" -eq 1
test -d "${databases[0]}/.git"
test -f "$NUNCIO_LOCKFILE"
cargo audit --file "$NUNCIO_LOCKFILE" --db "${databases[0]}" --no-fetch --deny warnings
+264
View File
@@ -0,0 +1,264 @@
name: Nuncio rebuild
on:
push:
branches: [feature/nuncio-google-first-rebuild, main, dev, develop, rc]
paths: ["rebuild/**", ".github/workflows/rebuild-ci.yml"]
pull_request:
paths: ["rebuild/**", ".github/workflows/rebuild-ci.yml"]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: rebuild-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
shell: bash
working-directory: rebuild
jobs:
rebuild-lint:
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04, macos-15]
runs-on: ${{ matrix.os }}
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "3.14"
- name: Verify test isolation preserves runner connectivity
if: runner.os == 'Linux'
run: python3 tests/egress/control_plane.py test-results/egress-controller scripts/egress.py
- name: Prepare pinned toolchain and offline dependencies
run: |
rustup show active-toolchain
python3 -m pip install ruff==0.16.7
cargo fetch --locked
cargo fetch --locked --manifest-path clients/smoke/Cargo.toml
- name: Run required offline checks with egress denied
run: python3 scripts/egress.py --evidence test-results/rebuild-lint-egress.json -- python3 scripts/verify.py --job rebuild-lint
- name: Retain command logs and numeric evidence
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: rebuild-lint-${{ matrix.os }}-evidence
path: |
rebuild/test-results/rebuild-lint/*.log
rebuild/test-results/rebuild-lint/results.json
rebuild/test-results/*-egress.json
rebuild/test-results/egress-controller/exit-*/*.json
if-no-files-found: warn
retention-days: 7
rebuild-mock-contract:
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04, macos-15]
runs-on: ${{ matrix.os }}
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "3.14"
- name: Verify test isolation preserves runner connectivity
if: runner.os == 'Linux'
run: python3 tests/egress/control_plane.py test-results/egress-controller scripts/egress.py
- name: Prepare pinned toolchain and offline dependencies
run: |
rustup show active-toolchain
python3 -m pip install ruff==0.16.7
cargo fetch --locked
cargo fetch --locked --manifest-path clients/smoke/Cargo.toml
- name: Run required offline checks with egress denied
run: python3 scripts/egress.py --evidence test-results/rebuild-mock-contract-egress.json -- python3 scripts/verify.py --job rebuild-mock-contract
- name: Retain command logs and numeric evidence
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: rebuild-mock-contract-${{ matrix.os }}-evidence
path: |
rebuild/test-results/rebuild-mock-contract/*.log
rebuild/test-results/rebuild-mock-contract/results.json
rebuild/test-results/*-egress.json
rebuild/test-results/egress-controller/exit-*/*.json
if-no-files-found: warn
retention-days: 7
rebuild-system:
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04]
runs-on: ${{ matrix.os }}
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "3.14"
- name: Verify test isolation preserves runner connectivity
if: runner.os == 'Linux'
run: python3 tests/egress/control_plane.py test-results/egress-controller scripts/egress.py
- name: Prepare pinned toolchain and offline dependencies
run: |
rustup show active-toolchain
python3 -m pip install ruff==0.16.7
cargo fetch --locked
cargo fetch --locked --manifest-path clients/smoke/Cargo.toml
- name: Run required offline checks with egress denied
run: python3 scripts/egress.py --evidence test-results/rebuild-system-egress.json -- python3 scripts/verify.py --job rebuild-system
- name: Retain command logs and numeric evidence
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: rebuild-system-${{ matrix.os }}-evidence
path: |
rebuild/test-results/rebuild-system/*.log
rebuild/test-results/rebuild-system/results.json
rebuild/test-results/*-egress.json
rebuild/test-results/egress-controller/exit-*/*.json
if-no-files-found: warn
retention-days: 7
rebuild-e2e:
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04, macos-15]
runs-on: ${{ matrix.os }}
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "3.14"
- name: Verify test isolation preserves runner connectivity
if: runner.os == 'Linux'
run: python3 tests/egress/control_plane.py test-results/egress-controller scripts/egress.py
- name: Prepare pinned toolchain and offline dependencies
run: |
rustup show active-toolchain
python3 -m pip install ruff==0.16.7
cargo fetch --locked
cargo fetch --locked --manifest-path clients/smoke/Cargo.toml
- name: Run required offline checks with egress denied
run: python3 scripts/egress.py --evidence test-results/rebuild-e2e-egress.json -- python3 scripts/verify.py --job rebuild-e2e
- name: Retain command logs and numeric evidence
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: rebuild-e2e-${{ matrix.os }}-evidence
path: |
rebuild/test-results/rebuild-e2e/*.log
rebuild/test-results/rebuild-e2e/results.json
rebuild/test-results/*-egress.json
rebuild/test-results/egress-controller/exit-*/*.json
if-no-files-found: warn
retention-days: 7
rebuild-imap:
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04]
runs-on: ${{ matrix.os }}
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "3.14"
- name: Verify test isolation preserves runner connectivity
if: runner.os == 'Linux'
run: python3 tests/egress/control_plane.py test-results/egress-controller scripts/egress.py
- name: Prepare pinned toolchain and offline dependencies
run: |
rustup show active-toolchain
python3 -m pip install ruff==0.16.7
cargo fetch --locked
cargo fetch --locked --manifest-path clients/smoke/Cargo.toml
- name: Pre-pull independent pinned mail servers
run: |
docker pull dovecot/dovecot:2.4.5@sha256:c807be4fb5a97d9c3a90770569d3a6c4cbdcb36742ad41f90409cbd929166553
docker pull axllent/mailpit:v1.31.1@sha256:98b916bd3c8d61f7633a52d3ea2f58d00620cb01ca57ab59edde68c347a95365
docker pull haproxy:3.2-alpine@sha256:6343ce34a132a5dceaa24767d739df2bd519f8f7c1079ae39e4821334e8eb42e
- name: Run required offline checks with egress denied
run: python3 scripts/egress.py --evidence test-results/rebuild-imap-egress.json -- python3 scripts/verify.py --job rebuild-imap
- name: Retain command logs and numeric evidence
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: rebuild-imap-${{ matrix.os }}-evidence
path: |
rebuild/test-results/rebuild-imap/*.log
rebuild/test-results/rebuild-imap/results.json
rebuild/test-results/*-egress.json
rebuild/test-results/egress-controller/exit-*/*.json
if-no-files-found: warn
retention-days: 7
rebuild-release-check:
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04, macos-15]
runs-on: ${{ matrix.os }}
timeout-minutes: 90
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "3.14"
- name: Verify test isolation preserves runner connectivity
if: runner.os == 'Linux'
run: python3 tests/egress/control_plane.py test-results/egress-controller scripts/egress.py
- name: Prepare pinned toolchain and offline dependencies
run: |
rustup show active-toolchain
python3 -m pip install ruff==0.16.7
cargo fetch --locked
cargo fetch --locked --manifest-path clients/smoke/Cargo.toml
- name: Prepare advisory tool and database
run: |
cargo install cargo-deny --version 0.19.8 --locked
cargo deny fetch db
- name: Run required offline checks with egress denied
run: python3 scripts/egress.py --evidence test-results/rebuild-release-check-egress.json -- python3 scripts/verify.py --job rebuild-release-check
- name: Verify local production archive
env:
NUNCIO_DESKTOP_OAUTH_JSON: ${{ github.event_name == 'push' && github.ref == 'refs/heads/feature/nuncio-google-first-rebuild' && secrets.NUNCIO_GOOGLE_DESKTOP_CLIENT_JSON || '' }}
run: python3 scripts/package-testing.py
- name: Retain verified Apple Silicon testing archive
if: runner.os == 'macOS' && github.event_name == 'push' && github.ref == 'refs/heads/feature/nuncio-google-first-rebuild'
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: nuncio-testing-aarch64-apple-darwin-${{ github.sha }}-attempt-${{ github.run_attempt }}
path: |
rebuild/dist/*-aarch64-apple-darwin-*.tar.gz
rebuild/dist/*-aarch64-apple-darwin-*.tar.gz.sha256
rebuild/dist/*-aarch64-apple-darwin-*-verification.json
if-no-files-found: error
retention-days: 14
- name: Retain command logs and numeric evidence
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: rebuild-release-check-${{ matrix.os }}-evidence
path: |
rebuild/test-results/rebuild-release-check/*.log
rebuild/test-results/rebuild-release-check/results.json
rebuild/test-results/*-egress.json
rebuild/test-results/egress-controller/exit-*/*.json
if-no-files-found: warn
retention-days: 7
+2 -2
View File
@@ -36,7 +36,7 @@ jobs:
uses: actions/checkout@v7
- name: Install Rust Toolchain
uses: dtolnay/rust-toolchain@stable
uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
with:
targets: ${{ matrix.target }}
@@ -87,7 +87,7 @@ jobs:
sha256sum * > SHA256SUMS.txt
- name: Create or Update GitHub Nightly Pre-Release
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3
with:
tag_name: dev-nightly
name: Nuncio Development Nightly Snapshot
+2 -2
View File
@@ -40,7 +40,7 @@ jobs:
uses: actions/checkout@v7
- name: Install Rust Toolchain
uses: dtolnay/rust-toolchain@stable
uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
with:
targets: ${{ matrix.target }}
@@ -111,7 +111,7 @@ jobs:
fi
- name: Create GitHub Release
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3
with:
tag_name: ${{ steps.prep.outputs.tag_name }}
name: ${{ steps.prep.outputs.release_name }}
Generated
+629 -940
View File
File diff suppressed because it is too large. Load diff
+13 -15
View File
@@ -35,39 +35,40 @@ unreachable = "deny"
[workspace.dependencies]
tokio = { version = "1.38", features = ["full"] }
async-trait = "0.1"
async-trait = "0.1.92"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
thiserror = "1.0"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
tracing-appender = "0.2"
sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite"] }
sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "sqlite"] }
ratatui = "0.28"
crossterm = { version = "0.28", features = ["event-stream"] }
clap = { version = "4.5", features = ["derive", "env"] }
clap = { version = "4.6.6", features = ["derive", "env"] }
zeroize = { version = "1.8", features = ["derive"] }
tempfile = "3.10"
keyring = "2.3"
age = "0.10"
aes-gcm = "0.10"
mail-parser = "0.9"
keyring = { version = "4.2.0", default-features = false, features = ["v1"] }
age = "0.12.1"
aes-gcm = "0.11.1"
getrandom = "0.4"
mail-parser = "0.11.9"
bytes = "1.6"
lettre = { version = "0.11", default-features = false, features = ["tokio1-rustls-tls", "builder", "smtp-transport"] }
lettre = { version = "0.11.23", default-features = false, features = ["tokio1-rustls-tls", "builder", "smtp-transport"] }
async-imap = { version = "0.11", default-features = false, features = ["runtime-tokio"] }
tokio-rustls = "0.26"
tokio-rustls = "0.26.5"
rustls = { version = "0.23", features = ["ring"] }
webpki-roots = "1.0"
rustls-pki-types = "1.7"
tokio-stream = "0.1"
futures = "0.3"
futures = "0.3.34"
wiremock = "0.6"
icalendar = "0.17"
chrono-tz = "0.10"
rrule = "0.10"
chrono = "0.4"
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
quick-xml = "0.41"
quick-xml = "0.42.0"
html2text = "0.12"
rpassword = "7.3"
sqlparser = "0.54"
@@ -76,7 +77,7 @@ regex = "1.10"
hmac = "0.12"
sha2 = "0.10"
hex = "0.4"
flate2 = "1.0"
flate2 = "1.1.10"
tar = "0.4"
zip = { version = "8.6", default-features = false, features = ["deflate"] }
tonic = "0.14"
@@ -91,6 +92,3 @@ tonic-prost-build = "0.14"
protoc-bin-vendored = "3.2"
subtle = "2.6"
fs4 = "0.13"
+77 -25
View File
@@ -1,33 +1,85 @@
# Nuncio
**A local-first mail, calendar, and contacts engine written in Rust.**
**A local-first mail and calendar engine written in Rust.**
> **Status: pre-alpha, under active reconstruction (2026-07-26).**
> Nuncio began as an AI-generated proof of concept. A forensic assessment found
> that while several library-layer components are genuinely well-built, the
> end-to-end product did not work and much of the prior documentation was
> inaccurate. The project is being rebuilt engine-first. **It is not usable yet,
> and published pre-1.0 releases are non-functional — do not install them.**
> See the **[live roadmap](https://koftwentytwo.github.io/nuncio/roadmap/)**
> (source: [`docs/ROADMAP.md`](docs/ROADMAP.md)) for the plan and
> [`docs/adr/0001-engine-first-grpc-architecture.md`](docs/adr/0001-engine-first-grpc-architecture.md)
> for the architecture decision.
> **Status: pre-alpha; active implementation is in [`rebuild/`](rebuild/README.md)
> (September 14, 2026).** Latest verified testing download: `bb70f95`. It includes
> readable CLI output, complete help, detailed startup/activity logs, guided
> account management, permanent installer paths and the Rustls security patch.
> All ten hosted rebuild jobs, seven security jobs and actual public fresh/update
> checks passed. The same source has two identical clean local archives and a
> full 35-command gate with 359 tests per workspace. Google registration and named
> live Google/MailPlus/native-keystore acceptance remain pending. See the
> [report](rebuild/docs/IMPLEMENTATION-REPORT.md) and [current state](rebuild/docs/SESSION-STATE.md).
## What Nuncio is
A background daemon (`nunciod`) owns all state, credentials, and protocol logic
(IMAP/SMTP, then JMAP, CalDAV, CardDAV), and **publishes a versioned gRPC API over
loopback**. User interfaces are separate native client projects that consume that
API — they hold no business logic and no data of their own. This makes feature
parity across clients a property of the published contract rather than a matter of
discipline.
A daemon (`nunciod`) owns encrypted SQLCipher storage, credentials, synchronization,
and durable operations. The active rebuild supports Google Gmail/Calendar and
IMAP/SMTP mail through an authenticated `nuncio.v2` gRPC API on numeric loopback.
The reference `nuncio-cli` consumes that API independently of engine/storage code.
Native applications are outside this workspace's implementation scope. Contacts,
JMAP, and CalDAV/CardDAV remain outside the active rebuild's approved provider set.
The only client in this repository is `nuncio-cli`, which serves as the reference
API consumer and the driver for end-to-end tests. Native GUIs (macOS/Windows), a
TUI, and an MCP bridge will live in separate repositories once the engine and its
API are solid.
Install the latest Apple Silicon testing build without cloning or compiling:
## Workspace layout
```sh
curl -fsSL https://raw.githubusercontent.com/KofTwentyTwo/nuncio/feature/nuncio-google-first-rebuild/install-testing.sh | bash
```
Requires macOS 15+, Python 3.11+ and authenticated GitHub CLI 2.100+;
[details and custom prefix](rebuild/docs/TESTING-INSTALL.md).
The permanent commands are in `~/.local/opt/nuncio-testing/bin/`. Rerun the
installer to update them; each verified update keeps this path. Start the daemon
with `~/.local/opt/nuncio-testing/bin/nunciod --profile laptop-qa`, then add an
account using `~/.local/opt/nuncio-testing/bin/nuncio-cli --profile laptop-qa account add`.
The current source reports startup stages and running activity at **info** level
(`--log-level debug` adds detail), and readable CLI output is now the default;
scripts use `--json`. Rustls 0.23.45 addresses RUSTSEC-2026-0285 across all three
lockfiles. The downloaded package also includes bounded IMAP catch-up, granular credential/recovery timings and safe CLI typo suggestions. The laptop startup pause still needs a retest.
Public fresh/update qualification passed at bb70f95. [CLI usability audit](rebuild/docs/CLI-USABILITY-AUDIT.md)
and [logging options](rebuild/docs/RUNNING.md#running-logs).
[Guided setup](rebuild/docs/ACCOUNT-SETUP.md) uses direct prompts and hidden passwords.
Follow the [Google setup walkthrough](rebuild/docs/GOOGLE-SETUP.md) to register
the app once and connect using a private local client file. Shared registration
has not yet been bundled into testing builds.
For Apple Silicon laptop testing, the [testing installer](rebuild/docs/TESTING-INSTALL.md)
downloads a verified successful CI package without compiling locally. It
activates each build at the same testing path and retains older versions for
recovery. Actual installation and update passed at `bb70f95`. It starts no service. No public release or live-provider readiness is implied.
The [proposed post-engine roadmap](rebuild/docs/POST-ENGINE-ROADMAP.md) starts
after engine/CLI acceptance and sequences a native Mac alpha, daily mail,
daily calendar, and a dependable personal release. It is future planning;
native implementation is outside the current goal.
## Active workspace and verification
Use the independent [`rebuild/Cargo.toml`](rebuild/Cargo.toml) workspace. Its
[README](rebuild/README.md), [running guide](rebuild/docs/RUNNING.md),
[account-management guide](rebuild/docs/ACCOUNT-MANAGEMENT.md), and
[API contract](rebuild/docs/API.md) describe the current source.
```sh
cd rebuild
cargo build --locked -p nunciod -p nuncio-cli
cargo fmt --all -- --check
cargo clippy --locked --workspace --all-targets -- -D warnings
cargo test --locked --workspace
```
These are basic Rust checks. The complete [offline verification](rebuild/docs/TESTING.md)
also runs test-feature builds, independent provider services, real daemon/CLI
subprocess tests, security/resource checks, and production-package checks.
## Original workspace
The root Cargo workspace and original roadmap remain preserved separately.
The July 26 assessment found the original proof of concept did not work end to
end and older documentation overstated its capabilities. Its library layout below
describes that original code, not the independent rebuild or a finished product.
Engine libraries composed by the daemon:
@@ -46,7 +98,7 @@ The former `nuncio-tui`, `nuncio-gui`, and `nuncio-mcp` shells have been moved t
`_reference/` (out of the workspace) and will be rebuilt as separate client
repositories (roadmap Phase 5).
## Build & test
## Original-workspace build and test
```bash
cargo build --workspace # build everything
@@ -61,10 +113,10 @@ full local gate as three separate commands — `cargo fmt --all -- --check`,
Cargo aliases are defined in [`.cargo/config.toml`](.cargo/config.toml).
## Documentation
## Original architecture and planning references
- **[Roadmap — live rendered page](https://koftwentytwo.github.io/nuncio/roadmap/)** — the visual plan to a client-ready backend (GitHub Pages)
- [`docs/ROADMAP.md`](docs/ROADMAP.md) — authoritative roadmap and target architecture (source for the page above)
- [`docs/ROADMAP.md`](docs/ROADMAP.md) — original-workspace roadmap (source for the page above); the rebuild follows its [approved specification](docs/superpowers/specs/2026-09-10-google-first-rebuild.md)
- [`docs/STORY-WORKFLOW.md`](docs/STORY-WORKFLOW.md) — how contributors execute a roadmap story (gates, patterns, review flow)
- [`docs/HANDOFF.md`](docs/HANDOFF.md) — development handoff: current state, branches, and the M1–M7 story index
- [`docs/BACKLOG.md`](docs/BACKLOG.md) — engineering-ready Phase 0 / Phase 1 stories
+7 -16
View File
@@ -359,14 +359,13 @@ fn resolved_name(
resolve: &ResolveResult<'_>,
element: &BytesStart<'_>,
) -> Result<Qname, MultistatusError> {
let local = String::from_utf8_lossy(element.local_name().into_inner()).into_owned();
let local = element.local_name().into_inner().to_string();
let namespace = match resolve {
ResolveResult::Bound(ns) => Some(String::from_utf8_lossy(ns.0).into_owned()),
ResolveResult::Bound(ns) => Some(ns.0.to_string()),
ResolveResult::Unbound => None,
ResolveResult::Unknown(prefix) => {
return Err(MultistatusError::Malformed(format!(
"undeclared namespace prefix `{}` on element `{local}`",
String::from_utf8_lossy(prefix)
"undeclared namespace prefix `{prefix}` on element `{local}`"
)))
}
};
@@ -410,27 +409,19 @@ pub fn parse_multistatus(
walker.end(&name);
}
Event::Text(text) => {
let decoded = text
.decode()
.map_err(|e| MultistatusError::Malformed(e.to_string()))?;
walker.push_text(&decoded);
walker.push_text(text);
}
Event::CData(cdata) => {
let decoded = cdata
.decode()
.map_err(|e| MultistatusError::Malformed(e.to_string()))?;
walker.push_text(&decoded);
walker.push_text(cdata);
}
Event::GeneralRef(reference) => {
let name = reference
.decode()
.map_err(|e| MultistatusError::Malformed(e.to_string()))?;
let name: &str = reference;
if let Some(character) = reference
.resolve_char_ref()
.map_err(|e| MultistatusError::Malformed(e.to_string()))?
{
walker.push_text(&character.to_string());
} else if let Some(expansion) = resolve_predefined_entity(&name) {
} else if let Some(expansion) = resolve_predefined_entity(name) {
walker.push_text(expansion);
} else {
// Nuncio declares no DTD, so an unknown entity has no expansion; a
+1 -1
View File
@@ -26,7 +26,7 @@ chrono.workspace = true
reqwest.workspace = true
hex.workspace = true
zeroize.workspace = true
aes-gcm.workspace = true
getrandom.workspace = true
[dev-dependencies]
tempfile.workspace = true
+8 -6
View File
@@ -23,6 +23,9 @@ pub enum ParseError {
/// AST depth limit exceeded (MAX_AST_DEPTH = 10).
#[error("AST recursion depth limit exceeded (max 10)")]
MaxDepthExceeded,
/// The operating system could not supply a random rule identifier.
#[error("cannot generate rule identifier: {0}")]
Randomness(String),
}
/// Custom SQL Dialect for Nuncio SQL (NSQL).
@@ -65,7 +68,7 @@ impl NsqlParser {
let now = chrono::Utc::now().timestamp();
Ok(FilterRule {
id: format!("rule-{}", generate_rule_id()),
id: format!("rule-{}", generate_rule_id()?),
name: name_str,
target_account,
priority,
@@ -611,17 +614,16 @@ fn split_comma_outside_quotes(text: &str) -> Vec<String> {
}
/// Generates a fresh, collision-resistant rule id from the same CSPRNG
/// (`aes-gcm`'s `OsRng`) that `nuncio-store` uses to source key material,
/// (the operating system) that `nuncio-store` uses to source key material,
/// rather than deriving the id from the rule's name/NSQL text. Two rules
/// created with identical name and NSQL text MUST still get distinct ids --
/// deriving the id from their content would make the second `CreateRule`
/// silently overwrite the first via `save_filter_rule`'s `INSERT OR
/// REPLACE`.
fn generate_rule_id() -> String {
use aes_gcm::aead::{rand_core::RngCore, OsRng};
fn generate_rule_id() -> Result<String, ParseError> {
let mut bytes = [0u8; 16];
OsRng.fill_bytes(&mut bytes);
hex::encode(bytes)
getrandom::fill(&mut bytes).map_err(|e| ParseError::Randomness(e.to_string()))?;
Ok(hex::encode(bytes))
}
#[cfg(test)]
+1
View File
@@ -22,6 +22,7 @@ tracing.workspace = true
keyring.workspace = true
age.workspace = true
aes-gcm.workspace = true
getrandom.workspace = true
tempfile.workspace = true
zeroize.workspace = true
hex = "0.4"
+35 -18
View File
@@ -1,6 +1,6 @@
//! Cryptographic payload encryption and `age` streaming attachment cipher.
use aes_gcm::aead::{Aead, KeyInit, OsRng};
use aes_gcm::aead::{Aead, KeyInit};
use aes_gcm::{Aes256Gcm, Nonce};
use std::io::{Read, Write};
use thiserror::Error;
@@ -28,12 +28,12 @@ impl PayloadCipher {
pub fn encrypt_bytes(key: &[u8; 32], plaintext: &[u8]) -> Result<Vec<u8>, CipherError> {
let cipher = Aes256Gcm::new(key.into());
let mut nonce_bytes = [0u8; Self::NONCE_LEN];
use aes_gcm::aead::rand_core::RngCore;
OsRng.fill_bytes(&mut nonce_bytes);
let nonce = Nonce::from_slice(&nonce_bytes);
getrandom::fill(&mut nonce_bytes)
.map_err(|e| CipherError::EncryptionFailed(format!("OS randomness failed: {e}")))?;
let nonce = Nonce::from(nonce_bytes);
let mut encrypted = cipher
.encrypt(nonce, plaintext)
.encrypt(&nonce, plaintext)
.map_err(|e| CipherError::EncryptionFailed(e.to_string()))?;
let mut output = Vec::with_capacity(Self::NONCE_LEN + encrypted.len());
@@ -53,10 +53,11 @@ impl PayloadCipher {
let (nonce_bytes, payload) = ciphertext.split_at(Self::NONCE_LEN);
let cipher = Aes256Gcm::new(key.into());
let nonce = Nonce::from_slice(nonce_bytes);
let nonce = Nonce::try_from(nonce_bytes)
.map_err(|e| CipherError::DecryptionFailed(format!("invalid nonce: {e}")))?;
cipher
.decrypt(nonce, payload)
.decrypt(&nonce, payload)
.map_err(|e| CipherError::DecryptionFailed(e.to_string()))
}
@@ -117,18 +118,17 @@ impl PayloadCipher {
passphrase: &str,
ciphertext: &[u8],
) -> Result<Vec<u8>, CipherError> {
let decryptor = match age::Decryptor::new(ciphertext) {
Ok(age::Decryptor::Passphrase(d)) => d,
Ok(_) => {
return Err(CipherError::DecryptionFailed(
"unexpected age header format".to_string(),
))
}
Err(e) => return Err(CipherError::DecryptionFailed(e.to_string())),
};
let decryptor = age::Decryptor::new(ciphertext)
.map_err(|e| CipherError::DecryptionFailed(e.to_string()))?;
if !decryptor.is_scrypt() {
return Err(CipherError::DecryptionFailed(
"unexpected age header format".to_string(),
));
}
let identity = age::scrypt::Identity::new(secret_service_passphrase(passphrase));
let mut reader = decryptor
.decrypt(&secret_service_passphrase(passphrase), None)
.decrypt(std::iter::once(&identity as &dyn age::Identity))
.map_err(|e| CipherError::DecryptionFailed(e.to_string()))?;
let mut plaintext = Vec::new();
@@ -141,13 +141,30 @@ impl PayloadCipher {
}
fn secret_service_passphrase(raw: &str) -> age::secrecy::SecretString {
age::secrecy::SecretString::new(raw.to_string())
age::secrecy::SecretString::from(raw.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn decrypts_payload_written_by_aes_gcm_0_10() {
let encrypted = include_bytes!("../tests/fixtures/aes-gcm-0.10.bin");
let decrypted = PayloadCipher::decrypt_bytes(&[42u8; 32], encrypted)
.expect("existing AES-GCM payload remains readable");
assert_eq!(decrypted, b"Nuncio synthetic migration fixture");
}
#[test]
fn decrypts_attachment_written_by_age_0_10() {
let encrypted = include_bytes!("../tests/fixtures/age-0.10.bin");
let decrypted =
PayloadCipher::decrypt_attachment_stream("synthetic-test-passphrase-only", encrypted)
.expect("existing age attachment remains readable");
assert_eq!(decrypted, b"Nuncio synthetic migration fixture");
}
#[test]
fn aes_256_gcm_encrypt_decrypt_roundtrip() {
let key = [42u8; 32];
+4 -4
View File
@@ -1,4 +1,3 @@
use aes_gcm::aead::{rand_core::RngCore, OsRng};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::Duration;
@@ -163,7 +162,7 @@ impl SecretVault for OsKeyring {
fn delete_secret(&self, key: &str) -> Result<(), VaultError> {
let entry = keyring::Entry::new(&self.service, key)
.map_err(|e| VaultError::StorageFailed(e.to_string()))?;
match entry.delete_password() {
match entry.delete_credential() {
Ok(()) => Ok(()),
Err(keyring::Error::NoEntry) => Err(VaultError::NotFound(key.to_string())),
Err(e) => Err(VaultError::StorageFailed(e.to_string())),
@@ -212,7 +211,7 @@ impl SecretManager {
}
/// Retrieve raw key material stored under `account`, generating and persisting
/// `len` bytes of cryptographically random key material via a CSPRNG (`OsRng`) on
/// `len` bytes of cryptographically random key material via the OS CSPRNG on
/// first use if none exists yet.
///
/// This implements the mandatory key lifecycle: on first run a random key is minted
@@ -230,7 +229,8 @@ impl SecretManager {
}),
Err(VaultError::NotFound(_)) => {
let mut bytes = vec![0u8; len];
OsRng.fill_bytes(&mut bytes);
getrandom::fill(&mut bytes)
.map_err(|e| VaultError::StorageFailed(format!("OS randomness failed: {e}")))?;
self.set_secret(account, &hex::encode(&bytes))?;
// Lifecycle visibility only: which purpose got a key minted, and how long it
// is, never the key material itself.
+16
View File
@@ -0,0 +1,16 @@
# Encryption compatibility fixtures
These synthetic payloads were generated independently with the previously locked
`aes-gcm = 0.10.3` and `age = 0.10.1`, before testing the upgraded adapters.
Both decrypt to the ASCII bytes `Nuncio synthetic migration fixture`.
- `aes-gcm-0.10.bin` uses the existing storage format: a twelve-byte nonce
(`012345678901`) followed by AES-256-GCM ciphertext and tag, with a 32-byte key
whose every byte is 42. The fixed nonce is test data only.
- `age-0.10.bin` uses age passphrase encryption with the public test passphrase
`synthetic-test-passphrase-only`.
No credentials or user data are present. Compatibility tests consume these
fixed old-format bytes instead of generating them with the version under test.
The generator and its dependency lock are retained in the local verification
receipt `rebuild/test-results/dependency-security/legacy-fixture-generator/`.
+1
View File
@@ -0,0 +1 @@
012345678901.e��r������яJ�zq$��d�0�KCg�+����E�B��'�٭q�
+5
View File
@@ -0,0 +1,5 @@
age-encryption.org/v1
-> scrypt SYSpYxGL/auJCYHGdz1DEw 14
dCw5WkkpPsedMNdLtlbUCFjhDZn0ExTC6sTOakFoyuU
--- hTLGEB7KaMtZMNBfhJVb9dRP8khFFA1HPLOxzWeSavM
nÅç® ›"Ù>vÓ©‘3@‰ÊG"²>u©'K©¿Ï␍%ŒïôÐ/$Í’žjK8µøû|Eá,Ø>€[È{Fe
+81
View File
@@ -0,0 +1,81 @@
# Nuncio Rebuild — Execution Goal
**Status:** Ready to adopt. Writing this goal has not started implementation or an active execution goal.
Build the complete personal engine and CLI in the [implementation plan](superpowers/plans/2026-09-10-google-first-rebuild.md), using its [specification](superpowers/specs/2026-09-10-google-first-rebuild.md). Google Gmail and Calendar come first; Synology MailPlus email through IMAP/SMTP follows. Native applications come later.
The first checkpoint is a usable Google reader. Full completion includes drafts and sending, mail changes, calendar changes, durable operations, background synchronization, encrypted storage, export/backup/recovery, local release artifacts, and provider acceptance. The stateful mock Gmail provider is mandatory; it also covers Google OAuth and Calendar so system and subprocess E2E tests exercise the real engine through the complete path.
## Copy this to start implementation
```text
Create an active goal to fully implement the Nuncio rebuild described in:
- docs/superpowers/specs/2026-09-10-google-first-rebuild.md
- docs/superpowers/plans/2026-09-10-google-first-rebuild.md
Read both documents, the September 10 engineering audit, repository guidance,
and any existing rebuild session state before making changes. Do not set a
token budget unless I give you one.
I approve the proposed scope and engineering defaults in those documents:
engine and CLI first; Google Gmail and Calendar, then Synology MailPlus
IMAP/SMTP; Rust, SQLCipher-backed SQLite, a versioned authenticated local gRPC
API, and a separate rebuild/ workspace. Native apps are outside this goal.
Implement every task and satisfy requirements R01 through R16. Build the
independent, stateful mock Google provider before the production Google
adapter. Use it for separate system tests and actual daemon/CLI subprocess
E2E tests. Validate remote send/copy/notification effects independently from
local results, especially after crashes and lost acknowledgements. Include
strict IMAP/SMTP testing against local independent server implementations.
I authorize the local feature branch or isolated worktree, new workspace,
manifests, dependencies, schema, API, local test services, CI workflow files,
documentation, and implementation fixes required by this plan. No issue is
required for this personal build. Preserve existing source, original data,
and unrelated work. Do not commit, push, merge, publish, install into my
normal environment, or change remote settings without my authorization.
Work through the plan autonomously, implementing and verifying each task.
Use inline execution unless I request delegation. Continue past intermediate
milestones; a scaffold, mock provider, or read-only client is not completion.
Resolve failures within scope and run the relevant checks. Do not weaken
requirements, mock validation, or assertions to manufacture a passing result.
Keep rebuild/docs/SESSION-STATE.md, rebuild/docs/TODO.md, and
rebuild/docs/VERIFICATION.md current so another session can resume exactly.
Record commands, exit statuses, test evidence, implementation locations,
remaining defects, and the next concrete action. Continue from that state
after context resets. Do not restart the planning process unnecessarily.
Ask early for any account setup or access that only I can supply, using a
secure credential-entry path. All automated tests must remain offline from
real providers. Prepare a concrete manual acceptance worksheet and obtain my
explicit authorization for named live accounts, recipients, calendars, and
actions before accessing them or sending mail/invitations. Continue all
independent implementation and offline verification while that is pending.
Do not mark the goal complete until the full scope works through the engine,
API, and CLI; all required offline checks pass; local production artifacts
are verified; and the authorized Google and Synology acceptance checks have
passed. If external access is missing, report the completed work and precise
pending condition honestly and follow the goal tool's blocked-state rules.
Finish with the requirement-to-evidence matrix, actual test results, local
artifact paths, operating/recovery instructions, provider compatibility
limits, and any remaining risks. Never claim that passing mocks proves live
provider compatibility or that local CI-equivalent checks prove remote CI ran.
```
## Completion checklist
- [ ] Google account setup, Gmail and Calendar reads/writes, offline search/agenda, attachments, and background refresh work through the real CLI.
- [ ] Synology IMAP/SMTP mail works with verified TLS, placements, UID recovery, and correct Sent-copy behavior.
- [ ] Local drafts and operation intent survive restarts, resync, and backup/restore. Unknown remote outcomes cannot trigger blind duplicate sends.
- [ ] Independent mock Google conformance, real system tests, subprocess E2E, IMAP/SMTP compatibility, failure, security, and release checks pass with no ignored required tests.
- [ ] Versioned API and local production artifacts are usable; test controls and credentials are excluded from shipped builds.
- [ ] Named disposable Google and Synology accounts pass separately authorized manual acceptance, with evidence and limitations recorded.
- [ ] Every requirement R01–R16 has implementation and verification evidence. No required work remains.
The proposed implementation details become accepted when you issue the execution prompt. Until then, this is a reviewable plan, and the existing root roadmap and application remain unchanged.
+5
View File
@@ -0,0 +1,5 @@
# Nuncio Rebuild Plan
Build an engine and CLI first: Google Gmail and Calendar, then Synology MailPlus email through IMAP/SMTP. The proposed full goal includes reads and writes, durable drafts and operations, background sync, encrypted storage, recovery, and verified local release artifacts. Native applications follow separately. An independent stateful mock Gmail/Google provider is required for system and actual daemon/CLI end-to-end tests.
Read the [implementation plan](superpowers/plans/2026-09-10-google-first-rebuild.md) for the 16 ordered tasks, files, interfaces, test gates, and completion evidence; read the [specification](superpowers/specs/2026-09-10-google-first-rebuild.md) for the architecture and behavior contract. Use the copyable prompt in the [execution goal](GOAL-nuncio-rebuild.md) to authorize and start the build. These documents are proposed and implementation has not started; the [audit](reviews/2026-09-10-nuncio-audit-and-rebuild.md) explains the evidence behind the restart.
@@ -0,0 +1,152 @@
# Nuncio: Engineering Audit and Rebuild Proposal
**Date:** September 10, 2026\
**Status:** Proposed direction for discussion; implementation not authorized\
**Reviewed baseline:** `dev` at `a268fd5a004977b65fd0347ceeb1c69755529aa3`\
**Product scope:** Engine and CLI first, native applications later; Google first, then IMAP/Synology MailPlus
## Executive summary
The current project contains substantial working engineering. It is still an unreliable foundation for daily mail and calendar use: protocol compatibility, data preservation, calendar semantics, and operation recovery have material gaps. A focused restart is reasonable, but the evidence does not justify discarding Rust, SQLite, gRPC, or every existing implementation. Restart the product scope and core data/operation model, then selectively reuse verified components.
The confirmed product direction is: **engine and CLI first, native applications later; Google first, then IMAP/Synology MailPlus**. The design below is a proposal, not an approved implementation plan or a change to the repository's accepted ADRs. No source, dependencies, Git refs, or live accounts were changed.
## Scope and verification
Reviewed the nine active Rust crates, daemon composition, protocol/storage/mutation paths, calendar and contact models, filter execution, gRPC/CLI contract, authentication, encryption, recovery/export, test seams, CI/release configuration, and repository planning/history. The archived GUI/TUI/MCP shells were considered reference material, not active products. The tree contains 39 RPCs and roughly 61,600 Rust lines including tests; size is not evidence of usable capabilities.
- `cargo fmt --all -- --check`: passed.
- `CARGO_NET_OFFLINE=true cargo check-all`: passed on pinned Rust 1.97.1.
- `CARGO_NET_OFFLINE=true RUST_TEST_THREADS=2 cargo test-all`: **788 passed, 0 failed, 0 ignored**, across 32 nonempty result groups. The initial sandboxed attempt stopped with local Wiremock port-binding denials; the authorized rerun with loopback listeners enabled passed. macOS SDK tooling emitted environment warnings during compilation.
- Reproduced CLI error status: pointing `system status --json` at an unavailable loopback endpoint with a dummy token returns an error object but **process exit code 0**. No keyring or live daemon was accessed.
- Local Git-history Gitleaks scan reported one historical Tauri updater `pubkey` field. It is public verification material, not a confirmed credential leak. No credential values were printed.
- No live Google, Synology, or other mailbox/calendar integration was exercised. Findings are source traces, test inspection, one CLI reproduction, and public protocol documentation. No current Rust dependency-vulnerability scan was run because `cargo-audit` is unavailable. This is not a penetration test or an exhaustive proof that no other defects exist.
The July forensic assessment describes an older, much less functional system. It must not be copied forward as current truth. Conversely, the August handoff's completed-feature statements and manual-use guide are also stale: mutation RPCs now exist, while calendar writes still do not. Code and fresh verification take precedence.
## Findings
P1 means a blocker for trusting the affected capability with daily work. P2 means significant usability, scale, or safety work before dependable use. Several findings are already acknowledged as incomplete in comments or planning documents; they remain product gaps even when deliberately deferred.
| Priority | Finding and consequence | Evidence |
|---|---|---|
| P1 | **Calendar identities can overwrite other accounts' events.** CalDAV derives IDs from response positions, while SQLite uses a global primary key and replacement writes. A second calendar's first event can replace the first calendar's first event. Reordered/windowed results also change identity. CardDAV uses account-scoped but still position-derived IDs. | [CalDAV identity](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-cal/src/caldav.rs#L135), [calendar table](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/db.rs#L1019), [save event](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/db.rs#L2320), [CardDAV identity](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-contacts/src/carddav.rs#L127) |
| P1 | **Calendar/contact deletions never reconcile.** Sync only upserts. A deleted remote contact remains locally; a deleted recurring master continues producing appointments. Empty, complete, partial, and malformed responses need distinct semantics. | [Calendar sync](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/calendar_sync.rs#L63), [contacts sync](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/contacts_sync.rs#L52) |
| P1 | **Calendar normalization changes appointments.** The parser returns the first VEVENT and drops exception/override structure. Recurrence rebuilds DTSTART in UTC, losing the original recurring wall-clock zone across DST. Date-only DTEND is turned into 23:59:59 on an exclusive end date, extending all-day events. | [Parser](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-cal/src/parser.rs#L104), [recurrence expansion](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-cal/src/rrule.rs#L30) |
| P1 | **JMAP uses incompatible request/response fields.** It sends `inMailboxes`, requests `isUnread`/`bodySnippet`, and deserializes numeric `receivedAt`. Standard JMAP uses `inMailbox`, keyword/body properties, and a date string. Fixtures reproduce the implementation's invented representation. | [Response model](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/jmap.rs#L162), [request construction](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/jmap.rs#L274), [fixture](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/tests/wiremock_jmap_test.rs#L55); [RFC 8621](https://www.rfc-editor.org/rfc/rfc8621.html#section-4.1) |
| P1 | **JMAP full enumeration is only one page.** The code marks one query/get result complete; sync removes local placements absent from it. A provider page cap therefore causes omissions or local disappearance during resync. Fixing the wire model alone is insufficient. | [Full enumeration](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/jmap.rs#L758), [placement reconciliation](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/sync.rs#L363); [JMAP pagination](https://www.rfc-editor.org/rfc/rfc8620.html#section-5.5) |
| P1 | **Quiet IMAP delta sync can emit an invalid FETCH.** No changed UIDs becomes an empty sequence string, which is still passed to `uid_fetch`. The mock accepts malformed syntax, hiding failures on strict servers and potentially preventing removal-only reconciliation. | [Empty set](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/imap.rs#L1222), [unconditional fetch](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/imap.rs#L898), [permissive mock](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/test_server.rs#L772); [IMAP grammar](https://www.rfc-editor.org/rfc/rfc3501.html#section-9) |
| P1 | **Common IMAP folder names are not quoted.** Raw SELECT/COPY/MOVE command construction breaks names containing spaces such as `Sent Items`. Account sync can stop before processing subsequent folders. | [SELECT](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/imap.rs#L1173), [QRESYNC SELECT](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/imap_raw.rs#L297), [COPY](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/imap.rs#L1756) |
| P1 | **Received attachments disappear at persistence.** MIME parsing extracts attachments, the message INSERT does not persist them, and readback returns an empty vector. Reading or auto-forwarding a received PDF loses its attachment. | [MIME parse](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/parser.rs#L107), [save](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/db.rs#L1708), [readback](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/db.rs#L3396), [forward](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/outbox.rs#L530) |
| P1 | **Uncertain remote outcomes are replayed indiscriminately.** `Unknown` always becomes retry. IMAP COPY may have succeeded without COPYUID or before a lost acknowledgement; repeating it creates copies. Forwarding has the analogous acceptance/acknowledgement ambiguity. A remote timeout is not proof that nothing happened. | [Retry policy](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/outbox.rs#L478), [COPY result classification](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/imap.rs#L128); [COPYUID semantics](https://www.rfc-editor.org/rfc/rfc4315.html#section-3) |
| P1 | **Filter actions lose their source placement.** The rule is evaluated against a placement, but enqueue drops that address. The executor correctly refuses to guess for messages occupying multiple folders, making normal multi-label/multi-folder rule actions fail permanently. | [Enqueue](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/sync.rs#L211), [ambiguity rejection](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/outbox.rs#L424) |
| P1 | **The filter claim and outbox are not one transaction.** The fire-once claim commits before local changes, queued actions, and execution logs. A crash or failed queue write in between suppresses work that was never durably recorded. | [Claim](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/sync.rs#L139), [subsequent writes](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/sync.rs#L174) |
| P1 | **Automatic repair resumes without durable local state.** Salvage restores accounts/filter/audit tables, then replaces the active database. It omits local contacts, queued actions/conflicts, and fire-once records. Upstream resync cannot restore local-only intent. A backup may permit manual recovery; the replacement is not a complete recovery. | [Salvaged table set](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/recovery.rs#L217), [replacement](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/recovery.rs#L254) |
| P1 | **Repair ignores failed WAL backup.** It ignores WAL copy failure, then removes the original WAL. Under I/O failure or disk pressure, committed transactions may disappear from both active state and the backup. | [Ignored copy](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/recovery.rs#L168), [WAL deletion](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/recovery.rs#L260) |
| P1 | **MBOX/EML exports are lossy.** They reconstruct plaintext messages instead of preserving MIME, losing HTML-only bodies, attachments, and original headers/identity. MBOX body separator lines are not escaped. Successful counts do not establish backup fidelity. | [MBOX writer](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-core/src/export.rs#L107), [EML writer](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-core/src/export.rs#L137) |
| P1 | **An export path can truncate the live database.** The daemon accepts an output path and uses `File::create`. An authenticated client's mistaken destination or symlink can overwrite the store. This is an accidental-data-loss hazard, not an unauthenticated exploit. | [Output creation](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/db.rs#L3594), [RPC boundary](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/grpc.rs#L3235) |
| P2 | **CLI errors return success to the shell.** The runner returns formatted strings and `main` always finishes with `Ok(())`. Reproduced with an unreachable daemon. Automation must currently parse the body to discover failure. | [CLI entry point](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-cli/src/main.rs#L35) |
| P2 | **Calendar command defaults are unusable.** `cal sync` supplies `i64::MAX` as its end, rejected by the production date conversion. The analogous listing bound fails recurrence expansion, which can fall back to raw masters. | [Defaults](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-cli/src/args.rs#L495), [conversion](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-cal/src/caldav.rs#L162), [fallback](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/grpc.rs#L1178) |
| P2 | **Configured background sync is not wired.** Intervals are persisted, but daemon sync waits for explicit commands/RPCs; IMAP IDLE helpers have no production caller. The active timers drain the outbox and optionally check updates. This is a manually triggered engine, not continuously current mail/calendar. | [Daemon command loop](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/main.rs#L116), [IDLE helper](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-mail/src/imap.rs#L2007) |
| P2 | **Client push can lose changes without a recovery signal.** The broadcast channel is best-effort; lag is logged server-side and silently skipped. Sync progress is explicitly dropped from gRPC. There is no cursor/revision for clients to detect a gap and refresh consistently. | [Event mapping](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/grpc.rs#L220), [lag handling](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/grpc.rs#L351) |
| P2 | **Startup reads the whole database to check 16 bytes.** Memory and startup cost grow with the entire mailbox database. | [Header probe](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/recovery.rs#L111) |
| P2 | **Webhook failure logs can contain URL credentials.** Raw `reqwest::Error` includes the failed URL; a secret in its path/query can leak despite host-only logging on other paths. | [Failure logging](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-filter/src/webhook.rs#L227) |
## Security, completeness, and engineering assessment
Useful security controls are real: loopback-only binding, constant-time bearer checking, an authentication canary across mounted services, OS-keyring abstraction, explicit decryption failures, and webhook address validation/pinning. Preserve these. Full mail confidentiality is not provided: FTS contains recoverable plaintext bodies ([explicit schema comment](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nuncio-store/src/db.rs#L1150)). Raw HTML reaches the API; sanitization and blocked remote resources remain a future renderer boundary, not a demonstrated current daemon XSS exploit.
Calendar is read-only at the API: Sync/List/Get, without create/edit/delete, RSVP, reminders, or free/busy. Contacts creation is local-only. Sending is real SMTP acceptance but bypasses durable submission storage and returns a generated nonpersisted identifier ([send design](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/crates/nunciod/src/send.rs#L33)). The active account/provider model has no Google OAuth or Gmail/Google Calendar adapter. Adding Gmail through the present IMAP path would not solve that missing Google-native account lifecycle.
Build hygiene is much stronger than the original prototype. Test confidence is uneven: tests often verify an adapter and a mock that share the same misunderstanding. Counts and descriptor goldens do not establish conformance, backward compatibility, or client feature parity. A shared API makes capabilities available; each client still needs implementation and acceptance coverage. The descriptor golden detects changes but can be replaced alongside a breaking change without a semantic compatibility check.
Module boundaries are reasonable at crate level but concentrated in implementation: `grpc.rs` is about 8,700 lines, `db.rs` about 6,800, and the CLI runner about 5,700, including tests. Handlers mix orchestration, mapping, storage, and domain decisions; the CLI also depends directly on most engine crates. In a rebuild, split by capability and put application operations between transport handlers and storage. Do not replace this with dozens of services or speculative interfaces.
CI scans JavaScript and Actions through CodeQL, not active Rust. Release publishing depends on successful binary builds and has no independent test/security dependency ([workflow](https://github.com/KofTwentyTwo/nuncio/blob/a268fd5a004977b65fd0347ceeb1c69755529aa3/.github/workflows/release.yml#L81)). Coverage is informational. Native-client packaging/codegen/runtime compatibility has not been validated in this audit. The repository is honestly labeled pre-alpha; preserve that posture until a complete personal workflow is demonstrated.
## Three ways forward
| Approach | Benefit | Cost | Assessment |
|---|---|---|---|
| Continue the existing roadmap | Keeps all accumulated code and issue history | Repairs broad protocol scope, stale contracts, and data-model problems before reaching Google-first usefulness | Viable, but poorly matched to the newly confirmed priority |
| Focused restart with selective reuse | Clean Google-first domain/API; retains proven Rust, SQLite, auth and test mechanisms | Requires a new vertical and careful review of reused code | Recommended |
| Entirely new implementation in another language | Removes legacy assumptions and can exploit different SDKs | Rebuilds working infrastructure, introduces new unknowns, and still faces the same mail/calendar semantics | No current evidence justifies this cost |
Keep the old checkout as reference. Do not bulk-copy its database schema, protocol fixtures, calendar entity, or backlog into the new core. Port components only when the new end-to-end tests demonstrate their behavior.
## Proposed architecture
One user, one daemon per machine, multiple accounts, one local SQLite database. The provider remains authoritative for synchronized objects; the local store is authoritative for drafts, queued operations, local settings, and action history. Those two categories need different reset/backup policies even if they live in one database.
```mermaid
flowchart TD
CLI[Reference CLI] --> API[Versioned gRPC API]
Native[Native apps later] --> API
API --> App[Application operations and queries]
App --> DB[SQLite: objects, raw payloads, search, operation journal]
App --> Sync[Account sync and operation workers]
Sync --> Google[Gmail and Google Calendar adapters]
Sync --> IMAP[IMAP and SMTP adapter for MailPlus]
Sync --> DAV[Optional later Synology Calendar adapter]
Sync --> Keys[OS keystore and account authorization]
```
Keep **Rust, Tokio, SQLite, protobuf/tonic, and Clap** as the default stack. This is a local modular application, not a reason to add Postgres, Redis, a broker, Kubernetes, or peer-to-peer synchronization. Keep daemon, CLI, schema, and integration fixtures in one workspace initially. Decide native-client repository layout when a real second client exists.
The daemon owns credentials, network access, storage, synchronization, and business decisions. The CLI owns argument parsing, display/JSON formatting, standard input/output, and correct exit status. Long-running commands return a durable operation ID; the CLI can wait or inspect it. Keep the existing loopback bearer model initially, with an explicit local pairing/bootstrap path and no network-server mode.
Use a small application layer with operations such as `list_messages`, `queue_send`, `set_labels`, `trash_message`, `list_occurrences`, and `update_event`. RPC handlers translate and validate requests, then call that layer. Provider adapters preserve provider capabilities instead of making all services pretend to be IMAP folders. Sharing application operations is more important than forcing mail and calendar into one universal entity.
### Providers and authorization
**Google first:** Gmail API for mail, Google Calendar API for calendar, desktop browser-based OAuth with PKCE/state and keychain-held refresh credentials. Validate authorization setup and refresh/revocation early. Google documents desktop loopback redirects and the token lifecycle; external OAuth projects left in Testing generally have seven-day refresh tokens for these scopes, so unattended use needs an intentional setup. [Desktop OAuth](https://developers.google.com/identity/protocols/oauth2/native-app), [token expiration](https://developers.google.com/identity/protocols/oauth2#expiration).
Gmail history IDs and Google Calendar sync tokens are separate cursors. Expired Gmail history yields 404 and requires full synchronization; expired Calendar sync tokens yield 410. Exhaust all pages and commit object changes plus the final cursor atomically. Reset only the affected synchronized projection, preserving drafts and queued intent. [Gmail sync](https://developers.google.com/workspace/gmail/api/guides/sync), [Calendar sync](https://developers.google.com/workspace/calendar/api/v3/reference/events/list).
**Synology second:** implement IMAP/SMTP with capability negotiation, strict TLS verification, correct mailbox encoding, UIDVALIDITY-aware addressing, incremental updates, deletion reconciliation, and safe mutation semantics. Do not assume a particular optional IMAP extension is enabled without testing the target configuration. Synology documents IMAP and SMTP client support. Calendar, if needed on the NAS, is a separate Synology Calendar/CalDAV integration; it is not implied by MailPlus support. [MailPlus specifications](https://www.synology.com/dsm/7.2/software_spec/mailplus), [Synology Calendar CalDAV](https://kb.synology.com/DSM/tutorial/How_to_Sync_Synology_Calendar_with_CalDAV_Clients/).
JMAP, Microsoft 365, contacts synchronization, and other providers are outside the first release unless requirements change. No public webhook endpoint is required for the initial local daemon: use scheduled incremental synchronization and catch-up after sleep. Add push only when its operational cost is justified.
### Data model and fidelity
- **Mail identity:** account-scoped local ID plus explicit provider identity. Separate messages, conversations, mailbox/label membership, raw MIME, and cached parsed views. Gmail labels retain label semantics; IMAP memberships retain mailbox + UIDVALIDITY + UID. Message-ID is metadata, not a globally unique primary key. Never merge messages across accounts just because headers match.
- **Payload preservation:** retain original RFC822/MIME when fetched, or record that it is not yet downloaded and can be fetched by provider identity. Attachments, originals, and exports must survive the fetch-store-read boundary. Raw bytes are the basis for faithful export; parsed views are rebuildable. Fetch large bodies/attachments on demand with size limits and streaming.
- **Calendar identity/time:** account + calendar + provider event ID; retain original provider representation, recurrence master and instance identity, original occurrence time, timezone, date-only versus date-time, exclusive ends, cancellations, overrides, attendees, and version/ETag. `iCalUID` alone does not identify a particular instance. Google explicitly models exceptions and cancelled instances. [Calendar event model](https://developers.google.com/workspace/calendar/api/v3/reference/events).
- **Calendar expansion:** initially use provider-expanded occurrences for a declared cached window while retaining canonical masters/exceptions. Report cache coverage honestly. Do not handwrite a recurrence engine; if fully offline expansion is required, qualify an existing library against DST/exception fixtures before adopting it.
- **Operations:** durable ID, explicit account/resource, desired change, precondition/version, attempts, last error, and state. Keep local-only records through provider resyncs. Use migrations and backup/restore tests before retaining valuable drafts or pending sends.
The persistence rule is simple: **save what cannot be reconstructed; rebuild only what can**. Recovery writes a separate candidate database, validates it, and preserves the original until explicitly promoted. Export streams to the client, whose file handling must not overwrite daemon-owned files.
For privacy, choose one coherent at-rest policy before real data: SQLite on an explicitly encrypted local volume is the simplest personal-tool baseline; if application-level encryption is required, use a vetted whole-database solution that also protects search and the journal. Do not recreate the present split where encrypted body columns coexist with plaintext copies in FTS. This choice needs a small packaging/backup spike rather than more custom cryptography.
### Synchronization, actions, and client state
Serialize conflicting work per account, bound concurrency across accounts, use cancellation and provider-aware backoff, and resync after wake/network recovery. Track initial sync, catching up, current, authentication required, offline, and failed separately. Keep partial snapshots from deleting cached objects. Show last successful synchronization and outstanding operations.
Persist local changes and outbound intent in one SQLite transaction. The worker performs remote I/O after commit. States should distinguish `queued`, `running`, `applied`, `conflict`, `failed`, and `outcome uncertain`. Retry only when safe for that operation. SMTP acceptance followed by connection loss does not support a universal exactly-once guarantee; reconcile where possible and expose uncertainty when it cannot be resolved. The same applies to forwarding and COPY.
A read/label change may be shown optimistically with pending status; sending should expose draft/queued/submitting/accepted/uncertain states. Require explicit sending account, preserve recipient arrays and reply headers, and keep a recoverable sent-operation record. Local operation deduplication prevents duplicate enqueue, not necessarily duplicate remote delivery.
Make client notifications recoverable without constructing a general event-sourcing platform: attach a database revision to snapshots and change notifications; a gap or expired cursor tells clients to refresh. Surface operation/sync progress. Golden protobuf descriptors are useful, but add semantic compatibility checks and a tiny generated native-client smoke test before freezing the public API.
## Implementation order and acceptance gates
| Stage | Deliverable | Evidence required to advance |
|---|---|---|
| 1. Walking skeleton | Daemon, CLI, SQLite, local auth, one Google account authorization/refresh path | Start/stop/restart, keyring locked/denied, daemon unavailable, JSON and nonzero errors, no sensitive logs |
| 2. Google read vertical | List/read/search Gmail, retrieve original MIME/attachments, Google Calendar agenda | Fixture mail/calendar visible through the actual CLI; restart preserves IDs; offline reads disclose cache coverage |
| 3. Reliable change ingestion | Gmail history, Calendar tokens, paging, deletion, background schedule, wake/catch-up | More than one page, expired cursors, duplicate pages, partial failures, external edits/deletes, interrupted transaction |
| 4. Daily-use write vertical | Draft/reply/send and label/trash actions; calendar create/edit/delete and invitation responses with explicit scope | Crash at each operation boundary; accepted-but-unacknowledged send; conflict with external client; DST and moved/cancelled recurring instance |
| 5. Personal dogfood gate | Backup/restore, lossless export, diagnostics, account removal, sustained use with one account | Deliberate disposable-account exercise first; then opted-in real-account use alongside the existing client; no unexplained missing/duplicate work |
| 6. Synology adapter | IMAP/SMTP using the same application contract | Strict independent test server, spaces/non-ASCII mailboxes, empty/no-change sync, UIDVALIDITY reset, remote moves/deletes, network interruption |
| 7. Native-client readiness | Freeze the exercised contract, generate a minimal native client, package daemon/CLI | Native client can authorize, query, mutate, observe progress, and recover from disconnect through the same API |
Calendar representation is designed at stage 1 even though calendar writes arrive later. Each stage delivers a runnable engine + API + CLI path. There should not be months of engine-only features waiting for a later API-completeness campaign. A reliable schedule requires the OAuth and provider-fixture spikes first. As an initial planning judgment, allow several months of part-time work; this is not a delivery commitment.
Tests must challenge implementation assumptions. Retain fast unit tests and mocked failure injection, but add fixtures independently grounded in provider schemas and strict local protocol servers. Specifically require multi-account identity, reordered DAV results, complete/incomplete paging, all-day exclusive ends, DST recurrence, attachment/export round trips, and accepted-but-acknowledgement-lost operations. Use live provider acceptance only as an explicit disposable-account exercise, separate from normal offline CI.
Defer the NSQL language, autonomous AI actions, MCP, plugins, WORM-style audit machinery, automatic salvage, bespoke updater, and simultaneous native clients. A plain action journal, safe storage, and a reliable daily workflow earn priority. Existing NSQL parsing and other useful libraries can return when a concrete workflow needs them.
## Next design discussion
The remaining product decision is the first daily workflow to optimize: inbox triage and reply, combined inbox/agenda review, or scheduling from email. Also confirm whether Synology is mail-only or includes Synology Calendar, what offline editing must support, and whether application-level encryption beyond encrypted local storage is required. These refine the proposal; they do not block the audit findings above.
@@ -0,0 +1,521 @@
# Nuncio Google-First Rebuild Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:executing-plans` to implement this plan task by task. Steps use checkboxes for tracking. Execute inline unless James requests delegation. Checkpoints record evidence; they do not require another approval for work already authorized by the execution goal.
**Goal:** Fully implement a dependable personal email/calendar engine and CLI for Google, followed by Synology MailPlus email, with independent provider mocks, system tests, subprocess E2E, recovery, and verified local release artifacts.
**Architecture:** A separate Rust workspace under `rebuild/` contains the headless engine, versioned protobuf contract, daemon, CLI, and unshipped test support. The daemon owns credentials, encrypted SQLite, synchronization, and durable operation execution. Provider projections can be rebuilt; drafts and operation intent survive resync and recovery.
**Tech Stack:** Rust 2021 / 1.97.1, Tokio, tonic/prost, Clap, serde, reqwest/rustls, rusqlite with SQLCipher/FTS5, OS keyring, established MIME and IMAP/SMTP libraries.
**Spec:** [Google-first rebuild specification](../specs/2026-09-10-google-first-rebuild.md). Read it together with this plan, the [audit](../../reviews/2026-09-10-nuncio-audit-and-rebuild.md), and the [execution goal](../../GOAL-nuncio-rebuild.md).
**Status:** Accepted and activated by James on September 10, 2026. Execution state is in `rebuild/docs/SESSION-STATE.md` within `.worktrees/nuncio-google-first-rebuild`. This plan governs the isolated rebuild; the root roadmap remains historical guidance for the original application. Latest steering requires full local mock services for both Google and Synology-compatible IMAP/SMTP; live-account acceptance is deferred.
## Global Constraints
- Primary personal runtime: macOS. Linux runs the offline CI suites. Native applications and Windows installation are outside this goal.
- Google Gmail and Google Calendar first; Synology MailPlus through IMAP/SMTP second. No Synology Calendar/CalDAV in this goal.
- Four production crates: `nuncio-engine`, `nuncio-proto`, `nunciod`, `nuncio-cli`. `nuncio-test-support` is never shipped.
- Use Rust 2021, the repository's pinned Rust 1.97.1, and warnings as errors. No production `unsafe`, `unwrap`, `expect`, `panic`, or placeholder success paths.
- Default endpoint `127.0.0.1:9421`; profile root `~/.nuncio-rebuild`; keyring service `mx.nuncio.rebuild`; API package `nuncio.v2`.
- SQLite/FTS/payloads/journal are encrypted through SQLCipher. Use a dedicated storage thread, typed requests, and memory temporary storage. No plaintext fallback.
- Blob chunks: 256 KiB. Default maximum single payload: 64 MiB, configurable downward. Bound raw, decoded, streamed, and decompressed sizes independently.
- Calendar agenda coverage: 30 days before today through 180 days after today, separate from canonical sync tokens.
- Default polling: 60 seconds; active-account limit: 2; request deadline: 30 seconds; retained change records: 10,000.
- All automated tests use synthetic data and local providers. No live credentials or live provider calls in tests or CI.
- Every capability needs engine behavior, authenticated RPC, CLI support, and offline E2E. Mock behavior alone cannot satisfy a product requirement.
- Preserve the original workspace, original data, unrelated changes, and old API. Do not migrate existing user state under this goal.
- Local implementation and verification are authorized only when the goal is adopted. Commits, pushes, merges, publication, installation into the user's PATH, and live provider actions require their stated authorization.
## Delivery sequence
| Milestone | Tasks | Reviewable outcome |
|---|---|---|
| A — foundation and test system | 01–04 | Encrypted daemon/CLI runs; independent mock Google, real test harness, and OAuth lifecycle work. |
| B — usable Google reader | 05–08 | Two-account Gmail and Calendar sync, offline mail/search/agenda, background recovery, observable API. |
| C — usable Google client | 09–11 | Durable drafts, sending, mail changes, calendar changes, conflicts, and uncertain outcomes. |
| D — full provider scope | 12–13 | Synology IMAP/SMTP plus safe export, backup, restore, and migrations. |
| E — completed release candidate | 14–16 | Failure/security checks, local packages, required CI definitions, and explicit provider acceptance evidence. |
Complete each milestone before starting the next. Complete Gmail and Calendar read-only acceptance before adding write operations. Establish the mock before implementing the production Google adapter. A milestone is a checkpoint, not permission to stop the overall goal.
## File and interface map
All paths in the following map are relative to `rebuild/`. Each task below identifies its files within this map. Create Rust module declarations and crate manifests as their first consumers need them; do not create empty speculative modules.
| Path | Responsibility |
|---|---|
| `Cargo.toml`, `Cargo.lock`, `rust-toolchain.toml`, `.cargo/config.toml` | Independent workspace, pinned toolchain/dependencies, lint settings. |
| `crates/nuncio-engine/src/domain/{identity,mail,calendar,operations}.rs` | Stable account-scoped identity, time, draft, and operation types. |
| `crates/nuncio-engine/src/store/{mod,worker,migrations,mail,calendar,operations,backup}.rs` | SQLCipher connection and typed storage operations on a dedicated thread. |
| `crates/nuncio-engine/src/secrets/{mod,keyring,test_store}.rs` | Secret interface, production OS keyring, feature-gated deterministic test secret store. |
| `crates/nuncio-engine/src/{engine,accounts,mail,calendar,operations,scheduler,changes}.rs` | Application operations; all provider/store composition remains here. |
| `crates/nuncio-engine/src/providers/google/{http,oauth,gmail,calendar}.rs` | Google wire types, HTTP requests, pagination, error classification. |
| `crates/nuncio-engine/src/providers/{imap,smtp}.rs` | IMAP placements, capability negotiation, SMTP delivery and Sent bookkeeping. |
| `crates/nuncio-proto/proto/nuncio/v2/{common,system,accounts,mail,calendar,operations,maintenance}.proto` | Public service contract and field documentation. |
| `crates/nuncio-proto/{build.rs,src/lib.rs,src/client.rs,tests/contract.rs}` | Bindings, authenticated client helper, descriptor compatibility test. |
| `crates/nunciod/src/{main,config,server}.rs`, `src/services/*.rs` | Lifecycle, configuration, one thin handler module per service, authentication. |
| `crates/nuncio-cli/src/{main,args,output}.rs`, `src/commands/*.rs` | CLI grammar, client-side file handling, output, exit statuses. |
| `crates/nuncio-test-support/src/google/{mod,state,oauth,gmail,calendar,faults}.rs` | Independent stateful Google HTTP service. |
| `crates/nuncio-test-support/src/{fixtures,process,imap}.rs`, `src/bin/mock-google.rs` | Synthetic fixtures, subprocess harness, independent mail-server orchestration. |
| `crates/nuncio-test-support/tests/support/{mod,system}.rs` | System harness composition using production engine/proto dev dependencies, excluded from the mock binary. |
| `crates/nuncio-test-support/fixtures/{google,mime,calendar}` | Hand-authored wire examples and expected bytes, with source/provenance notes. |
| `crates/nuncio-test-support/tests/*.rs` | Provider conformance, system, E2E, compatibility, and failure scenarios. |
| `scripts/{verify.py,package.py}`, `tests/imap/compose.yaml` | Reproducible validation, local artifacts, pinned local IMAP/SMTP services. |
| `docs/{SESSION-STATE.md,TODO.md,VERIFICATION.md,RUNNING.md,TESTING.md,API.md,RECOVERY.md}` | Execution state, evidence, operation and client contracts. |
`Engine` is the application entry point, holding provider registries, store handle, and workers. Its methods consume domain request types and return `Result<T, EngineError>`. RPC handlers convert protobuf values into these types; the CLI uses protobuf clients only. Avoid a universal provider trait that erases Gmail labels, IMAP placements, or calendar versions. Share transport/retry utilities where semantics actually match.
The operation envelope is common; action payloads are typed variants:
```rust
pub struct RequestIdentity {
pub account_id: AccountId,
pub request_id: String,
}
pub enum OperationState {
Queued,
Running,
Applied,
RetryWait,
Conflict,
Uncertain,
Failed,
Cancelled,
}
```
Task 01 defines `AccountId` as a validated opaque local identifier. Task 09 defines `OperationId`, `Operation`, the canonical payload fingerprint, and the full transition implementation. The request ID is client-generated; a new RPC transport request is not a new operation.
### RPC ownership
The names below are the required public methods, grouped by the task that introduces them. Define request/response fields alongside each vertical; reserve deleted field numbers and update the descriptor golden deliberately.
| Task | Service methods |
|---|---|
| 01 | `System.GetStatus`, `System.Shutdown` |
| 04 | `Accounts.BeginGoogleAuth`, `Accounts.GetAuthStatus`, `Accounts.ListAccounts`, `Accounts.DisconnectAccount` |
| 05 | `Accounts.SyncAccount`, `Accounts.GetSyncStatus`, `Accounts.CancelSync`, `Mail.ListCollections`, `Mail.ListMessages`, `Mail.GetMessage`, `Mail.SearchMessages`, `Mail.FetchMessage`, `Mail.DownloadRaw`, `Mail.DownloadAttachment` |
| 07 | `Calendar.ListCalendars`, `Calendar.GetEvent`, `Calendar.ListAgenda`, `Calendar.RefreshAgenda` |
| 08 | `System.WatchChanges` and expanded sync status in `System.GetStatus` |
| 09 | `Mail.SaveDraft`, `Mail.ListDrafts`, `Mail.GetDraft`, `Mail.DeleteDraft`, `Mail.PrepareReply`, `Mail.PrepareForward`, `Mail.UploadAttachment`; `Operations.ListOperations`, `Operations.GetOperation`, `Operations.CancelOperation`, `Operations.ResolveOperation` |
| 10 | `Mail.SendDraft`, `Mail.ChangeMessage` |
| 11 | `Calendar.ChangeEvent`, `Calendar.QueryFreeBusy` |
| 12 | `Accounts.AddImapAccount`; provider capability handling in existing mail methods |
| 13 | `Maintenance.CreateBackup`, `Maintenance.InspectBackup`, `Maintenance.RestoreBackup`, `Maintenance.RepairProjection` |
`PrepareReply` takes explicit reply/reply-all mode. `ChangeEvent` has typed create/update/delete/respond variants and explicit series/occurrence scope. Download/create-backup responses stream bounded byte chunks; uploads/restore use bounded client streams. File paths are client concerns, never arbitrary daemon file-read RPCs. Auth sessions and sync-run handles are inspectable separately from durable write operations.
## Execution method and verification commands
For each checked step, record the implementation path and evidence in `rebuild/docs/VERIFICATION.md`. Start each behavior with a failing assertion that demonstrates the intended failure, implement it, then rerun that assertion. Do not interpret a compiler error or a missing binary as proof that the behavior was exercised. Finish each task with its whole suite and applicable lint checks. Record the next task and unresolved issues in the local session-state files. Commit only if separately authorized.
Task 03 creates a Python standard-library runner with this stable interface, invoked from the repository root:
```bash
python3 rebuild/scripts/verify.py --suite google_mock_contract
python3 rebuild/scripts/verify.py --suite google_system
python3 rebuild/scripts/verify.py --suite google_e2e
python3 rebuild/scripts/verify.py --all
```
`--suite NAME` builds test-harness daemon/CLI binaries in `rebuild/target/test-harness`, sets absolute `NUNCIO_E2E_DAEMON` / `NUNCIO_E2E_CLI` paths, and runs `cargo test --manifest-path rebuild/Cargo.toml -p nuncio-test-support --test NAME`. `--all` runs fmt, Clippy, unit/contract/system/E2E/IMAP suites, then release-isolation checks. It uses `RUST_TEST_THREADS=2`, checks every subprocess exit status, and fails when required prerequisites or suites are missing. Test integration code fails with an actionable message if binary paths are absent; it never silently skips.
Run lint in both normal and test-harness configurations. Build release artifacts in a different target directory with production packages selected and **without** test features, avoiding Cargo feature unification contaminating release checks. The equivalent basic gates are:
```bash
cargo fmt --manifest-path rebuild/Cargo.toml --all -- --check
cargo clippy --manifest-path rebuild/Cargo.toml --workspace --all-targets -- -D warnings
cargo clippy --manifest-path rebuild/Cargo.toml --workspace --all-targets --features nunciod/test-harness -- -D warnings
```
The runner supplies the E2E binary environment when invoking workspace tests. Before Task 03 exists, run the exact package/test commands stated below. Root gates remain unchanged; run them if root runtime/build files are touched. Sandbox port denial is an environment restriction: request narrow escalation, retaining isolation and synthetic data.
## Task 01 — Isolated workspace, encrypted lifecycle, and status vertical
**Depends on:** Adoption of the execution goal. **Requirements:** R01, R12, R15.
**Files:** Create workspace manifests/config, `domain/identity.rs`, `store/{mod,worker,migrations}.rs`, `secrets/{mod,keyring}.rs`, engine entry point, System proto/client, daemon lifecycle, CLI system commands, `nuncio-engine/tests/store_lifecycle.rs`, and `nunciod/tests/lifecycle.rs`.
**Interfaces:** Define `AccountId` and `ProfileId` string newtypes with validated parsing; `SecretStore` get/put/delete of zeroizing secret bytes by profile/account/key purpose; `Store::open(StoreConfig, SecretStore)`; async typed store requests; `Engine::open(EngineConfig)`; `Engine::status()`; `Engine::shutdown()`. Status contains build/API version, profile ID, account states, storage state, revision, and protected data paths, never key material.
- [x] Record the starting SHA and dirty files; create/use `feature/nuncio-google-first-rebuild` only under the adopted goal's branch authorization. Carry plan/spec into an isolated worktree if one is needed; do not stash or move unrelated work. Record no issue required for this personal build when the goal says so.
- [x] Create the nested workspace, lints, manifests, and minimal store/lifecycle tests. Verify an encrypted store opens with its key, fails with a wrong key and ordinary SQLite, and retains data through close/reopen.
- [x] Implement the storage worker and migration transaction. Turn on foreign keys, WAL, bounded busy timeout, memory temp storage, and required SQLCipher settings before data access. Use bounded queues; never block a Tokio worker on SQLite. A second daemon cannot own the same store.
- [x] Implement OS keyring access and distinct database/API/provider key purposes. A missing key for an existing database fails closed; never generate a replacement key over it. Create data directories with restrictive permissions.
- [x] Implement authenticated status and graceful shutdown through the real CLI. Bind only loopback, acquire profile lock first, stop admission and workers before closing store. Read only bounded header bytes during startup checks.
- [x] Run `cargo test --manifest-path rebuild/Cargo.toml -p nuncio-engine --test store_lifecycle` and `cargo test --manifest-path rebuild/Cargo.toml -p nunciod --test lifecycle`; run the normal fmt/Clippy gates. Record wrong-key, second-owner, non-loopback, and missing-daemon exit results.
Core schema identity rule to test, not merely document:
```sql
CREATE TABLE accounts (id TEXT PRIMARY KEY NOT NULL);
CREATE TABLE messages (
account_id TEXT NOT NULL REFERENCES accounts(id),
id TEXT NOT NULL,
provider_id TEXT NOT NULL,
PRIMARY KEY (account_id, id),
UNIQUE (account_id, provider_id)
);
```
This is the identity fragment of migration 1; add message fields when Task 05 introduces them. Memberships reference `(account_id, id)`. Identical provider IDs in different accounts must coexist.
## Task 02 — Independent stateful mock Google
**Depends on:** 01. **Requirements:** R13, R14.
**Files:** Create test-support crate, Google modules, fixtures/provenance notes, `src/bin/mock-google.rs`, and `tests/google_mock_contract.rs`.
**Interfaces:** `MockGoogle::start(Seed) -> Result<MockGoogle, TestError>` binds ephemeral loopback sockets; `Seed::TwoAccounts` supplies deterministic synthetic account identifiers. `MockGoogle::base_url()`, `control()`, and async `shutdown()`. `GoogleControl` exposes typed seed/edit/delete operations, `set_page_cap(usize)`, `expire_cursor(account, scope)`, `inject(Fault)`, `snapshot()`, and `accepted_sends(account)`. Snapshots expose remote IDs, labels, events, and effect counters. Fault selectors identify endpoint, account, call ordinal, and before/after-acceptance phase.
- [x] Write raw HTTP conformance tests independent of engine/proto structs. Cover valid and invalid field names/types, identity, label semantics, pagination, OAuth state/PKCE/code reuse, and history/sync expiry.
- [x] Implement separate mock wire structs or hand-authored JSON schemas. Reject unsupported routes and invalid operations explicitly. Use synthetic MIME and fixed calendar occurrence fixtures grounded in the specification's official references.
- [x] Implement all required OAuth/Gmail/Calendar endpoint subsets from spec section 6. Store accounts, tokens, messages, labels, histories, calendars, versions, recurring instances, and side-effect counters independently of the client.
- [x] Implement opaque page tokens scoped to query/account/generation, with repeat requests returning the same page until expiry; keep provider sync cursors separate. Enforce configured page caps even if the client requests more. Include sparse history IDs above 2^53 as JSON strings. OAuth authorization codes, unlike page tokens, are one-use.
- [x] Add fault controls for every listed status/body/connection fault, including apply-then-drop. Prove the drop occurs after state/effect counters change. Keep controls on an out-of-band harness channel, inaccessible to the adapter's base URL.
- [x] Add standalone reset/seed/start operation, readiness file, deterministic shutdown, and usage instructions. Run `cargo test --manifest-path rebuild/Cargo.toml -p nuncio-test-support --test google_mock_contract`.
Required hand-authored history response fixture:
```json
{
"history": [{
"id": "9007199254740997",
"labelsAdded": [{
"message": {"id": "m-001", "threadId": "t-001"},
"labelIds": ["STARRED"]
}]
}],
"historyId": "9007199254741009",
"nextPageToken": "page-account-a-2"
}
```
The next response has no page token and returns the final durable `historyId`. Contract tests assert that using a page token as `startHistoryId` fails. The mock's fixture provenance includes a source URL and the field/behavior justified by it; credentials and personal content are forbidden.
## Task 03 — System and subprocess E2E harnesses
**Depends on:** 01–02. **Requirements:** R01, R12–R15.
**Files:** Create test-support `src/{process,fixtures}.rs` and `tests/support/{mod,system}.rs`; feature-gated engine test secret store and daemon test configuration; `scripts/verify.py`; `tests/{google_system,google_e2e,release_isolation}.rs`; `docs/TESTING.md`.
**Interfaces:** `SystemHarness` runs real Engine, SQLCipher, and authenticated RPC on loopback with real HTTP adapters. `E2eHarness` owns actual child binaries, profile directory, mock Google, and test credentials. Both support bounded shutdown; E2E additionally supports force-kill/restart while retaining profile and mock state. `CliOutput` contains `status: i32`, stdout/stderr bytes, and `json() -> Result<serde_json::Value, TestError>`. `E2eHarness::cli(&[&str]) -> Result<CliOutput, TestError>` invokes the binary directly without a shell. Fixtures and query helpers use public API; the mock never imports engine provider serialization types.
- [x] Add a process test for successful authenticated status, unauthenticated rejection, JSON stderr separation, and nonzero missing-daemon exit. Before implementation, assert the precise missing behavior.
- [x] Implement readiness-file plus authenticated-health waiting, deadline-based polling, child log capture with redaction, and cleanup on success/failure. Preserve logs on test failure. No fixed sleeps or global fixed ports.
- [x] Add test-only injection for loopback provider URLs, dummy secrets, clock/limits, and deterministic operation barriers. Compile it only with `test-harness`; production cannot enable it through an environment variable. Test credentials are synthetic and cannot name production keyring entries.
- [ ] Implement the runner defined above. Clear inherited provider credential/config variables for children, bind all mocks locally, and configure HTTP clients without ambient proxies in tests. CI denies external provider egress while allowing loopback and pre-pulled local containers.
- [x] Add release binary rejection of every test-only flag/env setting. Keep feature and production builds in separate target directories.
- [x] Run `python3 rebuild/scripts/verify.py --suite google_e2e` and `--suite release_isolation`. Capture the subprocess exit codes, not just logged result objects.
Required executable assertion shape in `google_e2e.rs`:
```rust
let h = E2eHarness::start(Seed::TwoAccounts).await?;
let output = h.cli(&["--json", "system", "status"]).await?;
assert_eq!(output.status, 0);
assert!(output.json()?["result"]["profile_id"].is_string());
h.shutdown().await?;
let offline = h.cli(&["--json", "system", "status"]).await?;
assert_eq!(offline.status, 4);
```
`shutdown` stops the daemon while leaving fixture paths available until harness drop; final drop cleans remaining children. Every later E2E uses this subprocess boundary.
## Task 04 — Google OAuth and account lifecycle
**Depends on:** 03. **Requirements:** R02, R15.
**Files:** Engine `accounts.rs`, Google `{http,oauth}.rs`, Accounts proto/handler/CLI, `tests/{google_system,google_e2e}.rs`.
**Interfaces:** `Engine::begin_google_auth(GoogleAuthRequest) -> AuthSession`; `auth_status(AuthSessionId)`; `list_accounts()`; `disconnect_account(AccountId)`. Session carries a browser URL and expiry, not verifier/token. `GoogleHttp` owns token refresh coordination, deadlines, redacted errors, and validated production endpoints.
- [x] Test full mock authorization redirect/code exchange/refresh with two accounts, denial, state mismatch, wrong PKCE, expired/reused code, and revoked refresh token.
- [x] Implement installed-app browser OAuth with loopback callback, random state and PKCE. CLI passes client registration configuration securely; scope requests cover the shipped Gmail/Calendar capabilities. Explain Google consent/test-user setup and any testing-mode token lifetime in `RUNNING.md`, verified against current official docs.
- [x] Persist account plus credential references with rollback/compensation on partial failure. Serialize refresh per account and preserve rotated refresh tokens. Never put token values into process arguments, logs, JSON results, database fields, or error URLs.
- [x] Implement disconnect as a durable pause plus credential removal, retaining cached data, drafts, and operation records. It neither purges data nor revokes the user's wider Google application grant silently. Reconnect must verify the provider account matches before resuming.
- [x] Add CLI `connect-google`, account list/disconnect, and inspectable authorization status. Harness follows mock browser redirects; production opens the actual consent page. Test one account requiring reauthorization while another continues.
- [x] Run `--suite google_system` and `--suite google_e2e`; prove offline test execution without a production Google client ID.
## Task 05 — Gmail initial sync, MIME, attachments, and query vertical
**Depends on:** 04. **Requirements:** R03, R12, R15.
**Files:** Engine Google `gmail.rs`, `domain/mail.rs`, `store/mail.rs`, `mail.rs`; Mail proto/handler/CLI; fixture MIME; `tests/{google_system,google_e2e}.rs`.
**Interfaces:** `Engine::sync_account(SyncRequest) -> SyncRunId`; mail list/get/search/collection queries return revision and coverage; `fetch_message(MessageRef)` explicitly downloads missing payload; raw/attachment downloads expose bounded streams. `MessageRef` includes account ID and local message ID. Initial sync stages provider IDs and pages under a sync-run generation.
- [x] Seed multiple pages, a message in several labels, shared thread membership, missing optional fields, HTML-only content, an exact multipart attachment, and duplicate provider IDs across accounts. Test complete ingestion through HTTP, then disable the provider and read/search/download locally.
- [x] Implement label/profile/message enumeration. Capture a start history cursor before initial listing; walk every page, ingest raw MIME/metadata, then catch up history from that start before marking the generation current. If the cursor expires, discard/restart that incomplete generation safely.
- [x] Preserve original MIME bytes, headers, attachment bytes, and raw provider IDs. Decode base64url and millisecond timestamp strings without floating-point conversion. Retain explicit body availability and enforce payload limits during streaming.
- [x] Create encrypted FTS indexes from normalized content, with account-scoped SQL parameters and bounded search expressions. Sanitize terminal control sequences for displayed text; raw downloads preserve original bytes.
- [x] Implement collection/message listing, read/search, explicit fetch, and streaming downloads through API and CLI. Use revision-bound keyset pagination. Return missing/too-large content honestly.
- [x] Run both Google suites. Assert SHA-256 of downloaded MIME/PDF equals hand-authored fixture hashes, and that label membership does not duplicate the message entity.
Test fragment, with fixture IDs defined by `Seed::TwoAccounts` and `connect_and_sync_google` performing OAuth and CLI sync:
```rust
let h = E2eHarness::start(Seed::TwoAccounts).await?;
let account = h.connect_and_sync_google("account-a").await?;
let output = h.cli(&["--json", "mail", "search", "--account", &account,
"--query", "synthetic-needle"]).await?;
assert_eq!(output.status, 0);
assert_eq!(output.json()?["result"]["items"].as_array().unwrap().len(), 1);
```
The harness helper returns the actual local account ID obtained from API output; it cannot preseed storage or fake sync results. Tests may use `unwrap` for assertions under test-only lint allowances.
## Task 06 — Gmail incremental sync and crash-safe reconciliation
**Depends on:** 05. **Requirements:** R03, R08, R14.
**Files:** Google `gmail.rs`, `store/mail.rs`, sync tables/migrations, `tests/google_system.rs`, `tests/google_e2e.rs`.
**Interfaces:** Sync scopes store typed cursor, query fingerprint, run status, and projection generation. Applying a page is idempotent; promoting a complete generation and final cursor is transactional. Engine startup resumes or safely restarts incomplete sync runs.
- [x] Test page overlap, deletion between list/get, last-page failure, label-only history, history expiry/404, and out-of-order IDs. Assert that a partial listing never deletes unseen local messages.
- [x] Implement history handling for additions, deletions, and label changes. Commit the final response's `historyId` only after every page is durably applied; do not use `nextPageToken` as a checkpoint.
- [x] Implement expired-history full reconciliation using staging. Delete only absent provider projection records after a completed authoritative run. Preserve drafts and operations even when their source message disappears.
- [x] Add test barriers before page commit, before generation promotion, and after promotion. Force-kill the real daemon at each barrier; restart and assert stable identities, complete membership, and correct final cursor behavior through mock request observations.
- [x] Run both Google suites with server page cap 1 and then larger caps. Assert the same final data and no lost changes after sync retries.
The transaction contract is:
```sql
BEGIN IMMEDIATE;
-- Apply the verified complete generation and its deletions here.
UPDATE sync_scopes SET cursor = ?1, active_generation = ?2
WHERE account_id = ?3 AND scope_id = ?4;
-- Insert a change-log revision for the same committed projection here.
COMMIT;
```
The comments mark transaction ordering, not permission to omit projection/change-log writes. Task 06 implements those writes against the schema introduced in Tasks 01/05.
## Task 07 — Google Calendar canonical sync and agenda
**Depends on:** 06. **Requirements:** R04, R12, R14.
**Files:** Google `calendar.rs`, `domain/calendar.rs`, `store/calendar.rs`, `calendar.rs`; Calendar proto/handler/CLI; fixed occurrence fixtures; both Google suites.
**Interfaces:** Calendar references carry account/calendar/event IDs. `EventTime` is `Date { date }` or `DateTime { rfc3339, time_zone }`. Engine calendar list/get/agenda operations return canonical version and occurrence coverage. `refresh_agenda(AgendaWindow)` is explicit and inspectable.
- [x] Test two accounts with the same event ID; all-day exclusive ends; DST; recurring masters; moved/cancelled exceptions; multiple calendars; deleted calendars; and token expiry/410. Expected occurrences are hand-authored, not generated by the production parser.
- [x] Implement paginated calendar discovery and canonical event sync with `singleEvents=false`, `showDeleted=true`, and a stable query fingerprint. Persist only the final `nextSyncToken`; never attach time bounds to incremental token requests.
- [x] Store raw event fields, ETag, recurrence identity, original start, attendee data, and tombstones. Stage full resets so a failed page cannot delete the old complete projection. Reconcile removed calendars only after complete discovery.
- [x] Implement a separate provider-expanded agenda cache for the spec's rolling window. Invalidate affected coverage after canonical changes; refresh expands recurring instances using Google, preserving cancellations and exceptions. No second local recurrence engine in this release.
- [x] Implement calendar list/get/agenda and explicit refresh in RPC/CLI. Validate date ranges and expose unavailable/stale/out-of-window coverage.
- [x] Run both Google suites. Verify fixture `all-day-2` has start `2026-10-31`, exclusive end `2026-11-02`, and occupies exactly October 31 and November 1 in the agenda.
Required public time representation:
```proto
message EventTime {
oneof value {
string date = 1;
ZonedDateTime date_time = 2;
}
}
message ZonedDateTime {
string rfc3339 = 1;
string time_zone = 2;
}
```
## Task 08 — Background operation and recoverable change stream
**Depends on:** 07. **Requirements:** R10, R12.
**Files:** Engine `scheduler.rs`, `changes.rs`, store change log; status/watch proto and CLI; Google system/E2E suites.
**Interfaces:** `Engine::watch_changes(after_revision)` returns committed revision events or a resnapshot-required error. Scheduler owns one account conflict lane, separate mail/calendar schedules, bounded global concurrency, cancellation tokens, and per-provider retry deadlines. Status identifies phase, last success, age, next attempt, error class, and incomplete coverage.
- [x] Test polling actually changes the local mailbox and calendar after a mock edit without a manual sync command. Test canceled work, one broken account, rate limit guidance, disconnect/reconnect, and simulated wake/catch-up.
- [x] Implement the concrete defaults from the spec and monotonic deadline scheduling. Coalesce repeated requests for the same scope; permit other accounts while one backs off. Credential revocation pauses that account instead of retrying forever.
- [x] Persist local changes and operation transitions with revisions in the same transaction. Stream from the bounded change log and signal overflow/expired revision explicitly. A slow consumer never silently misses updates.
- [x] Implement watch, status, sync wait, and consistent CLI exit mapping. A queue receipt is distinguishable from successful completion. Keep engine errors structured and safe for CLI diagnostics.
- [x] Run both Google suites. Pass milestone B only after reboot/offline read and background mail/calendar refresh work from actual CLI processes.
Required JSON failure shape, with no successful process exit:
```json
{"schema_version":1,"error":{"code":"unavailable","message":"Daemon is unavailable","retryable":true}}
```
Reader milestone B verified offline in `rebuild/test-results/task08`; operation enqueue/start/finish and audited resolution/resend revisions are additionally verified by operation_system and google_e2e (17 tests at the send crash-boundary checkpoint).
## Task 09 — Durable drafts and operation journal
**Depends on:** 08. **Requirements:** R05, R08, R12.
**Files:** Domain/store/application operations modules; draft methods in mail modules; Operations proto/handler/CLI; `tests/{operation_system,google_e2e}.rs`.
**Interfaces:** `Engine::save_draft(DraftInput) -> Draft`; draft list/get/delete; reply/forward preparation; `enqueue(RequestIdentity, OperationPayload) -> Operation`; get/list/cancel/resolve. `Operation` contains ID, account, request ID, state, timestamps, attempt receipts, preconditions, and redacted error. Typed payloads cover send, mail change, and calendar change.
- [x] Test draft persistence across forced termination and provider resync, multi-recipient reply-all with self-address exclusion, forwarding original attachments, and invalid recipient/header injection rejection. Upload attachments through chunked RPC, not daemon file paths.
- [x] Implement versioned local drafts with optimistic version checks. Compose replies with correct headers/thread context; preserve original attachments and MIME semantics. Draft editing cannot mutate an already enqueued send payload.
- [ ] Implement atomic desired-local-state plus operation intent, unique `(account_id, request_id)`, and canonical payload fingerprint. Same request returns the original operation; a changed payload under the same request ID fails.
- [ ] Implement the full state machine and transition constraints. Only queued/not-dispatched work is safely cancellable. Running work after restart enters reconciliation; classification never blindly resends it. Expose conflict/uncertain resolution with an explicit decision and audit trail.
- [ ] Implement draft and operation RPC/CLI methods. Resolution can acknowledge positively established success, abandon unresolved work, or explicitly create a new-send request with duplication risk shown; it cannot relabel unknown delivery as safely failed.
- [ ] Run `--suite operation_system` and `--suite google_e2e`. Assert queue and draft content survive backup-independent restart and full projection reset.
Required atomic uniqueness constraint:
```sql
CREATE UNIQUE INDEX operations_request_identity
ON operations(account_id, request_id);
```
On conflict, load and compare the canonical payload fingerprint inside the transaction. Never implement deduplication as a non-atomic “lookup, then insert.”
## Task 10 — Gmail send and mail mutations
**Depends on:** 09. **Requirements:** R05–R06, R08, R14.
**Files:** Google `gmail.rs`, engine `mail.rs`/`operations.rs`, Mail proto/CLI, both Google suites, operation suite.
**Interfaces:** `Engine::send_draft(SendDraftRequest) -> Operation`; `change_message(ChangeMessageRequest) -> Operation`. Provider executor accepts frozen MIME/desired labels and returns a receipt, definitive failure, conflict, or uncertainty. Mail action capability metadata distinguishes labels from folder placements.
- [x] Test compose/reply/reply-all/forward with attachments through the actual CLI and inspect MIME received by mock `messages.send`. Verify explicit From account, envelope recipients, thread ID and reply headers, and one accepted send per successful operation.
- [x] Implement send from frozen original bytes with stable Message-ID and request identity. Reconcile positively using provider receipts/search evidence when available; do not infer non-delivery from absence or claim a custom header creates Gmail idempotency.
- [x] Implement read/unread, star/unstar, archive, trash/untrash, and add/remove existing labels as typed actions. Preserve unrelated labels and classify repeatable desired-state actions separately from send.
- [x] Inject response loss after acceptance, kill the daemon, restart, and issue the same request ID again. Assert accepted-send count stays 1, while state is applied only with positive evidence or otherwise uncertain. A new explicit resend request is a separate operation.
- [x] Exercise definitive pre-acceptance failure, 429/5xx retry rules, draft edits after enqueue, cancel races, and concurrent duplicate requests. Run all three affected suites.
Required independent side-effect assertion:
```rust
assert_eq!(h.google.control().accepted_sends("account-a").await?, 1);
```
This must accompany CLI/journal assertions after every send crash/retry scenario; checking only that the local operation says “sent” is insufficient.
## Task 11 — Calendar writes, conflicts, responses, and free/busy
**Depends on:** 10. **Requirements:** R07–R08, R14.
**Files:** Google/calendar application/operation modules, Calendar proto/CLI, both Google suites.
**Interfaces:** `Engine::change_event(CalendarChangeRequest) -> Operation`; `query_free_busy(FreeBusyRequest) -> FreeBusyResult`. Requests specify action, identity, expected ETag where applicable, occurrence/series scope, field patch, and notification policy.
- [x] Test create/update/delete, attendee response, all-day edit, one moved occurrence, whole-series edit, and free/busy. Assert notification counts independently of final event objects.
- [x] Implement provider-compatible stable event IDs for creates and reconciliation. Updates/deletes use `If-Match`; 412 records a conflict and preserves local intent. Patch named fields so unknown/provider-managed data is not silently discarded.
- [x] Require explicit occurrence versus whole-series scope and notification policy. Reject unsupported “this and following.” Validate organizer/attendee permissions; RSVP changes only the selected user's supported response fields.
- [x] Handle apply-then-drop for create/update/delete without duplicate events or repeated notifications. Positively reconcile versions/content before retrying; unresolved notification outcome remains visible.
- [x] Implement JSON action-file validation, free/busy result coverage/errors, and CLI examples. Run both Google suites and finish milestone C only with all writes reachable through the CLI.
Concrete conflict acceptance: mock event ETag changes from `v1` to `v2` between read and edit; a request with `If-Match: v1` receives 412, leaves remote `v2` unchanged, and produces CLI exit 5 with an inspectable conflict operation.
Offline verification: `rebuild/test-results/task11/results.json`; all12commands exit0, normal/feature142 each (overlapping), separate mock25/system21/E2E26/operation6/release1. Live compatibility remains deferred and unverified.
## Task 12 — Synology MailPlus through IMAP/SMTP
**Depends on:** 11. **Requirements:** R02, R05–R06, R08–R09, R14–R15.
**Files:** Engine IMAP/SMTP adapters and account configuration, placement storage, Accounts/Mail proto/CLI capability handling, test-support `imap.rs`, `tests/{imap_contract,imap_system,imap_e2e}.rs`, `tests/imap/compose.yaml`.
**Interfaces:** `Engine::add_imap_account(ImapAccountConfig)` accepts host/port/TLS/account/Sent-folder configuration with credential entry separate from config. A placement reference is `(account, mailbox identity, UIDVALIDITY, UID)`. SMTP result and Sent-copy status are separate operation substeps.
- [x] Add a pinned local Dovecot instance and SMTP capture server, such as Mailpit, in the test composition. Bind mapped ports to loopback; use synthetic users and a local test CA. Use official maintained images and pin tested versions/digests at implementation time. Downloads are setup; tests contact local services only. References: [Dovecot Docker setup](https://doc.dovecot.org/main/installation/docker.html), [Mailpit Docker setup](https://mailpit.axllent.org/docs/install/docker/).
- [x] Write strict command/response tests for mailbox quoting/encoding, quiet delta, UIDVALIDITY reset, flags, deletion, truncated FETCH, and capability negotiation. Independently exercise the adapter against Dovecot, including servers without optional extensions.
- [x] Implement full/delta mail ingestion through the same storage/query contract. Discover folders and special-use roles. Do not assume message IDs or UIDs survive a UIDVALIDITY change; stage resync without deleting durable local state.
- [x] Implement read/star/archive/trash/restore and folder move/copy using explicit placement. Use safe UID-scoped expunge where available; if fallback cannot avoid unrelated deletion, return unsupported. Keep prior placement metadata for restore and report missing destination conflicts.
- [x] Implement SMTP sending and configurable server-auto-Sent versus client-append behavior. Accepted SMTP delivery is never retried because APPEND failed. Unknown send/COPY/APPEND outcomes enter reconciliation/uncertainty, with remote counters proving no blind repeat.
- [x] Implement validated production TLS, hostname checks, secure credential entry, and account capabilities in API/CLI. Never allow a remote plaintext-auth fallback. No SMTP/IMAP password in a config file or argv.
- [x] Run `--suite imap_contract`, `--suite imap_system`, and `--suite imap_e2e`, including actual CLI sends and downloaded-byte equality. Keep Google suites green. Record exact supported extensions; Dovecot passing does not claim live MailPlus compatibility.
Offline SMTP/Sent checkpoint: `rebuild/test-results/task12-smtp-server/results.json` all12commands0; independentIMAPcontract2(wrapping18Python),engine78,protoCLI4,IMAPsystem17,actualIMAPE2E6,Google21/26, nofailed/ignored. BothSentpolicies andTLS modes, negativeevidence/crash cases verified. Explicitresend follow-up status is in rebuild/docs/SESSION-STATE.md. ManualSMTPconfirmation and remaining profile/legacy/security work are still tracked; no live compatibility claim.
The empty-delta regression assertion is a protocol observation: when no new UID exists, the adapter issues no empty FETCH command, leaves the current checkpoint unchanged, and reports a successful quiet sync.
Latest offline Task12 evidence: rebuild/test-results/task12-smtp-confirmation (engine80/protoCLI8/IMAP19/Google6+21+26, all9commands0) and task12-smtp-profiles (IMAPsystem21/actualCLI9, fmt/bothClippy, all5commands0). SMTP manual confirmation/abandonment and missing-UIDPLUS/queued-endpoint profiles now verified. Full goal still requires Task13 recovery/migration fixtures, Task14 cross-cutting audit, Task15 artifacts/CI and Task16 acceptance; no live MailPlus compatibility claim.
## Task 13 — Export, backup, restore, migrations, and repair
**Depends on:** 12. **Requirements:** R11, R15.
**Files:** Store `backup.rs`/migration runner, Maintenance proto/handler/CLI, `tests/recovery_e2e.rs`, `docs/RECOVERY.md`.
**Interfaces:** Backup produces a consistent encrypted snapshot including drafts, operations/attempts, configuration, and projection state. Inspect validates format/version/integrity. Restore accepts a streamed backup into a new profile only. `repair_projection(account, scope)` stages a fresh provider projection, never discards durable state.
- [ ] Test raw EML and attachments byte-for-byte; reject output paths aliasing live database/WAL/config by canonical path and existing-file identity. Use atomic temporary-file completion, no success on partial writes, and no path traversal through supplied filenames.
- [ ] Implement SQLCipher-consistent backup from a defined database snapshot, including WAL-visible committed state. Encrypt the backup under a separate recovery passphrase using SQLCipher's supported key derivation/export facilities; read passphrase through hidden terminal input or protected input channel, never argv/config/logs. Preserve schema-version metadata explicitly across export and verify a real read after keying, since setting a key alone does not validate it. Do not build custom cryptography. Reference: [SQLCipher keying and export API](https://www.zetetic.net/sqlcipher/sqlcipher-api/).
- [ ] Exclude provider tokens and the daemon bearer token from backups. Document that the recovery passphrase is required and has no reset path. On restore, generate fresh profile/API keys and require provider reconnection; pending outbound work remains held for reconciliation, never auto-replayed.
- [ ] Implement inspection and restore into a new directory/profile with checksums, SQLCipher integrity verification, disk-space/error handling, atomic activation, and rollback. Wrong key, truncated backup, newer schema, or failed migration leaves source and target originals intact.
- [ ] Add migration fixtures from every supported rebuild schema. Test interrupted migrations, corruption of a copied database, failed WAL/backup operations, and repair while drafts/queued operations exist. Repair refuses to wipe an unreadable store; retain originals and provide a recovery error.
- [ ] Implement `backup create/inspect/restore` and `repair` CLI/RPC, plus dry-run reporting of affected projections. Run `--suite recovery_e2e` and retain restore comparison evidence for all durable tables and payload hashes.
Required recovery case: queue a send without dispatch, save a draft with a PDF, create backup, restore to a new profile, reconnect against the mock, and verify the draft/PDF/request identity survive while accepted-send count remains zero until explicit resume/reconciliation permits dispatch.
## Task 14 — Adversarial acceptance and resource bounds
**Depends on:** 13. **Requirements:** R01–R15.
**Files:** `tests/{multi_engine_system,security_system,resource_system}.rs`; relevant implementation fixes; `docs/VERIFICATION.md`.
**Interfaces:** Reuse real system/process harnesses and independent provider observations. Resource instrumentation reports queue depths, active requests, fetched bytes, storage page batches, and child RSS without content/secret leakage.
- [ ] Run three independent engines/profiles against one mock account. Apply external message/calendar changes while engines restart at different points. Require all provider projections to converge and assert separate send/copy/notification counters. Local drafts intentionally remain profile-local.
- [ ] Test all services without auth and with wrong/expired tokens, account crossover, revoked credentials, malicious MIME/HTML/terminal escapes, filename traversal, header injection, malformed provider IDs, and oversized streams.
- [ ] Search database, WAL, backup, temporary files, stdout/stderr, and retained logs for synthetic content/secret canaries. Also prove wrong-key/ordinary-SQLite rejection; byte scanning alone is not an encryption proof.
- [ ] Exercise 10,000 metadata messages across multiple pages and 16 MiB attachments; test rejection above the configured 64 MiB limit. Enforce request concurrency 2 and bounded queues. Report latency/RSS on the actual machine; fail unbounded growth, not an invented universal speed target.
- [ ] Run deterministic disconnect/retry/cancel/kill scenarios at every persisted-operation boundary. Classify findings by risk; fix all defects that violate an acceptance requirement, then rerun the affected suites. Do not weaken mocks or discard assertions to pass.
- [ ] Run all three new suites and `python3 rebuild/scripts/verify.py --all`. Record newly observed counts and zero ignored tests.
## Task 15 — Local release candidate, contract, and CI
**Depends on:** 14. **Requirements:** R12, R15–R16.
**Files:** `scripts/package.py`, contract descriptor/golden and tests, `docs/{API,RUNNING,TESTING}.md`, `README.md` within rebuild; root `.github/workflows/rebuild-ci.yml` as the only planned root build-integration addition.
**Interfaces:** `python3 rebuild/scripts/package.py --output rebuild/dist` builds production-only binaries into a clean target directory, verifies them, and emits versioned local archives with SHA-256, contract descriptor, license notices, and build metadata. No publishing or automatic installation.
- [ ] Pin the dependency lockfile and required build tools; document platform-specific SQLCipher prerequisites. Review dependency advisories/licenses and secret scan results. Address findings or explicitly record an accepted limitation; no fabricated clean security scan when a tool is unavailable.
- [ ] Freeze `nuncio.v2` field numbers for this release and add descriptor compatibility checks. Compile a tiny external generated-client status/watch smoke test with no engine dependency. This proves the native-client boundary without starting a native app project.
- [ ] Add CI jobs named `rebuild-lint`, `rebuild-mock-contract`, `rebuild-system`, `rebuild-e2e`, `rebuild-imap`, and `rebuild-release-check`, using the same runner on Linux and the platform-relevant macOS checks. Pre-pull local server images, then prevent external provider access. Write workflow files; do not change remote branch protection without authorization.
- [ ] Ensure CI and package checks reject test flags and exclude mock binary/test dependencies from production artifacts. Validate every required job fails when its underlying suite fails. Versioned artifacts are local release candidates, not a public functional-release claim.
- [ ] Write first-run/OAuth/credential setup, offline usage, action-file examples, JSON/exit semantics, backup recovery, install/uninstall instructions, and compatibility limits. Distinguish normal setup from manual live acceptance.
- [ ] Run `python3 rebuild/scripts/verify.py --all`, then the package command. Exercise extracted binaries in a temporary location. Run root fmt/Clippy/tests if root build behavior changed. Record artifact paths/hashes and local-equivalent CI results; remote CI remains unobserved until authorized push.
## Task 16 — Provider acceptance and final completion report
**Depends on:** 15; named disposable accounts and action authorization for live acceptance. **Requirements:** R01–R16.
**Files:** `docs/VERIFICATION.md`, `docs/SESSION-STATE.md`, `docs/TODO.md`, `docs/RUNNING.md` within rebuild.
- [ ] Ask early in execution for the prerequisites that only James can provide: Google installed-app registration and named disposable test identity, Synology hostname/TLS/account plus MailPlus version, and permitted test recipients/calendars/actions. Do not ask for plaintext credentials in chat. Continue independent offline implementation while access is pending.
- [ ] Prepare a concrete manual acceptance worksheet with exact commands, named resources, expected changes, cleanup steps, and evidence fields. Obtain explicit authorization before any live read/write/send/invite. These manual checks are separate from automated tests.
- [ ] On Google, verify actual OAuth/refresh, multi-page sync, external change reconciliation, raw/attachment fidelity, one authorized send/reply, label changes, and isolated calendar create/edit/respond/delete with named notification policy.
- [ ] On Synology, verify certificate/hostname, negotiated IMAP capabilities, folder encoding, UID/flag/deletion sync, one authorized SMTP submission, and actual Sent-copy behavior. Record server/package versions and any compatibility restrictions. Fix discovered product defects and rerun offline regressions before repeating the relevant manual check.
- [ ] Complete the requirement matrix below with source locations, actual test names and run artifacts, manual evidence, limitations, and package hashes. Mark each requirement passed, failed, or externally pending; do not convert missing evidence to passed.
- [ ] Produce final implementation status. The entire goal is complete only when required software, offline gates, local artifacts, and authorized provider acceptance pass. If live access remains pending, report “implemented and offline-verified; live acceptance pending” and preserve that outstanding goal condition. Follow the goal tool's actual blocked-state policy rather than marking success to end a turn.
## Requirement-to-task traceability
| Requirement | Implemented in tasks | Required evidence |
|---|---|---|
| R01 | 01, 03, 14 | lifecycle, actual daemon/CLI, lock/restart/auth tests |
| R02 | 04, 12, 16 | OAuth and account isolation, keyring/TLS, manual providers |
| R03 | 05–06, 16 | Gmail wire/system/E2E, MIME hashes, cursor/crash tests |
| R04 | 07, 16 | canonical/occurrence fixtures, cross-account/deletion/token tests |
| R05 | 09–10, 12 | draft durability, received MIME, attachment and send counters |
| R06 | 10, 12 | Gmail label and IMAP placement/move/copy observations |
| R07 | 11, 16 | event/RSVP/free-busy tests, versions and notification counts |
| R08 | 06, 09–12, 14 | journal state machine, repeated IDs, crash/uncertainty tests |
| R09 | 12, 16 | strict IMAP/SMTP plus independent server and live MailPlus |
| R10 | 08, 14 | actual automatic refresh, cancellation, backoff/isolation |
| R11 | 13 | byte fidelity, restored durable state, failed-recovery originals |
| R12 | 01, 03, 05, 07–13, 15 | CLI/RPC matrix, exits, streams, descriptor/client smoke |
| R13 | 02–03 | independently sourced mock conformance and fault-effect tests |
| R14 | 03–14 | required system and subprocess suites with no live network |
| R15 | 01, 03–05, 12–15 | encryption/secret/auth/TLS/limits/release-isolation evidence |
| R16 | 15–16 | reproducible local artifacts, CI definitions, manuals, report |
## Scope changes and completion discipline
The executor may choose private helper names and compatible library versions while preserving the defined interfaces and behavior. Record justified implementation refinements in session state. Changes to providers, product scope, encryption guarantees, send semantics, native-app scope, or completion requirements need James's decision. Missing credentials, a difficult test, or time spent is not permission to remove a requirement.
This plan deliberately has no fixed delivery-date promise. The highest-uncertainty work is synchronization under change, uncertain remote writes, calendar exceptions, and MailPlus compatibility. Resolve those risks with the specified tests and manual evidence. Do not claim success by completing only the scaffold, mock, read-only milestone, or an impressive test count.
@@ -0,0 +1,222 @@
# Nuncio Google-First Rebuild Specification
**Status:** Accepted September 10, 2026; implementation is active in the isolated feature worktree.\
**Date:** September 10, 2026\
**Baseline:** `dev` at `a268fd5a004977b65fd0347ceeb1c69755529aa3`\
**Evidence:** [Engineering audit](../../reviews/2026-09-10-nuncio-audit-and-rebuild.md)\
**Execution:** [Implementation plan](../plans/2026-09-10-google-first-rebuild.md) and [copyable goal](../../GOAL-nuncio-rebuild.md).
## 1. Outcome and scope
Deliver a local-first, single-user email and calendar engine, operated entirely through a reference CLI and a versioned authenticated gRPC API. Google Gmail and Google Calendar are the first providers; Synology MailPlus follows through IMAP/SMTP. Native applications are a subsequent project. The read-only Google milestone is an intermediate checkpoint, not completion of this goal.
User-confirmed requirements are engine/CLI first, Google before IMAP/Synology MailPlus, and a mock Gmail provider for system and end-to-end tests. This specification extends the mock to the Google Calendar and OAuth surfaces the engine also needs. The full write-capable completion boundary and defaults below are proposed for adoption with the execution goal.
### Required capabilities
| ID | Requirement |
|---|---|
| R01 | Foreground daemon, explicit profile/data directory, authenticated loopback API, health/status, graceful shutdown, exclusive store ownership, and restart. |
| R02 | Google browser OAuth, refresh/reauthorization, account list/disconnect, and Synology credentials stored through the OS keystore. At least two accounts work without identity or credential crossover. |
| R03 | Gmail initial and incremental sync, complete paging, label/thread identity, deletions, expired-history recovery, original MIME, attachments, offline reads, and local search. |
| R04 | Google Calendar discovery, canonical event sync, agenda occurrences, correct dates/timezones/recurrence exceptions, deletions, and expired-token recovery. |
| R05 | Durable local drafts, compose/reply/reply-all/forward, attachments, recipient arrays, explicit sending account, sending through Gmail or SMTP, and inspectable submission history. |
| R06 | Read/unread, star/unstar, archive/trash/restore, Gmail existing-label add/remove, and IMAP folder move/copy, with provider capabilities exposed. Permanent purge is excluded. |
| R07 | Google Calendar create/update/delete, single-instance and whole-series edit scope, attendee responses, and explicit attendee-notification policy. Free/busy query is included; reminder delivery is excluded. |
| R08 | Persisted operations, local idempotency, safe retry classification, cancellation before dispatch, conflicts, outcome uncertainty, and recovery after process death. |
| R09 | Synology IMAP/SMTP read/write parity for the mail operations above, including folder encoding, UIDVALIDITY, incremental updates, remote deletion, and Sent-copy behavior. |
| R10 | Scheduled synchronization, cancellation, bounded concurrency, provider-aware backoff, catch-up after sleep/network recovery, account isolation, and observable progress. |
| R11 | Faithful raw EML export, attachment download, encrypted consistent backup and restore, schema migration, account removal semantics, and repair that preserves originals. |
| R12 | CLI/API parity, deterministic pagination, JSON results, correct exit statuses, byte streaming, recoverable change notifications, and explicit cache coverage. |
| R13 | Stateful mock Google provider with independently grounded HTTP/OAuth schemas, fault injection, and independently inspectable remote state and side-effect counts. |
| R14 | Offline system tests through the real Google HTTP adapter and real encrypted store; subprocess E2E through the actual daemon and CLI. Strict IMAP/SMTP verification also required. |
| R15 | Confidential database/FTS/payload/journal storage, key separation, no plaintext secret logging, TLS validation, inert mail display, bounded inputs, and production exclusion of test hooks. |
| R16 | Reproducible build, required CI gates, local installation/uninstallation instructions, contract artifact, and evidence-backed acceptance report. |
### Explicit boundaries
- Primary personal runtime: macOS. Linux runs the offline CI suites. Windows/native-client installation is outside this goal; avoid unnecessary platform assumptions in the domain/API.
- Synology scope is MailPlus email. Synology Calendar/CalDAV is excluded until requested. Google Calendar supplies the calendar capability.
- No JMAP, CardDAV/contacts synchronization, NSQL/filter automation, MCP, AI, plugins, push-webhook infrastructure, custom updater, or peer-to-peer sync.
- Drafts are durable locally. Cross-device draft synchronization and scheduled send are excluded. Multiple ordinary clients may use the same upstream account; local-only state is not replicated between engines.
- Calendar supports editing one occurrence or the entire series. “This and following,” organizer transfer, calendar sharing administration, and local reminder delivery are explicitly unsupported.
- Existing custom Gmail labels and existing IMAP folders can be used. Creating/deleting label or folder hierarchies is excluded.
- This goal does not migrate or replace an existing Nuncio database, install a background login agent, publish a release, or merge the rebuild into the root workspace.
## 2. Implementation location and boundaries
Implement a separate Cargo workspace under `rebuild/` on a feature branch. This keeps the existing implementation available as reference and makes adoption reversible. The nested workspace has its own manifest, lockfile, tests, data namespace, and four production crates:
| Path | Responsibility |
|---|---|
| `rebuild/crates/nuncio-engine` | Domain/application operations, encrypted storage worker, providers, synchronization, operation execution, secrets. |
| `rebuild/crates/nuncio-proto` | `nuncio.v2` protobuf definitions, generated Rust bindings, thin authenticated client helper. |
| `rebuild/crates/nunciod` | Process lifecycle, dependency composition, gRPC handlers, production policy enforcement. |
| `rebuild/crates/nuncio-cli` | Parsing, formatting/JSON, stdin/stdout, exit codes, client-side files. No engine/database dependency in normal dependencies. |
| `rebuild/crates/nuncio-test-support` | Test-only mock Google server, synthetic fixtures, mock keystore, protocol harnesses, subprocess helpers. Never shipped. |
The test-support crate is an unshipped workspace member. Its mock implementation and wire fixtures must not depend on the engine's provider serialization/domain models. System-test composition may use the engine's public entry point through dev dependencies; keep that composition in test modules, separate from the standalone mock's dependency graph. A `mock-google` binary permits manual offline exercises. Injectable credentials/Google endpoints are compiled into test-harness daemon builds only; mock controls remain exclusively in the harness/mock process.
Use Rust 2021, the repository's pinned Rust 1.97.1, Tokio, tonic/prost, Clap, serde, reqwest/rustls, established MIME libraries, and SQLite/FTS5. For this execution contract, choose **SQLCipher-backed SQLite through rusqlite**, isolated behind a dedicated storage thread and typed request channel. This supplies whole-database encryption without a custom column cipher. Pin compatible dependencies in the new lockfile; do not weaken security to work around packaging issues. The original SQLx store is reference material, not a mandatory dependency.
SQLCipher protects database and WAL page content; configure temporary storage in memory and verify the behavior rather than assuming every transient file is encrypted. The driver has documented bundled SQLCipher build features. [SQLCipher design](https://www.zetetic.net/sqlcipher/design/), [rusqlite features](https://docs.rs/crate/rusqlite/latest).
Use `127.0.0.1:9421` by default, independent of the original daemon's port. Default data root is `~/.nuncio-rebuild`; keyring service is `mx.nuncio.rebuild`, with keys scoped by profile and account. Tests always use temporary directories and ephemeral ports. Bind and acquire the store lock before accepting work. Never fall back to the original profile or keyring namespace.
```mermaid
flowchart LR
CLI[Reference CLI] --> API[Authenticated gRPC API]
Native[Future native applications] -.-> API
API --> Engine[Engine and durable operations]
Engine --> Store[SQLCipher and encrypted payloads]
Engine --> Keys[OS keyring]
Engine --> Google[Gmail and Calendar HTTP adapters]
Engine --> Mail[IMAP and SMTP adapters]
Google --> Provider[Google or independent mock Google]
Mail --> Server[MailPlus or local test servers]
```
## 3. State model and invariants
### Storage ownership
Provider objects are synchronized projections. Local drafts, operation intent, conflict resolution, configuration, and backup metadata are durable user state. Resync replaces only a named provider projection. It cannot delete local drafts or queued work. Schema upgrades are versioned and transactional; an older binary rejects a newer store. Corruption or a wrong key fails closed and leaves original files intact.
Keep original MIME and attachment bytes inside encrypted, chunked blob tables initially. Chunk size is 256 KiB; maximum accepted single payload is 64 MiB by default and configurable downward. Oversized content retains metadata and an explicit `too_large` availability state. Streaming APIs never put a complete large attachment into a single protobuf message. Export only downloaded originals, or explicitly fetch them as a separate operation; missing payloads cannot become successful empty files.
### Logical tables
All entity foreign keys carry account scope. Local IDs are opaque, stable identifiers; provider IDs remain separately stored strings.
| Tables | Identity and important fields |
|---|---|
| `accounts` | Local account ID, provider, address, credential reference, capabilities, sync configuration; no password/token. |
| `messages`, `collections`, `memberships` | Local ID and provider ID; Gmail thread/label membership; IMAP membership address includes mailbox, UIDVALIDITY, UID. No uniqueness assumption for Message-ID. |
| `message_headers`, `blob_chunks`, `message_search` | Original and parsed representations, byte hash/size, payload availability, FTS in the encrypted database. |
| `calendars`, `calendar_objects`, `calendar_occurrences` | Account/calendar/provider IDs, raw provider JSON, ETag, master/instance identity, cancellations, canonical time fields, cached occurrence window/revision. |
| `sync_scopes`, `sync_runs`, `staged_changes` | Provider cursor and query fingerprint, old/new projection generation, page continuation, completeness, coverage, observed errors. |
| `drafts`, `operations`, `operation_attempts` | Immutable request fingerprint, frozen send payload, source/destination/precondition, lifecycle state, provider receipt, redacted error. |
| `change_log`, `schema_migrations`, `backup_metadata` | Monotonic local revision, migration version, checksummed backup format. |
A transaction promotes a completed sync generation and its durable cursor together. Staging pages avoids a huge in-memory mailbox and prevents partial responses from becoming authoritative deletion snapshots. Incremental page application may be idempotently persisted before cursor advancement; the final cursor advances only when the entire cycle is durably represented. Cursors are opaque strings with provider/type tags; do not increment history IDs or interpret them as floating-point values.
### Calendar representation
Represent time as a tagged value: `Date { date }` or `DateTime { rfc3339, time_zone }`. Date-only ends remain exclusive. Retain recurring master ID, instance ID, original start, exception/cancellation status, attendees, reminders as data, and raw provider fields. Unknown fields survive read-modify-write; edits patch named fields rather than reconstructing a reduced event.
Synchronize canonical Google event resources with `singleEvents=false` and `showDeleted=true`, using a fixed query shape compatible with sync tokens. Separately fetch provider-expanded instances for a rolling agenda window: 30 days before today through 180 days after today. No window bounds are attached to a canonical incremental token request. Changing the agenda window refreshes its occurrence cache. Canonical changes invalidate affected occurrence coverage until refreshed; stale occurrences are never presented as current. Requests outside cached coverage report that fact and can request a refresh. [Google event model](https://developers.google.com/workspace/calendar/api/v3/reference/events), [event listing constraints](https://developers.google.com/workspace/calendar/api/v3/reference/events/list).
## 4. User/API contract
Every user-visible capability has one application operation, a protobuf RPC, a CLI command, and an acceptance test. HTTP provider structs never become the public API. Services are `System`, `Accounts`, `Mail`, `Calendar`, `Operations`, and `Maintenance`; each is authenticated independently. RPC names are defined in the task plan.
Commands below are the proposed grammar, not claims about existing binaries. Required account/profile arguments remove implicit sender selection.
```text
nuncio-cli system status
nuncio-cli system watch --after REVISION
nuncio-cli system shutdown
nuncio-cli account connect-google --client-config FILE
nuncio-cli account auth-status --session ID
nuncio-cli account add-imap --config FILE
nuncio-cli account list
nuncio-cli account disconnect --account ID
nuncio-cli sync --account ID --wait
nuncio-cli sync status --run ID
nuncio-cli sync cancel --run ID
nuncio-cli mail list --account ID --collection ID
nuncio-cli mail collections --account ID
nuncio-cli mail read --account ID --message ID
nuncio-cli mail fetch --account ID --message ID --wait
nuncio-cli mail search --account ID --query TEXT
nuncio-cli mail raw --account ID --message ID --output FILE
nuncio-cli mail attachment --account ID --message ID --part ID --output FILE
nuncio-cli mail draft save --account ID --file FILE
nuncio-cli mail draft list --account ID
nuncio-cli mail draft show --account ID --draft ID
nuncio-cli mail draft delete --account ID --draft ID
nuncio-cli mail draft attach --account ID --draft ID --file FILE
nuncio-cli mail reply --account ID --message ID --body-file FILE
nuncio-cli mail reply-all --account ID --message ID --body-file FILE
nuncio-cli mail forward --account ID --message ID --to ADDRESS
nuncio-cli mail send --account ID --draft ID --request-id UUID --wait
nuncio-cli mail change --account ID --message ID --action ACTION --request-id UUID
nuncio-cli calendar list --account ID
nuncio-cli calendar agenda --account ID --from DATE --to DATE
nuncio-cli calendar refresh --account ID --from DATE --to DATE --wait
nuncio-cli calendar get --account ID --calendar ID --event ID
nuncio-cli calendar change --account ID --calendar ID --file FILE --request-id UUID
nuncio-cli calendar free-busy --account ID --file FILE
nuncio-cli operation list --account ID
nuncio-cli operation show --operation ID
nuncio-cli operation cancel --operation ID
nuncio-cli operation resolve --operation ID --decision DECISION
nuncio-cli backup create --output FILE
nuncio-cli backup inspect --file FILE
nuncio-cli backup restore --file FILE --new-profile NAME
nuncio-cli repair --account ID --scope SCOPE --dry-run
```
Reply/reply-all/forward create an editable local draft; send is a separate explicit action. Change payloads are versioned JSON with named action, resource identity, and expected version where supported. The CLI documents examples and validates them before dispatch. `--json` is global: stdout contains one versioned result object, or documented JSONL for streaming; diagnostics go to stderr. Binary output and JSON are mutually exclusive. Exit statuses: 0 success/accepted enqueue, 2 invalid input, 3 authorization required, 4 unavailable/offline miss, 5 conflict/uncertain outcome, 1 other failure. `--wait` returns 0 only after the requested operation reaches its successful terminal state.
Queries read the local store and report synchronization age/coverage. Missing content requires an explicit fetch operation; it is not replaced by empty text. Listing is keyset-paginated with a revision-bound cursor; a changed snapshot gives a refresh-required error instead of skipping objects silently. Change notifications carry monotonically increasing revisions; expired revisions require a new snapshot. Retain 10,000 change records initially. Native clients can recover after disconnect without replaying a permanent domain event log.
## 5. Operation and synchronization policy
Operation states are `queued`, `running`, `applied`, `retry_wait`, `conflict`, `uncertain`, `failed`, and `cancelled`. Persist request ID and canonical payload hash under an account-scoped unique constraint. Reusing the same request ID/payload returns the original operation; reusing it for a different payload is rejected. Persist desired local state plus operation intent atomically. Freeze sending bytes and Message-ID when enqueueing; draft edits after enqueue cannot alter that submission.
Remote calls happen after commit. A crash during `running` becomes reconciliation work, not an unconditional resend. Retry idempotent desired-state mutations only where provider semantics allow it. A failed send/COPY acknowledgement is uncertain until positively reconciled; absence in an eventually consistent read is not proof of failure. Never manufacture Gmail idempotency from a custom header. For an unresolved send, expose “accepted status unknown” and require an explicit new-send decision before risking duplication.
Gmail supports label operations rather than arbitrary folder moves. Archive removes INBOX; trash/untrash use their specific endpoints. IMAP mutations address a placement explicitly. Use MOVE where available; verified fallback may use COPY plus safe UID-scoped deletion. Without a safe expunge path, return unsupported/conflict rather than expunging unrelated messages. SMTP acceptance and saving a Sent copy are separate substeps; failure to append a Sent copy retries only the append/reconciliation, never the accepted send.
Calendar updates use ETag preconditions. A 412 creates a conflict with the observed version; do not silently overwrite. New Google events use a stable client-chosen provider ID where allowed, enabling reconciliation after uncertain creation. Notification policy is explicit in the command payload. Mock tests count invitation effects independently from stored event state. [Calendar conditional changes](https://developers.google.com/workspace/calendar/api/guides/version-resources).
Schedule mail and calendar independently per account. Default poll interval: 60 seconds; global active-account limit: 2; one conflicting sync/write sequence per account; network request deadline: 30 seconds. Read retries use bounded exponential backoff with jitter, honoring longer provider retry guidance. Persistent credential failure pauses the account and requests reauthorization. Other accounts continue. Inject limits and time sources for tests; do not pause Tokio time around real SQLite/blocking I/O.
## 6. Mock Google provider contract — required
The mock is a test product with its own state, HTTP request validation, and contract suite. It must exercise the production HTTP adapter; returning domain structs from a mock backend does not satisfy R13/R14. It may implement only the endpoints used by this release, but every supported endpoint follows its documented semantics and every unsupported one fails explicitly.
### HTTP surfaces
| Surface | Required subset |
|---|---|
| OAuth | Authorization redirect with state and PKCE, one-use authorization code, token exchange, refresh, expiry, revoked refresh token, scope denial. |
| Gmail account/mail | `users.getProfile`, labels list, messages list/get (metadata/full/raw), attachments get, threads get, history list. |
| Gmail changes | Message modify, trash/untrash, message send. Local drafts use raw MIME on send; Gmail draft synchronization is not required. |
| Calendar | Calendar-list list, events list/get/instances/insert/patch/delete, freeBusy query. |
Gmail IDs/date/cursor fields must use the documented JSON types. Fixtures include noncontiguous history IDs greater than 2^53, base64url MIME, multipart messages, omitted optional fields, and Gmail-specific label/thread behavior. A pagination token is not a synchronization cursor: final Gmail history uses the returned `historyId`; final Calendar synchronization uses `nextSyncToken`. [Gmail messages](https://developers.google.com/workspace/gmail/api/reference/rest/v1/users.messages), [Gmail history](https://developers.google.com/workspace/gmail/api/reference/rest/v1/users.history/list).
### State, control, and faults
- Seed two Google accounts, two calendars per account, a message visible through several labels, a multipart PDF attachment, HTML-only mail, and fixed expected recurring instances spanning DST. No real messages or personal identifiers.
- Controls create/edit/delete remote objects, reorder results, expire history/sync tokens, revoke access, and change provider versions independently of the daemon.
- Faults include small server page caps, duplicate page overlap, an object deleted between list/get, invalid cursor, 401/403/404/410/412/429/5xx, malformed JSON, truncated bodies, latency, disconnect before acceptance, and **apply the mutation then withhold/drop its response**.
- Record request counts, accepted sends, message copies, invitation notifications, and resulting remote objects. Test assertions query these observations through harness controls, independently of CLI output.
- Hand-authored response fixtures derive from linked official references. The mock cannot import production request/response structs, generate expectations from daemon output, silently accept misspelled fields, or pretend to implement undocumented idempotency.
- The mock's own tests prove valid requests succeed, invalid requests fail, page/cursor semantics work, and every fault has its intended effect. A mock failure is not fixed by weakening request validation to match a broken client.
- Provider endpoints bind loopback only. Test controls are out-of-band and unavailable to the production adapter. The standalone mock has a deterministic reset/seed command and announces bound ports through a readiness file, not fixed sleeps.
## 7. Verification and completion
1. **Unit:** domain identity/time conversion, MIME fidelity, state transitions, crypto/key handling, provider response decoding, query boundaries.
2. **Provider conformance:** mock Google validates independent wire fixtures; IMAP uses strict syntax tests plus a local independent server implementation and SMTP sink for protocol acceptance.
3. **System:** real application, encrypted store, provider HTTP adapters, and authenticated API against stateful mock Google. Direct storage seeding cannot stand in for provider ingestion.
4. **E2E:** actual `nunciod` and `nuncio-cli` subprocesses, HTTP mock, isolated profile, mock secrets, real restarts/forced termination. Test through commands; inspect provider side effects independently. Keep system and E2E as separate required gates.
5. **Release-build checks:** production binaries reject test endpoint/credential flags; all services reject unauthenticated calls; outputs contain no synthetic secret canaries; database/WAL/backup contain no plaintext content canary and cannot be opened with an ordinary SQLite connection or wrong key.
6. **Live acceptance:** separate, explicitly authorized manual exercise on named disposable Google and Synology accounts. No normal test or CI job accesses live services. Missing live access is recorded as pending and does not prevent completing independent code/test work. A fully provider-validated claim requires this evidence; passing the mock alone cannot supply it.
Security limits are concrete: no remote plaintext authentication, no disabled certificate verification, no credentials in configuration/argv/logs, no raw server error URLs with tokens, no secret-bearing Debug output, no automatic loading of remote HTML resources, and safe filenames on attachment/export paths. Terminal output removes escape/control sequences from untrusted fields; original byte downloads remain byte-for-byte original. HTML stays inert; a future renderer must impose its own sandbox.
Acceptance must include three independent engines converging against one mock account after external changes, plus separate send/copy/notification counts. The engine is not responsible for synchronizing local drafts between these profiles. Tests use event-driven readiness and bounded waits; no discarded assertions, ignored failures, invented success counts, or fixed sleeps as a substitute for synchronization.
The final report contains a requirement-to-test matrix for R01–R16, commands/exit statuses, newly measured test counts, protocol compatibility limitations, backup/restore evidence, artifact paths, dependency/security checks, and live-acceptance status. Building a mock or passing a read-only milestone does not complete the full goal.
## 8. Recovery and provider-specific identity details
Backups use a consistent SQLCipher snapshot, separately encrypted with a recovery passphrase through supported SQLCipher key derivation/export facilities. Read that passphrase through hidden terminal input or a protected input channel; never put it in argv, configuration, or logs. The passphrase is required to restore and has no reset path. Backups include durable drafts and operation records, but exclude provider credentials and API bearer tokens. Restore only into a new profile, require provider reconnection, and hold pending outbound work for reconciliation before dispatch. Projection repair must retain originals on corruption or failure; it cannot substitute an empty database for an unreadable one.
For Gmail, a message entity is unique by account plus Gmail message ID, with many label memberships. For IMAP, identify each remote placement by account, stable local mailbox identity, UIDVALIDITY, and UID. Initially retain a separate message record per IMAP placement rather than guessing cross-folder identity from Message-ID. Shared payload storage may use verified byte hashes, but identical bytes do not prove that two remote messages are one entity. Encode compound provider keys unambiguously, never by unsafe delimiter concatenation.
Account disconnect pauses the account and removes its credential reference/secret while preserving cached data, local drafts, and operation history. Reconnection must prove the same provider account identity. Destructive local account purge is outside this goal; disconnect must not masquerade as purge or silently erase durable user state.
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env bash
# Fetch the reviewed installer; it selects and verifies the latest testing build.
nuncio_install_testing() (
set -euo pipefail
umask 077
installer_commit='0c7083bf8edd4bd0334fef3b9d28ec4a62146d31'
installer_sha256='b8856606bcdcc211e2b34c147902109895818cc4825afd8e12b082d4f6d396a0'
for prerequisite in python3 gh curl shasum; do
if ! command -v "$prerequisite" >/dev/null 2>&1; then
printf 'Nuncio installer needs %s. Install Python 3.11+ and GitHub CLI 2.100+, then run gh auth login --hostname github.com.\n' "$prerequisite" >&2
exit 1
fi
done
if ! python3 -I -c 'import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)' 2>/dev/null; then
printf 'Nuncio installer needs Python 3.11 or newer available as python3.\n' >&2
exit 1
fi
nuncio_bootstrap_tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/nuncio-bootstrap.XXXXXXXX")
trap 'rm -rf -- "$nuncio_bootstrap_tmpdir"' EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
installer_file="$nuncio_bootstrap_tmpdir/install-testing.py"
installer_url="https://raw.githubusercontent.com/KofTwentyTwo/nuncio/$installer_commit/rebuild/scripts/install-testing.py"
if ! curl --fail --silent --show-error --location \
--proto '=https' --proto-redir '=https' --tlsv1.2 \
--connect-timeout 15 --max-time 120 --retry 2 --max-filesize 1048576 \
--output "$installer_file" "$installer_url"; then
printf 'Nuncio installer download failed; nothing was installed.\n' >&2
exit 1
fi
actual_sha256=$(shasum -a 256 "$installer_file")
if [[ "${actual_sha256%% *}" != "$installer_sha256" ]]; then
printf 'Nuncio installer checksum mismatch; nothing was executed or installed.\n' >&2
exit 1
fi
python3 -I "$installer_file" "$@" </dev/null
)
nuncio_install_testing "$@"
+2
View File
@@ -0,0 +1,2 @@
*.eml -text whitespace=cr-at-eol
*.pdf binary
+4
View File
@@ -0,0 +1,4 @@
/target/
/dist/
/test-results/
__pycache__/
+5
View File
@@ -0,0 +1,5 @@
# Rebuild guidance
This subtree implements the September 10, 2026 specification and implementation plan approved by James in the execution request. They govern rebuild scope; the root historical roadmap governs the original application. Read `docs/SESSION-STATE.md`, `docs/TODO.md`, and `docs/VERIFICATION.md` here before resuming. Continue inline. James authorized an initial checkpoint commit and subsequent commits/pushes after coherent changes pass relevant checks, on `feature/nuncio-google-first-rebuild` at `github.com/KofTwentyTwo/nuncio`. Merges, releases, installation, remote settings changes and live-provider acceptance remain unapproved.
Use Rust 2021 / 1.97.1, warnings as errors, and no production unsafe/unwrap/expect/panic. Keep the CLI independent of engine/storage code. All automated provider tests are local with synthetic credentials. SQLCipher encryption, account isolation, durable operation intent, independent provider conformance, and real subprocess E2E are required. Every capability needs engine, authenticated proto, CLI, and tested evidence. Never treat a milestone as completion of the whole goal.
+3855
View File
File diff suppressed because it is too large. Load diff
+77
View File
@@ -0,0 +1,77 @@
[workspace]
resolver = "2"
members = ["crates/nuncio-engine", "crates/nuncio-proto", "crates/nunciod", "crates/nuncio-cli", "crates/nuncio-test-support"]
[workspace.package]
version = "0.1.0"
edition = "2021"
rust-version = "1.97.1"
license = "MIT OR Apache-2.0"
[workspace.lints.rust]
warnings = "deny"
unsafe_code = "forbid"
unused_must_use = "deny"
[workspace.lints.clippy]
unwrap_used = "deny"
expect_used = "deny"
panic = "deny"
todo = "deny"
unimplemented = "deny"
[workspace.dependencies]
# Protocol libraries contain wire-level trace logging; compile it out in every build.
log = { version = "0.4", features = ["max_level_info", "release_max_level_info"] }
tracing = { version = "0.1", default-features = false, features = ["std"] }
tracing-subscriber = { version = "0.3.23", default-features = false, features = ["fmt"] }
tokio = { version = "1", features = ["full"] }
tokio-stream = { version = "0.1", features = ["net"] }
async-imap = { version = "0.11.3", default-features = false, features = ["runtime-tokio"] }
lettre = { version = "0.11.23", default-features = false, features = ["smtp-transport", "tokio1-rustls-tls"] }
tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] }
webpki-roots = "1"
thiserror = "2"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
zeroize = { version = "1", features = ["derive"] }
rusqlite = { version = "0.40.2", features = ["bundled-sqlcipher-vendored-openssl"] }
keyring = { version = "4.2.0", default-features = false, features = ["v1"] }
fs4 = "0.13"
rand = "0.8"
hex = "0.4"
subtle = "2.6"
uuid = { version = "1", features = ["v4", "serde"] }
tempfile = "3"
rustix = { version = "1.1.4", features = ["fs"] }
clap = { version = "4", features = ["derive"] }
tonic = "0.14"
tonic-prost = "0.14"
prost = "0.14"
prost-types = "0.14"
tonic-prost-build = "0.14"
protoc-bin-vendored = "3.2"
http = "1"
httpdate = "1"
axum = "0.8"
hyper = { version = "1", features = ["http1", "server"] }
hyper-util = { version = "0.1", features = ["tokio"] }
base64 = "0.22"
sha2 = "0.10"
url = "2"
percent-encoding = "2"
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
futures-util = "0.3"
chrono = { version = "0.4", features = ["serde"] }
mailparse = "0.16"
mail-parser = { version = "0.11.9", features = ["serde"] }
mail-builder = { version = "0.5.0", default-features = false }
email_address = { version = "0.2.9", default-features = false }
chrono-tz = "0.10"
rrule = "0.14"
regex = "1"
[profile.release]
lto = "thin"
codegen-units = 1
strip = true
+57
View File
@@ -0,0 +1,57 @@
# Nuncio rebuild
Independent Rust workspace for the approved Google-first engine and CLI rebuild.
It includes SQLCipher storage, authenticated local gRPC, Gmail/Calendar reads and
writes, IMAP folder/flag operations, SMTP with separate delivery/Sent-copy receipts,
encrypted backup/restore, migration, repair, and resource budgets. Current schema-23 account management uses 49 authenticated RPCs.
Guided `account add` includes hidden passwords, optional advanced MailPlus
settings and bundled Google registration support. The latest verified testing
build is `bb70f95`, with readable CLI output, complete help, detailed startup/activity
logs and the Rustls patch. All ten rebuild/seven security jobs and actual public
fresh/update checks passed. Two clean local archives are identical; the full
35-command gate passed 359 tests per workspace. [The report](docs/IMPLEMENTATION-REPORT.md)
records exact source/artifact evidence and the pending live/native acceptance.
Install the latest Apple Silicon testing build without cloning or compiling:
```sh
curl -fsSL https://raw.githubusercontent.com/KofTwentyTwo/nuncio/feature/nuncio-google-first-rebuild/install-testing.sh | bash
```
Requires macOS 15+, Python 3.11+ and authenticated GitHub CLI 2.100+;
[details and custom prefix](docs/TESTING-INSTALL.md).
The permanent commands are in `~/.local/opt/nuncio-testing/bin/`. Rerun the
installer to update them; each verified update keeps this path. Start the daemon
with `~/.local/opt/nuncio-testing/bin/nunciod --profile laptop-qa`, then add an
account using `~/.local/opt/nuncio-testing/bin/nuncio-cli --profile laptop-qa account add`.
The current source reports startup stages and running activity at **info** level
(`--log-level debug` adds detail), and readable CLI output is now the default;
scripts use `--json`. Rustls 0.23.45 addresses RUSTSEC-2026-0285 across all three
lockfiles. The downloaded package also includes bounded IMAP catch-up, granular credential/recovery timings and safe CLI typo suggestions. The laptop startup pause still needs a retest.
Public fresh/update qualification passed at bb70f95. [CLI usability audit](docs/CLI-USABILITY-AUDIT.md)
and [logging options](docs/RUNNING.md#running-logs).
[Guided setup](docs/ACCOUNT-SETUP.md) uses direct prompts and hidden passwords.
Follow the [Google setup walkthrough](docs/GOOGLE-SETUP.md) to register the app
once and connect using a private local client file. Shared registration has not
yet been bundled into testing builds.
- [Run locally](docs/RUNNING.md) and [verify/use local packages](docs/PACKAGING.md)
- [Account management](docs/ACCOUNT-MANAGEMENT.md)
- [Download a testing build without compiling](docs/TESTING-INSTALL.md) — actual CI download/install verified
- [Offline testing](docs/TESTING.md)
- [Mock Google service](docs/MOCK-GOOGLE.md)
- [API contract](docs/API.md), [API publication proposal](docs/API-PUBLICATION-PLAN.md), and [task progress estimates](docs/PROGRESS.md)
- [Development security scanning](docs/SECURITY-CI.md) and [open PR dispositions](docs/OPEN-PR-REPORT.md)
- [Proposed milestones after engine/CLI completion](docs/POST-ENGINE-ROADMAP.md)
- [Implementation report](docs/IMPLEMENTATION-REPORT.md)
- [Requirement evidence matrix](docs/REQUIREMENTS.md) and [compatibility limits](docs/COMPATIBILITY.md)
- [Session state](docs/SESSION-STATE.md), [remaining work](docs/TODO.md), and [verification evidence](docs/VERIFICATION.md)
- [Approved specification](../docs/superpowers/specs/2026-09-10-google-first-rebuild.md) and [implementation plan](../docs/superpowers/plans/2026-09-10-google-first-rebuild.md)
All automated provider acceptance uses synthetic local services. Live Google,
Synology MailPlus, and native-keystore acceptance remain deferred and unverified;
native apps are excluded. Earlier failures and later corrections remain in the
verification history. Historical archives qualify only their recorded source. The testing installer and API publication proposal are repository work;
current source-specific download verification is in the report. Public API distribution and a formal
release remain separate future work.
+1
View File
@@ -0,0 +1 @@
/target/
+2441
View File
File diff suppressed because it is too large. Load diff
+36
View File
@@ -0,0 +1,36 @@
[package]
name = "nuncio-api-smoke"
version = "0.1.0"
edition = "2021"
rust-version = "1.97.1"
license = "MIT OR Apache-2.0"
publish = false
# An independent consumer, deliberately outside the engine's Cargo workspace.
[workspace]
resolver = "2"
[lints.rust]
warnings = "deny"
unsafe_code = "forbid"
[lints.clippy]
unwrap_used = "deny"
expect_used = "deny"
panic = "deny"
[dependencies]
tonic = "0.14"
tonic-prost = "0.14"
prost = "0.14"
http = "1"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-std", "io-util", "time"] }
serde_json = "1"
zeroize = "1"
[build-dependencies]
tonic-prost-build = "0.14"
protoc-bin-vendored = "3.2"
[dev-dependencies]
nuncio-test-support = { path = "../../crates/nuncio-test-support" }
+20
View File
@@ -0,0 +1,20 @@
# Independent generated-client smoke
This small consumer has its own Cargo workspace and lockfile. Its build generates
a client directly from the published `nuncio.v2` System protobuf; it imports no
engine, store, CLI, daemon, or Nuncio protocol-client implementation. It reads
authenticated status and one replayed change through the real daemon's API.
The daemon must already have a committed change. Native apps are outside scope.
Build with `cargo build --locked --manifest-path rebuild/clients/smoke/Cargo.toml
--target-dir rebuild/target/external-client`. The binary takes one numeric loopback
HTTP endpoint argument and the complete authorization header value on stdin, then
EOF. Never put a real token in an argument, shell history, source or log. The
example has a15-second deadline and reports generic failure without token details.
Its subprocess integration test uses the existing independent mock and actual
test daemon/CLI selected by `NUNCIO_E2E_DAEMON` and `NUNCIO_E2E_CLI`. Credentials
are read from that synthetic profile and piped to the child. The test verifies
status/change identity, invalid authorization rejection, no credential output and
no provider sends/notifications. The mock is a dev-dependency only. Production
package inclusion and the main verification-runner integration remain pending.
+11
View File
@@ -0,0 +1,11 @@
fn main() -> Result<(), Box<dyn std::error::Error>> {
let directory = "../../crates/nuncio-proto/proto";
let path = format!("{directory}/nuncio/v2/system.proto");
let mut config = tonic_prost_build::Config::new();
config.protoc_executable(protoc_bin_vendored::protoc_bin_path()?);
tonic_prost_build::configure()
.build_server(false)
.compile_with_config(config, &[path.as_str()], &[directory])?;
println!("cargo:rerun-if-changed={path}");
Ok(())
}
+93
View File
@@ -0,0 +1,93 @@
use std::{io, net::IpAddr, time::Duration};
use tokio::io::AsyncReadExt;
use tonic::{
metadata::{Ascii, MetadataValue},
Request,
};
use zeroize::Zeroizing;
mod v2 {
tonic::include_proto!("nuncio.v2");
}
fn authorized<T>(body: T, token: &MetadataValue<Ascii>) -> Request<T> {
let mut request = Request::new(body);
request
.metadata_mut()
.insert("authorization", token.clone());
request
}
async fn run() -> Result<(), Box<dyn std::error::Error>> {
let arguments: Vec<_> = std::env::args().skip(1).collect();
if arguments.len() != 1 {
return Err(io::Error::other("Expected one loopback endpoint argument").into());
}
let uri: http::Uri = arguments[0].parse()?;
let host: IpAddr = uri
.host()
.ok_or_else(|| io::Error::other("Missing host"))?
.trim_matches(['[', ']'])
.parse()?;
if !host.is_loopback()
|| uri.scheme_str() != Some("http")
|| uri.query().is_some()
|| !matches!(uri.path(), "" | "/")
|| uri.authority().is_some_and(|a| a.as_str().contains('@'))
{
return Err(io::Error::other("Expected loopback HTTP endpoint").into());
}
let mut credential = Zeroizing::new(Vec::new());
tokio::io::stdin()
.take(1025)
.read_to_end(&mut credential)
.await?;
if credential.is_empty() || credential.len() > 1024 {
return Err(io::Error::other("Invalid authorization input").into());
}
let mut token: MetadataValue<Ascii> = std::str::from_utf8(&credential)?.trim().parse()?;
token.set_sensitive(true);
let channel = tonic::transport::Endpoint::from_shared(arguments[0].clone())?
.connect_timeout(Duration::from_secs(3))
.timeout(Duration::from_secs(10))
.connect()
.await?;
let mut client = v2::system_client::SystemClient::new(channel);
let status = client
.get_status(authorized(v2::GetStatusRequest {}, &token))
.await?
.into_inner();
let storage = status
.storage
.ok_or_else(|| io::Error::other("Missing storage status"))?;
if status.api_version != "nuncio.v2" || storage.revision == 0 {
return Err(io::Error::other("Expected v2 profile with a committed change").into());
}
let mut stream = client
.watch_changes(authorized(
v2::WatchChangesRequest { after_revision: 0 },
&token,
))
.await?
.into_inner();
let change = stream
.message()
.await?
.ok_or_else(|| io::Error::other("Missing change"))?;
println!(
"{}",
serde_json::json!({"api_version":status.api_version,"profile_id":status.profile_id,"revision":storage.revision,"change":{"revision":change.revision,"kind":change.kind,"account_id":change.account_id}})
);
Ok(())
}
#[tokio::main]
async fn main() {
if !matches!(
tokio::time::timeout(Duration::from_secs(15), run()).await,
Ok(Ok(()))
) {
eprintln!("Contract smoke failed");
std::process::exit(1);
}
}
+77
View File
@@ -0,0 +1,77 @@
#![allow(clippy::unwrap_used)]
use nuncio_test_support::{
google::Seed,
process::{isolated_command, E2eHarness},
};
use serde_json::Value;
use std::{path::Path, process::Stdio, time::Duration};
use tokio::io::AsyncWriteExt;
async fn invoke(h: &E2eHarness, credential: &str) -> std::process::Output {
let mut command = isolated_command(
Path::new(env!("CARGO_BIN_EXE_nuncio-api-smoke")),
&h.artifacts,
);
command
.arg(&h.endpoint)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let mut child = command.spawn().unwrap();
child
.stdin
.take()
.unwrap()
.write_all(credential.as_bytes())
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(20), child.wait_with_output())
.await
.unwrap()
.unwrap()
}
#[tokio::test]
async fn external_generated_client_reads_status_and_replays_authenticated_changes() {
let mut h = E2eHarness::start(Seed::TwoAccounts).await.unwrap();
let account = h.connect_google("alpha@example.test").await.unwrap();
let keys: Value = serde_json::from_slice(&std::fs::read(&h.secrets_file).unwrap()).unwrap();
let key = keys
.as_object()
.unwrap()
.iter()
.find(|(name, _)| name.ends_with("/profile/api"))
.unwrap()
.1
.as_str()
.unwrap();
let credential = zeroize::Zeroizing::new(format!("Bearer {key}"));
let output = invoke(&h, &credential).await;
assert_eq!(
output.status.code(),
Some(0),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let result: Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(result["api_version"], "nuncio.v2");
assert!(result["profile_id"].is_string());
assert_eq!(result["change"]["account_id"], account);
assert!(result["change"]["revision"].as_u64().unwrap() > 0);
assert!(result["change"]["revision"].as_u64().unwrap() <= result["revision"].as_u64().unwrap());
assert!(output.stderr.is_empty());
assert!(!String::from_utf8_lossy(&output.stdout).contains(key));
let denied = invoke(&h, "Bearer invalid").await;
assert_eq!(denied.status.code(), Some(1));
assert!(denied.stdout.is_empty());
assert!(!String::from_utf8_lossy(&denied.stderr).contains("invalid"));
let independent = h.google.control().snapshot().await;
assert!(independent
.mail
.values()
.all(|mail| mail.accepted_sends.is_empty()));
assert!(independent.calendars.values().all(|calendars| calendars
.values()
.all(|calendar| calendar.notifications.is_empty())));
h.shutdown().await.unwrap();
}
+29
View File
@@ -0,0 +1,29 @@
[package]
name = "nuncio-cli"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
[lints]
workspace = true
[features]
test-harness = ["nuncio-proto/test-harness"]
[dependencies]
nuncio-proto = { path = "../nuncio-proto" }
tokio.workspace = true
tonic.workspace = true
clap.workspace = true
chrono.workspace = true
serde_json.workspace = true
serde.workspace = true
keyring.workspace = true
zeroize.workspace = true
hex.workspace = true
uuid.workspace = true
sha2.workspace = true
tempfile.workspace = true
rustix = { workspace = true, features = ["termios"] }
futures-util.workspace = true
+391
View File
@@ -0,0 +1,391 @@
mod google;
mod imap;
pub(crate) mod wizard;
use crate::{args::AccountCommand, output::AppError, rpc_error};
use nuncio_proto::{
client::TokenInjector,
v2::{self, accounts_client::AccountsClient},
};
use serde::Deserialize;
use serde_json::{json, Value};
use std::{io::Read, path::Path};
use tonic::transport::Channel;
use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing};
type AccountClient =
AccountsClient<tonic::service::interceptor::InterceptedService<Channel, TokenInjector>>;
#[derive(Deserialize, Zeroize, ZeroizeOnDrop)]
struct Registration {
client_id: String,
#[serde(default)]
client_secret: Option<String>,
}
#[derive(Deserialize)]
struct Downloaded {
installed: Registration,
}
fn registration(path: &Path) -> Result<Registration, AppError> {
let metadata = std::fs::symlink_metadata(path).map_err(|_| AppError::invalid())?;
if !metadata.is_file() || metadata.file_type().is_symlink() || metadata.len() > 65536 {
return Err(AppError::invalid());
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
if metadata.permissions().mode() & 0o077 != 0 {
return Err(AppError {
code: "insecure_file",
message:
"OAuth client configuration must be readable only by its owner (chmod 600)",
sync_run: None,
operation: None,
recovery: None,
exit: 2,
});
}
}
let mut bytes = Zeroizing::new(Vec::new());
std::fs::File::open(path)
.map_err(|_| AppError::invalid())?
.take(65537)
.read_to_end(&mut bytes)
.map_err(|_| AppError::invalid())?;
if bytes.len() > 65536 {
return Err(AppError::invalid());
}
let config: Downloaded = serde_json::from_slice(&bytes).map_err(|_| AppError::invalid())?;
Ok(config.installed)
}
fn selected_registration(path: Option<std::path::PathBuf>) -> Result<Registration, AppError> {
if let Some(path) = path {
return registration(&path).map_err(|error| error.input_context(
"Cannot read Google registration; --client-config needs a private Desktop OAuth JSON with an installed.client_id field (at most 64 KiB). See docs/GOOGLE-SETUP.md."
));
}
let client_id = option_env!("NUNCIO_GOOGLE_CLIENT_ID").filter(|v| !v.trim().is_empty())
.ok_or_else(|| wizard::error("google_setup_required", "Google sign-in needs an app registration. Supply its Desktop JSON with --client-config; see docs/GOOGLE-SETUP.md for the one-time setup.", 2))?;
Ok(Registration {
client_id: client_id.to_owned(),
client_secret: option_env!("NUNCIO_GOOGLE_CLIENT_SECRET")
.filter(|v| !v.is_empty())
.map(str::to_owned),
})
}
pub async fn run(
command: AccountCommand,
channel: Channel,
injector: TokenInjector,
) -> Result<Value, AppError> {
let mut client = AccountsClient::with_interceptor(channel, injector);
match command {
AccountCommand::Add {
client_config,
no_browser,
} => wizard::run(&mut client, client_config, no_browser).await,
AccountCommand::EditImap {
account: id,
version,
config,
credentials_stdin,
} => {
let config = imap::configuration(&config)?;
let credentials = if credentials_stdin {
Some(
tokio::task::spawn_blocking(imap::credentials)
.await
.map_err(|_| AppError::invalid())??,
)
} else {
None
};
let result = client
.update_imap_account(v2::UpdateImapAccountRequest {
account_id: id,
version,
config: Some(config),
credentials,
})
.await
.map_err(rpc_error)?
.into_inner();
Ok(
json!({"account":account(result.account.ok_or_else(AppError::invalid)?),"capabilities":result.capabilities,"credential_cleanup_pending":result.credential_cleanup_pending}),
)
}
AccountCommand::AddGoogle {
client_config,
login_hint,
no_browser,
no_wait,
} => {
google::connect(
&mut client,
client_config,
login_hint,
None,
no_browser,
!no_wait,
)
.await
}
AccountCommand::ReauthGoogle {
account,
client_config,
no_browser,
no_wait,
} => {
google::connect(
&mut client,
client_config,
None,
Some(account),
no_browser,
!no_wait,
)
.await
}
AccountCommand::AuthWait {
session,
timeout_seconds,
} => google::wait(&mut client, session, timeout_seconds).await,
AccountCommand::ReauthImap {
account: id,
credentials_stdin: _,
} => {
let config = client
.get_imap_config(v2::AccountRequest {
account_id: id.clone(),
})
.await
.map_err(rpc_error)?
.into_inner()
.config
.ok_or_else(AppError::invalid)?;
let credentials = tokio::task::spawn_blocking(imap::credentials)
.await
.map_err(|_| AppError::invalid())??;
let result = client
.connect_imap(v2::ConnectImapRequest {
config: Some(config),
credentials: Some(credentials),
account_id: Some(id),
})
.await
.map_err(rpc_error)?
.into_inner();
Ok(
json!({"account":account(result.account.ok_or_else(AppError::invalid)?),"capabilities":result.capabilities,"credential_cleanup_pending":result.credential_cleanup_pending}),
)
}
AccountCommand::Show { account: id } => {
let value = client
.get_account(v2::AccountRequest { account_id: id })
.await
.map_err(rpc_error)?
.into_inner();
Ok(json!({"account": account(value)}))
}
AccountCommand::Edit {
account: id,
name,
version,
} => {
let value = client
.update_account(v2::UpdateAccountRequest {
account_id: id,
version,
display_name: name,
})
.await
.map_err(rpc_error)?
.into_inner();
Ok(account(value))
}
AccountCommand::Pause { account: id } => {
lifecycle(&mut client, id, v2::AccountLifecycleAction::Pause).await
}
AccountCommand::Resume { account: id } => {
lifecycle(&mut client, id, v2::AccountLifecycleAction::Resume).await
}
AccountCommand::Remove { account: id } => {
lifecycle(&mut client, id, v2::AccountLifecycleAction::Archive).await
}
AccountCommand::Restore { account: id } => {
lifecycle(&mut client, id, v2::AccountLifecycleAction::Restore).await
}
AccountCommand::AuthCancel { session: id } => Ok(session(
client
.cancel_google_auth(v2::GetAuthStatusRequest { session_id: id })
.await
.map_err(rpc_error)?
.into_inner(),
)),
AccountCommand::Purge {
account: id,
dry_run,
confirm,
} => {
if !dry_run && confirm.as_deref() != Some(&id) {
return Err(AppError::invalid());
}
let preview = client
.preview_account_purge(v2::AccountRequest {
account_id: id.clone(),
})
.await
.map_err(rpc_error)?
.into_inner();
if dry_run {
return Ok(json!(preview));
}
let result = client
.purge_account(v2::PurgeAccountRequest {
account_id: id.clone(),
version: preview.version,
revision: preview.revision,
confirm_account_id: confirm.ok_or_else(AppError::invalid)?,
})
.await
.map_err(rpc_error)?
.into_inner();
Ok(
json!({"account_id":id,"state":"purged","deleted":result.deleted,"credential_cleanup_pending":result.credential_cleanup_pending}),
)
}
AccountCommand::ConnectImap {
config,
credentials_stdin: _,
account: id,
} => {
let config = imap::configuration(&config)?;
let credentials = tokio::task::spawn_blocking(imap::credentials)
.await
.map_err(|_| AppError::invalid())??;
let result = client
.connect_imap(v2::ConnectImapRequest {
config: Some(config),
credentials: Some(credentials),
account_id: id,
})
.await
.map_err(rpc_error)?
.into_inner();
Ok(
json!({"account":account(result.account.ok_or_else(AppError::invalid)?),"capabilities":result.capabilities,"credential_cleanup_pending":result.credential_cleanup_pending}),
)
}
AccountCommand::ImapConfig { account: id } => {
let result = client
.get_imap_config(v2::AccountRequest {
account_id: id.clone(),
})
.await
.map_err(rpc_error)?
.into_inner();
Ok(
json!({"account_id":id,"config":imap::public_config(result.config.ok_or_else(AppError::invalid)?)?,"capabilities":result.capabilities}),
)
}
AccountCommand::ConnectGoogle {
client_config,
login_hint,
account,
no_browser,
wait,
} => {
google::connect(
&mut client,
Some(client_config),
login_hint,
account,
no_browser,
wait,
)
.await
}
AccountCommand::AuthStatus { session: id } => Ok(session(
client
.get_auth_status(v2::GetAuthStatusRequest { session_id: id })
.await
.map_err(rpc_error)?
.into_inner(),
)),
AccountCommand::List { include_archived } => {
let result = client
.list_accounts(v2::ListAccountsRequest { include_archived })
.await
.map_err(rpc_error)?
.into_inner();
Ok(json!({"accounts":result.accounts.into_iter().map(account).collect::<Vec<_>>()}))
}
AccountCommand::Disconnect { account: id } => {
client
.disconnect_account(v2::AccountRequest {
account_id: id.clone(),
})
.await
.map_err(rpc_error)?;
Ok(json!({"account_id":id,"state":"disconnected"}))
}
AccountCommand::Check { account: id } => Ok(account(
client
.check_account(v2::AccountRequest { account_id: id })
.await
.map_err(rpc_error)?
.into_inner(),
)),
}
}
fn session(value: v2::AuthSession) -> Value {
json!({"session_id":value.session_id,"browser_url":value.browser_url,"expires_at_ms":value.expires_at_ms,
"state":value.state,"account_id":value.account_id,"error_code":value.error_code,"warning_code":value.warning_code})
}
fn account(value: v2::Account) -> Value {
json!({"id":value.id,"provider":value.provider,"address":value.address,"state":value.state,"display_name":value.display_name,"version":value.version,"auth_state":value.auth_state,"identity":value.identity})
}
async fn lifecycle(
client: &mut AccountsClient<
tonic::service::interceptor::InterceptedService<Channel, TokenInjector>,
>,
id: String,
action: v2::AccountLifecycleAction,
) -> Result<Value, AppError> {
let result = client
.set_account_lifecycle(v2::SetAccountLifecycleRequest {
account_id: id,
action: action.into(),
})
.await
.map_err(rpc_error)?
.into_inner();
let mut value = account(result.account.ok_or_else(AppError::invalid)?);
value["credential_cleanup_pending"] = json!(result.credential_cleanup_pending);
Ok(value)
}
async fn open_browser(url: &str) -> bool {
#[cfg(target_os = "macos")]
let command = "/usr/bin/open";
#[cfg(not(target_os = "macos"))]
let command = "xdg-open";
match tokio::time::timeout(
std::time::Duration::from_secs(10),
tokio::process::Command::new(command)
.arg(url)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.kill_on_drop(true)
.status(),
)
.await
{
Ok(Ok(status)) if status.success() => true,
_ => {
eprintln!("Browser could not be opened; use the returned browser_url.");
false
}
}
}
@@ -0,0 +1,137 @@
use super::{open_browser, rpc_error, session, v2, AccountClient, AppError};
use serde_json::{json, Value};
use std::{path::PathBuf, time::Duration};
pub(super) async fn connect(
client: &mut AccountClient,
config: Option<PathBuf>,
login_hint: Option<String>,
account_id: Option<String>,
no_browser: bool,
wait_for_consent: bool,
) -> Result<Value, AppError> {
let registration = super::selected_registration(config)?;
connect_registered(
client,
registration,
login_hint,
account_id,
no_browser,
wait_for_consent,
)
.await
}
pub(super) async fn connect_registered(
client: &mut AccountClient,
mut registration: super::Registration,
login_hint: Option<String>,
account_id: Option<String>,
no_browser: bool,
wait_for_consent: bool,
) -> Result<Value, AppError> {
let mut interrupt = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())
.map_err(|_| AppError::unavailable())?;
let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.map_err(|_| AppError::unavailable())?;
let result = client
.begin_google_auth(v2::BeginGoogleAuthRequest {
client_id: std::mem::take(&mut registration.client_id),
client_secret: registration.client_secret.take(),
login_hint,
account_id,
})
.await
.map_err(rpc_error)?
.into_inner();
let opened = !no_browser && open_browser(&result.browser_url).await;
if wait_for_consent {
eprintln!(
"Complete Google consent in your browser: {}",
result.browser_url
);
return wait_with_signals(
client,
result.session_id,
300,
&mut interrupt,
&mut terminate,
)
.await;
}
let mut value = session(result);
value["browser_opened"] = json!(opened);
Ok(value)
}
pub(super) async fn wait(
client: &mut AccountClient,
id: String,
seconds: u64,
) -> Result<Value, AppError> {
let mut interrupt = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())
.map_err(|_| AppError::unavailable())?;
let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.map_err(|_| AppError::unavailable())?;
wait_with_signals(client, id, seconds, &mut interrupt, &mut terminate).await
}
async fn wait_with_signals(
client: &mut AccountClient,
id: String,
seconds: u64,
interrupt: &mut tokio::signal::unix::Signal,
terminate: &mut tokio::signal::unix::Signal,
) -> Result<Value, AppError> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(seconds);
loop {
let status = tokio::time::timeout_at(
deadline,
client.get_auth_status(v2::GetAuthStatusRequest {
session_id: id.clone(),
}),
)
.await
.map_err(|_| AppError {
code: "auth_timeout",
message: "Authorization status timed out; run account auth-status for this session",
exit: 5,
recovery: Some(json!({"session_id":id})),
operation: None,
sync_run: None,
})?
.map_err(rpc_error)?
.into_inner();
match status.state.as_str() {
"succeeded" => return Ok(session(status)),
"pending" | "completing" => {}
_ => {
return Err(auth_error(
"auth_failed",
"Google authorization did not complete; inspect the session status",
status,
))
}
}
tokio::select! {
_ = tokio::time::sleep_until(deadline) => return Err(auth_error("auth_timeout", "Google consent is still pending; complete it and run account auth-wait again", status)),
_ = tokio::time::sleep(Duration::from_millis(200)) => {}
_ = async { tokio::select! { _ = interrupt.recv() => {}, _ = terminate.recv() => {} } } => {
let status = client.cancel_google_auth(v2::GetAuthStatusRequest { session_id: id }).await.map_err(rpc_error)?.into_inner();
if status.state == "succeeded" { return Ok(session(status)); }
return Err(auth_error("auth_cancelled", "Google authorization was cancelled", status));
}
}
}
}
fn auth_error(code: &'static str, message: &'static str, status: v2::AuthSession) -> AppError {
AppError {
code,
message,
exit: 5,
recovery: Some(session(status)),
operation: None,
sync_run: None,
}
}
@@ -0,0 +1,166 @@
use crate::output::AppError;
use nuncio_proto::v2;
use serde::{Deserialize, Serialize};
use std::{
io::{IsTerminal, Read},
path::Path,
};
use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing};
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct Configuration {
schema_version: u32,
address: String,
imap: Endpoint,
smtp: Endpoint,
sent_policy: SentPolicy,
sent_folder: String,
archive_folder: Option<String>,
trash_folder: Option<String>,
trusted_ca_pem: Option<String>,
}
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct Endpoint {
host: String,
port: u16,
tls: Tls,
username: String,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
enum Tls {
Implicit,
StartTls,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
enum SentPolicy {
Server,
ClientAppend,
}
#[derive(Deserialize, Zeroize, ZeroizeOnDrop)]
#[serde(deny_unknown_fields)]
struct Credentials {
imap_password: String,
smtp_password: String,
}
pub(super) fn configuration(path: &Path) -> Result<v2::ImapAccountConfig, AppError> {
let file = std::fs::File::open(path).map_err(|_| AppError::invalid())?;
let meta = file.metadata().map_err(|_| AppError::invalid())?;
if !meta.is_file() || meta.len() > 256 * 1024 {
return Err(AppError::invalid());
}
let mut bytes = Vec::new();
file.take(256 * 1024 + 1)
.read_to_end(&mut bytes)
.map_err(|_| AppError::invalid())?;
decode_config(&bytes)
}
fn decode_config(bytes: &[u8]) -> Result<v2::ImapAccountConfig, AppError> {
if bytes.len() > 256 * 1024 {
return Err(AppError::invalid());
}
let config: Configuration = serde_json::from_slice(bytes).map_err(|_| AppError::invalid())?;
if config.schema_version != 1 {
return Err(AppError::invalid());
}
Ok(v2::ImapAccountConfig {
address: config.address,
imap: Some(endpoint(config.imap)),
smtp: Some(endpoint(config.smtp)),
sent_policy: match config.sent_policy {
SentPolicy::Server => v2::SentPolicy::Server,
SentPolicy::ClientAppend => v2::SentPolicy::ClientAppend,
}
.into(),
sent_folder: config.sent_folder,
archive_folder: config.archive_folder,
trash_folder: config.trash_folder,
trusted_ca_pem: config.trusted_ca_pem,
})
}
fn endpoint(e: Endpoint) -> v2::MailEndpoint {
v2::MailEndpoint {
host: e.host,
port: e.port.into(),
tls: match e.tls {
Tls::Implicit => v2::MailTls::Implicit,
Tls::StartTls => v2::MailTls::StartTls,
}
.into(),
username: e.username,
}
}
pub(super) fn credentials() -> Result<v2::ImapCredentials, AppError> {
let stdin = std::io::stdin();
if stdin.is_terminal() {
return Err(AppError {code:"credential_input",message:"Pipe a credential JSON object from a secure source; interactive terminal input is refused to prevent password echo",exit:2,sync_run:None,operation: None, recovery: None});
}
let mut bytes = Zeroizing::new(Vec::new());
stdin
.lock()
.take(16385)
.read_to_end(&mut bytes)
.map_err(|_| AppError::invalid())?;
if bytes.len() > 16384 {
return Err(AppError::invalid());
}
let mut value: Credentials = serde_json::from_slice(&bytes).map_err(|_| AppError::invalid())?;
Ok(v2::ImapCredentials {
imap_password: std::mem::take(&mut value.imap_password),
smtp_password: std::mem::take(&mut value.smtp_password),
})
}
pub(super) fn public_config(v: v2::ImapAccountConfig) -> Result<serde_json::Value, AppError> {
let endpoint = |v: v2::MailEndpoint| -> Result<Endpoint, AppError> {
Ok(Endpoint {
host: v.host,
port: u16::try_from(v.port).map_err(|_| AppError::invalid())?,
tls: match v2::MailTls::try_from(v.tls) {
Ok(v2::MailTls::Implicit) => Tls::Implicit,
Ok(v2::MailTls::StartTls) => Tls::StartTls,
_ => return Err(AppError::invalid()),
},
username: v.username,
})
};
let config = Configuration {
schema_version: 1,
address: v.address,
imap: endpoint(v.imap.ok_or_else(AppError::invalid)?)?,
smtp: endpoint(v.smtp.ok_or_else(AppError::invalid)?)?,
sent_policy: match v2::SentPolicy::try_from(v.sent_policy) {
Ok(v2::SentPolicy::Server) => SentPolicy::Server,
Ok(v2::SentPolicy::ClientAppend) => SentPolicy::ClientAppend,
_ => return Err(AppError::invalid()),
},
sent_folder: v.sent_folder,
archive_folder: v.archive_folder,
trash_folder: v.trash_folder,
trusted_ca_pem: v.trusted_ca_pem,
};
serde_json::to_value(config).map_err(|_| AppError::invalid())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn configuration_rejects_credential_fields_and_unknown_transport() {
let config = serde_json::json!({"schema_version":1,"address":"alpha@example.test","imap":{"host":"localhost","port":993,"tls":"implicit","username":"alpha@example.test"},"smtp":{"host":"localhost","port":587,"tls":"start_tls","username":"alpha@example.test"},"sent_policy":"client_append","sent_folder":"Sent"});
assert!(decode_config(config.to_string().as_bytes()).is_ok());
for (key, value) in [
("password", serde_json::json!("test-only-canary")),
("schema_version", serde_json::json!(2)),
] {
let mut bad = config.clone();
bad[key] = value;
assert!(decode_config(bad.to_string().as_bytes()).is_err());
}
let mut bad = config;
bad["imap"]["tls"] = serde_json::json!("none");
assert!(decode_config(bad.to_string().as_bytes()).is_err());
}
}
@@ -0,0 +1,343 @@
mod terminal;
use super::{account, google, selected_registration, AccountClient};
use crate::output::AppError;
use nuncio_proto::v2;
use serde_json::{json, Value};
use std::path::PathBuf;
use terminal::Terminal;
use zeroize::Zeroizing;
pub(crate) fn error(code: &'static str, message: &'static str, exit: u8) -> AppError {
AppError {
code,
message,
exit,
recovery: None,
operation: None,
sync_run: None,
}
}
pub(crate) fn preflight(json: bool) -> Result<(), AppError> {
Terminal::check(json)
}
pub(super) async fn run(
client: &mut AccountClient,
registration: Option<PathBuf>,
no_browser: bool,
) -> Result<Value, AppError> {
let terminal = Terminal::new()?;
let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.map_err(|_| AppError::invalid())?;
let connection = tokio::select! {
result = setup(&terminal, registration) => result,
_ = tokio::signal::ctrl_c() => Err(error("setup_cancelled", "Account setup cancelled; no sign-in was submitted.", 130)),
_ = terminate.recv() => Err(error("setup_cancelled", "Account setup cancelled; no sign-in was submitted.", 130)),
}?;
drop(terminal);
match connection {
Connection::Google {
registration,
address,
} => {
// OAuth owns cancellation once a session exists, so a late callback
// cannot admit an account after a successful cancellation.
let session = google::connect_registered(
client,
registration,
Some(address),
None,
no_browser,
true,
)
.await?;
let id = session["account_id"]
.as_str()
.ok_or_else(AppError::invalid)?;
let saved = client
.get_account(v2::AccountRequest {
account_id: id.to_owned(),
})
.await
.map_err(crate::rpc_error)?
.into_inner();
Ok(json!({"account":account(saved)}))
}
Connection::Imap {
config,
mut password,
mut smtp_password,
} => {
eprintln!("Checking secure IMAP and SMTP sign-in...");
let request = v2::ConnectImapRequest {
config: Some(*config),
credentials: Some(v2::ImapCredentials {
imap_password: std::mem::take(&mut *password),
smtp_password: std::mem::take(&mut *smtp_password),
}),
account_id: None,
};
let result = tokio::select! {
result = client.connect_imap(request) => result.map_err(|_| error("account_connection_failed", "Could not confirm sign-in. Check your server, login, password and TLS settings. Inspect account list before retrying.", 4)),
_ = tokio::signal::ctrl_c() => Err(error("setup_interrupted", "Stopped waiting for sign-in. Check account list before retrying; the connection may have completed.", 130)),
_ = terminate.recv() => Err(error("setup_interrupted", "Stopped waiting for sign-in. Check account list before retrying; the connection may have completed.", 130)),
}?.into_inner();
Ok(
json!({"account":account(result.account.ok_or_else(AppError::invalid)?),"credential_cleanup_pending":result.credential_cleanup_pending}),
)
}
}
}
enum Connection {
Google {
registration: super::Registration,
address: String,
},
Imap {
config: Box<v2::ImapAccountConfig>,
password: Zeroizing<String>,
smtp_password: Zeroizing<String>,
},
}
async fn setup(t: &Terminal, registration: Option<PathBuf>) -> Result<Connection, AppError> {
t.say("Add an account\n 1. Google (Gmail and Calendar)\n 2. Synology MailPlus / IMAP\nPress Ctrl-C at any time to cancel.")?;
let provider = loop {
match t
.ask("Provider", Some("1"))
.await?
.to_ascii_lowercase()
.as_str()
{
"1" | "google" | "gmail" => break true,
"2" | "imap" | "mailplus" | "synology" => break false,
_ => t.say("Choose 1 for Google or 2 for MailPlus / IMAP.")?,
}
};
let google_registration = if provider {
Some(selected_registration(registration)?)
} else {
None
};
let address = loop {
let value = t.ask("Email address", None).await?;
if valid_address(&value) {
break value;
}
t.say("Enter a complete email address, such as name@example.com.")?;
};
if let Some(registration) = google_registration {
t.say("Google will open in your browser. Nuncio requests Gmail and Calendar access, including write permissions. Connecting starts background mail/calendar synchronization.")?;
confirm(t).await?;
return Ok(Connection::Google {
registration,
address,
});
}
let host = loop {
let value = t.ask("Mail server", None).await?;
if valid_host(&value) {
break value;
}
t.say("Enter just the server hostname or IP address, without a URL or port.")?;
};
let username = t.ask("Username", Some(&address)).await?;
let password = t.password("Password (hidden)").await?;
if password.is_empty() {
return Err(error(
"empty_password",
"A password is required. Account setup stopped.",
2,
));
}
let mut smtp_password = Zeroizing::new(password.to_string());
let mut config = v2::ImapAccountConfig {
address,
imap: Some(v2::MailEndpoint {
host: host.clone(),
port: 993,
tls: v2::MailTls::Implicit.into(),
username: username.clone(),
}),
smtp: Some(v2::MailEndpoint {
host,
port: 587,
tls: v2::MailTls::StartTls.into(),
username,
}),
sent_policy: v2::SentPolicy::ClientAppend.into(),
sent_folder: "Sent".into(),
archive_folder: None,
trash_folder: None,
trusted_ca_pem: None,
};
t.say("Defaults: IMAP 993 with TLS; SMTP 587 with STARTTLS; same login for both. Sent-copy policy can be configured before sending. Connecting starts background mailbox synchronization.")?;
if yes(t, "Change advanced server settings?", false).await? {
advanced(t, &mut config, &mut smtp_password).await?;
}
confirm(t).await?;
Ok(Connection::Imap {
config: Box::new(config),
password,
smtp_password,
})
}
async fn yes(t: &Terminal, label: &str, default: bool) -> Result<bool, AppError> {
loop {
match t
.ask(label, Some(if default { "Y/n" } else { "y/N" }))
.await?
.to_ascii_lowercase()
.as_str()
{
"y/n" => return Ok(default),
"y" | "yes" => return Ok(true),
"n" | "no" => return Ok(false),
_ => t.say("Enter yes or no.")?,
}
}
}
async fn confirm(t: &Terminal) -> Result<(), AppError> {
if yes(t, "Connect and start syncing?", true).await? {
Ok(())
} else {
Err(error(
"setup_cancelled",
"Account setup cancelled; no sign-in was submitted.",
130,
))
}
}
fn valid_address(value: &str) -> bool {
value.len() <= 320
&& !value.chars().any(char::is_whitespace)
&& value.split_once('@').is_some_and(|(local, host)| {
!local.is_empty() && !host.contains('@') && valid_host(host)
})
}
fn valid_host(value: &str) -> bool {
!value.is_empty()
&& value.len() <= 253
&& (value.parse::<std::net::IpAddr>().is_ok()
|| value.split('.').all(|s| {
!s.is_empty()
&& s.len() <= 63
&& !s.starts_with('-')
&& !s.ends_with('-')
&& s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-')
}))
}
async fn port(t: &Terminal, label: &str, current: u32) -> Result<u32, AppError> {
loop {
if let Ok(value) = t
.ask(label, Some(&current.to_string()))
.await?
.parse::<u16>()
{
if value > 0 {
return Ok(u32::from(value));
}
}
t.say("Enter a port from 1 to 65535.")?;
}
}
async fn tls(t: &Terminal, label: &str, current: i32) -> Result<i32, AppError> {
loop {
match t
.ask(
label,
Some(if current == v2::MailTls::Implicit as i32 {
"1"
} else {
"2"
}),
)
.await?
.as_str()
{
"1" => return Ok(v2::MailTls::Implicit.into()),
"2" => return Ok(v2::MailTls::StartTls.into()),
_ => t.say(
"Choose 1 for TLS or 2 for STARTTLS. Unencrypted connections are not supported.",
)?,
}
}
}
async fn advanced(
t: &Terminal,
config: &mut v2::ImapAccountConfig,
password: &mut Zeroizing<String>,
) -> Result<(), AppError> {
let imap = config.imap.as_mut().ok_or_else(AppError::invalid)?;
imap.port = port(t, "IMAP port", imap.port).await?;
imap.tls = tls(t, "IMAP security: 1 TLS, 2 STARTTLS", imap.tls).await?;
let smtp = config.smtp.as_mut().ok_or_else(AppError::invalid)?;
loop {
let host = t.ask("SMTP server", Some(&smtp.host)).await?;
if valid_host(&host) {
smtp.host = host;
break;
}
t.say("Enter a server hostname or IP address.")?;
}
smtp.port = port(t, "SMTP port", smtp.port).await?;
smtp.tls = tls(
t,
"SMTP security: 1 TLS, 2 STARTTLS",
if smtp.port == 465 {
v2::MailTls::Implicit.into()
} else {
smtp.tls
},
)
.await?;
smtp.username = t.ask("SMTP username", Some(&smtp.username)).await?;
if yes(t, "Different SMTP password?", false).await? {
*password = t.password("SMTP password (hidden)").await?;
}
let ca = t.ask("Custom CA certificate file (optional)", None).await?;
if !ca.is_empty() {
use std::io::Read;
let mut text = String::new();
{
let file = std::fs::File::from(
rustix::fs::open(
ca.as_str(),
rustix::fs::OFlags::RDONLY
| rustix::fs::OFlags::NONBLOCK
| rustix::fs::OFlags::NOFOLLOW,
rustix::fs::Mode::empty(),
)
.map_err(|_| {
error(
"certificate_file",
"Could not read the custom CA certificate file.",
2,
)
})?,
);
if !file.metadata().map_err(|_| AppError::invalid())?.is_file() {
return Err(AppError::invalid());
}
file
}
.take(256 * 1024 + 1)
.read_to_string(&mut text)
.map_err(|_| AppError::invalid())?;
if text.len() > 256 * 1024 {
return Err(AppError::invalid());
}
config.trusted_ca_pem = Some(text);
}
config.sent_folder = t.ask("Sent folder", Some("Sent")).await?;
if yes(t, "Does the SMTP server save its own Sent copy?", false).await? {
config.sent_policy = v2::SentPolicy::Server.into();
}
let archive = t.ask("Archive folder (optional)", None).await?;
config.archive_folder = (!archive.is_empty()).then_some(archive);
let trash = t.ask("Trash folder (optional)", None).await?;
config.trash_folder = (!trash.is_empty()).then_some(trash);
Ok(())
}
@@ -0,0 +1,131 @@
use super::error;
use crate::output::AppError;
use rustix::{
fs::{fcntl_getfl, fcntl_setfl, OFlags},
termios::{tcgetattr, tcsetattr, LocalModes, OptionalActions, Termios},
};
use std::{
fs::File,
io::{IsTerminal, Write},
};
use zeroize::Zeroizing;
pub struct Terminal {
file: File,
original: Termios,
}
fn failed() -> AppError {
error(
"terminal_input",
"Could not read the terminal securely; account setup stopped.",
2,
)
}
impl Terminal {
pub fn check(json: bool) -> Result<(), AppError> {
if json || !std::io::stdin().is_terminal() || !std::io::stderr().is_terminal() {
return Err(error("interactive_terminal_required", "Run account add in an interactive terminal. For scripts, use account add-google or account add-imap.", 2));
}
Ok(())
}
pub fn new() -> Result<Self, AppError> {
Self::check(false)?;
let file = File::open("/dev/tty").map_err(|_| failed())?;
let original = tcgetattr(&file).map_err(|_| failed())?;
let flags = fcntl_getfl(&file).map_err(|_| failed())?;
fcntl_setfl(&file, flags | OFlags::NONBLOCK).map_err(|_| failed())?;
Ok(Self { file, original })
}
pub fn say(&self, text: &str) -> Result<(), AppError> {
writeln!(std::io::stderr().lock(), "{text}").map_err(|_| failed())
}
pub async fn ask(&self, label: &str, default: Option<&str>) -> Result<String, AppError> {
let value = self.line(label, default, false).await?;
let value = value.trim();
Ok(if value.is_empty() {
default.unwrap_or("").to_owned()
} else {
value.to_owned()
})
}
pub async fn password(&self, label: &str) -> Result<Zeroizing<String>, AppError> {
self.line(label, None, true).await
}
async fn line(
&self,
label: &str,
default: Option<&str>,
secret: bool,
) -> Result<Zeroizing<String>, AppError> {
let mut attributes = self.original.clone();
if secret {
attributes
.local_modes
.remove(LocalModes::ECHO | LocalModes::ECHONL);
}
tcsetattr(&self.file, OptionalActions::Now, &attributes).map_err(|_| failed())?;
{
let mut output = std::io::stderr().lock();
if let Some(value) = default {
write!(output, "{label} [{value}]: ")
} else {
write!(output, "{label}: ")
}
.map_err(|_| failed())?;
output.flush().map_err(|_| failed())?;
}
let mut bytes = Zeroizing::new(Vec::new());
loop {
let mut chunk = Zeroizing::new([0u8; 1024]);
match rustix::io::read(&self.file, &mut *chunk) {
Ok(0) => return Err(error("setup_cancelled", "Account setup cancelled.", 130)),
Ok(n) => {
bytes.extend_from_slice(&chunk[..n]);
if bytes.len() > 4096 {
return Err(error(
"input_too_long",
"That value is too long; account setup stopped.",
2,
));
}
if bytes.last() == Some(&b'\n') {
break;
}
}
Err(rustix::io::Errno::AGAIN | rustix::io::Errno::INTR) => {
tokio::time::sleep(std::time::Duration::from_millis(20)).await
}
Err(_) => return Err(failed()),
}
}
tcsetattr(&self.file, OptionalActions::Now, &self.original).map_err(|_| failed())?;
if secret {
self.say("")?;
}
if bytes.last() == Some(&b'\n') {
bytes.pop();
}
if bytes.last() == Some(&b'\r') {
bytes.pop();
}
let value = std::str::from_utf8(&bytes).map_err(|_| failed())?;
if value.chars().any(|c| {
c.is_control() || matches!(c, '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}')
}) {
return Err(error(
"invalid_input",
"Control characters are not allowed in setup values.",
2,
));
}
Ok(Zeroizing::new(value.to_owned()))
}
}
impl Drop for Terminal {
fn drop(&mut self) {
let _ = tcsetattr(&self.file, OptionalActions::Now, &self.original);
}
}
+900
View File
@@ -0,0 +1,900 @@
use clap::{Parser, Subcommand};
use std::path::PathBuf;
#[derive(Parser)]
#[command(version, about = "Mail and calendar on your machine", arg_required_else_help = true, color = clap::ColorChoice::Never, after_help = crate::help_text::ROOT)]
pub struct Args {
/// Output the versioned JSON result or error for scripts; default output is readable text.
#[arg(long, global = true)]
pub json: bool,
/// Loopback HTTP address of the running daemon.
#[arg(long, global = true, default_value = "http://127.0.0.1:9421")]
pub endpoint: String,
/// Use this explicit local data directory instead of the named profile default.
#[arg(long, global = true)]
pub data_dir: Option<PathBuf>,
/// Local data profile used by the daemon; one profile can contain multiple accounts.
#[arg(long, global = true, default_value = "default")]
pub profile: String,
#[cfg(feature = "test-harness")]
/// Synthetic keystore file for offline test harnesses; absent from production builds.
#[arg(long, global = true)]
pub test_secrets_file: Option<PathBuf>,
#[command(subcommand)]
pub command: Command,
}
#[derive(Subcommand)]
pub enum Command {
/// Rebuild a provider projection while preserving drafts and operation history.
#[command(after_help = crate::help_text::REPAIR)]
Repair(RepairArgs),
/// Create, inspect or restore encrypted local backups.
#[command(arg_required_else_help = true, after_help = crate::help_text::BACKUP)]
Backup {
#[command(subcommand)]
command: BackupCommand,
},
/// Inspect queued work, delivery outcomes and recovery decisions.
#[command(name = "operation", visible_alias = "operations")]
#[command(arg_required_else_help = true, after_help = crate::help_text::OPERATIONS)]
Operations {
#[command(subcommand)]
command: OperationsCommand,
},
/// Read cached calendars and manage Google calendar events.
#[command(arg_required_else_help = true, after_help = crate::help_text::CALENDAR)]
Calendar {
#[command(subcommand)]
command: CalendarCommand,
},
/// Download mail changes for one account; returns a sync run unless --wait is used.
#[command(after_help = crate::help_text::SYNC)]
Sync {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Reconcile the complete remote mail listing instead of only incremental changes.
#[arg(long)]
full: bool,
/// Wait for completion and report the final state; queued work remains inspectable after a timeout.
#[arg(long)]
wait: bool,
},
/// List, read, search, draft and send mail for one account.
#[command(arg_required_else_help = true, after_help = crate::help_text::MAIL)]
Mail {
#[command(subcommand)]
command: MailCommand,
},
/// Add accounts or manage one saved account by its ID.
#[command(
arg_required_else_help = true,
after_help = "Work on one account:\n 1. Run 'nuncio-cli account list' and find the matching address or display_name.\n 2. Copy that entry's id and pass it as --account ACCOUNT_ID after the action.\n Use the local ID, not the email address or display name.\n 3. Use the same --profile as your daemon (for example --profile laptop-qa).\n A profile can hold multiple accounts; each command selects its account.\n\nExamples (replace ACCOUNT_ID and VERSION with values from list/show):\n nuncio-cli account list\n nuncio-cli account show --account ACCOUNT_ID\n nuncio-cli account edit --account ACCOUNT_ID --name 'Personal' --version VERSION\n nuncio-cli account pause --account ACCOUNT_ID\n nuncio-cli account resume --account ACCOUNT_ID\n nuncio-cli account remove --account ACCOUNT_ID\n nuncio-cli mail list --account ACCOUNT_ID\n\nRemoval archives local data by default. Use restore to recover an archived account.\nRun 'nuncio-cli account ACTION --help' for the options for any action."
)]
Account {
#[command(subcommand)]
command: AccountCommand,
},
/// Check the daemon, follow changes, inspect sync runs or shut it down.
#[command(arg_required_else_help = true, after_help = crate::help_text::SYSTEM)]
System {
#[command(subcommand)]
command: SystemCommand,
},
}
#[derive(clap::Args)]
pub struct RepairArgs {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
pub account: String,
/// Provider cache to rebuild: mail or calendar; local drafts and history are preserved.
#[arg(long, value_enum)]
pub scope: RepairScope,
/// Preview the local change without applying it or contacting the provider.
#[arg(long)]
pub dry_run: bool,
/// Wait for completion and report the final state; queued work remains inspectable after a timeout.
#[arg(long, conflicts_with = "dry_run")]
pub wait: bool,
/// Inclusive start date, YYYY-MM-DD; use together with --to.
#[arg(long, requires = "to")]
pub from: Option<String>,
/// Exclusive end date, YYYY-MM-DD; use together with --from.
#[arg(long, requires = "from")]
pub to: Option<String>,
}
#[derive(Clone, Copy, clap::ValueEnum)]
pub enum RepairScope {
Mail,
Calendar,
}
#[derive(Subcommand)]
pub enum BackupCommand {
/// Create an encrypted snapshot; read recovery-passphrase JSON from protected stdin.
#[command(after_help = crate::help_text::BACKUP)]
Create {
/// New destination file on this machine; existing files are never overwritten.
#[arg(long)]
output: PathBuf,
},
/// Inspect a backup; read recovery-passphrase JSON from protected stdin.
#[command(after_help = crate::help_text::BACKUP)]
Inspect {
/// Existing encrypted Nuncio backup file.
#[arg(long)]
file: PathBuf,
},
/// Restore into a new sibling profile; all pending writes remain held.
#[command(after_help = crate::help_text::BACKUP)]
Restore {
/// Existing encrypted Nuncio backup file to restore into a new profile.
#[arg(long)]
file: PathBuf,
/// New sibling profile name (1-64 letters, digits, hyphens or underscores); must not exist.
#[arg(long,value_parser=parse_backup_profile)]
new_profile: String,
},
}
fn parse_backup_profile(value: &str) -> Result<String, String> {
if value.is_empty()
|| value.len() > 64
|| !value
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_'))
{
return Err("Profile names use 1–64 ASCII letters, digits, hyphens or underscores".into());
}
Ok(value.into())
}
#[derive(Subcommand)]
pub enum CalendarCommand {
/// Explicit online availability query; reports per-calendar coverage and errors.
#[command(after_help = crate::help_text::FREE_BUSY)]
FreeBusy {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Version1 query JSON with RFC3339 bounds, time zone, and provider calendar IDs.
#[arg(long)]
file: PathBuf,
},
/// Create, update, delete or respond to an event using an explicit JSON action.
#[command(after_help = crate::help_text::CALENDAR_CHANGE)]
Change {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local calendar ID from calendar list.
#[arg(long)]
calendar: String,
/// A UUID for this action (create once with uuidgen); reuse only for the identical retry.
#[arg(long)]
#[arg(value_parser = parse_request_id)]
request_id: String,
/// Version1 Calendar action JSON with explicit scope and notification policy.
#[arg(long)]
file: PathBuf,
/// Wait for completion and report the final state; queued work remains inspectable after a timeout.
#[arg(long)]
wait: bool,
},
/// List cached calendars and their local IDs.
List {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Number of items per page (1-100).
#[arg(long, default_value_t = 100)]
#[arg(value_parser = clap::value_parser!(u32).range(1..=100))]
page_size: u32,
/// Opaque next_page_token from the previous result; keep the same query and page size.
#[arg(long)]
page_token: Option<String>,
},
/// Read one cached event by local calendar and event IDs.
Get {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local calendar ID from calendar list.
#[arg(long)]
calendar: String,
/// Local event ID from calendar agenda.
#[arg(long)]
event: String,
},
/// List cached occurrences in a date window; reports cache coverage.
Agenda {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Inclusive start date, YYYY-MM-DD; use together with --to.
#[arg(long)]
from: String,
/// Exclusive upper date, interpreted in each calendar's time zone.
#[arg(long)]
to: String,
/// Number of items per page (1-100).
#[arg(long, default_value_t = 100)]
#[arg(value_parser = clap::value_parser!(u32).range(1..=100))]
page_size: u32,
/// Opaque next_page_token from the previous result; keep the same query and page size.
#[arg(long)]
page_token: Option<String>,
},
/// Download calendars and events; optionally refresh a paired date window.
Refresh {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Inclusive start date, YYYY-MM-DD; use together with --to.
#[arg(long, requires = "to")]
from: Option<String>,
/// Exclusive end date, YYYY-MM-DD; use together with --from.
#[arg(long, requires = "from")]
to: Option<String>,
/// Wait for completion and report the final state; queued work remains inspectable after a timeout.
#[arg(long)]
wait: bool,
},
}
#[derive(Subcommand)]
pub enum AccountCommand {
/// Add an account with guided prompts and hidden password entry.
#[command(
after_help = "Run in an interactive terminal. This guided command does not accept --json.\nFor scripts, use add-google or connect-imap; see their --help for private input formats."
)]
Add {
/// Private Google Desktop OAuth JSON; needed when the build has no bundled registration.
#[arg(long)]
client_config: Option<PathBuf>,
/// Print the Google consent URL instead of opening a browser.
#[arg(long)]
no_browser: bool,
},
/// Probe and save public IMAP/SMTP settings for the same IMAP principal.
#[command(after_help = crate::help_text::IMAP_SETUP)]
EditImap {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
/// Current configuration version from account show.
#[arg(long)]
version: u64,
/// Public IMAP/SMTP settings JSON; passwords are supplied separately.
#[arg(long)]
config: PathBuf,
/// Optionally replace passwords using protected stdin input.
#[arg(long)]
credentials_stdin: bool,
},
/// Add a Google account and wait for browser consent by default.
#[command(after_help = crate::help_text::GOOGLE_SETUP)]
AddGoogle {
/// Private Google Desktop OAuth JSON; needed when the build has no bundled registration.
#[arg(long)]
client_config: Option<PathBuf>,
/// Google email address to suggest in browser sign-in.
#[arg(long)]
login_hint: Option<String>,
/// Print the consent URL so you can open it yourself.
#[arg(long)]
no_browser: bool,
/// Return the sign-in session immediately; inspect it with account auth-status.
#[arg(long)]
no_wait: bool,
},
/// Reauthenticate exactly this saved Google identity.
#[command(after_help = crate::help_text::GOOGLE_SETUP)]
ReauthGoogle {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
/// Private Google Desktop OAuth JSON; needed when the build has no bundled registration.
#[arg(long)]
client_config: Option<PathBuf>,
/// Print the consent URL so you can open it yourself.
#[arg(long)]
no_browser: bool,
/// Return the sign-in session immediately; inspect it with account auth-status.
#[arg(long)]
no_wait: bool,
},
/// Replace passwords for the saved IMAP/SMTP configuration.
#[command(after_help = crate::help_text::IMAP_SETUP)]
ReauthImap {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
/// Read credential JSON from a secure pipe; use account add for hidden interactive entry.
#[arg(long, required = true)]
credentials_stdin: bool,
},
/// Wait for terminal browser-consent status; pending consent returns a timeout error.
AuthWait {
/// Sign-in session_id returned by the Google connection command.
#[arg(long)]
session: String,
/// Maximum seconds to wait for Google consent (1-300).
#[arg(long, default_value_t = 300, value_parser=clap::value_parser!(u64).range(1..=300))]
timeout_seconds: u64,
},
/// Read saved identity, display name, lifecycle and configuration version.
Show {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
},
/// Change the local display name using the version from account show.
Edit {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
/// New local display name; the provider login stays the same.
#[arg(long)]
name: String,
/// Current version from account show; protects against overwriting a newer edit.
#[arg(long)]
version: u64,
},
/// Stop provider work while retaining credentials and downloaded data.
Pause {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
},
/// Resume background synchronization for a paused account.
Resume {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
},
/// Archive locally and disconnect credentials; downloaded data is retained.
#[command(visible_alias = "archive")]
Remove {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
},
/// Unarchive a saved account. Reauthenticate separately to reconnect.
Restore {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
},
/// Permanently delete an archived account from this profile; remote data and backups remain.
Purge {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
/// Preview the local change without applying it or contacting the provider.
#[arg(long, conflicts_with = "confirm")]
dry_run: bool,
/// Exact account ID. Required unless --dry-run is supplied.
#[arg(long, required_unless_present = "dry_run")]
confirm: Option<String>,
},
/// Cancel a pending Google browser sign-in session.
AuthCancel {
/// Sign-in session_id returned by the Google connection command.
#[arg(long)]
session: String,
},
/// Authenticate IMAP and SMTP with required TLS; passwords arrive only on stdin.
#[command(visible_alias = "add-imap")]
#[command(after_help = crate::help_text::IMAP_SETUP)]
ConnectImap {
/// Version1 public configuration JSON; no passwords.
#[arg(long)]
config: PathBuf,
/// Read a bounded credential JSON object from a pipe (terminal input is refused).
#[arg(long, required = true)]
credentials_stdin: bool,
/// Reconnect this exact saved IMAP endpoint and username.
/// Use the local ID from account list, not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: Option<String>,
},
/// Read saved public configuration and the last authenticated capability snapshot.
ImapConfig {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
},
/// Start Google sign-in using a Desktop registration file; add-google waits by default.
#[command(after_help = crate::help_text::GOOGLE_SETUP)]
ConnectGoogle {
/// Downloaded Desktop OAuth registration JSON, readable only by its owner.
#[arg(long)]
client_config: PathBuf,
/// Google email address to suggest in browser sign-in.
#[arg(long)]
login_hint: Option<String>,
/// Reconnect this saved identity; choosing a different Google user fails.
/// Use the local ID from account list, not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: Option<String>,
/// Return the browser URL without opening it automatically.
#[arg(long)]
no_browser: bool,
/// Wait up to five minutes for consent; add-google waits by default.
#[arg(long)]
wait: bool,
},
/// Inspect a Google browser sign-in session by its session ID.
AuthStatus {
/// Sign-in session_id returned by the Google connection command.
#[arg(long)]
session: String,
},
/// List saved accounts and their IDs, addresses, names and states (local read).
#[command(display_order = 0)]
List {
/// Also show archived accounts so they can be restored or permanently deleted.
#[arg(long)]
include_archived: bool,
},
/// Remove saved credentials and stop provider work; keep the local account and data.
Disconnect {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
},
/// Explicit online credential/identity check. Account list reads local state.
Check {
/// Local account ID from account list; not an email address or display name.
#[arg(long, value_name = "ACCOUNT_ID")]
account: String,
},
}
#[derive(Subcommand)]
pub enum SystemCommand {
/// Committed changes as JSONL; save the last revision to resume later.
Watch {
/// Last observed revision; get an initial revision from system status.
#[arg(long)]
after: u64,
/// Stop successfully after this many changes; otherwise watch until interrupted.
#[arg(long,value_parser=clap::value_parser!(u32).range(1..))]
limit: Option<u32>,
},
/// Show daemon health, storage, account counts and background activity.
Status,
/// Gracefully stop the daemon after draining active workers.
Shutdown,
/// Inspect the state and error code of a saved synchronization run.
SyncStatus {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Sync run ID returned by sync, mail fetch, calendar refresh or repair.
#[arg(long)]
run: String,
},
/// Cancel a queued or running sync; keep the last complete local cache.
CancelSync {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Sync run ID returned by sync, mail fetch, calendar refresh or repair.
#[arg(long)]
run: String,
},
}
#[derive(clap::Args)]
pub struct MailQueryArgs {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
pub account: String,
/// Local label or folder ID from mail collections; omit to search all cached mail.
#[arg(long)]
pub collection: Option<String>,
/// Number of items per page (1-100).
#[arg(long, default_value_t = 100)]
#[arg(value_parser = clap::value_parser!(u32).range(1..=100))]
pub page_size: u32,
/// Opaque next_page_token from the previous result; keep the same query and page size.
#[arg(long)]
pub page_token: Option<String>,
}
#[derive(Subcommand)]
pub enum MailCommand {
/// Report supported actions and provider placement semantics.
Capabilities {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
},
/// Durably request an explicit read/star/archive/trash/label state from a JSON file.
#[command(after_help = crate::help_text::MAIL_CHANGE)]
Change {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local message ID from mail list or mail search.
#[arg(long)]
message: String,
/// A UUID for this action (create once with uuidgen); reuse only for the identical retry.
#[arg(long)]
#[arg(value_parser = parse_request_id)]
request_id: String,
/// Action JSON with schema_version 1; see the example below.
#[arg(long)]
file: PathBuf,
/// Wait for completion and report the final state; queued work remains inspectable after a timeout.
#[arg(long)]
wait: bool,
},
/// Durably enqueue an immutable submission; inspect its operation for delivery.
Send {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local draft ID from mail draft list or mail draft show.
#[arg(long)]
draft: String,
/// A UUID for this action (create once with uuidgen); reuse only for the identical retry.
#[arg(long)]
#[arg(value_parser = parse_request_id)]
request_id: String,
/// Current object version from show; protects against overwriting newer changes.
#[arg(long,value_parser=clap::value_parser!(u64).range(1..))]
version: Option<u64>,
/// Wait for completion and report the final state; queued work remains inspectable after a timeout.
#[arg(long)]
wait: bool,
},
/// Create a local reply draft; sending is a separate explicit command.
Reply {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local message ID from mail list or mail search.
#[arg(long)]
message: String,
/// UTF-8 text file for the new message body.
#[arg(long)]
body_file: PathBuf,
},
/// Reply to all visible recipients, excluding this account's address.
ReplyAll {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local message ID from mail list or mail search.
#[arg(long)]
message: String,
/// UTF-8 text file for the new message body.
#[arg(long)]
body_file: PathBuf,
},
/// Create a local forward with original bodies and attachments.
Forward {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local message ID from mail list or mail search.
#[arg(long)]
message: String,
/// Recipient email address; repeat --to for multiple recipients.
#[arg(long, required = true)]
to: Vec<String>,
/// UTF-8 text file for the new message body.
#[arg(long)]
body_file: Option<PathBuf>,
},
/// Create and manage local drafts and their attachments.
#[command(arg_required_else_help = true, after_help = crate::help_text::DRAFT)]
Draft {
#[command(subcommand)]
command: DraftCommand,
},
/// List downloaded message summaries and local message IDs.
List {
#[command(flatten)]
args: MailQueryArgs,
},
/// Search the local mail cache; no provider request is made.
Search {
#[command(flatten)]
args: MailQueryArgs,
/// Words to find in the downloaded mail cache; quote multiple words.
#[arg(long)]
query: String,
},
/// List cached Gmail labels or IMAP folders and their local IDs.
Collections {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
},
/// Show cached message metadata, body availability and attachment IDs.
Read {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local message ID from mail list or mail search.
#[arg(long)]
message: String,
},
/// Download the original message and attachments from the provider.
Fetch {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local message ID from mail list or mail search.
#[arg(long)]
message: String,
/// Wait for completion and report the final state; queued work remains inspectable after a timeout.
#[arg(long)]
wait: bool,
},
/// Export the cached original message to a new EML file.
Raw {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local message ID from mail list or mail search.
#[arg(long)]
message: String,
/// New destination file on this machine; existing files are never overwritten.
#[arg(long)]
output: PathBuf,
},
/// Save a cached attachment to a new file; get its ID from mail read.
Attachment {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local message ID from mail list or mail search.
#[arg(long)]
message: String,
/// Local attachment ID from mail read.
#[arg(long)]
attachment: String,
/// New destination file on this machine; existing files are never overwritten.
#[arg(long)]
output: PathBuf,
},
/// Save cached text or HTML to a new file without rendering it.
Body {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local message ID from mail list or mail search.
#[arg(long)]
message: String,
/// Body representation to export: text or inert HTML.
#[arg(long,value_parser=["text","html"])]
kind: String,
/// New destination file on this machine; existing files are never overwritten.
#[arg(long)]
output: PathBuf,
},
}
#[derive(Subcommand)]
pub enum DraftCommand {
/// Add a local file to a draft using its current version.
Attach {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local draft ID from mail draft list or mail draft show.
#[arg(long)]
draft: String,
/// Current object version from show; protects against overwriting newer changes.
#[arg(long,value_parser=clap::value_parser!(u64).range(1..))]
version: u64,
/// Stable local attachment file, at most 64 MiB.
#[arg(long)]
file: PathBuf,
/// Attachment media type, for example application/pdf.
#[arg(long, default_value = "application/octet-stream")]
mime_type: String,
/// Optional MIME character set for a text attachment, for example utf-8.
#[arg(long)]
charset: Option<String>,
/// Optional MIME Content-ID for referencing an inline attachment.
#[arg(long)]
content_id: Option<String>,
/// Mark the attachment for inline display; it remains inert in the CLI.
#[arg(long)]
inline: bool,
},
/// Create or update a local draft from JSON; does not send it.
#[command(after_help = crate::help_text::DRAFT_SAVE)]
Save {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local draft ID from mail draft list or mail draft show.
#[arg(long, requires = "version")]
draft: Option<String>,
/// Current object version from show; protects against overwriting newer changes.
#[arg(long,requires="draft",value_parser=clap::value_parser!(u64).range(1..))]
version: Option<u64>,
/// Draft content JSON containing recipients, subject and text; see the example below.
#[arg(long)]
file: PathBuf,
},
/// Show draft content, attachments and the version needed for editing.
Show {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local draft ID from mail draft list or mail draft show.
#[arg(long)]
draft: String,
},
/// List local drafts and their IDs for one account.
List {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Number of items per page (1-100).
#[arg(long,default_value_t=100,value_parser=clap::value_parser!(u32).range(1..=100))]
page_size: u32,
/// Opaque next_page_token from the previous result; keep the same query and page size.
#[arg(long)]
page_token: Option<String>,
},
/// Delete a local draft using its current version; does not delete remote mail.
Delete {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local draft ID from mail draft list or mail draft show.
#[arg(long)]
draft: String,
/// Current object version from show; protects against overwriting newer changes.
#[arg(long,value_parser=clap::value_parser!(u64).range(1..))]
version: u64,
},
}
#[derive(Subcommand)]
pub enum OperationsCommand {
/// Inspect provider evidence; optionally resume only proven safe work.
Reconcile {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local operation ID from operation list or the command that queued it.
#[arg(long)]
operation: String,
/// A UUID for this action (create once with uuidgen); reuse only for the identical retry.
#[arg(long)]
#[arg(value_parser = parse_request_id)]
request_id: String,
/// Current object version from show; protects against overwriting newer changes.
#[arg(long,value_parser=clap::value_parser!(u64).range(1..))]
version: u64,
/// Permit safe desired-state retries or finishing an already-proven copy.
/// This never authorizes an ambiguous send or COPY replay.
#[arg(long)]
resume_safe: bool,
/// Wait for completion and report the final state; queued work remains inspectable after a timeout.
#[arg(long)]
wait: bool,
},
/// Wait up to 30 seconds for provider acceptance or an inspectable outcome.
Wait {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local operation ID from operation list or the command that queued it.
#[arg(long)]
operation: String,
},
/// Audit a decision about uncertain work. Resend can duplicate delivery.
#[command(after_help = crate::help_text::RESOLVE)]
Resolve {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local operation ID from operation list or the command that queued it.
#[arg(long)]
operation: String,
/// Current object version from show; protects against overwriting newer changes.
#[arg(long,value_parser=clap::value_parser!(u64).range(1..))]
version: u64,
/// Decision JSON: abandon, confirm_applied or resend; see the example below.
#[arg(long)]
file: std::path::PathBuf,
/// Required for resend: a prior submission may already have been delivered.
#[arg(long)]
accept_duplicate_risk: bool,
},
/// List durable operations and their current outcomes for one account.
List {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Number of items per page (1-100).
#[arg(long,default_value_t=100,value_parser=clap::value_parser!(u32).range(1..=100))]
page_size: u32,
/// Opaque next_page_token from the previous result; keep the same query and page size.
#[arg(long)]
page_token: Option<String>,
},
/// Durable attempts and their evidence, newest first.
Attempts {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local operation ID from operation list or the command that queued it.
#[arg(long)]
operation: String,
/// Number of items per page (1-100).
#[arg(long,default_value_t=25,value_parser=clap::value_parser!(u32).range(1..=100))]
page_size: u32,
/// Opaque next_page_token from the previous result; keep the same query and page size.
#[arg(long)]
page_token: Option<String>,
},
/// Show one operation, its evidence and any required recovery decision.
Show {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local operation ID from operation list or the command that queued it.
#[arg(long)]
operation: String,
},
/// Cancel work before provider dispatch; completed or uncertain effects cannot be undone.
Cancel {
/// Local account ID from account list; not an email address or display name.
#[arg(long)]
#[arg(value_name = "ACCOUNT_ID")]
account: String,
/// Local operation ID from operation list or the command that queued it.
#[arg(long)]
operation: String,
},
}
fn parse_request_id(value: &str) -> Result<String, &'static str> {
uuid::Uuid::parse_str(value)
.map(|id| id.to_string())
.map_err(|_| "Request IDs must be UUIDs")
}
+297
View File
@@ -0,0 +1,297 @@
mod secret;
mod upload;
use crate::{args::BackupCommand, output::AppError};
use nuncio_proto::{
client::TokenInjector,
v2::{self, maintenance_client::MaintenanceClient, system_client::SystemClient},
};
use sha2::{Digest, Sha256};
use std::{
io::Write,
path::{Path, PathBuf},
sync::atomic::Ordering,
};
use tonic::{service::interceptor::InterceptedService, transport::Channel};
type Transport = InterceptedService<Channel, TokenInjector>;
const LIMIT: u64 = 1 << 40;
pub async fn run(
command: BackupCommand,
channel: Channel,
injector: TokenInjector,
) -> Result<serde_json::Value, AppError> {
let mut client = MaintenanceClient::with_interceptor(channel.clone(), injector.clone());
match command {
BackupCommand::Create { output } => {
let status = SystemClient::with_interceptor(channel, injector)
.get_status(v2::GetStatusRequest {})
.await
.map_err(crate::rpc_error)?
.into_inner();
let output = destination(&output, &status.protected_paths)?;
let secret = secret::read()?;
create(&mut client, secret, output).await
}
BackupCommand::Inspect { file } => {
let (stream, failed) = upload::prepare(&file, secret::read()?, None)?;
let result = client.inspect_backup(stream).await;
if failed.load(Ordering::Relaxed) {
return Err(file_error());
}
let info = result.map_err(|e| rpc_error(e, None))?.into_inner();
validate_info(&info)?;
crate::mail::json(info)
}
BackupCommand::Restore { file, new_profile } => {
let (stream, failed) =
upload::prepare(&file, secret::read()?, Some(new_profile.clone()))?;
let result = client.restore_backup(stream).await;
if failed.load(Ordering::Relaxed) {
return Err(file_error());
}
let report = result
.map_err(|e| rpc_error(e, Some(&new_profile)))?
.into_inner();
if uuid::Uuid::parse_str(&report.profile_id).is_err()
|| !Path::new(&report.directory).is_absolute()
|| report.schema_version == 0
|| report
.backup
.as_ref()
.is_none_or(|v| validate_info(v).is_err())
{
return Err(invalid_response());
}
crate::mail::json(report)
}
}
}
async fn create(
client: &mut MaintenanceClient<Transport>,
secret: v2::RecoverySecret,
output: PathBuf,
) -> Result<serde_json::Value, AppError> {
use v2::backup_chunk::Content;
let parent = output.parent().ok_or_else(file_error)?;
let mut file = tempfile::NamedTempFile::new_in(parent).map_err(|_| file_error())?;
let mut stream = client
.create_backup(v2::CreateBackupRequest {
secret: Some(secret),
})
.await
.map_err(|e| rpc_error(e, None))?
.into_inner();
let Some(v2::BackupChunk {
content: Some(Content::Info(info)),
}) = stream.message().await.map_err(|e| rpc_error(e, None))?
else {
return Err(invalid_response());
};
validate_info(&info)?;
let mut received = 0_u64;
let mut hash = Sha256::new();
let mut complete = false;
while let Some(frame) = stream.message().await.map_err(|e| rpc_error(e, None))? {
if complete {
return Err(invalid_response());
}
match frame.content {
Some(Content::Data(data)) => {
if data.offset != received
|| data.data.is_empty()
|| data.data.len() > 262144
|| received
.checked_add(data.data.len() as u64)
.is_none_or(|n| n > info.byte_length)
{
return Err(invalid_response());
}
file.write_all(&data.data).map_err(|_| file_error())?;
hash.update(&data.data);
received += data.data.len() as u64;
}
Some(Content::Complete(digest)) => {
if digest.byte_length != received
|| digest.byte_length != info.byte_length
|| digest.sha256 != info.sha256
{
return Err(invalid_response());
}
complete = true;
}
_ => return Err(invalid_response()),
}
}
if !complete || received != info.byte_length || format!("{:x}", hash.finalize()) != info.sha256
{
return Err(invalid_response());
}
file.as_file().sync_all().map_err(|_| file_error())?;
file.persist_noclobber(&output).map_err(|_| file_error())?;
std::fs::File::open(parent).and_then(|f|f.sync_all()).map_err(|_|AppError {code:"backup_output_uncertain",message:"Backup file was completed but directory sync failed; inspect the output before retrying",exit:5,operation:None,sync_run:None,recovery:Some(serde_json::json!({"output":output}))})?;
Ok(serde_json::json!({"output":output,"backup":info}))
}
fn validate_info(info: &v2::BackupInspection) -> Result<(), AppError> {
if info.format_version != 1
|| info.schema_version == 0
|| info.created_at_ms < 0
|| info.byte_length == 0
|| info.byte_length > LIMIT
|| !valid_hash(&info.sha256)
{
return Err(invalid_response());
}
Ok(())
}
fn valid_hash(value: &str) -> bool {
value.len() == 64
&& value
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
fn destination(path: &Path, protected: &[String]) -> Result<PathBuf, AppError> {
crate::output_file::destination(path, protected).map_err(|e| match e {
crate::output_file::OutputPathError::Destination => file_error(),
crate::output_file::OutputPathError::Protection => invalid_response(),
})
}
fn file_error() -> AppError {
AppError{code:"backup_file_io",message:"Backup input must be a stable regular file and output must be a new unprotected writable path",exit:1,operation:None,sync_run:None,recovery:None}
}
fn invalid_response() -> AppError {
AppError {
code: "invalid_backup_response",
message: "Daemon returned an incomplete or invalid backup response",
exit: 1,
operation: None,
sync_run: None,
recovery: None,
}
}
fn rpc_error(status: tonic::Status, new_profile: Option<&str>) -> AppError {
let code = status
.metadata()
.get("nuncio-recovery-code")
.and_then(|v| v.to_str().ok());
if code == Some("cleanup_incomplete") {
if let Some(id) = status
.metadata()
.get("nuncio-restored-profile-id")
.and_then(|v| v.to_str().ok())
.and_then(|v| uuid::Uuid::parse_str(v).ok())
{
return AppError{code:"restore_cleanup_incomplete",message:"Restore failed; remove only the reported new profile's database/API key entries when secure storage is available",exit:1,operation:None,sync_run:None,recovery:Some(serde_json::json!({"profile_id":id.to_string(),"new_profile":new_profile}))};
}
}
if code == Some("activation_uncertain")
|| (new_profile.is_some()
&& matches!(
status.code(),
tonic::Code::Unavailable
| tonic::Code::DeadlineExceeded
| tonic::Code::Cancelled
| tonic::Code::Unknown
))
{
return AppError {
code: "restore_outcome_uncertain",
message: "Restore may have activated; inspect the named target profile before retrying",
exit: 5,
operation: None,
sync_run: None,
recovery: Some(serde_json::json!({"new_profile":new_profile})),
};
}
if status.code() == tonic::Code::AlreadyExists {
return AppError {
code: "restore_target_exists",
message: "Restore target already exists; originals were preserved",
exit: 5,
operation: None,
sync_run: None,
recovery: None,
};
}
crate::rpc_error(status)
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
#[test]
fn restore_transport_loss_reports_target_inspection_without_blind_retry() {
for code in [
tonic::Code::Unknown,
tonic::Code::Cancelled,
tonic::Code::Unavailable,
tonic::Code::DeadlineExceeded,
] {
let error = super::rpc_error(
tonic::Status::new(code, "synthetic lost response"),
Some("recovered"),
);
assert_eq!(error.code, "restore_outcome_uncertain");
assert_eq!(error.exit, 5);
assert_eq!(
error.recovery,
Some(serde_json::json!({"new_profile":"recovered"}))
);
}
let ordinary = super::rpc_error(tonic::Status::unknown("synthetic transport error"), None);
assert_eq!(ordinary.code, "request_outcome_unknown");
assert_eq!(ordinary.exit, 4);
let invalid = super::rpc_error(
tonic::Status::invalid_argument("synthetic invalid input"),
Some("recovered"),
);
assert_eq!(invalid.exit, 2);
}
#[test]
fn output_refuses_existing_files_links_and_future_protected_sidecars() {
let temp = tempfile::tempdir().unwrap();
let original = temp.path().join("store.db");
std::fs::write(&original, b"original").unwrap();
let protected = vec![
original.to_string_lossy().into_owned(),
temp.path()
.join("store.db-wal")
.to_string_lossy()
.into_owned(),
];
assert!(super::destination(&original, &protected).is_err());
assert!(super::destination(&temp.path().join("store.db-wal"), &protected).is_err());
let hard = temp.path().join("hardlink");
std::fs::hard_link(&original, &hard).unwrap();
assert!(super::destination(&hard, &protected).is_err());
#[cfg(unix)]
{
let alias = temp.path().join("alias");
std::os::unix::fs::symlink(temp.path(), &alias).unwrap();
assert!(super::destination(&alias.join("store.db-wal"), &protected).is_err());
let dangling = temp.path().join("dangling");
std::os::unix::fs::symlink(temp.path().join("missing"), &dangling).unwrap();
assert!(super::destination(&dangling, &protected).is_err());
}
assert!(super::destination(&temp.path().join("new.nuncio"), &protected).is_ok());
assert_eq!(std::fs::read(original).unwrap(), b"original");
}
#[test]
fn cleanup_failure_retains_only_a_valid_profile_identifier_for_recovery() {
let mut status = tonic::Status::internal("server text must not be printed");
status.metadata_mut().insert(
"nuncio-recovery-code",
"cleanup_incomplete".parse().unwrap(),
);
let id = uuid::Uuid::new_v4().to_string();
status
.metadata_mut()
.insert("nuncio-restored-profile-id", id.parse().unwrap());
let error = super::rpc_error(status, Some("recovered"));
assert_eq!(error.exit, 1);
assert_eq!(error.code, "restore_cleanup_incomplete");
assert_eq!(error.recovery.unwrap()["profile_id"], id);
assert!(!error.message.contains("server text"));
}
}
@@ -0,0 +1,66 @@
use crate::output::AppError;
use std::io::{IsTerminal, Read};
use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing};
#[derive(serde::Deserialize, Zeroize, ZeroizeOnDrop)]
#[serde(deny_unknown_fields)]
struct Input {
passphrase: String,
}
pub(super) fn read() -> Result<nuncio_proto::v2::RecoverySecret, AppError> {
let stdin = std::io::stdin();
if stdin.is_terminal() {
return Err(AppError{code:"recovery_input",message:"Pipe recovery-passphrase JSON from a secure source; terminal input is refused to prevent secret echo",exit:2,operation:None,sync_run:None,recovery:None});
}
let mut bytes = Zeroizing::new(Vec::new());
stdin
.lock()
.take(32769)
.read_to_end(&mut bytes)
.map_err(|_| AppError::invalid())?;
decode(&bytes)
}
fn decode(bytes: &[u8]) -> Result<nuncio_proto::v2::RecoverySecret, AppError> {
if bytes.len() > 32768 {
return Err(AppError::invalid());
}
let mut value: Input = serde_json::from_slice(bytes).map_err(|_| AppError::invalid())?;
let phrase = &value.passphrase;
let raw = phrase
.strip_prefix("x'")
.or_else(|| phrase.strip_prefix("X'"))
.and_then(|v| v.strip_suffix('\''))
.is_some_and(|v| {
matches!(v.len(), 64 | 96 | 160) && v.bytes().all(|b| b.is_ascii_hexdigit())
});
if phrase.len() > 4096
|| phrase.chars().count() < 12
|| phrase.trim().is_empty()
|| phrase.contains('\0')
|| raw
{
return Err(AppError::invalid());
}
Ok(nuncio_proto::v2::RecoverySecret {
passphrase: std::mem::take(&mut value.passphrase),
})
}
#[cfg(test)]
mod tests {
#[test]
fn secret_input_rejects_unknown_fields_and_raw_key_literals() {
assert!(super::decode(br#"{"passphrase":"synthetic secure recovery phrase"}"#).is_ok());
for invalid in [
br#"{"passphrase":"short"}"#.as_slice(),
br#"{"passphrase":"synthetic secure recovery phrase","token":"unexpected"}"#,
b"not-json",
] {
assert!(super::decode(invalid).is_err());
}
for prefix in ["x", "X"] {
for n in [64, 96, 160] {
let value = serde_json::json!({"passphrase":format!("{prefix}'{}'","1".repeat(n))});
assert!(super::decode(value.to_string().as_bytes()).is_err());
}
}
}
}
@@ -0,0 +1,111 @@
use super::{file_error, AppError, LIMIT};
use nuncio_proto::v2::{self, backup_upload_chunk::Content};
use sha2::{Digest, Sha256};
use std::{
io::{Read, Seek},
path::Path,
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
};
use tokio::io::AsyncReadExt;
pub(super) fn prepare(
path: &Path,
secret: v2::RecoverySecret,
new_profile: Option<String>,
) -> Result<
(
impl futures_util::Stream<Item = v2::BackupUploadChunk> + Send + 'static,
Arc<AtomicBool>,
),
AppError,
> {
let fd = rustix::fs::open(
path,
rustix::fs::OFlags::RDONLY
| rustix::fs::OFlags::NOFOLLOW
| rustix::fs::OFlags::NONBLOCK
| rustix::fs::OFlags::CLOEXEC,
rustix::fs::Mode::empty(),
)
.map_err(|_| file_error())?;
let mut file = std::fs::File::from(fd);
let metadata = file.metadata().map_err(|_| file_error())?;
let length = metadata.len();
if !metadata.is_file() || length == 0 || length > LIMIT {
return Err(file_error());
}
for suffix in ["-wal", "-shm", "-journal"] {
let mut companion = path.as_os_str().to_os_string();
companion.push(suffix);
match std::fs::symlink_metadata(&companion) {
Ok(_) => return Err(file_error()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(_) => return Err(file_error()),
}
}
let mut hash = Sha256::new();
let mut remaining = length;
let mut buffer = vec![0; 262144];
while remaining > 0 {
let n = remaining.min(buffer.len() as u64) as usize;
file.read_exact(&mut buffer[..n])
.map_err(|_| file_error())?;
hash.update(&buffer[..n]);
remaining -= n as u64;
}
if file.read(&mut buffer[..1]).map_err(|_| file_error())? != 0 {
return Err(file_error());
}
file.rewind().map_err(|_| file_error())?;
let header = v2::BackupUploadHeader {
secret: Some(secret),
byte_length: length,
sha256: format!("{:x}", hash.finalize()),
new_profile,
};
let failed = Arc::new(AtomicBool::new(false));
let stream = futures_util::stream::unfold(
(
Some(header),
tokio::fs::File::from_std(file),
0_u64,
failed.clone(),
),
move |(header, mut file, offset, failed)| async move {
if failed.load(Ordering::Relaxed) {
return None;
}
if let Some(header) = header {
return Some((
v2::BackupUploadChunk {
content: Some(Content::Header(header)),
},
(None, file, offset, failed),
));
}
let n = (length - offset).min(262144) as usize;
let mut data = vec![0; n.max(1)];
if n == 0 {
if matches!(file.read(&mut data).await, Ok(0)) {
return None;
}
} else if file.read_exact(&mut data).await.is_ok() {
return Some((
v2::BackupUploadChunk {
content: Some(Content::Data(v2::BackupData { offset, data })),
},
(None, file, offset + n as u64, failed),
));
}
failed.store(true, Ordering::Relaxed);
Some((
v2::BackupUploadChunk { content: None },
(None, file, offset, failed),
))
},
);
Ok((stream, failed))
}
+172
View File
@@ -0,0 +1,172 @@
mod action;
mod free_busy;
use crate::{
args::CalendarCommand,
mail::{json, wait_run},
output::AppError,
rpc_error,
};
use nuncio_proto::{
client::TokenInjector,
v2::{self, calendar_client::CalendarClient, system_client::SystemClient},
};
use serde_json::{json as value, Value};
use tonic::transport::Channel;
pub async fn run(
command: CalendarCommand,
channel: Channel,
token: TokenInjector,
) -> Result<Value, AppError> {
let mut client = CalendarClient::with_interceptor(channel.clone(), token.clone())
.max_decoding_message_size(16 * 1024 * 1024);
match command {
CalendarCommand::FreeBusy { account, file } => {
let mut request = free_busy::read(&file).map_err(|error| error.input_context(
"Invalid availability query: --file needs schema_version 1, RFC3339 from/to, time_zone and provider_calendar_ids (at most 128 KiB). See 'calendar free-busy --help'."
))?;
request.account_id = account;
json(
client
.query_free_busy(request)
.await
.map_err(rpc_error)?
.into_inner(),
)
}
CalendarCommand::Change {
account,
calendar,
request_id,
file,
wait,
} => {
let mut request = action::read(&file).map_err(|error| error.input_context(
"Invalid calendar action: --file needs schema_version 1, action, scope, notifications and matching event fields (at most 1 MiB). See 'calendar change --help'."
))?;
request.account_id = account;
request.calendar_id = calendar;
request.request_id = request_id;
let op = client
.change_event(request)
.await
.map_err(rpc_error)?
.into_inner();
if wait {
crate::operations::wait_for(op, channel, token).await
} else {
json(op)
}
}
CalendarCommand::List {
account,
page_size,
page_token,
} => json(
client
.list_calendars(v2::ListCalendarsRequest {
account_id: account,
page_size,
page_token,
})
.await
.map_err(rpc_error)?
.into_inner(),
),
CalendarCommand::Get {
account,
calendar,
event: id,
} => {
let response = client
.get_event(v2::CalendarEventRequest {
account_id: account,
calendar_id: calendar,
event_id: id,
})
.await
.map_err(rpc_error)?
.into_inner();
Ok(
value!({"revision":response.revision,"event":event(response.event.ok_or_else(invalid_response)?)?,"coverage":response.coverage}),
)
}
CalendarCommand::Agenda {
account,
from,
to,
page_size,
page_token,
} => {
let response = client
.list_agenda(v2::ListAgendaRequest {
account_id: account,
window: Some(v2::AgendaWindow { from, to }),
page_size,
page_token,
})
.await
.map_err(rpc_error)?
.into_inner();
let events = response
.items
.into_iter()
.map(event)
.collect::<Result<Vec<_>, _>>()?;
Ok(
value!({"revision":response.revision,"items":events,"coverage":response.coverage,"next_page_token":response.next_page_token}),
)
}
CalendarCommand::Refresh {
account,
from,
to,
wait,
} => {
let window = match (from, to) {
(Some(from), Some(to)) => Some(v2::AgendaWindow { from, to }),
(None, None) => None,
_ => return Err(AppError::invalid()),
};
let run = client
.refresh_agenda(v2::RefreshAgendaRequest {
account_id: account,
window,
})
.await
.map_err(rpc_error)?
.into_inner();
wait_run(
&mut SystemClient::with_interceptor(channel, token),
run,
wait,
)
.await
}
}
}
fn event(e: v2::CalendarEvent) -> Result<Value, AppError> {
Ok(
value!({"id":e.id,"account_id":e.account_id,"calendar_id":e.calendar_id,"provider_id":e.provider_id,"summary":e.summary,"etag":e.etag,"status":e.status,
"start":e.start.map(time).transpose()?,"end":e.end.map(time).transpose()?,"original_start":e.original_start.map(time).transpose()?,"recurring_provider_id":e.recurring_provider_id,"recurring_event_id":e.recurring_event_id,"provider_json":e.provider_json}),
)
}
fn time(t: v2::EventTime) -> Result<Value, AppError> {
Ok(match t.value.ok_or_else(invalid_response)? {
v2::event_time::Value::Date(date) => value!({"date":date}),
v2::event_time::Value::DateTime(t) => {
value!({"date_time":{"rfc3339":t.rfc3339,"time_zone":t.time_zone}})
}
})
}
fn invalid_response() -> AppError {
AppError {
code: "invalid_response",
message: "Engine returned an invalid Calendar response",
sync_run: None,
operation: None,
recovery: None,
exit: 1,
}
}
@@ -0,0 +1,327 @@
#[cfg(test)]
mod tests {
#[test]
#[allow(clippy::unwrap_used)]
fn calendar_action_files_are_versioned_strict_and_preserve_explicit_policy() {
let temp = tempfile::tempdir().unwrap();
let path = temp.path().join("action.json");
for invalid in [
r#"{"schema_version":1,"action":"create","scope":"single","notifications":"none","event":{"summary":null}}"#,
r#"{"action":"delete","event_id":"event","expected_etag":"\"v1\"","scope":"single","notifications":"none"}"#,
r#"{"schema_version":2,"action":"delete","event_id":"event","expected_etag":"\"v1\"","scope":"single","notifications":"none"}"#,
r#"{"schema_version":1,"action":"delete","event_id":"event","expected_etag":"\"v1\"","scope":"following","notifications":"none"}"#,
r#"{"schema_version":1,"action":"delete","event_id":"event","expected_etag":"\"v1\"","scope":"single"}"#,
r#"{"schema_version":1,"action":"create","scope":"single","notifications":"none","event":{"organizer":"cannot transfer"}}"#,
r#"{"schema_version":1,"action":"create","scope":"single","notifications":"none","event":{"start":{"date":"2026-10-01","extra":true}}}"#,
r#"{"schema_version":1,"action":"respond","event_id":"event","expected_etag":"\"v1\"","scope":"single","notifications":"all","response":"maybe"}"#,
r#"{"schema_version":1,"schema_version":2,"action":"delete","event_id":"event","expected_etag":"\"v1\"","scope":"single","notifications":"none"}"#,
] {
std::fs::write(&path, invalid).unwrap();
assert!(super::read(&path).is_err(), "accepted {invalid}");
}
std::fs::write(&path,r#"{"schema_version":1,"action":"respond","event_id":"event","expected_etag":"\"v1\"","scope":"single","notifications":"external_only","response":"declined","comment":"Unavailable"}"#).unwrap();
let q = super::read(&path).map_err(|e| e.code).unwrap();
assert_eq!(q.scope, nuncio_proto::v2::CalendarEditScope::Single as i32);
assert_eq!(
q.notification_policy,
nuncio_proto::v2::CalendarNotificationPolicy::ExternalOnly as i32
);
assert!(
matches!(q.action,Some(nuncio_proto::v2::change_event_request::Action::Respond(v)) if v.response==nuncio_proto::v2::CalendarResponseStatus::Declined as i32 && v.comment.as_deref()==Some("Unavailable"))
);
std::fs::write(&path, vec![b' '; 1024 * 1024 + 1]).unwrap();
assert!(super::read(&path).is_err());
}
}
use crate::output::AppError;
use nuncio_proto::v2::{self, change_event_request::Action};
use serde::Deserialize;
use std::{io::Read, path::Path};
#[derive(Deserialize)]
#[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)]
enum ActionFile {
Create {
schema_version: u32,
scope: Scope,
notifications: Policy,
event: Fields,
},
Update {
schema_version: u32,
scope: Scope,
notifications: Policy,
event_id: String,
expected_etag: String,
#[serde(default)]
patch: Fields,
#[serde(default)]
clear_fields: Vec<String>,
},
Delete {
schema_version: u32,
scope: Scope,
notifications: Policy,
event_id: String,
expected_etag: String,
},
Respond {
schema_version: u32,
scope: Scope,
notifications: Policy,
event_id: String,
expected_etag: String,
response: Response,
#[serde(default, deserialize_with = "present")]
comment: Option<String>,
},
}
#[derive(Deserialize)]
#[serde(rename_all = "snake_case")]
enum Scope {
Single,
Series,
}
#[derive(Deserialize)]
#[serde(rename_all = "snake_case")]
enum Policy {
None,
All,
ExternalOnly,
}
#[derive(Deserialize)]
#[serde(rename_all = "snake_case")]
enum Response {
NeedsAction,
Accepted,
Declined,
Tentative,
}
#[derive(Default, Deserialize)]
#[serde(deny_unknown_fields)]
struct Fields {
#[serde(default, deserialize_with = "present")]
summary: Option<String>,
#[serde(default, deserialize_with = "present")]
description: Option<String>,
#[serde(default, deserialize_with = "present")]
location: Option<String>,
#[serde(default, deserialize_with = "present")]
start: Option<Time>,
#[serde(default, deserialize_with = "present")]
end: Option<Time>,
#[serde(default, deserialize_with = "present")]
recurrence: Option<Vec<String>>,
#[serde(default, deserialize_with = "present")]
attendees: Option<Vec<Attendee>>,
#[serde(default, deserialize_with = "present")]
reminders: Option<Reminders>,
#[serde(default, deserialize_with = "present")]
guests_can_modify: Option<bool>,
#[serde(default, deserialize_with = "present")]
guests_can_invite_others: Option<bool>,
#[serde(default, deserialize_with = "present")]
guests_can_see_other_guests: Option<bool>,
#[serde(default, deserialize_with = "present")]
transparency: Option<String>,
#[serde(default, deserialize_with = "present")]
visibility: Option<String>,
#[serde(default, deserialize_with = "present")]
color_id: Option<String>,
}
#[derive(Deserialize)]
#[serde(untagged, deny_unknown_fields)]
enum Time {
Date { date: String },
Timed { date_time: ZonedTime },
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct ZonedTime {
rfc3339: String,
#[serde(default)]
time_zone: String,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Attendee {
email: String,
#[serde(default, deserialize_with = "present")]
display_name: Option<String>,
#[serde(default, deserialize_with = "present")]
optional: Option<bool>,
#[serde(default, deserialize_with = "present")]
resource: Option<bool>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Reminders {
use_default: bool,
#[serde(default)]
overrides: Vec<Reminder>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Reminder {
method: String,
minutes: u32,
}
fn present<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>
where
D: serde::Deserializer<'de>,
T: Deserialize<'de>,
{
T::deserialize(deserializer).map(Some)
}
pub(super) fn read(path: &Path) -> Result<v2::ChangeEventRequest, AppError> {
let mut bytes = Vec::new();
crate::drafts::regular_file(path)?
.take(1024 * 1024 + 1)
.read_to_end(&mut bytes)
.map_err(|_| AppError::invalid())?;
if bytes.len() > 1024 * 1024 {
return Err(AppError::invalid());
}
let input: ActionFile = serde_json::from_slice(&bytes).map_err(|_| AppError::invalid())?;
let (version, scope, policy, action) = match input {
ActionFile::Create {
schema_version,
scope,
notifications,
event,
} => (
schema_version,
scope,
notifications,
Action::Create(v2::CalendarCreate {
event: Some(fields(event)),
}),
),
ActionFile::Update {
schema_version,
scope,
notifications,
event_id,
expected_etag,
patch,
clear_fields,
} => (
schema_version,
scope,
notifications,
Action::Update(v2::CalendarUpdate {
event_id,
expected_etag,
fields: Some(fields(patch)),
clear_fields,
}),
),
ActionFile::Delete {
schema_version,
scope,
notifications,
event_id,
expected_etag,
} => (
schema_version,
scope,
notifications,
Action::Delete(v2::CalendarDelete {
event_id,
expected_etag,
}),
),
ActionFile::Respond {
schema_version,
scope,
notifications,
event_id,
expected_etag,
response,
comment,
} => (
schema_version,
scope,
notifications,
Action::Respond(v2::CalendarRespond {
event_id,
expected_etag,
response: match response {
Response::NeedsAction => v2::CalendarResponseStatus::NeedsAction,
Response::Accepted => v2::CalendarResponseStatus::Accepted,
Response::Declined => v2::CalendarResponseStatus::Declined,
Response::Tentative => v2::CalendarResponseStatus::Tentative,
} as i32,
comment,
}),
),
};
if version != 1 {
return Err(AppError::invalid());
}
Ok(v2::ChangeEventRequest {
scope: match scope {
Scope::Single => v2::CalendarEditScope::Single,
Scope::Series => v2::CalendarEditScope::Series,
} as i32,
notification_policy: match policy {
Policy::None => v2::CalendarNotificationPolicy::None,
Policy::All => v2::CalendarNotificationPolicy::All,
Policy::ExternalOnly => v2::CalendarNotificationPolicy::ExternalOnly,
} as i32,
action: Some(action),
..Default::default()
})
}
fn fields(f: Fields) -> v2::CalendarEventFields {
v2::CalendarEventFields {
summary: f.summary,
description: f.description,
location: f.location,
start: f.start.map(time),
end: f.end.map(time),
recurrence: f.recurrence.map(|rules| v2::CalendarRecurrence { rules }),
attendees: f.attendees.map(|a| v2::CalendarAttendees {
items: a
.into_iter()
.map(|a| v2::CalendarAttendee {
email: a.email,
display_name: a.display_name,
optional: a.optional,
resource: a.resource,
})
.collect(),
}),
reminders: f.reminders.map(|r| v2::CalendarReminders {
use_default: r.use_default,
overrides: r
.overrides
.into_iter()
.map(|r| v2::CalendarReminder {
method: r.method,
minutes: r.minutes,
})
.collect(),
}),
guests_can_modify: f.guests_can_modify,
guests_can_invite_others: f.guests_can_invite_others,
guests_can_see_other_guests: f.guests_can_see_other_guests,
transparency: f.transparency,
visibility: f.visibility,
color_id: f.color_id,
}
}
fn time(t: Time) -> v2::EventTime {
v2::EventTime {
value: Some(match t {
Time::Date { date } => v2::event_time::Value::Date(date),
Time::Timed { date_time: t } => v2::event_time::Value::DateTime(v2::ZonedDateTime {
rfc3339: t.rfc3339,
time_zone: t.time_zone,
}),
}),
}
}
@@ -0,0 +1,39 @@
use crate::output::AppError;
use nuncio_proto::v2::FreeBusyRequest;
use serde::Deserialize;
use std::{io::Read, path::Path};
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Query {
schema_version: u32,
from: String,
to: String,
time_zone: String,
provider_calendar_ids: Vec<String>,
}
pub(super) fn read(path: &Path) -> Result<FreeBusyRequest, AppError> {
let file = std::fs::File::open(path).map_err(|_| AppError::invalid())?;
let meta = file.metadata().map_err(|_| AppError::invalid())?;
const MAX: u64 = 128 * 1024;
if !meta.is_file() || meta.len() > MAX {
return Err(AppError::invalid());
}
let mut bytes = Vec::new();
file.take(MAX + 1)
.read_to_end(&mut bytes)
.map_err(|_| AppError::invalid())?;
if bytes.len() > MAX as usize {
return Err(AppError::invalid());
}
let q: Query = serde_json::from_slice(&bytes).map_err(|_| AppError::invalid())?;
if q.schema_version != 1 {
return Err(AppError::invalid());
}
Ok(FreeBusyRequest {
account_id: String::new(),
from: q.from,
to: q.to,
time_zone: q.time_zone,
provider_calendar_ids: q.provider_calendar_ids,
})
}
+63
View File
@@ -0,0 +1,63 @@
use crate::{
args::{Args, Command, SystemCommand},
output::AppError,
rpc_error,
};
use nuncio_proto::v2::{system_client::SystemClient, WatchChangesRequest};
use std::io::Write;
pub async fn run(args: Args) -> Result<(), AppError> {
let (after, limit) = match &args.command {
Command::System {
command: SystemCommand::Watch { after, limit },
} => (*after, *limit),
_ => return Err(AppError::invalid()),
};
let (channel, token) = crate::connect(&args).await?;
let mut stream = SystemClient::with_interceptor(channel, token)
.watch_changes(WatchChangesRequest {
after_revision: after,
})
.await
.map_err(rpc_error)?
.into_inner();
let mut previous = after;
let mut emitted = 0_u64;
loop {
let change = tokio::select! {
result=stream.message()=>result.map_err(rpc_error)?.ok_or_else(AppError::unavailable)?,
result=tokio::signal::ctrl_c()=>{result.map_err(|_|AppError::unavailable())?;return Ok(());}
};
if previous.checked_add(1) != Some(change.revision) {
return Err(AppError {
code: "resnapshot_required",
message: "Change revisions are discontinuous; take a new snapshot",
sync_run: None,
operation: None,
recovery: None,
exit: 5,
});
}
previous = change.revision;
let value = serde_json::json!({"schema_version":1,"result":change});
let line = crate::output::render(&value, true).map_err(|_| output_error())?;
let mut stdout = std::io::stdout().lock();
writeln!(stdout, "{line}")
.and_then(|()| stdout.flush())
.map_err(|_| output_error())?;
emitted += 1;
if limit.is_some_and(|n| emitted >= u64::from(n)) {
return Ok(());
}
}
}
fn output_error() -> AppError {
AppError {
code: "output_failed",
message: "Could not write change output",
sync_run: None,
operation: None,
recovery: None,
exit: 1,
}
}
@@ -0,0 +1,68 @@
use crate::{args::Args, output::AppError};
use std::{fs::File, io::Read, path::PathBuf};
use zeroize::Zeroizing;
pub fn load(args: &Args) -> Result<Zeroizing<String>, AppError> {
if args.profile.is_empty()
|| args.profile.len() > 64
|| !args
.profile
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
{
return Err(AppError::invalid());
}
let directory = match &args.data_dir {
Some(path) => path.clone(),
None => PathBuf::from(std::env::var_os("HOME").ok_or_else(AppError::auth)?)
.join(".nuncio-rebuild")
.join(&args.profile),
};
let mut bytes = Vec::new();
File::open(directory.join("profile.json"))
.map_err(|_| AppError::auth())?
.take(4097)
.read_to_end(&mut bytes)
.map_err(|_| AppError::auth())?;
if bytes.len() > 4096 {
return Err(AppError::auth());
}
let manifest: serde_json::Value =
serde_json::from_slice(&bytes).map_err(|_| AppError::auth())?;
if manifest["version"] != 1 {
return Err(AppError::auth());
}
let id = uuid::Uuid::parse_str(manifest["id"].as_str().ok_or_else(AppError::auth)?)
.map_err(|_| AppError::auth())?;
let secret = read_secret(args, &format!("{id}/profile/api"))?;
if secret.len() != 64 || !secret.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return Err(AppError::auth());
}
Ok(Zeroizing::new(format!("Bearer {}", secret.as_str())))
}
fn read_secret(_args: &Args, name: &str) -> Result<Zeroizing<String>, AppError> {
#[cfg(feature = "test-harness")]
if let Some(path) = &_args.test_secrets_file {
let mut bytes = Zeroizing::new(Vec::new());
File::open(path)
.map_err(|_| AppError::auth())?
.take(1_048_577)
.read_to_end(&mut bytes)
.map_err(|_| AppError::auth())?;
if bytes.len() > 1_048_576 {
return Err(AppError::auth());
}
let mut values: std::collections::BTreeMap<String, String> =
serde_json::from_slice(&bytes).map_err(|_| AppError::auth())?;
return values
.remove(name)
.map(Zeroizing::new)
.ok_or_else(AppError::auth);
}
let entry = keyring::Entry::new("mx.nuncio.rebuild", name).map_err(|_| AppError::auth())?;
entry
.get_password()
.map(Zeroizing::new)
.map_err(|_| AppError::auth())
}
@@ -0,0 +1,99 @@
use crate::{
drafts::{file_error, regular_file},
output::AppError,
};
use nuncio_proto::v2::{
draft_upload_chunk::Part, DraftUploadChunk, DraftUploadData, DraftUploadHeader,
};
use sha2::{Digest, Sha256};
use std::{
io::{Read, Seek},
path::Path,
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
};
use tokio::io::AsyncReadExt;
const CHUNK: usize = 256 * 1024;
const LIMIT: u64 = 64 * 1024 * 1024;
pub(crate) fn prepare(
path: &Path,
mut header: DraftUploadHeader,
) -> Result<
(
impl futures_util::Stream<Item = DraftUploadChunk> + Send + 'static,
Arc<AtomicBool>,
),
AppError,
> {
let mut file = regular_file(path)?;
let length = file.metadata().map_err(|_| file_error())?.len();
if length > LIMIT {
return Err(AppError::invalid());
}
let mut remaining = length;
let mut digest = Sha256::new();
let mut buffer = vec![0; CHUNK];
while remaining > 0 {
let count = remaining.min(CHUNK as u64) as usize;
file.read_exact(&mut buffer[..count])
.map_err(|_| file_error())?;
digest.update(&buffer[..count]);
remaining -= count as u64;
}
if file.read(&mut buffer[..1]).map_err(|_| file_error())? != 0 {
return Err(file_error());
}
file.rewind().map_err(|_| file_error())?;
header.byte_length = length;
header.sha256 = hex::encode(digest.finalize());
let failed = Arc::new(AtomicBool::new(false));
let state = (
Some(header),
tokio::fs::File::from_std(file),
0_u64,
failed.clone(),
);
// Pull-based reads apply HTTP/2 backpressure without an unowned producer.
// A changed or unreadable file sends an invalid frame, so EOF cannot commit
// a successfully read prefix as a complete attachment.
let stream = futures_util::stream::unfold(
state,
move |(header, mut file, offset, failed)| async move {
if failed.load(Ordering::Relaxed) {
return None;
}
if let Some(header) = header {
return Some((
DraftUploadChunk {
part: Some(Part::Header(header)),
},
(None, file, offset, failed),
));
}
let count = (length - offset).min(CHUNK as u64) as usize;
let mut data = vec![0; count.max(1)];
if count == 0 {
if matches!(file.read(&mut data).await, Ok(0)) {
return None;
}
} else if file.read_exact(&mut data[..count]).await.is_ok() {
return Some((
DraftUploadChunk {
part: Some(Part::Data(DraftUploadData { offset, data })),
},
(None, file, offset + count as u64, failed),
));
}
failed.store(true, Ordering::Relaxed);
Some((
DraftUploadChunk { part: None },
(None, file, offset, failed),
))
},
);
Ok((stream, failed))
}
+183
View File
@@ -0,0 +1,183 @@
use crate::{args::DraftCommand, mail::json, output::AppError, rpc_error};
use nuncio_proto::{
client::TokenInjector,
v2::{self, mail_client::MailClient},
};
use std::io::Read;
use tonic::{service::interceptor::InterceptedService, transport::Channel};
pub async fn run(
client: &mut MailClient<InterceptedService<Channel, TokenInjector>>,
command: DraftCommand,
) -> Result<serde_json::Value, AppError> {
match command {
DraftCommand::Attach {
account,
draft,
version,
file,
mime_type,
charset,
content_id,
inline,
} => {
let (stream, failed) = crate::draft_upload::prepare(
&file,
v2::DraftUploadHeader {
account_id: account,
draft_id: draft,
expected_version: version,
filename: file
.file_name()
.and_then(|name| name.to_str())
.map(str::to_owned),
mime_type,
parameters: charset.map(|s| ("charset".into(), s)).into_iter().collect(),
content_id,
disposition: if inline { "inline" } else { "attachment" }.into(),
byte_length: 0,
sha256: String::new(),
},
)?;
let result = client.upload_draft_attachment(stream).await;
if failed.load(std::sync::atomic::Ordering::Relaxed) {
return Err(file_error());
}
json(result.map_err(rpc_error)?.into_inner())
}
DraftCommand::Save {
account,
draft,
version,
file,
} => {
let mut bytes = Vec::new();
regular_file(&file)?
.take(8 * 1024 * 1024 + 1)
.read_to_end(&mut bytes)
.map_err(|_| file_error())?;
if bytes.len() > 8 * 1024 * 1024 {
return Err(AppError::invalid().input_context(
"Draft JSON exceeds 8 MiB; attach large files with 'mail draft attach'.",
));
}
let content: v2::DraftContent =
serde_json::from_slice(&bytes).map_err(|_| AppError::invalid().input_context(
"Invalid draft JSON in --file; use recipient arrays, subject and text as shown in 'mail draft save --help'."
))?;
json(
client
.save_draft(v2::SaveDraftRequest {
account_id: account,
draft_id: draft,
expected_version: version,
content: Some(content),
})
.await
.map_err(rpc_error)?
.into_inner(),
)
}
DraftCommand::Show { account, draft } => json(
client
.get_draft(v2::DraftRequest {
account_id: account,
draft_id: draft,
})
.await
.map_err(rpc_error)?
.into_inner(),
),
DraftCommand::List {
account,
page_size,
page_token,
} => json(
client
.list_drafts(v2::ListDraftsRequest {
account_id: account,
page_size,
page_token,
})
.await
.map_err(rpc_error)?
.into_inner(),
),
DraftCommand::Delete {
account,
draft,
version,
} => json(
client
.delete_draft(v2::DeleteDraftRequest {
account_id: account,
draft_id: draft,
expected_version: version,
})
.await
.map_err(rpc_error)?
.into_inner(),
),
}
}
pub(crate) async fn prepare(
client: &mut MailClient<InterceptedService<Channel, TokenInjector>>,
account: String,
message: String,
kind: &str,
body_file: Option<std::path::PathBuf>,
to: Vec<String>,
) -> Result<serde_json::Value, AppError> {
let body = body_file
.map(|path| {
let mut bytes = Vec::new();
regular_file(&path)?
.take(1024 * 1024 + 1)
.read_to_end(&mut bytes)
.map_err(|_| file_error())?;
if bytes.len() > 1024 * 1024 {
return Err(AppError::invalid());
}
String::from_utf8(bytes).map_err(|_| AppError::invalid())
})
.transpose()?;
json(
client
.prepare_draft(v2::PrepareDraftRequest {
account_id: account,
message_id: message,
kind: kind.into(),
body,
to: to
.into_iter()
.map(|address| v2::DraftRecipient {
address,
name: None,
})
.collect(),
})
.await
.map_err(rpc_error)?
.into_inner(),
)
}
pub(crate) fn regular_file(path: &std::path::Path) -> Result<std::fs::File, AppError> {
if !std::fs::metadata(path).map_err(|_| file_error())?.is_file() {
return Err(file_error());
}
let file = std::fs::File::open(path).map_err(|_| file_error())?;
if !file.metadata().map_err(|_| file_error())?.is_file() {
return Err(file_error());
}
Ok(file)
}
pub(crate) fn file_error() -> AppError {
AppError {
code: "file_io",
message: "Could not read the input file; use an existing, readable regular file that stays unchanged during the command",
exit: 1,
sync_run: None,
operation: None,
recovery: None,
}
}
+175
View File
@@ -0,0 +1,175 @@
pub const ROOT: &str = r#"Getting started (keep nunciod running in another terminal):
nuncio-cli --profile laptop-qa system status
nuncio-cli --profile laptop-qa account add
nuncio-cli --profile laptop-qa account list
nuncio-cli --profile laptop-qa mail list --account ACCOUNT_ID
Replace ACCOUNT_ID with the local id shown by account list. Use the same
--profile as the daemon on every command. Results are readable text by default;
add --json for the stable machine format. Run COMMAND --help for details."#;
pub const MAIL: &str = r#"Read mail:
nuncio-cli account list
nuncio-cli sync --account ACCOUNT_ID --wait
nuncio-cli mail list --account ACCOUNT_ID
nuncio-cli mail read --account ACCOUNT_ID --message MESSAGE_ID
nuncio-cli mail search --account ACCOUNT_ID --query 'meeting notes'
ACCOUNT_ID comes from account list; MESSAGE_ID comes from mail list/search.
Use the same --profile as your daemon. List, read, search and exports use the
local cache; sync and fetch download from the provider. An empty list can mean
the account has not synchronized yet. Draft commands save locally; send is an
explicit separate action. Run 'nuncio-cli mail COMMAND --help' for its options."#;
pub const SYNC: &str = r#"Example:
nuncio-cli --profile laptop-qa sync --account ACCOUNT_ID --wait
Find ACCOUNT_ID with account list. This synchronizes mail; use calendar refresh
for calendars. Without --wait, inspect the returned run ID with system sync-status.
--wait waits up to five minutes. --full reconciles remote mail without deleting
local drafts or operation history."#;
pub const CALENDAR: &str = r#"Examples:
nuncio-cli calendar refresh --account ACCOUNT_ID --wait
nuncio-cli calendar list --account ACCOUNT_ID
nuncio-cli calendar agenda --account ACCOUNT_ID --from 2026-10-01 --to 2026-10-08
nuncio-cli calendar get --account ACCOUNT_ID --calendar CALENDAR_ID --event EVENT_ID
Account IDs come from account list, calendar IDs from calendar list, and event
IDs from agenda. Use local IDs, not provider IDs. Date windows are YYYY-MM-DD
with an exclusive end date. List/get/agenda read the cache and report coverage;
refresh/free-busy contact Google. Calendar writes require an explicit action,
edit scope and notification policy. Synology calendar is not supported."#;
pub const SYSTEM: &str = r#"Examples:
nuncio-cli system status
nuncio-cli system sync-status --account ACCOUNT_ID --run SYNC_RUN_ID
nuncio-cli system cancel-sync --account ACCOUNT_ID --run SYNC_RUN_ID
nuncio-cli system watch --after REVISION
nuncio-cli system shutdown
Use the same --profile and --endpoint as your daemon. Run IDs come from sync,
fetch, refresh or repair. Watch always emits JSONL; obtain an initial revision
from status and keep the last observed revision to resume after disconnecting."#;
pub const DRAFT: &str = r#"Examples:
nuncio-cli mail draft save --account ACCOUNT_ID --file draft.json
nuncio-cli mail draft list --account ACCOUNT_ID
nuncio-cli mail draft show --account ACCOUNT_ID --draft DRAFT_ID
nuncio-cli mail draft attach --account ACCOUNT_ID --draft DRAFT_ID --version VERSION --file report.pdf
These commands change local drafts only. Copy the latest version from draft show
before updating, attaching or deleting. Send separately with mail send. Use
'mail draft save --help' for the JSON format."#;
pub const OPERATIONS: &str = r#"Examples:
nuncio-cli operation list --account ACCOUNT_ID
nuncio-cli operation show --account ACCOUNT_ID --operation OPERATION_ID
nuncio-cli operation attempts --account ACCOUNT_ID --operation OPERATION_ID
nuncio-cli operation wait --account ACCOUNT_ID --operation OPERATION_ID
Use IDs returned by write commands or operation list. Queued is not delivered;
inspect the state and provider receipts. A timeout does not cancel the operation.
An uncertain send may already have been delivered: inspect/reconcile its evidence
before making an explicit recovery decision. Use show for the current version."#;
pub const REPAIR: &str = r#"Examples:
nuncio-cli repair --account ACCOUNT_ID --scope mail --dry-run
nuncio-cli repair --account ACCOUNT_ID --scope mail --wait
Preview first. Repair rebuilds a provider cache and retains drafts and operation
history; it does not erase the account. Calendar date bounds require
--scope calendar and paired --from/--to dates. The old complete cache remains
available if a rebuild fails. This is not an immediate cache eviction command."#;
pub const BACKUP: &str = r#"Examples (read the passphrase from an owner-only file or secure pipe):
nuncio-cli backup create --output backup.nuncio < /private/path/recovery.json
nuncio-cli backup inspect --file backup.nuncio < /private/path/recovery.json
nuncio-cli backup restore --file backup.nuncio --new-profile recovered < /private/path/recovery.json
Recovery input is a JSON object with one field: passphrase. Use at least twelve
characters. Keep it private and out of shell history, chat and Git. The daemon
must be running. Restore creates a new sibling profile, preserves the source,
and holds pending remote writes for explicit recovery. It never replaces the
current profile. Full procedure: the package's docs/RECOVERY.md."#;
pub const MAIL_CHANGE: &str = r#"Example action.json:
{"schema_version":1,"action":"read","read":true}
Queue it:
nuncio-cli mail change --account ACCOUNT_ID --message MESSAGE_ID --request-id REQUEST_ID --file action.json --wait
Other actions: star (starred boolean), archive (no extra fields), trash (trashed
boolean; false restores), label (collection_id and present boolean), move/copy
(destination_collection_id). IDs come from mail collections. Check mail
capabilities for provider support. Use a UUID request ID (create once with uuidgen) for each new intent;
reuse it only with the identical account/action/payload after a failed response."#;
pub const DRAFT_SAVE: &str = r#"Example draft.json:
{"to":[{"address":"recipient@example.test"}],"subject":"Hello","text":"A short message."}
Create a local draft:
nuncio-cli mail draft save --account ACCOUNT_ID --file draft.json
For an update, add --draft DRAFT_ID --version VERSION from draft show. Recipient
fields to/cc/bcc are arrays of address objects. Optional html, in_reply_to and
references fields preserve message content/threading. Attach files separately
with draft attach. Saving does not send; mail send requires an explicit account,
draft and UUID request ID (create once with uuidgen)."#;
pub const CALENDAR_CHANGE: &str = r#"Example create.json (a one-day event; end date is exclusive):
{"schema_version":1,"action":"create","scope":"single","notifications":"none","event":{"summary":"Test event","start":{"date":"2026-10-01"},"end":{"date":"2026-10-02"}}}
Queue it:
nuncio-cli calendar change --account ACCOUNT_ID --calendar CALENDAR_ID --request-id REQUEST_ID --file create.json --wait
Actions: create/update/delete/respond. Scope: single or series. Notifications:
none/all/external_only. Update/delete/respond require event_id and expected_etag
from calendar get. REQUEST_ID must be a UUID. Update uses patch and optional clear_fields. Respond uses
response: accepted/declined/tentative/needs_action. Timed values use date_time:
{"rfc3339":"2026-10-01T09:00:00-05:00","time_zone":"America/Chicago"}.
Details and supported fields: the package's docs/RUNNING.md."#;
pub const FREE_BUSY: &str = r#"Example query.json:
{"schema_version":1,"from":"2026-10-01T00:00:00Z","to":"2026-10-02T00:00:00Z","time_zone":"UTC","provider_calendar_ids":["primary"]}
Run: nuncio-cli calendar free-busy --account ACCOUNT_ID --file query.json
This is an online Google query. Unlike other calendar commands, the file uses
provider calendar IDs (from calendar list), not Nuncio's local calendar IDs.
Check each calendar's returned coverage and errors before treating it as free."#;
pub const RESOLVE: &str = r#"Example decision.json:
{"decision":"abandon","reason":"Reviewed independently; do not attempt this again"}
Run: nuncio-cli operation resolve --account ACCOUNT_ID --operation OPERATION_ID --version VERSION --file decision.json
Read show/attempts first. Abandon prevents further attempts but cannot undo a
remote effect. confirm_applied requires independently verified evidence;
resend requires a new UUID request_id, reason and --accept-duplicate-risk. A resend
can deliver a second copy. Detailed evidence formats: docs/RUNNING.md in the
package. Never infer delivery from a local timeout."#;
pub const GOOGLE_SETUP: &str = r#"Example (private Desktop OAuth JSON downloaded from Google Cloud):
nuncio-cli account add-google --client-config /private/path/nuncio-google.json
Protect the JSON with chmod 600 and keep it outside Git and chat. The same
--client-config option is available on account add and reauth-google. Use the
same --profile as your daemon. First-time registration and browser consent:
docs/GOOGLE-SETUP.md in the installed package. For an existing account, reauth
must sign in as that saved identity; account list shows its local account ID."#;
pub const IMAP_SETUP: &str = r#"For interactive setup with hidden password entry, use account add.
For scripts, keep public settings and private credentials separate:
nuncio-cli account connect-imap --config mailplus.json --credentials-stdin < /private/path/credentials.json
Public config (schema_version 1): address, imap and smtp endpoints, sent_policy,
and sent_folder. Each endpoint needs host, port, tls (implicit or start_tls),
and username. Optional fields: archive_folder, trash_folder, trusted_ca_pem.
The two credential JSON fields are imap_password and smtp_password; read them
from a protected file or secure pipe, never command arguments or shell history.
Config is limited to 256 KiB and credentials to 16 KiB. Use account imap-config
--json for an existing account's public settings; edit-imap also requires its
current --version from account show. Reauth-imap reads only credentials and
uses saved settings. Complete examples and Sent-copy policy selection:
docs/ACCOUNT-SETUP.md and docs/RUNNING.md in the installed package."#;
+118
View File
@@ -0,0 +1,118 @@
use serde_json::Value;
use std::fmt::Write;
pub fn render(value: &Value) -> String {
let mut output = String::new();
if value.get("items").is_some_and(Value::is_array)
&& value.pointer("/coverage/state").and_then(Value::as_str) == Some("unavailable")
{
output.push_str("No completed local mail snapshot is available.\nMail becomes visible after the first successful sync. Check 'nuncio-cli system status' for progress or errors, using the same --profile.\n\n");
}
fields(&mut output, value, 0);
output.trim_end().to_owned()
}
fn fields(output: &mut String, value: &Value, depth: usize) {
let indent = " ".repeat(depth);
match value {
Value::Object(values) if !values.is_empty() => {
for (key, value) in values {
if key.ends_with("_at_ms") || key == "internal_date_ms" {
if let Some(time) = value
.as_i64()
.and_then(chrono::DateTime::from_timestamp_millis)
{
let _ = writeln!(
output,
"{indent}{}: {}",
label(key.trim_end_matches("_ms")),
time.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
);
continue;
}
}
let label = label(key);
match value {
Value::String(text) if text.contains('\n') => {
let _ = writeln!(output, "{indent}{label}:");
for line in text.lines() {
let _ = writeln!(output, "{indent} | {}", safe(line));
}
}
Value::Array(items) if items.is_empty() => {
let _ = writeln!(output, "{indent}{label}: No items");
}
Value::Array(_) | Value::Object(_) => {
let _ = writeln!(output, "{indent}{label}:");
fields(output, value, depth + 1);
}
_ => {
let _ = writeln!(output, "{indent}{label}: {}", scalar(value));
}
}
}
}
Value::Array(items) if !items.is_empty() => {
for (index, item) in items.iter().enumerate() {
if matches!(item, Value::Object(_) | Value::Array(_)) {
let _ = writeln!(output, "{indent}{}.", index + 1);
fields(output, item, depth + 1);
} else {
let _ = writeln!(output, "{indent}- {}", scalar(item));
}
}
}
Value::Array(_) => {
let _ = writeln!(output, "{indent}No items");
}
Value::Object(_) => {
let _ = writeln!(output, "{indent}No details");
}
_ => {
let _ = writeln!(output, "{indent}{}", scalar(value));
}
}
}
fn label(key: &str) -> String {
let mut words: Vec<_> = key.split('_').map(str::to_owned).collect();
for (index, word) in words.iter_mut().enumerate() {
if matches!(
word.as_str(),
"id" | "api" | "url" | "tls" | "uid" | "smtp" | "imap"
) {
*word = word.to_uppercase();
} else if index == 0 {
let mut letters = word.chars();
if let Some(first) = letters.next() {
*word = first.to_uppercase().collect::<String>() + letters.as_str();
}
}
}
safe(&words.join(" "))
}
fn scalar(value: &Value) -> String {
match value {
Value::Null => "Not set".into(),
Value::Bool(true) => "yes".into(),
Value::Bool(false) => "no".into(),
Value::String(text) if text.is_empty() => "(empty)".into(),
Value::String(text) => safe(text),
_ => value.to_string(),
}
}
fn safe(text: &str) -> String {
let mut output = String::new();
for character in text.chars() {
if character.is_control()
|| matches!(character, '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}')
{
let _ = write!(output, "\\u{:04x}", character as u32);
} else {
output.push(character);
}
}
output
}
+531
View File
@@ -0,0 +1,531 @@
use crate::{args::MailCommand, output::AppError, rpc_error};
use nuncio_proto::{
client::TokenInjector,
v2::{self, mail_client::MailClient, system_client::SystemClient},
};
use serde_json::Value;
use sha2::{Digest, Sha256};
use std::{io::Write, path::PathBuf, time::Duration};
use tonic::{service::interceptor::InterceptedService, transport::Channel};
type Transport = InterceptedService<Channel, TokenInjector>;
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod action_tests {
#[test]
fn action_files_require_version_one_and_reject_unknown_fields_and_oversized_input() {
let temporary = tempfile::tempdir().unwrap();
let path = temporary.path().join("action.json");
for invalid in [
r#"{"action":"archive"}"#,
r#"{"schema_version":2,"action":"archive"}"#,
r#"{"schema_version":"1","action":"archive"}"#,
r#"{"schema_version":1,"action":"archive","unexpected":true}"#,
r#"{"schema_version":1,"action":"read"}"#,
] {
std::fs::write(&path, invalid).unwrap();
assert!(super::read_action(&path).is_err(), "accepted {invalid}");
}
std::fs::write(
&path,
format!(
"{{\"schema_version\":1,\"action\":\"archive\"}}{}",
" ".repeat(8192)
),
)
.unwrap();
assert!(super::read_action(&path).is_err());
std::fs::write(
&path,
r#"{"schema_version":1,"action":"read","read":false}"#,
)
.unwrap();
assert!(
matches!(super::read_action(&path),Ok(nuncio_proto::v2::change_message_request::Action::Read(v)) if !v.read)
);
}
}
#[derive(serde::Deserialize)]
#[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)]
enum ActionFile {
Read {
read: bool,
},
Star {
starred: bool,
},
Archive {},
Move {
destination_collection_id: String,
},
Copy {
destination_collection_id: String,
},
Trash {
trashed: bool,
},
Label {
collection_id: String,
present: bool,
},
}
fn read_action(path: &std::path::Path) -> Result<v2::change_message_request::Action, AppError> {
use std::io::Read;
use v2::change_message_request::Action;
let mut bytes = Vec::new();
crate::drafts::regular_file(path)?
.take(8193)
.read_to_end(&mut bytes)
.map_err(|_| AppError::invalid())?;
if bytes.len() > 8192 {
return Err(AppError::invalid());
}
let mut value: serde_json::Value =
serde_json::from_slice(&bytes).map_err(|_| AppError::invalid())?;
let fields = value.as_object_mut().ok_or_else(AppError::invalid)?;
if fields
.remove("schema_version")
.as_ref()
.and_then(serde_json::Value::as_u64)
!= Some(1)
{
return Err(AppError::invalid());
}
let action: ActionFile = serde_json::from_value(value).map_err(|_| AppError::invalid())?;
Ok(match action {
ActionFile::Read { read } => Action::Read(v2::MailReadChange { read }),
ActionFile::Star { starred } => Action::Star(v2::MailStarChange { starred }),
ActionFile::Archive {} => Action::Archive(v2::MailArchiveChange {}),
ActionFile::Move {
destination_collection_id,
} => Action::Move(v2::MailFolderChange {
destination_collection_id,
}),
ActionFile::Copy {
destination_collection_id,
} => Action::Copy(v2::MailFolderChange {
destination_collection_id,
}),
ActionFile::Trash { trashed } => Action::Trash(v2::MailTrashChange { trashed }),
ActionFile::Label {
collection_id,
present,
} => Action::Label(v2::MailLabelChange {
collection_id,
present,
}),
})
}
pub(crate) fn json(value: impl serde::Serialize) -> Result<Value, AppError> {
serde_json::to_value(value).map_err(|_| invalid_response())
}
pub async fn run(
mut command: MailCommand,
channel: Channel,
token: TokenInjector,
) -> Result<Value, AppError> {
if let MailCommand::Fetch {
account,
message,
wait,
} = command
{
return sync(
&mut SystemClient::with_interceptor(channel, token),
account,
false,
Some(message),
wait,
)
.await;
}
if let MailCommand::Raw { output, .. }
| MailCommand::Body { output, .. }
| MailCommand::Attachment { output, .. } = &mut command
{
let status = SystemClient::with_interceptor(channel.clone(), token.clone())
.get_status(v2::GetStatusRequest {})
.await
.map_err(rpc_error)?
.into_inner();
*output =
crate::output_file::destination(output, &status.protected_paths).map_err(|error| {
match error {
crate::output_file::OutputPathError::Destination => file_error(),
crate::output_file::OutputPathError::Protection => invalid_response(),
}
})?;
}
let mut client = MailClient::with_interceptor(channel.clone(), token.clone());
match command {
MailCommand::Capabilities { account } => json(
client
.get_capabilities(v2::MailCapabilitiesRequest {
account_id: account,
})
.await
.map_err(rpc_error)?
.into_inner(),
),
MailCommand::Change {
account,
message,
request_id,
file,
wait,
} => {
let action = read_action(&file).map_err(|error| error.input_context(
"Invalid mail action file: --file needs schema_version 1 and the fields for its action (at most 8 KiB). See 'mail change --help'."
))?;
let op = client
.change_message(v2::ChangeMessageRequest {
account_id: account,
message_id: message,
request_id,
action: Some(action),
})
.await
.map_err(rpc_error)?
.into_inner();
if wait {
crate::operations::wait_for(op, channel, token).await
} else {
json(op)
}
}
MailCommand::Send {
account,
draft,
request_id,
version,
wait,
} => {
let operation = client
.send_draft(v2::SendDraftRequest {
account_id: account,
draft_id: draft,
request_id,
expected_version: version,
})
.await
.map_err(rpc_error)?
.into_inner();
if wait {
crate::operations::wait_for(operation, channel, token).await
} else {
json(operation)
}
}
MailCommand::Reply {
account,
message,
body_file,
} => {
crate::drafts::prepare(
&mut client,
account,
message,
"reply",
Some(body_file),
vec![],
)
.await
}
MailCommand::ReplyAll {
account,
message,
body_file,
} => {
crate::drafts::prepare(
&mut client,
account,
message,
"reply_all",
Some(body_file),
vec![],
)
.await
}
MailCommand::Forward {
account,
message,
to,
body_file,
} => crate::drafts::prepare(&mut client, account, message, "forward", body_file, to).await,
MailCommand::Draft { command } => crate::drafts::run(&mut client, command).await,
MailCommand::List { args } => list(&mut client, args, None).await,
MailCommand::Search { args, query } => list(&mut client, args, Some(query)).await,
MailCommand::Collections { account } => json(
client
.list_collections(v2::ListCollectionsRequest {
account_id: account,
})
.await
.map_err(rpc_error)?
.into_inner(),
),
MailCommand::Read { account, message } => {
let detail = client
.get_message(v2::MessageRequest {
account_id: account.clone(),
message_id: message.clone(),
})
.await
.map_err(rpc_error)?
.into_inner();
let mut result = json(detail)?;
for kind in ["text", "html"] {
let mut bytes = Vec::new();
let downloaded = download(
&mut client,
v2::DownloadMailRequest {
account_id: account.clone(),
message_id: message.clone(),
kind: kind.into(),
attachment_id: None,
},
&mut bytes,
)
.await;
match downloaded {
Ok(_) => {
result[kind] = Value::String(
String::from_utf8(bytes).map_err(|_| invalid_response())?,
);
}
Err(e) if e.code == "not_found" => {
result[kind] = Value::Null;
}
Err(e) => return Err(e),
}
}
Ok(result)
}
MailCommand::Raw {
account,
message,
output,
} => save(&mut client, account, message, "raw".into(), None, output).await,
MailCommand::Body {
account,
message,
kind,
output,
} => save(&mut client, account, message, kind, None, output).await,
MailCommand::Attachment {
account,
message,
attachment,
output,
} => {
save(
&mut client,
account,
message,
"attachment".into(),
Some(attachment),
output,
)
.await
}
MailCommand::Fetch { .. } => Err(AppError::invalid()),
}
}
async fn list(
client: &mut MailClient<Transport>,
q: crate::args::MailQueryArgs,
query: Option<String>,
) -> Result<Value, AppError> {
json(
client
.list_messages(v2::ListMailRequest {
account_id: q.account,
collection_id: q.collection,
query,
page_size: q.page_size,
page_token: q.page_token,
})
.await
.map_err(rpc_error)?
.into_inner(),
)
}
pub async fn sync(
client: &mut SystemClient<Transport>,
account: String,
full: bool,
fetch: Option<String>,
wait: bool,
) -> Result<Value, AppError> {
let run = client
.start_sync(v2::StartSyncRequest {
account_id: account,
full,
fetch_message_id: fetch,
})
.await
.map_err(rpc_error)?
.into_inner();
wait_run(client, run, wait).await
}
pub(crate) async fn wait_run(
client: &mut SystemClient<Transport>,
mut run: v2::SyncRun,
wait: bool,
) -> Result<Value, AppError> {
if wait {
let deadline = tokio::time::Instant::now() + Duration::from_secs(300);
loop {
match run.state.as_str() {
"succeeded" => break,
"failed" | "cancelled" => {
return Err((match run.error_code.as_deref() {
Some("unavailable") => AppError {
code: "provider_unavailable",
message: "Provider synchronization failed; inspect the sync run",
sync_run: None,
operation: None,
recovery: None,
exit: 4,
},
Some("authorization_required" | "scope_denied") => AppError::auth(),
Some("history_expired") => AppError {
code: "history_expired",
message: "Provider history expired repeatedly; retry synchronization",
sync_run: None,
operation: None,
recovery: None,
exit: 4,
},
_ => AppError {
code: "sync_failed",
message: "Synchronization did not complete; inspect the sync run",
sync_run: None,
operation: None,
recovery: None,
exit: 1,
},
})
.with_sync(&run))
}
"queued" | "running" => {}
_ => return Err(invalid_response()),
}
if tokio::time::Instant::now() >= deadline {
return Err(AppError {
code: "wait_timeout",
message: "Synchronization is still pending; inspect the sync run",
sync_run: None,
operation: None,
recovery: None,
exit: 4,
}
.with_sync(&run));
}
tokio::time::sleep(Duration::from_millis(50)).await;
run = client
.get_sync_run(v2::SyncRunRequest {
account_id: run.account_id.clone(),
run_id: run.id.clone(),
})
.await
.map_err(|error| rpc_error(error).with_sync(&run))?
.into_inner();
}
}
json(run)
}
async fn save(
client: &mut MailClient<Transport>,
account: String,
message: String,
kind: String,
attachment: Option<String>,
output: PathBuf,
) -> Result<Value, AppError> {
let parent = output
.parent()
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or(std::path::Path::new("."));
let mut file = tempfile::NamedTempFile::new_in(parent).map_err(|_| file_error())?;
let (length, hash) = download(
client,
v2::DownloadMailRequest {
account_id: account,
message_id: message,
kind,
attachment_id: attachment,
},
file.as_file_mut(),
)
.await?;
file.as_file().sync_all().map_err(|_| file_error())?;
file.persist_noclobber(&output).map_err(|_| file_error())?;
std::fs::File::open(parent)
.and_then(|f| f.sync_all())
.map_err(|_| file_error())?;
Ok(serde_json::json!({"output":output,"byte_length":length,"sha256":hash}))
}
async fn download(
client: &mut MailClient<Transport>,
request: v2::DownloadMailRequest,
writer: &mut impl Write,
) -> Result<(u64, String), AppError> {
let mut stream = client
.download(request)
.await
.map_err(rpc_error)?
.into_inner();
let mut offset = 0;
let mut manifest = None;
let mut hasher = Sha256::new();
let mut finished = false;
while let Some(chunk) = stream.message().await.map_err(rpc_error)? {
if finished
|| chunk.offset != offset
|| chunk.data.len() > 262144
|| chunk.total_bytes > 64 * 1024 * 1024
|| chunk.sha256.len() != 64
{
return Err(invalid_response());
}
let expected = manifest.get_or_insert((chunk.total_bytes, chunk.sha256.clone()));
if expected != &(chunk.total_bytes, chunk.sha256)
|| offset + chunk.data.len() as u64 > chunk.total_bytes
{
return Err(invalid_response());
}
if chunk.data.is_empty() && chunk.total_bytes != 0 {
return Err(invalid_response());
}
hasher.update(&chunk.data);
writer.write_all(&chunk.data).map_err(|_| file_error())?;
offset += chunk.data.len() as u64;
finished = chunk.eof;
}
let (total, hash) = manifest.ok_or_else(invalid_response)?;
if !finished || offset != total || hex::encode(hasher.finalize()) != hash {
return Err(invalid_response());
}
Ok((total, hash))
}
fn invalid_response() -> AppError {
AppError {
code: "invalid_response",
message: "Daemon returned an incomplete or invalid response",
sync_run: None,
operation: None,
recovery: None,
exit: 1,
}
}
fn file_error() -> AppError {
AppError {
code: "file_io",
message: "Could not write output; destination must be a new writable path",
sync_run: None,
operation: None,
recovery: None,
exit: 1,
}
}
+329
View File
@@ -0,0 +1,329 @@
mod accounts;
mod args;
mod backup;
mod calendar;
mod changes;
mod credentials;
mod draft_upload;
mod drafts;
mod help_text;
mod human;
mod mail;
mod operations;
mod output;
mod output_file;
mod repair;
mod usage;
use args::{Args, Command, SystemCommand};
use clap::Parser;
use nuncio_proto::{
client::{ClientError, TokenInjector},
v2::{system_client::SystemClient, GetStatusRequest, ShutdownRequest},
};
use output::AppError;
#[tokio::main]
async fn main() -> std::process::ExitCode {
let args = match Args::try_parse() {
Ok(args) => args,
Err(error)
if matches!(
error.kind(),
clap::error::ErrorKind::DisplayHelp | clap::error::ErrorKind::DisplayVersion
) =>
{
return if error.print().is_ok() {
std::process::ExitCode::SUCCESS
} else {
std::process::ExitCode::from(1)
};
}
Err(error)
if error.kind() == clap::error::ErrorKind::DisplayHelpOnMissingArgumentOrSubcommand
&& !std::env::args_os().any(|value| value == "--json") =>
{
return if error.print().is_ok() {
std::process::ExitCode::from(2)
} else {
std::process::ExitCode::from(1)
};
}
Err(error) => {
return if std::env::args_os().any(|value| value == "--json") {
output::emit(Err(AppError::invalid()), true)
} else {
usage::emit(&error)
};
}
};
let json = args.json;
if matches!(
&args.command,
Command::System {
command: SystemCommand::Watch { .. }
}
) {
return match changes::run(args).await {
Ok(()) => std::process::ExitCode::SUCCESS,
Err(e) => output::emit(Err(e), true),
};
}
let guided = matches!(
&args.command,
Command::Account {
command: args::AccountCommand::Add { .. }
}
);
if guided {
if let Err(error) = accounts::wizard::preflight(json) {
return output::emit(Err(error), json);
}
return output::emit_setup(run(args).await);
}
output::emit(run(args).await, json)
}
async fn connect(args: &Args) -> Result<(tonic::transport::Channel, TokenInjector), AppError> {
if std::env::vars_os().any(|(name, _)| name.to_string_lossy().starts_with("NUNCIO_TEST_")) {
return Err(AppError::invalid());
}
let channel = nuncio_proto::client::connect_with_timeout(
&args.endpoint,
std::time::Duration::from_secs(if matches!(args.command, Command::Backup { .. }) {
3600
} else {
30
}),
)
.await
.map_err(|error| match error {
ClientError::InvalidEndpoint => AppError::invalid(),
ClientError::Authorization => AppError::auth(),
ClientError::Unavailable => AppError::unavailable(),
})?;
let token = credentials::load(args)?;
let injector = TokenInjector::new(token).map_err(|_| AppError::auth())?;
Ok((channel, injector))
}
async fn run(args: Args) -> Result<serde_json::Value, AppError> {
let (channel, injector) = connect(&args).await?;
if let Command::Repair(command) = args.command {
return repair::run(command, channel, injector).await;
}
if let Command::Backup { command } = args.command {
return backup::run(command, channel, injector).await;
}
if let Command::Operations { command } = args.command {
return operations::run(command, channel, injector).await;
}
if let Command::Calendar { command } = args.command {
return calendar::run(command, channel, injector).await;
}
if let Command::Account { command } = args.command {
return accounts::run(command, channel, injector).await;
}
if let Command::Mail { command } = args.command {
return mail::run(command, channel, injector).await;
}
let mut client = SystemClient::with_interceptor(channel, injector);
match args.command {
Command::Repair(_) => Err(AppError::invalid()),
Command::Backup { .. } => Err(AppError::invalid()),
Command::Operations { .. } => Err(AppError::invalid()),
Command::System {
command: SystemCommand::Watch { .. },
} => Err(AppError::invalid()),
Command::Calendar { .. } => Err(AppError::invalid()),
Command::Account { .. } => Err(AppError::invalid()),
Command::Mail { .. } => Err(AppError::invalid()),
Command::Sync {
account,
full,
wait,
} => mail::sync(&mut client, account, full, None, wait).await,
Command::System {
command: SystemCommand::SyncStatus { account, run },
} => {
let run = client
.get_sync_run(nuncio_proto::v2::SyncRunRequest {
account_id: account,
run_id: run,
})
.await
.map_err(rpc_error)?
.into_inner();
mail::json(run)
}
Command::System {
command: SystemCommand::CancelSync { account, run },
} => {
let run = client
.cancel_sync(nuncio_proto::v2::SyncRunRequest {
account_id: account,
run_id: run,
})
.await
.map_err(rpc_error)?
.into_inner();
mail::json(run)
}
Command::System {
command: SystemCommand::Status,
} => {
let response = client
.get_status(GetStatusRequest {})
.await
.map_err(rpc_error)?
.into_inner();
let storage = response.storage.ok_or_else(AppError::unavailable)?;
Ok(
serde_json::json!({"version":response.version, "api_version":response.api_version, "profile_id":response.profile_id,
"storage":{"schema_version":storage.schema_version,"account_count":storage.account_count,"revision":storage.revision},
"protected_paths":response.protected_paths,"background_sync":response.background_sync,"scheduler_error":response.scheduler_error,"operation_worker_error":response.operation_worker_error,"sync":response.sync,"resources":response.resources}),
)
}
Command::System {
command: SystemCommand::Shutdown,
} => {
client
.shutdown(ShutdownRequest {})
.await
.map_err(rpc_error)?;
Ok(serde_json::json!({"shutdown_requested":true}))
}
}
}
fn rpc_error(status: tonic::Status) -> AppError {
let guidance = rpc_guidance(&status);
let mut error = match status.code() {
tonic::Code::OutOfRange => AppError {
code: "resnapshot_required",
message:
"Change history is unavailable; take a new snapshot and resume from its revision",
sync_run: None,
operation: None,
recovery: None,
exit: 5,
},
tonic::Code::Unauthenticated | tonic::Code::PermissionDenied => AppError::auth(),
tonic::Code::InvalidArgument => AppError::invalid(),
tonic::Code::Unavailable | tonic::Code::DeadlineExceeded => AppError::unavailable(),
tonic::Code::Unknown | tonic::Code::Cancelled => AppError {
code: "request_outcome_unknown",
message: "Request outcome is unknown; inspect state or retry the identical request",
sync_run: None,
operation: None,
recovery: None,
exit: 4,
},
tonic::Code::NotFound => AppError {
code: "not_found",
message: "Resource not found for this account; copy its local ID from the matching list command and check --account",
sync_run: None,
operation: None,
recovery: None,
exit: 4,
},
tonic::Code::FailedPrecondition | tonic::Code::Aborted => AppError {
code: "conflict",
message: "This action conflicts with current state; inspect the account/resource and use its latest version or ETag before retrying",
sync_run: None,
operation: None,
recovery: None,
exit: 5,
},
_ => AppError {
code: "operation_failed",
message: "The engine could not complete this action; inspect the daemon log and any returned operation or sync run",
sync_run: None,
operation: None,
recovery: None,
exit: 1,
},
};
if let Some(message) = guidance {
error.message = message;
}
error
}
fn rpc_guidance(status: &tonic::Status) -> Option<&'static str> {
use tonic::Code;
// Recognize only fixed API messages; never print arbitrary server details.
Some(match (status.code(), status.message()) {
(Code::Unauthenticated, "Account authorization is required") => "Account sign-in is required. Use account add for a new account, or account reauth-google / account reauth-imap for a saved account; see their --help.",
(Code::Unauthenticated, "Required Google permissions were not granted") => "Required Google permissions were not granted. Repeat Google sign-in and grant the requested Gmail and Calendar access; see docs/GOOGLE-SETUP.md.",
(Code::FailedPrecondition, "Mail server TLS verification failed") => "Mail server TLS verification failed. Check the server hostname, certificate and trusted CA in the account settings; use account add or account edit-imap.",
(Code::FailedPrecondition, "Mail server does not support a required protocol capability") => "The mail server lacks a required protocol capability; compare its settings with docs/COMPATIBILITY.md.",
(Code::FailedPrecondition, "Provider account identity does not match the saved account") => "Sign-in selected a different account. Reauthenticate using the saved identity, or use account add for a separate account.",
(Code::FailedPrecondition, "Account lifecycle does not permit this action; restore archived accounts and resolve remote-effect uncertainty before deletion") => "Account state prevents this action. Inspect account show; restore an archived account before reconnecting and resolve uncertain operations before permanent deletion.",
(Code::Aborted, "Account configuration changed; read the latest version") => "The account configuration changed. Read account show and repeat the edit using its current --version.",
(Code::FailedPrecondition, "The local resource changed; read its latest version before editing") => "The resource changed. Read its show/get result and repeat the edit using the current version or ETag.",
(Code::FailedPrecondition, "The query snapshot changed; restart pagination") => "Cached results changed during pagination. Start the list/search again without --page-token, then use its new page tokens.",
(Code::FailedPrecondition, "Different work is already active for this account and scope") => "Other work is active for this account and scope. Inspect its sync run or operation and wait for it to finish before retrying.",
(Code::Unavailable, "Provider service is unavailable") => "The provider is unavailable. Check account status and daemon logs; inspect any queued operation before retrying a write.",
(Code::Unavailable, "Provider service requested a later attempt") => "The provider requested a delay. Wait before checking again; queued work retains its retry schedule.",
(Code::ResourceExhausted, "Too many pending authorization sessions or accounts") => "Too many accounts or pending sign-in sessions. Inspect account list and cancel unused sessions with account auth-cancel.",
(Code::ResourceExhausted, "Query result is too large; request a smaller page") => "The result exceeds the response limit. Retry with a smaller --page-size.",
(Code::InvalidArgument, "Invalid account configuration") => "The account configuration is invalid. Check account setup help, or use account add for guided input; see docs/ACCOUNT-SETUP.md.",
(Code::Internal, "Secure credential storage is unavailable") => "The secure credential store is unavailable. Unlock the OS credential store and inspect the daemon log before retrying account setup.",
_ => return None,
})
}
#[cfg(test)]
mod rpc_error_tests {
#[test]
fn provider_errors_explain_the_right_recovery_without_echoing_server_text() {
use tonic::{Code, Status};
for (code, message, guidance) in [
(
Code::Unauthenticated,
"Account authorization is required",
"reauth",
),
(
Code::Unauthenticated,
"Required Google permissions were not granted",
"Google permissions",
),
(
Code::FailedPrecondition,
"Mail server TLS verification failed",
"certificate",
),
(
Code::Unavailable,
"Provider service is unavailable",
"provider",
),
(
Code::FailedPrecondition,
"The query snapshot changed; restart pagination",
"--page-token",
),
] {
let error = super::rpc_error(Status::new(code, message));
assert!(
error.message.contains(guidance),
"wrong recovery for {message}: {}",
error.message
);
assert!(!error.message.contains("start 'nunciod"));
}
let local = super::rpc_error(Status::unauthenticated("Profile authorization required"));
assert!(local.message.contains("--profile"));
for code in [
Code::Unauthenticated,
Code::FailedPrecondition,
Code::Unavailable,
Code::Internal,
] {
let error = super::rpc_error(Status::new(code, "private-server-token\u{1b}[2J"));
assert!(!error.message.contains("private-server-token"));
assert!(!error.message.contains('\u{1b}'));
}
}
}
+207
View File
@@ -0,0 +1,207 @@
use crate::{args::OperationsCommand, mail::json, output::AppError, rpc_error};
use nuncio_proto::{
client::TokenInjector,
v2::{self, operations_client::OperationsClient, OperationRequest},
};
use tonic::transport::Channel;
pub(crate) async fn wait_for(
mut operation: v2::Operation,
channel: Channel,
token: TokenInjector,
) -> Result<serde_json::Value, AppError> {
let mut client = OperationsClient::with_interceptor(channel.clone(), token.clone())
.max_decoding_message_size(8 * 1024 * 1024);
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30);
loop {
match operation.state.as_str() {
"applied"=>return json(operation),
"failed"=>return Err(AppError{code:"operation_failed",message:"Provider rejected the operation; inspect its attempts",exit:1,sync_run:None,operation: None, recovery: None}.with_operation(&operation)),
"conflict"|"cancelled"=>return Err(AppError{code:"operation_not_applied",message:"Operation was cancelled or has a conflict",exit:5,sync_run:None,operation: None, recovery: None}.with_operation(&operation)),
"uncertain" if !operation.needs_reconciliation=>return Err(AppError{code:"operation_uncertain",message:"Provider outcome remains uncertain; inspect the operation before deciding what to do",exit:5,sync_run:None,operation: None, recovery: None}.with_operation(&operation)),
"queued"|"running"|"retry_wait"|"uncertain"=>{},
_=>return Err(AppError::unavailable().with_operation(&operation)),
}
let request = OperationRequest {
account_id: operation.account_id.clone(),
operation_id: operation.id.clone(),
};
tokio::select! {
_=tokio::time::sleep_until(deadline)=>return Err(AppError{code:"operation_wait_timeout",message:"Wait timed out; the durable operation remains inspectable",exit:4,sync_run:None,operation: None, recovery: None}.with_operation(&operation)),
result=async {tokio::time::sleep(std::time::Duration::from_millis(100)).await;client.get_operation(request).await}=>{
operation=result.map_err(|e|rpc_error(e).with_operation(&operation))?.into_inner();
}
}
}
}
pub(crate) async fn run(
command: OperationsCommand,
channel: Channel,
token: TokenInjector,
) -> Result<serde_json::Value, AppError> {
let mut client = OperationsClient::with_interceptor(channel.clone(), token.clone())
.max_decoding_message_size(8 * 1024 * 1024);
match command {
OperationsCommand::List {
account,
page_size,
page_token,
} => {
return json(
client
.list_operations(v2::ListOperationsRequest {
account_id: account,
page_size,
page_token,
})
.await
.map_err(rpc_error)?
.into_inner(),
)
}
OperationsCommand::Attempts {
account,
operation,
page_size,
page_token,
} => {
return json(
client
.list_attempts(v2::ListOperationAttemptsRequest {
account_id: account,
operation_id: operation,
page_size,
page_token,
})
.await
.map_err(rpc_error)?
.into_inner(),
)
}
_ => {}
}
let result = match command {
OperationsCommand::Reconcile {
account,
operation,
request_id,
version,
resume_safe,
wait,
} => {
let operation = client
.reconcile_operation(v2::ReconcileOperationRequest {
account_id: account,
operation_id: operation,
request_id,
expected_version: version,
mode: if resume_safe {
v2::ReconciliationMode::ResumeSafe
} else {
v2::ReconciliationMode::Observe
} as i32,
})
.await
.map_err(rpc_error)?
.into_inner();
return if wait {
wait_for(operation, channel, token).await
} else {
json(operation)
};
}
OperationsCommand::Wait { account, operation } => {
let operation = client
.get_operation(OperationRequest {
account_id: account,
operation_id: operation,
})
.await
.map_err(rpc_error)?
.into_inner();
return wait_for(operation, channel, token).await;
}
OperationsCommand::List { .. } | OperationsCommand::Attempts { .. } => {
return Err(AppError::invalid())
}
OperationsCommand::Show { account, operation } => {
client
.get_operation(OperationRequest {
account_id: account,
operation_id: operation,
})
.await
}
OperationsCommand::Cancel { account, operation } => {
client
.cancel_operation(OperationRequest {
account_id: account,
operation_id: operation,
})
.await
}
OperationsCommand::Resolve {
account,
operation,
version,
file,
accept_duplicate_risk,
} => {
let decision = read_decision(&file, accept_duplicate_risk).map_err(|error| error.input_context(
"Invalid recovery decision: --file needs a supported decision and its evidence/reason (at most 16 KiB). See 'operation resolve --help'."
))?;
client
.resolve_operation(v2::ResolveOperationRequest {
account_id: account,
operation_id: operation,
expected_version: version,
decision: Some(decision),
})
.await
}
}
.map_err(rpc_error)?
.into_inner();
json(result)
}
#[derive(serde::Deserialize)]
#[serde(tag = "decision", rename_all = "snake_case", deny_unknown_fields)]
enum DecisionFile {
Abandon { reason: String },
ConfirmApplied { evidence: v2::OperationConfirmation },
Resend { request_id: String, reason: String },
}
fn read_decision(
path: &std::path::Path,
accept_duplicate_risk: bool,
) -> Result<v2::resolve_operation_request::Decision, AppError> {
use std::io::{Read, Write};
use v2::resolve_operation_request::Decision;
let mut bytes = Vec::new();
crate::drafts::regular_file(path)?
.take(16 * 1024 + 1)
.read_to_end(&mut bytes)
.map_err(|_| crate::drafts::file_error())?;
if bytes.len() > 16 * 1024 {
return Err(AppError::invalid());
}
let decision: DecisionFile = serde_json::from_slice(&bytes).map_err(|_| AppError::invalid())?;
if accept_duplicate_risk && !matches!(decision, DecisionFile::Resend { .. }) {
return Err(AppError::invalid());
}
Ok(match decision {
DecisionFile::Abandon { reason } => Decision::Abandon(v2::OperationAbandonment { reason }),
DecisionFile::ConfirmApplied { evidence } => Decision::ConfirmApplied(evidence),
DecisionFile::Resend { request_id, reason } => {
if !accept_duplicate_risk {
return Err(AppError{code:"duplicate_risk_not_accepted",message:"Resend may duplicate a message already delivered. Explicit --accept-duplicate-risk is required.",exit:2,sync_run:None,operation: None, recovery: None});
}
writeln!(std::io::stderr().lock(),"Resend may duplicate a message already delivered; creating a separate send request.").map_err(|_|AppError::unavailable())?;
Decision::Resend(v2::OperationResend {
request_id,
reason,
accept_duplicate_risk: true,
})
}
})
}
+229
View File
@@ -0,0 +1,229 @@
use std::io::Write;
pub struct AppError {
pub recovery: Option<serde_json::Value>,
pub operation: Option<Box<nuncio_proto::v2::Operation>>,
pub sync_run: Option<Box<nuncio_proto::v2::SyncRun>>,
pub code: &'static str,
pub message: &'static str,
pub exit: u8,
}
impl AppError {
pub fn input_context(mut self, message: &'static str) -> Self {
if self.code == "invalid_input" {
self.message = message;
}
self
}
pub fn with_operation(mut self, operation: &nuncio_proto::v2::Operation) -> Self {
self.operation = Some(Box::new(operation.clone()));
self
}
pub fn with_sync(mut self, run: &nuncio_proto::v2::SyncRun) -> Self {
self.sync_run = Some(Box::new(run.clone()));
self
}
pub fn invalid() -> Self {
Self {
code: "invalid_input",
message: "Input was rejected; check the required options, IDs and file format with this command's --help",
sync_run: None,
operation: None,
recovery: None,
exit: 2,
}
}
pub fn auth() -> Self {
Self {
code: "authorization_required",
message: "Profile authorization is unavailable; start the daemon with the same --profile and unlock your OS credential store",
sync_run: None,
operation: None,
recovery: None,
exit: 3,
}
}
pub fn unavailable() -> Self {
Self {
code: "unavailable",
message: "Cannot reach Nuncio; start 'nunciod --profile PROFILE' and use the same --profile and --endpoint here",
sync_run: None,
operation: None,
recovery: None,
exit: 4,
}
}
}
pub fn emit(result: Result<serde_json::Value, AppError>, json: bool) -> std::process::ExitCode {
let (value, exit) = match result {
Ok(value) => (serde_json::json!({"schema_version":1, "result":value}), 0),
Err(error) => {
if !json {
return emit_human_error(error);
}
let _ = writeln!(std::io::stderr().lock(), "{}", error.message);
let mut detail = serde_json::json!({"code":error.code, "message":error.message, "retryable":error.exit == 4});
if let Some(run) = error.sync_run {
detail["sync_run"] = serde_json::json!(run);
}
if let Some(operation) = error.operation {
detail["operation"] = serde_json::json!(operation);
}
if let Some(recovery) = error.recovery {
detail["recovery"] = recovery;
}
(
serde_json::json!({"schema_version":1, "error":detail}),
error.exit,
)
}
};
let rendered = render(&value, json);
match rendered {
Ok(rendered) if writeln!(std::io::stdout().lock(), "{rendered}").is_ok() => {
std::process::ExitCode::from(exit)
}
_ => std::process::ExitCode::from(1),
}
}
fn emit_human_error(error: AppError) -> std::process::ExitCode {
let mut text = format!("Error: {}\n", error.message);
if let Some(operation) = error.operation {
let detail = serde_json::json!({"operation_id":operation.id,"account_id":operation.account_id,
"state":operation.state,"error_code":operation.error_code,"needs_reconciliation":operation.needs_reconciliation});
text.push_str(&super::human::render(&detail));
text.push_str("\nInspect with: nuncio-cli operation show --account ACCOUNT_ID --operation OPERATION_ID\n");
}
if let Some(run) = error.sync_run {
let detail = serde_json::json!({"sync_run_id":run.id,"account_id":run.account_id,
"state":run.state,"error_code":run.error_code});
text.push_str(&super::human::render(&detail));
text.push_str("\nInspect with: nuncio-cli system sync-status --account ACCOUNT_ID --run SYNC_RUN_ID\n");
}
if let Some(recovery) = error.recovery {
text.push_str("Recovery details:\n");
text.push_str(&super::human::render(&recovery));
text.push('\n');
}
match std::io::stderr().lock().write_all(text.as_bytes()) {
Ok(()) => std::process::ExitCode::from(error.exit),
Err(_) => std::process::ExitCode::from(1),
}
}
pub fn emit_setup(result: Result<serde_json::Value, AppError>) -> std::process::ExitCode {
match result {
Ok(value) => {
let address = render(&value["account"]["address"], true);
let Ok(address) = address else {
return std::process::ExitCode::from(1);
};
let text = format!("Account connected: {address}\nBackground sync is enabled. Use account list to view your accounts.");
if writeln!(std::io::stdout().lock(), "{text}").is_err() {
return std::process::ExitCode::from(1);
}
if value["credential_cleanup_pending"] == true {
let _ = writeln!(
std::io::stderr().lock(),
"An older credential still needs cleanup. Run account check for details."
);
}
std::process::ExitCode::SUCCESS
}
Err(error) => {
let _ = writeln!(std::io::stderr().lock(), "{}", error.message);
std::process::ExitCode::from(error.exit)
}
}
}
pub(crate) fn render(value: &serde_json::Value, json: bool) -> Result<String, serde_json::Error> {
if !json {
return Ok(super::human::render(value.get("result").unwrap_or(value)));
}
let rendered = if json {
serde_json::to_string(&value)
} else {
serde_json::to_string_pretty(&value)
}?;
// JSON already quotes C0 controls. Also quote C1 and direction controls so
// terminal output is inert while a JSON consumer can recover original text.
let mut safe = String::with_capacity(rendered.len());
for ch in rendered.chars() {
if matches!(ch,'\u{7f}'..='\u{9f}'|'\u{202a}'..='\u{202e}'|'\u{2066}'..='\u{2069}') {
use std::fmt::Write;
let _ = write!(safe, "\\u{:04x}", ch as u32);
} else {
safe.push(ch)
}
}
Ok(safe)
}
#[cfg(test)]
mod tests {
#[test]
#[allow(clippy::unwrap_used)]
fn human_dates_are_utc_and_durations_stay_numeric() {
let value = serde_json::json!({"created_at_ms":0,"elapsed_ms":1500});
let output = super::render(&value, false).unwrap();
assert!(
output.contains("Created at: 1970-01-01T00:00:00.000Z"),
"{output}"
);
assert!(output.contains("Elapsed ms: 1500"), "{output}");
}
#[test]
#[allow(clippy::unwrap_used)]
fn human_multiline_content_stays_visibly_inside_its_field() {
let value = serde_json::json!({"text":"Hello\nState: applied\n\u{1b}[2J"});
let output = super::render(&value, false).unwrap();
assert!(
output.contains("Text:\n | Hello\n | State: applied"),
"{output}"
);
assert!(!output.contains('\u{1b}'));
}
#[test]
#[allow(clippy::unwrap_used)]
fn human_output_keeps_ids_pagination_and_empty_results_readable() {
let value = serde_json::json!({"schema_version":1,"result":{
"items":[{"id":"message-17","subject":"Meeting notes","read":false}],
"next_page_token":"page-2","revision":7}});
let output = super::render(&value, false).unwrap();
assert!(output.contains("Subject: Meeting notes"), "{output}");
assert!(output.contains("ID: message-17"), "{output}");
assert!(output.contains("Read: no"), "{output}");
assert!(output.contains("Next page token: page-2"), "{output}");
assert!(!output.contains("schema_version"));
assert!(!output.trim_start().starts_with('{'));
let empty = super::render(&serde_json::json!({"items":[]}), false).unwrap();
assert!(empty.contains("No items"), "{empty}");
assert_eq!(
serde_json::from_str::<serde_json::Value>(&super::render(&value, true).unwrap())
.unwrap(),
value
);
}
#[test]
#[allow(clippy::unwrap_used)]
fn terminal_controls_are_escaped_without_changing_json_content() {
let value = serde_json::json!({"subject":"a\u{1b}[2J\u{9b}2J\u{202e}hidden"});
for json in [true, false] {
let output = super::render(&value, json).unwrap();
assert!(!output.contains(['\u{1b}', '\u{9b}', '\u{202e}']));
if json {
assert_eq!(
serde_json::from_str::<serde_json::Value>(&output).unwrap(),
value
);
} else {
assert!(output.contains("hidden"));
}
}
}
}
@@ -0,0 +1,34 @@
use std::path::{Path, PathBuf};
pub(crate) enum OutputPathError {
Destination,
Protection,
}
pub(crate) fn destination(path: &Path, protected: &[String]) -> Result<PathBuf, OutputPathError> {
// All existing destinations are refused, including hard links and dangling
// symlinks. Canonical parents also protect files (such as WAL) not yet present.
match std::fs::symlink_metadata(path) {
Ok(_) => return Err(OutputPathError::Destination),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(_) => return Err(OutputPathError::Destination),
}
let parent = path
.parent()
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or(Path::new("."));
let parent = std::fs::canonicalize(parent).map_err(|_| OutputPathError::Destination)?;
let output = parent.join(path.file_name().ok_or(OutputPathError::Destination)?);
for path in protected {
let path = Path::new(path);
let name = path.file_name().ok_or(OutputPathError::Protection)?;
let parent = path.parent().ok_or(OutputPathError::Protection)?;
let canonical = std::fs::canonicalize(parent)
.map_err(|_| OutputPathError::Protection)?
.join(name);
if canonical == output {
return Err(OutputPathError::Destination);
}
}
Ok(output)
}
+66
View File
@@ -0,0 +1,66 @@
use crate::{
args::{RepairArgs, RepairScope},
output::AppError,
};
use nuncio_proto::{
client::TokenInjector,
v2::{self, maintenance_client::MaintenanceClient, system_client::SystemClient},
};
use tonic::transport::Channel;
pub async fn run(
args: RepairArgs,
channel: Channel,
injector: TokenInjector,
) -> Result<serde_json::Value, AppError> {
let scope = match args.scope {
RepairScope::Mail => v2::ProjectionScope::Mail,
RepairScope::Calendar => v2::ProjectionScope::Calendar,
};
let window = match (args.from, args.to) {
(Some(from), Some(to)) if scope == v2::ProjectionScope::Calendar => {
Some(v2::AgendaWindow { from, to })
}
(None, None) => None,
_ => return Err(AppError::invalid()),
};
let response = MaintenanceClient::with_interceptor(channel.clone(), injector.clone())
.repair_projection(v2::RepairProjectionRequest {
account_id: args.account.clone(),
scope: scope.into(),
dry_run: args.dry_run,
window,
})
.await
.map_err(crate::rpc_error)?
.into_inner();
if response.account_id != args.account
|| response.scope != scope as i32
|| response.dry_run != args.dry_run
|| response.projection.is_none()
|| response.preserved.is_none()
|| response.run.is_some() == args.dry_run
|| response
.run
.as_ref()
.is_some_and(|r| r.account_id != args.account)
{
return Err(AppError {
code: "invalid_repair_response",
message: "Daemon returned an incomplete or mismatched repair response",
exit: 1,
operation: None,
sync_run: None,
recovery: None,
});
}
let mut result = crate::mail::json(&response)?;
if let Some(run) = response.run {
result["run"] = crate::mail::wait_run(
&mut SystemClient::with_interceptor(channel, injector),
run,
args.wait,
)
.await?;
}
Ok(result)
}
+121
View File
@@ -0,0 +1,121 @@
use clap::error::{ContextKind, ErrorKind};
use clap::CommandFactory;
use std::io::Write;
/// Use parser-owned argument names and usage, never rejected user values.
pub fn emit(error: &clap::Error) -> std::process::ExitCode {
let reason = match error.kind() {
ErrorKind::MissingRequiredArgument => "Missing required arguments",
ErrorKind::MissingSubcommand => "Choose a command from the help below",
ErrorKind::UnknownArgument => "Unrecognized option or extra argument",
ErrorKind::InvalidSubcommand => "Unrecognized command",
ErrorKind::InvalidValue | ErrorKind::ValueValidation => "An option has an invalid value",
ErrorKind::ArgumentConflict => "These options cannot be used together",
ErrorKind::TooManyValues | ErrorKind::TooFewValues | ErrorKind::WrongNumberOfValues => {
"An option has the wrong number of values"
}
_ => "Invalid command or argument",
};
let mut text = format!("Error: {reason}.\n");
// For these kinds InvalidArg describes a declared option. For unknown
// arguments/subcommands it contains user input and must not be printed.
if matches!(
error.kind(),
ErrorKind::MissingRequiredArgument
| ErrorKind::InvalidValue
| ErrorKind::ValueValidation
| ErrorKind::ArgumentConflict
| ErrorKind::TooFewValues
| ErrorKind::TooManyValues
| ErrorKind::WrongNumberOfValues
) {
if let Some(arg) = error.get(ContextKind::InvalidArg) {
text.push_str(&format!(" {arg}\n"));
}
}
let (mut command, path) = selected_command();
if let Some(clap::error::ContextValue::String(suggestion)) =
error.get(ContextKind::SuggestedArg)
{
if let Some(name) = command
.get_arguments()
.filter_map(|argument| argument.get_long())
.find(|name| suggestion == &format!("--{name}"))
{
text.push_str(&format!("Did you mean '--{name}'?\n"));
}
}
if matches!(
error.kind(),
ErrorKind::InvalidValue | ErrorKind::ValueValidation
) {
if let Some(clap::error::ContextValue::String(argument)) =
error.get(ContextKind::InvalidArg)
{
let name = argument
.split_whitespace()
.next()
.and_then(|name| name.strip_prefix("--"));
if let Some(help) = command
.get_arguments()
.find(|arg| arg.get_long() == name)
.and_then(|arg| arg.get_help())
{
text.push_str(&format!("Expected: {help}\n"));
}
}
}
text.push_str(&format!("\n{}\n", command.render_usage()));
if text.contains("--account") {
text.push_str("\nFind the local account ID with 'nuncio-cli account list'.\nUse the same --profile as your daemon; --account selects one account inside it.\nExample: nuncio-cli --profile laptop-qa mail list --account ACCOUNT_ID\n");
}
text.push_str(&format!(
"\nRun '{path} --help' for its options and examples.\n"
));
match std::io::stderr().lock().write_all(text.as_bytes()) {
Ok(()) => std::process::ExitCode::from(2),
Err(_) => std::process::ExitCode::from(1),
}
}
fn selected_command() -> (clap::Command, String) {
let mut command = crate::args::Args::command();
command.build();
let mut path = String::from("nuncio-cli");
let mut arguments = std::env::args_os().skip(1);
while let Some(argument) = arguments.next() {
let Some(argument) = argument.to_str() else {
break;
};
if argument == "--" {
break;
}
if let Some(option) = argument.strip_prefix("--") {
let (name, inline) = option
.split_once('=')
.map_or((option, false), |(name, _)| (name, true));
let Some(option) = command.get_arguments().find(|a| a.get_long() == Some(name)) else {
break;
};
if option.get_action().takes_values() && !inline {
arguments.next();
}
continue;
}
let Some(next) = command
.get_subcommands()
.find(|child| {
child.get_name() == argument
|| child.get_all_aliases().any(|alias| alias == argument)
})
.cloned()
else {
break;
};
path.push(' ');
path.push_str(next.get_name());
command = next;
}
(command.bin_name(path.clone()), path)
}
+51
View File
@@ -0,0 +1,51 @@
#![allow(clippy::unwrap_used)]
use std::process::Command;
#[test]
fn backup_commands_are_available_and_secrets_or_traversal_are_not_accepted_in_arguments() {
for (command, flag) in [
("create", "--output"),
("inspect", "--file"),
("restore", "--new-profile"),
] {
let result = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.args(["backup", command, "--help"])
.output()
.unwrap();
assert!(result.status.success(), "missing backup {command}");
let text = String::from_utf8(result.stdout).unwrap();
assert!(text.contains(flag));
assert!(!text.contains("--passphrase"));
}
for args in [
vec![
"--json",
"backup",
"create",
"--output",
"unused.nuncio",
"--passphrase",
"never-print-secret-canary",
],
vec![
"--json",
"backup",
"restore",
"--file",
"unused.nuncio",
"--new-profile",
"../outside",
],
] {
let result = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.args(args)
.output()
.unwrap();
assert_eq!(result.status.code(), Some(2));
let stdout = String::from_utf8(result.stdout).unwrap();
let stderr = String::from_utf8(result.stderr).unwrap();
assert!(!stdout.contains("never-print-secret-canary"));
assert!(!stderr.contains("never-print-secret-canary"));
let value: serde_json::Value = serde_json::from_str(&stdout).unwrap();
assert_eq!(value["error"]["code"], "invalid_input");
}
}
@@ -0,0 +1,130 @@
#![cfg(feature = "test-harness")]
#![allow(clippy::unwrap_used, clippy::expect_used)]
use std::{
path::{Path, PathBuf},
process::Stdio,
time::Duration,
};
use tokio::{
io::{AsyncBufReadExt, BufReader},
process::{Child, Command},
};
async fn start(directory: &Path, secrets: &Path) -> (Child, String) {
let ready_file = secrets.with_extension("ready.json");
if ready_file.exists() {
std::fs::remove_file(&ready_file).unwrap();
}
let daemon = std::env::var_os("NUNCIO_E2E_DAEMON")
.expect("Set NUNCIO_E2E_DAEMON to the test-harness daemon binary");
let mut child = Command::new(daemon)
.env_clear()
.args(["--bind", "127.0.0.1:0", "--data-dir"])
.arg(directory)
.arg("--test-secrets-file")
.arg(secrets)
.arg("--ready-file")
.arg(&ready_file)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true)
.spawn()
.unwrap();
let mut line = String::new();
let mut output = BufReader::new(child.stdout.take().unwrap());
let bytes = tokio::time::timeout(Duration::from_secs(10), output.read_line(&mut line))
.await
.unwrap()
.unwrap();
assert!(
bytes > 0,
"actual daemon must initialize and report readiness with a mock keystore"
);
let ready: serde_json::Value = serde_json::from_str(&line).unwrap();
let file: serde_json::Value =
serde_json::from_slice(&std::fs::read(&ready_file).unwrap()).unwrap();
assert_eq!(ready, file);
(child, ready["endpoint"].as_str().unwrap().to_owned())
}
async fn cli(
directory: &Path,
secrets: &Path,
endpoint: &str,
action: &str,
) -> std::process::Output {
tokio::time::timeout(
Duration::from_secs(5),
Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.arg("--json")
.arg("--data-dir")
.arg(directory)
.arg("--test-secrets-file")
.arg(secrets)
.args(["--endpoint", endpoint, "system", action])
.output(),
)
.await
.unwrap()
.unwrap()
}
#[tokio::test]
async fn actual_cli_authenticates_and_restarts_actual_daemon_without_os_keyring() {
let temporary = tempfile::tempdir().unwrap();
let directory: PathBuf = temporary.path().join("profile");
let secrets = temporary.path().join("synthetic-secrets.json");
let (mut child, endpoint) = start(&directory, &secrets).await;
let status = cli(&directory, &secrets, &endpoint, "status").await;
assert_eq!(status.status.code(), Some(0));
let value: serde_json::Value = serde_json::from_slice(&status.stdout).unwrap();
let id = value["result"]["profile_id"].as_str().unwrap().to_owned();
assert_eq!(value["result"]["api_version"], "nuncio.v2");
assert!(!String::from_utf8_lossy(&status.stdout).contains("Bearer"));
let mut wrong: serde_json::Value =
serde_json::from_slice(&std::fs::read(&secrets).unwrap()).unwrap();
wrong[format!("{id}/profile/api")] = serde_json::Value::String("00".repeat(32));
let wrong_path = temporary.path().join("wrong-secrets.json");
std::fs::write(&wrong_path, serde_json::to_vec(&wrong).unwrap()).unwrap();
assert_eq!(
cli(&directory, &wrong_path, &endpoint, "status")
.await
.status
.code(),
Some(3)
);
assert!(cli(&directory, &secrets, &endpoint, "shutdown")
.await
.status
.success());
assert!(tokio::time::timeout(Duration::from_secs(5), child.wait())
.await
.unwrap()
.unwrap()
.success());
assert_eq!(
cli(&directory, &secrets, &endpoint, "status")
.await
.status
.code(),
Some(4)
);
let (mut child, endpoint) = start(&directory, &secrets).await;
let status = cli(&directory, &secrets, &endpoint, "status").await;
assert_eq!(
serde_json::from_slice::<serde_json::Value>(&status.stdout).unwrap()["result"]
["profile_id"],
id
);
assert!(cli(&directory, &secrets, &endpoint, "shutdown")
.await
.status
.success());
assert!(tokio::time::timeout(Duration::from_secs(5), child.wait())
.await
.unwrap()
.unwrap()
.success());
}
+205
View File
@@ -0,0 +1,205 @@
#![allow(clippy::unwrap_used)]
use std::process::Command;
#[test]
fn unreachable_daemon_returns_json_error_and_nonzero_exit() {
let output = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.args([
"--json",
"--endpoint",
"http://127.0.0.1:1",
"system",
"status",
])
.output()
.unwrap();
assert_eq!(output.status.code(), Some(4));
let value: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(value["error"]["code"], "unavailable");
assert!(value.get("result").is_none());
}
#[test]
fn invalid_input_is_an_error_and_help_succeeds() {
let invalid = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.args(["--json", "made-up-command"])
.output()
.unwrap();
assert_eq!(invalid.status.code(), Some(2));
let value: serde_json::Value = serde_json::from_slice(&invalid.stdout).unwrap();
assert_eq!(value["error"]["code"], "invalid_input");
let help = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.arg("--help")
.output()
.unwrap();
assert!(help.status.success());
}
#[test]
fn account_without_an_action_shows_help_before_connecting_or_creating_a_profile() {
let temp = tempfile::tempdir().unwrap();
let data = temp.path().join("unused-profile");
for action in [vec!["account"], vec!["account", "--help"]] {
let explicit_help = action.len() == 2;
let output = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.args(["--endpoint", "http://127.0.0.1:1", "--data-dir"])
.arg(&data)
.args(action)
.output()
.unwrap();
assert_eq!(
output.status.code(),
Some(if explicit_help { 0 } else { 2 })
);
let help = if explicit_help {
assert!(output.stderr.is_empty());
String::from_utf8(output.stdout).unwrap()
} else {
assert!(output.stdout.is_empty());
String::from_utf8(output.stderr).unwrap()
};
assert!(help.contains("Usage: nuncio-cli account"), "{help}");
assert!(help.contains("Commands:"), "{help}");
assert!(!data.exists());
}
}
#[test]
fn account_without_an_action_preserves_json_errors_for_scripts() {
for args in [["--json", "account"], ["account", "--json"]] {
let output = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.args(args)
.output()
.unwrap();
assert_eq!(output.status.code(), Some(2));
let value: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(value["error"]["code"], "invalid_input");
assert!(value.get("result").is_none());
}
}
#[test]
fn missing_mail_account_explains_selection_without_creating_a_profile() {
let temp = tempfile::tempdir().unwrap();
let data = temp.path().join("unused-profile");
let output = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.args(["--profile", "laptop-qa", "--data-dir"])
.arg(&data)
.args(["mail", "list"])
.output()
.unwrap();
assert_eq!(output.status.code(), Some(2));
assert!(output.stdout.is_empty());
let error = String::from_utf8(output.stderr).unwrap();
assert!(error.contains("--account <ACCOUNT_ID>"), "{error}");
assert!(error.contains("account list"), "{error}");
assert!(error.contains("Usage: nuncio-cli mail list"), "{error}");
assert!(!error.contains("schema_version"));
assert!(!data.exists());
}
#[test]
fn normal_runtime_errors_are_actionable_text_and_json_remains_explicit() {
let output = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.args(["--endpoint", "http://127.0.0.1:1", "system", "status"])
.output()
.unwrap();
assert_eq!(output.status.code(), Some(4));
assert!(output.stdout.is_empty());
let error = String::from_utf8(output.stderr).unwrap();
assert!(error.contains("nunciod"), "{error}");
assert!(error.contains("--profile"), "{error}");
assert!(!error.contains("schema_version"));
}
#[test]
fn helpful_parse_errors_never_echo_untrusted_argument_values() {
let secret = "private-token-DO-NOT-ECHO\u{1b}[2J";
for args in [
vec![
"mail",
"list",
"--account",
"local-account",
"--page-size",
secret,
],
vec!["mail", secret],
vec!["mail", "list", secret],
] {
let output = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.args(args)
.output()
.unwrap();
assert_eq!(output.status.code(), Some(2));
assert!(output.stdout.is_empty());
let error = String::from_utf8(output.stderr).unwrap();
assert!(error.contains("Usage:"), "{error}");
assert!(!error.contains("private-token"));
assert!(!error.contains('\u{1b}'));
}
}
#[test]
fn write_request_ids_explain_the_required_uuid_before_connecting() {
let output = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.args([
"mail",
"send",
"--account",
"account",
"--draft",
"draft",
"--request-id",
"send-1",
])
.output()
.unwrap();
assert_eq!(output.status.code(), Some(2));
assert!(output.stdout.is_empty());
let error = String::from_utf8(output.stderr).unwrap();
assert!(error.contains("--request-id"), "{error}");
assert!(error.contains("UUID"), "{error}");
}
#[test]
fn misspelled_options_suggest_declared_names_without_echoing_supplied_values() {
let temp = tempfile::tempdir().unwrap();
let data = temp.path().join("unused-profile");
for option in ["--acount", "--acount=private-token-canary"] {
let output = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.args(["--data-dir"])
.arg(&data)
.args(["mail", "list", option, "private-value-canary"])
.output()
.unwrap();
assert_eq!(output.status.code(), Some(2));
assert!(output.stdout.is_empty());
let error = String::from_utf8(output.stderr).unwrap();
assert!(error.contains("Did you mean '--account'?"), "{error}");
assert!(!error.contains("private-token-canary"));
assert!(!error.contains("private-value-canary"));
assert!(!data.exists());
}
let output = Command::new(env!("CARGO_BIN_EXE_nuncio-cli"))
.env_clear()
.args(["--json", "mail", "list", "--acount", "private-value-canary"])
.output()
.unwrap();
assert_eq!(output.status.code(), Some(2));
let value: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(value["error"]["code"], "invalid_input");
assert!(!String::from_utf8(output.stdout)
.unwrap()
.contains("private-value-canary"));
}
+45
View File
@@ -0,0 +1,45 @@
#![allow(clippy::unwrap_used)]
#[test]
fn repair_exposes_explicit_scope_dry_run_and_wait_without_ambiguous_date_options() {
let binary = env!("CARGO_BIN_EXE_nuncio-cli");
let help = std::process::Command::new(binary)
.args(["repair", "--help"])
.output()
.unwrap();
assert!(help.status.success());
let text = String::from_utf8(help.stdout).unwrap();
for option in ["--scope", "--dry-run", "--wait", "--from", "--to"] {
assert!(text.contains(option));
}
for args in [
vec!["repair", "--account", "synthetic", "--scope", "unknown"],
vec![
"repair",
"--account",
"synthetic",
"--scope",
"mail",
"--dry-run",
"--wait",
],
vec![
"repair",
"--account",
"synthetic",
"--scope",
"calendar",
"--from",
"2026-10-01",
],
] {
assert_eq!(
std::process::Command::new(binary)
.args(args)
.output()
.unwrap()
.status
.code(),
Some(2)
);
}
}
+51
View File
@@ -0,0 +1,51 @@
[package]
name = "nuncio-engine"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
[lints]
workspace = true
[features]
test-harness = []
[dependencies]
tracing.workspace = true
log.workspace = true
tokio.workspace = true
async-imap.workspace = true
lettre.workspace = true
tokio-rustls.workspace = true
webpki-roots.workspace = true
thiserror.workspace = true
serde.workspace = true
serde_json.workspace = true
zeroize.workspace = true
rusqlite.workspace = true
keyring.workspace = true
fs4.workspace = true
rand.workspace = true
hex.workspace = true
uuid.workspace = true
tempfile.workspace = true
rustix.workspace = true
url.workspace = true
reqwest.workspace = true
futures-util.workspace = true
base64.workspace = true
sha2.workspace = true
subtle.workspace = true
axum.workspace = true
hyper.workspace = true
hyper-util.workspace = true
mail-parser.workspace = true
mail-builder.workspace = true
email_address.workspace = true
chrono.workspace = true
chrono-tz.workspace = true
httpdate.workspace = true
[dev-dependencies]
tempfile.workspace = true
@@ -0,0 +1,830 @@
mod imap;
mod management;
use crate::{
providers::google::{
http::{GoogleApi, GoogleHttp, GoogleRead, TokenResponse, UserInfo},
oauth,
},
secrets::SecretStore,
store::{ConnectedAccount, Store, StoredAccount},
};
pub use imap::ImapConnection;
use serde::{Deserialize, Serialize};
use std::{
collections::BTreeMap,
sync::Arc,
time::{Duration, Instant},
};
use tokio::sync::{watch, Mutex};
use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing};
enum GoogleWrite<'a> {
Gmail {
sender: Option<&'a str>,
path: &'a [&'a str],
body: Vec<u8>,
},
Calendar(&'a crate::store::CalendarChangePayload),
}
#[derive(Debug, thiserror::Error)]
pub enum AccountError {
#[error("Account configuration changed; read the latest version")]
VersionConflict,
#[error("Account lifecycle does not permit this action; restore archived accounts and resolve remote-effect uncertainty before deletion")]
Lifecycle,
#[error("Mail server TLS verification failed")]
Tls,
#[error("Mail server does not support a required protocol capability")]
Unsupported,
#[error("Invalid account configuration")]
Invalid,
#[error("Account or authorization session was not found")]
NotFound,
#[error("Account authorization is required")]
Authorization,
#[error("Required Google permissions were not granted")]
ScopeDenied,
#[error("Provider account identity does not match the saved account")]
IdentityMismatch,
#[error("Provider service is unavailable")]
Unavailable,
#[error("Provider service requested a later attempt")]
RetryAfter(i64),
#[error("Provider returned an invalid response")]
Provider,
#[error("Secure credential storage is unavailable")]
Secret,
#[error("Account storage is unavailable")]
Storage,
#[error("Too many pending authorization sessions or accounts")]
Limit,
}
impl AccountError {
pub fn code(&self) -> &'static str {
match self {
Self::VersionConflict => "version_conflict",
Self::Lifecycle => "account_lifecycle",
Self::Tls => "tls_verification",
Self::Unsupported => "unsupported",
Self::Invalid => "invalid_input",
Self::NotFound => "not_found",
Self::Authorization => "authorization_required",
Self::ScopeDenied => "scope_denied",
Self::IdentityMismatch => "identity_mismatch",
Self::Unavailable | Self::RetryAfter(_) => "unavailable",
Self::Provider => "invalid_provider_response",
Self::Secret => "credential_storage",
Self::Storage => "storage",
Self::Limit => "limit",
}
}
}
impl From<crate::store::StoreError> for AccountError {
fn from(error: crate::store::StoreError) -> Self {
match error {
crate::store::StoreError::NotFound => Self::NotFound,
crate::store::StoreError::VersionConflict => Self::VersionConflict,
crate::store::StoreError::AccountLifecycle => Self::Lifecycle,
crate::store::StoreError::InvalidInput | crate::store::StoreError::InvalidAccount => {
Self::Invalid
}
crate::store::StoreError::Busy => Self::Unavailable,
_ => Self::Storage,
}
}
}
#[derive(Serialize, Deserialize, Zeroize, ZeroizeOnDrop)]
#[serde(deny_unknown_fields)]
pub struct GoogleRegistration {
pub client_id: String,
#[serde(default)]
pub client_secret: Option<String>,
}
impl GoogleRegistration {
fn validate(&self, synthetic: bool) -> Result<(), AccountError> {
let valid_id = if synthetic {
self.client_id == "nuncio-test-client"
} else {
self.client_id.ends_with(".apps.googleusercontent.com")
&& self.client_id.len() <= 256
&& self
.client_id
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'.' || b == b'-')
};
if !valid_id
|| self.client_secret.as_ref().is_some_and(|s| {
s.is_empty()
|| s.len() > 4096
|| s.chars().any(char::is_control)
|| synthetic && s != "synthetic-client-secret"
})
{
return Err(AccountError::Invalid);
}
Ok(())
}
}
pub struct GoogleAuthRequest {
pub registration: GoogleRegistration,
pub login_hint: Option<String>,
pub account: Option<String>,
}
#[derive(Clone, Serialize)]
pub struct AuthSession {
pub session_id: String,
pub browser_url: String,
pub expires_at_ms: i64,
pub state: String,
pub account_id: Option<String>,
pub error_code: Option<String>,
pub warning_code: Option<String>,
}
#[derive(Clone, Serialize)]
pub struct Account {
pub id: String,
pub provider: String,
pub address: String,
pub state: String,
pub display_name: String,
pub version: u64,
pub auth_state: String,
pub identity: Option<String>,
}
impl From<StoredAccount> for Account {
fn from(row: StoredAccount) -> Self {
Self {
id: row.account.id,
provider: row.account.provider,
address: row.account.address,
state: row.state,
display_name: row.display_name,
version: row.version,
auth_state: row.auth_state,
identity: row.subject,
}
}
}
#[derive(Serialize, Deserialize, Zeroize, ZeroizeOnDrop)]
pub(crate) struct Credentials {
pub client_id: String,
pub client_secret: Option<String>,
pub refresh_token: String,
}
struct Access {
token: Zeroizing<String>,
expires: Instant,
}
pub(crate) struct SessionSlot {
pub status: AuthSession,
pub expected_account: Option<String>,
pub cancel: watch::Sender<bool>,
}
pub(crate) struct Accounts {
store: Store,
profile: String,
secrets: Arc<dyn SecretStore>,
pub http: GoogleHttp,
pub sessions: Mutex<BTreeMap<String, SessionSlot>>,
gates: Mutex<BTreeMap<String, Arc<Mutex<Option<Access>>>>>,
mutations: Mutex<()>,
pub stop: watch::Sender<bool>,
pub tasks: Mutex<Vec<tokio::task::JoinHandle<()>>>,
}
impl Accounts {
pub async fn new(
store: Store,
profile: String,
secrets: Arc<dyn SecretStore>,
http: GoogleHttp,
) -> Result<Arc<Self>, AccountError> {
let service = Arc::new(Self {
store,
profile,
secrets,
http,
sessions: Mutex::new(BTreeMap::new()),
gates: Mutex::new(BTreeMap::new()),
mutations: Mutex::new(()),
stop: watch::channel(false).0,
tasks: Mutex::new(Vec::new()),
});
service.cleanup().await?;
Ok(service)
}
pub async fn begin(
self: &Arc<Self>,
request: GoogleAuthRequest,
) -> Result<AuthSession, AccountError> {
let _mutation = self.mutations.lock().await;
request.registration.validate(self.http.synthetic)?;
if request
.login_hint
.as_ref()
.is_some_and(|s| s.is_empty() || s.len() > 320 || s.chars().any(char::is_control))
{
return Err(AccountError::Invalid);
}
if let Some(id) = &request.account {
let row = self.row(id).await?;
if row.archived {
return Err(AccountError::Lifecycle);
}
if row.account.provider != "google" || row.subject.is_none() {
return Err(AccountError::Invalid);
}
}
oauth::begin(self.clone(), request).await
}
pub async fn auth_status(&self, id: &str) -> Result<AuthSession, AccountError> {
validate_id(id)?;
self.sessions
.lock()
.await
.get(id)
.map(|s| s.status.clone())
.ok_or(AccountError::NotFound)
}
pub async fn list(&self) -> Result<Vec<Account>, AccountError> {
self.list_including_archived(false).await
}
pub async fn list_including_archived(
&self,
include_archived: bool,
) -> Result<Vec<Account>, AccountError> {
let _mutation = self.mutations.lock().await;
let mut result = Vec::new();
for row in self.store.accounts().await? {
let row = self.row(&row.id).await?;
if include_archived || !row.archived {
result.push(row.into());
}
}
Ok(result)
}
pub async fn disconnect(&self, id: &str) -> Result<(), AccountError> {
self.row(id).await?;
let gate = self.gate(id).await;
let mut access = gate.lock().await;
let _mutation = self.mutations.lock().await;
self.row(id).await?;
for slot in self.sessions.lock().await.values_mut() {
if slot.expected_account.as_deref() == Some(id)
&& matches!(slot.status.state.as_str(), "pending" | "completing")
{
slot.status.state = "cancelled".into();
slot.status.browser_url.clear();
slot.cancel.send_replace(true);
}
}
self.store
.set_account_state(id.into(), "disconnected".into())
.await?;
*access = None;
tracing::info!(account_id = id, "Account disconnected");
self.cleanup().await
}
pub async fn check(&self, id: &str) -> Result<Account, AccountError> {
self.row(id).await?;
let gate = self.gate(id).await;
let mut access = gate.lock().await;
let row = self.row(id).await?;
if row.account.provider == "imap" {
return self.check_imap(row).await;
}
if row.account.provider != "google" {
return Err(AccountError::Invalid);
}
if row.state != "connected" {
return Err(AccountError::Authorization);
}
for scope in ["gmail", "calendar"] {
if let Some(deadline) = self
.store
.provider_retry_deadline(id.into(), scope.into())
.await?
{
if deadline > self.http.clock.now_ms() {
return Err(AccountError::RetryAfter(deadline));
}
}
}
let reference = row
.credential_ref
.as_ref()
.ok_or(AccountError::Authorization)?;
let result = async {
if access.as_ref().is_none_or(|a| a.expires <= Instant::now()) {
*access = Some(self.refresh(reference).await?);
}
let current = access.as_ref().ok_or(AccountError::Authorization)?;
let info = match self.http.userinfo(&current.token).await {
Err(AccountError::Authorization) => {
*access = Some(self.refresh(reference).await?);
self.http
.userinfo(&access.as_ref().ok_or(AccountError::Authorization)?.token)
.await?
}
other => other?,
};
if row.subject.as_deref() != Some(&info.sub) {
return Err(AccountError::IdentityMismatch);
}
Ok(())
}
.await;
if matches!(
result,
Err(AccountError::Authorization
| AccountError::ScopeDenied
| AccountError::IdentityMismatch)
) {
let _mutation = self.mutations.lock().await;
self.store
.set_account_state(id.into(), "needs_auth".into())
.await?;
*access = None;
self.cleanup().await?;
}
if let Err(AccountError::RetryAfter(deadline)) = &result {
self.auth_retry_after(id, *deadline).await?;
}
result?;
Ok(row.into())
}
async fn refresh(&self, reference: &str) -> Result<Access, AccountError> {
let bytes = self
.secret_get(reference)
.await?
.ok_or(AccountError::Authorization)?;
let mut credentials: Credentials =
serde_json::from_slice(&bytes).map_err(|_| AccountError::Secret)?;
let response = self.http.refresh(&credentials).await?;
if let Some(refresh) = &response.refresh_token {
credentials.refresh_token.zeroize();
credentials.refresh_token = refresh.clone();
self.secret_put(
reference,
Zeroizing::new(serde_json::to_vec(&credentials).map_err(|_| AccountError::Secret)?),
)
.await?;
}
Ok(access(&response))
}
pub async fn gmail_get(
&self,
id: &str,
path: &[&str],
query: &[(&str, &str)],
limit: usize,
) -> Result<serde_json::Value, crate::mail::MailError> {
self.google_get(id, GoogleApi::Gmail, path, query, limit)
.await
}
pub async fn gmail_write(
&self,
id: &str,
sender: Option<&str>,
path: &[&str],
body: Vec<u8>,
retry_at_ms: i64,
) -> Result<crate::providers::google::send::WriteResult, AccountError> {
self.google_write(id, GoogleWrite::Gmail { sender, path, body }, retry_at_ms)
.await
}
pub async fn calendar_write(
&self,
id: &str,
intent: &crate::store::CalendarChangePayload,
retry_at_ms: i64,
) -> Result<crate::providers::google::send::WriteResult, AccountError> {
self.google_write(id, GoogleWrite::Calendar(intent), retry_at_ms)
.await
}
async fn google_write(
&self,
id: &str,
request: GoogleWrite<'_>,
retry_at_ms: i64,
) -> Result<crate::providers::google::send::WriteResult, AccountError> {
use crate::providers::google::send::WriteResult;
let (api, sender) = match &request {
GoogleWrite::Gmail { sender, .. } => (GoogleApi::Gmail, *sender),
GoogleWrite::Calendar(_) => (GoogleApi::Calendar, None),
};
let gate = self.gate(id).await;
let mut cached = gate.lock().await;
let row = self.row(id).await?;
if row.account.provider != "google" {
return Err(AccountError::Invalid);
}
if sender.is_some_and(|sender| !row.account.address.eq_ignore_ascii_case(sender)) {
return Err(AccountError::IdentityMismatch);
}
if row.state != "connected" {
return Err(AccountError::Authorization);
}
if let Some(at) = self
.store
.provider_retry_deadline(id.into(), api.scope().into())
.await?
{
if at > self.http.clock.now_ms() {
return Err(AccountError::RetryAfter(at));
}
}
let reference = row
.credential_ref
.as_deref()
.ok_or(AccountError::Authorization)?;
let mut result = async {
if cached.as_ref().is_none_or(|a| a.expires <= Instant::now()) {
*cached = Some(self.refresh(reference).await?);
}
let current = cached.as_ref().ok_or(AccountError::Authorization)?;
Ok::<_, AccountError>(match request {
GoogleWrite::Gmail { path, body, .. } => {
self.http
.write_gmail(&current.token, path, body, retry_at_ms)
.await
}
GoogleWrite::Calendar(intent) => {
self.http
.write_calendar(&current.token, intent, retry_at_ms)
.await
}
})
}
.await;
if let Ok(WriteResult::Rejected {
code: "authorization_required",
retry_at_ms,
..
}) = &result
{
let retry_at_ms = *retry_at_ms;
result = match self.refresh(reference).await {
Ok(access) => {
*cached = Some(access);
Ok(WriteResult::Rejected {
code: "access_token_rejected",
retry_at_ms,
authorization: false,
})
}
Err(error) => Err(error),
};
}
if matches!(
&result,
Ok(WriteResult::Rejected {
authorization: true,
..
}) | Err(AccountError::Authorization | AccountError::ScopeDenied)
) {
let _mutation = self.mutations.lock().await;
self.store
.set_account_state(id.into(), "needs_auth".into())
.await?;
*cached = None;
}
match &result {
Ok(outcome) => {
if let Some(at) = outcome.retry_deadline() {
if self
.store
.provider_retry_after(id.into(), api.scope().into(), at)
.await
.is_err()
{
return Ok(WriteResult::Uncertain {
code: "post_submission_storage_failed",
retry_at_ms: Some(at),
});
}
}
}
Err(AccountError::RetryAfter(at)) => self.auth_retry_after(id, *at).await?,
_ => {}
}
result
}
pub async fn calendar_get(
&self,
id: &str,
path: &[&str],
query: &[(&str, &str)],
limit: usize,
) -> Result<serde_json::Value, crate::sync_error::SyncError> {
self.google_get(id, GoogleApi::Calendar, path, query, limit)
.await
}
async fn google_get(
&self,
id: &str,
api: GoogleApi,
path: &[&str],
query: &[(&str, &str)],
limit: usize,
) -> Result<serde_json::Value, crate::sync_error::SyncError> {
self.google_read(id, api, GoogleRead::Get { path, query, limit })
.await
}
pub async fn calendar_free_busy(
&self,
id: &str,
body: &serde_json::Value,
) -> Result<serde_json::Value, crate::sync_error::SyncError> {
self.google_read(id, GoogleApi::Calendar, GoogleRead::FreeBusy(body))
.await
}
async fn google_read(
&self,
id: &str,
api: GoogleApi,
input: GoogleRead<'_>,
) -> Result<serde_json::Value, crate::sync_error::SyncError> {
use crate::mail::MailError;
self.row(id).await?;
let gate = self.gate(id).await;
let mut access = gate.lock().await;
let row = self.row(id).await?;
if row.account.provider != "google" {
return Err(AccountError::Invalid.into());
}
if row.state != "connected" {
return Err(AccountError::Authorization.into());
}
if let Some(deadline) = self
.store
.provider_retry_deadline(id.into(), api.scope().into())
.await?
{
if deadline > self.http.clock.now_ms() {
return Err(MailError::RetryAfter(deadline));
}
}
let reference = row
.credential_ref
.as_ref()
.ok_or(AccountError::Authorization)?;
let result = async {
if access.as_ref().is_none_or(|a| a.expires <= Instant::now()) {
*access = Some(self.refresh(reference).await?);
}
let current = access.as_ref().ok_or(AccountError::Authorization)?;
match self.http.read(api, &current.token, input).await {
Err(MailError::Account(AccountError::Authorization)) => {
*access = Some(self.refresh(reference).await?);
self.http
.read(
api,
&access.as_ref().ok_or(AccountError::Authorization)?.token,
input,
)
.await
}
other => other,
}
}
.await;
if matches!(
result,
Err(MailError::Account(
AccountError::Authorization | AccountError::ScopeDenied
))
) {
let _mutation = self.mutations.lock().await;
self.store
.set_account_state(id.into(), "needs_auth".into())
.await?;
*access = None;
self.cleanup().await?;
}
match &result {
Err(MailError::RetryAfter(deadline)) => {
self.store
.provider_retry_after(id.into(), api.scope().into(), *deadline)
.await?
}
Err(MailError::Account(AccountError::RetryAfter(deadline))) => {
self.auth_retry_after(id, *deadline).await?
}
_ => {}
}
result
}
async fn auth_retry_after(&self, id: &str, deadline: i64) -> Result<(), AccountError> {
// OAuth is shared by both APIs; a token endpoint throttle applies to both.
for scope in ["gmail", "calendar"] {
self.store
.provider_retry_after(id.into(), scope.into(), deadline)
.await?;
}
Ok(())
}
pub async fn finish(
&self,
session: &str,
request: &GoogleAuthRequest,
info: UserInfo,
token: TokenResponse,
) -> Result<(String, bool), AccountError> {
let id = if let Some(id) = &request.account {
id.clone()
} else {
let mut found = None;
for row in self.store.accounts().await? {
let row = self.row(&row.id).await?;
if row.account.provider == "google" && row.subject.as_deref() == Some(&info.sub) {
found = Some(row.account.id);
break;
}
}
found.unwrap_or_else(|| uuid::Uuid::new_v4().to_string())
};
let gate = self.gate(&id).await;
let mut cached = gate.lock().await;
let _mutation = self.mutations.lock().await;
if self
.sessions
.lock()
.await
.get(session)
.is_none_or(|s| s.status.state != "completing")
{
return Err(AccountError::Authorization);
}
if let Some(row) = self.store.account(id.clone()).await? {
if row.archived {
return Err(AccountError::Lifecycle);
}
if row.account.provider != "google" || row.subject.as_deref() != Some(&info.sub) {
return Err(AccountError::IdentityMismatch);
}
} else if request.account.is_some() {
return Err(AccountError::NotFound);
} else if self.store.status().await?.account_count >= 100 {
return Err(AccountError::Limit);
}
let reference = format!(
"{}/account/{id}/google/{}",
self.profile,
uuid::Uuid::new_v4()
);
let credentials = Credentials {
client_id: request.registration.client_id.clone(),
client_secret: request.registration.client_secret.clone(),
refresh_token: token
.refresh_token
.as_ref()
.ok_or(AccountError::Authorization)?
.clone(),
};
self.store.prepare_credential(reference.clone()).await?;
let saved = self
.secret_put(
&reference,
Zeroizing::new(serde_json::to_vec(&credentials).map_err(|_| AccountError::Secret)?),
)
.await;
if let Err(error) = saved {
let _ = self.cleanup().await;
return Err(error);
}
if let Err(error) = self
.store
.connect_google(ConnectedAccount {
id: id.clone(),
subject: info.sub,
address: info.email,
credential_ref: reference,
})
.await
{
let _ = self.cleanup().await;
return Err(error.into());
}
*cached = Some(access(&token));
// The new account reference is already committed. A failed deletion of an
// obsolete secret remains durably queued; report that independently.
let cleanup_pending = self.cleanup().await.is_err();
if let Some(slot) = self.sessions.lock().await.get_mut(session) {
slot.status.state = "succeeded".into();
slot.status.account_id = Some(id.clone());
slot.status.warning_code = cleanup_pending.then(|| "credential_cleanup_pending".into());
}
tracing::info!(account_id = %id, provider = "google", cleanup_pending, "Account connected");
Ok((id, cleanup_pending))
}
async fn row(&self, id: &str) -> Result<StoredAccount, AccountError> {
validate_id(id)?;
self.store
.account(id.into())
.await?
.ok_or(AccountError::NotFound)
}
async fn gate(&self, id: &str) -> Arc<Mutex<Option<Access>>> {
self.gates
.lock()
.await
.entry(id.into())
.or_insert_with(|| Arc::new(Mutex::new(None)))
.clone()
}
fn validate_reference(&self, reference: &str) -> Result<(), AccountError> {
let parts: Vec<_> = reference.split('/').collect();
if parts.len() != 5
|| parts[0] != self.profile
|| parts[1] != "account"
|| !matches!(parts[3], "google" | "imap")
|| uuid::Uuid::parse_str(parts[2]).is_err()
|| uuid::Uuid::parse_str(parts[4]).is_err()
{
return Err(AccountError::Secret);
}
Ok(())
}
async fn secret_get(
&self,
reference: &str,
) -> Result<Option<Zeroizing<Vec<u8>>>, AccountError> {
self.validate_reference(reference)?;
let secrets = self.secrets.clone();
let reference = reference.to_owned();
let value = tokio::task::spawn_blocking(move || secrets.get(&reference))
.await
.map_err(|_| AccountError::Secret)?
.map_err(|_| AccountError::Secret)?;
if value.as_ref().is_some_and(|v| v.len() > 65536) {
return Err(AccountError::Secret);
}
Ok(value)
}
async fn secret_put(
&self,
reference: &str,
value: Zeroizing<Vec<u8>>,
) -> Result<(), AccountError> {
self.validate_reference(reference)?;
let secrets = self.secrets.clone();
let reference = reference.to_owned();
tokio::task::spawn_blocking(move || secrets.put(&reference, &value))
.await
.map_err(|_| AccountError::Secret)?
.map_err(|_| AccountError::Secret)
}
async fn cleanup(&self) -> Result<(), AccountError> {
let pending = self.store.credential_cleanup().await?;
let total = pending.len();
let started = Instant::now();
tracing::info!(pending = total, "Credential cleanup started");
for (index, reference) in pending.into_iter().enumerate() {
let deletion_started = Instant::now();
tracing::info!(
completed = index,
remaining = total - index,
"Deleting obsolete credential"
);
self.validate_reference(&reference)?;
let secrets = self.secrets.clone();
let name = reference.clone();
tokio::task::spawn_blocking(move || secrets.delete(&name))
.await
.map_err(|_| AccountError::Secret)?
.map_err(|_| AccountError::Secret)?;
self.store.finish_credential_cleanup(reference).await?;
tracing::info!(
completed = index + 1,
remaining = total - index - 1,
elapsed_ms = deletion_started.elapsed().as_millis() as u64,
"Credential deletion complete"
);
}
tracing::info!(
completed = total,
elapsed_ms = started.elapsed().as_millis() as u64,
"Credential cleanup complete"
);
Ok(())
}
pub async fn shutdown(&self) {
self.stop.send_replace(true);
for task in self.tasks.lock().await.drain(..) {
let _ = task.await;
}
}
}
fn access(response: &TokenResponse) -> Access {
Access {
token: Zeroizing::new(response.access_token.clone()),
expires: Instant::now() + Duration::from_secs(response.expires_in.saturating_sub(30)),
}
}
fn validate_id(id: &str) -> Result<(), AccountError> {
uuid::Uuid::parse_str(id)
.map(|_| ())
.map_err(|_| AccountError::Invalid)
}
@@ -0,0 +1,306 @@
use super::{Account, AccountError, Accounts};
use crate::{
domain::imap_account::{ImapAccountConfig, ImapCapabilities, ImapCredentials},
providers::imap::probe,
store::{ConnectedAccount, StoredAccount},
};
use zeroize::Zeroizing;
pub struct ImapConnection {
pub account: Account,
pub capabilities: ImapCapabilities,
pub credential_cleanup_pending: bool,
}
impl Accounts {
pub(crate) async fn imap_session(
&self,
id: &str,
) -> Result<crate::providers::imap::Connection, AccountError> {
let gate = self.gate(id).await;
let _guard = gate.lock().await;
let row = self.row(id).await?;
if row.account.provider != "imap" {
return Err(AccountError::Invalid);
}
if row.state != "connected" {
return Err(AccountError::Authorization);
}
let secret = self
.secret_get(
row.credential_ref
.as_deref()
.ok_or(AccountError::Authorization)?,
)
.await?
.ok_or(AccountError::Authorization)?;
let credentials: ImapCredentials =
serde_json::from_slice(&secret).map_err(|_| AccountError::Secret)?;
credentials.validate().map_err(|_| AccountError::Secret)?;
let (config, previous) = self.store.imap_config(id.into()).await?;
if config
.identity()
.map_err(|_| AccountError::Storage)?
.as_str()
!= row.subject.as_deref().ok_or(AccountError::Storage)?
{
return Err(AccountError::IdentityMismatch);
}
match crate::providers::imap::open(&config, &credentials, self.http.resources.clone())
.await
.map_err(AccountError::from)
{
Ok(connection) => {
let mut capabilities = connection.capabilities.clone();
capabilities.smtp_utf8 = previous.smtp_utf8;
capabilities.eight_bit_mime = previous.eight_bit_mime;
self.store
.update_imap_capabilities(id.into(), capabilities)
.await?;
Ok(connection)
}
Err(AccountError::Authorization) => {
self.store
.set_account_state(id.into(), "needs_auth".into())
.await?;
self.cleanup().await?;
Err(AccountError::Authorization)
}
Err(error) => Err(error),
}
}
pub(crate) async fn smtp_session(
&self,
id: &str,
expected: &crate::domain::imap_account::MailEndpoint,
) -> Result<crate::providers::imap::smtp::Session, AccountError> {
let gate = self.gate(id).await;
let _guard = gate.lock().await;
let row = self.row(id).await?;
if row.account.provider != "imap" {
return Err(AccountError::Invalid);
}
if row.state != "connected" {
return Err(AccountError::Authorization);
}
let (config, _) = self.store.imap_config(id.into()).await?;
if &config.smtp != expected
|| config
.identity()
.map_err(|_| AccountError::Storage)?
.as_str()
!= row.subject.as_deref().ok_or(AccountError::Storage)?
{
return Err(AccountError::IdentityMismatch);
}
let secret = self
.secret_get(
row.credential_ref
.as_deref()
.ok_or(AccountError::Authorization)?,
)
.await?
.ok_or(AccountError::Authorization)?;
let credentials: ImapCredentials =
serde_json::from_slice(&secret).map_err(|_| AccountError::Secret)?;
credentials.validate().map_err(|_| AccountError::Secret)?;
let result =
crate::providers::imap::open_smtp(&config, &credentials, self.http.resources.clone())
.await
.map_err(AccountError::from);
if matches!(result, Err(AccountError::Authorization)) {
self.store
.set_account_state(id.into(), "needs_auth".into())
.await?;
self.cleanup().await?;
}
result
}
pub async fn imap_config(
&self,
id: String,
) -> Result<(ImapAccountConfig, ImapCapabilities), AccountError> {
let row = self.row(&id).await?;
if row.account.provider != "imap" {
return Err(AccountError::Invalid);
}
Ok(self.store.imap_config(id).await?)
}
pub async fn connect_imap(
&self,
config: ImapAccountConfig,
credentials: ImapCredentials,
requested: Option<String>,
) -> Result<ImapConnection, AccountError> {
self.connect_imap_inner(config, credentials, requested, None)
.await
}
pub async fn update_imap(
&self,
id: String,
version: u64,
config: ImapAccountConfig,
credentials: Option<ImapCredentials>,
) -> Result<ImapConnection, AccountError> {
let row = self.row(&id).await?;
if row.version != version {
return Err(AccountError::VersionConflict);
}
let credentials = match credentials {
Some(credentials) => credentials,
None => {
let reference = row.credential_ref.ok_or(AccountError::Authorization)?;
let secret = self
.secret_get(&reference)
.await?
.ok_or(AccountError::Authorization)?;
serde_json::from_slice(&secret).map_err(|_| AccountError::Secret)?
}
};
self.connect_imap_inner(config, credentials, Some(id), Some(version))
.await
}
async fn connect_imap_inner(
&self,
config: ImapAccountConfig,
credentials: ImapCredentials,
requested: Option<String>,
expected_version: Option<u64>,
) -> Result<ImapConnection, AccountError> {
credentials.validate().map_err(|_| AccountError::Invalid)?;
let reconnect = requested.is_some();
let identity = config.identity().map_err(|_| AccountError::Invalid)?;
let id = if let Some(id) = requested {
let row = self.row(&id).await?;
if row.account.provider != "imap" || row.subject.as_deref() != Some(&identity) {
return Err(AccountError::IdentityMismatch);
}
id
} else {
let mut found = None;
for account in self.store.accounts().await? {
let row = self.row(&account.id).await?;
if row.account.provider == "imap" && row.subject.as_deref() == Some(&identity) {
found = Some(account.id);
break;
}
}
found.unwrap_or_else(|| uuid::Uuid::new_v4().to_string())
};
let gate = self.gate(&id).await;
let _credentials = gate.lock().await;
let _mutation = self.mutations.lock().await;
if let Some(row) = self.store.account(id.clone()).await? {
if expected_version.is_some_and(|version| row.version != version) {
return Err(AccountError::VersionConflict);
}
if row.archived {
return Err(AccountError::Lifecycle);
}
if row.account.provider != "imap" || row.subject.as_deref() != Some(&identity) {
return Err(AccountError::IdentityMismatch);
}
} else if reconnect {
return Err(AccountError::NotFound);
} else if self.store.status().await?.account_count >= 100 {
return Err(AccountError::Limit);
}
let mut stop = self.stop.subscribe();
if *stop.borrow() {
return Err(AccountError::Unavailable);
}
let capabilities = tokio::select! {
_=stop.changed()=>return Err(AccountError::Unavailable),
result=probe(&config,&credentials,self.http.resources.clone())=>result.map_err(AccountError::from)?,
};
let reference = format!(
"{}/account/{id}/imap/{}",
self.profile,
uuid::Uuid::new_v4()
);
self.store.prepare_credential(reference.clone()).await?;
let saved = self
.secret_put(
&reference,
Zeroizing::new(serde_json::to_vec(&credentials).map_err(|_| AccountError::Secret)?),
)
.await;
if let Err(error) = saved {
let _ = self.cleanup().await;
return Err(error);
}
if let Err(error) = self
.store
.connect_imap_versioned(
ConnectedAccount {
id: id.clone(),
subject: identity,
address: config.address.clone(),
credential_ref: reference,
},
config,
capabilities.clone(),
expected_version,
)
.await
{
let _ = self.cleanup().await;
return Err(error.into());
}
let credential_cleanup_pending = self.cleanup().await.is_err();
tracing::info!(account_id = %id, provider = "imap", credential_cleanup_pending, "Account connected");
Ok(ImapConnection {
account: self.row(&id).await?.into(),
capabilities,
credential_cleanup_pending,
})
}
pub(super) async fn check_imap(&self, row: StoredAccount) -> Result<Account, AccountError> {
if row.state != "connected" {
return Err(AccountError::Authorization);
}
let reference = row
.credential_ref
.as_ref()
.ok_or(AccountError::Authorization)?;
let secret = self
.secret_get(reference)
.await?
.ok_or(AccountError::Authorization)?;
let credentials: ImapCredentials =
serde_json::from_slice(&secret).map_err(|_| AccountError::Secret)?;
credentials.validate().map_err(|_| AccountError::Secret)?;
let (config, _) = self.store.imap_config(row.account.id.clone()).await?;
if config
.identity()
.map_err(|_| AccountError::Storage)?
.as_str()
!= row.subject.as_deref().ok_or(AccountError::Storage)?
{
return Err(AccountError::IdentityMismatch);
}
let mut stop = self.stop.subscribe();
if *stop.borrow() {
return Err(AccountError::Unavailable);
}
let result = tokio::select! {
_=stop.changed()=>return Err(AccountError::Unavailable),
result=probe(&config,&credentials,self.http.resources.clone())=>result.map_err(AccountError::from),
};
match result {
Ok(caps) => {
self.store
.update_imap_capabilities(row.account.id.clone(), caps)
.await?;
Ok(row.into())
}
Err(AccountError::Authorization) => {
self.store
.set_account_state(row.account.id, "needs_auth".into())
.await?;
self.cleanup().await?;
Err(AccountError::Authorization)
}
Err(error) => Err(error),
}
}
}
@@ -0,0 +1,122 @@
use super::{Account, AccountError, Accounts, AuthSession};
use crate::store::{AccountLifecycle, AccountPurgePreview};
impl Accounts {
pub async fn show(&self, id: &str) -> Result<Account, AccountError> {
Ok(self.row(id).await?.into())
}
pub async fn edit_name(
&self,
id: &str,
version: u64,
name: String,
) -> Result<Account, AccountError> {
self.row(id).await?;
let gate = self.gate(id).await;
let _access = gate.lock().await;
let _mutation = self.mutations.lock().await;
let account = self
.store
.edit_account_name(id.into(), version, name)
.await?;
tracing::info!(account_id = id, "Account display name updated");
Ok(account.into())
}
pub async fn lifecycle(
&self,
id: &str,
action: AccountLifecycle,
) -> Result<(Account, bool), AccountError> {
self.row(id).await?;
let gate = self.gate(id).await;
let mut access = gate.lock().await;
let _mutation = self.mutations.lock().await;
#[cfg(feature = "test-harness")]
if matches!(action, AccountLifecycle::Archive) {
self.management_checkpoint("account_before_archive").await?;
}
let row = self
.store
.change_account_lifecycle(id.into(), action)
.await?;
let pending = if matches!(action, AccountLifecycle::Archive) {
#[cfg(feature = "test-harness")]
self.management_checkpoint("account_after_archive").await?;
self.cancel_account_sessions(id).await;
*access = None;
self.cleanup().await.is_err()
} else {
false
};
tracing::info!(account_id = id, state = %row.state, archived = row.archived, cleanup_pending = pending, "Account lifecycle changed");
Ok((row.into(), pending))
}
pub async fn purge(
&self,
id: &str,
version: u64,
revision: u64,
) -> Result<(AccountPurgePreview, bool), AccountError> {
self.row(id).await?;
let gate = self.gate(id).await;
let mut access = gate.lock().await;
let _mutation = self.mutations.lock().await;
#[cfg(feature = "test-harness")]
self.management_checkpoint("account_before_purge").await?;
let preview = self
.store
.purge_account(id.into(), version, revision)
.await?;
#[cfg(feature = "test-harness")]
self.management_checkpoint("account_after_purge").await?;
self.cancel_account_sessions(id).await;
*access = None;
let pending = self.cleanup().await.is_err();
tracing::info!(
account_id = id,
cleanup_pending = pending,
"Account permanently deleted"
);
Ok((preview, pending))
}
#[cfg(feature = "test-harness")]
async fn management_checkpoint(&self, name: &str) -> Result<(), AccountError> {
if let Some(config) = &self.http.test_config {
config
.checkpoint(name, self.stop.subscribe())
.await
.map_err(|_| AccountError::Unavailable)?;
}
Ok(())
}
async fn cancel_account_sessions(&self, id: &str) {
for slot in self.sessions.lock().await.values_mut() {
// An untargeted consent flow has no known identity until its callback finishes.
if (slot.expected_account.is_none() || slot.expected_account.as_deref() == Some(id))
&& matches!(slot.status.state.as_str(), "pending" | "completing")
{
slot.status.state = "cancelled".into();
slot.status.browser_url.clear();
slot.cancel.send_replace(true);
}
}
}
pub async fn cancel_auth(&self, id: &str) -> Result<AuthSession, AccountError> {
super::validate_id(id)?;
let _mutation = self.mutations.lock().await;
let mut sessions = self.sessions.lock().await;
let slot = sessions.get_mut(id).ok_or(AccountError::NotFound)?;
if matches!(slot.status.state.as_str(), "pending" | "completing") {
slot.status.state = "cancelled".into();
slot.status.browser_url.clear();
slot.cancel.send_replace(true);
}
Ok(slot.status.clone())
}
}
@@ -0,0 +1,545 @@
use crate::{
accounts::{AccountError, Accounts},
domain::calendar::{AgendaWindow, CalendarObject},
store::{CalendarCatalogEntry, Store, StoreError, StoredCalendar, SyncRun},
sync_error::SyncError,
};
use serde_json::Value;
use std::{
collections::{BTreeMap, BTreeSet},
sync::Arc,
};
use tokio::sync::{watch, Mutex};
struct Job {
window: AgendaWindow,
repair: bool,
stop: watch::Sender<bool>,
task: tokio::task::JoinHandle<()>,
}
struct EventsResult {
cursor: Option<String>,
time_zone: String,
}
pub(crate) struct CalendarSync {
accounts: Arc<Accounts>,
store: Store,
coordinator: Arc<crate::coordination::Coordinator>,
jobs: Mutex<BTreeMap<(String, String), Job>>,
}
impl CalendarSync {
pub fn new(
accounts: Arc<Accounts>,
store: Store,
coordinator: Arc<crate::coordination::Coordinator>,
) -> Arc<Self> {
Arc::new(Self {
accounts,
store,
coordinator,
jobs: Mutex::new(BTreeMap::new()),
})
}
pub async fn free_busy(
&self,
q: crate::domain::free_busy::FreeBusyRequest,
) -> Result<crate::domain::free_busy::FreeBusyResult, SyncError> {
let body = q.provider_body()?;
let _permit = self.coordinator.acquire(&q.account_id).await?;
let value = self
.accounts
.calendar_free_busy(&q.account_id, &body)
.await?;
let now = self.accounts.http.clock.now_ms();
tokio::task::spawn_blocking(move || {
crate::domain::free_busy::FreeBusyResult::from_google(&q, value, now)
})
.await
.map_err(|_| SyncError::Provider)?
}
pub async fn start(
self: &Arc<Self>,
account: String,
window: Option<AgendaWindow>,
) -> Result<SyncRun, SyncError> {
self.start_mode(account, window, false).await
}
pub async fn repair(
self: &Arc<Self>,
account: String,
window: AgendaWindow,
) -> Result<SyncRun, SyncError> {
self.start_mode(account, Some(window), true).await
}
async fn start_mode(
self: &Arc<Self>,
account: String,
window: Option<AgendaWindow>,
repair: bool,
) -> Result<SyncRun, SyncError> {
let row = self
.store
.account(account.clone())
.await?
.ok_or(StoreError::NotFound)?;
if row.account.provider != "google" {
return Err(AccountError::Invalid.into());
}
if row.state != "connected" {
return Err(AccountError::Authorization.into());
}
let window = window
.unwrap_or(AgendaWindow::rolling(self.now()?).map_err(|_| StoreError::InvalidInput)?);
AgendaWindow::new(&window.from, &window.to).map_err(|_| StoreError::InvalidInput)?;
let mut jobs = self.jobs.lock().await;
jobs.retain(|_, j| !j.task.is_finished());
if let Some(((.., id), job)) = jobs.iter().find(|((a, _), _)| a == &account) {
if job.window == window && (!repair || job.repair) {
return Ok(self.store.sync_run(account, id.clone()).await?);
}
return Err(StoreError::Busy.into());
}
let admission = self.accounts.http.resources.job()?;
let run = self
.store
.start_calendar_run(account.clone(), self.now()?)
.await?;
let service = self.clone();
let work_run = run.clone();
let job_window = window.clone();
let (stop, mut stopped) = watch::channel(false);
let task = tokio::spawn(async move {
let _admission = admission;
let started = std::time::Instant::now();
let work_stop = stopped.clone();
let work = async {
let _permit = service
.coordinator
.acquire_sync(
&service.store,
&service.accounts.http.clock,
&work_run.account_id,
"calendar",
)
.await?;
service
.synchronize(&work_run, window, repair, work_stop)
.await
};
let result =
tokio::select! {result=work=>result,_=stopped.changed()=>Err(SyncError::Cancelled)};
if result.is_ok() {
tracing::info!(account_id = %work_run.account_id, run_id = %work_run.id, scope = %work_run.scope, elapsed_ms = started.elapsed().as_millis() as u64, "Sync completed");
}
if let Err(e) = result {
let state = if matches!(e, SyncError::Cancelled) {
"cancelled"
} else {
"failed"
};
let recorded = service
.store
.finish_sync_run_error(
work_run.account_id,
work_run.id,
state.into(),
e.code().into(),
service.now().unwrap_or(work_run.started_at_ms),
)
.await;
if recorded.is_err() {
tracing::error!("Sync failure could not be recorded in storage");
}
}
});
jobs.insert(
(account, run.id.clone()),
Job {
stop,
task,
window: job_window,
repair,
},
);
Ok(run)
}
async fn synchronize(
&self,
run: &SyncRun,
window: AgendaWindow,
repair: bool,
stop: watch::Receiver<bool>,
) -> Result<(), SyncError> {
self.store
.begin_sync_run(run.account_id.clone(), run.id.clone(), None)
.await?;
let mut page: Option<String> = None;
let mut seen = BTreeSet::new();
loop {
let mut query = vec![("maxResults", "100")];
if let Some(page) = &page {
query.push(("pageToken", page));
}
let response = self
.accounts
.calendar_get(
&run.account_id,
&["users", "me", "calendarList"],
&query,
4 * 1024 * 1024,
)
.await?;
for value in items(&response)? {
let access = optional(value, "accessRole")?.unwrap_or_else(|| "none".into());
self.store
.stage_calendar_entry(
run.account_id.clone(),
run.id.clone(),
CalendarCatalogEntry {
provider_id: required(value, "id")?,
summary: optional(value, "summary")?,
time_zone: optional(value, "timeZone")?,
access_role: access,
is_primary: value
.get("primary")
.and_then(Value::as_bool)
.unwrap_or(false),
provider_json: serde_json::to_string(value)
.map_err(|_| SyncError::Provider)?,
},
)
.await?;
}
page = next(&response, &mut seen)?;
self.accounts.http.resources.page_stored();
if page.is_none() {
break;
}
}
if repair {
return self.repair_staged(run, window, stop).await;
}
self.store
.promote_calendar_catalog(run.account_id.clone(), run.id.clone())
.await?;
let mut processed = 0;
for calendar in self
.store
.calendars(run.account_id.clone())
.await?
.into_iter()
.filter(|c| !c.retired)
{
if matches!(calendar.access_role.as_str(), "none" | "freeBusyReader") {
continue;
}
self.canonical(run, &calendar, &mut processed).await?;
// Capture the canonical revision after its promotion; another change
// must not label an older expanded window current.
let refreshed = self
.store
.calendars(run.account_id.clone())
.await?
.into_iter()
.find(|c| c.id == calendar.id)
.ok_or(StoreError::NotFound)?;
self.occurrences(run, &refreshed, &window, &mut processed)
.await?;
}
self.store
.finish_calendar_run(run.account_id.clone(), run.id.clone(), self.now()?)
.await?;
Ok(())
}
async fn repair_staged(
&self,
run: &SyncRun,
window: AgendaWindow,
_stop: watch::Receiver<bool>,
) -> Result<(), SyncError> {
let calendars = self
.store
.calendar_repair_targets(run.account_id.clone(), run.id.clone())
.await?;
let mut checkpoints = Vec::new();
let mut processed = 0;
for mut calendar in calendars {
if matches!(calendar.access_role.as_str(), "none" | "freeBusyReader") {
continue;
}
let canonical = self
.events(run, &calendar, None, None, &mut processed, true)
.await?;
calendar.time_zone = Some(canonical.time_zone.clone());
let expanded = self
.events(run, &calendar, None, Some(&window), &mut processed, true)
.await?;
if canonical.time_zone != expanded.time_zone {
return Err(SyncError::Provider);
}
checkpoints.push(crate::store::CalendarRepairCheckpoint {
calendar_id: calendar.id,
cursor: canonical.cursor.ok_or(SyncError::Provider)?,
time_zone: canonical.time_zone,
});
}
#[cfg(feature = "test-harness")]
if let Some(config) = &self.accounts.http.test_config {
config
.checkpoint("calendar-repair-before-promotion", _stop)
.await
.map_err(|_| SyncError::Cancelled)?;
}
self.store
.promote_calendar_repair(
run.account_id.clone(),
run.id.clone(),
window,
checkpoints,
self.now()?,
)
.await?;
Ok(())
}
async fn canonical(
&self,
run: &SyncRun,
calendar: &StoredCalendar,
processed: &mut u64,
) -> Result<(), SyncError> {
let mut cursor = self
.store
.calendar_cursor(run.account_id.clone(), calendar.id.clone())
.await?;
for attempt in 0..2 {
let result = self
.events(run, calendar, cursor.as_deref(), None, processed, false)
.await;
match result {
Ok(result) => {
if cursor.is_some() && calendar.time_zone.as_deref() != Some(&result.time_zone)
{
if attempt != 0 {
return Err(SyncError::Provider);
}
self.store
.clear_calendar_staging(
run.account_id.clone(),
run.id.clone(),
calendar.id.clone(),
false,
)
.await?;
cursor = None;
continue;
}
self.store
.promote_calendar_objects(
run.account_id.clone(),
run.id.clone(),
calendar.id.clone(),
crate::store::CalendarCheckpoint {
cursor: result.cursor.ok_or(SyncError::Provider)?,
full: cursor.is_none(),
time_zone: result.time_zone,
},
self.now()?,
)
.await?;
return Ok(());
}
Err(SyncError::CalendarExpired) if attempt == 0 => {
self.store
.clear_calendar_staging(
run.account_id.clone(),
run.id.clone(),
calendar.id.clone(),
false,
)
.await?;
cursor = None;
}
Err(e) => return Err(e),
}
}
Err(SyncError::CalendarExpired)
}
async fn occurrences(
&self,
run: &SyncRun,
calendar: &StoredCalendar,
window: &AgendaWindow,
processed: &mut u64,
) -> Result<(), SyncError> {
let result = self
.events(run, calendar, None, Some(window), processed, false)
.await?;
if calendar.time_zone.as_deref() != Some(&result.time_zone) {
return Err(SyncError::Provider);
}
self.store
.promote_calendar_occurrences(
run.account_id.clone(),
run.id.clone(),
calendar.id.clone(),
window.clone(),
calendar.canonical_revision,
self.now()?,
)
.await?;
Ok(())
}
async fn events(
&self,
run: &SyncRun,
calendar: &StoredCalendar,
cursor: Option<&str>,
window: Option<&AgendaWindow>,
processed: &mut u64,
repair: bool,
) -> Result<EventsResult, SyncError> {
let bounds = window
.map(|w| {
w.provider_bounds(calendar.time_zone.as_deref().ok_or(SyncError::Provider)?)
.map_err(|_| SyncError::Provider)
})
.transpose()?;
let mut page: Option<String> = None;
let mut seen = BTreeSet::new();
let mut response_zone: Option<String> = None;
loop {
let mut query = vec![
("maxResults", "100"),
(
"singleEvents",
if window.is_some() { "true" } else { "false" },
),
("showDeleted", "true"),
];
if let Some(token) = cursor {
query.push(("syncToken", token));
}
if let Some(page) = &page {
query.push(("pageToken", page));
}
if let Some((from, to)) = &bounds {
query.extend([("timeMin", from.as_str()), ("timeMax", to.as_str())]);
}
let response = self
.accounts
.calendar_get(
&run.account_id,
&["calendars", &calendar.provider_id, "events"],
&query,
32 * 1024 * 1024,
)
.await?;
let page_zone = optional(&response, "timeZone")?;
let zone = page_zone
.as_deref()
.or(calendar.time_zone.as_deref())
.ok_or(SyncError::Provider)?;
if zone.parse::<chrono_tz::Tz>().is_err()
|| response_zone
.as_deref()
.is_some_and(|previous| previous != zone)
{
return Err(SyncError::Provider);
}
response_zone = Some(zone.to_owned());
for value in items(&response)? {
let object = CalendarObject::from_google(value.clone(), zone)
.map_err(|_| SyncError::Provider)?;
if repair {
self.store
.stage_calendar_repair_event(
run.account_id.clone(),
run.id.clone(),
calendar.id.clone(),
window.is_some(),
object,
)
.await?;
} else {
self.store
.stage_calendar_event(
run.account_id.clone(),
run.id.clone(),
calendar.id.clone(),
window.is_some(),
object,
)
.await?;
}
*processed += 1;
}
page = next(&response, &mut seen)?;
self.accounts.http.resources.page_stored();
self.store
.sync_run_progress(
run.account_id.clone(),
run.id.clone(),
*processed,
page.clone(),
)
.await?;
if page.is_none() {
return Ok(EventsResult {
time_zone: response_zone.ok_or(SyncError::Provider)?,
cursor: if window.is_some() {
None
} else {
Some(required(&response, "nextSyncToken")?)
},
});
}
}
}
pub async fn cancel(&self, account: &str, id: &str) -> Result<SyncRun, SyncError> {
self.store.sync_run(account.into(), id.into()).await?;
if let Some(job) = self.jobs.lock().await.remove(&(account.into(), id.into())) {
job.stop.send_replace(true);
job.task.await.map_err(|_| SyncError::Unavailable)?;
}
Ok(self.store.sync_run(account.into(), id.into()).await?)
}
pub async fn shutdown(&self) {
let jobs = std::mem::take(&mut *self.jobs.lock().await);
for job in jobs.values() {
job.stop.send_replace(true);
}
for (_, job) in jobs {
let _ = job.task.await;
}
}
fn now(&self) -> Result<i64, StoreError> {
Ok(self.accounts.http.clock.now_ms())
}
}
fn items(value: &Value) -> Result<&[Value], SyncError> {
match value.get("items") {
None => Ok(&[]),
Some(Value::Array(a)) if a.len() <= 2500 => Ok(a),
_ => Err(SyncError::Provider),
}
}
fn optional(value: &Value, key: &str) -> Result<Option<String>, SyncError> {
match value.get(key) {
None => Ok(None),
Some(Value::String(s)) if s.len() <= 8192 => Ok(Some(s.clone())),
_ => Err(SyncError::Provider),
}
}
fn required(value: &Value, key: &str) -> Result<String, SyncError> {
optional(value, key)?
.filter(|s| !s.is_empty() && !s.chars().any(char::is_control))
.ok_or(SyncError::Provider)
}
fn next(value: &Value, seen: &mut BTreeSet<String>) -> Result<Option<String>, SyncError> {
let token = optional(value, "nextPageToken")?;
if let Some(token) = &token {
if token.is_empty() || seen.len() >= 100_000 || !seen.insert(token.clone()) {
return Err(SyncError::Provider);
}
}
Ok(token)
}
@@ -0,0 +1,46 @@
use crate::store::{ChangeEvent, Store, StoreError};
use std::collections::VecDeque;
use tokio::sync::watch;
pub struct ChangeReader {
store: Store,
changed: watch::Receiver<u64>,
after: u64,
buffer: VecDeque<ChangeEvent>,
}
impl ChangeReader {
pub(crate) async fn open(store: Store, after: u64) -> Result<Self, StoreError> {
let changed = store.revision.clone();
let page = store.changes_after(after, 100).await?;
Ok(Self {
store,
changed,
after,
buffer: page.changes.into(),
})
}
pub async fn next(&mut self) -> Result<ChangeEvent, StoreError> {
loop {
if let Some(change) = self.buffer.pop_front() {
self.after = change.revision;
return Ok(change);
}
// Subscribe before querying, so a commit between the query and wait
// cannot be lost. The watch value is only a wake-up hint; SQL is truth.
self.changed.borrow_and_update();
self.buffer = self
.store
.changes_after(self.after, 100)
.await?
.changes
.into();
if !self.buffer.is_empty() {
continue;
}
self.changed
.changed()
.await
.map_err(|_| StoreError::Unavailable)?;
}
}
}
+63
View File
@@ -0,0 +1,63 @@
use std::time::Instant;
pub(crate) struct Clock {
started: Instant,
test_wall_ms: Option<i64>,
#[cfg(feature = "test-harness")]
test_offset_ms: std::sync::atomic::AtomicI64,
}
impl Clock {
pub fn new(test_wall_ms: Option<i64>) -> Self {
Self {
started: Instant::now(),
test_wall_ms,
#[cfg(feature = "test-harness")]
test_offset_ms: std::sync::atomic::AtomicI64::new(0),
}
}
pub fn now_ms(&self) -> i64 {
if let Some(base) = self.test_wall_ms {
#[cfg(feature = "test-harness")]
let base = base.saturating_add(
self.test_offset_ms
.load(std::sync::atomic::Ordering::Relaxed),
);
base.saturating_add(
i64::try_from(self.started.elapsed().as_millis()).unwrap_or(i64::MAX),
)
} else {
chrono::Utc::now().timestamp_millis()
}
}
#[cfg(feature = "test-harness")]
pub fn set_test_offset(&self, offset: i64) {
self.test_offset_ms
.store(offset, std::sync::atomic::Ordering::Relaxed);
}
pub async fn wait_until(&self, wall_ms: i64) {
let mut previous_wall = self.now_ms();
let mut previous_tick = tokio::time::Instant::now();
let mut deadline = previous_tick.checked_add(std::time::Duration::from_millis(
wall_ms.saturating_sub(previous_wall).max(0) as u64,
));
let mut tick = tokio::time::interval(std::time::Duration::from_millis(100));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tick.tick().await;
let now = self.now_ms();
let instant = tokio::time::Instant::now();
let elapsed = i64::try_from(instant.duration_since(previous_tick).as_millis())
.unwrap_or(i64::MAX);
if now.saturating_sub(previous_wall) > elapsed.saturating_add(1000) {
deadline = instant.checked_add(std::time::Duration::from_millis(
wall_ms.saturating_sub(now).max(0) as u64,
));
}
if deadline.is_some_and(|at| at <= instant) {
return;
}
previous_wall = now;
previous_tick = instant;
}
}
}
@@ -0,0 +1,133 @@
use crate::store::StoreError;
use std::{
collections::BTreeMap,
sync::{Arc, Weak},
};
use tokio::sync::{Mutex, OwnedMutexGuard, OwnedSemaphorePermit, Semaphore};
pub(crate) struct Coordinator {
active: Arc<Semaphore>,
admitted: Arc<Semaphore>,
accounts: Mutex<BTreeMap<String, Weak<Mutex<()>>>>,
}
pub(crate) struct AccountPermit {
_active: OwnedSemaphorePermit,
_admitted: OwnedSemaphorePermit,
_sequence: OwnedMutexGuard<()>,
}
impl Coordinator {
pub fn admitted_requests(&self) -> u64 {
(64 - self.admitted.available_permits()) as u64
}
pub fn new() -> Arc<Self> {
Arc::new(Self {
active: Arc::new(Semaphore::new(2)),
admitted: Arc::new(Semaphore::new(64)),
accounts: Mutex::new(BTreeMap::new()),
})
}
pub async fn acquire_sync(
&self,
store: &crate::store::Store,
clock: &crate::clock::Clock,
account: &str,
scope: &str,
) -> Result<AccountPermit, StoreError> {
loop {
if let Some(at) = store
.provider_retry_deadline(account.into(), scope.into())
.await?
{
clock.wait_until(at).await;
}
let permit = self.acquire(account).await?;
// A preceding account request may have just recorded new guidance.
// Release both permits before waiting so other work remains possible.
if store
.provider_retry_deadline(account.into(), scope.into())
.await?
.is_none_or(|at| at <= clock.now_ms())
{
return Ok(permit);
}
drop(permit);
}
}
pub async fn acquire(&self, account: &str) -> Result<AccountPermit, StoreError> {
// Bound retained callers before they wait for account sequencing.
let admitted = self
.admitted
.clone()
.try_acquire_owned()
.map_err(|_| StoreError::Busy)?;
let lane = {
let mut accounts = self.accounts.lock().await;
accounts.retain(|_, lane| lane.strong_count() != 0);
if let Some(lane) = accounts.get(account).and_then(Weak::upgrade) {
lane
} else {
let lane = Arc::new(Mutex::new(()));
accounts.insert(account.to_owned(), Arc::downgrade(&lane));
lane
}
};
// Waiting on one account must not consume another account's global slot.
let sequence = lane.lock_owned().await;
let active = self
.active
.clone()
.acquire_owned()
.await
.map_err(|_| StoreError::Unavailable)?;
Ok(AccountPermit {
_active: active,
_admitted: admitted,
_sequence: sequence,
})
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
use super::*;
use futures_util::{stream::FuturesUnordered, StreamExt};
use std::time::Duration;
#[tokio::test]
async fn account_waiters_have_bounded_admission_and_cancellation_releases_capacity() {
let coordinator = Coordinator::new();
let first = coordinator.acquire("same-account").await.unwrap();
let mut waiting = FuturesUnordered::new();
for _ in 0..64 {
waiting.push(coordinator.acquire("same-account"));
}
let rejected = tokio::time::timeout(Duration::from_secs(1), waiting.next()).await;
assert!(rejected.is_ok(),"the 65th admitted account request must fail instead of joining an unbounded wait queue");
assert!(matches!(rejected.unwrap(), Some(Err(StoreError::Busy))));
drop(waiting);
let other =
tokio::time::timeout(Duration::from_secs(1), coordinator.acquire("other-account"))
.await
.unwrap()
.unwrap();
let mut third = Box::pin(coordinator.acquire("third-account"));
assert!(
futures_util::poll!(&mut third).is_pending(),
"only two accounts may be active"
);
drop(other);
let third = tokio::time::timeout(Duration::from_secs(1), third)
.await
.unwrap()
.unwrap();
drop(third);
drop(first);
let fresh =
tokio::time::timeout(Duration::from_secs(1), coordinator.acquire("same-account"))
.await
.unwrap()
.unwrap();
drop(fresh);
}
}
@@ -0,0 +1,252 @@
use chrono::{DateTime, Datelike, Duration, NaiveDate, NaiveDateTime, TimeZone, Utc};
use chrono_tz::Tz;
use serde::{Deserialize, Serialize};
use serde_json::Value;
#[derive(Debug, thiserror::Error)]
pub enum CalendarError {
#[error("Calendar data or event time is invalid")]
Invalid,
#[error("Agenda requires increasing ISO dates within a 366-day window")]
InvalidWindow,
#[error("Calendar resource exceeds the supported size")]
TooLarge,
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(untagged)]
pub enum EventTime {
Date { date: String },
DateTime { date_time: ZonedDateTime },
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct ZonedDateTime {
pub rfc3339: String,
pub time_zone: String,
}
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AgendaWindow {
pub from: String,
pub to: String,
}
impl AgendaWindow {
pub fn new(from: &str, to: &str) -> Result<Self, CalendarError> {
let a = date(from)?;
let b = date(to)?;
let days = b.signed_duration_since(a).num_days();
if !(1..=366).contains(&days) {
return Err(CalendarError::InvalidWindow);
}
Ok(Self {
from: from.into(),
to: to.into(),
})
}
pub fn rolling(now_ms: i64) -> Result<Self, CalendarError> {
let today = DateTime::<Utc>::from_timestamp_millis(now_ms)
.ok_or(CalendarError::InvalidWindow)?
.date_naive();
let from = today
.checked_sub_signed(Duration::days(30))
.ok_or(CalendarError::InvalidWindow)?;
// 'Through 180 days after today' includes that date; upper bounds are exclusive.
let to = today
.checked_add_signed(Duration::days(181))
.ok_or(CalendarError::InvalidWindow)?;
Self::new(&from.to_string(), &to.to_string())
}
pub fn provider_bounds(&self, zone: &str) -> Result<(String, String), CalendarError> {
Self::new(&self.from, &self.to)?;
let zone: Tz = zone.parse().map_err(|_| CalendarError::Invalid)?;
Ok((
midnight(&self.from, zone)?.to_rfc3339(),
midnight(&self.to, zone)?.to_rfc3339(),
))
}
pub fn bounds_ms(&self, zone: &str) -> Result<(i64, i64), CalendarError> {
let (from, to) = self.provider_bounds(zone)?;
Ok((
DateTime::parse_from_rfc3339(&from)
.map_err(|_| CalendarError::Invalid)?
.timestamp_millis(),
DateTime::parse_from_rfc3339(&to)
.map_err(|_| CalendarError::Invalid)?
.timestamp_millis(),
))
}
}
#[derive(Clone, Serialize, Deserialize)]
pub struct CalendarObject {
pub provider_id: String,
pub summary: Option<String>,
pub etag: Option<String>,
pub status: String,
pub start: Option<EventTime>,
pub end: Option<EventTime>,
pub original_start: Option<EventTime>,
pub recurring_provider_id: Option<String>,
pub provider_json: String,
pub start_ms: Option<i64>,
pub end_ms: Option<i64>,
}
impl CalendarObject {
pub fn from_google(value: Value, calendar_zone: &str) -> Result<Self, CalendarError> {
let provider_json = serde_json::to_string(&value).map_err(|_| CalendarError::Invalid)?;
if provider_json.len() > 1024 * 1024 {
return Err(CalendarError::TooLarge);
}
let provider_id = text(&value, "id")?.ok_or(CalendarError::Invalid)?;
let status = text(&value, "status")?.unwrap_or_else(|| "confirmed".into());
if !matches!(status.as_str(), "confirmed" | "tentative" | "cancelled") {
return Err(CalendarError::Invalid);
}
let start = value
.get("start")
.map(|v| time(v, calendar_zone))
.transpose()?;
let end = value
.get("end")
.map(|v| time(v, calendar_zone))
.transpose()?;
let original_start = value
.get("originalStartTime")
.map(|v| time(v, calendar_zone))
.transpose()?;
let recurring_provider_id = text(&value, "recurringEventId")?;
if status != "cancelled" && (start.is_none() || end.is_none()) {
return Err(CalendarError::Invalid);
}
if recurring_provider_id.is_some() && original_start.is_none() {
return Err(CalendarError::Invalid);
}
let start_ms = start
.as_ref()
.or(original_start.as_ref())
.map(|t| instant(t, calendar_zone))
.transpose()?;
let end_ms = end
.as_ref()
.map(|t| instant(t, calendar_zone))
.transpose()?;
if let (Some(start), Some(end)) = (&start, &end) {
if matches!(start, EventTime::Date { .. }) != matches!(end, EventTime::Date { .. })
|| start_ms >= end_ms
{
return Err(CalendarError::Invalid);
}
}
Ok(Self {
provider_id,
summary: text(&value, "summary")?,
etag: text(&value, "etag")?,
status,
start,
end,
original_start,
recurring_provider_id,
provider_json,
start_ms,
end_ms,
})
}
}
fn text(value: &Value, key: &str) -> Result<Option<String>, CalendarError> {
match value.get(key) {
None => Ok(None),
Some(Value::String(s))
if s.len() <= 65536
&& (!matches!(key, "id" | "etag" | "recurringEventId")
|| !s.is_empty() && s.len() <= 2048 && !s.chars().any(char::is_control)) =>
{
Ok(Some(s.clone()))
}
_ => Err(CalendarError::Invalid),
}
}
fn date(value: &str) -> Result<NaiveDate, CalendarError> {
if value.len() != 10
|| !value.bytes().enumerate().all(|(i, c)| {
if i == 4 || i == 7 {
c == b'-'
} else {
c.is_ascii_digit()
}
})
{
return Err(CalendarError::InvalidWindow);
}
let date =
NaiveDate::parse_from_str(value, "%Y-%m-%d").map_err(|_| CalendarError::InvalidWindow)?;
if !(1..=9999).contains(&date.year()) {
return Err(CalendarError::InvalidWindow);
}
Ok(date)
}
fn midnight(value: &str, zone: Tz) -> Result<DateTime<Tz>, CalendarError> {
zone.from_local_datetime(
&date(value)?
.and_hms_opt(0, 0, 0)
.ok_or(CalendarError::Invalid)?,
)
.single()
.ok_or(CalendarError::Invalid)
}
fn time(value: &Value, calendar_zone: &str) -> Result<EventTime, CalendarError> {
let value = value.as_object().ok_or(CalendarError::Invalid)?;
if value.contains_key("date") == value.contains_key("dateTime") {
return Err(CalendarError::Invalid);
}
if let Some(date_value) = value.get("date") {
let s = date_value.as_str().ok_or(CalendarError::Invalid)?;
date(s)?;
return Ok(EventTime::Date { date: s.into() });
}
let raw = value
.get("dateTime")
.and_then(Value::as_str)
.ok_or(CalendarError::Invalid)?;
if raw.len() > 64 {
return Err(CalendarError::Invalid);
}
let explicit_zone = value
.get("timeZone")
.map(|v| v.as_str().ok_or(CalendarError::Invalid))
.transpose()?;
let zone: Tz = explicit_zone
.unwrap_or(calendar_zone)
.parse()
.map_err(|_| CalendarError::Invalid)?;
let normalized = if let Ok(parsed) = DateTime::parse_from_rfc3339(raw) {
if !(1..=9999).contains(&parsed.year()) {
return Err(CalendarError::Invalid);
}
raw.to_owned()
} else {
if explicit_zone.is_none() {
return Err(CalendarError::Invalid);
}
let naive = NaiveDateTime::parse_from_str(raw, "%Y-%m-%dT%H:%M:%S%.f")
.map_err(|_| CalendarError::Invalid)?;
zone.from_local_datetime(&naive)
.single()
.ok_or(CalendarError::Invalid)?
.to_rfc3339()
};
Ok(EventTime::DateTime {
date_time: ZonedDateTime {
rfc3339: normalized,
time_zone: zone.to_string(),
},
})
}
fn instant(time: &EventTime, calendar_zone: &str) -> Result<i64, CalendarError> {
match time {
EventTime::Date { date } => Ok(midnight(
date,
calendar_zone.parse().map_err(|_| CalendarError::Invalid)?,
)?
.timestamp_millis()),
EventTime::DateTime { date_time } => Ok(DateTime::parse_from_rfc3339(&date_time.rfc3339)
.map_err(|_| CalendarError::Invalid)?
.timestamp_millis()),
}
}
@@ -0,0 +1,273 @@
use super::calendar::CalendarObject;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
mod fields;
#[derive(Debug, thiserror::Error)]
pub enum ChangeError {
#[error("Calendar action has invalid, unsupported, or oversized fields")]
Invalid,
#[error("Calendar action is not permitted for this account or event")]
Permission,
#[error("Calendar action scope does not match the event")]
Scope,
#[error("The selected account must match exactly one event attendee")]
Attendee,
}
#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Scope {
Single,
Series,
}
#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Notifications {
None,
All,
ExternalOnly,
}
impl Notifications {
pub fn parameter(self) -> &'static str {
match self {
Self::None => "none",
Self::All => "all",
Self::ExternalOnly => "externalOnly",
}
}
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)]
pub enum CalendarAction {
Create {
scope: Scope,
notifications: Notifications,
event: Value,
},
Update {
event_id: String,
expected_etag: String,
scope: Scope,
notifications: Notifications,
patch: Value,
},
Delete {
event_id: String,
expected_etag: String,
scope: Scope,
notifications: Notifications,
},
Respond {
event_id: String,
expected_etag: String,
scope: Scope,
notifications: Notifications,
response: Response,
#[serde(default, skip_serializing_if = "Option::is_none")]
comment: Option<String>,
},
}
#[derive(Clone, Copy, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum Response {
NeedsAction,
Accepted,
Declined,
Tentative,
}
impl CalendarAction {
pub fn from_value(value: Value, zone: &str) -> Result<Self, ChangeError> {
if serde_json::to_vec(&value)
.map_err(|_| ChangeError::Invalid)?
.len()
> 1024 * 1024
{
return Err(ChangeError::Invalid);
}
let action: Self = serde_json::from_value(value).map_err(|_| ChangeError::Invalid)?;
if let Some((event, etag)) = action.identity() {
if event.is_empty()
|| event.len() > 128
|| event.chars().any(char::is_control)
|| etag.len() > 2048
|| etag.len() < 3
|| !etag.starts_with('"')
|| !etag.ends_with('"')
|| !etag.bytes().all(|b| (32..=126).contains(&b))
|| etag[1..etag.len() - 1].contains('"')
{
return Err(ChangeError::Invalid);
}
}
match &action {
Self::Create { event, .. } => {
fields::validate(event)?;
validate_event(event.clone(), zone)?;
action.check_scope(event)?;
}
Self::Update { patch, .. } => fields::validate(patch)?,
Self::Respond { comment, .. }
if comment
.as_ref()
.is_some_and(|v| v.len() > 65536 || v.contains('\0')) =>
{
return Err(ChangeError::Invalid)
}
_ => (),
}
Ok(action)
}
pub fn identity(&self) -> Option<(&str, &str)> {
match self {
Self::Create { .. } => None,
Self::Update {
event_id,
expected_etag,
..
}
| Self::Delete {
event_id,
expected_etag,
..
}
| Self::Respond {
event_id,
expected_etag,
..
} => Some((event_id, expected_etag)),
}
}
pub fn notification_parameter(&self) -> &'static str {
match self {
Self::Create { notifications, .. }
| Self::Update { notifications, .. }
| Self::Delete { notifications, .. }
| Self::Respond { notifications, .. } => notifications.parameter(),
}
}
pub fn provider_patch(
&self,
base: Option<&Value>,
zone: &str,
account_email: &str,
role: &str,
) -> Result<Value, ChangeError> {
if !matches!(role, "owner" | "writer" | "writerWithoutPrivateAccess") {
return Err(ChangeError::Permission);
}
if let Self::Create { event, .. } = self {
return Ok(event.clone());
}
let base = base.ok_or(ChangeError::Invalid)?;
if role == "writerWithoutPrivateAccess" && base["visibility"] == "private"
|| base["status"] == "cancelled"
|| base["locked"] == true
|| base.get("eventType").is_some_and(|v| v != "default")
{
return Err(ChangeError::Permission);
}
self.check_scope(base)?;
match self {
Self::Update { patch, .. } => {
let organizer = base.get("organizer").is_none_or(|o| {
o.get("self").and_then(Value::as_bool).unwrap_or_else(|| {
o["email"]
.as_str()
.is_some_and(|e| e.eq_ignore_ascii_case(account_email))
})
});
if !organizer
&& [
"attendees",
"guestsCanModify",
"guestsCanInviteOthers",
"guestsCanSeeOtherGuests",
]
.iter()
.any(|k| patch.get(*k).is_some())
{
return Err(ChangeError::Permission);
}
let mut merged = base.clone();
merge(&mut merged, patch);
validate_event(merged, zone)?;
Ok(patch.clone())
}
Self::Respond {
response, comment, ..
} => {
let attendees = base["attendees"].as_array().ok_or(ChangeError::Attendee)?;
let matches: Vec<_> = attendees
.iter()
.filter(|a| {
a["email"]
.as_str()
.is_some_and(|e| e.eq_ignore_ascii_case(account_email))
})
.collect();
if matches.len() != 1 {
return Err(ChangeError::Attendee);
}
let mut attendee = json!({"email":matches[0]["email"],"responseStatus":response});
if let Some(comment) = comment {
attendee["comment"] = json!(comment);
}
// Native partial response avoids replacing other attendees or copying
// provider-owned fields back into a write request.
Ok(json!({"attendeesOmitted":true,"attendees":[attendee]}))
}
Self::Delete { .. } => Ok(Value::Null),
Self::Create { .. } => Err(ChangeError::Invalid),
}
}
fn check_scope(&self, base: &Value) -> Result<(), ChangeError> {
let scope = match self {
Self::Create { scope, .. }
| Self::Update { scope, .. }
| Self::Delete { scope, .. }
| Self::Respond { scope, .. } => *scope,
};
let master = base
.get("recurrence")
.is_some_and(|v| v.as_array().is_some_and(|a| !a.is_empty()))
&& base.get("recurringEventId").is_none();
if (scope == Scope::Series) != master {
return Err(ChangeError::Scope);
}
Ok(())
}
}
fn validate_event(mut value: Value, zone: &str) -> Result<(), ChangeError> {
if value.get("id").is_none() {
value["id"] = json!("validation");
}
CalendarObject::from_google(value.clone(), zone).map_err(|_| ChangeError::Invalid)?;
if let Some(rules) = value.get("recurrence") {
let rules = rules.as_array().ok_or(ChangeError::Invalid)?;
if rules.is_empty() {
return Err(ChangeError::Invalid);
}
if value["start"].get("dateTime").is_some()
&& (value["start"].get("timeZone").is_none() || value["end"].get("timeZone").is_none())
{
return Err(ChangeError::Invalid);
}
}
Ok(())
}
pub(crate) fn merge(target: &mut Value, patch: &Value) {
if let (Some(target), Some(patch)) = (target.as_object_mut(), patch.as_object()) {
for (key, value) in patch {
if value.is_null() {
target.remove(key);
} else if value.is_object() {
merge(target.entry(key.clone()).or_insert(json!({})), value);
} else {
target.insert(key.clone(), value.clone());
}
}
} else {
*target = patch.clone();
}
}
@@ -0,0 +1,117 @@
use super::ChangeError;
use serde_json::Value;
use std::collections::BTreeSet;
pub(super) fn validate(value: &Value) -> Result<(), ChangeError> {
let fields = value
.as_object()
.filter(|v| !v.is_empty())
.ok_or(ChangeError::Invalid)?;
for (key, value) in fields {
let valid = match key.as_str() {
"summary" | "description" | "location" => text(value, 65536) || value.is_null(),
"colorId" => text(value, 32) || value.is_null(),
"start" | "end" => time(value),
"transparency" => ["opaque", "transparent"].iter().any(|s| value == s),
"visibility" => ["default", "public", "private", "confidential"]
.iter()
.any(|s| value == s),
"guestsCanModify" | "guestsCanInviteOthers" | "guestsCanSeeOtherGuests" => {
value.is_boolean()
}
"recurrence" => {
value.is_null()
|| value.as_array().is_some_and(|a| {
!a.is_empty()
&& a.len() <= 128
&& a.iter().all(|v| {
v.as_str().is_some_and(|s| {
s.len() <= 8192
&& !s.chars().any(char::is_control)
&& [
"RRULE:", "EXRULE:", "RDATE:", "RDATE;", "EXDATE:",
"EXDATE;",
]
.iter()
.any(|prefix| s.starts_with(prefix))
})
})
})
}
"attendees" => attendees(value),
"reminders" => reminders(value),
_ => false,
};
if !valid {
return Err(ChangeError::Invalid);
}
}
Ok(())
}
fn text(value: &Value, max: usize) -> bool {
value
.as_str()
.is_some_and(|s| s.len() <= max && !s.contains('\0'))
}
fn time(value: &Value) -> bool {
value.as_object().is_some_and(|o| {
!o.is_empty()
&& o.iter().all(|(k, v)| match k.as_str() {
// Explicit null removes the old representation when changing date/time kind.
"date" | "dateTime" => v.is_null() || text(v, 64),
"timeZone" => {
v.is_null()
|| v.as_str()
.is_some_and(|s| s.parse::<chrono_tz::Tz>().is_ok())
}
_ => false,
})
})
}
fn attendees(value: &Value) -> bool {
let Some(attendees) = value.as_array().filter(|a| a.len() <= 1000) else {
return false;
};
let mut emails = BTreeSet::new();
attendees.iter().all(|a| {
let Some(o) = a.as_object() else {
return false;
};
let Some(email) = o.get("email").and_then(Value::as_str).filter(|s| {
s.len() <= 320
&& email_address::EmailAddress::is_valid(s)
&& !s.chars().any(char::is_control)
}) else {
return false;
};
emails.insert(email.to_ascii_lowercase())
&& o.iter().all(|(k, v)| match k.as_str() {
"email" => true,
"displayName" => text(v, 65536),
"optional" | "resource" => v.is_boolean(),
// Responses belong to the explicit selected-account RSVP action.
"responseStatus" => v == "needsAction",
_ => false,
})
})
}
fn reminders(value: &Value) -> bool {
if value.is_null() {
return true;
}
let Some(o) = value.as_object() else {
return false;
};
o.iter().all(|(k, v)| match k.as_str() {
"useDefault" => v.is_boolean(),
"overrides" => v.as_array().is_some_and(|a| {
a.len() <= 5
&& a.iter().all(|v| {
v.as_object().is_some_and(|o| o.len() == 2)
&& (v["method"] == "email" || v["method"] == "popup")
&& v["minutes"].as_u64().is_some_and(|m| m <= 40320)
})
}),
_ => false,
}) && !(value["useDefault"] == true && value.get("overrides").is_some())
}
@@ -0,0 +1,88 @@
use serde::{Deserialize, Serialize};
pub const MAX_DRAFT_BODY_BYTES: usize = 1024 * 1024;
pub const MAX_DRAFT_RECIPIENTS: usize = 1000;
#[derive(Debug, thiserror::Error)]
pub enum DraftError {
#[error("Original message is unavailable or cannot be safely prepared")]
Source,
#[error("Invalid recipient address")]
Recipient,
#[error("Invalid mail header")]
Header,
#[error("Draft exceeds the supported size or recipient count")]
TooLarge,
}
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct Recipient {
pub address: String,
#[serde(default)]
pub name: Option<String>,
}
impl Recipient {
pub fn validate(&self) -> Result<(), DraftError> {
if self.address.len() > 320
|| self.address.trim() != self.address
|| self.address.chars().any(char::is_control)
|| email_address::EmailAddress::parse_with_options(
&self.address,
email_address::Options {
allow_display_text: false,
..Default::default()
},
)
.is_err()
{
return Err(DraftError::Recipient);
}
if self
.name
.as_ref()
.is_some_and(|name| name.len() > 256 || name.chars().any(char::is_control))
{
return Err(DraftError::Header);
}
Ok(())
}
}
#[derive(Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(default, deny_unknown_fields)]
pub struct DraftContent {
pub to: Vec<Recipient>,
pub cc: Vec<Recipient>,
pub bcc: Vec<Recipient>,
pub subject: String,
pub text: Option<String>,
pub html: Option<String>,
}
impl DraftContent {
pub fn validate(&self) -> Result<(), DraftError> {
let recipients = self
.to
.len()
.saturating_add(self.cc.len())
.saturating_add(self.bcc.len());
let body = self
.text
.as_ref()
.map_or(0, String::len)
.saturating_add(self.html.as_ref().map_or(0, String::len));
if recipients > MAX_DRAFT_RECIPIENTS
|| body > MAX_DRAFT_BODY_BYTES
|| self.subject.len() > 8192
{
return Err(DraftError::TooLarge);
}
if self.subject.chars().any(char::is_control) {
return Err(DraftError::Header);
}
for recipient in self.to.iter().chain(&self.cc).chain(&self.bcc) {
recipient.validate()?;
}
Ok(())
}
}
@@ -0,0 +1,172 @@
use crate::{store::StoreError, sync_error::SyncError};
use chrono::{DateTime, FixedOffset};
use serde::Serialize;
use serde_json::{json, Value};
use std::collections::BTreeSet;
#[derive(Clone, Debug)]
pub struct FreeBusyRequest {
pub account_id: String,
pub from: String,
pub to: String,
pub time_zone: String,
// Provider calendar identifiers; arbitrary groups are not expanded.
pub calendar_ids: Vec<String>,
}
#[derive(Debug, Serialize)]
pub struct FreeBusyResult {
pub from: String,
pub to: String,
pub time_zone: String,
pub checked_at_ms: i64,
pub complete: bool,
pub calendars: Vec<CalendarAvailability>,
}
#[derive(Debug, Serialize)]
pub struct CalendarAvailability {
pub provider_calendar_id: String,
pub complete: bool,
pub busy: Vec<BusyPeriod>,
pub errors: Vec<AvailabilityError>,
}
#[derive(Debug, Serialize)]
pub struct BusyPeriod {
pub from: String,
pub to: String,
}
#[derive(Debug, Serialize)]
pub struct AvailabilityError {
pub domain: String,
pub reason: String,
}
impl FreeBusyRequest {
pub fn validate(&self) -> Result<(), StoreError> {
let invalid = StoreError::InvalidInput;
let lower = timestamp(&self.from).map_err(|_| StoreError::InvalidInput)?;
let upper = timestamp(&self.to).map_err(|_| StoreError::InvalidInput)?;
if lower >= upper
|| upper - lower > chrono::Duration::days(366)
|| self.time_zone.len() > 128
|| self.time_zone.parse::<chrono_tz::Tz>().is_err()
|| self.calendar_ids.is_empty()
|| self.calendar_ids.len() > 50
|| self.calendar_ids.iter().any(|id| {
id.is_empty()
|| id.len() > 2048
|| id.chars().any(|c| c.is_control() || c.is_whitespace())
})
|| self.calendar_ids.iter().collect::<BTreeSet<_>>().len() != self.calendar_ids.len()
{
return Err(invalid);
}
Ok(())
}
pub fn provider_body(&self) -> Result<Value, StoreError> {
self.validate()?;
Ok(
json!({"timeMin":self.from,"timeMax":self.to,"timeZone":self.time_zone,"items":self.calendar_ids.iter().map(|id|json!({"id":id})).collect::<Vec<_>>()}),
)
}
}
impl FreeBusyResult {
pub fn from_google(
q: &FreeBusyRequest,
value: Value,
checked_at_ms: i64,
) -> Result<Self, SyncError> {
q.validate()?;
let lower = timestamp(&q.from)?;
let upper = timestamp(&q.to)?;
if value["kind"] != "calendar#freeBusy"
|| timestamp(text(&value, "timeMin")?)? != lower
|| timestamp(text(&value, "timeMax")?)? != upper
{
return Err(SyncError::Provider);
}
if value
.get("groups")
.is_some_and(|v| v.as_object().is_none_or(|g| !g.is_empty()))
{
return Err(SyncError::Provider);
}
let map = value["calendars"].as_object().ok_or(SyncError::Provider)?;
if map.len() != q.calendar_ids.len() {
return Err(SyncError::Provider);
}
let zone = q
.time_zone
.parse::<chrono_tz::Tz>()
.map_err(|_| SyncError::Provider)?;
let mut calendars = Vec::new();
let mut total = 0;
for id in &q.calendar_ids {
let item = map
.get(id)
.filter(|v| v.is_object())
.ok_or(SyncError::Provider)?;
let mut errors = Vec::new();
if let Some(v) = item.get("errors") {
let entries = v
.as_array()
.filter(|a| a.len() <= 100)
.ok_or(SyncError::Provider)?;
for error in entries {
errors.push(AvailabilityError {
domain: text(error, "domain")?.into(),
reason: text(error, "reason")?.into(),
});
}
}
let mut busy = Vec::new();
match item.get("busy") {
Some(v) => {
let periods = v.as_array().ok_or(SyncError::Provider)?;
total += periods.len();
if total > 20_000 {
return Err(SyncError::TooLarge);
}
for period in periods {
let from = timestamp(text(period, "start")?)?;
let to = timestamp(text(period, "end")?)?;
if from >= to || from >= upper || to <= lower {
return Err(SyncError::Provider);
}
busy.push(BusyPeriod {
from: from.max(lower).with_timezone(&zone).to_rfc3339(),
to: to.min(upper).with_timezone(&zone).to_rfc3339(),
});
}
}
None if !errors.is_empty() => {}
None => return Err(SyncError::Provider),
}
calendars.push(CalendarAvailability {
provider_calendar_id: id.clone(),
complete: errors.is_empty(),
busy,
errors,
});
}
Ok(Self {
from: q.from.clone(),
to: q.to.clone(),
time_zone: q.time_zone.clone(),
checked_at_ms,
complete: calendars.iter().all(|c| c.complete),
calendars,
})
}
}
fn text<'a>(v: &'a Value, key: &str) -> Result<&'a str, SyncError> {
v[key]
.as_str()
.filter(|s| !s.is_empty() && s.len() <= 2048 && !s.chars().any(char::is_control))
.ok_or(SyncError::Provider)
}
fn timestamp(v: &str) -> Result<DateTime<FixedOffset>, SyncError> {
if v.len() > 128 {
return Err(SyncError::Provider);
}
DateTime::parse_from_rfc3339(v).map_err(|_| SyncError::Provider)
}
@@ -0,0 +1,39 @@
use serde::{Deserialize, Serialize};
use std::{fmt, str::FromStr};
use uuid::Uuid;
#[derive(Debug, thiserror::Error)]
#[error("Invalid resource identifier")]
pub struct IdentityError;
macro_rules! identity {
($name:ident) => {
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(transparent)]
pub struct $name(Uuid);
impl $name {
pub fn generate() -> Self {
Self(Uuid::new_v4())
}
}
impl FromStr for $name {
type Err = IdentityError;
fn from_str(value: &str) -> Result<Self, Self::Err> {
Uuid::parse_str(value).map(Self).map_err(|_| IdentityError)
}
}
impl fmt::Display for $name {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(formatter)
}
}
};
}
identity!(AccountId);
identity!(ProfileId);
identity!(ImapMailboxId);
@@ -0,0 +1,296 @@
use super::{
identity::{AccountId, ImapMailboxId},
imap_account::validate_mailbox,
};
use base64::{engine::general_purpose::STANDARD_NO_PAD, Engine as _};
use serde::{Deserialize, Serialize};
use std::{collections::BTreeSet, num::NonZeroU32};
#[derive(Debug, thiserror::Error)]
#[error("Invalid IMAP mailbox, placement, flag or UID input")]
pub struct ImapDataError;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ImapPlacement {
pub account_id: AccountId,
pub mailbox_id: ImapMailboxId,
pub uid_validity: NonZeroU32,
pub uid: NonZeroU32,
}
impl ImapPlacement {
pub fn new(
account_id: AccountId,
mailbox_id: ImapMailboxId,
uid_validity: u32,
uid: u32,
) -> Result<Self, ImapDataError> {
Ok(Self {
account_id,
mailbox_id,
uid_validity: NonZeroU32::new(uid_validity).ok_or(ImapDataError)?,
uid: NonZeroU32::new(uid).ok_or(ImapDataError)?,
})
}
pub fn provider_id(&self) -> Result<String, ImapDataError> {
// No RFC Message-ID, mutable folder name, or ambiguous delimiter join.
serde_json::to_string(&(1, "imap", self)).map_err(|_| ImapDataError)
}
pub fn from_provider_id(account: AccountId, id: &str) -> Result<Self, ImapDataError> {
if id.len() > 1024 {
return Err(ImapDataError);
}
let (version, provider, placement): (u8, String, Self) =
serde_json::from_str(id).map_err(|_| ImapDataError)?;
if version != 1 || provider != "imap" || placement.account_id != account {
return Err(ImapDataError);
}
Ok(placement)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(try_from = "String", into = "String")]
pub struct MailboxName(String);
impl TryFrom<String> for MailboxName {
type Error = ImapDataError;
fn try_from(value: String) -> Result<Self, Self::Error> {
Self::from_unicode(&value)
}
}
impl From<MailboxName> for String {
fn from(value: MailboxName) -> Self {
value.0
}
}
impl MailboxName {
pub fn from_unicode(name: &str) -> Result<Self, ImapDataError> {
validate_mailbox(name).map_err(|_| ImapDataError)?;
Ok(Self(if name.eq_ignore_ascii_case("INBOX") {
"INBOX".into()
} else {
name.into()
}))
}
pub fn as_str(&self) -> &str {
&self.0
}
pub fn wire_name(&self) -> String {
encode_name(&self.0)
}
pub fn from_wire(wire: &str) -> Result<Self, ImapDataError> {
if wire.is_empty() || wire.len() > 8192 || !wire.bytes().all(|b| (0x20..=0x7e).contains(&b))
{
return Err(ImapDataError);
}
let mut out = String::new();
let mut remaining = wire;
while let Some((literal, tail)) = remaining.split_once('&') {
out.push_str(literal);
let (encoded, tail) = tail.split_once('-').ok_or(ImapDataError)?;
if encoded.is_empty() {
out.push('&');
} else {
if !encoded
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b','))
{
return Err(ImapDataError);
}
let bytes = STANDARD_NO_PAD
.decode(encoded.replace(',', "/"))
.map_err(|_| ImapDataError)?;
if bytes.is_empty() || bytes.len() % 2 != 0 {
return Err(ImapDataError);
}
let units = bytes
.chunks_exact(2)
.map(|b| u16::from_be_bytes([b[0], b[1]]))
.collect::<Vec<_>>();
let decoded = String::from_utf16(&units).map_err(|_| ImapDataError)?;
if decoded.chars().any(|c| (' '..='~').contains(&c)) {
return Err(ImapDataError);
}
out.push_str(&decoded);
}
remaining = tail;
}
out.push_str(remaining);
// Reject noncanonical shift forms before folding the special INBOX name.
if encode_name(&out) != wire {
return Err(ImapDataError);
}
Self::from_unicode(&out)
}
}
fn encode_name(name: &str) -> String {
let mut out = String::new();
let mut unicode = Vec::new();
for ch in name.chars() {
if (' '..='~').contains(&ch) {
encode_run(&mut out, &mut unicode);
if ch == '&' {
out.push_str("&-");
} else {
out.push(ch);
}
} else {
let mut units = [0; 2];
for unit in ch.encode_utf16(&mut units) {
unicode.extend_from_slice(&unit.to_be_bytes());
}
}
}
encode_run(&mut out, &mut unicode);
out
}
fn encode_run(out: &mut String, bytes: &mut Vec<u8>) {
if !bytes.is_empty() {
out.push('&');
out.push_str(&STANDARD_NO_PAD.encode(&bytes).replace('/', ","));
out.push('-');
bytes.clear();
}
}
pub fn uid_batches(ids: &[NonZeroU32], batch: usize) -> Result<Vec<String>, ImapDataError> {
if batch == 0 || batch > 1000 || ids.len() > 1_000_000 {
return Err(ImapDataError);
}
let ordered = ids
.iter()
.map(|n| n.get())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
Ok(ordered
.chunks(batch)
.map(|chunk| {
let mut ranges = Vec::new();
let mut rest = chunk;
while let Some((&start, tail)) = rest.split_first() {
let mut end = start;
rest = tail;
while let Some((&next, tail)) = rest.split_first() {
if end.checked_add(1) != Some(next) {
break;
}
end = next;
rest = tail;
}
ranges.push(if start == end {
start.to_string()
} else {
format!("{start}:{end}")
});
}
ranges.join(",")
})
.collect())
}
pub fn new_uid_range(highest: u32, uid_next: u32) -> Result<Option<String>, ImapDataError> {
let end = uid_next.checked_sub(1).ok_or(ImapDataError)?;
if highest >= end {
return Ok(None);
}
let start = highest.checked_add(1).ok_or(ImapDataError)?;
Ok(Some(format!("{start}:{end}")))
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(try_from = "Vec<String>", into = "Vec<String>")]
pub struct ImapFlags(Vec<String>);
impl TryFrom<Vec<String>> for ImapFlags {
type Error = ImapDataError;
fn try_from(value: Vec<String>) -> Result<Self, Self::Error> {
Self::new(value)
}
}
impl From<ImapFlags> for Vec<String> {
fn from(value: ImapFlags) -> Self {
value.0
}
}
impl ImapFlags {
pub fn new(flags: Vec<String>) -> Result<Self, ImapDataError> {
if flags.len() > 128 {
return Err(ImapDataError);
}
let mut values = BTreeSet::new();
for flag in flags {
let keyword = flag.strip_prefix('\\').unwrap_or(&flag);
if flag.len() > 255
|| keyword.is_empty()
|| !keyword
.bytes()
.all(|b| b.is_ascii_graphic() && !b"(){%*\"\\]".contains(&b))
{
return Err(ImapDataError);
}
let canonical = match flag.to_ascii_lowercase().as_str() {
"\\seen" => "\\Seen",
"\\flagged" => "\\Flagged",
"\\answered" => "\\Answered",
"\\deleted" => "\\Deleted",
"\\draft" => "\\Draft",
"\\recent" => "\\Recent",
_ => &flag,
};
values.insert(canonical.to_owned());
}
Ok(Self(values.into_iter().collect()))
}
pub fn values(&self) -> &[String] {
&self.0
}
pub fn seen(&self) -> bool {
self.0.iter().any(|f| f == "\\Seen")
}
pub fn flagged(&self) -> bool {
self.0.iter().any(|f| f == "\\Flagged")
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ImapMailboxState {
pub name: MailboxName,
pub delimiter: Option<char>,
pub attributes: Vec<String>,
pub uid_validity: Option<u32>,
pub uid_next: Option<u32>,
pub highest_mod_seq: Option<u64>,
}
impl ImapMailboxState {
pub fn validated(mut self) -> Result<Self, ImapDataError> {
self.attributes = ImapFlags::new(self.attributes)?.into();
if self
.delimiter
.is_some_and(|c| !c.is_ascii() || c.is_control())
|| self.highest_mod_seq == Some(0)
{
return Err(ImapDataError);
}
let noselect = self
.attributes
.iter()
.any(|a| a.eq_ignore_ascii_case("\\Noselect"));
if noselect {
if self.uid_validity.is_some()
|| self.uid_next.is_some()
|| self.highest_mod_seq.is_some()
{
return Err(ImapDataError);
}
} else if self.uid_validity.is_none_or(|v| v == 0) || self.uid_next.is_none_or(|v| v == 0) {
return Err(ImapDataError);
}
Ok(self)
}
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ImapMessageState {
pub placement: ImapPlacement,
pub flags: ImapFlags,
}
@@ -0,0 +1,168 @@
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{fmt, net::IpAddr};
use zeroize::{Zeroize, ZeroizeOnDrop};
#[derive(Debug, thiserror::Error)]
#[error("Invalid IMAP/SMTP account configuration or credential input")]
pub struct ImapConfigError;
#[derive(Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum MailTls {
Implicit,
StartTls,
}
#[derive(Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum SentPolicy {
Server,
ClientAppend,
}
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct MailEndpoint {
pub host: String,
pub port: u16,
pub tls: MailTls,
pub username: String,
}
impl MailEndpoint {
fn canonicalize(&mut self) -> Result<(), ImapConfigError> {
if self.port == 0
|| self.username.is_empty()
|| self.username.len() > 1024
|| self.username.chars().any(char::is_control)
|| self.host.len() > 1024
|| self.host.chars().any(char::is_control)
{
return Err(ImapConfigError);
}
if let Ok(ip) = self.host.parse::<IpAddr>() {
self.host = ip.to_string();
return Ok(());
}
self.host = match url::Host::parse(&self.host).map_err(|_| ImapConfigError)? {
url::Host::Ipv4(ip) => ip.to_string(),
url::Host::Ipv6(ip) => ip.to_string(),
url::Host::Domain(domain) => {
let domain = domain.strip_suffix('.').unwrap_or(&domain);
if domain.len() > 253
|| domain.split('.').any(|label| {
label.is_empty()
|| label.len() > 63
|| label.starts_with('-')
|| label.ends_with('-')
|| !label
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'-')
})
{
return Err(ImapConfigError);
}
domain.to_ascii_lowercase()
}
};
Ok(())
}
}
#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
pub struct ImapAccountConfig {
pub address: String,
pub imap: MailEndpoint,
pub smtp: MailEndpoint,
pub sent_policy: SentPolicy,
pub sent_folder: String,
pub archive_folder: Option<String>,
pub trash_folder: Option<String>,
pub trusted_ca_pem: Option<String>,
}
impl ImapAccountConfig {
pub fn canonicalized(mut self) -> Result<Self, ImapConfigError> {
super::drafts::Recipient {
address: self.address.clone(),
name: None,
}
.validate()
.map_err(|_| ImapConfigError)?;
self.imap.canonicalize()?;
self.smtp.canonicalize()?;
for folder in std::iter::once(&self.sent_folder)
.chain(self.archive_folder.iter())
.chain(self.trash_folder.iter())
{
validate_mailbox(folder)?;
}
if self
.trusted_ca_pem
.as_ref()
.is_some_and(|pem| pem.is_empty() || pem.len() > 128 * 1024 || pem.contains('\0'))
{
return Err(ImapConfigError);
}
// X.509 parsing and hostname verification occur before authentication
// or persistence, in the transport that consumes these trust roots.
Ok(self)
}
pub fn identity(&self) -> Result<String, ImapConfigError> {
let canonical = self.clone().canonicalized()?;
// IMAP has no Google-style immutable subject. Pin the authenticated
// principal to its canonical endpoint and exact (case-sensitive) user.
let encoded = serde_json::to_vec(&("imap-principal-v1", &canonical.imap))
.map_err(|_| ImapConfigError)?;
Ok(format!("imap:{}", hex::encode(Sha256::digest(encoded))))
}
}
pub fn validate_mailbox(name: &str) -> Result<(), ImapConfigError> {
if name.is_empty() || name.len() > 2048 || name.chars().any(char::is_control) {
Err(ImapConfigError)
} else {
Ok(())
}
}
#[derive(Serialize, Deserialize, Zeroize, ZeroizeOnDrop)]
#[serde(deny_unknown_fields)]
pub struct ImapCredentials {
pub imap_password: String,
pub smtp_password: String,
}
impl ImapCredentials {
pub fn validate(&self) -> Result<(), ImapConfigError> {
if [&self.imap_password, &self.smtp_password]
.iter()
.any(|value| value.is_empty() || value.len() > 4096 || value.contains('\0'))
{
Err(ImapConfigError)
} else {
Ok(())
}
}
}
impl fmt::Debug for ImapCredentials {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("ImapCredentials([redacted])")
}
}
#[derive(Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ImapCapabilities {
pub move_messages: bool,
pub uidplus: bool,
pub condstore: bool,
pub qresync: bool,
pub idle: bool,
pub smtp_utf8: bool,
pub eight_bit_mime: bool,
}
@@ -0,0 +1,182 @@
use mail_parser::{Encoding, MessageParser, MimeHeaders, PartType};
use serde::{Deserialize, Serialize};
pub const MAX_PAYLOAD_BYTES: usize = 64 * 1024 * 1024;
pub const BLOB_CHUNK_BYTES: usize = 256 * 1024;
#[derive(Debug, thiserror::Error)]
pub enum MailError {
#[error("Mail content exceeds the configured payload limit")]
TooLarge,
#[error("Original mail content could not be decoded")]
InvalidMime,
#[error("Mail structure exceeds the supported part or header count")]
TooComplex,
}
#[derive(Clone, Serialize, Deserialize)]
pub struct Header {
pub name: String,
pub value: String,
}
pub struct DecodedAttachment {
pub part_index: u32,
pub filename: Option<String>,
pub mime_type: String,
pub content_id: Option<String>,
pub bytes: Vec<u8>,
}
pub struct DecodedMail {
pub subject: Option<String>,
pub headers: Vec<Header>,
/// Original decoded text/plain content; absent for HTML-only messages.
pub text: Option<String>,
/// Original decoded HTML; stored inertly and never loaded into a browser.
pub html: Option<String>,
pub search_text: String,
pub attachments: Vec<DecodedAttachment>,
}
/// Called on the bounded blocking ingestion worker, not on a Tokio reactor.
/// The caller retains the input bytes as the authoritative downloadable original.
pub fn decode_mime(raw: &[u8], limit: usize) -> Result<DecodedMail, MailError> {
with_parsed(raw, limit, |message| decode(message, raw, limit))?
}
pub(crate) fn with_parsed<T>(
raw: &[u8],
limit: usize,
inspect: impl FnOnce(&mail_parser::Message<'_>) -> T,
) -> Result<T, MailError> {
if limit == 0 || limit > MAX_PAYLOAD_BYTES || raw.len() > limit {
return Err(MailError::TooLarge);
}
let message = MessageParser::default()
.parse(raw)
.ok_or(MailError::InvalidMime)?;
let result = inspect(&message);
// Nested unencoded message/rfc822 structures use recursive value types. Drain
// them iteratively so dropping a parsed attachment tree does not recurse.
let mut pending = vec![message];
while let Some(mut message) = pending.pop() {
for part in message.parts.drain(..) {
if let PartType::Message(nested) = part.body {
pending.push(nested);
}
}
}
Ok(result)
}
pub(crate) fn decode(
message: &mail_parser::Message<'_>,
raw: &[u8],
limit: usize,
) -> Result<DecodedMail, MailError> {
if message.parts.is_empty() {
return Err(MailError::InvalidMime);
}
if message.parts.len() > 4096 || message.headers().len() > 4096 {
return Err(MailError::TooComplex);
}
let mut headers = Vec::new();
for header in message.headers() {
let bytes = raw
.get(header.offset_start as usize..header.offset_end as usize)
.ok_or(MailError::InvalidMime)?;
headers.push(Header {
name: header.name.as_str().into(),
value: String::from_utf8_lossy(bytes).trim().into(),
});
}
let mut text = Vec::new();
let mut html = Vec::new();
let mut search = Vec::new();
let mut decoded_bytes = 0;
for id in &message.text_body {
let part = message
.parts
.get(*id as usize)
.ok_or(MailError::InvalidMime)?;
if part.is_encoding_problem {
return Err(MailError::InvalidMime);
}
if let PartType::Text(value) = &part.body {
add_size(&mut decoded_bytes, value.len(), limit)?;
text.push(value.as_ref());
}
}
for id in &message.html_body {
let part = message
.parts
.get(*id as usize)
.ok_or(MailError::InvalidMime)?;
if part.is_encoding_problem {
return Err(MailError::InvalidMime);
}
if let PartType::Html(value) = &part.body {
add_size(&mut decoded_bytes, value.len(), limit)?;
html.push(value.as_ref());
}
}
for index in 0..message.text_body.len() {
if let Some(value) = message.body_text(index) {
search.push(value.into_owned());
}
}
let mut attachments = Vec::new();
for id in &message.attachments {
let part = message
.parts
.get(*id as usize)
.ok_or(MailError::InvalidMime)?;
if part.is_encoding_problem {
return Err(MailError::InvalidMime);
}
let encoded = raw
.get(part.offset_body as usize..part.offset_end as usize)
.ok_or(MailError::InvalidMime)?;
// Decode transfer encoding only. contents() converts text attachments to
// UTF-8, which would change a downloaded ISO-8859-1/UTF-16 attachment.
let bytes = match part.encoding {
Encoding::None => encoded.to_vec(),
Encoding::Base64 => mail_parser::decoders::base64::base64_decode(encoded)
.ok_or(MailError::InvalidMime)?,
Encoding::QuotedPrintable => {
mail_parser::decoders::quoted_printable::quoted_printable_decode(encoded)
.ok_or(MailError::InvalidMime)?
}
};
add_size(&mut decoded_bytes, bytes.len(), limit)?;
let mime_type = part
.content_type()
.map(|value| match value.subtype() {
Some(subtype) => format!("{}/{subtype}", value.ctype()),
None => value.ctype().into(),
})
.unwrap_or_else(|| "application/octet-stream".into());
attachments.push(DecodedAttachment {
part_index: *id,
filename: part.attachment_name().map(str::to_owned),
mime_type,
content_id: part.content_id().map(str::to_owned),
bytes,
});
}
let search_text = search.join("\n");
if search_text.len() > limit {
return Err(MailError::TooLarge);
}
Ok(DecodedMail {
subject: message.subject().map(str::to_owned),
headers,
text: (!text.is_empty()).then(|| text.join("\n")),
html: (!html.is_empty()).then(|| html.join("\n")),
search_text,
attachments,
})
}
fn add_size(total: &mut usize, size: usize, limit: usize) -> Result<(), MailError> {
if size > limit.saturating_sub(*total) {
return Err(MailError::TooLarge);
}
*total += size;
Ok(())
}
@@ -0,0 +1,39 @@
use serde::{Deserialize, Serialize};
pub struct MailCapabilities {
pub account_id: String,
pub provider: String,
pub placement_model: String,
pub read_unread: bool,
pub star_unstar: bool,
pub archive: bool,
pub trash_restore: bool,
pub existing_labels: bool,
pub move_copy: bool,
}
/// Explicit desired states, never toggles based on a potentially stale cache.
#[derive(Clone, Serialize, Deserialize)]
#[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)]
pub enum MailAction {
Read {
read: bool,
},
Star {
starred: bool,
},
Archive {},
Move {
destination_collection_id: String,
},
Copy {
destination_collection_id: String,
},
Trash {
trashed: bool,
},
Label {
collection_id: String,
present: bool,
},
}
@@ -0,0 +1,11 @@
pub mod calendar;
pub mod calendar_change;
pub mod drafts;
pub mod free_busy;
pub mod identity;
pub mod imap;
pub mod imap_account;
pub mod mail;
pub mod mail_change;
pub mod prepare;
pub mod submission;
@@ -0,0 +1,264 @@
use super::{
drafts::{DraftContent, DraftError, Recipient},
mail,
};
use mail_parser::{Address, MimeHeaders};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum PrepareKind {
Reply,
ReplyAll,
Forward,
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DraftContext {
pub source_message_id: String,
pub kind: String,
pub original_subject: String,
pub thread_id: Option<String>,
pub in_reply_to: Option<String>,
pub references: Vec<String>,
}
pub struct PreparedAttachment {
pub filename: Option<String>,
pub mime_type: String,
pub parameters: BTreeMap<String, String>,
pub content_id: Option<String>,
pub disposition: String,
pub bytes: Vec<u8>,
}
pub struct PreparedDraft {
pub content: DraftContent,
pub context: DraftContext,
pub attachments: Vec<PreparedAttachment>,
}
pub fn prepare(
raw: &[u8],
self_address: &str,
source: &str,
thread: Option<&str>,
kind: PrepareKind,
body: Option<String>,
to: Vec<Recipient>,
) -> Result<PreparedDraft, DraftError> {
if kind != PrepareKind::Forward && !to.is_empty() {
return Err(DraftError::Recipient);
}
let request = DraftContent {
to,
text: body,
..Default::default()
};
request.validate()?;
mail::with_parsed(raw, mail::MAX_PAYLOAD_BYTES, |message| {
let mut decoded =
mail::decode(message, raw, mail::MAX_PAYLOAD_BYTES).map_err(|_| DraftError::Source)?;
let subject = decoded.subject.take().unwrap_or_default();
let mut content = request;
let mut context = DraftContext {
source_message_id: source.into(),
original_subject: subject.clone(),
kind: match kind {
PrepareKind::Reply => "reply",
PrepareKind::ReplyAll => "reply_all",
PrepareKind::Forward => "forward",
}
.into(),
thread_id: None,
in_reply_to: None,
references: Vec::new(),
};
let mut attachments = Vec::new();
if kind == PrepareKind::Forward {
content.subject = prefix(&subject, "Fwd:");
let intro = format!(
"{}\n\n---------- Forwarded message ----------\nSubject: {}\n\n",
content.text.as_deref().unwrap_or_default(),
subject
);
content.html = decoded
.html
.map(|html| format!("<pre>{}</pre>{html}", escape(&intro)));
content.text = Some(format!(
"{intro}{}",
decoded.text.unwrap_or(decoded.search_text)
));
if decoded.attachments.len() > 256 {
return Err(DraftError::TooLarge);
}
for attachment in decoded.attachments {
let part = message
.parts
.get(attachment.part_index as usize)
.ok_or(DraftError::Source)?;
let mut parameters = BTreeMap::new();
if let Some(attributes) = part.content_type().and_then(|t| t.attributes.as_ref()) {
for attribute in attributes {
let name = attribute.name.to_ascii_lowercase();
if parameters
.insert(name, attribute.value.to_string())
.is_some()
{
return Err(DraftError::Header);
}
}
}
validate_parameters(&parameters)?;
let disposition = if part
.content_disposition()
.is_some_and(|d| d.ctype().eq_ignore_ascii_case("inline"))
{
"inline"
} else {
"attachment"
};
attachments.push(PreparedAttachment {
filename: attachment.filename,
mime_type: attachment.mime_type,
parameters,
content_id: attachment.content_id,
disposition: disposition.into(),
bytes: attachment.bytes,
});
}
} else {
content.subject = prefix(&subject, "Re:");
let mut seen = BTreeSet::from([self_address.to_ascii_lowercase()]);
add(
message.reply_to().or_else(|| message.from()),
&mut content.to,
&mut seen,
)?;
// A reply to one's own sent message addresses the original To list.
if kind == PrepareKind::ReplyAll || content.to.is_empty() {
for addresses in message.all_to() {
add(Some(addresses), &mut content.to, &mut seen)?;
}
}
if kind == PrepareKind::ReplyAll {
for addresses in message.all_cc() {
add(Some(addresses), &mut content.cc, &mut seen)?;
}
}
context.in_reply_to = message.message_id().map(str::to_owned);
if let Some(id) = &context.in_reply_to {
let inherited = message.references().as_text_list().or_else(|| {
message
.in_reply_to()
.as_text_list()
.filter(|ids| ids.len() == 1)
});
if let Some(ids) = inherited {
context.references.extend(ids.iter().map(|s| s.to_string()));
}
if context.references.last() != Some(id) {
context.references.push(id.clone());
}
context.thread_id = thread.map(str::to_owned);
}
}
content.validate()?;
context.validate()?;
Ok(PreparedDraft {
content,
context,
attachments,
})
})
.map_err(|_| DraftError::Source)?
}
fn add(
addresses: Option<&Address<'_>>,
target: &mut Vec<Recipient>,
seen: &mut BTreeSet<String>,
) -> Result<(), DraftError> {
if let Some(addresses) = addresses {
for address in addresses.iter() {
let recipient = Recipient {
address: address
.address
.as_deref()
.ok_or(DraftError::Recipient)?
.into(),
name: address.name.as_deref().map(str::to_owned),
};
recipient.validate()?;
if seen.insert(recipient.address.to_ascii_lowercase()) {
target.push(recipient);
}
}
}
Ok(())
}
fn prefix(subject: &str, prefix: &str) -> String {
if subject
.get(..prefix.len())
.is_some_and(|s| s.eq_ignore_ascii_case(prefix))
{
subject.into()
} else {
format!("{prefix} {subject}")
}
}
fn escape(value: &str) -> String {
value
.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
}
pub(crate) fn mime_token(value: &str) -> bool {
!value.is_empty()
&& value
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&b))
}
pub(crate) fn validate_parameters(parameters: &BTreeMap<String, String>) -> Result<(), DraftError> {
if parameters.len() > 32
|| parameters
.iter()
.map(|(k, v)| k.len() + v.len())
.sum::<usize>()
> 8192
|| parameters.iter().any(|(k, v)| {
!mime_token(k) || k != &k.to_ascii_lowercase() || v.chars().any(char::is_control)
})
{
return Err(DraftError::Header);
}
Ok(())
}
impl DraftContext {
pub fn validate(&self) -> Result<(), DraftError> {
let valid_id = |id: &str| {
id.len() <= 998
&& id
.split_once('@')
.is_some_and(|(a, b)| !a.is_empty() && !b.is_empty())
&& id
.bytes()
.all(|b| b.is_ascii_graphic() && !b"<>\"\\()".contains(&b))
};
if self.source_message_id.is_empty()
|| self.source_message_id.len() > 128
|| !matches!(self.kind.as_str(), "reply" | "reply_all" | "forward")
|| self.original_subject.len() > 8192
|| self.original_subject.chars().any(char::is_control)
|| self
.thread_id
.as_ref()
.is_some_and(|s| s.is_empty() || s.len() > 8192 || s.chars().any(char::is_control))
|| self.references.len() > 100
|| self.references.iter().map(String::len).sum::<usize>() > 16384
|| self.in_reply_to.as_ref().is_some_and(|id| !valid_id(id))
|| self.references.iter().any(|id| !valid_id(id))
{
return Err(DraftError::Header);
}
Ok(())
}
}
@@ -0,0 +1,324 @@
use super::{
drafts::{DraftContent, DraftError, Recipient},
mail::MAX_PAYLOAD_BYTES,
prepare::{mime_token, validate_parameters, DraftContext, PreparedAttachment},
};
use base64::{engine::general_purpose::STANDARD, Engine as _};
use mail_builder::{
headers::{address::Address, content_type::ContentType},
mime::MimePart,
MessageBuilder,
};
use std::{collections::BTreeSet, io::Write};
mod sent_fingerprint;
pub use sent_fingerprint::SentFingerprint;
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum SubmissionTransport {
Gmail,
Smtp,
}
#[derive(Clone)]
pub struct FrozenMessage {
pub wire: Vec<u8>,
/// SMTP retains Bcc locally, but never transmits that header to recipients.
/// Gmail needs Bcc in its submitted MIME to supply envelope recipients.
pub sent_copy: Option<Vec<u8>>,
pub recipients: Vec<String>,
pub thread_id: Option<String>,
pub message_id: String,
}
pub fn freeze(
sender: &str,
content: &DraftContent,
context: Option<&DraftContext>,
attachments: &[PreparedAttachment],
message_id: &str,
now_ms: i64,
transport: SubmissionTransport,
) -> Result<FrozenMessage, DraftError> {
content.validate()?;
Recipient {
address: sender.into(),
name: None,
}
.validate()?;
validate_identity(message_id, now_ms)?;
let mut seen = BTreeSet::new();
let recipients = content
.to
.iter()
.chain(&content.cc)
.chain(&content.bcc)
.filter(|r| seen.insert(r.address.to_ascii_lowercase()))
.map(|r| r.address.clone())
.collect::<Vec<_>>();
if recipients.is_empty() {
return Err(DraftError::Recipient);
}
if attachments.len() > 256
|| attachments.iter().map(|a| a.bytes.len()).sum::<usize>() > MAX_PAYLOAD_BYTES
{
return Err(DraftError::TooLarge);
}
let mut builder = MessageBuilder::new()
.from(sender)
.subject(content.subject.as_str())
.message_id(message_id)
.date(now_ms / 1000);
if !content.to.is_empty() {
builder = builder.to(addresses(&content.to));
}
if !content.cc.is_empty() {
builder = builder.cc(addresses(&content.cc));
}
if !content.bcc.is_empty() {
builder = builder.bcc(addresses(&content.bcc));
}
let mut thread_id = None;
if let Some(context) = context {
context.validate()?;
if context.kind != "forward" {
if let Some(id) = context.in_reply_to.as_deref() {
builder = builder.in_reply_to(id);
}
if !context.references.is_empty() {
builder = builder.references(
context
.references
.iter()
.map(String::as_str)
.collect::<Vec<_>>(),
);
}
if subject_key(&content.subject) == subject_key(&context.original_subject) {
thread_id = context.thread_id.clone();
}
}
}
let mut inline = Vec::new();
let mut ordinary = Vec::new();
for attachment in attachments {
let part = attachment_part(attachment)?;
if attachment.disposition == "inline" && content.html.is_some() {
inline.push(part);
} else {
ordinary.push(part);
}
}
let html = content.html.as_deref().map(|html| {
let html = MimePart::new("text/html", html);
if inline.is_empty() {
html
} else {
let mut related = vec![html];
related.extend(inline);
MimePart::new("multipart/related", related)
}
});
let body = match (content.text.as_deref(), html) {
(Some(text), Some(html)) => MimePart::new(
"multipart/alternative",
vec![MimePart::new("text/plain", text), html],
),
(Some(text), None) => MimePart::new("text/plain", text),
(None, Some(html)) => html,
(None, None) => MimePart::new("text/plain", ""),
};
let body = if ordinary.is_empty() {
body
} else {
let mut parts = vec![body];
parts.extend(ordinary);
MimePart::new("multipart/mixed", parts)
};
let mut output = LimitedBytes::default();
builder
.body(body)
.write_to(&mut output)
.map_err(|_| DraftError::TooLarge)?;
// The DATA terminator follows a complete CRLF line. Freeze this byte now
// so SMTP transport and the independently retained Sent copy stay identical.
if transport == SubmissionTransport::Smtp && !output.0.ends_with(b"\r\n") {
output
.write_all(b"\r\n")
.map_err(|_| DraftError::TooLarge)?;
}
let (wire, sent_copy) = if transport == SubmissionTransport::Smtp && !content.bcc.is_empty() {
let wire = without_bcc(&output.0)?;
(wire, Some(output.0))
} else {
(output.0, None)
};
Ok(FrozenMessage {
wire,
sent_copy,
recipients,
thread_id,
message_id: message_id.into(),
})
}
fn validate_identity(message_id: &str, now_ms: i64) -> Result<(), DraftError> {
if now_ms < 0
|| now_ms / 1000 > 253402300799
|| message_id.len() > 998
|| !message_id
.split_once('@')
.is_some_and(|(a, b)| !a.is_empty() && !b.is_empty())
|| !message_id
.bytes()
.all(|b| b.is_ascii_graphic() && !b"<>\"\\()".contains(&b))
{
return Err(DraftError::Header);
}
Ok(())
}
/// A deliberate new submission changes only its outer identity and timestamp.
/// The frozen body, including any nested messages, remains byte-for-byte intact.
pub fn reidentify(original: &[u8], message_id: &str, now_ms: i64) -> Result<Vec<u8>, DraftError> {
use mail_builder::headers::{date::Date, message_id::MessageId, Header};
use mail_parser::HeaderName;
validate_identity(message_id, now_ms)?;
if original.len() > MAX_PAYLOAD_BYTES {
return Err(DraftError::TooLarge);
}
let parsed = mail_parser::MessageParser::default()
.parse_headers(original)
.ok_or(DraftError::Source)?;
for name in [HeaderName::MessageId, HeaderName::Date] {
if parsed.headers().iter().filter(|h| h.name == name).count() != 1 {
return Err(DraftError::Source);
}
}
let mut output = LimitedBytes::default();
let write = |result: std::io::Result<()>| result.map_err(|_| DraftError::TooLarge);
write(output.write_all(b"Message-ID: "))?;
MessageId::new(message_id)
.write_header(&mut output, 12)
.map_err(|_| DraftError::TooLarge)?;
write(output.write_all(b"Date: "))?;
Date::new(now_ms / 1000)
.write_header(&mut output, 6)
.map_err(|_| DraftError::TooLarge)?;
let mut offset = 0;
for header in parsed
.headers()
.iter()
.filter(|h| matches!(h.name, HeaderName::MessageId | HeaderName::Date))
{
let start = header.offset_field as usize;
let end = header.offset_end as usize;
write(output.write_all(original.get(offset..start).ok_or(DraftError::Source)?))?;
offset = end;
}
write(output.write_all(original.get(offset..).ok_or(DraftError::Source)?))?;
Ok(output.0)
}
fn addresses(recipients: &[Recipient]) -> Address<'_> {
Address::new_list(
recipients
.iter()
.map(|r| Address::new_address(r.name.as_deref(), r.address.as_str()))
.collect(),
)
}
fn subject_key(mut subject: &str) -> &str {
subject = subject.trim();
while subject
.get(..3)
.is_some_and(|s| s.eq_ignore_ascii_case("re:"))
{
subject = subject[3..].trim_start();
}
subject
}
fn attachment_part(a: &PreparedAttachment) -> Result<MimePart<'_>, DraftError> {
if !a
.mime_type
.split_once('/')
.is_some_and(|(a, b)| mime_token(a) && mime_token(b))
|| a.mime_type.len() > 255
|| !matches!(a.disposition.as_str(), "attachment" | "inline")
|| a.filename
.as_ref()
.is_some_and(|s| s.is_empty() || s.len() > 1024 || s.chars().any(char::is_control))
|| a.content_id.as_ref().is_some_and(|s| {
s.is_empty()
|| s.len() > 255
|| !s
.bytes()
.all(|b| b.is_ascii_graphic() && !b"<>\"\\()[];,:".contains(&b))
})
{
return Err(DraftError::Header);
}
validate_parameters(&a.parameters)?;
let mut content_type = ContentType::new(a.mime_type.as_str());
for (key, value) in &a.parameters {
content_type = content_type.attribute(key.as_str(), value.as_str());
}
let mut disposition = ContentType::new(a.disposition.as_str());
if let Some(filename) = a.filename.as_deref() {
disposition = disposition.attribute("filename", filename);
}
let mut part = if a.mime_type.eq_ignore_ascii_case("message/rfc822")
|| a.mime_type.to_ascii_lowercase().starts_with("multipart/")
{
// MIME forbids base64 for these container types. Preserve the original
// nested bytes; the SMTP executor must negotiate their transport needs.
MimePart::new(content_type, a.bytes.as_slice()).transfer_encoding("8bit")
} else {
// Treat every leaf attachment as opaque bytes, including inline text.
// Explicit base64 avoids body newline/charset normalization by encoders.
let mut encoded = LimitedBytes::default();
for chunk in a.bytes.chunks(57) {
encoded
.write_all(STANDARD.encode(chunk).as_bytes())
.map_err(|_| DraftError::TooLarge)?;
encoded
.write_all(b"\r\n")
.map_err(|_| DraftError::TooLarge)?;
}
MimePart::new(content_type, encoded.0).transfer_encoding("base64")
}
.header("Content-Disposition", disposition);
if let Some(id) = a.content_id.as_deref() {
part = part.cid(id);
}
Ok(part)
}
fn without_bcc(original: &[u8]) -> Result<Vec<u8>, DraftError> {
let headers = mail_parser::MessageParser::default()
.parse_headers(original)
.ok_or(DraftError::Source)?;
let mut output = Vec::with_capacity(original.len());
let mut offset = 0;
for header in headers
.headers()
.iter()
.filter(|h| h.name == mail_parser::HeaderName::Bcc)
{
let start = header.offset_field as usize;
let end = header.offset_end as usize;
output.extend_from_slice(original.get(offset..start).ok_or(DraftError::Source)?);
offset = end;
}
output.extend_from_slice(original.get(offset..).ok_or(DraftError::Source)?);
Ok(output)
}
#[derive(Default)]
struct LimitedBytes(Vec<u8>);
impl Write for LimitedBytes {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
if bytes.len() > MAX_PAYLOAD_BYTES.saturating_sub(self.0.len()) {
return Err(std::io::Error::other("MIME exceeds payload limit"));
}
self.0.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
@@ -0,0 +1,101 @@
use super::DraftError;
use mail_parser::{HeaderName, HeaderValue};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
/// A versioned content comparison, not a proof that a provider performed a write.
/// SMTP acceptance and a fresh, unique placement observation are separate facts.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SentFingerprint {
version: u32,
headers: String,
body: String,
bcc: Option<String>,
}
impl SentFingerprint {
pub fn from_mime(raw: &[u8]) -> Result<Self, DraftError> {
if raw.len() > crate::domain::mail::MAX_PAYLOAD_BYTES {
return Err(DraftError::TooLarge);
}
let body = raw
.windows(4)
.position(|w| w == b"\r\n\r\n")
.ok_or(DraftError::Header)?
+ 4;
if body > 1024 * 1024 {
return Err(DraftError::TooLarge);
}
let message = mail_parser::MessageParser::default()
.parse_headers(raw)
.ok_or(DraftError::Header)?;
if message.headers().len() > 512
|| message.message_id().is_none()
|| message.from().is_none()
|| message.date().is_none()
{
return Err(DraftError::Header);
}
let value = |name: HeaderName<'static>| -> Result<Option<&HeaderValue<'_>>, DraftError> {
let mut values = message.headers().iter().filter(|h| h.name == name);
let first = values.next().map(|h| &h.value);
if values.next().is_some() {
return Err(DraftError::Header);
}
Ok(first)
};
let mut headers = Vec::new();
// Compare parsed values so valid folding/encoded header forms remain
// equivalent. Trace headers are deliberately outside message identity.
for name in [
HeaderName::MessageId,
HeaderName::From,
HeaderName::Sender,
HeaderName::ReplyTo,
HeaderName::To,
HeaderName::Cc,
HeaderName::Date,
HeaderName::Subject,
HeaderName::InReplyTo,
HeaderName::References,
HeaderName::MimeVersion,
HeaderName::ContentType,
HeaderName::ContentTransferEncoding,
] {
headers.push(value(name)?);
}
let headers = serde_json::to_vec(&headers).map_err(|_| DraftError::Header)?;
let bcc = value(HeaderName::Bcc)?
.map(|v| {
serde_json::to_vec(v)
.map(|bytes| hex::encode(Sha256::digest(bytes)))
.map_err(|_| DraftError::Header)
})
.transpose()?;
Ok(Self {
version: 1,
headers: hex::encode(Sha256::digest(headers)),
body: hex::encode(Sha256::digest(&raw[body..])),
bcc,
})
}
pub fn matches(&self, observed: &Self) -> bool {
let valid = |value: &Self| {
value.version == 1
&& std::iter::once(&value.headers)
.chain(std::iter::once(&value.body))
.chain(value.bcc.iter())
.all(|hash| {
hash.len() == 64
&& hash
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
})
};
valid(self)
&& valid(observed)
&& self.headers == observed.headers
&& self.body == observed.body
&& (observed.bcc.is_none() || self.bcc == observed.bcc)
}
}
+852
View File
@@ -0,0 +1,852 @@
use crate::{
domain::identity::ProfileId,
profile,
secrets::{SecretError, SecretStore},
store::{Store, StoreError, StoreStatus},
};
use serde::Serialize;
use std::{path::PathBuf, sync::Arc};
use zeroize::Zeroizing;
mod recovery;
mod repair;
pub use crate::maintenance::{BackupInput, BackupUpload};
pub use recovery::ProfileRestoreReport;
pub use repair::RepairResult;
#[derive(Debug, thiserror::Error)]
pub enum EngineError {
#[error(transparent)]
Account(#[from] crate::accounts::AccountError),
#[cfg(feature = "test-harness")]
#[error(
"Test configuration requires loopback endpoints, valid limits, and synthetic credentials"
)]
InvalidTestConfiguration,
#[error("Engine is unavailable")]
Unavailable,
#[error("Profile metadata is missing or invalid; original files were preserved")]
InvalidProfile,
#[error("Required profile key is missing or invalid; original files were preserved")]
MissingKey,
#[error("Restore was not activated, but new key cleanup failed for profile {profile_id}; remove only that profile's database/API key entries")]
RestoreCleanupIncomplete { profile_id: String },
#[error("Restore recovery requires inspection for profile {profile_id}; files and keys were preserved where ownership or activation could not be proven")]
RestoreRecoveryPending { profile_id: String },
#[error(transparent)]
Storage(#[from] StoreError),
#[error(transparent)]
Secret(#[from] SecretError),
}
pub struct EngineConfig {
pub directory: PathBuf,
pub secrets: Arc<dyn SecretStore>,
}
pub struct Engine {
maintenance: crate::maintenance::Maintenance,
operations: crate::operations::OperationWorker,
send_admission: tokio::sync::Semaphore,
compose_slots: Arc<tokio::sync::Semaphore>,
scheduler: crate::scheduler::Scheduler,
accounts: Arc<crate::accounts::Accounts>,
mail: Arc<crate::mail::MailSync>,
calendar: Arc<crate::calendar::CalendarSync>,
#[cfg(feature = "test-harness")]
test_config: Option<crate::test_controls::TestConfig>,
store: Store,
profile_id: ProfileId,
directory: PathBuf,
authorization: Zeroizing<String>,
_profile_lock: Arc<std::fs::File>,
}
#[derive(Serialize)]
pub struct EngineStatus {
pub resources: crate::resources::ResourceStatus,
pub operation_worker_error: Option<String>,
pub sync: Vec<crate::sync_status::SyncScopeStatus>,
pub background_sync: bool,
pub scheduler_error: Option<String>,
pub version: String,
pub api_version: String,
pub profile_id: String,
pub storage: StoreStatus,
pub protected_paths: Vec<PathBuf>,
}
impl Engine {
pub async fn change_message(
&self,
input: crate::store::EnqueueMailChange,
) -> Result<crate::store::Operation, StoreError> {
self.store
.enqueue_mail_change(input, self.accounts.http.clock.now_ms())
.await
}
pub async fn mail_capabilities(
&self,
account: String,
) -> Result<crate::domain::mail_change::MailCapabilities, StoreError> {
let row = self
.store
.account(account.clone())
.await?
.ok_or(StoreError::NotFound)?;
let google = row.account.provider == "google";
Ok(crate::domain::mail_change::MailCapabilities {
account_id: account,
provider: row.account.provider.clone(),
placement_model: if google { "labels" } else { "folders" }.into(),
read_unread: google || row.account.provider == "imap",
star_unstar: google || row.account.provider == "imap",
archive: google || row.account.provider == "imap",
trash_restore: google || row.account.provider == "imap",
existing_labels: google,
move_copy: row.account.provider == "imap",
})
}
pub async fn send_draft(
&self,
account: String,
draft_id: String,
request_id: String,
expected_version: Option<u64>,
) -> Result<crate::store::Operation, StoreError> {
if let Some(original) = self
.store
.find_send_request(
account.clone(),
request_id.clone(),
draft_id.clone(),
expected_version,
)
.await?
{
return Ok(original);
}
let _admission = self
.send_admission
.try_acquire()
.map_err(|_| StoreError::Busy)?;
let permit = self
.compose_slots
.clone()
.acquire_owned()
.await
.map_err(|_| StoreError::Unavailable)?;
if let Some(original) = self
.store
.find_send_request(
account.clone(),
request_id.clone(),
draft_id.clone(),
expected_version,
)
.await?
{
return Ok(original);
}
let owner = self
.store
.account(account.clone())
.await?
.ok_or(StoreError::NotFound)?;
let draft = self
.store
.get_draft(account.clone(), draft_id.clone())
.await?;
if expected_version.is_some_and(|v| v != draft.version) {
return Err(StoreError::VersionConflict);
}
let mut attachments = Vec::with_capacity(draft.attachments.len());
for a in &draft.attachments {
let bytes = self
.store
.read_blob(
account.clone(),
crate::store::StoredBlob {
id: a.blob_id.clone(),
byte_length: a.byte_length,
sha256: a.sha256.clone(),
},
)
.await?;
attachments.push(crate::domain::prepare::PreparedAttachment {
filename: a.filename.clone(),
mime_type: a.mime_type.clone(),
parameters: a.parameters.clone(),
content_id: a.content_id.clone(),
disposition: a.disposition.clone(),
bytes,
});
}
let now = self.accounts.http.clock.now_ms();
let version = draft.version;
let sender = owner.account.address;
let frozen_sender = sender.clone();
let transport = match owner.account.provider.as_str() {
"google" => crate::domain::submission::SubmissionTransport::Gmail,
"imap" => crate::domain::submission::SubmissionTransport::Smtp,
_ => return Err(StoreError::InvalidInput),
};
let (frozen, permit) = tokio::task::spawn_blocking(move || {
let message_id = format!("{}@nuncio.invalid", uuid::Uuid::new_v4());
let result = crate::domain::submission::freeze(
&frozen_sender,
&draft.content,
draft.context.as_ref(),
&attachments,
&message_id,
now,
transport,
);
(result, permit)
})
.await
.map_err(|_| StoreError::Unavailable)?;
self.store
.enqueue_send_guarded(
crate::store::EnqueueSend {
account_id: account,
request_id,
draft_id,
expected_version,
snapshot_version: version,
sender,
frozen: frozen.map_err(|_| StoreError::InvalidInput)?,
},
now,
Some(permit),
)
.await
}
pub async fn operation(
&self,
account: String,
id: String,
) -> Result<crate::store::Operation, StoreError> {
self.store.get_operation(account, id).await
}
pub async fn list_operations(
&self,
account: String,
size: u32,
token: Option<String>,
) -> Result<crate::store::OperationPage, StoreError> {
self.store.query_operations(account, size, token).await
}
pub async fn operation_attempts(
&self,
account: String,
id: String,
size: u32,
token: Option<String>,
) -> Result<crate::store::OperationAttemptPage, StoreError> {
self.store
.query_operation_attempts(account, id, size, token)
.await
}
pub async fn cancel_operation(
&self,
account: String,
id: String,
) -> Result<crate::store::Operation, StoreError> {
self.store
.cancel_operation(account, id, self.accounts.http.clock.now_ms())
.await
}
pub async fn reconcile_operation(
&self,
input: crate::store::ReconcileOperation,
) -> Result<crate::store::Operation, StoreError> {
let operation = self
.store
.request_reconciliation(input, self.accounts.http.clock.now_ms())
.await?;
#[cfg(feature = "test-harness")]
if let Some(config) = &self.accounts.http.test_config {
let (_sender, stop) = tokio::sync::watch::channel(false);
config
.checkpoint("reconciliation_after_admission", stop)
.await
.map_err(|_| StoreError::Unavailable)?;
}
Ok(operation)
}
pub async fn resolve_operation(
&self,
input: crate::store::ResolveOperation,
) -> Result<crate::store::Operation, StoreError> {
use crate::{
domain::submission::{reidentify, FrozenMessage},
store::ResolutionDecision,
};
if let Some(original) = self.store.find_resolution(input.clone()).await? {
return Ok(original);
}
if !matches!(input.decision, ResolutionDecision::Resend { .. }) {
return self
.store
.resolve_operation(input, None, self.accounts.http.clock.now_ms())
.await;
}
let _admission = self
.send_admission
.try_acquire()
.map_err(|_| StoreError::Busy)?;
let permit = self
.compose_slots
.clone()
.acquire_owned()
.await
.map_err(|_| StoreError::Unavailable)?;
if let Some(original) = self.store.find_resolution(input.clone()).await? {
return Ok(original);
}
let payload = self
.store
.send_payload(input.account_id.clone(), input.operation_id.clone())
.await?;
let wire = self
.store
.read_blob(input.account_id.clone(), payload.wire)
.await?;
let sent_copy = match payload.sent_copy {
Some(blob) => Some(self.store.read_blob(input.account_id.clone(), blob).await?),
None => None,
};
let now = self.accounts.http.clock.now_ms();
let (frozen, permit) = tokio::task::spawn_blocking(move || {
let result = (|| {
let message_id = format!("{}@nuncio.invalid", uuid::Uuid::new_v4());
Ok::<_, StoreError>(FrozenMessage {
wire: reidentify(&wire, &message_id, now)
.map_err(|_| StoreError::InvalidInput)?,
sent_copy: sent_copy
.as_ref()
.map(|b| reidentify(b, &message_id, now))
.transpose()
.map_err(|_| StoreError::InvalidInput)?,
recipients: payload.recipients,
thread_id: payload.thread_id,
message_id,
})
})();
(result, permit)
})
.await
.map_err(|_| StoreError::Unavailable)?;
self.store
.resolve_operation_guarded(input, Some(frozen?), now, Some(permit))
.await
}
pub async fn prepare_draft(
&self,
account: String,
message: String,
kind: crate::domain::prepare::PrepareKind,
body: Option<String>,
to: Vec<crate::domain::drafts::Recipient>,
) -> Result<crate::store::Draft, StoreError> {
let permit = self
.compose_slots
.clone()
.try_acquire_owned()
.map_err(|_| StoreError::Busy)?;
let source = self
.store
.draft_source(account.clone(), message.clone())
.await?;
let raw = self.store.read_blob(account.clone(), source.raw).await?;
let (prepared, permit) = tokio::task::spawn_blocking(move || {
let prepared = crate::domain::prepare::prepare(
&raw,
&source.address,
&message,
source.thread_id.as_deref(),
kind,
body,
to,
);
(prepared, permit)
})
.await
.map_err(|_| StoreError::Unavailable)?;
self.store
.create_prepared_draft(
account,
prepared.map_err(|_| StoreError::InvalidInput)?,
self.accounts.http.clock.now_ms(),
permit,
)
.await
}
pub async fn begin_draft_upload(
&self,
input: crate::store::DraftUploadInput,
) -> Result<crate::store::DraftUpload, StoreError> {
self.store
.begin_draft_upload(input, self.accounts.http.clock.now_ms())
.await
}
pub async fn append_draft_upload(
&self,
upload: &crate::store::DraftUpload,
offset: u64,
data: Vec<u8>,
) -> Result<(), StoreError> {
self.store.append_draft_upload(upload, offset, data).await
}
pub async fn finish_draft_upload(
&self,
upload: crate::store::DraftUpload,
) -> Result<crate::store::Draft, StoreError> {
self.store
.finish_draft_upload(upload, self.accounts.http.clock.now_ms())
.await
}
pub async fn save_draft(
&self,
input: crate::store::SaveDraft,
) -> Result<crate::store::Draft, StoreError> {
self.store
.save_draft(input, self.accounts.http.clock.now_ms())
.await
}
pub async fn get_draft(
&self,
account: String,
id: String,
) -> Result<crate::store::Draft, StoreError> {
self.store.get_draft(account, id).await
}
pub async fn list_drafts(
&self,
account: String,
size: u32,
token: Option<String>,
) -> Result<crate::store::DraftPage, StoreError> {
self.store.query_drafts(account, size, token).await
}
pub async fn delete_draft(
&self,
account: String,
id: String,
version: u64,
) -> Result<(), StoreError> {
self.store.delete_draft(account, id, version).await
}
pub async fn watch_changes(
&self,
after: u64,
) -> Result<crate::changes::ChangeReader, StoreError> {
self.store.watch_changes(after).await
}
#[cfg(feature = "test-harness")]
pub async fn open_for_test(
config: EngineConfig,
settings: crate::test_controls::TestConfig,
) -> Result<Self, EngineError> {
settings
.validate()
.map_err(|_| EngineError::InvalidTestConfiguration)?;
if !config.secrets.is_synthetic() {
return Err(EngineError::InvalidTestConfiguration);
}
let http = crate::providers::google::http::GoogleHttp::for_test(&settings)?;
let mut engine = Self::open_inner(config, http).await?;
engine.test_config = Some(settings);
Ok(engine)
}
#[cfg(feature = "test-harness")]
pub fn test_config(&self) -> Option<&crate::test_controls::TestConfig> {
self.test_config.as_ref()
}
pub async fn open(config: EngineConfig) -> Result<Self, EngineError> {
Self::open_inner(
config,
crate::providers::google::http::GoogleHttp::production()?,
)
.await
}
async fn open_inner(
config: EngineConfig,
http: crate::providers::google::http::GoogleHttp,
) -> Result<Self, EngineError> {
#[cfg(feature = "test-harness")]
http.refresh_test_clock().await?;
tracing::info!("Preparing local profile");
let directory = config.directory.clone();
let secrets = config.secrets.clone();
let profile = tokio::task::spawn_blocking(move || {
profile::prepare(&directory, config.secrets.as_ref())
})
.await
.map_err(|_| EngineError::Unavailable)??;
let profile_lock = Arc::new(profile.lock);
let options = crate::store::StoreOpenOptions {
#[cfg(feature = "test-harness")]
test_config: http.test_config.clone(),
};
tracing::info!("Opening encrypted database");
let store =
Store::open_with_options(&config.directory, profile.database_key, options).await?;
tracing::info!("Checking interrupted restore jobs");
if let Err(error) = recovery::recover_restores(
&store,
config.directory.clone(),
profile.id,
secrets.clone(),
profile_lock.clone(),
)
.await
{
let _ = store.close().await;
return Err(error);
}
let maintenance = crate::maintenance::Maintenance::new(
config.directory.clone(),
secrets.clone(),
store.clone(),
profile_lock.clone(),
);
tracing::info!("Recovering local drafts and durable operations");
store.recover_draft_uploads().await?;
store.recover_operations(http.clock.now_ms()).await?;
tracing::info!("Loading account state and credential cleanup");
let accounts =
crate::accounts::Accounts::new(store.clone(), profile.id.to_string(), secrets, http)
.await?;
tracing::info!("Recovering interrupted sync runs");
let sync_recovery_started = std::time::Instant::now();
let mail = crate::mail::MailSync::new(accounts.clone(), store.clone()).await?;
tracing::info!(
elapsed_ms = sync_recovery_started.elapsed().as_millis() as u64,
"Interrupted sync recovery complete"
);
let calendar = crate::calendar::CalendarSync::new(
accounts.clone(),
store.clone(),
mail.coordinator.clone(),
);
tracing::info!("Recovery checks complete");
tracing::info!("Starting background workers");
let scheduler =
crate::scheduler::Scheduler::start(store.clone(), mail.clone(), calendar.clone())
.await?;
let compose_slots = Arc::new(tokio::sync::Semaphore::new(1));
let operations = crate::operations::OperationWorker::start(
store.clone(),
accounts.clone(),
mail.coordinator.clone(),
compose_slots.clone(),
);
tracing::info!(
background_sync = scheduler.enabled,
"Background workers started"
);
Ok(Self {
maintenance,
operations,
send_admission: tokio::sync::Semaphore::new(64),
compose_slots,
scheduler,
accounts,
mail,
calendar,
#[cfg(feature = "test-harness")]
test_config: None,
store,
profile_id: profile.id,
directory: config.directory,
authorization: profile.authorization,
_profile_lock: profile_lock,
})
}
pub async fn status(&self) -> Result<EngineStatus, EngineError> {
let protected_directory =
std::fs::canonicalize(&self.directory).map_err(StoreError::from)?;
let (storage, schedules) = self.store.status_with_schedules().await?;
let now = self.accounts.http.clock.now_ms();
let (poll, enabled) = self.accounts.http.sync_policy();
let mut resources = self.accounts.http.resources.snapshot();
resources.account_requests = self.mail.coordinator.admitted_requests();
resources.account_request_limit = 64;
resources.store_queue_depth = self.store.queue_depth();
resources.store_queue_limit = 64;
Ok(EngineStatus {
resources,
operation_worker_error: self.operations.error(),
sync: schedules
.into_iter()
.map(|s| crate::sync_status::SyncScopeStatus::from_schedule(s, now, poll, enabled))
.collect(),
background_sync: self.scheduler.enabled,
scheduler_error: self.scheduler.error(),
version: env!("CARGO_PKG_VERSION").into(),
api_version: "nuncio.v2".into(),
profile_id: self.profile_id.to_string(),
storage,
protected_paths: [
"store.db",
"store.db-wal",
"store.db-shm",
"store.db-journal",
"store.lock",
"profile.json",
"profile.lock",
]
.iter()
.map(|name| protected_directory.join(name))
.collect(),
})
}
pub async fn shutdown(self) -> Result<(), EngineError> {
self.scheduler.shutdown().await;
self.operations.shutdown().await;
self.calendar.shutdown().await;
self.mail.shutdown().await;
self.accounts.shutdown().await;
self.store.close().await.map_err(EngineError::from)
}
pub async fn connect_imap(
&self,
config: crate::domain::imap_account::ImapAccountConfig,
credentials: crate::domain::imap_account::ImapCredentials,
account: Option<String>,
) -> Result<crate::accounts::ImapConnection, crate::accounts::AccountError> {
let config = config
.canonicalized()
.map_err(|_| crate::accounts::AccountError::Invalid)?;
let lane = account.clone().unwrap_or(
config
.identity()
.map_err(|_| crate::accounts::AccountError::Invalid)?,
);
let _permit = self.mail.coordinator.acquire(&lane).await?;
self.accounts
.connect_imap(config, credentials, account)
.await
}
pub async fn imap_config(
&self,
id: String,
) -> Result<
(
crate::domain::imap_account::ImapAccountConfig,
crate::domain::imap_account::ImapCapabilities,
),
crate::accounts::AccountError,
> {
self.accounts.imap_config(id).await
}
pub async fn update_imap_account(
&self,
id: String,
version: u64,
config: crate::domain::imap_account::ImapAccountConfig,
credentials: Option<crate::domain::imap_account::ImapCredentials>,
) -> Result<crate::accounts::ImapConnection, crate::accounts::AccountError> {
let config = config
.canonicalized()
.map_err(|_| crate::accounts::AccountError::Invalid)?;
let _permit = self.mail.coordinator.acquire(&id).await?;
self.accounts
.update_imap(id, version, config, credentials)
.await
}
pub async fn begin_google_auth(
&self,
request: crate::accounts::GoogleAuthRequest,
) -> Result<crate::accounts::AuthSession, crate::accounts::AccountError> {
self.accounts.begin(request).await
}
pub async fn auth_status(
&self,
id: &str,
) -> Result<crate::accounts::AuthSession, crate::accounts::AccountError> {
self.accounts.auth_status(id).await
}
pub async fn list_accounts(
&self,
) -> Result<Vec<crate::accounts::Account>, crate::accounts::AccountError> {
self.accounts.list().await
}
pub async fn disconnect_account(&self, id: &str) -> Result<(), crate::accounts::AccountError> {
let _permit = self.mail.coordinator.acquire(id).await?;
self.accounts.disconnect(id).await
}
pub async fn show_account(
&self,
id: &str,
) -> Result<crate::accounts::Account, crate::accounts::AccountError> {
self.accounts.show(id).await
}
pub async fn list_accounts_including_archived(
&self,
include_archived: bool,
) -> Result<Vec<crate::accounts::Account>, crate::accounts::AccountError> {
self.accounts
.list_including_archived(include_archived)
.await
}
pub async fn edit_account_name(
&self,
id: &str,
version: u64,
name: String,
) -> Result<crate::accounts::Account, crate::accounts::AccountError> {
let _permit = self.mail.coordinator.acquire(id).await?;
self.accounts.edit_name(id, version, name).await
}
pub async fn change_account_lifecycle(
&self,
id: &str,
action: crate::store::AccountLifecycle,
) -> Result<(crate::accounts::Account, bool), crate::accounts::AccountError> {
let _permit = self.mail.coordinator.acquire(id).await?;
self.accounts.lifecycle(id, action).await
}
pub async fn preview_account_purge(
&self,
id: String,
) -> Result<crate::store::AccountPurgePreview, crate::accounts::AccountError> {
Ok(self.store.preview_account_purge(id).await?)
}
pub async fn purge_account(
&self,
id: &str,
version: u64,
revision: u64,
) -> Result<(crate::store::AccountPurgePreview, bool), crate::accounts::AccountError> {
let _permit = self.mail.coordinator.acquire(id).await?;
self.accounts.purge(id, version, revision).await
}
pub async fn cancel_google_auth(
&self,
id: &str,
) -> Result<crate::accounts::AuthSession, crate::accounts::AccountError> {
self.accounts.cancel_auth(id).await
}
pub async fn check_account(
&self,
id: &str,
) -> Result<crate::accounts::Account, crate::accounts::AccountError> {
let _permit = self.mail.coordinator.acquire(id).await?;
self.accounts.check(id).await
}
pub fn authorization(&self) -> Zeroizing<String> {
self.authorization.clone()
}
pub async fn sync_account(
&self,
account: String,
full: bool,
fetch: Option<String>,
) -> Result<crate::store::SyncRun, crate::mail::MailError> {
self.mail.start(account, full, fetch).await
}
pub async fn sync_run(
&self,
account: String,
run: String,
) -> Result<crate::store::SyncRun, StoreError> {
self.store.sync_run(account, run).await
}
pub async fn cancel_sync(
&self,
account: &str,
run: &str,
) -> Result<crate::store::SyncRun, crate::mail::MailError> {
if self.store.sync_run(account.into(), run.into()).await?.scope == "calendar" {
self.calendar.cancel(account, run).await
} else {
self.mail.cancel(account, run).await
}
}
pub async fn list_mail(
&self,
query: crate::store::MailQuery,
) -> Result<crate::store::MailPage, StoreError> {
self.store.query_mail(query).await
}
pub async fn get_mail(
&self,
account: String,
id: String,
) -> Result<crate::store::MailDetail, StoreError> {
self.store.get_mail(account, id).await
}
pub async fn mail_collections(
&self,
account: String,
) -> Result<(u64, Vec<crate::store::MailCollection>), StoreError> {
self.store.mail_collections(account).await
}
pub async fn mail_blob(
&self,
account: String,
id: String,
kind: String,
attachment: Option<String>,
) -> Result<crate::store::StoredBlob, StoreError> {
self.store.mail_blob(account, id, kind, attachment).await
}
pub async fn blob_chunk(
&self,
account: String,
id: String,
ordinal: u64,
) -> Result<Vec<u8>, StoreError> {
self.store.blob_chunk(account, id, ordinal).await
}
pub async fn query_free_busy(
&self,
input: crate::domain::free_busy::FreeBusyRequest,
) -> Result<crate::domain::free_busy::FreeBusyResult, crate::sync_error::SyncError> {
self.calendar.free_busy(input).await
}
pub async fn change_event(
&self,
input: crate::store::EnqueueCalendarChange,
) -> Result<crate::store::Operation, crate::store::StoreError> {
self.store
.enqueue_calendar_change(input, self.accounts.http.clock.now_ms())
.await
}
pub async fn refresh_calendar(
&self,
account: String,
window: Option<crate::domain::calendar::AgendaWindow>,
) -> Result<crate::store::SyncRun, crate::sync_error::SyncError> {
self.calendar.start(account, window).await
}
pub async fn calendar_list(
&self,
account: String,
size: u32,
token: Option<String>,
) -> Result<crate::store::CalendarCatalogPage, StoreError> {
self.store.calendar_catalog_page(account, size, token).await
}
pub async fn calendar_event(
&self,
account: String,
calendar: String,
event: String,
) -> Result<crate::store::CalendarEventResult, StoreError> {
self.store.calendar_event(account, calendar, event).await
}
pub async fn calendar_agenda(
&self,
account: String,
window: crate::domain::calendar::AgendaWindow,
size: u32,
token: Option<String>,
) -> Result<crate::store::CalendarAgendaPage, StoreError> {
self.store
.calendar_agenda(account, window, size, token)
.await
}
}
@@ -0,0 +1,230 @@
mod journal;
use super::{Engine, EngineConfig, EngineError};
use crate::{
profile::RestoreIdentity,
store::{stage_restore_with, RestoreReport, StoreError},
};
pub(super) use journal::recover as recover_restores;
use serde::Serialize;
use std::path::PathBuf;
use zeroize::Zeroizing;
#[derive(Debug, Serialize)]
pub struct ProfileRestoreReport {
pub profile_id: String,
pub directory: PathBuf,
pub restore: RestoreReport,
}
impl Engine {
pub async fn create_backup(
&self,
passphrase: Zeroizing<String>,
) -> Result<crate::store::BackupArtifact, StoreError> {
self.maintenance
.create(passphrase, self.accounts.http.clock.now_ms())
.await
}
pub async fn begin_backup_upload(
&self,
byte_length: u64,
sha256: String,
) -> Result<super::BackupUpload, StoreError> {
self.maintenance.upload(byte_length, sha256).await
}
pub async fn inspect_backup(
&self,
input: super::BackupInput,
passphrase: Zeroizing<String>,
) -> Result<crate::store::BackupInspection, StoreError> {
tokio::task::spawn_blocking(move || {
let result = crate::store::inspect_backup(&input.path(), passphrase);
drop(input);
result
})
.await
.map_err(|_| StoreError::Unavailable)?
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub async fn restore_backup(
&self,
input: super::BackupInput,
passphrase: Zeroizing<String>,
new_profile: String,
) -> Result<ProfileRestoreReport, EngineError> {
if !input.belongs_to(&self._profile_lock) {
return Err(StoreError::InvalidPath.into());
}
let config = self.maintenance.restore_config(&new_profile)?;
let now = self.accounts.http.clock.now_ms();
let profile_lock = self._profile_lock.clone();
let source_id = self.profile_id;
let journal = journal::Journal {
store: self.store.clone(),
owner: journal::canonical_owner(&self.directory)?,
runtime: tokio::runtime::Handle::current(),
};
#[cfg(feature = "test-harness")]
let test_config = self.test_config.clone();
tokio::task::spawn_blocking(move || {
// Cancellation can outlive Engine shutdown. Another engine must
// not recover this profile while its restore is still mutating it.
let _profile_lock = profile_lock;
// The input owns maintenance admission throughout cleanup and
// restore, including if the caller cancels this blocking work.
journal.runtime.block_on(recover_restores(
&journal.store,
journal.owner.clone(),
source_id,
config.secrets.clone(),
_profile_lock.clone(),
))?;
let result = restore(
config,
input.path(),
passphrase,
now,
Some(&journal),
#[cfg(feature = "test-harness")]
test_config.as_ref(),
);
let cleanup = input.cleanup();
if let Ok(report) = &result {
cleanup.map_err(|_| StoreError::RestoreActivationUncertain)?;
// Sync the upload deletion before retiring its durable record.
std::fs::File::open(&journal.owner)
.and_then(|f| f.sync_all())
.map_err(|_| StoreError::RestoreActivationUncertain)?;
match journal
.runtime
.block_on(journal.store.finish_restore_job(report.profile_id.clone()))
{
Ok(()) | Err(StoreError::Unavailable) => {}
Err(_) => return Err(StoreError::RestoreActivationUncertain.into()),
}
// Unavailable means the source Store already shut down. The
// retained row is reconciled at startup, keeping activated keys.
}
result
})
.await
.map_err(|_| EngineError::Unavailable)?
}
/// Builds a disconnected new profile; it does not start the restored engine.
/// Blocking restore work owns its resources until completion even if the
/// awaiting caller disappears. API admission must cover that full lifetime.
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub async fn restore_profile(
config: EngineConfig,
source: PathBuf,
passphrase: Zeroizing<String>,
) -> Result<ProfileRestoreReport, EngineError> {
let now = chrono::Utc::now().timestamp_millis();
tokio::task::spawn_blocking(move || {
restore(
config,
source,
passphrase,
now,
None,
#[cfg(feature = "test-harness")]
None,
)
})
.await
.map_err(|_| EngineError::Unavailable)?
}
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn restore(
config: EngineConfig,
source: PathBuf,
passphrase: Zeroizing<String>,
now: i64,
journal: Option<&journal::Journal>,
#[cfg(feature = "test-harness")] test_config: Option<&crate::test_controls::TestConfig>,
) -> Result<ProfileRestoreReport, EngineError> {
match std::fs::symlink_metadata(&config.directory) {
Ok(_) => return Err(StoreError::Io(std::io::ErrorKind::AlreadyExists).into()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(StoreError::from(e).into()),
}
let parent = config.directory.parent().ok_or(StoreError::InvalidPath)?;
let identity = RestoreIdentity::new(config.secrets.as_ref())?;
let stage = stage_restore_with(
&source,
passphrase,
identity.database_key.clone(),
parent,
now,
|stage, parent, stage_id| {
if let Some(journal) = journal {
journal.record(
identity.id,
&source,
&config.directory,
stage,
parent,
stage_id,
)?;
}
#[cfg(feature = "test-harness")]
if let Some(config) = test_config {
config
.blocking_checkpoint("restore_stage_owned")
.map_err(|_| StoreError::Unavailable)?;
}
Ok(())
},
)?;
// Persist before keys or rename; a detached worker may finish after its
// source store shuts down, so it needs no later database write to activate.
if let Some(journal) = journal {
journal.activation_intent(identity.id)?;
}
#[cfg(feature = "test-harness")]
if let Some(config) = test_config {
config
.blocking_checkpoint("restore_after_stage")
.map_err(|_| EngineError::Unavailable)?;
}
if let Err(error) = identity.initialize(
stage.directory(),
config.secrets.as_ref(),
#[cfg(feature = "test-harness")]
test_config,
) {
identity.rollback(config.secrets.as_ref())?;
return Err(error);
}
let report = match stage.activate(&config.directory) {
Ok(report) => report,
// Activation succeeded: these keys now belong to a real restored
// profile and must remain available, even when its fsync failed.
Err(StoreError::RestoreActivationUncertain) => {
return Err(StoreError::RestoreActivationUncertain.into())
}
Err(error) => {
identity.rollback(config.secrets.as_ref())?;
return Err(error.into());
}
};
#[cfg(feature = "test-harness")]
if let Some(config) = test_config {
config
.blocking_checkpoint("restore_after_activation")
.map_err(|_| EngineError::Unavailable)?;
}
Ok(ProfileRestoreReport {
profile_id: identity.id.to_string(),
directory: config.directory,
restore: report,
})
}
#[cfg(test)]
mod journal_tests;
@@ -0,0 +1,217 @@
use crate::store::CleanupDirectory;
use crate::{
domain::identity::ProfileId,
engine::EngineError,
secrets::SecretStore,
store::{DirectoryIdentity, OwnedDirectory, RestoreJob, Store, StoreError},
};
use rustix::fs::{Mode, OFlags};
use std::{
fs::File,
io::Read,
path::{Path, PathBuf},
sync::Arc,
};
pub(super) struct Journal {
pub store: Store,
pub owner: PathBuf,
pub runtime: tokio::runtime::Handle,
}
impl Journal {
pub fn record(
&self,
id: ProfileId,
source: &Path,
target: &Path,
stage: &Path,
parent: DirectoryIdentity,
identity: DirectoryIdentity,
) -> Result<(), StoreError> {
let upload = source.parent().ok_or(StoreError::InvalidPath)?;
let owner = upload.parent().ok_or(StoreError::InvalidPath)?;
if canonical_owner(owner)? != self.owner {
return Err(StoreError::InvalidPath);
}
let job = RestoreJob {
profile_id: id,
owner: DirectoryIdentity::at(&self.owner)?,
parent,
stage: OwnedDirectory {
name: name(stage)?,
identity,
},
upload: OwnedDirectory {
name: name(upload)?,
identity: DirectoryIdentity::at(upload)?,
},
target: name(target)?,
};
self.runtime.block_on(self.store.record_restore_job(job))
}
pub fn activation_intent(&self, id: ProfileId) -> Result<(), StoreError> {
self.runtime
.block_on(self.store.activate_restore_job(id.to_string()))
}
}
fn name(path: &Path) -> Result<String, StoreError> {
path.file_name()
.and_then(|s| s.to_str())
.map(str::to_owned)
.ok_or(StoreError::InvalidPath)
}
pub(super) fn canonical_owner(owner: &Path) -> Result<PathBuf, StoreError> {
let identity = DirectoryIdentity::at(owner)?;
let canonical = std::fs::canonicalize(owner)?;
if !identity.matches(&canonical)? {
return Err(StoreError::InvalidPath);
}
Ok(canonical)
}
pub(in crate::engine) async fn recover(
store: &Store,
owner: PathBuf,
source_id: ProfileId,
secrets: Arc<dyn SecretStore>,
profile_lock: Arc<File>,
) -> Result<(), EngineError> {
let owner = canonical_owner(&owner)?;
for (job, activating) in store.restore_jobs().await? {
if job.profile_id == source_id {
return Err(StoreError::KeyOrCorrupt.into());
}
let id = job.profile_id.to_string();
let directory = owner.clone();
let secrets = secrets.clone();
let profile_lock = profile_lock.clone();
tokio::task::spawn_blocking(move || {
let _profile_lock = profile_lock;
cleanup(&job, activating, &directory, secrets.as_ref())
})
.await
.map_err(|_| EngineError::Unavailable)??;
store.finish_restore_job(id).await?;
}
Ok(())
}
fn pending(job: &RestoreJob) -> EngineError {
EngineError::RestoreRecoveryPending {
profile_id: job.profile_id.to_string(),
}
}
fn cleanup(
job: &RestoreJob,
activating: bool,
owner: &Path,
secrets: &dyn SecretStore,
) -> Result<(), EngineError> {
let parent = owner.parent().ok_or(StoreError::InvalidPath)?;
if !job.owner.matches(owner)? || !job.parent.matches(parent)? {
return Err(pending(job));
}
let stage = parent.join(&job.stage.name);
let target = parent.join(&job.target);
let activated = job.stage.identity.matches(&target)?;
if activated {
if !activating || !manifest_matches(&target, job.profile_id)? {
return Err(pending(job));
}
// The renamed directory owns both keys, including after a lost reply.
} else {
let stage_owned = job.stage.identity.matches(&stage)?;
if (!stage_owned && exists(&stage)?)
|| (activating && !stage_owned && keys_exist(job, secrets)?)
{
return Err(pending(job));
}
let upload = CleanupDirectory::prepare(owner, job.owner, &job.upload, &["upload.nuncio"])
.map_err(|_| pending(job))?;
let stage = CleanupDirectory::prepare(
parent,
job.parent,
&job.stage,
&[
"source.nuncio",
"source.nuncio-wal",
"source.nuncio-shm",
"source.nuncio-journal",
"store.db",
"store.db-wal",
"store.db-shm",
"store.db-journal",
"store.lock",
"profile.json",
],
)
.map_err(|_| pending(job))?;
crate::profile::rollback_restore_keys(job.profile_id, secrets)?;
if let Some(stage) = stage {
stage.remove()?;
}
if let Some(upload) = upload {
upload.remove()?;
}
return Ok(());
}
if let Some(upload) =
CleanupDirectory::prepare(owner, job.owner, &job.upload, &["upload.nuncio"])
.map_err(|_| pending(job))?
{
upload.remove()?;
}
Ok(())
}
fn keys_exist(job: &RestoreJob, secrets: &dyn SecretStore) -> Result<bool, EngineError> {
for suffix in ["database", "api"] {
if secrets
.get(&format!("{}/profile/{suffix}", job.profile_id))?
.is_some()
{
return Ok(true);
}
}
Ok(false)
}
fn exists(path: &Path) -> Result<bool, StoreError> {
match std::fs::symlink_metadata(path) {
Ok(_) => Ok(true),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(e.into()),
}
}
fn open_directory(path: &Path) -> Result<File, StoreError> {
Ok(File::from(
rustix::fs::open(
path,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)
.map_err(std::io::Error::from)?,
))
}
fn manifest_matches(directory: &Path, id: ProfileId) -> Result<bool, StoreError> {
let dir = open_directory(directory)?;
let file = File::from(
rustix::fs::openat(
&dir,
"profile.json",
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC,
Mode::empty(),
)
.map_err(std::io::Error::from)?,
);
if !file.metadata()?.is_file() {
return Ok(false);
}
let mut bytes = Vec::new();
file.take(4097).read_to_end(&mut bytes)?;
if bytes.len() > 4096 {
return Ok(false);
}
let value: serde_json::Value =
serde_json::from_slice(&bytes).map_err(|_| StoreError::KeyOrCorrupt)?;
Ok(value["version"] == 1 && value["id"].as_str() == Some(id.to_string().as_str()))
}
@@ -0,0 +1,401 @@
#![allow(clippy::unwrap_used)]
use super::journal::*;
use crate::{
secrets::{SecretError, SecretStore},
store::{DirectoryIdentity, OwnedDirectory, RestoreJob, Store},
};
use std::{
collections::BTreeMap,
fs,
path::PathBuf,
sync::{
atomic::{AtomicBool, Ordering},
Arc, Condvar, Mutex,
},
};
use zeroize::Zeroizing;
#[derive(Default)]
struct Secrets {
keys: Mutex<BTreeMap<String, Zeroizing<Vec<u8>>>>,
fail_delete: AtomicBool,
delete_barrier: Mutex<Option<DeleteBarrier>>,
}
type ReleaseSignal = Arc<(Mutex<bool>, Condvar)>;
struct DeleteBarrier {
entered: tokio::sync::oneshot::Sender<()>,
release: ReleaseSignal,
}
struct Release(ReleaseSignal);
impl Drop for Release {
fn drop(&mut self) {
*self.0 .0.lock().unwrap() = true;
self.0 .1.notify_all();
}
}
impl SecretStore for Secrets {
fn get(&self, n: &str) -> Result<Option<Zeroizing<Vec<u8>>>, SecretError> {
Ok(self.keys.lock().unwrap().get(n).cloned())
}
fn put(&self, n: &str, v: &[u8]) -> Result<(), SecretError> {
self.keys
.lock()
.unwrap()
.insert(n.into(), Zeroizing::new(v.to_vec()));
Ok(())
}
fn delete(&self, n: &str) -> Result<(), SecretError> {
if self.fail_delete.load(Ordering::SeqCst) {
return Err(SecretError);
}
let barrier = self.delete_barrier.lock().unwrap().take();
if let Some(barrier) = barrier {
let _ = barrier.entered.send(());
let mut ready = barrier.release.0.lock().unwrap();
while !*ready {
ready = barrier.release.1.wait(ready).unwrap();
}
}
self.keys.lock().unwrap().remove(n);
Ok(())
}
}
struct Fixture {
temp: tempfile::TempDir,
owner: PathBuf,
store: Store,
job: RestoreJob,
secrets: Arc<Secrets>,
}
impl Fixture {
async fn new() -> Self {
let temp = tempfile::tempdir().unwrap();
let owner = temp.path().join("source");
let store = Store::open(&owner, Zeroizing::new(vec![42; 32]))
.await
.unwrap();
let stage = temp.path().join(".restore-owned");
let upload = owner.join(".maintenance-owned");
fs::create_dir(&stage).unwrap();
fs::create_dir(&upload).unwrap();
let job = RestoreJob {
profile_id: crate::domain::identity::ProfileId::generate(),
owner: DirectoryIdentity::at(&owner).unwrap(),
parent: DirectoryIdentity::at(temp.path()).unwrap(),
stage: OwnedDirectory {
name: ".restore-owned".into(),
identity: DirectoryIdentity::at(&stage).unwrap(),
},
upload: OwnedDirectory {
name: ".maintenance-owned".into(),
identity: DirectoryIdentity::at(&upload).unwrap(),
},
target: "restored".into(),
};
fs::write(stage.join("store.db"), b"owned encrypted stage canary").unwrap();
fs::write(
stage.join("profile.json"),
serde_json::json!({"version":1,"id":job.profile_id}).to_string(),
)
.unwrap();
fs::write(
upload.join("upload.nuncio"),
b"owned encrypted upload canary",
)
.unwrap();
let secrets = Arc::new(Secrets::default());
for suffix in ["database", "api"] {
secrets
.put(&format!("{}/profile/{suffix}", job.profile_id), &[77; 32])
.unwrap();
}
secrets
.put("unrelated/profile/database", &[99; 32])
.unwrap();
store.record_restore_job(job.clone()).await.unwrap();
store
.activate_restore_job(job.profile_id.to_string())
.await
.unwrap();
Self {
temp,
owner,
store,
job,
secrets,
}
}
async fn recover(&self) -> Result<(), crate::engine::EngineError> {
use fs4::fs_std::FileExt;
let lock = fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(self.owner.join("profile.lock"))
.unwrap();
assert!(lock.try_lock_exclusive().unwrap());
recover(
&self.store,
self.owner.clone(),
crate::domain::identity::ProfileId::generate(),
self.secrets.clone(),
Arc::new(lock),
)
.await
}
fn stage(&self) -> PathBuf {
self.temp.path().join(&self.job.stage.name)
}
fn target(&self) -> PathBuf {
self.temp.path().join(&self.job.target)
}
fn keys(&self) -> BTreeMap<String, Zeroizing<Vec<u8>>> {
self.secrets.keys.lock().unwrap().clone()
}
}
#[tokio::test]
async fn uncertain_paths_preserve_keys_files_and_retryable_cleanup_authority() {
for scenario in [
"stage-replaced",
"stage-symlink",
"stage-missing",
"target-wrong-id",
"owner-copied",
"owner-symlink",
"unexpected-stage-entry",
"upload-replaced",
] {
let mut f = Fixture::new().await;
let keys = f.keys();
match scenario {
"stage-replaced" => {
fs::rename(f.stage(), f.temp.path().join("moved")).unwrap();
fs::create_dir(f.stage()).unwrap();
fs::write(f.stage().join("store.db"), b"unrelated replacement").unwrap();
}
"stage-symlink" => {
fs::rename(f.stage(), f.temp.path().join("moved")).unwrap();
std::os::unix::fs::symlink(f.temp.path().join("moved"), f.stage()).unwrap();
}
"stage-missing" => {
fs::rename(f.stage(), f.temp.path().join("moved")).unwrap();
}
"target-wrong-id" => {
fs::rename(f.stage(), f.target()).unwrap();
fs::write(f.target().join("profile.json"),serde_json::json!({"version":1,"id":crate::domain::identity::ProfileId::generate()}).to_string()).unwrap();
}
"owner-copied" => {
f.owner = f.temp.path().join("copied");
fs::create_dir(&f.owner).unwrap();
}
"owner-symlink" => {
let moved = f.temp.path().join("moved-owner");
fs::rename(&f.owner, &moved).unwrap();
std::os::unix::fs::symlink(&moved, &f.owner).unwrap();
}
"unexpected-stage-entry" => {
fs::write(f.stage().join("unrelated"), b"preserve this").unwrap();
}
"upload-replaced" => {
let upload = f.owner.join(&f.job.upload.name);
fs::rename(&upload, f.owner.join("moved-upload")).unwrap();
fs::create_dir(&upload).unwrap();
fs::write(upload.join("upload.nuncio"), b"unrelated replacement").unwrap();
}
_ => unreachable!(),
}
assert!(f.recover().await.is_err(), "{scenario}");
assert!(f.keys() == keys, "keys changed for {scenario}");
assert_eq!(f.store.restore_jobs().await.unwrap().len(), 1);
assert!(f.recover().await.is_err(), "second recovery: {scenario}");
assert!(f.keys() == keys, "second recovery changed keys: {scenario}");
f.store.close().await.unwrap();
}
}
#[tokio::test]
async fn failed_key_cleanup_is_retryable_and_activated_target_retains_keys() {
let f = Fixture::new().await;
let keys = f.keys();
f.secrets.fail_delete.store(true, Ordering::SeqCst);
assert!(f.recover().await.is_err());
assert!(f.keys() == keys);
assert!(f.stage().join("store.db").exists());
assert_eq!(f.store.restore_jobs().await.unwrap().len(), 1);
f.secrets.fail_delete.store(false, Ordering::SeqCst);
f.recover().await.unwrap();
assert!(!f.stage().exists());
assert!(!f.owner.join(&f.job.upload.name).exists());
assert_eq!(f.keys().len(), 1);
assert!(f.store.restore_jobs().await.unwrap().is_empty());
f.recover().await.unwrap();
f.store.close().await.unwrap();
let f = Fixture::new().await;
let keys = f.keys();
fs::rename(f.stage(), f.target()).unwrap();
f.recover().await.unwrap();
assert!(f.keys() == keys);
assert_eq!(
fs::read(f.target().join("store.db")).unwrap(),
b"owned encrypted stage canary"
);
assert!(!f.owner.join(&f.job.upload.name).exists());
assert!(f.store.restore_jobs().await.unwrap().is_empty());
f.recover().await.unwrap();
f.store.close().await.unwrap();
}
#[tokio::test]
async fn backup_excludes_cleanup_authority_while_source_retains_it() {
let f = Fixture::new().await;
let keys = f.keys();
let phrase: Zeroizing<String> = Zeroizing::new("synthetic cleanup backup passphrase".into());
let backup = f.store.create_backup(phrase.clone(), 10).await.unwrap();
let c = rusqlite::Connection::open(backup.path()).unwrap();
c.pragma_update(None, "key", phrase.as_str()).unwrap();
assert_eq!(
c.query_row("SELECT count(*) FROM restore_cleanup_jobs", [], |r| r
.get::<_, i64>(0))
.unwrap(),
0
);
assert_eq!(f.store.restore_jobs().await.unwrap().len(), 1);
assert!(f.keys() == keys);
c.close().unwrap();
drop(backup);
f.store.close().await.unwrap();
}
#[tokio::test]
async fn journal_accepts_parent_aliases_without_accepting_another_owners_upload() {
let f = Fixture::new().await;
let alias = f.temp.path().join("parent-alias");
std::os::unix::fs::symlink(f.temp.path(), &alias).unwrap();
let source = alias
.join("source")
.join(&f.job.upload.name)
.join("upload.nuncio");
let other = f.temp.path().join("other-owner");
fs::create_dir(&other).unwrap();
let other_source = other.join(".maintenance-other/upload.nuncio");
let journal = Journal {
store: f.store.clone(),
owner: canonical_owner(&f.owner).unwrap(),
runtime: tokio::runtime::Handle::current(),
};
let id = crate::domain::identity::ProfileId::generate();
let stage = f.stage();
let target = f.target();
let parent = f.job.parent;
let identity = f.job.stage.identity;
tokio::task::spawn_blocking(move || {
journal
.record(id, &source, &target, &stage, parent, identity)
.unwrap();
assert!(matches!(
journal.record(
crate::domain::identity::ProfileId::generate(),
&other_source,
&target,
&stage,
parent,
identity,
),
Err(crate::store::StoreError::InvalidPath)
));
})
.await
.unwrap();
let jobs = f.store.restore_jobs().await.unwrap();
assert_eq!(jobs.len(), 2);
let (recorded, activating) = jobs.iter().find(|(job, _)| job.profile_id == id).unwrap();
assert!(!activating);
assert_eq!(recorded.owner, f.job.owner);
assert_eq!(recorded.upload.identity, f.job.upload.identity);
f.store.close().await.unwrap();
}
#[tokio::test]
async fn cancelled_startup_retains_profile_ownership_until_cleanup_worker_finishes() {
use crate::{
domain::identity::ProfileId,
engine::{Engine, EngineConfig, EngineError},
};
use std::time::Duration;
let f = Fixture::new().await;
let source_id = ProfileId::generate();
fs::write(
f.owner.join("profile.json"),
serde_json::json!({"version":1,"id":source_id}).to_string(),
)
.unwrap();
f.secrets
.put(&format!("{source_id}/profile/database"), &[42; 32])
.unwrap();
f.secrets
.put(&format!("{source_id}/profile/api"), &[55; 32])
.unwrap();
let original = f
.keys()
.into_iter()
.filter(|(n, _)| !n.starts_with(&f.job.profile_id.to_string()))
.collect::<BTreeMap<_, _>>();
let manifest = fs::read(f.owner.join("profile.json")).unwrap();
f.store.clone().close().await.unwrap();
let (entered, waiting) = tokio::sync::oneshot::channel();
let release = Release(Arc::new((Mutex::new(false), Condvar::new())));
*f.secrets.delete_barrier.lock().unwrap() = Some(DeleteBarrier {
entered,
release: release.0.clone(),
});
let config = || EngineConfig {
directory: f.owner.clone(),
secrets: f.secrets.clone(),
};
let caller = tokio::spawn(Engine::open(config()));
tokio::time::timeout(Duration::from_secs(10), waiting)
.await
.unwrap()
.unwrap();
caller.abort();
assert!(caller.await.err().unwrap().is_cancelled());
let opened = tokio::time::timeout(Duration::from_secs(10), Engine::open(config()))
.await
.unwrap();
let retained = matches!(
opened,
Err(EngineError::Storage(crate::store::StoreError::Locked))
);
if let Ok(engine) = opened {
engine.shutdown().await.unwrap();
}
drop(release);
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
let engine = loop {
match Engine::open(config()).await {
Ok(engine) => break engine,
Err(error) => {
assert!(
matches!(
error,
EngineError::Storage(crate::store::StoreError::Locked)
),
"startup after cleanup failed: {error}"
);
assert!(tokio::time::Instant::now() < deadline);
tokio::time::sleep(Duration::from_millis(10)).await;
}
}
};
assert_eq!(
engine.status().await.unwrap().profile_id,
source_id.to_string()
);
engine.shutdown().await.unwrap();
assert!(f.keys() == original);
assert_eq!(fs::read(f.owner.join("profile.json")).unwrap(), manifest);
assert!(!f.stage().exists());
assert!(!f.owner.join(&f.job.upload.name).exists());
assert!(
retained,
"source reopened while a cancelled startup's cleanup worker was still active"
);
}
@@ -0,0 +1,59 @@
use super::Engine;
use crate::{
domain::calendar::AgendaWindow,
store::{ProjectionScope, RepairPreview, StoreError, SyncRun},
sync_error::SyncError,
};
use serde::Serialize;
#[derive(Serialize)]
pub struct RepairResult {
#[serde(flatten)]
pub preview: RepairPreview,
pub dry_run: bool,
pub window: Option<AgendaWindow>,
pub run: Option<SyncRun>,
}
impl Engine {
pub async fn repair_projection(
&self,
account: String,
scope: ProjectionScope,
window: Option<AgendaWindow>,
dry_run: bool,
) -> Result<RepairResult, SyncError> {
let window = match scope {
ProjectionScope::Mail => {
if window.is_some() {
return Err(StoreError::InvalidInput.into());
}
None
}
ProjectionScope::Calendar => Some(match window {
Some(w) => {
AgendaWindow::new(&w.from, &w.to).map_err(|_| StoreError::InvalidInput)?
}
None => AgendaWindow::rolling(self.accounts.http.clock.now_ms())
.map_err(|_| StoreError::InvalidInput)?,
}),
};
let preview = self.store.preview_repair(account.clone(), scope).await?;
let run = if dry_run {
None
} else {
Some(match scope {
ProjectionScope::Mail => self.mail.start(account, true, None).await?,
ProjectionScope::Calendar => {
self.calendar
.repair(account, window.clone().ok_or(StoreError::InvalidInput)?)
.await?
}
})
};
Ok(RepairResult {
preview,
dry_run,
window,
run,
})
}
}
+20
View File
@@ -0,0 +1,20 @@
pub mod accounts;
pub mod calendar;
pub mod changes;
mod clock;
mod coordination;
pub mod domain;
pub mod engine;
pub mod mail;
mod maintenance;
mod operations;
mod profile;
mod providers;
pub mod resources;
mod scheduler;
pub mod secrets;
pub mod store;
pub mod sync_error;
pub mod sync_status;
#[cfg(feature = "test-harness")]
pub mod test_controls;
+175
View File
@@ -0,0 +1,175 @@
use crate::{
accounts::{AccountError, Accounts},
store::{Store, StoreError, SyncRun},
};
use std::{collections::BTreeMap, sync::Arc};
use tokio::sync::{watch, Mutex};
pub use crate::sync_error::SyncError as MailError;
struct Job {
mode: &'static str,
fetch: Option<String>,
stop: watch::Sender<bool>,
task: tokio::task::JoinHandle<()>,
}
pub(crate) struct MailSync {
pub accounts: Arc<Accounts>,
pub store: Store,
pub coordinator: Arc<crate::coordination::Coordinator>,
jobs: Mutex<BTreeMap<(String, String), Job>>,
}
impl MailSync {
pub async fn new(accounts: Arc<Accounts>, store: Store) -> Result<Arc<Self>, StoreError> {
store
.recover_sync_runs(accounts.http.clock.now_ms())
.await?;
Ok(Arc::new(Self {
accounts,
store,
coordinator: crate::coordination::Coordinator::new(),
jobs: Mutex::new(BTreeMap::new()),
}))
}
pub async fn start(
self: &Arc<Self>,
account: String,
force_full: bool,
fetch: Option<String>,
) -> Result<SyncRun, MailError> {
let row = self
.store
.account(account.clone())
.await?
.ok_or(StoreError::NotFound)?;
if !matches!(row.account.provider.as_str(), "google" | "imap") {
return Err(AccountError::Invalid.into());
}
if row.state != "connected" {
return Err(AccountError::Authorization.into());
}
let cursor = self.store.mail_coverage(account.clone()).await?.cursor;
let mode = if fetch.is_some() {
"fetch"
} else if force_full || cursor.is_none() {
"full"
} else {
"delta"
};
let mut jobs = self.jobs.lock().await;
jobs.retain(|_, j| !j.task.is_finished());
if let Some(((.., id), job)) = jobs.iter().find(|((a, _), _)| a == &account) {
if job.fetch == fetch && (!force_full || job.mode == "full") {
return Ok(self.store.sync_run(account, id.clone()).await?);
}
return Err(StoreError::Busy.into());
}
let admission = self.accounts.http.resources.job()?;
let run = self
.store
.start_mail_run(account.clone(), mode.into(), self.now()?)
.await?;
let (stop, mut stopped) = watch::channel(false);
let service = self.clone();
let work_run = run.clone();
let job_fetch = fetch.clone();
let task = tokio::spawn(async move {
let _admission = admission;
let started = std::time::Instant::now();
let work_stop = stopped.clone();
let work = async {
let _permit = service
.coordinator
.acquire_sync(
&service.store,
&service.accounts.http.clock,
&work_run.account_id,
&work_run.scope,
)
.await?;
if work_run.scope == "imap" {
if let Some(message) = fetch {
crate::providers::imap::sync::fetch(&service, &work_run, message, work_stop)
.await
} else {
crate::providers::imap::sync::sync(&service, &work_run, work_stop).await
}
} else {
crate::providers::google::gmail::sync(
&service, &work_run, cursor, fetch, work_stop,
)
.await
}
};
let result =
tokio::select! {result=work=>result,_=stopped.changed()=>Err(MailError::Cancelled)};
if result.is_ok() {
tracing::info!(account_id = %work_run.account_id, run_id = %work_run.id, scope = %work_run.scope, elapsed_ms = started.elapsed().as_millis() as u64, "Sync completed");
}
if let Err(error) = result {
let state = if matches!(error, MailError::Cancelled) {
"cancelled"
} else {
"failed"
};
let recorded = service
.store
.finish_sync_run_error(
work_run.account_id,
work_run.id,
state.into(),
error.code().into(),
service.now().unwrap_or(work_run.started_at_ms),
)
.await;
if recorded.is_err() {
tracing::error!("Sync failure could not be recorded in storage");
}
}
});
jobs.insert(
(account, run.id.clone()),
Job {
stop,
task,
mode,
fetch: job_fetch,
},
);
Ok(run)
}
pub async fn cancel(&self, account: &str, run: &str) -> Result<SyncRun, MailError> {
self.store.sync_run(account.into(), run.into()).await?;
let mut jobs = self.jobs.lock().await;
if let Some(job) = jobs.remove(&(account.into(), run.into())) {
job.stop.send_replace(true);
job.task.await.map_err(|_| MailError::Unavailable)?;
}
Ok(self.store.sync_run(account.into(), run.into()).await?)
}
pub async fn shutdown(&self) {
let jobs = std::mem::take(&mut *self.jobs.lock().await);
for job in jobs.values() {
job.stop.send_replace(true);
}
for (_, job) in jobs {
let _ = job.task.await;
}
}
pub fn now(&self) -> Result<i64, StoreError> {
Ok(self.accounts.http.clock.now_ms())
}
pub async fn checkpoint(
&self,
_name: &str,
_stop: watch::Receiver<bool>,
) -> Result<(), MailError> {
#[cfg(feature = "test-harness")]
if let Some(config) = &self.accounts.http.test_config {
config
.checkpoint(_name, _stop)
.await
.map_err(|_| MailError::Cancelled)?;
}
Ok(())
}
}
@@ -0,0 +1,231 @@
use crate::{
engine::EngineConfig,
secrets::SecretStore,
store::{
BackupArtifact, CleanupDirectory, DirectoryIdentity, OwnedDirectory, Store, StoreError,
},
};
use sha2::{Digest, Sha256};
use std::{
fs::{File, OpenOptions},
io::Write,
path::PathBuf,
sync::Arc,
};
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use zeroize::Zeroizing;
use crate::store::MAX_BACKUP_BYTES;
const CHUNK_BYTES: usize = 262144;
pub(crate) struct Maintenance {
directory: PathBuf,
secrets: Arc<dyn SecretStore>,
store: Store,
admission: Arc<Semaphore>,
profile_lock: Arc<File>,
}
pub(crate) struct MaintenanceLease {
_permit: OwnedSemaphorePermit,
_profile_lock: Arc<File>,
}
impl Maintenance {
pub fn new(
directory: PathBuf,
secrets: Arc<dyn SecretStore>,
store: Store,
profile_lock: Arc<File>,
) -> Self {
Self {
directory,
secrets,
store,
admission: Arc::new(Semaphore::new(1)),
profile_lock,
}
}
fn lease(&self) -> Result<MaintenanceLease, StoreError> {
Ok(MaintenanceLease {
_permit: self
.admission
.clone()
.try_acquire_owned()
.map_err(|_| StoreError::Busy)?,
_profile_lock: self.profile_lock.clone(),
})
}
pub async fn create(
&self,
phrase: Zeroizing<String>,
now: i64,
) -> Result<BackupArtifact, StoreError> {
let lease = self.lease()?;
let result = self
.store
.create_backup_with_guard(phrase, now, Some(lease))
.await?;
if result.inspection().byte_length > MAX_BACKUP_BYTES {
return Err(StoreError::ResultTooLarge);
}
Ok(result)
}
pub async fn upload(&self, length: u64, hash: String) -> Result<BackupUpload, StoreError> {
if length == 0
|| length > MAX_BACKUP_BYTES
|| hash.len() != 64
|| !hash
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
{
return Err(StoreError::InvalidInput);
}
let lease = self.lease()?;
let parent = self.directory.clone();
tokio::task::spawn_blocking(move || {
crate::store::backup_preflight(&parent, length, 1)?;
let directory = tempfile::Builder::new()
.prefix(".maintenance-")
.tempdir_in(&parent)?;
let parent_identity = DirectoryIdentity::at(&parent)?;
let owned = OwnedDirectory {
name: directory
.path()
.file_name()
.and_then(|s| s.to_str())
.ok_or(StoreError::InvalidPath)?
.to_owned(),
identity: DirectoryIdentity::at(directory.path())?,
};
File::open(&parent)?.sync_all()?;
let mut options = OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let file = options.open(directory.path().join("upload.nuncio"))?;
Ok(BackupUpload {
file,
directory,
parent_identity,
owned,
lease,
length,
expected_hash: hash,
received: 0,
hash: Sha256::new(),
})
})
.await
.map_err(|_| StoreError::Unavailable)?
}
pub fn restore_config(&self, name: &str) -> Result<EngineConfig, StoreError> {
if name.is_empty()
|| name.len() > 64
|| !name
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_'))
{
return Err(StoreError::InvalidInput);
}
let directory = std::fs::canonicalize(&self.directory)?;
let parent = directory.parent().ok_or(StoreError::InvalidPath)?;
Ok(EngineConfig {
directory: parent.join(name),
secrets: self.secrets.clone(),
})
}
}
/// A single ordered upload. Each blocking write owns the file, private directory
/// and admission permit, including after cancellation of its awaiting caller.
pub struct BackupUpload {
file: File,
directory: tempfile::TempDir,
parent_identity: DirectoryIdentity,
owned: OwnedDirectory,
lease: MaintenanceLease,
length: u64,
expected_hash: String,
received: u64,
hash: Sha256,
}
impl BackupUpload {
pub async fn append(mut self, offset: u64, bytes: Vec<u8>) -> Result<Self, StoreError> {
if offset != self.received
|| bytes.is_empty()
|| bytes.len() > CHUNK_BYTES
|| self
.received
.checked_add(bytes.len() as u64)
.is_none_or(|n| n > self.length)
{
return Err(StoreError::InvalidInput);
}
tokio::task::spawn_blocking(move || {
self.file.write_all(&bytes)?;
self.hash.update(&bytes);
self.received += bytes.len() as u64;
Ok(self)
})
.await
.map_err(|_| StoreError::Unavailable)?
}
pub async fn finish(self) -> Result<BackupInput, StoreError> {
tokio::task::spawn_blocking(move || {
if self.received != self.length
|| format!("{:x}", self.hash.clone().finalize()) != self.expected_hash
{
return Err(StoreError::InvalidInput);
}
self.file.sync_all()?;
drop(self.file);
File::open(self.directory.path())?.sync_all()?;
Ok(BackupInput {
directory: self.directory,
parent_identity: self.parent_identity,
owned: self.owned,
_lease: self.lease,
})
})
.await
.map_err(|_| StoreError::Unavailable)?
}
}
/// A checksum-verified encrypted input. It retains admission through inspection
/// or restore; neither its path nor arbitrary caller files are accepted by RPCs.
pub struct BackupInput {
directory: tempfile::TempDir,
parent_identity: DirectoryIdentity,
owned: OwnedDirectory,
_lease: MaintenanceLease,
}
impl BackupInput {
pub(crate) fn belongs_to(&self, profile_lock: &Arc<File>) -> bool {
Arc::ptr_eq(&self._lease._profile_lock, profile_lock)
}
pub(crate) fn cleanup(mut self) -> Result<(), StoreError> {
self.directory.disable_cleanup(true);
let parent = self
.directory
.path()
.parent()
.ok_or(StoreError::InvalidPath)?;
if let Some(directory) = CleanupDirectory::prepare(
parent,
self.parent_identity,
&self.owned,
&["upload.nuncio"],
)? {
directory.remove()?;
}
Ok(())
}
pub(crate) fn path(&self) -> PathBuf {
self.directory.path().join("upload.nuncio")
}
}
@@ -0,0 +1,622 @@
mod calendar_change;
mod imap_flags;
mod imap_transfer;
mod mail_change;
mod smtp;
use crate::{
accounts::{AccountError, Accounts},
coordination::Coordinator,
providers::google::send::WriteResult,
store::{AttemptKind, AttemptOutcome, OperationReceipt, Store, StoreError},
};
use base64::{
engine::general_purpose::{URL_SAFE, URL_SAFE_NO_PAD},
Engine as _,
};
use std::{collections::BTreeSet, sync::Arc, time::Duration};
use tokio::{
sync::{watch, Semaphore},
task::JoinSet,
};
pub(crate) struct OperationWorker {
stop: watch::Sender<bool>,
task: Option<tokio::task::JoinHandle<()>>,
error: watch::Receiver<Option<String>>,
}
enum PreparedWrite {
Smtp {
_memory: tokio::sync::OwnedSemaphorePermit,
raw: Vec<u8>,
payload: crate::store::SendPayload,
intent: Box<crate::store::SmtpIntent>,
},
Send {
memory: tokio::sync::OwnedSemaphorePermit,
raw: Vec<u8>,
payload: crate::store::SendPayload,
},
Change(crate::store::MailChangePayload),
ImapFlags(crate::store::ImapFlagPayload),
ImapTransfer {
payload: crate::store::ImapTransferPayload,
_memory: tokio::sync::OwnedSemaphorePermit,
},
Calendar(crate::store::CalendarChangePayload),
}
struct Service {
store: Store,
accounts: Arc<Accounts>,
coordinator: Arc<Coordinator>,
composition: Arc<Semaphore>,
}
impl OperationWorker {
pub fn start(
store: Store,
accounts: Arc<Accounts>,
coordinator: Arc<Coordinator>,
composition: Arc<Semaphore>,
) -> Self {
let (stop, _) = watch::channel(false);
let (failed, error) = watch::channel(None);
let service = Arc::new(Service {
store,
accounts,
coordinator,
composition,
});
let stopping = stop.clone();
tracing::debug!("Operation worker starting");
let task = tokio::spawn(async move {
if run(service, stopping).await.is_err() {
tracing::error!(
error_code = "operation_worker_unavailable",
"Operation worker stopped unexpectedly"
);
failed.send_replace(Some("operation_worker_unavailable".into()));
}
});
Self {
stop,
task: Some(task),
error,
}
}
pub fn error(&self) -> Option<String> {
self.error.borrow().clone()
}
pub async fn shutdown(mut self) {
self.stop.send_replace(true);
if let Some(task) = self.task.take() {
let _ = task.await;
}
}
}
impl Drop for OperationWorker {
fn drop(&mut self) {
self.stop.send_replace(true);
if let Some(task) = self.task.take() {
task.abort();
}
}
}
async fn run(service: Arc<Service>, stop: watch::Sender<bool>) -> Result<(), StoreError> {
let mut stopped = stop.subscribe();
let mut jobs = JoinSet::new();
let mut accounts = BTreeSet::new();
let mut tick = tokio::time::interval(Duration::from_millis(250));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let result = loop {
if *stopped.borrow() {
break Ok(());
}
tokio::select! {
_=stopped.changed()=>break Ok(()),
finished=jobs.join_next(),if !jobs.is_empty()=>{
match finished {
Some(Ok((account,Ok(()))))=>{accounts.remove(&account);},
_=>break Err(StoreError::Unavailable),
}
},
_=tick.tick()=>{
#[cfg(feature="test-harness")]
if service.accounts.http.refresh_test_clock().await.is_err() { break Err(StoreError::Unavailable); }
let pending=match service.store.ready_operations(service.accounts.http.clock.now_ms()).await {
Ok(pending)=>pending,Err(error)=>break Err(error),
};
for work in pending {
if accounts.contains(&work.account_id) { continue; }
let Ok(admission)=service.accounts.http.resources.job() else { break; };
if !accounts.insert(work.account_id.clone()) { continue; }
let service=service.clone();let stopped=stop.subscribe();
jobs.spawn(async move {
let _admission=admission;
let account=work.account_id.clone();
let result=process(service.clone(),work,stopped.clone()).await;
let _=service.checkpoint("operation_job_finished",stopped).await;
(account,result)
});
}
}
}
};
stop.send_replace(true);
while jobs.join_next().await.is_some() {}
result
}
async fn process(
service: Arc<Service>,
work: crate::store::ReadyOperation,
mut stop: watch::Receiver<bool>,
) -> Result<(), StoreError> {
let preparation_stop = stop.clone();
let preparation = async {
service
.checkpoint("operation_before_dispatch", preparation_stop)
.await?;
let imap = service
.store
.account(work.account_id.clone())
.await?
.filter(|row| row.state == "connected")
.ok_or(StoreError::AccountLifecycle)?
.account
.provider
== "imap";
let lane = service
.coordinator
.acquire_sync(
&service.store,
&service.accounts.http.clock,
&work.account_id,
if imap {
"imap"
} else if work.kind == "calendar_change" {
"calendar"
} else {
"gmail"
},
)
.await?;
if service
.store
.account(work.account_id.clone())
.await?
.is_none_or(|row| row.state != "connected")
{
return Err(StoreError::AccountLifecycle);
}
let prepared = if work.kind == "calendar_change" {
PreparedWrite::Calendar(
service
.store
.calendar_change_payload(work.account_id.clone(), work.id.clone())
.await?,
)
} else if work.kind == "mail_change" && imap {
match service
.store
.imap_change_payload(work.account_id.clone(), work.id.clone())
.await?
{
crate::store::ImapChangePayload::Flags(payload) => {
PreparedWrite::ImapFlags(payload)
}
crate::store::ImapChangePayload::Transfer(payload) => PreparedWrite::ImapTransfer {
payload,
_memory: service
.composition
.clone()
.acquire_owned()
.await
.map_err(|_| StoreError::Unavailable)?,
},
}
} else if work.kind == "mail_change" {
PreparedWrite::Change(
service
.store
.mail_change_payload(work.account_id.clone(), work.id.clone())
.await?,
)
} else {
let memory = service
.composition
.clone()
.acquire_owned()
.await
.map_err(|_| StoreError::Unavailable)?;
let payload = service
.store
.send_payload(work.account_id.clone(), work.id.clone())
.await?;
let raw = service
.store
.read_blob(work.account_id.clone(), payload.wire.clone())
.await?;
if imap {
let intent = service
.store
.smtp_intent(work.account_id.clone(), work.id.clone())
.await?;
PreparedWrite::Smtp {
_memory: memory,
raw,
payload,
intent: Box::new(intent),
}
} else {
PreparedWrite::Send {
memory,
raw,
payload,
}
}
};
Ok::<_, StoreError>((lane, prepared))
};
let (_lane, prepared) = tokio::select! {
result=preparation=>match result {
Ok(prepared)=>prepared,
// No attempt has started; leave durable intent ready for the next tick.
Err(StoreError::Busy | StoreError::AccountLifecycle)=>return Ok(()),
Err(StoreError::NotFound) if service.store.account(work.account_id.clone()).await?.is_none()=>return Ok(()),
Err(error)=>return Err(error),
},
_=stop.changed()=>return Ok(()),
};
if *stop.borrow() {
return Ok(());
}
let kind = if work.state == "uncertain" {
AttemptKind::Reconcile
} else {
AttemptKind::Dispatch
};
let attempt = match service
.store
.begin_operation_attempt(
work.account_id.clone(),
work.id.clone(),
kind,
service.accounts.http.clock.now_ms(),
)
.await
{
Ok(attempt) => attempt,
Err(StoreError::VersionConflict | StoreError::Busy) => return Ok(()),
Err(error) => return Err(error),
};
let interrupted = || AttemptOutcome::Uncertain {
code: "interrupted".into(),
};
let policy = service
.store
.operation_execution_policy(work.account_id.clone(), work.id.clone())
.await?;
let retry_ordinal = policy.retry_ordinal(attempt.ordinal)?;
let outcome = if *stop.borrow() {
interrupted()
} else {
let work_stop = stop.clone();
let work_call = async {
service
.checkpoint("operation_after_attempt", work_stop.clone())
.await?;
let result = match prepared {
PreparedWrite::Smtp {
_memory,
raw,
payload,
intent,
} => {
service
.smtp(
&work.id,
&intent,
&payload,
&raw,
(kind, attempt.ordinal, policy),
work_stop.clone(),
)
.await?
}
PreparedWrite::Calendar(payload) => {
service
.change_calendar(&work.account_id, &payload, kind, retry_ordinal)
.await?
}
PreparedWrite::ImapTransfer { payload, _memory } => {
service
.transfer(
&work.account_id,
&work.id,
&payload,
(kind, attempt.ordinal, policy),
work_stop.clone(),
)
.await?
}
PreparedWrite::ImapFlags(payload) => {
service
.imap_flags(&work.account_id, &payload, kind, retry_ordinal)
.await?
}
PreparedWrite::Change(payload) => {
service
.change(&work.account_id, &payload, kind, retry_ordinal)
.await?
}
PreparedWrite::Send {
memory,
raw,
payload,
} => {
if kind == AttemptKind::Dispatch {
let (body, _memory) = tokio::task::spawn_blocking(move || {
let mut body = serde_json::json!({"raw":URL_SAFE_NO_PAD.encode(raw)});
if let Some(id) = payload.thread_id {
body["threadId"] = id.into();
}
(serde_json::to_vec(&body), memory)
})
.await
.map_err(|_| StoreError::Unavailable)?;
service
.send(
&work.account_id,
&payload.sender,
body.map_err(|_| StoreError::InvalidInput)?,
retry_ordinal,
)
.await?
} else {
let _memory = memory;
service
.reconcile(&work.account_id, &payload.message_id, &raw)
.await?
}
}
};
service
.checkpoint("operation_before_receipt", work_stop.clone())
.await?;
Ok::<_, StoreError>(result)
};
tokio::select! {
result=work_call=>match result {
Ok(outcome)=>outcome,
Err(_)=>AttemptOutcome::Uncertain{code:"operation_processing_failed".into()},
},
_=stop.changed()=>interrupted(),
_=tokio::time::sleep(Duration::from_secs(60))=>AttemptOutcome::Uncertain{code:"operation_deadline".into()},
}
};
service
.store
.finish_operation_attempt(
work.account_id,
work.id,
attempt.ordinal,
outcome,
service.accounts.http.clock.now_ms(),
)
.await?;
Ok(())
}
impl Service {
async fn checkpoint(
&self,
_name: &str,
_stop: watch::Receiver<bool>,
) -> Result<(), StoreError> {
#[cfg(feature = "test-harness")]
if let Some(config) = &self.accounts.http.test_config {
config
.checkpoint(_name, _stop)
.await
.map_err(|_| StoreError::Unavailable)?;
}
Ok(())
}
async fn send(
&self,
account: &str,
sender: &str,
body: Vec<u8>,
ordinal: u32,
) -> Result<AttemptOutcome, StoreError> {
let now = self.accounts.http.clock.now_ms();
let base = (1000_i64 << ordinal.saturating_sub(1).min(8)).min(240_000);
let jitter = i64::from(rand::random::<u32>()) % (base / 4 + 1);
let retry = now.saturating_add(base + jitter);
let result = self
.accounts
.gmail_write(account, Some(sender), &["messages", "send"], body, retry)
.await;
Ok(match result {
Ok(WriteResult::Acknowledged(value)) => {
let provider_id = value["id"]
.as_str()
.ok_or(StoreError::InvalidInput)?
.to_owned();
AttemptOutcome::Applied(OperationReceipt {
kind: "google_send".into(),
source: "acknowledgement".into(),
provider_id: Some(provider_id),
etag: None,
})
}
Ok(WriteResult::Rejected {
code,
retry_at_ms,
authorization,
}) => AttemptOutcome::Rejected {
code: code.into(),
retry_at_ms: if ordinal < 8 || authorization {
retry_at_ms
} else {
None
},
},
Ok(WriteResult::Uncertain { code, .. }) => {
AttemptOutcome::Uncertain { code: code.into() }
}
Err(error) => AttemptOutcome::Rejected {
code: error.code().into(),
retry_at_ms: match error {
AccountError::RetryAfter(at) => Some(at),
AccountError::Authorization | AccountError::ScopeDenied => Some(retry),
AccountError::Unavailable | AccountError::Secret if ordinal < 8 => Some(retry),
_ => None,
},
},
})
}
async fn reconcile(
&self,
account: &str,
message_id: &str,
original: &[u8],
) -> Result<AttemptOutcome, StoreError> {
let result = self.find_sent(account, message_id, original).await;
Ok(match result {
Ok(Some(id)) => AttemptOutcome::Applied(OperationReceipt {
kind: "google_send".into(),
source: "positive_read".into(),
provider_id: Some(id),
etag: None,
}),
_ => AttemptOutcome::Uncertain {
code: "no_positive_send_evidence".into(),
},
})
}
async fn find_sent(
&self,
account: &str,
message_id: &str,
original: &[u8],
) -> Result<Option<String>, crate::mail::MailError> {
use crate::mail::MailError;
let search = format!("rfc822msgid:{message_id}");
let mut token = None;
let mut pages = BTreeSet::new();
let mut ids = BTreeSet::new();
let mut found = None;
for _ in 0..100 {
let mut query = vec![
("q", search.as_str()),
("labelIds", "SENT"),
("includeSpamTrash", "true"),
("maxResults", "100"),
];
if let Some(token) = token.as_deref() {
query.push(("pageToken", token));
}
let page = self
.accounts
.gmail_get(account, &["messages"], &query, 2 * 1024 * 1024)
.await?;
let messages = match page.get("messages") {
Some(value) => value.as_array().ok_or(MailError::Provider)?.as_slice(),
None => &[],
};
if messages.len() > 500 {
return Err(MailError::TooLarge);
}
for message in messages {
let id = message["id"]
.as_str()
.filter(|s| !s.is_empty() && s.len() <= 2048)
.ok_or(MailError::Provider)?;
if !ids.insert(id.to_owned()) {
continue;
}
if ids.len() > 1000 {
return Err(MailError::TooLarge);
}
let limit = self.accounts.http.max_payload_bytes.saturating_mul(4) / 3 + 65536;
let remote = self
.accounts
.gmail_get(account, &["messages", id], &[("format", "raw")], limit)
.await?;
if remote["id"] != id
|| !remote["labelIds"]
.as_array()
.is_some_and(|labels| labels.iter().any(|l| l == "SENT"))
{
continue;
}
let encoded = remote["raw"].as_str().ok_or(MailError::Provider)?;
let bytes = URL_SAFE_NO_PAD
.decode(encoded)
.or_else(|_| URL_SAFE.decode(encoded))
.map_err(|_| MailError::Provider)?;
if bytes.len() > self.accounts.http.max_payload_bytes {
return Err(MailError::TooLarge);
}
if matching_sent(original, &bytes, message_id) {
if found.is_some() {
return Ok(None);
}
found = Some(id.into());
}
}
match page.get("nextPageToken") {
None => return Ok(found),
Some(value) => {
let next = value
.as_str()
.filter(|s| !s.is_empty() && s.len() <= 4096)
.ok_or(MailError::Provider)?
.to_owned();
if !pages.insert(next.clone()) {
return Err(MailError::Provider);
}
token = Some(next);
}
}
}
Err(MailError::TooLarge)
}
}
fn matching_sent(original: &[u8], remote: &[u8], message_id: &str) -> bool {
use mail_parser::HeaderName;
let parser = mail_parser::MessageParser::default();
let (Some(a), Some(b)) = (parser.parse_headers(original), parser.parse_headers(remote)) else {
return false;
};
if b.message_id() != Some(message_id) {
return false;
}
for name in [
HeaderName::MessageId,
HeaderName::From,
HeaderName::To,
HeaderName::Cc,
HeaderName::Bcc,
HeaderName::Subject,
HeaderName::ContentType,
HeaderName::ContentTransferEncoding,
] {
if !a
.headers()
.iter()
.filter(|h| h.name == name)
.map(|h| &h.value)
.eq(b
.headers()
.iter()
.filter(|h| h.name == name)
.map(|h| &h.value))
{
return false;
}
}
let body = |raw: &[u8]| raw.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4);
match (body(original), body(remote)) {
(Some(a), Some(b)) => original[a..] == remote[b..],
_ => false,
}
}
@@ -0,0 +1,190 @@
use super::Service;
use crate::{
accounts::AccountError,
domain::calendar::CalendarObject,
providers::google::send::WriteResult,
store::{
AttemptKind, AttemptOutcome, CalendarChangePayload, CalendarChangeReceipt,
CalendarRetryEvidence, CalendarWriteKind, StoreError,
},
sync_error::SyncError,
};
impl Service {
pub(super) async fn change_calendar(
&self,
account: &str,
payload: &CalendarChangePayload,
kind: AttemptKind,
ordinal: u32,
) -> Result<AttemptOutcome, StoreError> {
let now = self.accounts.http.clock.now_ms();
let base = (1000_i64 << ordinal.saturating_sub(1).min(8)).min(240_000);
let retry = now.saturating_add(base + i64::from(rand::random::<u32>()) % (base / 4 + 1));
if kind == AttemptKind::Reconcile {
return Ok(self
.reconcile_calendar(account, payload, ordinal, retry)
.await);
}
Ok(
match self.accounts.calendar_write(account, payload, retry).await {
Ok(WriteResult::Acknowledged(value)) => {
let receipt = if payload.kind == CalendarWriteKind::Delete && value.is_null() {
CalendarChangeReceipt::Deleted {
provider_event_id: payload.provider_event_id.clone(),
}
} else {
CalendarChangeReceipt::Event(value)
};
if valid_receipt(payload, &receipt) {
AttemptOutcome::AppliedCalendar {
source: "acknowledgement".into(),
receipt,
}
} else {
AttemptOutcome::Uncertain {
code: "invalid_calendar_response".into(),
}
}
}
Ok(WriteResult::Uncertain { code, .. }) => {
AttemptOutcome::Uncertain { code: code.into() }
}
Ok(WriteResult::Rejected {
code: "calendar_precondition_failed",
..
}) => match self.read_calendar(account, payload).await {
Ok(observed) if valid_event(payload, &observed) => {
AttemptOutcome::ObservedCalendarConflict {
code: "calendar_precondition_failed".into(),
observed,
}
}
_ => AttemptOutcome::Conflict {
code: "calendar_precondition_failed".into(),
},
},
Ok(WriteResult::Rejected {
code: "resource_not_found",
..
}) => AttemptOutcome::Conflict {
code: "calendar_resource_missing".into(),
},
Ok(WriteResult::Rejected {
code,
retry_at_ms,
authorization,
}) => AttemptOutcome::Rejected {
code: code.into(),
retry_at_ms: if ordinal < 8 || authorization {
retry_at_ms
} else {
None
},
},
Err(error) => AttemptOutcome::Rejected {
code: error.code().into(),
retry_at_ms: match error {
AccountError::RetryAfter(at) => Some(at),
AccountError::Authorization | AccountError::ScopeDenied => Some(retry),
AccountError::Unavailable | AccountError::Secret if ordinal < 8 => {
Some(retry)
}
_ => None,
},
},
},
)
}
async fn read_calendar(
&self,
account: &str,
payload: &CalendarChangePayload,
) -> Result<serde_json::Value, SyncError> {
self.accounts
.calendar_get(
account,
&[
"calendars",
&payload.provider_calendar_id,
"events",
&payload.provider_event_id,
],
&[],
1024 * 1024,
)
.await
}
async fn reconcile_calendar(
&self,
account: &str,
payload: &CalendarChangePayload,
ordinal: u32,
retry: i64,
) -> AttemptOutcome {
let defer = |code: &str, at: i64| {
if ordinal < 8 {
AttemptOutcome::ReconcileLater {
code: code.into(),
retry_at_ms: at,
}
} else {
AttemptOutcome::Uncertain { code: code.into() }
}
};
match self.read_calendar(account, payload).await {
Ok(value) if valid_event(payload, &value) => {
let receipt = CalendarChangeReceipt::Event(value.clone());
if payload.satisfied_by(&receipt) {
return AttemptOutcome::AppliedCalendar {
source: "positive_read".into(),
receipt,
};
}
let evidence = CalendarRetryEvidence::Unchanged(value.clone());
if payload.permits_retry(&evidence) && ordinal < 8 {
return AttemptOutcome::RepeatableCalendar {
code: "unchanged_remote_version".into(),
retry_at_ms: retry,
evidence,
};
}
AttemptOutcome::ObservedCalendarConflict {
code: "calendar_remote_changed".into(),
observed: value,
}
}
Err(SyncError::NotFound) => match payload.kind {
CalendarWriteKind::Create if ordinal < 8 => AttemptOutcome::RepeatableCalendar {
code: "stable_create_id_absent".into(),
retry_at_ms: retry,
evidence: CalendarRetryEvidence::Missing {
provider_event_id: payload.provider_event_id.clone(),
},
},
CalendarWriteKind::Delete => AttemptOutcome::AppliedCalendar {
source: "positive_read".into(),
receipt: CalendarChangeReceipt::Deleted {
provider_event_id: payload.provider_event_id.clone(),
},
},
_ => AttemptOutcome::Conflict {
code: "calendar_resource_missing".into(),
},
},
Err(SyncError::RetryAfter(at)) => defer("calendar_reconciliation_delayed", at),
_ => defer("calendar_reconciliation_unavailable", retry),
}
}
}
fn valid_event(payload: &CalendarChangePayload, value: &serde_json::Value) -> bool {
value["id"] == payload.provider_event_id
&& CalendarObject::from_google(value.clone(), &payload.time_zone).is_ok()
}
fn valid_receipt(payload: &CalendarChangePayload, receipt: &CalendarChangeReceipt) -> bool {
payload.satisfied_by(receipt)
&& match receipt {
CalendarChangeReceipt::Event(value) => valid_event(payload, value),
CalendarChangeReceipt::Deleted { .. } => true,
}
}
@@ -0,0 +1,106 @@
use super::Service;
use crate::{
accounts::AccountError,
providers::imap::{
flags::{self, Observation},
MailError,
},
store::{AttemptKind, AttemptOutcome, ImapFlagPayload, StoreError},
};
impl Service {
pub(super) async fn imap_flags(
&self,
account: &str,
payload: &ImapFlagPayload,
kind: AttemptKind,
ordinal: u32,
) -> Result<AttemptOutcome, StoreError> {
let base = (1000_i64 << ordinal.saturating_sub(1).min(8)).min(240_000);
let retry = self
.accounts
.http
.clock
.now_ms()
.saturating_add(base + i64::from(rand::random::<u32>()) % (base / 4 + 1));
let unavailable = |code: &str, transient: bool| match kind {
AttemptKind::Reconcile if ordinal < 8 => AttemptOutcome::ReconcileLater {
code: code.into(),
retry_at_ms: retry,
},
AttemptKind::Reconcile => AttemptOutcome::Uncertain { code: code.into() },
AttemptKind::Dispatch => AttemptOutcome::Rejected {
code: code.into(),
retry_at_ms: (transient && ordinal < 8).then_some(retry),
},
};
let mut connection = match self.accounts.imap_session(account).await {
Ok(connection) => connection,
Err(error) => {
return Ok(unavailable(
error.code(),
matches!(
error,
AccountError::Unavailable
| AccountError::Authorization
| AccountError::Secret
),
))
}
};
let observed =
match flags::observe(&mut connection, payload, kind == AttemptKind::Dispatch).await {
Ok(value) => value,
Err(error) => {
return Ok(unavailable(
"imap_preflight_unavailable",
matches!(error, MailError::Unavailable),
))
}
};
match observed {
Observation::EpochChanged => {
return Ok(AttemptOutcome::Conflict {
code: "imap_uid_validity_changed".into(),
})
}
Observation::Missing => {
return Ok(AttemptOutcome::Conflict {
code: "remote_message_missing".into(),
})
}
Observation::Message(message) if payload.satisfied_by(&message) => {
return Ok(AttemptOutcome::AppliedImapFlags {
source: "positive_read".into(),
message,
})
}
Observation::Message(_) => {}
}
if kind == AttemptKind::Reconcile {
// An authenticated positive read established current desired-state
// absence. Only this idempotent flag operation may be dispatched again.
return Ok(if ordinal < 8 {
AttemptOutcome::Repeatable {
code: "desired_state_not_observed".into(),
retry_at_ms: retry,
}
} else {
AttemptOutcome::Conflict {
code: "imap_flags_unconfirmed".into(),
}
});
}
Ok(match flags::change(&mut connection, payload).await {
Ok(Observation::Message(message)) if payload.satisfied_by(&message) => {
AttemptOutcome::AppliedImapFlags {
source: "acknowledgement".into(),
message,
}
}
_ => AttemptOutcome::Uncertain {
code: "imap_flags_acknowledgement_unknown".into(),
},
})
}
}
Loaded 100 of 504 files, more files were not shown because too many files have changed in this diff. Show more