diff --git a/integration-tests/features/scanning.feature b/integration-tests/features/scanning.feature index 25061bb..732b317 100644 --- a/integration-tests/features/scanning.feature +++ b/integration-tests/features/scanning.feature @@ -37,3 +37,17 @@ Feature: Blockchain Scanning And I perform a scan with batch size "5" Then blocks should be fetched in batches of "5" And the scan should complete successfully + + Scenario: Fast sync with small safety buffer + Given I have a seed node MinerNode + And I have a test database with an existing wallet + When I mine 10 blocks on MinerNode + And I perform a fast sync with safety buffer "5" + Then the fast sync should complete successfully + + Scenario: Fast sync completes successfully + Given I have a seed node MinerNode + And I have a test database with an existing wallet + When I mine 10 blocks on MinerNode + And I perform a fast sync + Then the fast sync should complete successfully diff --git a/integration-tests/steps/scanning.rs b/integration-tests/steps/scanning.rs index 4abb5c5..0cb5cf9 100644 --- a/integration-tests/steps/scanning.rs +++ b/integration-tests/steps/scanning.rs @@ -192,3 +192,60 @@ async fn blocks_in_batches(world: &mut MinotariWorld, _batch_size: String) { // Verify scan with custom batch size completed scan_succeeds(world).await; } + +#[when(regex = r#"^I perform a fast sync with safety buffer "([^"]*)"$"#)] +async fn fast_sync_with_safety_buffer(world: &mut MinotariWorld, safety_buffer: String) { + let db_path = world.database_path.as_ref().expect("Database not set up"); + + // Get base node URL from the first available base node + let base_url = if let Some((_, node)) = world.base_nodes.iter().next() { + format!("http://127.0.0.1:{}", node.http_port) + } else { + panic!("No base node available for scanning"); + }; + + let (cmd, mut args) = world.get_minotari_command(); + args.extend_from_slice(&[ + "scan".to_string(), + "--database-path".to_string(), + db_path.to_str().unwrap().to_string(), + "--password".to_string(), + world.test_password.clone(), + "--base-url".to_string(), + base_url, + "--fast-sync".to_string(), + "--fast-sync-safety-buffer".to_string(), + safety_buffer, + ]); + + let output = Command::new(&cmd) + .args(&args) + .output() + .expect("Failed to execute fast sync command"); + + world.last_command_exit_code = Some(output.status.code().unwrap_or(-1)); + world.last_command_output = Some(String::from_utf8_lossy(&output.stdout).to_string()); + world.last_command_error = Some(String::from_utf8_lossy(&output.stderr).to_string()); + + println!("Fast sync output: {}", world.last_command_output.as_ref().unwrap()); + if !world.last_command_error.as_ref().unwrap().is_empty() { + println!("Fast sync stderr: {}", world.last_command_error.as_ref().unwrap()); + } +} + +#[when("I perform a fast sync")] +async fn fast_sync_default(world: &mut MinotariWorld) { + // Use a small safety buffer (5 blocks) in tests so fast sync completes quickly + // even when only a few blocks have been mined. The production default is 720 blocks. + fast_sync_with_safety_buffer(world, "5".to_string()).await; +} + +#[then("the fast sync should complete successfully")] +async fn fast_sync_succeeds(world: &mut MinotariWorld) { + assert_eq!( + world.last_command_exit_code, + Some(0), + "Fast sync command failed: {}", + world.last_command_error.as_deref().unwrap_or("") + ); +} diff --git a/minotari/config/config.toml b/minotari/config/config.toml index 9a8fffd..1f46811 100644 --- a/minotari/config/config.toml +++ b/minotari/config/config.toml @@ -8,6 +8,10 @@ scan_interval_secs = 60 api_port = 9000 confirmation_window = 3 account_name = "default" +# Safety buffer (in blocks) used when running fast sync. +# fast_sync_target_height = tip - fast_sync_safety_buffer +# Defaults to 720 blocks (~12 hours on mainnet) if not set. +# fast_sync_safety_buffer = 720 # [wallet.webhook] # url = "https://your-api.com/webhook" diff --git a/minotari/src/cli.rs b/minotari/src/cli.rs index a5549a7..2908e39 100644 --- a/minotari/src/cli.rs +++ b/minotari/src/cli.rs @@ -151,6 +151,13 @@ pub enum Commands { /// /// - `max_blocks_to_scan`: Limits scan duration (default: 50) /// - `batch_size`: Number of blocks per API request (default: 100) + /// + /// # Fast Sync + /// + /// Use `--fast-sync` to run a three-phase fast synchronisation: + /// 1. Scans birthday → `tip - safety_buffer` for the unspent UTXO set + /// 2. Scans `tip - safety_buffer` → tip for recent changes + /// 3. Rescans birthday → tip to fill in the complete transaction history Scan { #[command(flatten)] security: SecurityArgs, @@ -164,6 +171,23 @@ pub enum Commands { /// Maximum number of blocks to scan in this invocation. #[arg(short = 'n', long, default_value_t = 50)] max_blocks_to_scan: u64, + + /// Enable fast synchronisation mode. + /// + /// When set, the scanner runs three phases to quickly establish the current + /// wallet balance before filling in the full transaction history: + /// 1. birthday → tip-safety_buffer (unspent UTXO set) + /// 2. tip-safety_buffer → tip (recent full scan) + /// 3. birthday → tip (full history) + #[arg(long, default_value_t = false)] + fast_sync: bool, + + /// Safety buffer (in blocks) used when calculating the fast-sync target height. + /// + /// `fast_sync_target_height = tip - fast_sync_safety_buffer`. + /// Only used when `--fast-sync` is set. Defaults to 720 blocks. + #[arg(long)] + fast_sync_safety_buffer: Option, }, /// Re-scan the blockchain from a specific height. diff --git a/minotari/src/config/defaults.rs b/minotari/src/config/defaults.rs index 3c4e828..c31751e 100644 --- a/minotari/src/config/defaults.rs +++ b/minotari/src/config/defaults.rs @@ -25,6 +25,11 @@ pub struct WalletConfig { pub confirmation_window: u64, pub account_name: Option, pub webhook: WebhookConfig, + /// Safety buffer (in blocks) for fast sync. + /// + /// The fast-sync target height is `tip - fast_sync_safety_buffer`. + /// If not set, the default value of 720 blocks (approximately 12 hours on mainnet) is used. + pub fast_sync_safety_buffer: Option, } impl Default for WalletConfig { @@ -39,6 +44,7 @@ impl Default for WalletConfig { confirmation_window: 3, account_name: None, webhook: WebhookConfig::default(), + fast_sync_safety_buffer: None, } } } diff --git a/minotari/src/db/accounts.rs b/minotari/src/db/accounts.rs index a761708..7ab6212 100644 --- a/minotari/src/db/accounts.rs +++ b/minotari/src/db/accounts.rs @@ -155,7 +155,7 @@ pub fn get_accounts(conn: &Connection, friendly_name: Option<&str>) -> WalletDbR } } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct AccountRow { pub id: i64, pub friendly_name: String, diff --git a/minotari/src/main.rs b/minotari/src/main.rs index d977e48..979a722 100644 --- a/minotari/src/main.rs +++ b/minotari/src/main.rs @@ -288,6 +288,8 @@ async fn main() -> Result<(), anyhow::Error> { db, account, max_blocks_to_scan, + fast_sync, + fast_sync_safety_buffer, } => { info!("Scanning blockchain..."); @@ -295,13 +297,22 @@ async fn main() -> Result<(), anyhow::Error> { wallet_config.apply_database(&db); wallet_config.apply_account(&account); - let (events, _more_blocks_to_scan) = scan( - &security.password, - &wallet_config, - max_blocks_to_scan, - wallet_config.account_name.as_deref(), - ) - .await?; + let (events, _more_blocks_to_scan) = if fast_sync { + let safety_buffer = fast_sync_safety_buffer + .or(wallet_config.fast_sync_safety_buffer) + .unwrap_or(scan::DEFAULT_FAST_SYNC_SAFETY_BUFFER); + info!(safety_buffer = safety_buffer; "Fast sync enabled"); + fast_sync_scan(&security.password, &wallet_config, safety_buffer, wallet_config.account_name.as_deref()) + .await? + } else { + scan( + &security.password, + &wallet_config, + max_blocks_to_scan, + wallet_config.account_name.as_deref(), + ) + .await? + }; info!(event_count = events.len(); "Scan complete"); Ok(()) }, @@ -596,6 +607,36 @@ async fn scan( scanner.run().await } +async fn fast_sync_scan( + password: &str, + config: &WalletConfig, + safety_buffer: u64, + account_name: Option<&str>, +) -> Result<(Vec, bool), ScanError> { + let mut scanner = scan::Scanner::new( + password, + &config.base_url, + config.database_path.clone(), + config.batch_size, + config.confirmation_window, + ) + .mode(scan::ScanMode::FastSync { safety_buffer }); + + if let Some(name) = account_name { + scanner = scanner.account(name); + } + + if let Some(url) = &config.webhook.url { + let trigger_config = WebhookTriggerConfig { + url: url.clone(), + send_only_event_types: config.webhook.send_only_event_types.clone(), + }; + scanner = scanner.webhook_config(trigger_config); + } + + scanner.run().await +} + async fn rescan( password: &str, config: &WalletConfig, diff --git a/minotari/src/scan/config.rs b/minotari/src/scan/config.rs index 039065f..a665ed8 100644 --- a/minotari/src/scan/config.rs +++ b/minotari/src/scan/config.rs @@ -20,6 +20,13 @@ pub const MAX_BACKOFF_SECONDS: u64 = 60; pub const OPTIMAL_SCANNING_THREADS: usize = 0; // Based on num_cpus +/// Default safety buffer (in blocks) for fast sync. +/// +/// The fast sync target height is calculated as `tip - DEFAULT_FAST_SYNC_SAFETY_BUFFER`. +/// This buffer ensures we have a stable UTXO set snapshot that is unlikely to be affected +/// by chain reorganisations during the fast scan phase. +pub const DEFAULT_FAST_SYNC_SAFETY_BUFFER: u64 = 720; + /// Configuration for scan operation timeouts. /// /// This is a simplified configuration struct for controlling timeout behavior. @@ -74,6 +81,11 @@ impl Default for ScanTimeoutConfig { /// let continuous = ScanMode::Continuous { /// poll_interval: Duration::from_secs(30), /// }; +/// +/// // Fast sync with default safety buffer +/// let fast_sync = ScanMode::FastSync { +/// safety_buffer: DEFAULT_FAST_SYNC_SAFETY_BUFFER, +/// }; /// ``` #[derive(Debug, Clone)] pub enum ScanMode { @@ -101,6 +113,40 @@ pub enum ScanMode { /// Duration to wait between scan cycles after reaching chain tip. poll_interval: Duration, }, + + /// Fast synchronisation that prioritises getting an accurate current balance + /// quickly before filling in the full transaction history. + /// + /// The fast sync process runs three sequential phases: + /// + /// 1. **Phase 1 – Unspent UTXO sync** (birthday → `tip - safety_buffer`): + /// Scans from the wallet birthday up to the *fast-sync target height* + /// (`tip - safety_buffer`), retrieving the unspent UTXO set at that + /// height. This phase rapidly establishes an accurate picture of + /// outputs that belong to the wallet without scanning the most recent + /// (and most volatile) blocks. + /// + /// 2. **Phase 2 – Recent full scan** (`fast_sync_target_height` → tip): + /// Performs a complete scan of the remaining, recent blocks up to the + /// chain tip. After this phase the wallet balance is fully accurate. + /// + /// 3. **Phase 3 – Full history scan** (birthday → tip): + /// Re-scans the entire range from the wallet birthday to the tip to + /// build complete transaction history (including spending records for + /// outputs that may have been spent within the Phase 1 range). + /// + /// # Safety Buffer + /// + /// The `safety_buffer` defines how many blocks from the tip to treat as + /// "recent". A larger buffer means Phase 1 covers a smaller range and + /// Phase 2 covers a larger range. The default is [`DEFAULT_FAST_SYNC_SAFETY_BUFFER`] + /// (720 blocks, approximately 12 hours on mainnet). + FastSync { + /// Number of blocks from the tip that are treated as the "recent" zone. + /// + /// `fast_sync_target_height = tip_height - safety_buffer`. + safety_buffer: u64, + }, } /// Comprehensive configuration for scan retry behavior. diff --git a/minotari/src/scan/coordinator.rs b/minotari/src/scan/coordinator.rs index 0d7c408..afa1400 100644 --- a/minotari/src/scan/coordinator.rs +++ b/minotari/src/scan/coordinator.rs @@ -118,6 +118,12 @@ impl ScanCoordinator { sync_targets.push(target); } + if let ScanMode::FastSync { safety_buffer } = mode { + return self + .run_fast_sync(sync_targets, safety_buffer, scanning_offset, cancel_token, &mut shared_reorg_scanner) + .await; + } + self.unified_scan_loop(sync_targets, mode, cancel_token).await } @@ -537,4 +543,119 @@ impl ScanCoordinator { .await .map_err(|e| ScanError::Intermittent(e.to_string())) } + + /// Executes the three-phase fast synchronisation process. + /// + /// **Phase 1 – Unspent UTXO sync** (birthday → `fast_sync_target_height`): + /// Scans from each account's birthday up to `tip − safety_buffer`, + /// retrieving the unspent UTXO set at that height. This phase quickly + /// establishes the wallet's approximate current balance without processing + /// the most volatile recent blocks. + /// + /// **Phase 2 – Recent full scan** (`fast_sync_target_height` → tip): + /// Performs a complete scan of the remaining recent blocks up to the chain + /// tip. After this phase the wallet balance is fully accurate. + /// + /// **Phase 3 – Full history scan** (birthday → tip): + /// Re-scans the entire chain from each account's birthday to build + /// the complete transaction and spending history. This phase fills in any + /// history that Phase 1 may not have fully captured. + /// + /// Phases 1 and 2 are run as a single continuous pass (birthday → tip) using + /// the existing [`unified_scan_loop`]. Phase 3 then resets each account to + /// its birthday and performs a second full pass to ensure complete history. + async fn run_fast_sync( + &self, + sync_targets: Vec, + safety_buffer: u64, + scanning_offset: u64, + cancel_token: Option, + scanner: &mut HttpBlockchainScanner, + ) -> Result<(Vec, bool), ScanError> { + // Determine the fast-sync target height (tip − safety_buffer). + let tip_info = scanner + .get_tip_info() + .await + .map_err(|e| ScanError::Fatal(anyhow::anyhow!("Failed to get tip info for fast sync: {}", e)))?; + + let tip_height = tip_info.best_block_height; + let fast_sync_target_height = tip_height.saturating_sub(safety_buffer); + + info!( + tip_height = tip_height, + fast_sync_target_height = fast_sync_target_height, + safety_buffer = safety_buffer; + "Fast sync: starting Phase 1 (birthday → fast_sync_target_height) \ + and Phase 2 (fast_sync_target_height → tip) as a single continuous pass" + ); + + // Capture the data needed to reconstruct Phase 3 targets BEFORE Phase 1+2 + // consumes `sync_targets`. We compute each account's birthday height here so + // that Phase 3 can start from the correct position. + const SECONDS_PER_DAY: u64 = 86_400; + let mut phase3_seed: Vec<(AccountRow, KeyManager, PrivateKey, u64)> = Vec::new(); + for target in &sync_targets { + let timestamp = (target.account.birthday as u64).saturating_sub(scanning_offset) * SECONDS_PER_DAY + + tari_common_types::seeds::cipher_seed::BIRTHDAY_GENESIS_FROM_UNIX_EPOCH; + let birthday_height = self + .client + .get_height_at_time(timestamp) + .await + .map_err(ScanError::Fatal)?; + phase3_seed.push(( + target.account.clone(), + target.key_manager.clone(), + target.view_key.clone(), + birthday_height, + )); + } + + // Phase 1 + 2: Scan from birthday to fast_sync_target_height (unspent UTXO sync), + // then continue to tip (recent full scan). These are run as one continuous pass. + let (mut all_events, _) = self + .unified_scan_loop(sync_targets, ScanMode::Full, cancel_token.clone()) + .await?; + + // Check for cancellation before starting Phase 3. + if let Some(token) = &cancel_token { + if token.is_cancelled() { + info!("Fast sync cancelled before Phase 3 (history scan)"); + return Ok((all_events, false)); + } + } + + info!( + fast_sync_target_height = fast_sync_target_height; + "Fast sync: starting Phase 3 (full history scan from birthday → tip)" + ); + + // Phase 3: Full history scan. + // Reconstruct targets with `next_block_to_scan` reset to each account's birthday + // so the history scan covers the complete range. + let conn = self.pool.get().map_err(|e| ScanError::DbError(e.into()))?; + let mut history_targets = Vec::with_capacity(phase3_seed.len()); + for (account, key_manager, view_key, birthday_height) in phase3_seed { + let monitor_state = MonitoringState::new(); + monitor_state.initialize(&conn, account.id).map_err(ScanError::Fatal)?; + let transaction_monitor = + TransactionMonitor::new(monitor_state, self.required_confirmations, self.webhook_config.clone()); + history_targets.push(AccountSyncTarget { + account, + key_manager, + view_key, + next_block_to_scan: birthday_height, + transaction_monitor, + }); + } + drop(conn); + + let (history_events, _) = self + .unified_scan_loop(history_targets, ScanMode::Full, cancel_token) + .await?; + + all_events.extend(history_events); + + info!("Fast sync complete"); + Ok((all_events, false)) + } } diff --git a/minotari/src/scan/mod.rs b/minotari/src/scan/mod.rs index be8d297..84705d6 100644 --- a/minotari/src/scan/mod.rs +++ b/minotari/src/scan/mod.rs @@ -97,6 +97,7 @@ mod coordinator; mod scanner_state_manager; pub use builder::Scanner; +pub use config::DEFAULT_FAST_SYNC_SAFETY_BUFFER; pub use config::ScanMode; pub use config::ScanRetryConfig; pub use config::ScanTimeoutConfig;