diff --git a/crates/pevm/Cargo.toml b/crates/pevm/Cargo.toml index 0904f955..b8643354 100644 --- a/crates/pevm/Cargo.toml +++ b/crates/pevm/Cargo.toml @@ -21,6 +21,7 @@ alloy-trie.workspace = true bitflags.workspace = true dashmap.workspace = true hashbrown.workspace = true +rayon.workspace = true rustc-hash.workspace = true serde.workspace = true smallvec.workspace = true diff --git a/crates/pevm/benches/gigagas.rs b/crates/pevm/benches/gigagas.rs index 6bfe257d..97b0752a 100644 --- a/crates/pevm/benches/gigagas.rs +++ b/crates/pevm/benches/gigagas.rs @@ -97,6 +97,7 @@ pub fn bench_raw_transfers(c: &mut Criterion) { value: U256::from(1), gas_limit: common::RAW_TRANSFER_GAS_LIMIT, gas_price: 1, + nonce: 1, ..TxEnv::default() } }) diff --git a/crates/pevm/src/mv_memory.rs b/crates/pevm/src/mv_memory.rs index e4e54253..d1693bd5 100644 --- a/crates/pevm/src/mv_memory.rs +++ b/crates/pevm/src/mv_memory.rs @@ -1,4 +1,5 @@ use std::{ + cell::UnsafeCell, collections::{BTreeMap, HashSet}, sync::Mutex, }; @@ -8,8 +9,8 @@ use dashmap::DashMap; use revm::state::Bytecode; use crate::{ - BuildIdentityHasher, BuildSuffixHasher, MemoryEntry, MemoryLocationHash, ReadOrigin, ReadSet, - TxIdx, TxVersion, WriteSet, + BuildIdentityHasher, BuildSuffixHasher, MemoryEntry, MemoryLocation, MemoryLocationHash, + ReadOrigin, ReadSet, TxIdx, TxVersion, WriteSet, hash_deterministic, }; #[derive(Default, Debug)] @@ -21,23 +22,117 @@ struct LastLocations { type LazyAddresses = HashSet; +// Per-location multi-version storage, in one of two layouts: +// +// Sparse — a BTreeMap keyed by tx_idx, used for most memory locations (storage slots, +// code hashes, etc.) where only a handful of transactions write to the same location. +// Supports O(log n) range queries required by validate_read_locations. +// +// Dense — a contiguous boxed slice indexed directly by tx_idx, used for lazy addresses +// (beneficiary, fee recipients, raw-transfer accounts) which can receive writes from +// nearly every transaction in the block. Dense gives O(1) indexed writes and +// cache-friendly sequential iteration during post-processing, replacing the +// pointer-chasing BTreeMap for these hot paths. +// +// SAFETY invariant for Dense: the Block-STM scheduler ensures at most one thread writes +// to a given tx_idx slot at a time, so UnsafeCell slots on different indices never race. +pub(crate) enum MvEntries { + Sparse(BTreeMap), + Dense(Box<[UnsafeCell>]>), +} + +// SAFETY: UnsafeCell inside Dense is protected by the scheduler's per-tx-idx exclusion. +unsafe impl Send for MvEntries {} +unsafe impl Sync for MvEntries {} + +impl std::fmt::Debug for MvEntries { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Sparse(m) => write!(f, "Sparse(len={})", m.len()), + Self::Dense(s) => write!(f, "Dense(len={})", s.len()), + } + } +} + +impl MvEntries { + fn new_dense(block_size: usize) -> Self { + Self::Dense( + (0..block_size) + .map(|_| UnsafeCell::new(None)) + .collect::>() + .into_boxed_slice(), + ) + } + + // Iterate prior entries with key < `tx_idx` in decreasing order. Unified across + // Sparse (BTreeMap range) and Dense (linear backward scan, skipping empty slots). + // VM reads and validate_read_locations both rely on this ordering to find the + // closest prior writer and to detect Estimate markers from aborted incarnations. + pub(crate) fn iter_back_below(&self, tx_idx: TxIdx) -> MvIter<'_> { + match self { + Self::Sparse(map) => MvIter::Sparse(map.range(..tx_idx)), + Self::Dense(slots) => MvIter::Dense { + slots, + cursor: tx_idx.min(slots.len()), + }, + } + } + + // Iterate Dense slots in tx_idx order, returning references to entries or None. + // SAFETY: must be called only in single-threaded post-processing after all writes finish. + pub(crate) fn dense_iter(&self) -> impl Iterator> { + let Self::Dense(slots) = self else { unreachable!() }; + slots.iter().map(|cell| unsafe { (*cell.get()).as_ref() }) + } +} + +pub(crate) enum MvIter<'a> { + Sparse(std::collections::btree_map::Range<'a, TxIdx, MemoryEntry>), + Dense { + slots: &'a [UnsafeCell>], + cursor: usize, + }, +} + +impl<'a> MvIter<'a> { + // Yield the next entry below the current cursor (most-recent-first). Returns the + // tx_idx and a reference to its MemoryEntry, or None if exhausted. + pub(crate) fn next_back(&mut self) -> Option<(TxIdx, &'a MemoryEntry)> { + match self { + Self::Sparse(r) => r.next_back().map(|(k, v)| (*k, v)), + Self::Dense { slots, cursor } => { + while *cursor > 0 { + *cursor -= 1; + // SAFETY: validation/read paths run after the corresponding record() + // with happens-before edges from the scheduler's atomic state. + if let Some(entry) = unsafe { (*slots[*cursor].get()).as_ref() } { + return Some((*cursor, entry)); + } + } + None + } + } + } +} + /// The `MvMemory` contains shared memory in a form of a multi-version data /// structure for values written and read by different transactions. It stores /// multiple writes for each memory location, along with a value and an associated /// version of a corresponding transaction. #[derive(Debug)] pub struct MvMemory { - /// The list of transaction incarnations and written values for each memory location + /// Per-location multi-version entries. + /// Sparse (BTreeMap) for most locations; Dense (Vec indexed by tx_idx) for lazy addresses. // No more hashing is required as we already identify memory locations by their hash - // in the read & write sets. [dashmap] having a dedicated interface for this use case - // (that skips hashing for [u64] keys) would make our code cleaner and "faster". - // Nevertheless, the compiler should be good enough to optimize these cases anyway. - pub(crate) data: DashMap, BuildIdentityHasher>, - /// Last read & written locations of each transaction + // in the read & write sets. + pub(crate) data: DashMap, + /// Total number of transactions; needed to size new Dense entries added at runtime. + block_size: usize, + /// Last read & written locations of each transaction. last_locations: Vec>, - /// Lazy addresses that need full evaluation at the end of the block + /// Lazy addresses that need full evaluation at the end of the block. lazy_addresses: Mutex, - /// New bytecodes deployed in this block + /// New bytecodes deployed in this block. pub(crate) new_bytecodes: DashMap, } @@ -47,28 +142,37 @@ impl MvMemory { estimated_locations: impl IntoIterator)>, lazy_addresses: impl IntoIterator, ) -> Self { - // TODO: Fine-tune the number of shards, like to the next number of two from the - // number of worker threads. - let data = DashMap::default(); - // We preallocate estimated locations to avoid restructuring trees at runtime - // while holding a write lock. Ideally [dashmap] would have a lock-free - // construction API. This is acceptable for now as it's a non-congested one-time - // cost. + let data: DashMap = + DashMap::default(); + + // Pre-populate estimated locations with Sparse Estimate entries to avoid + // BTreeMap rebalancing under lock at runtime. for (location_hash, estimated_tx_idxs) in estimated_locations { data.insert( location_hash, - estimated_tx_idxs - .into_iter() - .map(|tx_idx| (tx_idx, MemoryEntry::Estimate)) - .collect(), + MvEntries::Sparse( + estimated_tx_idxs + .into_iter() + .map(|tx_idx| (tx_idx, MemoryEntry::Estimate)) + .collect(), + ), ); } + + // Lazy addresses use Dense entries so writes are O(1) indexed and post-processing + // iterates a contiguous slice instead of a pointer-chasing BTreeMap. + let lazy_addresses_vec: Vec
= lazy_addresses.into_iter().collect(); + for &address in &lazy_addresses_vec { + let hash = hash_deterministic(MemoryLocation::Basic(address)); + // Overwrite any Sparse entry (e.g. from estimated_locations) with Dense. + data.insert(hash, MvEntries::new_dense(block_size)); + } + Self { data, + block_size, last_locations: (0..block_size).map(|_| Mutex::default()).collect(), - lazy_addresses: Mutex::new(LazyAddresses::from_iter(lazy_addresses)), - // TODO: Fine-tune the number of shards, like to the next number of two from the - // number of worker threads. + lazy_addresses: Mutex::new(LazyAddresses::from_iter(lazy_addresses_vec)), new_bytecodes: DashMap::default(), } } @@ -76,7 +180,29 @@ impl MvMemory { pub(crate) fn add_lazy_addresses(&self, new_lazy_addresses: impl IntoIterator) { let mut lazy_addresses = self.lazy_addresses.lock().unwrap(); for address in new_lazy_addresses { - lazy_addresses.insert(address); + if !lazy_addresses.insert(address) { + continue; // already registered; Dense entry already exists in data + } + let hash = hash_deterministic(MemoryLocation::Basic(address)); + // Upgrade the location from Sparse to Dense atomically under the DashMap shard + // write lock. Holding the lock through the entire replace+migrate prevents a + // concurrent record() from writing to a Sparse we're about to drop. + let mut entry_ref = self + .data + .entry(hash) + .or_insert_with(|| MvEntries::Sparse(BTreeMap::new())); + let old = std::mem::replace(entry_ref.value_mut(), MvEntries::new_dense(self.block_size)); + // Migrate all existing Sparse entries (Data + Estimate) into Dense. Preserving + // Estimate is required so concurrent VM reads still see "blocked, wait for + // re-execution" instead of silently reading a stale prior value. + if let MvEntries::Sparse(map) = old { + let MvEntries::Dense(slots) = entry_ref.value() else { unreachable!() }; + for (tx_idx, entry) in map { + // SAFETY: no concurrent writes during add_lazy_addresses migration + // since we hold the DashMap shard write lock. + unsafe { *slots[tx_idx].get() = Some(entry) }; + } + } } } @@ -93,14 +219,21 @@ impl MvMemory { let mut last_locations = index_mutex!(self.last_locations, tx_version.tx_idx); last_locations.read = read_set; - // TODO: Group updates by shard to avoid locking operations. - // Remove old locations that aren't written to anymore. + // Remove stale writes from the previous incarnation. let mut last_location_idx = 0; while last_location_idx < last_locations.write.len() { let prev_location = unsafe { last_locations.write.get_unchecked(last_location_idx) }; if write_set.iter().all(|(l, _)| l != prev_location) { - if let Some(mut written_transactions) = self.data.get_mut(prev_location) { - written_transactions.remove(&tx_version.tx_idx); + if let Some(mut e) = self.data.get_mut(prev_location) { + match e.value_mut() { + MvEntries::Dense(slots) => { + // SAFETY: see struct-level invariant. + unsafe { *slots[tx_version.tx_idx].get() = None }; + } + MvEntries::Sparse(map) => { + map.remove(&tx_version.tx_idx); + } + } } last_locations.write.swap_remove(last_location_idx); } else { @@ -112,10 +245,42 @@ impl MvMemory { let mut wrote_new_location = false; for (location, value) in write_set { - self.data.entry(location).or_default().insert( - tx_version.tx_idx, - MemoryEntry::Data(tx_version.tx_incarnation, value), - ); + // Fast path: Dense entries allow lockless indexed writes via a read lock. + // Clone value for the Dense case; move it for Sparse (avoids Clone on cold path). + let used_dense = if let Some(e) = self.data.get(&location) { + if let MvEntries::Dense(slots) = e.value() { + // SAFETY: see struct-level invariant. + unsafe { + *slots[tx_version.tx_idx].get() = + Some(MemoryEntry::Data(tx_version.tx_incarnation, value.clone())); + }; + true + } else { + false + } + } else { + false + }; + + if !used_dense { + // Sparse path: acquire write lock. Handle race where add_lazy_addresses + // upgraded this location to Dense between our get() check and entry(). + let new_entry = MemoryEntry::Data(tx_version.tx_incarnation, value); + let mut entry_ref = self + .data + .entry(location) + .or_insert_with(|| MvEntries::Sparse(BTreeMap::new())); + match entry_ref.value_mut() { + MvEntries::Sparse(map) => { + map.insert(tx_version.tx_idx, new_entry); + } + MvEntries::Dense(slots) => { + // SAFETY: see struct-level invariant. + unsafe { *slots[tx_version.tx_idx].get() = Some(new_entry) }; + } + } + } + if !last_locations.write.contains(&location) { last_locations.write.push(location); wrote_new_location = true; @@ -139,29 +304,25 @@ impl MvMemory { // can be aborted at most once). pub(crate) fn validate_read_locations(&self, tx_idx: TxIdx) -> bool { for (location, prior_origins) in &index_mutex!(self.last_locations, tx_idx).read { - if let Some(written_transactions) = self.data.get(location) { - let mut iter = written_transactions.range(..tx_idx); + if let Some(entries) = self.data.get(location) { + let mut iter = entries.iter_back_below(tx_idx); for prior_origin in prior_origins { if let ReadOrigin::MvMemory(prior_version) = prior_origin { - // Found something: Must match version. + // Found something: must match version. Estimate (pending re-exec) + // or a different (closest_idx, incarnation) means the read is stale. if let Some((closest_idx, MemoryEntry::Data(tx_incarnation, ..))) = iter.next_back() { - if closest_idx != &prior_version.tx_idx + if closest_idx != prior_version.tx_idx || &prior_version.tx_incarnation != tx_incarnation { return false; } - } - // The previously read value is now cleared - // or marked with ESTIMATE. - else { + } else { return false; } - } - // Read from storage but there is now something - // in between! - else if iter.next_back().is_some() { + } else if iter.next_back().is_some() { + // Read from storage but there is now something in between. return false; } } @@ -180,8 +341,16 @@ impl MvMemory { // that read them. pub(crate) fn convert_writes_to_estimates(&self, tx_idx: TxIdx) { for location in &index_mutex!(self.last_locations, tx_idx).write { - if let Some(mut written_transactions) = self.data.get_mut(location) { - written_transactions.insert(tx_idx, MemoryEntry::Estimate); + if let Some(mut e) = self.data.get_mut(location) { + match e.value_mut() { + MvEntries::Dense(slots) => { + // SAFETY: see struct-level invariant. + unsafe { *slots[tx_idx].get() = Some(MemoryEntry::Estimate) }; + } + MvEntries::Sparse(map) => { + map.insert(tx_idx, MemoryEntry::Estimate); + } + } } } } diff --git a/crates/pevm/src/pevm.rs b/crates/pevm/src/pevm.rs index 3ecc2312..de747b54 100644 --- a/crates/pevm/src/pevm.rs +++ b/crates/pevm/src/pevm.rs @@ -1,11 +1,13 @@ use std::{ + cell::UnsafeCell, fmt::Debug, num::NonZeroUsize, - sync::{Mutex, OnceLock, mpsc}, + sync::{OnceLock, mpsc}, thread, }; use alloy_primitives::{TxNonce, U256}; +use rayon::prelude::*; use alloy_rpc_types_eth::{Block, BlockTransactions}; use hashbrown::HashMap; use revm::{ @@ -20,7 +22,7 @@ use crate::{ chain::PevmChain, compat::get_block_env, hash_deterministic, - mv_memory::MvMemory, + mv_memory::{MvEntries, MvMemory}, scheduler::Scheduler, storage::StorageWrapper, vm::{ExecutionError, PevmTxExecutionResult, Vm, VmExecutionError, VmExecutionResult}, @@ -100,11 +102,31 @@ impl AsyncDropper { } } +// Per-tx slot for execution results. UnsafeCell allows worker threads to write through +// a shared reference without taking a lock; the Block-STM scheduler guarantees at most +// one writer per tx_idx at a time, so writes to different slots never race. +#[derive(Debug, Default)] +struct ExecutionResultSlot(UnsafeCell>); + +// SAFETY: see struct-level invariant. +unsafe impl Sync for ExecutionResultSlot {} + +impl ExecutionResultSlot { + fn write(&self, value: PevmTxExecutionResult) { + // SAFETY: see struct-level invariant. + unsafe { *self.0.get() = Some(value) }; + } + + fn take(&mut self) -> Option { + self.0.get_mut().take() + } +} + // TODO: Port more recyclable resources into here. #[derive(Debug, Default)] /// The main pevm struct that executes blocks. pub struct Pevm { - execution_results: Vec>>, + execution_results: Vec, abort_reason: OnceLock, dropper: AsyncDropper<(MvMemory, Scheduler)>, } @@ -189,7 +211,7 @@ impl Pevm { if additional > 0 { self.execution_results.reserve(additional); for _ in 0..additional { - self.execution_results.push(Mutex::new(None)); + self.execution_results.push(ExecutionResultSlot::default()); } } @@ -242,8 +264,10 @@ impl Pevm { let mut fully_evaluated_results = Vec::with_capacity(block_size); let mut cumulative_gas_used: u64 = 0; - for i in 0..block_size { - let mut execution_result = index_mutex!(self.execution_results, i).take().unwrap(); + // After thread::scope returns we hold exclusive access to self.execution_results, + // so we can take() through &mut without any locking. + for slot in self.execution_results.iter_mut().take(block_size) { + let mut execution_result = slot.take().unwrap(); cumulative_gas_used = cumulative_gas_used.saturating_add(execution_result.receipt.cumulative_gas_used); execution_result.receipt.cumulative_gas_used = cumulative_gas_used; @@ -254,115 +278,125 @@ impl Pevm { // and raw transfer recipients that may have been atomically updated. for address in mv_memory.consume_lazy_addresses() { let location_hash = hash_deterministic(MemoryLocation::Basic(address)); - if let Some(write_history) = mv_memory.data.get(&location_hash) { - let mut balance = U256::ZERO; - let mut nonce = 0; - // Read from storage if the first multi-version entry is not an absolute value. - if !matches!( - write_history.first_key_value(), - Some((_, MemoryEntry::Data(_, MemoryValue::Basic(_)))) - ) && let Ok(Some(account)) = storage.basic(&address) - { - balance = account.balance; - nonce = account.nonce; - } - // Accounts that take implicit writes like the beneficiary account can be contract! - let code_hash = match storage.code_hash(&address) { - Ok(code_hash) => code_hash, + + let Some(entries) = mv_memory.data.get(&location_hash) else { + continue; + }; + + // Load code once — lazy addresses can be contracts (e.g. fee recipient). + let code_hash = match storage.code_hash(&address) { + Ok(ch) => ch, + Err(err) => return Err(PevmError::StorageError(err.to_string())), + }; + let code = if let Some(ch) = &code_hash { + match storage.code_by_hash(ch) { + Ok(c) => c, Err(err) => return Err(PevmError::StorageError(err.to_string())), - }; - let code = if let Some(code_hash) = &code_hash { - match storage.code_by_hash(code_hash) { - Ok(code) => code, - Err(err) => return Err(PevmError::StorageError(err.to_string())), - } - } else { - None - }; - - for (tx_idx, memory_entry) in write_history.iter() { - let tx = chain.tx_env(unsafe { txs.get_unchecked(*tx_idx) }); - match memory_entry { - MemoryEntry::Data(_, MemoryValue::Basic(info)) => { - // We fall back to sequential execution when reading a self-destructed account, - // so an empty account here would be a bug - debug_assert!(!(info.balance.is_zero() && info.nonce == 0)); - balance = info.balance; - nonce = info.nonce; - } - MemoryEntry::Data(_, MemoryValue::LazyRecipient(addition)) => { - balance = balance.saturating_add(*addition); - } - MemoryEntry::Data(_, MemoryValue::LazySender(subtraction)) => { - // We must re-do extra sender balance checks as we mock - // the max value in [Vm] during execution. Ideally we - // can turn off these redundant checks in revm. - // Ideally we would share these calculations with revm - // (using their utility functions). - let mut max_fee = U256::from(tx.gas_limit) - .saturating_mul(U256::from(tx.gas_price)) - .saturating_add(tx.value); - max_fee = max_fee.saturating_add( - U256::from(tx.total_blob_gas()) - .saturating_mul(U256::from(tx.max_fee_per_blob_gas)), - ); - if balance < max_fee { - Err(ExecutionError::Transaction( - InvalidTransaction::LackOfFundForMaxFee { - balance: Box::new(balance), - fee: Box::new(max_fee), - }, - ))? - } - balance = balance.saturating_sub(*subtraction); - nonce += 1; - } - // TODO: Better error handling - _ => unreachable!(), - } - // Assert that evaluated nonce is correct when address is caller. - if tx.caller == address { - let executed_nonce = if nonce == 0 { - return Err(PevmError::UnreachableError); - } else { - nonce - 1 + } + } else { + None + }; + let eip161 = chain.is_eip_161_enabled(spec_id); + + match entries.value() { + MvEntries::Dense(_) => { + // Dense path: all lazy addresses land here after add_lazy_addresses + // upgrades them. Contiguous slice enables cache-friendly sequential + // prefix-sum and a parallel rayon state update. + let (mut balance, mut nonce) = storage + .basic(&address) + .ok() + .flatten() + .map(|acc| (acc.balance, acc.nonce)) + .unwrap_or_default(); + + // Sequential pass: compute running (balance, nonce) after each tx. + // Errors (insufficient balance) are returned immediately. + let mut prefix_states: Vec> = + Vec::with_capacity(block_size); + for (tx_idx, entry_opt) in entries.dense_iter().enumerate() { + let Some(entry) = entry_opt else { + prefix_states.push(None); + continue; }; - if tx.nonce != executed_nonce { - // TODO: Consider falling back to sequential instead - return Err(PevmError::NonceMismatch { - tx_idx: *tx_idx, - tx_nonce: tx.nonce, - executed_nonce, - }); + match entry { + MemoryEntry::Data(_, MemoryValue::Basic(info)) => { + // We fall back to sequential on self-destruct, so empty is a bug. + debug_assert!(!(info.balance.is_zero() && info.nonce == 0)); + balance = info.balance; + nonce = info.nonce; + } + MemoryEntry::Data(_, MemoryValue::LazyRecipient(addition)) => { + balance = balance.saturating_add(*addition); + } + MemoryEntry::Data(_, MemoryValue::LazySender(subtraction)) => { + // Re-do sender balance checks: we mocked MAX balance during + // execution so revm skipped these. Can't share revm's helpers. + let tx = chain.tx_env(unsafe { txs.get_unchecked(tx_idx) }); + let mut max_fee = U256::from(tx.gas_limit) + .saturating_mul(U256::from(tx.gas_price)) + .saturating_add(tx.value); + max_fee = max_fee.saturating_add( + U256::from(tx.total_blob_gas()) + .saturating_mul(U256::from(tx.max_fee_per_blob_gas)), + ); + if balance < max_fee { + return Err(ExecutionError::Transaction( + InvalidTransaction::LackOfFundForMaxFee { + balance: Box::new(balance), + fee: Box::new(max_fee), + }, + ))?; + } + balance = balance.saturating_sub(*subtraction); + nonce += 1; + if tx.caller == address { + let executed_nonce = if nonce == 0 { + return Err(PevmError::UnreachableError); + } else { + nonce - 1 + }; + if tx.nonce != executed_nonce { + return Err(PevmError::NonceMismatch { + tx_idx, + tx_nonce: tx.nonce, + executed_nonce, + }); + } + } + } + _ => unreachable!(), } + prefix_states.push(Some((balance, nonce))); } - // SAFETY: The multi-version data structure should not leak an index over block size. - let tx_result = unsafe { fully_evaluated_results.get_unchecked_mut(*tx_idx) }; - let account = tx_result.state.entry(address).or_default(); - // TODO: Deduplicate this logic with [PevmTxExecutionResult::from_revm] - if chain.is_eip_161_enabled(spec_id) - && code_hash.is_none() - && nonce == 0 - && balance == U256::ZERO - { - *account = None; - } else if let Some(account) = account { - // Explicit write: only overwrite the account info in case there are storage changes - // Code cannot change midblock here as we're falling back to sequential execution - // on reading a self-destructed contract. - account.balance = balance; - account.nonce = nonce; - } else { - // Implicit write: e.g. gas payments to the beneficiary account, - // which doesn't have explicit writes in [tx_result.state] - *account = Some(EvmAccount { - balance, - nonce, - code_hash, - code: code.clone(), - storage: HashMap::default(), + + // Parallel pass: apply (balance, nonce) to each tx_result's state. + fully_evaluated_results + .par_iter_mut() + .zip(prefix_states.par_iter()) + .for_each(|(tx_result, state_opt)| { + let Some(&(bal, nonce)) = state_opt.as_ref() else { + return; + }; + let account = tx_result.state.entry(address).or_default(); + if eip161 && code_hash.is_none() && nonce == 0 && bal == U256::ZERO { + *account = None; + } else if let Some(existing) = account { + existing.balance = bal; + existing.nonce = nonce; + } else { + *account = Some(EvmAccount { + balance: bal, + nonce, + code_hash, + code: code.clone(), + storage: HashMap::default(), + }); + } }); - } + } + MvEntries::Sparse(_) => { + unreachable!("Lazy addresses should have been upgraded to dense entries in add_lazy_addresses, so we shouldn't have any sparse entries left. Found sparse entries for address {:?} with code hash {:?} and code {:?}.", address, code_hash, code); } } } @@ -412,8 +446,10 @@ impl Pevm { execution_result, flags, }) => { - *index_mutex!(self.execution_results, tx_version.tx_idx) = - Some(execution_result); + // SAFETY: scheduler ensures only one thread executes a given tx_idx + // at a time, so this lockless write never races with another writer. + unsafe { self.execution_results.get_unchecked(tx_version.tx_idx) } + .write(execution_result); scheduler.finish_execution(tx_version, flags) } }; diff --git a/crates/pevm/src/vm.rs b/crates/pevm/src/vm.rs index e5760be9..1e14fe04 100644 --- a/crates/pevm/src/vm.rs +++ b/crates/pevm/src/vm.rs @@ -16,7 +16,8 @@ use smallvec::SmallVec; use crate::{ AccountBasic, BuildIdentityHasher, BuildSuffixHasher, EvmAccount, FinishExecFlags, MemoryEntry, MemoryLocation, MemoryLocationHash, MemoryValue, ReadOrigin, ReadOrigins, ReadSet, Storage, - TxIdx, TxVersion, WriteSet, chain::PevmChain, hash_deterministic, mv_memory::MvMemory, + TxIdx, TxVersion, WriteSet, chain::PevmChain, hash_deterministic, + mv_memory::MvMemory, }; /// The execution error from the underlying EVM executor. @@ -216,7 +217,7 @@ impl<'a, S: Storage> VmDb<'a, S> { // contention in [MvMemory] if let Some(written_transactions) = self.mv_memory.data.get(&location_hash) && let Some((tx_idx, MemoryEntry::Data(tx_incarnation, value))) = - written_transactions.range(..self.tx_idx).next_back() + written_transactions.iter_back_below(self.tx_idx).next_back() { match value { MemoryValue::SelfDestructed => { @@ -226,7 +227,7 @@ impl<'a, S: Storage> VmDb<'a, S> { Self::push_origin( read_origins, ReadOrigin::MvMemory(TxVersion { - tx_idx: *tx_idx, + tx_idx, tx_incarnation: *tx_incarnation, }), )?; @@ -283,13 +284,13 @@ impl Database for VmDb<'_, S> { if self.tx_idx > 0 && let Some(written_transactions) = self.mv_memory.data.get(&location_hash) { - let mut iter = written_transactions.range(..self.tx_idx); + let mut iter = written_transactions.iter_back_below(self.tx_idx); // Fully evaluate lazy updates loop { match iter.next_back() { Some((blocking_idx, MemoryEntry::Estimate)) => { - return Err(ReadError::Blocking(*blocking_idx)); + return Err(ReadError::Blocking(blocking_idx)); } Some((closest_idx, MemoryEntry::Data(tx_incarnation, value))) => { // About to push a new origin @@ -298,7 +299,7 @@ impl Database for VmDb<'_, S> { return Err(ReadError::InconsistentRead); } let origin = ReadOrigin::MvMemory(TxVersion { - tx_idx: *closest_idx, + tx_idx: closest_idx, tx_incarnation: *tx_incarnation, }); // Inconsistent: new origin is different from the previous! @@ -444,20 +445,20 @@ impl Database for VmDb<'_, S> { if self.tx_idx > 0 && let Some(written_transactions) = self.mv_memory.data.get(&location_hash) && let Some((closest_idx, entry)) = - written_transactions.range(..self.tx_idx).next_back() + written_transactions.iter_back_below(self.tx_idx).next_back() { match entry { MemoryEntry::Data(tx_incarnation, MemoryValue::Storage(value)) => { Self::push_origin( read_origins, ReadOrigin::MvMemory(TxVersion { - tx_idx: *closest_idx, + tx_idx: closest_idx, tx_incarnation: *tx_incarnation, }), )?; return Ok(*value); } - MemoryEntry::Estimate => return Err(ReadError::Blocking(*closest_idx)), + MemoryEntry::Estimate => return Err(ReadError::Blocking(closest_idx)), _ => return Err(ReadError::InvalidMemoryValueType), } }