From ced3d020c230d4b57fa664ed93197730baa949cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=9D=E5=80=89=E6=B0=B4=E5=B8=8C?= Date: Tue, 25 Nov 2025 14:15:07 +0800 Subject: [PATCH 1/6] chore: cleanup --- page_table_entry/src/arch/aarch64.rs | 3 ++- page_table_entry/src/arch/loongarch64.rs | 4 +++- page_table_entry/src/arch/riscv.rs | 1 + page_table_entry/src/arch/x86_64.rs | 1 + page_table_entry/src/lib.rs | 10 ++++----- page_table_multiarch/src/arch/aarch64.rs | 6 +++-- page_table_multiarch/src/arch/loongarch64.rs | 11 ++++++---- page_table_multiarch/src/arch/riscv.rs | 23 +++++++++----------- page_table_multiarch/src/arch/x86_64.rs | 8 ++++--- page_table_multiarch/src/bits64.rs | 8 +++++-- 10 files changed, 44 insertions(+), 31 deletions(-) diff --git a/page_table_entry/src/arch/aarch64.rs b/page_table_entry/src/arch/aarch64.rs index 9d2ba040..b97d3b5e 100644 --- a/page_table_entry/src/arch/aarch64.rs +++ b/page_table_entry/src/arch/aarch64.rs @@ -1,7 +1,8 @@ //! AArch64 VMSAv8-64 translation table format descriptors. -use aarch64_cpu::registers::MAIR_EL1; use core::fmt; + +use aarch64_cpu::registers::MAIR_EL1; use memory_addr::PhysAddr; use crate::{GenericPTE, MappingFlags}; diff --git a/page_table_entry/src/arch/loongarch64.rs b/page_table_entry/src/arch/loongarch64.rs index bab6a9b9..f12933de 100644 --- a/page_table_entry/src/arch/loongarch64.rs +++ b/page_table_entry/src/arch/loongarch64.rs @@ -4,10 +4,12 @@ //! //! -use crate::{GenericPTE, MappingFlags}; use core::fmt; + use memory_addr::PhysAddr; +use crate::{GenericPTE, MappingFlags}; + bitflags::bitflags! { /// Page-table entry flags. /// diff --git a/page_table_entry/src/arch/riscv.rs b/page_table_entry/src/arch/riscv.rs index 4e7ee634..137ca8bd 100644 --- a/page_table_entry/src/arch/riscv.rs +++ b/page_table_entry/src/arch/riscv.rs @@ -1,6 +1,7 @@ //! RISC-V page table entries. use core::fmt; + use memory_addr::PhysAddr; use crate::{GenericPTE, MappingFlags}; diff --git a/page_table_entry/src/arch/x86_64.rs b/page_table_entry/src/arch/x86_64.rs index 8c720a75..7d6f0738 100644 --- a/page_table_entry/src/arch/x86_64.rs +++ b/page_table_entry/src/arch/x86_64.rs @@ -1,6 +1,7 @@ //! x86 page table entries on 64-bit paging. use core::fmt; + use memory_addr::PhysAddr; pub use x86_64::structures::paging::page_table::PageTableFlags as PTF; diff --git a/page_table_entry/src/lib.rs b/page_table_entry/src/lib.rs index 71f51dce..ae7fe163 100644 --- a/page_table_entry/src/lib.rs +++ b/page_table_entry/src/lib.rs @@ -2,11 +2,11 @@ #![cfg_attr(doc, feature(doc_cfg))] #![doc = include_str!("../README.md")] -mod arch; +use core::fmt; -use core::fmt::{self, Debug}; use memory_addr::PhysAddr; +mod arch; pub use self::arch::*; bitflags::bitflags! { @@ -29,16 +29,16 @@ bitflags::bitflags! { } } -impl Debug for MappingFlags { +impl fmt::Debug for MappingFlags { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - Debug::fmt(&self.0, f) + fmt::Debug::fmt(&self.0, f) } } /// A generic page table entry. /// /// All architecture-specific page table entry types implement this trait. -pub trait GenericPTE: Debug + Clone + Copy + Sync + Send + Sized { +pub trait GenericPTE: fmt::Debug + Clone + Copy + Sync + Send + Sized { /// Creates a page table entry point to a terminate page or block. fn new_page(paddr: PhysAddr, flags: MappingFlags, is_huge: bool) -> Self; /// Creates a page table entry point to a next level page table. diff --git a/page_table_multiarch/src/arch/aarch64.rs b/page_table_multiarch/src/arch/aarch64.rs index fa191a14..81edcadb 100644 --- a/page_table_multiarch/src/arch/aarch64.rs +++ b/page_table_multiarch/src/arch/aarch64.rs @@ -1,6 +1,8 @@ //! AArch64 specific page table structures. use core::arch::asm; + +use memory_addr::VirtAddr; use page_table_entry::aarch64::A64PTE; use crate::{PageTable64, PagingMetaData}; @@ -12,7 +14,7 @@ impl PagingMetaData for A64PagingMetaData { const LEVELS: usize = 4; const PA_MAX_BITS: usize = 48; const VA_MAX_BITS: usize = 48; - type VirtAddr = memory_addr::VirtAddr; + type VirtAddr = VirtAddr; fn vaddr_is_valid(vaddr: usize) -> bool { let top_bits = vaddr >> Self::VA_MAX_BITS; @@ -20,7 +22,7 @@ impl PagingMetaData for A64PagingMetaData { } #[inline] - fn flush_tlb(vaddr: Option) { + fn flush_tlb(vaddr: Option) { unsafe { if let Some(vaddr) = vaddr { // TLB Invalidate by VA, All ASID, EL1, Inner Shareable diff --git a/page_table_multiarch/src/arch/loongarch64.rs b/page_table_multiarch/src/arch/loongarch64.rs index 04a527b0..69081fed 100644 --- a/page_table_multiarch/src/arch/loongarch64.rs +++ b/page_table_multiarch/src/arch/loongarch64.rs @@ -1,9 +1,12 @@ //! LoongArch64 specific page table structures. -use crate::{PageTable64, PagingMetaData}; use core::arch::asm; + +use memory_addr::VirtAddr; use page_table_entry::loongarch64::LA64PTE; +use crate::{PageTable64, PagingMetaData}; + /// Metadata of LoongArch64 page tables. #[derive(Copy, Clone, Debug)] pub struct LA64MetaData; @@ -44,10 +47,10 @@ impl PagingMetaData for LA64MetaData { const LEVELS: usize = 4; const PA_MAX_BITS: usize = 48; const VA_MAX_BITS: usize = 48; - type VirtAddr = memory_addr::VirtAddr; + type VirtAddr = VirtAddr; #[inline] - fn flush_tlb(vaddr: Option) { + fn flush_tlb(vaddr: Option) { unsafe { if let Some(vaddr) = vaddr { // @@ -82,4 +85,4 @@ impl PagingMetaData for LA64MetaData { /// 4 levels: /// /// using page table dir3, dir2, dir1 and pt, ignore dir4 -pub type LA64PageTable = PageTable64; +pub type LA64PageTable = PageTable64; diff --git a/page_table_multiarch/src/arch/riscv.rs b/page_table_multiarch/src/arch/riscv.rs index 8674ea4c..3cf4509a 100644 --- a/page_table_multiarch/src/arch/riscv.rs +++ b/page_table_multiarch/src/arch/riscv.rs @@ -1,16 +1,9 @@ //! RISC-V specific page table structures. -use crate::{PageTable64, PagingMetaData}; +use memory_addr::VirtAddr; use page_table_entry::riscv::Rv64PTE; -#[inline] -fn riscv_flush_tlb(vaddr: Option) { - if let Some(vaddr) = vaddr { - riscv::asm::sfence_vma(0, vaddr.as_usize()) - } else { - riscv::asm::sfence_vma_all(); - } -} +use crate::{PageTable64, PagingMetaData}; /// A virtual address that can be used in RISC-V Sv39 and Sv48 page tables. pub trait SvVirtAddr: memory_addr::MemoryAddr + Send + Sync { @@ -18,10 +11,14 @@ pub trait SvVirtAddr: memory_addr::MemoryAddr + Send + Sync { fn flush_tlb(vaddr: Option); } -impl SvVirtAddr for memory_addr::VirtAddr { +impl SvVirtAddr for VirtAddr { #[inline] fn flush_tlb(vaddr: Option) { - riscv_flush_tlb(vaddr.map(|vaddr| vaddr.into())) + if let Some(vaddr) = vaddr { + riscv::asm::sfence_vma(0, vaddr.as_usize()) + } else { + riscv::asm::sfence_vma_all(); + } } } @@ -60,7 +57,7 @@ impl PagingMetaData for Sv48MetaData { } /// Sv39: Page-Based 39-bit (3 levels) Virtual-Memory System. -pub type Sv39PageTable = PageTable64, Rv64PTE, H>; +pub type Sv39PageTable = PageTable64, Rv64PTE, H>; /// Sv48: Page-Based 48-bit (4 levels) Virtual-Memory System. -pub type Sv48PageTable = PageTable64, Rv64PTE, H>; +pub type Sv48PageTable = PageTable64, Rv64PTE, H>; diff --git a/page_table_multiarch/src/arch/x86_64.rs b/page_table_multiarch/src/arch/x86_64.rs index 6f9c37dd..4e800e29 100644 --- a/page_table_multiarch/src/arch/x86_64.rs +++ b/page_table_multiarch/src/arch/x86_64.rs @@ -1,8 +1,10 @@ //! x86 specific page table structures. -use crate::{PageTable64, PagingMetaData}; +use memory_addr::VirtAddr; use page_table_entry::x86_64::X64PTE; +use crate::{PageTable64, PagingMetaData}; + /// metadata of x86_64 page tables. pub struct X64PagingMetaData; @@ -10,10 +12,10 @@ impl PagingMetaData for X64PagingMetaData { const LEVELS: usize = 4; const PA_MAX_BITS: usize = 52; const VA_MAX_BITS: usize = 48; - type VirtAddr = memory_addr::VirtAddr; + type VirtAddr = VirtAddr; #[inline] - fn flush_tlb(vaddr: Option) { + fn flush_tlb(vaddr: Option) { unsafe { if let Some(vaddr) = vaddr { x86::tlb::flush(vaddr.into()); diff --git a/page_table_multiarch/src/bits64.rs b/page_table_multiarch/src/bits64.rs index 18a01033..a831426e 100644 --- a/page_table_multiarch/src/bits64.rs +++ b/page_table_multiarch/src/bits64.rs @@ -1,8 +1,12 @@ -use crate::{GenericPTE, PagingHandler, PagingMetaData}; -use crate::{MappingFlags, PageSize, PagingError, PagingResult, TlbFlush, TlbFlushAll}; use core::marker::PhantomData; + use memory_addr::{MemoryAddr, PAGE_SIZE_4K, PhysAddr}; +use crate::{ + GenericPTE, MappingFlags, PageSize, PagingError, PagingHandler, PagingMetaData, PagingResult, + TlbFlush, TlbFlushAll, +}; + const ENTRY_COUNT: usize = 512; const fn p4_index(vaddr: usize) -> usize { From ac7d1c8d0cfb7235d9bdbbcd18f9cff800e80724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=9D=E5=80=89=E6=B0=B4=E5=B8=8C?= Date: Fri, 21 Nov 2025 23:09:10 +0800 Subject: [PATCH 2/6] feat: easier patch --- Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 93002b63..65fb26af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,4 +17,7 @@ rust-version = "1.85" [workspace.dependencies] memory_addr = "0.4" -page_table_entry = { path = "page_table_entry", version = "0.5" } +page_table_entry = "0.5" + +[patch.crates-io] +page_table_entry = { path = "./page_table_entry" } From eddf1949cb5db59b4f5e1320a238091450efbf1c Mon Sep 17 00:00:00 2001 From: mivik Date: Thu, 17 Jul 2025 16:08:56 +0800 Subject: [PATCH 3/6] feat: add remap --- page_table_multiarch/src/bits64.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/page_table_multiarch/src/bits64.rs b/page_table_multiarch/src/bits64.rs index a831426e..35a0bdc0 100644 --- a/page_table_multiarch/src/bits64.rs +++ b/page_table_multiarch/src/bits64.rs @@ -100,6 +100,26 @@ impl PageTable64 PagingResult { + let (entry, size) = self.get_entry_mut(vaddr)?; + entry.set_paddr(paddr); + entry.set_flags(flags, size.is_huge()); + M::flush_tlb(Some(vaddr)); + Ok(size) + } + /// Updates the flags of the mapping starts with `vaddr`. /// /// Returns the page size of the mapping. From 4594a765e361bca2a8b6ef6773a37ae6d8539840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=9D=E5=80=89=E6=B0=B4=E5=B8=8C?= Date: Tue, 25 Nov 2025 14:23:53 +0800 Subject: [PATCH 4/6] refactor: modify page table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 朝倉水希 --- Cargo.lock | 7 + page_table_multiarch/Cargo.toml | 1 + page_table_multiarch/README.md | 2 +- page_table_multiarch/src/arch/aarch64.rs | 4 +- page_table_multiarch/src/arch/loongarch64.rs | 4 +- page_table_multiarch/src/arch/riscv.rs | 6 +- page_table_multiarch/src/arch/x86_64.rs | 4 +- page_table_multiarch/src/bits64.rs | 626 +++++++++---------- page_table_multiarch/src/lib.rs | 66 +- page_table_multiarch/tests/alloc_tests.rs | 18 +- 10 files changed, 346 insertions(+), 392 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6acfe1ca..bbd3ceb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "tock-registers", ] +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "axerrno" version = "0.1.2" @@ -105,6 +111,7 @@ dependencies = [ name = "page_table_multiarch" version = "0.5.7" dependencies = [ + "arrayvec", "axerrno 0.1.2", "bitmaps", "log", diff --git a/page_table_multiarch/Cargo.toml b/page_table_multiarch/Cargo.toml index 23348bd9..1eee6350 100644 --- a/page_table_multiarch/Cargo.toml +++ b/page_table_multiarch/Cargo.toml @@ -19,6 +19,7 @@ copy-from = ["dep:bitmaps"] [dependencies] axerrno = { version = "0.1", optional = true } +arrayvec = { version = "0.7", default-features = false } bitmaps = { version = "3.2", default-features = false, optional = true } log = "0.4" memory_addr.workspace = true diff --git a/page_table_multiarch/README.md b/page_table_multiarch/README.md index 04973f19..9ab0de47 100644 --- a/page_table_multiarch/README.md +++ b/page_table_multiarch/README.md @@ -66,6 +66,6 @@ let flags = MappingFlags::READ | MappingFlags::WRITE; let mut pt = X64PageTable::::try_new().unwrap(); assert!(pt.root_paddr().is_aligned_4k()); -assert!(pt.map(vaddr, paddr, PageSize::Size4K, flags).is_ok()); +assert!(pt.modify().map(vaddr, paddr, PageSize::Size4K, flags).is_ok()); assert_eq!(pt.query(vaddr), Ok((paddr, flags, PageSize::Size4K))); ``` diff --git a/page_table_multiarch/src/arch/aarch64.rs b/page_table_multiarch/src/arch/aarch64.rs index 81edcadb..06365291 100644 --- a/page_table_multiarch/src/arch/aarch64.rs +++ b/page_table_multiarch/src/arch/aarch64.rs @@ -5,7 +5,7 @@ use core::arch::asm; use memory_addr::VirtAddr; use page_table_entry::aarch64::A64PTE; -use crate::{PageTable64, PagingMetaData}; +use crate::{PageTable64, PageTable64Mut, PagingMetaData}; /// Metadata of AArch64 page tables. pub struct A64PagingMetaData; @@ -38,3 +38,5 @@ impl PagingMetaData for A64PagingMetaData { /// AArch64 VMSAv8-64 translation table. pub type A64PageTable = PageTable64; +/// Mutable reference to an AArch64 VMSAv8-64 translation table. +pub type A64PageTableMut<'a, H> = PageTable64Mut<'a, A64PagingMetaData, A64PTE, H>; diff --git a/page_table_multiarch/src/arch/loongarch64.rs b/page_table_multiarch/src/arch/loongarch64.rs index 69081fed..b3b66702 100644 --- a/page_table_multiarch/src/arch/loongarch64.rs +++ b/page_table_multiarch/src/arch/loongarch64.rs @@ -5,7 +5,7 @@ use core::arch::asm; use memory_addr::VirtAddr; use page_table_entry::loongarch64::LA64PTE; -use crate::{PageTable64, PagingMetaData}; +use crate::{PageTable64, PageTable64Mut, PagingMetaData}; /// Metadata of LoongArch64 page tables. #[derive(Copy, Clone, Debug)] @@ -86,3 +86,5 @@ impl PagingMetaData for LA64MetaData { /// /// using page table dir3, dir2, dir1 and pt, ignore dir4 pub type LA64PageTable = PageTable64; +/// Mutable reference to a loongarch64 page table. +pub type LA64PageTableMut<'a, H> = PageTable64Mut<'a, LA64MetaData, LA64PTE, H>; diff --git a/page_table_multiarch/src/arch/riscv.rs b/page_table_multiarch/src/arch/riscv.rs index 3cf4509a..930f3748 100644 --- a/page_table_multiarch/src/arch/riscv.rs +++ b/page_table_multiarch/src/arch/riscv.rs @@ -3,7 +3,7 @@ use memory_addr::VirtAddr; use page_table_entry::riscv::Rv64PTE; -use crate::{PageTable64, PagingMetaData}; +use crate::{PageTable64, PageTable64Mut, PagingMetaData}; /// A virtual address that can be used in RISC-V Sv39 and Sv48 page tables. pub trait SvVirtAddr: memory_addr::MemoryAddr + Send + Sync { @@ -58,6 +58,10 @@ impl PagingMetaData for Sv48MetaData { /// Sv39: Page-Based 39-bit (3 levels) Virtual-Memory System. pub type Sv39PageTable = PageTable64, Rv64PTE, H>; +/// Mutable reference to a RISC-V Sv39 page table. +pub type Sv39PageTableMut<'a, H> = PageTable64Mut<'a, Sv39MetaData, Rv64PTE, H>; /// Sv48: Page-Based 48-bit (4 levels) Virtual-Memory System. pub type Sv48PageTable = PageTable64, Rv64PTE, H>; +/// Mutable reference to a RISC-V Sv48 page table. +pub type Sv48PageTableMut<'a, H> = PageTable64Mut<'a, Sv48MetaData, Rv64PTE, H>; diff --git a/page_table_multiarch/src/arch/x86_64.rs b/page_table_multiarch/src/arch/x86_64.rs index 4e800e29..577164ad 100644 --- a/page_table_multiarch/src/arch/x86_64.rs +++ b/page_table_multiarch/src/arch/x86_64.rs @@ -3,7 +3,7 @@ use memory_addr::VirtAddr; use page_table_entry::x86_64::X64PTE; -use crate::{PageTable64, PagingMetaData}; +use crate::{PageTable64, PageTable64Mut, PagingMetaData}; /// metadata of x86_64 page tables. pub struct X64PagingMetaData; @@ -28,3 +28,5 @@ impl PagingMetaData for X64PagingMetaData { /// x86_64 page table. pub type X64PageTable = PageTable64; +/// Mutable reference to an x86_64 page table. +pub type X64PageTableMut<'a, H> = PageTable64Mut<'a, X64PagingMetaData, X64PTE, H>; diff --git a/page_table_multiarch/src/bits64.rs b/page_table_multiarch/src/bits64.rs index 35a0bdc0..3d38a348 100644 --- a/page_table_multiarch/src/bits64.rs +++ b/page_table_multiarch/src/bits64.rs @@ -1,10 +1,10 @@ -use core::marker::PhantomData; +use core::{marker::PhantomData, ops::Deref}; +use arrayvec::ArrayVec; use memory_addr::{MemoryAddr, PAGE_SIZE_4K, PhysAddr}; use crate::{ GenericPTE, MappingFlags, PageSize, PagingError, PagingHandler, PagingMetaData, PagingResult, - TlbFlush, TlbFlushAll, }; const ENTRY_COUNT: usize = 512; @@ -55,6 +55,246 @@ impl PageTable64 PagingResult<(PhysAddr, MappingFlags, PageSize)> { + let (entry, size) = self.get_entry(vaddr)?; + if !entry.is_present() { + return Err(PagingError::NotMapped); + } + let off = size.align_offset(vaddr.into()); + Ok((entry.paddr().add(off), entry.flags(), size)) + } + + /// Gets a mutable reference to modify the page table. + /// + /// The TLB will be flushed automatically when the mutable reference is + /// dropped. + pub fn modify(&mut self) -> PageTable64Mut<'_, M, PTE, H> { + PageTable64Mut::new(self) + } +} + +// Private implements. +impl PageTable64 { + fn alloc_table() -> PagingResult { + if let Some(paddr) = H::alloc_frame() { + let ptr = H::phys_to_virt(paddr).as_mut_ptr(); + unsafe { core::ptr::write_bytes(ptr, 0, PAGE_SIZE_4K) }; + Ok(paddr) + } else { + Err(PagingError::NoMemory) + } + } + + fn table_of<'a>(&self, paddr: PhysAddr) -> &'a [PTE] { + let ptr = H::phys_to_virt(paddr).as_ptr() as _; + unsafe { core::slice::from_raw_parts(ptr, ENTRY_COUNT) } + } + + fn next_table<'a>(&self, entry: &PTE) -> PagingResult<&'a [PTE]> { + if entry.paddr().as_usize() == 0 { + Err(PagingError::NotMapped) + } else if entry.is_huge() { + Err(PagingError::MappedToHugePage) + } else { + Ok(self.table_of(entry.paddr())) + } + } + + fn get_entry(&self, vaddr: M::VirtAddr) -> PagingResult<(&PTE, PageSize)> { + let vaddr: usize = vaddr.into(); + let p3 = if M::LEVELS == 3 { + self.table_of(self.root_paddr()) + } else if M::LEVELS == 4 { + let p4 = self.table_of(self.root_paddr()); + let p4e = &p4[p4_index(vaddr)]; + self.next_table(p4e)? + } else { + unreachable!() + }; + let p3e = &p3[p3_index(vaddr)]; + if p3e.is_huge() { + return Ok((p3e, PageSize::Size1G)); + } + + let p2 = self.next_table(p3e)?; + let p2e = &p2[p2_index(vaddr)]; + if p2e.is_huge() { + return Ok((p2e, PageSize::Size2M)); + } + + let p1 = self.next_table(p2e)?; + let p1e = &p1[p1_index(vaddr)]; + Ok((p1e, PageSize::Size4K)) + } + + fn dealloc_tree(&self, table_paddr: PhysAddr, level: usize) { + // don't free the entries in last level, they are not array. + if level < M::LEVELS - 1 { + for entry in self.table_of(table_paddr) { + if self.next_table(entry).is_ok() { + self.dealloc_tree(entry.paddr(), level + 1); + } + } + } + H::dealloc_frame(table_paddr); + } +} + +impl Drop for PageTable64 { + fn drop(&mut self) { + let root = self.table_of(self.root_paddr); + #[allow(unused_variables)] + for (i, entry) in root.iter().enumerate() { + #[cfg(feature = "copy-from")] + if self.borrowed_entries.get(i) { + continue; + } + if self.next_table(entry).is_ok() { + self.dealloc_tree(entry.paddr(), 1); + } + } + H::dealloc_frame(self.root_paddr()); + } +} + +// TODO: tune threshold +const SMALL_FLUSH_THRESHOLD: usize = 16; + +enum ToFlush { + None, + Addresses(ArrayVec), + Full, +} + +/// A mutable reference to a [`PageTable64`], allowing modifications and +/// flushing the TLB on drop. +pub struct PageTable64Mut<'a, M: PagingMetaData, PTE: GenericPTE, H: PagingHandler> { + inner: &'a mut PageTable64, + flush: ToFlush, +} + +impl Deref for PageTable64Mut<'_, M, PTE, H> { + type Target = PageTable64; + + fn deref(&self) -> &Self::Target { + self.inner + } +} + +impl<'a, M: PagingMetaData, PTE: GenericPTE, H: PagingHandler> PageTable64Mut<'a, M, PTE, H> { + fn new(inner: &'a mut PageTable64) -> Self { + Self { + inner, + flush: ToFlush::None, + } + } + + fn flush(&mut self, vaddr: M::VirtAddr) { + match self.flush { + ToFlush::None => { + let mut addresses = ArrayVec::new(); + addresses.push(vaddr); + self.flush = ToFlush::Addresses(addresses); + } + ToFlush::Addresses(ref mut addrs) => { + if addrs.try_push(vaddr).is_err() { + self.flush = ToFlush::Full; + } + } + ToFlush::Full => {} + } + } + + fn table_of_mut(&mut self, paddr: PhysAddr) -> &'a mut [PTE] { + let ptr = H::phys_to_virt(paddr).as_mut_ptr() as _; + unsafe { core::slice::from_raw_parts_mut(ptr, ENTRY_COUNT) } + } + + fn next_table_mut(&mut self, entry: &PTE) -> PagingResult<&'a mut [PTE]> { + if entry.paddr().as_usize() == 0 { + Err(PagingError::NotMapped) + } else if entry.is_huge() { + Err(PagingError::MappedToHugePage) + } else { + Ok(self.table_of_mut(entry.paddr())) + } + } + + fn next_table_mut_or_create(&mut self, entry: &mut PTE) -> PagingResult<&'a mut [PTE]> { + if entry.is_unused() { + let paddr = PageTable64::::alloc_table()?; + *entry = GenericPTE::new_table(paddr); + Ok(self.table_of_mut(paddr)) + } else { + self.next_table_mut(entry) + } + } + + fn get_entry_mut(&mut self, vaddr: M::VirtAddr) -> PagingResult<(&mut PTE, PageSize)> { + let vaddr: usize = vaddr.into(); + let p3 = if M::LEVELS == 3 { + self.table_of_mut(self.root_paddr()) + } else if M::LEVELS == 4 { + let p4 = self.table_of_mut(self.root_paddr()); + let p4e = &mut p4[p4_index(vaddr)]; + self.next_table_mut(p4e)? + } else { + unreachable!() + }; + let p3e = &mut p3[p3_index(vaddr)]; + if p3e.is_huge() { + return Ok((p3e, PageSize::Size1G)); + } + + let p2 = self.next_table_mut(p3e)?; + let p2e = &mut p2[p2_index(vaddr)]; + if p2e.is_huge() { + return Ok((p2e, PageSize::Size2M)); + } + + let p1 = self.next_table_mut(p2e)?; + let p1e = &mut p1[p1_index(vaddr)]; + Ok((p1e, PageSize::Size4K)) + } + + fn get_entry_mut_or_create( + &mut self, + vaddr: M::VirtAddr, + page_size: PageSize, + ) -> PagingResult<&mut PTE> { + let vaddr: usize = vaddr.into(); + let p3 = if M::LEVELS == 3 { + self.table_of_mut(self.root_paddr()) + } else if M::LEVELS == 4 { + let p4 = self.table_of_mut(self.root_paddr()); + let p4e = &mut p4[p4_index(vaddr)]; + self.next_table_mut_or_create(p4e)? + } else { + unreachable!() + }; + let p3e = &mut p3[p3_index(vaddr)]; + if page_size == PageSize::Size1G { + return Ok(p3e); + } + + let p2 = self.next_table_mut_or_create(p3e)?; + let p2e = &mut p2[p2_index(vaddr)]; + if page_size == PageSize::Size2M { + return Ok(p2e); + } + + let p1 = self.next_table_mut_or_create(p2e)?; + let p1e = &mut p1[p1_index(vaddr)]; + Ok(p1e) + } + /// Maps a virtual page to a physical frame with the given `page_size` /// and mapping `flags`. /// @@ -70,7 +310,7 @@ impl PageTable64 PagingResult> { + ) -> PagingResult { // `vaddr` does not need to be page-aligned here; `get_entry_mut_or_create` // internally maps `vaddr` to its corresponding page table entry (PTE). let entry = self.get_entry_mut_or_create(vaddr, page_size)?; @@ -78,26 +318,8 @@ impl PageTable64 PagingResult<(PageSize, TlbFlush)> { - let (entry, size) = self.get_entry_mut(vaddr)?; - entry.set_paddr(paddr); - entry.set_flags(flags, size.is_huge()); - Ok((size, TlbFlush::new(vaddr))) + self.flush(vaddr); + Ok(()) } /// Remap the mapping starts with `vaddr`, updates both the physical address @@ -116,7 +338,7 @@ impl PageTable64 PageTable64 PagingResult<(PageSize, TlbFlush)> { + pub fn protect(&mut self, vaddr: M::VirtAddr, flags: MappingFlags) -> PagingResult { let (entry, size) = self.get_entry_mut(vaddr)?; if !entry.is_present() { return Err(PagingError::NotMapped); } entry.set_flags(flags, size.is_huge()); - Ok((size, TlbFlush::new(vaddr))) + self.flush(vaddr); + Ok(size) } /// Unmaps the mapping starts with `vaddr`. /// /// Returns [`Err(PagingError::NotMapped)`](PagingError::NotMapped) if the /// mapping is not present. - pub fn unmap(&mut self, vaddr: M::VirtAddr) -> PagingResult<(PhysAddr, PageSize, TlbFlush)> { + pub fn unmap( + &mut self, + vaddr: M::VirtAddr, + ) -> PagingResult<(PhysAddr, MappingFlags, PageSize)> { let (entry, size) = self.get_entry_mut(vaddr)?; if !entry.is_present() { entry.clear(); return Err(PagingError::NotMapped); } let paddr = entry.paddr(); + let flags = entry.flags(); entry.clear(); - Ok((paddr, size, TlbFlush::new(vaddr))) - } - - /// Queries the result of the mapping starts with `vaddr`. - /// - /// Returns the physical address of the target frame, mapping flags, and - /// the page size. - /// - /// Returns [`Err(PagingError::NotMapped)`](PagingError::NotMapped) if the - /// mapping is not present. - pub fn query(&self, vaddr: M::VirtAddr) -> PagingResult<(PhysAddr, MappingFlags, PageSize)> { - let (entry, size) = self.get_entry(vaddr)?; - if !entry.is_present() { - return Err(PagingError::NotMapped); - } - let off = size.align_offset(vaddr.into()); - Ok((entry.paddr().add(off), entry.flags(), size)) + self.flush(vaddr); + Ok((paddr, flags, size)) } /// Maps a contiguous virtual memory region to a contiguous physical memory @@ -175,14 +383,12 @@ impl PageTable64 PageTable64 PagingResult> { + ) -> PagingResult { let mut vaddr_usize: usize = vaddr.into(); let mut size = size; if !PageSize::Size4K.is_aligned(vaddr_usize) || !PageSize::Size4K.is_aligned(size) { @@ -225,37 +430,22 @@ impl PageTable64 {paddr:#x?}, {e:?}") })?; - if flush_tlb_by_page { - M::flush_tlb(Some(vaddr)); - } - if flush_tlb_by_page { - tlb.flush(); - } else { - tlb.ignore(); - } vaddr_usize += page_size as usize; size -= page_size as usize; } - Ok(TlbFlushAll::new()) + Ok(()) } /// Unmaps a contiguous virtual memory region. /// - /// The region must be mapped before using [`PageTable64::map_region`], or - /// unexpected behaviors may occur. It can deal with huge pages automatically. - /// - /// When `flush_tlb_by_page` is true, it will flush the TLB immediately after - /// mapping each page. Otherwise, the TLB flush should by handled by the caller. - pub fn unmap_region( - &mut self, - vaddr: M::VirtAddr, - size: usize, - flush_tlb_by_page: bool, - ) -> PagingResult> { + /// The region must be mapped before using [`Self::map_region`], or + /// unexpected behaviors may occur. It can deal with huge pages + /// automatically. + pub fn unmap_region(&mut self, vaddr: M::VirtAddr, size: usize) -> PagingResult { let mut vaddr_usize: usize = vaddr.into(); let mut size = size; trace!( @@ -266,37 +456,29 @@ impl PageTable64 0 { let vaddr = vaddr_usize.into(); - let (_, page_size, tlb) = self + let (_, _, page_size) = self .unmap(vaddr) .inspect_err(|e| error!("failed to unmap page: {vaddr_usize:#x?}, {e:?}"))?; - if flush_tlb_by_page { - tlb.flush(); - } else { - tlb.ignore(); - } assert!(page_size.is_aligned(vaddr_usize)); assert!(page_size as usize <= size); vaddr_usize += page_size as usize; size -= page_size as usize; } - Ok(TlbFlushAll::new()) + Ok(()) } /// Updates mapping flags of a contiguous virtual memory region. /// - /// The region must be mapped before using [`PageTable64::map_region`], or - /// unexpected behaviors may occur. It can deal with huge pages automatically. - /// - /// When `flush_tlb_by_page` is true, it will flush the TLB immediately after - /// mapping each page. Otherwise, the TLB flush should by handled by the caller. + /// The region must be mapped before using [`Self::map_region`], or + /// unexpected behaviors may occur. It can deal with huge pages + /// automatically. pub fn protect_region( &mut self, vaddr: M::VirtAddr, size: usize, flags: MappingFlags, - flush_tlb_by_page: bool, - ) -> PagingResult> { + ) -> PagingResult { let mut vaddr_usize: usize = vaddr.into(); let mut size = size; trace!( @@ -308,52 +490,30 @@ impl PageTable64 0 { let vaddr = vaddr_usize.into(); - let (page_size, tlb) = self - .protect(vaddr, flags) - .inspect_err(|e| error!("failed to protect page: {vaddr_usize:#x?}, {e:?}"))?; - if flush_tlb_by_page { - tlb.flush(); - } else { - tlb.ignore(); - } + let page_size = match self.protect(vaddr, flags) { + Ok(page_size) => { + assert!(page_size.is_aligned(vaddr_usize)); + assert!(page_size as usize <= size); + + page_size + } + Err(PagingError::NotMapped) => PageSize::Size4K, + Err(e) => { + error!("failed to protect page: {vaddr_usize:#x?}, {e:?}"); + return Err(e); + } + }; - assert!(page_size.is_aligned(vaddr_usize)); - assert!(page_size as usize <= size); vaddr_usize += page_size as usize; size -= page_size as usize; } - Ok(TlbFlushAll::new()) - } - - /// Walk the page table recursively. - /// - /// When reaching a page table entry, call `pre_func` and `post_func` on the - /// entry if they are provided. The max number of enumerations in one table - /// is limited by `limit`. `pre_func` and `post_func` are called before and - /// after recursively walking the page table. - /// - /// The arguments of `*_func` are: - /// - Current level (starts with `0`): `usize` - /// - The index of the entry in the current-level table: `usize` - /// - The virtual address that is mapped to the entry: `M::VirtAddr` - /// - The reference of the entry: [`&PTE`](GenericPTE) - pub fn walk(&self, limit: usize, pre_func: Option<&F>, post_func: Option<&F>) -> PagingResult - where - F: Fn(usize, usize, M::VirtAddr, &PTE), - { - self.walk_recursive( - self.table_of(self.root_paddr()), - 0, - 0.into(), - limit, - pre_func, - post_func, - ) + Ok(()) } - /// Copy entries from another page table within the given virtual memory range. + /// Copy entries from another page table within the given virtual memory + /// range. #[cfg(feature = "copy-from")] - pub fn copy_from(&mut self, other: &Self, start: M::VirtAddr, size: usize) { + pub fn copy_from(&mut self, other: &PageTable64, start: M::VirtAddr, size: usize) { if size == 0 { return; } @@ -372,215 +532,33 @@ impl PageTable64 PageTable64 { - fn alloc_table() -> PagingResult { - if let Some(paddr) = H::alloc_frame() { - let ptr = H::phys_to_virt(paddr).as_mut_ptr(); - unsafe { core::ptr::write_bytes(ptr, 0, PAGE_SIZE_4K) }; - Ok(paddr) - } else { - Err(PagingError::NoMemory) - } - } - - fn table_of<'a>(&self, paddr: PhysAddr) -> &'a [PTE] { - let ptr = H::phys_to_virt(paddr).as_ptr() as _; - unsafe { core::slice::from_raw_parts(ptr, ENTRY_COUNT) } - } - - fn table_of_mut<'a>(&mut self, paddr: PhysAddr) -> &'a mut [PTE] { - let ptr = H::phys_to_virt(paddr).as_mut_ptr() as _; - unsafe { core::slice::from_raw_parts_mut(ptr, ENTRY_COUNT) } - } - - fn next_table<'a>(&self, entry: &PTE) -> PagingResult<&'a [PTE]> { - if entry.paddr().as_usize() == 0 { - Err(PagingError::NotMapped) - } else if entry.is_huge() { - Err(PagingError::MappedToHugePage) - } else { - Ok(self.table_of(entry.paddr())) - } - } - - fn next_table_mut<'a>(&mut self, entry: &PTE) -> PagingResult<&'a mut [PTE]> { - if entry.paddr().as_usize() == 0 { - Err(PagingError::NotMapped) - } else if entry.is_huge() { - Err(PagingError::MappedToHugePage) - } else { - Ok(self.table_of_mut(entry.paddr())) - } - } - fn next_table_mut_or_create<'a>(&mut self, entry: &mut PTE) -> PagingResult<&'a mut [PTE]> { - if entry.is_unused() { - let paddr = Self::alloc_table()?; - *entry = GenericPTE::new_table(paddr); - Ok(self.table_of_mut(paddr)) - } else { - self.next_table_mut(entry) - } - } - - fn get_entry(&self, vaddr: M::VirtAddr) -> PagingResult<(&PTE, PageSize)> { - let vaddr: usize = vaddr.into(); - let p3 = if M::LEVELS == 3 { - self.table_of(self.root_paddr()) - } else if M::LEVELS == 4 { - let p4 = self.table_of(self.root_paddr()); - let p4e = &p4[p4_index(vaddr)]; - self.next_table(p4e)? - } else { - unreachable!() - }; - let p3e = &p3[p3_index(vaddr)]; - if p3e.is_huge() { - return Ok((p3e, PageSize::Size1G)); - } - - let p2 = self.next_table(p3e)?; - let p2e = &p2[p2_index(vaddr)]; - if p2e.is_huge() { - return Ok((p2e, PageSize::Size2M)); - } - - let p1 = self.next_table(p2e)?; - let p1e = &p1[p1_index(vaddr)]; - Ok((p1e, PageSize::Size4K)) - } - - fn get_entry_mut(&mut self, vaddr: M::VirtAddr) -> PagingResult<(&mut PTE, PageSize)> { - let vaddr: usize = vaddr.into(); - let p3 = if M::LEVELS == 3 { - self.table_of_mut(self.root_paddr()) - } else if M::LEVELS == 4 { - let p4 = self.table_of_mut(self.root_paddr()); - let p4e = &mut p4[p4_index(vaddr)]; - self.next_table_mut(p4e)? - } else { - unreachable!() - }; - let p3e = &mut p3[p3_index(vaddr)]; - if p3e.is_huge() { - return Ok((p3e, PageSize::Size1G)); - } - - let p2 = self.next_table_mut(p3e)?; - let p2e = &mut p2[p2_index(vaddr)]; - if p2e.is_huge() { - return Ok((p2e, PageSize::Size2M)); - } - - let p1 = self.next_table_mut(p2e)?; - let p1e = &mut p1[p1_index(vaddr)]; - Ok((p1e, PageSize::Size4K)) - } - - fn get_entry_mut_or_create( - &mut self, - vaddr: M::VirtAddr, - page_size: PageSize, - ) -> PagingResult<&mut PTE> { - let vaddr: usize = vaddr.into(); - let p3 = if M::LEVELS == 3 { - self.table_of_mut(self.root_paddr()) - } else if M::LEVELS == 4 { - let p4 = self.table_of_mut(self.root_paddr()); - let p4e = &mut p4[p4_index(vaddr)]; - self.next_table_mut_or_create(p4e)? - } else { - unreachable!() - }; - let p3e = &mut p3[p3_index(vaddr)]; - if page_size == PageSize::Size1G { - return Ok(p3e); - } - - let p2 = self.next_table_mut_or_create(p3e)?; - let p2e = &mut p2[p2_index(vaddr)]; - if page_size == PageSize::Size2M { - return Ok(p2e); - } - - let p1 = self.next_table_mut_or_create(p2e)?; - let p1e = &mut p1[p1_index(vaddr)]; - Ok(p1e) - } - - fn walk_recursive( - &self, - table: &[PTE], - level: usize, - start_vaddr: M::VirtAddr, - limit: usize, - pre_func: Option<&F>, - post_func: Option<&F>, - ) -> PagingResult - where - F: Fn(usize, usize, M::VirtAddr, &PTE), - { - let start_vaddr_usize: usize = start_vaddr.into(); - let mut n = 0; - for (i, entry) in table.iter().enumerate() { - let vaddr_usize = start_vaddr_usize + (i << (12 + (M::LEVELS - 1 - level) * 9)); - let vaddr = vaddr_usize.into(); - - if entry.is_present() { - if let Some(func) = pre_func { - func(level, i, vaddr, entry); - } - if level < M::LEVELS - 1 && !entry.is_huge() { - let table_entry = self.next_table(entry)?; - self.walk_recursive(table_entry, level + 1, vaddr, limit, pre_func, post_func)?; - } - if let Some(func) = post_func { - func(level, i, vaddr, entry); - } - n += 1; - if n >= limit { - break; + /// Finishes modifications and flushes the TLB if necessary. + pub fn finish(&mut self) { + #[cfg(not(docsrs))] + match &self.flush { + ToFlush::None => {} + ToFlush::Addresses(addrs) => { + for vaddr in addrs.iter() { + M::flush_tlb(Some(*vaddr)); } } - } - Ok(()) - } - - fn dealloc_tree(&self, table_paddr: PhysAddr, level: usize) { - // don't free the entries in last level, they are not array. - if level < M::LEVELS - 1 { - for entry in self.table_of(table_paddr) { - if self.next_table(entry).is_ok() { - self.dealloc_tree(entry.paddr(), level + 1); - } + ToFlush::Full => { + M::flush_tlb(None); } } - H::dealloc_frame(table_paddr); + self.flush = ToFlush::None; } } -impl Drop for PageTable64 { +impl Drop for PageTable64Mut<'_, M, PTE, H> { fn drop(&mut self) { - let root = self.table_of(self.root_paddr); - #[allow(unused_variables)] - for (i, entry) in root.iter().enumerate() { - #[cfg(feature = "copy-from")] - if self.borrowed_entries.get(i) { - continue; - } - if self.next_table(entry).is_ok() { - self.dealloc_tree(entry.paddr(), 1); - } - } - H::dealloc_frame(self.root_paddr()); + self.finish(); } } diff --git a/page_table_multiarch/src/lib.rs b/page_table_multiarch/src/lib.rs index e3063f0e..8b2accc7 100644 --- a/page_table_multiarch/src/lib.rs +++ b/page_table_multiarch/src/lib.rs @@ -8,16 +8,17 @@ extern crate log; mod arch; mod bits64; -use core::{fmt::Debug, marker::PhantomData}; +use core::fmt::Debug; use memory_addr::{MemoryAddr, PhysAddr, VirtAddr}; - -pub use self::arch::*; -pub use self::bits64::PageTable64; - #[doc(no_inline)] pub use page_table_entry::{GenericPTE, MappingFlags}; +pub use self::{ + arch::*, + bits64::{PageTable64, PageTable64Mut}, +}; + /// The error type for page table operation failures. #[derive(Debug, PartialEq, Clone, Copy)] pub enum PagingError { @@ -62,8 +63,8 @@ pub trait PagingMetaData: Sync + Send { /// The virtual address to be translated in this page table. /// - /// This associated type allows more flexible use of page tables structs like [`PageTable64`], - /// for example, to implement EPTs. + /// This associated type allows more flexible use of page tables structs + /// like [`PageTable64`], for example, to implement EPTs. type VirtAddr: MemoryAddr; // (^)it can be converted from/to usize and it's trivially copyable @@ -83,8 +84,8 @@ pub trait PagingMetaData: Sync + Send { /// Flushes the TLB. /// - /// If `vaddr` is [`None`], flushes the entire TLB. Otherwise, flushes the TLB - /// entry at the given virtual address. + /// If `vaddr` is [`None`], flushes the entire TLB. Otherwise, flushes the + /// TLB entry at the given virtual address. fn flush_tlb(vaddr: Option); } @@ -97,7 +98,8 @@ pub trait PagingHandler: Sized { fn dealloc_frame(paddr: PhysAddr); /// Returns a virtual address that maps to the given physical address. /// - /// Used to access the physical memory directly in page table implementation. + /// Used to access the physical memory directly in page table + /// implementation. fn phys_to_virt(paddr: PhysAddr) -> VirtAddr; } @@ -136,47 +138,3 @@ impl From for usize { size as usize } } - -/// This type indicates the mapping of a virtual address has been changed. -/// -/// The caller can call [`TlbFlush::flush`] to flush TLB entries related to -/// the given virtual address, or call [`TlbFlush::ignore`] if it knowns the -/// TLB will be flushed later. -#[must_use] -pub struct TlbFlush(M::VirtAddr, PhantomData); - -impl TlbFlush { - pub(crate) const fn new(vaddr: M::VirtAddr) -> Self { - Self(vaddr, PhantomData) - } - - /// Don't flush the TLB and silence the “must be used” warning. - pub fn ignore(self) {} - - /// Flush the the TLB by the given virtual address to ensure the mapping - /// changes take effect. - pub fn flush(self) { - M::flush_tlb(Some(self.0)) - } -} - -/// This type indicates the page table mappings have been changed. -/// -/// The caller can call [`TlbFlushAll::flush_all`] to flush the entire TLB, or call -/// [`TlbFlushAll::ignore`] if it knowns the TLB will be flushed later. -#[must_use] -pub struct TlbFlushAll(PhantomData); - -impl TlbFlushAll { - pub(crate) const fn new() -> Self { - Self(PhantomData) - } - - /// Don't flush the TLB and silence the “must be used” warning. - pub fn ignore(self) {} - - /// Flush the entire TLB. - pub fn flush_all(self) { - M::flush_tlb(None) - } -} diff --git a/page_table_multiarch/tests/alloc_tests.rs b/page_table_multiarch/tests/alloc_tests.rs index 2ce9550e..5e84976b 100644 --- a/page_table_multiarch/tests/alloc_tests.rs +++ b/page_table_multiarch/tests/alloc_tests.rs @@ -55,7 +55,9 @@ fn run_test_for, PTE: GenericPTE>() -> Pa let mut table = PageTable64::>::try_new().unwrap(); let mut pages = HashSet::new(); let mut rng = SmallRng::seed_from_u64(1234); + for _ in 0..2048 { + let mut table = table.modify(); if rng.random_ratio(3, 4) || pages.is_empty() { // insert a mapping let addr = loop { @@ -64,18 +66,16 @@ fn run_test_for, PTE: GenericPTE>() -> Pa break addr; } }; - table - .map( - VirtAddr::from_usize(addr as usize), - PhysAddr::from_usize((rng.random::() & vaddr_mask) as usize), - PageSize::Size4K, - MappingFlags::READ | MappingFlags::WRITE, - )? - .ignore(); + table.map( + VirtAddr::from_usize(addr as usize), + PhysAddr::from_usize((rng.random::() & vaddr_mask) as usize), + PageSize::Size4K, + MappingFlags::READ | MappingFlags::WRITE, + )?; } else { // remove a mapping let addr = *pages.iter().next().unwrap(); - table.unmap(VirtAddr::from_usize(addr as usize))?.2.ignore(); + table.unmap(VirtAddr::from_usize(addr as usize))?; pages.remove(&addr); } } From 736931ee5128b13e8d9b7df3eb78e831b4e061e8 Mon Sep 17 00:00:00 2001 From: li041 Date: Thu, 25 Dec 2025 21:38:59 +0800 Subject: [PATCH 5/6] feat(smp): support cross-core TLB flush --- Cargo.lock | 12 ++++ page_table_multiarch/Cargo.toml | 2 + page_table_multiarch/src/arch/aarch64.rs | 11 +++- page_table_multiarch/src/arch/loongarch64.rs | 58 ++++++++++++-------- page_table_multiarch/src/arch/riscv.rs | 19 +++++-- page_table_multiarch/src/arch/x86_64.rs | 24 ++++++-- page_table_multiarch/src/lib.rs | 15 +++++ 7 files changed, 106 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbd3ceb6..4c8f766c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,17 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" +[[package]] +name = "crate_interface" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70272a03a2cef15589bac05d3d15c023752f5f8f2da8be977d983a9d9e6250fb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -114,6 +125,7 @@ dependencies = [ "arrayvec", "axerrno 0.1.2", "bitmaps", + "crate_interface", "log", "memory_addr", "page_table_entry", diff --git a/page_table_multiarch/Cargo.toml b/page_table_multiarch/Cargo.toml index 1eee6350..04c63d29 100644 --- a/page_table_multiarch/Cargo.toml +++ b/page_table_multiarch/Cargo.toml @@ -16,11 +16,13 @@ rust-version.workspace = true default = [] axerrno = ["dep:axerrno"] copy-from = ["dep:bitmaps"] +smp = [] [dependencies] axerrno = { version = "0.1", optional = true } arrayvec = { version = "0.7", default-features = false } bitmaps = { version = "3.2", default-features = false, optional = true } +crate_interface = "0.1.4" log = "0.4" memory_addr.workspace = true page_table_entry.workspace = true diff --git a/page_table_multiarch/src/arch/aarch64.rs b/page_table_multiarch/src/arch/aarch64.rs index 06365291..28f91ac3 100644 --- a/page_table_multiarch/src/arch/aarch64.rs +++ b/page_table_multiarch/src/arch/aarch64.rs @@ -27,10 +27,17 @@ impl PagingMetaData for A64PagingMetaData { if let Some(vaddr) = vaddr { // TLB Invalidate by VA, All ASID, EL1, Inner Shareable const VA_MASK: usize = (1 << 44) - 1; // VA[55:12] => bits[43:0] - asm!("tlbi vaae1is, {}; dsb sy; isb", in(reg) ((vaddr.as_usize() >> 12) & VA_MASK)) + let va = (vaddr.as_usize() >> 12) & VA_MASK; + #[cfg(feature = "smp")] + asm!("tlbi vaae1is, {}; dsb sy; isb", in(reg) va); + #[cfg(not(feature = "smp"))] + asm!("tlbi vaae1, {}; dsb sy; isb", in(reg) va); } else { + #[cfg(feature = "smp")] // TLB Invalidate by VMID, All at stage 1, EL1 - asm!("tlbi vmalle1; dsb sy; isb") + asm!("tlbi vmalle1is; dsb sy; isb"); + #[cfg(not(feature = "smp"))] + asm!("tlbi vmalle1; dsb sy; isb"); } } } diff --git a/page_table_multiarch/src/arch/loongarch64.rs b/page_table_multiarch/src/arch/loongarch64.rs index b3b66702..d591457a 100644 --- a/page_table_multiarch/src/arch/loongarch64.rs +++ b/page_table_multiarch/src/arch/loongarch64.rs @@ -7,6 +7,34 @@ use page_table_entry::loongarch64::LA64PTE; use crate::{PageTable64, PageTable64Mut, PagingMetaData}; +#[inline] +fn local_flush_tlb(vaddr: Option) { + unsafe { + if let Some(vaddr) = vaddr { + // + // + // Only after all previous load/store access operations are completely + // executed, the DBAR 0 instruction can be executed; and only after the + // execution of DBAR 0 is completed, all subsequent load/store access + // operations can be executed. + // + // + // + // formats: invtlb op, asid, addr + // + // op 0x5: Clear all page table entries with G=0 and ASID equal to the + // register specified ASID, and VA equal to the register specified VA. + // + // When the operation indicated by op does not require an ASID, the + // general register rj should be set to r0. + asm!("dbar 0; invtlb 0x05, $r0, {reg}", reg = in(reg) vaddr.as_usize()); + } else { + // op 0x0: Clear all page table entries + asm!("dbar 0; invtlb 0x00, $r0, $r0"); + } + } +} + /// Metadata of LoongArch64 page tables. #[derive(Copy, Clone, Debug)] pub struct LA64MetaData; @@ -51,30 +79,14 @@ impl PagingMetaData for LA64MetaData { #[inline] fn flush_tlb(vaddr: Option) { - unsafe { - if let Some(vaddr) = vaddr { - // - // - // Only after all previous load/store access operations are completely - // executed, the DBAR 0 instruction can be executed; and only after the - // execution of DBAR 0 is completed, all subsequent load/store access - // operations can be executed. - // - // - // - // formats: invtlb op, asid, addr - // - // op 0x5: Clear all page table entries with G=0 and ASID equal to the - // register specified ASID, and VA equal to the register specified VA. - // - // When the operation indicated by op does not require an ASID, the - // general register rj should be set to r0. - asm!("dbar 0; invtlb 0x05, $r0, {reg}", reg = in(reg) vaddr.as_usize()); - } else { - // op 0x0: Clear all page table entries - asm!("dbar 0; invtlb 0x00, $r0, $r0"); - } + #[cfg(feature = "smp")] + { + use crate::__TlbFlushIf_mod; + use crate_interface::call_interface; + + call_interface!(TlbFlushIf::flush_all(vaddr)); } + local_flush_tlb(vaddr); } } diff --git a/page_table_multiarch/src/arch/riscv.rs b/page_table_multiarch/src/arch/riscv.rs index 930f3748..0a4b69fc 100644 --- a/page_table_multiarch/src/arch/riscv.rs +++ b/page_table_multiarch/src/arch/riscv.rs @@ -5,6 +5,14 @@ use page_table_entry::riscv::Rv64PTE; use crate::{PageTable64, PageTable64Mut, PagingMetaData}; +fn local_flush_tlb(vaddr: Option) { + if let Some(vaddr) = vaddr { + riscv::asm::sfence_vma(0, vaddr.as_usize()) + } else { + riscv::asm::sfence_vma_all(); + } +} + /// A virtual address that can be used in RISC-V Sv39 and Sv48 page tables. pub trait SvVirtAddr: memory_addr::MemoryAddr + Send + Sync { /// Flush the TLB. @@ -14,11 +22,14 @@ pub trait SvVirtAddr: memory_addr::MemoryAddr + Send + Sync { impl SvVirtAddr for VirtAddr { #[inline] fn flush_tlb(vaddr: Option) { - if let Some(vaddr) = vaddr { - riscv::asm::sfence_vma(0, vaddr.as_usize()) - } else { - riscv::asm::sfence_vma_all(); + #[cfg(feature = "smp")] + { + use crate::__TlbFlushIf_mod; + use crate_interface::call_interface; + + call_interface!(TlbFlushIf::flush_all(vaddr)); } + local_flush_tlb(vaddr); } } diff --git a/page_table_multiarch/src/arch/x86_64.rs b/page_table_multiarch/src/arch/x86_64.rs index 577164ad..7ab41565 100644 --- a/page_table_multiarch/src/arch/x86_64.rs +++ b/page_table_multiarch/src/arch/x86_64.rs @@ -5,6 +5,17 @@ use page_table_entry::x86_64::X64PTE; use crate::{PageTable64, PageTable64Mut, PagingMetaData}; +#[inline] +fn local_flush_tlb(vaddr: Option) { + unsafe { + if let Some(vaddr) = vaddr { + x86::tlb::flush(vaddr.into()); + } else { + x86::tlb::flush_all(); + } + } +} + /// metadata of x86_64 page tables. pub struct X64PagingMetaData; @@ -16,13 +27,14 @@ impl PagingMetaData for X64PagingMetaData { #[inline] fn flush_tlb(vaddr: Option) { - unsafe { - if let Some(vaddr) = vaddr { - x86::tlb::flush(vaddr.into()); - } else { - x86::tlb::flush_all(); - } + #[cfg(feature = "smp")] + { + use crate::__TlbFlushIf_mod; + use crate_interface::call_interface; + + call_interface!(TlbFlushIf::flush_all(vaddr)); } + local_flush_tlb(vaddr); } } diff --git a/page_table_multiarch/src/lib.rs b/page_table_multiarch/src/lib.rs index 8b2accc7..c186b142 100644 --- a/page_table_multiarch/src/lib.rs +++ b/page_table_multiarch/src/lib.rs @@ -138,3 +138,18 @@ impl From for usize { size as usize } } + +/// Interface for performing TLB flush operations on SMP systems. +/// +/// This trait is used by the page table subsystem to request +/// architecture- or platform-specific TLB invalidation. +#[cfg(feature = "smp")] +#[crate_interface::def_interface] +pub trait TlbFlushIf { + /// Flush TLB entries. + /// + /// If `vaddr` is `None`, flush all TLB entries. + /// If `Some(vaddr)` is provided, flush the TLB entry corresponding + /// to the given virtual address. + fn flush_all(vaddr: Option); +} From 3c0da97701f2d52dc07f40243327f97ee63c5b74 Mon Sep 17 00:00:00 2001 From: li041 Date: Sun, 28 Dec 2025 21:08:18 +0800 Subject: [PATCH 6/6] add necessary data sync barrier --- page_table_multiarch/src/arch/aarch64.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/page_table_multiarch/src/arch/aarch64.rs b/page_table_multiarch/src/arch/aarch64.rs index 28f91ac3..9524d8f9 100644 --- a/page_table_multiarch/src/arch/aarch64.rs +++ b/page_table_multiarch/src/arch/aarch64.rs @@ -29,15 +29,15 @@ impl PagingMetaData for A64PagingMetaData { const VA_MASK: usize = (1 << 44) - 1; // VA[55:12] => bits[43:0] let va = (vaddr.as_usize() >> 12) & VA_MASK; #[cfg(feature = "smp")] - asm!("tlbi vaae1is, {}; dsb sy; isb", in(reg) va); + asm!("dsb ishst; tlbi vaae1is, {}; dsb sy; isb", in(reg) va); #[cfg(not(feature = "smp"))] - asm!("tlbi vaae1, {}; dsb sy; isb", in(reg) va); + asm!("dsb nshst; tlbi vaae1, {}; dsb sy; isb", in(reg) va); } else { #[cfg(feature = "smp")] // TLB Invalidate by VMID, All at stage 1, EL1 - asm!("tlbi vmalle1is; dsb sy; isb"); + asm!("dsb ishst; tlbi vmalle1is; dsb sy; isb"); #[cfg(not(feature = "smp"))] - asm!("tlbi vmalle1; dsb sy; isb"); + asm!("dsb nshst; tlbi vmalle1; dsb sy; isb"); } } }