feat(smp): support cross-core TLB flush

Cargo.toml

@@ -17,4 +17,7 @@ rust-version = "1.85"
 
 [workspace.dependencies]
 memory_addr = "0.4"
-page_table_entry = { path = "page_table_entry", version = "0.5" }
+page_table_entry = "0.5"
+
+[patch.crates-io]
+page_table_entry = { path = "./page_table_entry" }

page_table_entry/src/arch/aarch64.rs

@@ -1,7 +1,8 @@
 //! AArch64 VMSAv8-64 translation table format descriptors.
 
-use aarch64_cpu::registers::MAIR_EL1;
 use core::fmt;
+
+use aarch64_cpu::registers::MAIR_EL1;
 use memory_addr::PhysAddr;
 
 use crate::{GenericPTE, MappingFlags};

page_table_entry/src/arch/loongarch64.rs

@@ -4,10 +4,12 @@
 //!
 //! <https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#section-multi-level-page-table-structure-supported-by-page-walking>
 
-use crate::{GenericPTE, MappingFlags};
 use core::fmt;
+
 use memory_addr::PhysAddr;
 
+use crate::{GenericPTE, MappingFlags};
+
 bitflags::bitflags! {
     /// Page-table entry flags.
     ///

page_table_entry/src/arch/riscv.rs

@@ -1,6 +1,7 @@
 //! RISC-V page table entries.
 
 use core::fmt;
+
 use memory_addr::PhysAddr;
 
 use crate::{GenericPTE, MappingFlags};

page_table_entry/src/arch/x86_64.rs

@@ -1,6 +1,7 @@
 //! x86 page table entries on 64-bit paging.
 
 use core::fmt;
+
 use memory_addr::PhysAddr;
 
 pub use x86_64::structures::paging::page_table::PageTableFlags as PTF;

page_table_entry/src/lib.rs

@@ -2,11 +2,11 @@
 #![cfg_attr(doc, feature(doc_cfg))]
 #![doc = include_str!("../README.md")]
 
-mod arch;
+use core::fmt;
 
-use core::fmt::{self, Debug};
 use memory_addr::PhysAddr;
 
+mod arch;
 pub use self::arch::*;
 
 bitflags::bitflags! {
@@ -29,16 +29,16 @@ bitflags::bitflags! {
     }
 }
 
-impl Debug for MappingFlags {
+impl fmt::Debug for MappingFlags {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        Debug::fmt(&self.0, f)
+        fmt::Debug::fmt(&self.0, f)
     }
 }
 
 /// A generic page table entry.
 ///
 /// All architecture-specific page table entry types implement this trait.
-pub trait GenericPTE: Debug + Clone + Copy + Sync + Send + Sized {
+pub trait GenericPTE: fmt::Debug + Clone + Copy + Sync + Send + Sized {
     /// Creates a page table entry point to a terminate page or block.
     fn new_page(paddr: PhysAddr, flags: MappingFlags, is_huge: bool) -> Self;
     /// Creates a page table entry point to a next level page table.

page_table_multiarch/Cargo.toml

@@ -16,10 +16,13 @@ rust-version.workspace = true
 default = []
 axerrno = ["dep:axerrno"]
 copy-from = ["dep:bitmaps"]
+smp = []
 
 [dependencies]
 axerrno = { version = "0.1", optional = true }
+arrayvec = { version = "0.7", default-features = false }
 bitmaps = { version = "3.2", default-features = false, optional = true }
+crate_interface = "0.1.4"
 log = "0.4"
 memory_addr.workspace = true
 page_table_entry.workspace = true

page_table_multiarch/README.md

@@ -66,6 +66,6 @@ let flags = MappingFlags::READ | MappingFlags::WRITE;
 let mut pt = X64PageTable::<PagingHandlerImpl>::try_new().unwrap();
 
 assert!(pt.root_paddr().is_aligned_4k());
-assert!(pt.map(vaddr, paddr, PageSize::Size4K, flags).is_ok());
+assert!(pt.modify().map(vaddr, paddr, PageSize::Size4K, flags).is_ok());
 assert_eq!(pt.query(vaddr), Ok((paddr, flags, PageSize::Size4K)));
 ```

page_table_multiarch/src/arch/aarch64.rs

@@ -1,9 +1,11 @@
 //! AArch64 specific page table structures.
 
 use core::arch::asm;
+
+use memory_addr::VirtAddr;
 use page_table_entry::aarch64::A64PTE;
 
-use crate::{PageTable64, PagingMetaData};
+use crate::{PageTable64, PageTable64Mut, PagingMetaData};
 
 /// Metadata of AArch64 page tables.
 pub struct A64PagingMetaData;
@@ -12,27 +14,36 @@ impl PagingMetaData for A64PagingMetaData {
     const LEVELS: usize = 4;
     const PA_MAX_BITS: usize = 48;
     const VA_MAX_BITS: usize = 48;
-    type VirtAddr = memory_addr::VirtAddr;
+    type VirtAddr = VirtAddr;
 
     fn vaddr_is_valid(vaddr: usize) -> bool {
         let top_bits = vaddr >> Self::VA_MAX_BITS;
         top_bits == 0 || top_bits == 0xffff
     }
 
     #[inline]
-    fn flush_tlb(vaddr: Option<memory_addr::VirtAddr>) {
+    fn flush_tlb(vaddr: Option<VirtAddr>) {
         unsafe {
             if let Some(vaddr) = vaddr {
                 // TLB Invalidate by VA, All ASID, EL1, Inner Shareable
                 const VA_MASK: usize = (1 << 44) - 1; // VA[55:12] => bits[43:0]
-                asm!("tlbi vaae1is, {}; dsb sy; isb", in(reg) ((vaddr.as_usize() >> 12) & VA_MASK))
+                let va = (vaddr.as_usize() >> 12) & VA_MASK;
+                #[cfg(feature = "smp")]
+                asm!("dsb ishst; tlbi vaae1is, {}; dsb sy; isb", in(reg) va);
+                #[cfg(not(feature = "smp"))]
+                asm!("dsb nshst; tlbi vaae1, {}; dsb sy; isb", in(reg) va);
             } else {
+                #[cfg(feature = "smp")]
                 // TLB Invalidate by VMID, All at stage 1, EL1
-                asm!("tlbi vmalle1; dsb sy; isb")
+                asm!("dsb ishst; tlbi vmalle1is; dsb sy; isb");
+                #[cfg(not(feature = "smp"))]
+                asm!("dsb nshst; tlbi vmalle1; dsb sy; isb");
             }
         }
     }
 }
 
 /// AArch64 VMSAv8-64 translation table.
 pub type A64PageTable<H> = PageTable64<A64PagingMetaData, A64PTE, H>;
+/// Mutable reference to an AArch64 VMSAv8-64 translation table.
+pub type A64PageTableMut<'a, H> = PageTable64Mut<'a, A64PagingMetaData, A64PTE, H>;

page_table_multiarch/src/arch/loongarch64.rs

@@ -1,9 +1,40 @@
 //! LoongArch64 specific page table structures.
 
-use crate::{PageTable64, PagingMetaData};
 use core::arch::asm;
+
+use memory_addr::VirtAddr;
 use page_table_entry::loongarch64::LA64PTE;
 
+use crate::{PageTable64, PageTable64Mut, PagingMetaData};
+
+#[inline]
+fn local_flush_tlb(vaddr: Option<memory_addr::VirtAddr>) {
+    unsafe {
+        if let Some(vaddr) = vaddr {
+            // <https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_dbar>
+            //
+            // Only after all previous load/store access operations are completely
+            // executed, the DBAR 0 instruction can be executed; and only after the
+            // execution of DBAR 0 is completed, all subsequent load/store access
+            // operations can be executed.
+            //
+            // <https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_invtlb>
+            //
+            // formats: invtlb op, asid, addr
+            //
+            // op 0x5: Clear all page table entries with G=0 and ASID equal to the
+            // register specified ASID, and VA equal to the register specified VA.
+            //
+            // When the operation indicated by op does not require an ASID, the
+            // general register rj should be set to r0.
+            asm!("dbar 0; invtlb 0x05, $r0, {reg}", reg = in(reg) vaddr.as_usize());
+        } else {
+            // op 0x0: Clear all page table entries
+            asm!("dbar 0; invtlb 0x00, $r0, $r0");
+        }
+    }
+}
+
 /// Metadata of LoongArch64 page tables.
 #[derive(Copy, Clone, Debug)]
 pub struct LA64MetaData;
@@ -44,34 +75,18 @@ impl PagingMetaData for LA64MetaData {
     const LEVELS: usize = 4;
     const PA_MAX_BITS: usize = 48;
     const VA_MAX_BITS: usize = 48;
-    type VirtAddr = memory_addr::VirtAddr;
+    type VirtAddr = VirtAddr;
 
     #[inline]
-    fn flush_tlb(vaddr: Option<memory_addr::VirtAddr>) {
-        unsafe {
-            if let Some(vaddr) = vaddr {
-                // <https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_dbar>
-                //
-                // Only after all previous load/store access operations are completely
-                // executed, the DBAR 0 instruction can be executed; and only after the
-                // execution of DBAR 0 is completed, all subsequent load/store access
-                // operations can be executed.
-                //
-                // <https://loongson.github.io/LoongArch-Documentation/LoongArch-Vol1-EN.html#_invtlb>
-                //
-                // formats: invtlb op, asid, addr
-                //
-                // op 0x5: Clear all page table entries with G=0 and ASID equal to the
-                // register specified ASID, and VA equal to the register specified VA.
-                //
-                // When the operation indicated by op does not require an ASID, the
-                // general register rj should be set to r0.
-                asm!("dbar 0; invtlb 0x05, $r0, {reg}", reg = in(reg) vaddr.as_usize());
-            } else {
-                // op 0x0: Clear all page table entries
-                asm!("dbar 0; invtlb 0x00, $r0, $r0");
-            }
+    fn flush_tlb(vaddr: Option<VirtAddr>) {
+        #[cfg(feature = "smp")]
+        {
+            use crate::__TlbFlushIf_mod;
+            use crate_interface::call_interface;
+
+            call_interface!(TlbFlushIf::flush_all(vaddr));
         }
+        local_flush_tlb(vaddr);
     }
 }
 
@@ -82,4 +97,6 @@ impl PagingMetaData for LA64MetaData {
 /// 4 levels:
 ///
 /// using page table dir3, dir2, dir1 and pt, ignore dir4
-pub type LA64PageTable<I> = PageTable64<LA64MetaData, LA64PTE, I>;
+pub type LA64PageTable<H> = PageTable64<LA64MetaData, LA64PTE, H>;
+/// Mutable reference to a loongarch64 page table.
+pub type LA64PageTableMut<'a, H> = PageTable64Mut<'a, LA64MetaData, LA64PTE, H>;

page_table_multiarch/src/arch/riscv.rs

@@ -1,10 +1,11 @@
 //! RISC-V specific page table structures.
 
-use crate::{PageTable64, PagingMetaData};
+use memory_addr::VirtAddr;
 use page_table_entry::riscv::Rv64PTE;
 
-#[inline]
-fn riscv_flush_tlb(vaddr: Option<memory_addr::VirtAddr>) {
+use crate::{PageTable64, PageTable64Mut, PagingMetaData};
+
+fn local_flush_tlb(vaddr: Option<memory_addr::VirtAddr>) {
     if let Some(vaddr) = vaddr {
         riscv::asm::sfence_vma(0, vaddr.as_usize())
     } else {
@@ -18,10 +19,17 @@ pub trait SvVirtAddr: memory_addr::MemoryAddr + Send + Sync {
     fn flush_tlb(vaddr: Option<Self>);
 }
 
-impl SvVirtAddr for memory_addr::VirtAddr {
+impl SvVirtAddr for VirtAddr {
     #[inline]
     fn flush_tlb(vaddr: Option<Self>) {
-        riscv_flush_tlb(vaddr.map(|vaddr| vaddr.into()))
+        #[cfg(feature = "smp")]
+        {
+            use crate::__TlbFlushIf_mod;
+            use crate_interface::call_interface;
+
+            call_interface!(TlbFlushIf::flush_all(vaddr));
+        }
+        local_flush_tlb(vaddr);
     }
 }
 
@@ -60,7 +68,11 @@ impl<VA: SvVirtAddr> PagingMetaData for Sv48MetaData<VA> {
 }
 
 /// Sv39: Page-Based 39-bit (3 levels) Virtual-Memory System.
-pub type Sv39PageTable<H> = PageTable64<Sv39MetaData<memory_addr::VirtAddr>, Rv64PTE, H>;
+pub type Sv39PageTable<H> = PageTable64<Sv39MetaData<VirtAddr>, Rv64PTE, H>;
+/// Mutable reference to a RISC-V Sv39 page table.
+pub type Sv39PageTableMut<'a, H> = PageTable64Mut<'a, Sv39MetaData<VirtAddr>, Rv64PTE, H>;
 
 /// Sv48: Page-Based 48-bit (4 levels) Virtual-Memory System.
-pub type Sv48PageTable<H> = PageTable64<Sv48MetaData<memory_addr::VirtAddr>, Rv64PTE, H>;
+pub type Sv48PageTable<H> = PageTable64<Sv48MetaData<VirtAddr>, Rv64PTE, H>;
+/// Mutable reference to a RISC-V Sv48 page table.
+pub type Sv48PageTableMut<'a, H> = PageTable64Mut<'a, Sv48MetaData<VirtAddr>, Rv64PTE, H>;

page_table_multiarch/src/arch/x86_64.rs

@@ -1,28 +1,44 @@
 //! x86 specific page table structures.
 
-use crate::{PageTable64, PagingMetaData};
+use memory_addr::VirtAddr;
 use page_table_entry::x86_64::X64PTE;
 
+use crate::{PageTable64, PageTable64Mut, PagingMetaData};
+
+#[inline]
+fn local_flush_tlb(vaddr: Option<memory_addr::VirtAddr>) {
+    unsafe {
+        if let Some(vaddr) = vaddr {
+            x86::tlb::flush(vaddr.into());
+        } else {
+            x86::tlb::flush_all();
+        }
+    }
+}
+
 /// metadata of x86_64 page tables.
 pub struct X64PagingMetaData;
 
 impl PagingMetaData for X64PagingMetaData {
     const LEVELS: usize = 4;
     const PA_MAX_BITS: usize = 52;
     const VA_MAX_BITS: usize = 48;
-    type VirtAddr = memory_addr::VirtAddr;
+    type VirtAddr = VirtAddr;
 
     #[inline]
-    fn flush_tlb(vaddr: Option<memory_addr::VirtAddr>) {
-        unsafe {
-            if let Some(vaddr) = vaddr {
-                x86::tlb::flush(vaddr.into());
-            } else {
-                x86::tlb::flush_all();
-            }
+    fn flush_tlb(vaddr: Option<VirtAddr>) {
+        #[cfg(feature = "smp")]
+        {
+            use crate::__TlbFlushIf_mod;
+            use crate_interface::call_interface;
+
+            call_interface!(TlbFlushIf::flush_all(vaddr));
         }
+        local_flush_tlb(vaddr);
     }
 }
 
 /// x86_64 page table.
 pub type X64PageTable<H> = PageTable64<X64PagingMetaData, X64PTE, H>;
+/// Mutable reference to an x86_64 page table.
+pub type X64PageTableMut<'a, H> = PageTable64Mut<'a, X64PagingMetaData, X64PTE, H>;