Add CCA feature

This is WIP

Signed-off-by: Matias Ezequiel Vara Larsen <[email protected]>

Author: MatiasVara

Makefile

@@ -27,6 +27,9 @@ ifeq ($(SEV),1)
     INIT_SRC += $(SNP_INIT_SRC)
 	BUILD_INIT = 0
 endif
+ifeq ($(CCA), 1)
+    FEATURE_FLAGS := --features cca
+endif
 ifeq ($(GPU),1)
     FEATURE_FLAGS += --features gpu
 endif

src/arch/src/aarch64/fdt.rs

@@ -285,7 +285,10 @@ fn create_psci_node(fdt: &mut FdtWriter) -> Result<()> {
     // Two methods available: hvc and smc.
     // As per documentation, PSCI calls between a guest and hypervisor may use the HVC conduit instead of SMC.
     // So, since we are using kvm, we need to use hvc.
+    #[cfg(not(feature = "cca"))]
     fdt.property_string("method", "hvc")?;
+    #[cfg(feature = "cca")]
+    fdt.property_string("method", "smc")?;
     fdt.end_node(node)?;
 
     Ok(())

src/arch/src/aarch64/linux/regs.rs

@@ -125,8 +125,10 @@ arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5);
 /// * `boot_ip` - Starting instruction pointer.
 /// * `mem` - Reserved DRAM for current VM.
 pub fn setup_regs(vcpu: &VcpuFd, cpu_id: u8, boot_ip: u64, mem: &GuestMemoryMmap) -> Result<()> {
-    // Get the register index of the PSTATE (Processor State) register.
+    // PSTATE cannot be accesed from the host in CCA
+    #[cfg(not(feature = "cca"))]
     #[allow(deref_nullptr)]
+    // Get the register index of the PSTATE (Processor State) register.
     vcpu.set_one_reg(arm64_core_reg!(pstate), &PSTATE_FAULT_BITS_64.to_le_bytes())
         .map_err(Error::SetCoreRegister)?;
 

src/arch/src/aarch64/mod.rs

@@ -69,7 +69,7 @@ pub fn arch_memory_regions(size: usize) -> (ArchMemoryInfo, Vec<(GuestAddress, u
     } else {
         vec![
             (GuestAddress(layout::DRAM_MEM_START), dram_size),
-            (GuestAddress(shm_start_addr), MMIO_SHM_SIZE as usize),
+            //(GuestAddress(shm_start_addr), MMIO_SHM_SIZE as usize),
         ]
     };
 

src/vmm/src/builder.rs

@@ -12,6 +12,8 @@ use std::io;
 use std::os::fd::AsRawFd;
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex};
+use std::cmp::max;
+use cca::Algo;
 
 use super::{Error, Vmm};
 
@@ -68,7 +70,9 @@ use vm_memory::mmap::MmapRegion;
 #[cfg(any(target_arch = "aarch64", feature = "tee"))]
 use vm_memory::Bytes;
 use vm_memory::GuestMemory;
-use vm_memory::{GuestAddress, GuestMemoryMmap};
+use vm_memory::{GuestAddress, GuestMemoryMmap, GuestMemoryRegion, Address};
+
+use kvm_bindings::KVM_ARM_VCPU_REC;
 
 #[cfg(feature = "efi")]
 static EDK2_BINARY: &[u8] = include_bytes!("../../../edk2/KRUN_EFI.silent.fd");
@@ -559,7 +563,7 @@ pub fn build_microvm(
         )?;
     }
 
-    #[cfg(not(feature = "tee"))]
+    #[cfg(all(not(feature = "tee"), not(feature = "cca")))]
     let _shm_region = Some(VirtioShmRegion {
         host_addr: guest_memory
             .get_host_address(GuestAddress(arch_memory_info.shm_start_addr))
@@ -568,6 +572,27 @@ pub fn build_microvm(
         size: arch_memory_info.shm_size as usize,
     });
 
+    // YOU CAN'T ACCESS GUEST MEMORY AT THIS POINT
+    /*
+    #[cfg(feature = "cca")]
+    {
+        let _ = vm.realm.configure_measurement(vm.fd(), Algo::AlgoSha256);
+        vm.realm.create_realm_descriptor(vm.fd()).unwrap();
+
+        for (_index, region) in guest_memory.iter().enumerate() {
+            vm.realm.populate(vm.fd(), region.start_addr().raw_value(), region.len()).unwrap();
+        }
+        let feature = KVM_ARM_VCPU_REC as i32;
+
+        // not really sure if the finalize and the activate should go here
+        for vcpu in vcpus.iter() {
+             vcpu.fd.vcpu_finalize(&feature).unwrap();
+        }
+
+        vm.realm.activate(vm.fd()).unwrap();
+    }
+    */
+
     let mut vmm = Vmm {
         guest_memory,
         arch_memory_info,
@@ -647,6 +672,7 @@ pub fn build_microvm(
     #[cfg(not(feature = "tee"))]
     let initrd_config = None;
 
+    // after this point guest memory and regs are not accesible anymore
     vmm.configure_system(
         vcpus.as_slice(),
         &initrd_config,
@@ -809,7 +835,7 @@ fn load_cmdline(vmm: &Vmm) -> std::result::Result<(), StartMicrovmError> {
     .map_err(StartMicrovmError::LoadCommandline)
 }
 
-#[cfg(all(target_os = "linux", not(feature = "tee")))]
+#[cfg(all(target_os = "linux", not(feature = "tee"), not(feature = "cca")))]
 pub(crate) fn setup_vm(
     guest_memory: &GuestMemoryMmap,
 ) -> std::result::Result<Vm, StartMicrovmError> {
@@ -824,6 +850,29 @@ pub(crate) fn setup_vm(
         .map_err(StartMicrovmError::Internal)?;
     Ok(vm)
 }
+#[cfg(all(target_os = "linux", feature = "cca"))]
+pub(crate) fn setup_vm(
+    guest_memory: &GuestMemoryMmap,
+) -> std::result::Result<Vm, StartMicrovmError> {
+    let kvm = KvmContext::new()
+        .map_err(Error::KvmContext)
+        .map_err(StartMicrovmError::Internal)?;
+
+    // calculate max_addr for max_ipa
+    let mut max_addr = 0;
+    for (_index, region) in guest_memory.iter().enumerate() {
+        max_addr = max(max_addr, region.start_addr().raw_value() + region.len() - 1);
+    }
+
+    let mut vm = Vm::new(kvm.fd(), max_addr as usize)
+        .map_err(Error::Vm)
+        .map_err(StartMicrovmError::Internal)?;
+
+    vm.memory_init(guest_memory, kvm.max_memslots(), true)
+        .map_err(Error::Vm)
+        .map_err(StartMicrovmError::Internal)?;
+    Ok(vm)
+}
 #[cfg(all(target_os = "linux", feature = "tee"))]
 pub(crate) fn setup_vm(
     kvm: &KvmContext,
@@ -1021,7 +1070,7 @@ fn create_vcpus_aarch64(
 ) -> super::Result<Vec<Vcpu>> {
     let mut vcpus = Vec::with_capacity(vcpu_config.vcpu_count as usize);
     for cpu_index in 0..vcpu_config.vcpu_count {
-        let mut vcpu = Vcpu::new_aarch64(
+        let mut vcpu: Vcpu = Vcpu::new_aarch64(
             cpu_index,
             vm.fd(),
             exit_evt.try_clone().map_err(Error::EventFd)?,

src/vmm/src/lib.rs

@@ -64,6 +64,11 @@ use polly::event_manager::{self, EventManager, Subscriber};
 use utils::epoll::{EpollEvent, EventSet};
 use utils::eventfd::EventFd;
 use vm_memory::GuestMemoryMmap;
+use cca::Algo;
+use kvm_bindings::KVM_ARM_VCPU_REC;
+use vm_memory::GuestMemory;
+use vm_memory::GuestMemoryRegion;
+use vm_memory::Address;
 
 /// Success exit code.
 pub const FC_EXIT_CODE_OK: u8 = 0;
@@ -299,6 +304,25 @@ impl Vmm {
                 _smbios_oem_strings,
             )
             .map_err(Error::ConfigureSystem)?;
+
+            // after activation guest not accesible anymore
+            #[cfg(feature = "cca")]
+            {
+                let _ = self.vm.realm.configure_measurement(self.vm.fd(), Algo::AlgoSha256);
+                self.vm.realm.create_realm_descriptor(self.vm.fd()).unwrap();
+
+                for (_index, region) in self.guest_memory.iter().enumerate() {
+                    self.vm.realm.populate(self.vm.fd(), region.start_addr().raw_value(), region.len()).unwrap();
+                }
+                let feature = KVM_ARM_VCPU_REC as i32;
+
+                // not really sure if the finalize and the activate should go here
+                for vcpu in vcpus.iter() {
+                     vcpu.fd.vcpu_finalize(&feature).unwrap();
+                }
+
+                self.vm.realm.activate(self.vm.fd()).unwrap();
+            }
         }
 
         #[cfg(all(target_arch = "aarch64", target_os = "macos"))]

src/vmm/src/linux/vstate.rs

@@ -11,6 +11,7 @@ use std::cell::Cell;
 use std::fmt::{Display, Formatter};
 use std::io;
 use std::os::fd::RawFd;
+use std::cmp::max;
 
 #[cfg(feature = "tee")]
 use std::os::unix::io::RawFd;
@@ -48,8 +49,9 @@ use kvm_bindings::{
     KVM_MAX_CPUID_ENTRIES, KVM_PIT_SPEAKER_DUMMY,
 };
 use kvm_bindings::{
-    kvm_create_guest_memfd, kvm_userspace_memory_region, kvm_userspace_memory_region2,
-    KVM_API_VERSION, KVM_MEM_GUEST_MEMFD,
+    kvm_create_guest_memfd, kvm_userspace_memory_region, kvm_userspace_memory_region2, kvm_memory_attributes,
+    KVM_API_VERSION, KVM_MEM_GUEST_MEMFD, KVM_VM_TYPE_ARM_REALM, KVM_VM_TYPE_ARM_IPA_SIZE_MASK,
+    KVM_MEMORY_ATTRIBUTE_PRIVATE
 };
 use kvm_ioctls::*;
 use utils::eventfd::EventFd;
@@ -65,6 +67,9 @@ use sev::launch::sev as sev_launch;
 #[cfg(feature = "amd-sev")]
 use sev::launch::snp;
 
+#[cfg(feature = "cca")]
+use cca::Realm;
+
 /// Signal number (SIGRTMIN) used to kick Vcpus.
 pub(crate) const VCPU_RTSIG_OFFSET: i32 = 0;
 
@@ -483,11 +488,14 @@ pub struct Vm {
 
     #[cfg(feature = "amd-sev")]
     pub tee: Tee,
+
+    #[cfg(feature = "cca")]
+    pub realm: Realm,
 }
 
 impl Vm {
     /// Constructs a new `Vm` using the given `Kvm` instance.
-    #[cfg(not(feature = "tee"))]
+    #[cfg(all(not(feature = "tee"), not(feature = "cca")))]
     pub fn new(kvm: &Kvm) -> Result<Self> {
         //create fd for interacting with kvm-vm specific functions
         let vm_fd = kvm.create_vm().map_err(Error::VmFd)?;
@@ -511,6 +519,22 @@ impl Vm {
         })
     }
 
+    #[cfg(feature = "cca")]
+    pub fn new(kvm: &Kvm, max_ipa: usize) -> Result<Self> {
+        //create fd for interacting with kvm-vm specific functions
+        let ipa_bits = max(64u32 - max_ipa.leading_zeros()- 1, 32) + 1;
+        let vm_fd = kvm.create_vm_with_type((KVM_VM_TYPE_ARM_REALM | (ipa_bits & KVM_VM_TYPE_ARM_IPA_SIZE_MASK)).into()).map_err(Error::VmFd)?;
+
+        let realm  = Realm::new().unwrap();
+
+        Ok(Vm {
+            fd: vm_fd,
+            #[cfg(target_arch = "aarch64")]
+            irqchip_handle: None,
+            realm,
+        })
+    }
+
     #[cfg(feature = "amd-sev")]
     pub fn new(kvm: &Kvm, tee_config: &TeeConfig) -> Result<Self> {
         //create fd for interacting with kvm-vm specific functions
@@ -581,7 +605,7 @@ impl Vm {
                     .create_guest_memfd(gmem)
                     .map_err(Error::CreateGuestMemfd)?;
 
-                let memory_region = kvm_userspace_memory_region2 {
+                let memory_region: kvm_userspace_memory_region2 = kvm_userspace_memory_region2 {
                     slot: index as u32,
                     flags: KVM_MEM_GUEST_MEMFD,
                     guest_phys_addr: region.start_addr().raw_value(),
@@ -600,6 +624,14 @@ impl Vm {
                         .set_user_memory_region2(memory_region)
                         .map_err(Error::SetUserMemoryRegion2)?;
                 };
+                // set private by default when using guestmemfd
+                let attr = kvm_memory_attributes {
+                    address: region.start_addr().raw_value(),
+                    size: region.len(),
+                    attributes: KVM_MEMORY_ATTRIBUTE_PRIVATE as u64,
+                    flags: 0,
+                };
+                self.fd.set_memory_attributes(attr).unwrap();
             } else {
                 let memory_region = kvm_userspace_memory_region {
                     slot: index as u32,
@@ -808,7 +840,7 @@ type VcpuCell = Cell<Option<*mut Vcpu>>;
 
 /// A wrapper around creating and using a kvm-based VCPU.
 pub struct Vcpu {
-    fd: VcpuFd,
+    pub fd: VcpuFd,
     id: u8,
     mmio_bus: Option<devices::Bus>,
     #[allow(dead_code)]
@@ -1267,6 +1299,12 @@ impl Vcpu {
                     info!("Received KVM_EXIT_SHUTDOWN signal");
                     Ok(VcpuEmulation::Stopped)
                 }
+                VcpuExit::MemoryFault {flags, gpa, size} => {
+                    println!("ignore memoryfault at {} {}", gpa, size);
+                    // TODO: To setup the setmemoryproperties I need to have access to vm
+                    // but vm is not a shared resource
+                    Ok(VcpuEmulation::Handled)
+                }
                 // Documentation specifies that below kvm exits are considered
                 // errors.
                 VcpuExit::FailEntry(reason, vcpu) => {
@@ -1280,6 +1318,7 @@ impl Vcpu {
                 r => {
                     // TODO: Are we sure we want to finish running a vcpu upon
                     // receiving a vm exit that is not necessarily an error?
+                    println!("error! {:?}", r);
                     error!("Unexpected exit reason on vcpu run: {:?}", r);
                     Err(Error::VcpuUnhandledKvmExit)
                 }