Scrub memory before platform-specific invalidation

kernel/src/address.rs

@@ -5,6 +5,7 @@
 // Author: Carlos López <[email protected]>
 
 use crate::types::{PAGE_SHIFT, PAGE_SIZE};
+use crate::utils::{align_down, align_up};
 use core::fmt;
 use core::ops;
 
@@ -42,7 +43,12 @@ pub trait Address:
 
     #[inline]
     fn align_up(&self, align: InnerAddr) -> Self {
-        Self::from((self.bits() + (align - 1)) & !(align - 1))
+        Self::from(align_up((*self).into(), align))
+    }
+
+    #[inline]
+    fn align_down(&self, align: InnerAddr) -> Self {
+        Self::from(align_down((*self).into(), align))
     }
 
     #[inline]
@@ -52,7 +58,7 @@ pub trait Address:
 
     #[inline]
     fn page_align(&self) -> Self {
-        Self::from(self.bits() & !(PAGE_SIZE - 1))
+        self.align_down(PAGE_SIZE)
     }
 
     #[inline]

kernel/src/config.rs

@@ -159,7 +159,7 @@ impl SvsmConfig<'_> {
         }
     }
 
-    pub fn invalidate_boot_data(&self) -> bool {
+    pub fn clean_up_boot_data(&self) -> bool {
         match self {
             SvsmConfig::FirmwareConfig(_) => false,
             SvsmConfig::IgvmConfig(_) => true,

kernel/src/mm/virtualrange.rs

@@ -10,6 +10,7 @@ use crate::error::SvsmError;
 use crate::types::{PAGE_SHIFT, PAGE_SHIFT_2M, PAGE_SIZE, PAGE_SIZE_2M};
 use crate::utils::bitmap_allocator::{BitmapAllocator, BitmapAllocator1024};
 use crate::utils::MemoryRegion;
+use core::cmp::min;
 use core::fmt::Debug;
 
 use super::{
@@ -48,8 +49,9 @@ impl VirtualRange {
     }
 
     pub fn alloc(&mut self, page_count: usize, alignment: usize) -> Result<VirtAddr, SvsmError> {
-        // Always reserve an extra page to leave a guard between virtual memory allocations
-        match self.bits.alloc(page_count + 1, alignment) {
+        // Reserve an extra page, if possible, to leave a guard between virtual memory allocations
+        let npages = min(page_count + 1, self.page_count);
+        match self.bits.alloc(npages, alignment) {
             Some(offset) => Ok(self.start_virt + (offset << self.page_shift)),
             None => Err(SvsmError::Mem),
         }

kernel/src/platform/tdp.rs

@@ -10,7 +10,7 @@ use crate::cpu::cpuid::CpuidResult;
 use crate::cpu::percpu::PerCpu;
 use crate::error::SvsmError;
 use crate::io::IOPort;
-use crate::mm::{virt_to_frame, PerCPUPageMappingGuard};
+use crate::mm::virt_to_frame;
 use crate::platform::{PageEncryptionMasks, PageStateChangeOp, PageValidateOp, SvsmPlatform};
 use crate::types::PageSize;
 use crate::utils::immut_after_init::ImmutAfterInitCell;
@@ -101,14 +101,9 @@ impl SvsmPlatform for TdpPlatform {
         region: MemoryRegion<PhysAddr>,
         op: PageValidateOp,
     ) -> Result<(), SvsmError> {
-        match op {
-            PageValidateOp::Validate => {
-                td_accept_memory(region.start().into(), region.len().try_into().unwrap());
-            }
-            PageValidateOp::Invalidate => {
-                let mapping = PerCPUPageMappingGuard::create(region.start(), region.end(), 0)?;
-                zero_mem_region(mapping.virt_addr(), mapping.virt_addr() + region.len());
-            }
+        // There is no need for page invalidation on TDP
+        if let PageValidateOp::Validate = op {
+            td_accept_memory(region.start().into(), region.len().try_into().unwrap());
         }
         Ok(())
     }

kernel/src/svsm.rs

@@ -45,7 +45,7 @@ use svsm::platform::{init_platform_type, SvsmPlatformCell, SVSM_PLATFORM};
 use svsm::requests::{request_loop, request_processing_main, update_mappings};
 use svsm::sev::utils::{rmp_adjust, RMPFlags};
 use svsm::sev::{secrets_page, secrets_page_mut};
-use svsm::svsm_paging::{init_page_table, invalidate_early_boot_memory};
+use svsm::svsm_paging::{clean_up_early_boot_memory, init_page_table};
 use svsm::task::exec_user;
 use svsm::task::{create_kernel_task, schedule_init};
 use svsm::types::{PageSize, GUEST_VMPL, PAGE_SIZE};
@@ -423,8 +423,8 @@ pub extern "C" fn svsm_main() {
     populate_ram_fs(LAUNCH_INFO.kernel_fs_start, LAUNCH_INFO.kernel_fs_end)
         .expect("Failed to unpack FS archive");
 
-    invalidate_early_boot_memory(&**SVSM_PLATFORM, &config, launch_info)
-        .expect("Failed to invalidate early boot memory");
+    clean_up_early_boot_memory(&**SVSM_PLATFORM, &config, launch_info)
+        .expect("Failed to clean up early boot memory");
 
     let cpus = config.load_cpu_info().expect("Failed to load ACPI tables");
     let mut nr_cpus = 0;

kernel/src/svsm_paging.rs

@@ -9,11 +9,12 @@ use crate::config::SvsmConfig;
 use crate::error::SvsmError;
 use crate::igvm_params::IgvmParams;
 use crate::mm::pagetable::{PTEntryFlags, PageTable};
-use crate::mm::PageBox;
+use crate::mm::{PageBox, PerCPUPageMappingGuard};
 use crate::platform::{PageStateChangeOp, PageValidateOp, SvsmPlatform};
-use crate::types::PageSize;
-use crate::utils::MemoryRegion;
+use crate::types::{PageSize, PAGE_SHIFT, PAGE_SHIFT_2M, PAGE_SIZE, PAGE_SIZE_2M};
+use crate::utils::{align_down, page_align, zero_mem_region, MemoryRegion};
 use bootlib::kernel_launch::KernelLaunchInfo;
+use core::cmp::min;
 
 struct IgvmParamInfo<'a> {
     virt_addr: VirtAddr,
@@ -94,18 +95,70 @@ pub fn init_page_table(
     Ok(pgtable)
 }
 
-fn invalidate_boot_memory_region(
+fn scrub_phys_memory_region(region: MemoryRegion<PhysAddr>) -> Result<(), SvsmError> {
+    let mut start = region.start();
+    let mut maxlen_4k = 512 * PAGE_SIZE; // 2M
+    let mut maxlen_2m = 512 * PAGE_SIZE_2M; // 1G
+    let region_end_4k = region.end().page_align_up();
+    let region_end_2m = region.end().align_up(PAGE_SIZE_2M);
+
+    while start < region.end() {
+        let mut map_start = start.page_align();
+        let (mapping, map_end) = if maxlen_4k == 0 && maxlen_2m == 0 {
+            // No more space for temporary mappings
+            return Err(SvsmError::Mem);
+        } else if (maxlen_4k > 0 && (region_end_4k - map_start) < PAGE_SIZE_2M) || maxlen_2m == 0 {
+            // Use 4K mappings if
+            // - The mapping size < a 2M huge page, or
+            // - There is no space for 2M temporary mappings
+            let map_end = min(map_start + maxlen_4k, region_end_4k);
+            match PerCPUPageMappingGuard::create(map_start, map_end, 0) {
+                Ok(m) => (m, map_end),
+                Err(SvsmError::Mem) => {
+                    maxlen_4k = page_align((map_end - map_start) >> 1);
+                    continue;
+                }
+                Err(e) => return Err(e),
+            }
+        } else {
+            // Use 2M mappings if
+            // - The mapping size >= a 2M huge page, or
+            // - There is no space for 4K temporary mappings
+            map_start = start.align_down(PAGE_SIZE_2M);
+            let map_end = min(map_start + maxlen_2m, region_end_2m);
+            match PerCPUPageMappingGuard::create(map_start, map_end, PAGE_SHIFT_2M - PAGE_SHIFT) {
+                Ok(m) => (m, map_end),
+                Err(SvsmError::Mem) => {
+                    maxlen_2m = align_down((map_end - map_start) >> 1, PAGE_SIZE_2M);
+                    continue;
+                }
+                Err(e) => return Err(e),
+            }
+        };
+        let off = start - map_start;
+        let len = min(map_end, region.end()) - map_start;
+        zero_mem_region(mapping.virt_addr() + off, mapping.virt_addr() + len);
+        start = map_end;
+    }
+    Ok(())
+}
+
+fn clean_up_boot_memory_region(
     platform: &dyn SvsmPlatform,
     config: &SvsmConfig<'_>,
     region: MemoryRegion<PhysAddr>,
 ) -> Result<(), SvsmError> {
     log::info!(
-        "Invalidating boot region {:018x}-{:018x}",
+        "Cleaning up boot region {:018x}-{:018x}",
         region.start(),
         region.end()
     );
 
     if !region.is_empty() {
+        // Some platforms (such as TDP) do not need page invalidation.
+        // Scrub the memory region first to make sure its content is
+        // always at least wiped.
+        scrub_phys_memory_region(region)?;
         platform.validate_physical_page_range(region, PageValidateOp::Invalidate)?;
 
         if config.page_state_change_required() {
@@ -116,49 +169,49 @@ fn invalidate_boot_memory_region(
     Ok(())
 }
 
-pub fn invalidate_early_boot_memory(
+pub fn clean_up_early_boot_memory(
     platform: &dyn SvsmPlatform,
     config: &SvsmConfig<'_>,
     launch_info: &KernelLaunchInfo,
 ) -> Result<(), SvsmError> {
-    // Early boot memory must be invalidated after changing to the SVSM page
-    // page table to avoid invalidating page tables currently in use.  Always
-    // invalidate stage 2 memory, unless firmware is loaded into low memory.
-    // Also invalidate the boot data if required.
+    // Early boot memory must be cleaned up after changing to the SVSM page
+    // page table to avoid destroying page tables currently in use.  Always
+    // clean up stage 2 memory, unless firmware is loaded into low memory.
+    // Also clean up the boot data if required.
     if !config.fw_in_low_memory() {
         let lowmem_region = MemoryRegion::new(PhysAddr::null(), 640 * 1024);
-        invalidate_boot_memory_region(platform, config, lowmem_region)?;
+        clean_up_boot_memory_region(platform, config, lowmem_region)?;
     }
 
     let stage2_base = PhysAddr::from(launch_info.stage2_start);
     let stage2_end = PhysAddr::from(launch_info.stage2_end);
     let stage2_region = MemoryRegion::from_addresses(stage2_base, stage2_end);
-    invalidate_boot_memory_region(platform, config, stage2_region)?;
+    clean_up_boot_memory_region(platform, config, stage2_region)?;
 
-    if config.invalidate_boot_data() {
+    if config.clean_up_boot_data() {
         let kernel_elf_size =
             launch_info.kernel_elf_stage2_virt_end - launch_info.kernel_elf_stage2_virt_start;
         let kernel_elf_region = MemoryRegion::new(
             PhysAddr::new(launch_info.kernel_elf_stage2_virt_start.try_into().unwrap()),
             kernel_elf_size.try_into().unwrap(),
         );
-        invalidate_boot_memory_region(platform, config, kernel_elf_region)?;
+        clean_up_boot_memory_region(platform, config, kernel_elf_region)?;
 
         let kernel_fs_size = launch_info.kernel_fs_end - launch_info.kernel_fs_start;
         if kernel_fs_size > 0 {
             let kernel_fs_region = MemoryRegion::new(
                 PhysAddr::new(launch_info.kernel_fs_start.try_into().unwrap()),
                 kernel_fs_size.try_into().unwrap(),
             );
-            invalidate_boot_memory_region(platform, config, kernel_fs_region)?;
+            clean_up_boot_memory_region(platform, config, kernel_fs_region)?;
         }
 
         if launch_info.stage2_igvm_params_size > 0 {
             let igvm_params_region = MemoryRegion::new(
                 PhysAddr::new(launch_info.stage2_igvm_params_phys_addr.try_into().unwrap()),
                 launch_info.stage2_igvm_params_size as usize,
             );
-            invalidate_boot_memory_region(platform, config, igvm_params_region)?;
+            clean_up_boot_memory_region(platform, config, igvm_params_region)?;
         }
     }
 

kernel/src/utils/mod.rs

@@ -11,5 +11,6 @@ pub mod util;
 
 pub use memory_region::MemoryRegion;
 pub use util::{
-    align_down, align_up, halt, is_aligned, overlap, page_align_up, page_offset, zero_mem_region,
+    align_down, align_up, halt, is_aligned, overlap, page_align, page_align_up, page_offset,
+    zero_mem_region,
 };

kernel/src/utils/util.rs

@@ -41,6 +41,10 @@ pub fn page_align_up(x: usize) -> usize {
     align_up(x, PAGE_SIZE)
 }
 
+pub fn page_align(x: usize) -> usize {
+    align_down(x, PAGE_SIZE)
+}
+
 pub fn page_offset(x: usize) -> usize {
     x & (PAGE_SIZE - 1)
 }