[host] adds init-paging feature (#639)

This is an extra feature flag that allows disabling our paging setup. This is needed to further enable custom guests and
should not impact any of our current workloads. init-paging is on by default.

Signed-off-by: danbugs <danilochiarlone@gmail.com>

Author: danbugs

Justfile

@@ -75,8 +75,8 @@ test-like-ci config=default-target hypervisor="kvm":
     @# with default features
     just test {{config}} {{ if hypervisor == "mshv3" {"mshv3"} else {""} }}
 
-    @# with only one driver enabled + seccomp
-    just test {{config}} seccomp,build-metadata,{{ if hypervisor == "mshv" {"mshv2"} else if hypervisor == "mshv3" {"mshv3"} else {"kvm"} }}
+    @# with only one driver enabled + seccomp + build-metadata + init-paging
+    just test {{config}} seccomp,build-metadata,init-paging,{{ if hypervisor == "mshv" {"mshv2"} else if hypervisor == "mshv3" {"mshv3"} else {"kvm"} }}
 
     @# make sure certain cargo features compile
     cargo check -p hyperlight-host --features crashdump
@@ -94,32 +94,32 @@ test target=default-target features="": (test-unit target features) (test-isolat
 
 # runs unit tests
 test-unit target=default-target features="":
-    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --lib
+    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --lib
 
 # runs tests that requires being run separately, for example due to global state
 test-isolated target=default-target features="":
-    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- sandbox::uninitialized::tests::test_trace_trace --exact --ignored
-    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- sandbox::uninitialized::tests::test_log_trace --exact --ignored
-    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- sandbox::initialized_multi_use::tests::create_1000_sandboxes --exact --ignored
-    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- sandbox::outb::tests::test_log_outb_log --exact --ignored
-    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- mem::shared_mem::tests::test_drop --exact --ignored
-    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --test integration_test -- log_message --exact --ignored
+    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- sandbox::uninitialized::tests::test_trace_trace --exact --ignored
+    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- sandbox::uninitialized::tests::test_log_trace --exact --ignored
+    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- sandbox::initialized_multi_use::tests::create_1000_sandboxes --exact --ignored
+    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- sandbox::outb::tests::test_log_outb_log --exact --ignored
+    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- mem::shared_mem::tests::test_drop --exact --ignored
+    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --test integration_test -- log_message --exact --ignored
     @# metrics tests
-    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F function_call_metrics," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- metrics::tests::test_metrics_are_emitted --exact 
+    cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F function_call_metrics,init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host --lib -- metrics::tests::test_metrics_are_emitted --exact 
 # runs integration tests. Guest can either be "rust" or "c"
 test-integration guest target=default-target features="":
     @# run execute_on_heap test with feature "executable_heap" on and off
     {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap {{ if features =="" {" --features executable_heap"} else {"--features executable_heap," + features} }} -- --ignored
     {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap {{ if features =="" {""} else {"--features " + features} }} -- --ignored
 
     @# run the rest of the integration tests
-    {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test -p hyperlight-host {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --test '*'
+    {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test -p hyperlight-host {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --test '*'
 
 # runs seccomp tests
 test-seccomp target=default-target features="":
     @# run seccomp test with feature "seccomp" on and off
     cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib {{ if features =="" {''} else { "--features " + features } }} -- --ignored
-    cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features {{ if features =~"mshv3" {"--features mshv3"} else {"--features mshv2,kvm" } }} --lib -- --ignored
+    cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features {{ if features =~"mshv3" {"--features init-paging,mshv3"} else {"--features mshv2,init-paging,kvm" } }} --lib -- --ignored
 
 # runs tests that ensure compilation fails when it should
 test-compilation-fail target=default-target:

src/hyperlight_host/Cargo.toml

@@ -119,7 +119,7 @@ cfg_aliases = "0.2.1"
 built = { version = "0.8.0", optional = true, features = ["chrono", "git2"] }
 
 [features]
-default = ["kvm", "mshv2", "seccomp", "build-metadata"]
+default = ["kvm", "mshv2", "seccomp", "build-metadata", "init-paging"]
 seccomp = ["dep:seccompiler"]
 function_call_metrics = []
 executable_heap = []
@@ -134,6 +134,7 @@ mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]
 gdb = ["dep:gdbstub", "dep:gdbstub_arch"]
 fuzzing = ["hyperlight-common/fuzzing"]
 build-metadata = ["dep:built"]
+init-paging = []
 
 [[bench]]
 name = "benchmarks"

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -61,14 +61,14 @@ use super::gdb::{
 #[cfg(gdb)]
 use super::handlers::DbgMemAccessHandlerWrapper;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
+#[cfg(feature = "init-paging")]
 use super::{
     CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR, CR4_OSXMMEXCPT, CR4_PAE,
-    EFER_LMA, EFER_LME, EFER_NX, EFER_SCE, Hypervisor, InterruptHandle, LinuxInterruptHandle,
-    VirtualCPU,
+    EFER_LMA, EFER_LME, EFER_NX, EFER_SCE,
 };
+use super::{HyperlightExit, Hypervisor, InterruptHandle, LinuxInterruptHandle, VirtualCPU};
 #[cfg(gdb)]
 use crate::HyperlightError;
-use crate::hypervisor::HyperlightExit;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
 use crate::sandbox::SandboxConfiguration;
@@ -436,11 +436,12 @@ impl HypervLinuxDriver {
     }
 
     #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
-    fn setup_initial_sregs(vcpu: &mut VcpuFd, pml4_addr: u64) -> Result<()> {
+    fn setup_initial_sregs(vcpu: &mut VcpuFd, _pml4_addr: u64) -> Result<()> {
+        #[cfg(feature = "init-paging")]
         let sregs = SpecialRegisters {
             cr0: CR0_PE | CR0_MP | CR0_ET | CR0_NE | CR0_AM | CR0_PG | CR0_WP,
             cr4: CR4_PAE | CR4_OSFXSR | CR4_OSXMMEXCPT,
-            cr3: pml4_addr,
+            cr3: _pml4_addr,
             efer: EFER_LME | EFER_LMA | EFER_SCE | EFER_NX,
             cs: SegmentRegister {
                 type_: 11,
@@ -457,6 +458,16 @@ impl HypervLinuxDriver {
             },
             ..Default::default()
         };
+
+        #[cfg(not(feature = "init-paging"))]
+        let sregs = SpecialRegisters {
+            cs: SegmentRegister {
+                base: 0,
+                selector: 0,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
         vcpu.set_sregs(&sregs)?;
         Ok(())
     }

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

@@ -41,10 +41,12 @@ use super::surrogate_process::SurrogateProcess;
 use super::surrogate_process_manager::*;
 use super::windows_hypervisor_platform::{VMPartition, VMProcessor};
 use super::wrappers::{HandleWrapper, WHvFPURegisters};
+#[cfg(feature = "init-paging")]
 use super::{
     CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR, CR4_OSXMMEXCPT, CR4_PAE,
-    EFER_LMA, EFER_LME, EFER_NX, EFER_SCE, HyperlightExit, Hypervisor, InterruptHandle, VirtualCPU,
+    EFER_LMA, EFER_LME, EFER_NX, EFER_SCE,
 };
+use super::{HyperlightExit, Hypervisor, InterruptHandle, VirtualCPU};
 use crate::hypervisor::fpu::FP_CONTROL_WORD_DEFAULT;
 use crate::hypervisor::wrappers::WHvGeneralRegisters;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
@@ -125,9 +127,10 @@ impl HypervWindowsDriver {
         })
     }
 
-    fn setup_initial_sregs(proc: &mut VMProcessor, pml4_addr: u64) -> Result<()> {
+    fn setup_initial_sregs(proc: &mut VMProcessor, _pml4_addr: u64) -> Result<()> {
+        #[cfg(feature = "init-paging")]
         proc.set_registers(&[
-            (WHvX64RegisterCr3, WHV_REGISTER_VALUE { Reg64: pml4_addr }),
+            (WHvX64RegisterCr3, WHV_REGISTER_VALUE { Reg64: _pml4_addr }),
             (
                 WHvX64RegisterCr4,
                 WHV_REGISTER_VALUE {
@@ -158,6 +161,19 @@ impl HypervWindowsDriver {
                 },
             ),
         ])?;
+
+        #[cfg(not(feature = "init-paging"))]
+        proc.set_registers(&[(
+            WHvX64RegisterCs,
+            WHV_REGISTER_VALUE {
+                Segment: WHV_X64_SEGMENT_REGISTER {
+                    Base: 0,
+                    Selector: 0,
+                    ..Default::default()
+                },
+            },
+        )])?;
+
         Ok(())
     }
 

src/hyperlight_host/src/hypervisor/kvm.rs

@@ -35,11 +35,12 @@ use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, KvmDebug
 #[cfg(gdb)]
 use super::handlers::DbgMemAccessHandlerWrapper;
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
+#[cfg(feature = "init-paging")]
 use super::{
     CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR, CR4_OSXMMEXCPT, CR4_PAE,
-    EFER_LMA, EFER_LME, EFER_NX, EFER_SCE, HyperlightExit, Hypervisor, InterruptHandle,
-    LinuxInterruptHandle, VirtualCPU,
+    EFER_LMA, EFER_LME, EFER_NX, EFER_SCE,
 };
+use super::{HyperlightExit, Hypervisor, InterruptHandle, LinuxInterruptHandle, VirtualCPU};
 #[cfg(gdb)]
 use crate::HyperlightError;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
@@ -390,14 +391,21 @@ impl KVMDriver {
     }
 
     #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
-    fn setup_initial_sregs(vcpu_fd: &mut VcpuFd, pml4_addr: u64) -> Result<()> {
+    fn setup_initial_sregs(vcpu_fd: &mut VcpuFd, _pml4_addr: u64) -> Result<()> {
         // setup paging and IA-32e (64-bit) mode
         let mut sregs = vcpu_fd.get_sregs()?;
-        sregs.cr3 = pml4_addr;
-        sregs.cr4 = CR4_PAE | CR4_OSFXSR | CR4_OSXMMEXCPT;
-        sregs.cr0 = CR0_PE | CR0_MP | CR0_ET | CR0_NE | CR0_AM | CR0_PG | CR0_WP;
-        sregs.efer = EFER_LME | EFER_LMA | EFER_SCE | EFER_NX;
-        sregs.cs.l = 1; // required for 64-bit mode
+        cfg_if::cfg_if! {
+            if #[cfg(feature = "init-paging")] {
+                sregs.cr3 = _pml4_addr;
+                sregs.cr4 = CR4_PAE | CR4_OSFXSR | CR4_OSXMMEXCPT;
+                sregs.cr0 = CR0_PE | CR0_MP | CR0_ET | CR0_NE | CR0_AM | CR0_PG | CR0_WP;
+                sregs.efer = EFER_LME | EFER_LMA | EFER_SCE | EFER_NX;
+                sregs.cs.l = 1; // required for 64-bit mode
+            } else {
+                sregs.cs.base = 0;
+                sregs.cs.selector = 0;
+            }
+        }
         vcpu_fd.set_sregs(&sregs)?;
         Ok(())
     }

src/hyperlight_host/src/hypervisor/mod.rs

@@ -75,20 +75,24 @@ use self::handlers::{
 };
 use crate::mem::ptr::RawPtr;
 
-pub(crate) const CR4_PAE: u64 = 1 << 5;
-pub(crate) const CR4_OSFXSR: u64 = 1 << 9;
-pub(crate) const CR4_OSXMMEXCPT: u64 = 1 << 10;
-pub(crate) const CR0_PE: u64 = 1;
-pub(crate) const CR0_MP: u64 = 1 << 1;
-pub(crate) const CR0_ET: u64 = 1 << 4;
-pub(crate) const CR0_NE: u64 = 1 << 5;
-pub(crate) const CR0_WP: u64 = 1 << 16;
-pub(crate) const CR0_AM: u64 = 1 << 18;
-pub(crate) const CR0_PG: u64 = 1 << 31;
-pub(crate) const EFER_LME: u64 = 1 << 8;
-pub(crate) const EFER_LMA: u64 = 1 << 10;
-pub(crate) const EFER_SCE: u64 = 1;
-pub(crate) const EFER_NX: u64 = 1 << 11;
+cfg_if::cfg_if! {
+    if #[cfg(feature = "init-paging")] {
+        pub(crate) const CR4_PAE: u64 = 1 << 5;
+        pub(crate) const CR4_OSFXSR: u64 = 1 << 9;
+        pub(crate) const CR4_OSXMMEXCPT: u64 = 1 << 10;
+        pub(crate) const CR0_PE: u64 = 1;
+        pub(crate) const CR0_MP: u64 = 1 << 1;
+        pub(crate) const CR0_ET: u64 = 1 << 4;
+        pub(crate) const CR0_NE: u64 = 1 << 5;
+        pub(crate) const CR0_WP: u64 = 1 << 16;
+        pub(crate) const CR0_AM: u64 = 1 << 18;
+        pub(crate) const CR0_PG: u64 = 1 << 31;
+        pub(crate) const EFER_LME: u64 = 1 << 8;
+        pub(crate) const EFER_LMA: u64 = 1 << 10;
+        pub(crate) const EFER_SCE: u64 = 1;
+        pub(crate) const EFER_NX: u64 = 1 << 11;
+    }
+}
 
 /// These are the generic exit reasons that we can handle from a Hypervisor the Hypervisors run method is responsible for mapping from
 /// the hypervisor specific exit reasons to these generic ones

src/hyperlight_host/src/mem/elf.rs

@@ -19,6 +19,9 @@ use goblin::elf::reloc::{R_AARCH64_NONE, R_AARCH64_RELATIVE};
 #[cfg(target_arch = "x86_64")]
 use goblin::elf::reloc::{R_X86_64_NONE, R_X86_64_RELATIVE};
 use goblin::elf::{Elf, ProgramHeaders, Reloc};
+#[cfg(not(feature = "init-paging"))]
+use goblin::elf32::program_header::PT_LOAD;
+#[cfg(feature = "init-paging")]
 use goblin::elf64::program_header::PT_LOAD;
 
 use crate::{Result, log_then_return, new_error};