From 03f75ff7b11f065c269a311e2a27abdcc79fe03d Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Tue, 3 Aug 2021 21:18:29 +0000 Subject: [PATCH 1/4] vfio-ioctls: add mshv feature gate and crates Signed-off-by: Wei Liu --- Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 28205c7..240ca43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ edition = "2018" [features] default = ["kvm"] kvm = ["kvm-ioctls", "kvm-bindings"] +mshv = ["mshv-ioctls", "mshv-bindings"] [dependencies] byteorder = ">=1.2.1" @@ -19,3 +20,5 @@ kvm-ioctls = { version = "~0", optional = true } vfio-bindings = "~0" vm-memory = { version = "0.6", features = ["backend-mmap"] } vmm-sys-util = "0.8" +mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional = true } +mshv-ioctls = { git = "https://github.com/rust-vmm/mshv", branch = "main", optional = true } From d3e3905c3dd6ce4b5611ff0ab8eee972aea724f5 Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Tue, 3 Aug 2021 21:28:58 +0000 Subject: [PATCH 2/4] vfio_device: add MSHV support Rename KvmSetDeviceAttr to SetDeviceAttr and modify private helper functions to work with both hypervisors. Signed-off-by: Wei Liu --- src/vfio_device.rs | 54 ++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/src/vfio_device.rs b/src/vfio_device.rs index 1e8f0e6..a2f5ac3 100644 --- a/src/vfio_device.rs +++ b/src/vfio_device.rs @@ -22,6 +22,12 @@ use kvm_bindings::{ #[cfg(feature = "kvm")] use kvm_ioctls::DeviceFd; use log::{debug, error, warn}; +#[cfg(feature = "mshv")] +use mshv_bindings::{ + mshv_device_attr, MSHV_DEV_VFIO_GROUP, MSHV_DEV_VFIO_GROUP_ADD, MSHV_DEV_VFIO_GROUP_DEL, +}; +#[cfg(feature = "mshv")] +use mshv_ioctls::DeviceFd; use vfio_bindings::bindings::vfio::*; use vm_memory::{Address, GuestMemory, GuestMemoryRegion, MemoryRegionAddress}; use vmm_sys_util::errno::Error as SysError; @@ -46,8 +52,7 @@ pub enum VfioError { UnsetContainer, ContainerSetIOMMU, GroupGetDeviceFD, - #[cfg(feature = "kvm")] - KvmSetDeviceAttr(SysError), + SetDeviceAttr(SysError), VfioDeviceGetInfo, VfioDeviceGetRegionInfo(SysError), InvalidPath, @@ -87,9 +92,8 @@ impl fmt::Display for VfioError { "failed to set container's IOMMU driver type as VfioType1V2" ), VfioError::GroupGetDeviceFD => write!(f, "failed to get vfio device fd"), - #[cfg(feature = "kvm")] - VfioError::KvmSetDeviceAttr(e) => { - write!(f, "failed to set KVM vfio device's attribute: {}", e) + VfioError::SetDeviceAttr(e) => { + write!(f, "failed to set vfio device's attribute: {}", e) } VfioError::VfioDeviceGetInfo => { write!(f, "failed to get vfio device's info or info doesn't match") @@ -123,8 +127,7 @@ impl std::error::Error for VfioError { VfioError::UnsetContainer => None, VfioError::ContainerSetIOMMU => None, VfioError::GroupGetDeviceFD => None, - #[cfg(feature = "kvm")] - VfioError::KvmSetDeviceAttr(e) => Some(e), + VfioError::SetDeviceAttr(e) => Some(e), VfioError::VfioDeviceGetInfo => None, VfioError::VfioDeviceGetRegionInfo(e) => Some(e), VfioError::InvalidPath => None, @@ -229,9 +232,10 @@ impl VfioContainer { Ok(()) } - #[cfg(feature = "kvm")] - fn kvm_device_add_group(&self, group_fd: RawFd) -> Result<()> { + fn device_add_group(&self, group_fd: RawFd) -> Result<()> { let group_fd_ptr = &group_fd as *const i32; + + #[cfg(feature = "kvm")] let dev_attr = kvm_device_attr { flags: 0, group: KVM_DEV_VFIO_GROUP, @@ -239,14 +243,22 @@ impl VfioContainer { addr: group_fd_ptr as u64, }; + #[cfg(feature = "mshv")] + let dev_attr = mshv_device_attr { + flags: 0, + group: MSHV_DEV_VFIO_GROUP, + attr: u64::from(MSHV_DEV_VFIO_GROUP_ADD), + addr: group_fd_ptr as u64, + }; + self.device_fd .set_device_attr(&dev_attr) - .map_err(VfioError::KvmSetDeviceAttr) + .map_err(VfioError::SetDeviceAttr) } - #[cfg(feature = "kvm")] - fn kvm_device_del_group(&self, group_fd: RawFd) -> Result<()> { + fn device_del_group(&self, group_fd: RawFd) -> Result<()> { let group_fd_ptr = &group_fd as *const i32; + #[cfg(feature = "kvm")] let dev_attr = kvm_device_attr { flags: 0, group: KVM_DEV_VFIO_GROUP, @@ -254,9 +266,17 @@ impl VfioContainer { addr: group_fd_ptr as u64, }; + #[cfg(feature = "mshv")] + let dev_attr = mshv_device_attr { + flags: 0, + group: MSHV_DEV_VFIO_GROUP, + attr: u64::from(MSHV_DEV_VFIO_GROUP_DEL), + addr: group_fd_ptr as u64, + }; + self.device_fd .set_device_attr(&dev_attr) - .map_err(VfioError::KvmSetDeviceAttr) + .map_err(VfioError::SetDeviceAttr) } fn get_group(&self, group_id: u32) -> Result> { @@ -287,9 +307,8 @@ impl VfioContainer { } } - // Add the new group object to the KVM driver. - #[cfg(feature = "kvm")] - if let Err(e) = self.kvm_device_add_group(group.as_raw_fd()) { + // Add the new group object to the hypervisor driver. + if let Err(e) = self.device_add_group(group.as_raw_fd()) { let _ = unsafe { ioctl_with_ref(&*group, VFIO_GROUP_UNSET_CONTAINER(), &self.as_raw_fd()) }; return Err(e); @@ -310,8 +329,7 @@ impl VfioContainer { // - one reference cloned in VfioDevice.drop() and passed into here // - one reference held by the groups hashmap if Arc::strong_count(&group) == 3 { - #[cfg(feature = "kvm")] - match self.kvm_device_del_group(group.as_raw_fd()) { + match self.device_del_group(group.as_raw_fd()) { Ok(_) => {} Err(e) => { error!("Could not delete VFIO group: {:?}", e); From b1ea4b36dc77dc5787df15986eebc7ade1694416 Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Mon, 6 Sep 2021 15:48:18 +0000 Subject: [PATCH 3/4] vfio_device: remove RawFd from function parameters Generally speaking functions take RawFd should be marked as unsafe. device_add_group and device_del group are such functions. Instead of marks them as unsafe, we can make these two functions take a reference to VfioGroup. This is far more elegant. Signed-off-by: Wei Liu --- src/vfio_device.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/vfio_device.rs b/src/vfio_device.rs index a2f5ac3..d8186d3 100644 --- a/src/vfio_device.rs +++ b/src/vfio_device.rs @@ -232,8 +232,8 @@ impl VfioContainer { Ok(()) } - fn device_add_group(&self, group_fd: RawFd) -> Result<()> { - let group_fd_ptr = &group_fd as *const i32; + fn device_add_group(&self, group: &VfioGroup) -> Result<()> { + let group_fd_ptr = &group.as_raw_fd() as *const i32; #[cfg(feature = "kvm")] let dev_attr = kvm_device_attr { @@ -256,8 +256,8 @@ impl VfioContainer { .map_err(VfioError::SetDeviceAttr) } - fn device_del_group(&self, group_fd: RawFd) -> Result<()> { - let group_fd_ptr = &group_fd as *const i32; + fn device_del_group(&self, group: &VfioGroup) -> Result<()> { + let group_fd_ptr = &group.as_raw_fd() as *const i32; #[cfg(feature = "kvm")] let dev_attr = kvm_device_attr { flags: 0, @@ -308,7 +308,7 @@ impl VfioContainer { } // Add the new group object to the hypervisor driver. - if let Err(e) = self.device_add_group(group.as_raw_fd()) { + if let Err(e) = self.device_add_group(&group) { let _ = unsafe { ioctl_with_ref(&*group, VFIO_GROUP_UNSET_CONTAINER(), &self.as_raw_fd()) }; return Err(e); @@ -329,7 +329,7 @@ impl VfioContainer { // - one reference cloned in VfioDevice.drop() and passed into here // - one reference held by the groups hashmap if Arc::strong_count(&group) == 3 { - match self.device_del_group(group.as_raw_fd()) { + match self.device_del_group(&group) { Ok(_) => {} Err(e) => { error!("Could not delete VFIO group: {:?}", e); From 9d6916f1a969df1967142c183c347c150ab9e7b0 Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Mon, 6 Sep 2021 16:45:40 +0000 Subject: [PATCH 4/4] vfio_device: make buildkite happy Rust does not support mutually exclusive features. Buildkite enables all features. These two factors combined causes pipeline failures. Turn the feature gates in code a bit. Whenever "kvm" is specify, surface KVM's definitions. Make available MSHV if and only if "mshv" is specified. Add custom tests for MSHV. Signed-off-by: Wei Liu --- .buildkite/custom-tests.json | 14 ++++++++++++++ src/vfio_device.rs | 8 ++++---- 2 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 .buildkite/custom-tests.json diff --git a/.buildkite/custom-tests.json b/.buildkite/custom-tests.json new file mode 100644 index 0000000..f67bf1f --- /dev/null +++ b/.buildkite/custom-tests.json @@ -0,0 +1,14 @@ +{ + "tests": [ + { + "test_name": "build-mshv", + "command": "cargo build --release --no-default-features --features mshv", + "platform": ["x86_64"] + }, + { + "test_name": "clippy-mshv", + "command": "cargo clippy --workspace --bins --examples --benches --no-default-features --features mshv --all-targets -- -D warnings", + "platform": ["x86_64"] + } + ] +} diff --git a/src/vfio_device.rs b/src/vfio_device.rs index d8186d3..e72c5e4 100644 --- a/src/vfio_device.rs +++ b/src/vfio_device.rs @@ -22,11 +22,11 @@ use kvm_bindings::{ #[cfg(feature = "kvm")] use kvm_ioctls::DeviceFd; use log::{debug, error, warn}; -#[cfg(feature = "mshv")] +#[cfg(all(feature = "mshv", not(feature = "kvm")))] use mshv_bindings::{ mshv_device_attr, MSHV_DEV_VFIO_GROUP, MSHV_DEV_VFIO_GROUP_ADD, MSHV_DEV_VFIO_GROUP_DEL, }; -#[cfg(feature = "mshv")] +#[cfg(all(feature = "mshv", not(feature = "kvm")))] use mshv_ioctls::DeviceFd; use vfio_bindings::bindings::vfio::*; use vm_memory::{Address, GuestMemory, GuestMemoryRegion, MemoryRegionAddress}; @@ -243,7 +243,7 @@ impl VfioContainer { addr: group_fd_ptr as u64, }; - #[cfg(feature = "mshv")] + #[cfg(all(feature = "mshv", not(feature = "kvm")))] let dev_attr = mshv_device_attr { flags: 0, group: MSHV_DEV_VFIO_GROUP, @@ -266,7 +266,7 @@ impl VfioContainer { addr: group_fd_ptr as u64, }; - #[cfg(feature = "mshv")] + #[cfg(all(feature = "mshv", not(feature = "kvm")))] let dev_attr = mshv_device_attr { flags: 0, group: MSHV_DEV_VFIO_GROUP,