Add implementation of adding vCPUs to VMM

Actually add the vCPUs to the VMM on API request. Must add at least 1
vCPU, and the total number of vCPUs after addition must be less than
MAX_SUPPORTED_VCPUS.

Signed-off-by: James Curtis <jxcurtis@amazon.co.uk>

src/firecracker/src/api_server/request/hotplug.rs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 use vmm::logger::{IncMetric, METRICS};
 use vmm::rpc_interface::VmmAction;
-use vmm::vmm_config::hotplug::{HotplugVcpuConfig, HotplugVcpuError};
+use vmm::vmm_config::hotplug::HotplugVcpuConfig;
 
 use super::super::parsed_request::{ParsedRequest, RequestError};
 use super::{Body, Method, StatusCode};
@@ -41,11 +41,12 @@ fn parse_put_vcpu_hotplug(body: &Body) -> Result<ParsedRequest, RequestError> {
 #[cfg(test)]
 mod tests {
 
-    use hotplug::{parse_put_hotplug, parse_put_vcpu_hotplug};
-    use vmm::{logger::{IncMetric, METRICS}, rpc_interface::VmmAction, vmm_config::hotplug::{HotplugVcpuConfig, HotplugVcpuError}};
-    use crate::api_server::parsed_request::tests::vmm_action_from_request;
+    use hotplug::parse_put_hotplug;
+    use vmm::rpc_interface::VmmAction;
+    use vmm::vmm_config::hotplug::HotplugVcpuConfig;
 
     use super::super::*;
+    use crate::api_server::parsed_request::tests::vmm_action_from_request;
 
     #[test]
     fn test_parse_put_hotplug() {
@@ -60,15 +61,11 @@ mod tests {
             "vcpu_count": 4
         }"#;
 
-        let expected_config = HotplugVcpuConfig {
-            vcpu_count: 4
-        };
+        let expected_config = HotplugVcpuConfig { vcpu_count: 4 };
 
         assert_eq!(
             vmm_action_from_request(parse_put_hotplug(&Body::new(body), Some("vcpu")).unwrap()),
             VmmAction::HotplugVcpu(expected_config)
         );
     }
-
-
 }

src/vmm/src/builder.rs

@@ -145,13 +145,15 @@ impl std::convert::From<linux_loader::cmdline::Error> for StartMicrovmError {
 }
 
 #[cfg_attr(target_arch = "aarch64", allow(unused))]
+#[allow(clippy::too_many_arguments)]
 fn create_vmm_and_vcpus(
     instance_info: &InstanceInfo,
     event_manager: &mut EventManager,
     guest_memory: GuestMemoryMmap,
     uffd: Option<Uffd>,
     track_dirty_pages: bool,
     vcpu_count: u8,
+    seccomp_filters: BpfThreadMap,
     kvm_capabilities: Vec<KvmCapability>,
 ) -> Result<(Vmm, Vec<Vcpu>), StartMicrovmError> {
     use self::StartMicrovmError::*;
@@ -229,6 +231,7 @@ fn create_vmm_and_vcpus(
         uffd,
         vcpus_handles: Vec::new(),
         vcpus_exit_evt,
+        seccomp_filters,
         resource_allocator,
         mmio_device_manager,
         #[cfg(target_arch = "x86_64")]
@@ -309,6 +312,7 @@ pub fn build_microvm_for_boot(
         None,
         track_dirty_pages,
         vm_resources.vm_config.vcpu_count,
+        seccomp_filters.clone(),
         cpu_template.kvm_capabilities.clone(),
     )?;
 
@@ -478,6 +482,7 @@ pub fn build_microvm_from_snapshot(
         uffd,
         vm_resources.vm_config.track_dirty_pages,
         vm_resources.vm_config.vcpu_count,
+        seccomp_filters.clone(),
         microvm_state.vm_state.kvm_cap_modifiers.clone(),
     )?;
 
@@ -1155,6 +1160,7 @@ pub mod tests {
             uffd: None,
             vcpus_handles: Vec::new(),
             vcpus_exit_evt,
+            seccomp_filters: crate::seccomp_filters::get_empty_filters(),
             resource_allocator: ResourceAllocator::new().unwrap(),
             mmio_device_manager,
             #[cfg(target_arch = "x86_64")]

src/vmm/src/lib.rs

@@ -123,12 +123,14 @@ use device_manager::resources::ResourceAllocator;
 #[cfg(target_arch = "x86_64")]
 use devices::acpi::vmgenid::VmGenIdError;
 use event_manager::{EventManager as BaseEventManager, EventOps, Events, MutEventSubscriber};
-use seccompiler::BpfProgram;
+use seccompiler::{BpfProgram, BpfThreadMap};
 use userfaultfd::Uffd;
 use utils::epoll::EventSet;
 use utils::eventfd::EventFd;
 use utils::terminal::Terminal;
 use utils::u64_to_usize;
+use vmm_config::hotplug::{HotplugVcpuConfig, HotplugVcpuError};
+use vmm_config::machine_config::MAX_SUPPORTED_VCPUS;
 use vstate::vcpu::{self, KvmVcpuConfigureError, StartThreadedError, VcpuSendEventError};
 
 use crate::arch::DeviceType;
@@ -143,7 +145,7 @@ use crate::devices::virtio::balloon::{
 use crate::devices::virtio::block::device::Block;
 use crate::devices::virtio::net::Net;
 use crate::devices::virtio::{TYPE_BALLOON, TYPE_BLOCK, TYPE_NET};
-use crate::logger::{error, info, warn, MetricsError, METRICS};
+use crate::logger::{error, info, warn, IncMetric, MetricsError, METRICS};
 use crate::persist::{MicrovmState, MicrovmStateError, VmInfo};
 use crate::rate_limiter::BucketUpdate;
 use crate::snapshot::Persist;
@@ -317,6 +319,7 @@ pub struct Vmm {
     vcpus_handles: Vec<VcpuHandle>,
     // Used by Vcpus and devices to initiate teardown; Vmm should never write here.
     vcpus_exit_evt: EventFd,
+    seccomp_filters: BpfThreadMap,
 
     // Allocator for guest resrouces
     resource_allocator: ResourceAllocator,
@@ -600,6 +603,50 @@ impl Vmm {
         Ok(cpu_configs)
     }
 
+    /// Adds new vCPUs to VMM.
+    pub fn hotplug_vcpus(&mut self, config: HotplugVcpuConfig) -> Result<(), HotplugVcpuError> {
+        #[allow(clippy::cast_possible_truncation)]
+        if config.vcpu_count < 1 {
+            return Err(HotplugVcpuError::VcpuCountTooLow);
+        } else if self.vcpus_handles.len() as u8 + config.vcpu_count > MAX_SUPPORTED_VCPUS {
+            return Err(HotplugVcpuError::VcpuCountTooHigh);
+        }
+
+        // Create and start new vcpus
+        let mut vcpus = Vec::with_capacity(config.vcpu_count as usize);
+
+        #[allow(clippy::cast_possible_truncation)]
+        let start_idx = self.vcpus_handles.len() as u8;
+        for cpu_idx in start_idx..(start_idx + config.vcpu_count) {
+            let exit_evt = self
+                .vcpus_exit_evt
+                .try_clone()
+                .map_err(HotplugVcpuError::EventFd)?;
+            let vcpu =
+                Vcpu::new(cpu_idx, &self.vm, exit_evt).map_err(HotplugVcpuError::VcpuCreate)?;
+            vcpus.push(vcpu);
+        }
+
+        self.start_vcpus(
+            vcpus,
+            self.seccomp_filters
+                .get("vcpu")
+                .ok_or_else(|| HotplugVcpuError::MissingSeccompFilters("vcpu".to_string()))?
+                .clone(),
+        )
+        .map_err(HotplugVcpuError::VcpuStart)?;
+
+        #[allow(clippy::cast_lossless)]
+        METRICS
+            .hotplug
+            .vcpu_hotplug_count
+            .add(config.vcpu_count as u64);
+
+        // Update VM config to reflect new CPUs added
+
+        Ok(())
+    }
+
     /// Retrieves the KVM dirty bitmap for each of the guest's memory regions.
     pub fn reset_dirty_bitmap(&self) {
         self.guest_memory

src/vmm/src/logger/metrics.rs

@@ -835,6 +835,7 @@ impl VmmMetrics {
 pub struct HotplugMetrics {
     pub vcpu_hotplug_request_count: SharedIncMetric,
     pub vcpu_hotplug_request_fails: SharedIncMetric,
+    pub vcpu_hotplug_count: SharedIncMetric,
 }
 
 impl HotplugMetrics {
@@ -844,6 +845,7 @@ impl HotplugMetrics {
         Self {
             vcpu_hotplug_request_count: SharedIncMetric::new(),
             vcpu_hotplug_request_fails: SharedIncMetric::new(),
+            vcpu_hotplug_count: SharedIncMetric::new(),
         }
     }
 }

src/vmm/src/rpc_interface.rs

@@ -663,7 +663,13 @@ impl RuntimeApiController {
             GetVmmVersion => Ok(VmmData::VmmVersion(
                 self.vmm.lock().expect("Poisoned lock").version(),
             )),
-            HotplugVcpu(config) => Ok(VmmData::Empty),
+            HotplugVcpu(config) => self
+                .vmm
+                .lock()
+                .expect("Poisoned lock")
+                .hotplug_vcpus(config)
+                .map(|_| VmmData::Empty)
+                .map_err(VmmActionError::HotplugVcpu),
             PatchMMDS(value) => self.patch_mmds(value),
             Pause => self.pause(),
             PutMMDS(value) => self.put_mmds(value),
@@ -868,6 +874,7 @@ mod tests {
     use std::path::PathBuf;
 
     use seccompiler::BpfThreadMap;
+    use vmm_config::hotplug::HotplugVcpuError;
 
     use super::*;
     use crate::cpu_config::templates::test_utils::build_test_template;
@@ -892,6 +899,7 @@ mod tests {
                     | (BootSource(_), BootSource(_))
                     | (CreateSnapshot(_), CreateSnapshot(_))
                     | (DriveConfig(_), DriveConfig(_))
+                    | (HotplugVcpu(_), HotplugVcpu(_))
                     | (InternalVmm(_), InternalVmm(_))
                     | (LoadSnapshot(_), LoadSnapshot(_))
                     | (MachineConfig(_), MachineConfig(_))
@@ -1099,6 +1107,7 @@ mod tests {
         pub update_block_device_path_called: bool,
         pub update_block_device_vhost_user_config_called: bool,
         pub update_net_rate_limiters_called: bool,
+        pub hotplug_vcpus_called: bool,
         // when `true`, all self methods are forced to fail
         pub force_errors: bool,
     }
@@ -1209,6 +1218,13 @@ mod tests {
             Ok(())
         }
 
+        pub fn hotplug_vcpus(&mut self, _: HotplugVcpuConfig) -> Result<(), HotplugVcpuError> {
+            if self.force_errors {
+                return Err(HotplugVcpuError::VcpuCountTooHigh);
+            }
+            self.hotplug_vcpus_called = true;
+            Ok(())
+        }
         pub fn instance_info(&self) -> InstanceInfo {
             InstanceInfo::default()
         }
@@ -1823,6 +1839,12 @@ mod tests {
             VmmAction::SendCtrlAltDel,
             VmmActionError::OperationNotSupportedPreBoot,
         );
+
+        #[cfg(target_arch = "x86_64")]
+        check_preboot_request_err(
+            VmmAction::HotplugVcpu(HotplugVcpuConfig { vcpu_count: 4 }),
+            VmmActionError::OperationNotSupportedPreBoot,
+        );
     }
 
     fn check_runtime_request<F>(request: VmmAction, check_success: F)
@@ -2052,6 +2074,27 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_runtime_hotplug_vcpu() {
+        let req = VmmAction::HotplugVcpu(HotplugVcpuConfig { vcpu_count: 4 });
+        check_runtime_request(req, |result, vmm| {
+            assert_eq!(result, Ok(VmmData::Empty));
+            assert!(vmm.hotplug_vcpus_called)
+        });
+
+        let req = VmmAction::HotplugVcpu(HotplugVcpuConfig { vcpu_count: 0 });
+        check_runtime_request_err(
+            req,
+            VmmActionError::HotplugVcpu(HotplugVcpuError::VcpuCountTooLow),
+        );
+
+        let req = VmmAction::HotplugVcpu(HotplugVcpuConfig { vcpu_count: 33 });
+        check_runtime_request_err(
+            req,
+            VmmActionError::HotplugVcpu(HotplugVcpuError::VcpuCountTooHigh),
+        );
+    }
+
     #[test]
     fn test_runtime_disallowed() {
         check_runtime_request_err(

src/vmm/src/vmm_config/hotplug.rs

@@ -1,19 +1,20 @@
 // Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-use serde::{Deserialize, Serialize};
 use std::io;
+
+use serde::{Deserialize, Serialize};
+
 use crate::vstate::vcpu::VcpuError;
 use crate::StartVcpusError;
 
 /// Firecracker aims to support small scale workloads only, so limit the maximum
 /// vCPUs supported.
-pub const MAX_SUPPORTED_VCPUS: u8 = 32;
 
 /// Errors associated with hot-plugging vCPUs
 #[derive(Debug, thiserror::Error, displaydoc::Display)]
 pub enum HotplugVcpuError {
-    /// The number of vCPUs added must be greater than 0. 
+    /// The number of vCPUs added must be greater than 0.
     VcpuCountTooLow,
     /// The number of vCPUs added must be less than 32.
     VcpuCountTooHigh,
@@ -23,6 +24,8 @@ pub enum HotplugVcpuError {
     VcpuCreate(VcpuError),
     /// Failed to start vCPUs
     VcpuStart(StartVcpusError),
+    /// No seccomp filter for thread category: {0}
+    MissingSeccompFilters(String),
 }
 
 /// Config for hotplugging vCPUS

tests/host_tools/fcmetrics.py

@@ -152,6 +152,11 @@ def validate_fc_metrics(metrics):
             "mmds_count",
             "vmm_version_count",
         ],
+        "hotplug": [
+            "vcpu_hotplug_request_count",
+            "vcpu_hotplug_request_fails",
+            "vcpu_hotplug_count",
+        ],
         "i8042": [
             "error_count",
             "missed_read_count",
@@ -209,6 +214,8 @@ def validate_fc_metrics(metrics):
             "boot_source_fails",
             "drive_count",
             "drive_fails",
+            "hotplug",
+            "hotplug_fails",
             "logger_count",
             "logger_fails",
             "machine_cfg_count",

tools/create_snapshot_artifact/main.py

@@ -24,7 +24,6 @@
 from framework.utils_cpuid import CpuVendor, get_cpu_vendor
 from host_tools.cargo_build import get_firecracker_binaries
 
-
 # pylint: enable=wrong-import-position
 
 # Default IPv4 address to route MMDS requests.