feat: Add ability to resume snapshots and write back changes to the b…

…acking file continously, add API endpoint to `msync` the files backing snapshots

resources/seccomp/aarch64-unknown-linux-musl.json

@@ -35,6 +35,10 @@
             {
                 "syscall": "fsync"
             },
+            {
+                "syscall": "msync",
+                "comment": "Used for live migration to sync dirty pages"
+            },
             {
                 "syscall": "close"
             },

resources/seccomp/x86_64-unknown-linux-musl.json

@@ -35,6 +35,10 @@
             {
                 "syscall": "fsync"
             },
+            {
+                "syscall": "msync",
+                "comment": "Used for live migration to sync dirty pages"
+            },
             {
                 "syscall": "close"
             },

src/firecracker/src/api_server/parsed_request.rs

@@ -23,7 +23,9 @@ use super::request::machine_configuration::{
 use super::request::metrics::parse_put_metrics;
 use super::request::mmds::{parse_get_mmds, parse_patch_mmds, parse_put_mmds};
 use super::request::net::{parse_patch_net, parse_put_net};
-use super::request::snapshot::{parse_patch_vm_state, parse_put_snapshot};
+use super::request::snapshot::{
+    parse_patch_vm_state, parse_put_snapshot, parse_put_snapshot_nomemory,
+};
 use super::request::version::parse_get_version;
 use super::request::vsock::parse_put_vsock;
 use super::ApiServer;
@@ -97,6 +99,9 @@ impl TryFrom<&Request> for ParsedRequest {
                 parse_put_net(body, path_tokens.next())
             }
             (Method::Put, "snapshot", Some(body)) => parse_put_snapshot(body, path_tokens.next()),
+            (Method::Put, "snapshot-nomemory", Some(body)) => {
+                parse_put_snapshot_nomemory(body, path_tokens.next())
+            }
             (Method::Put, "vsock", Some(body)) => parse_put_vsock(body),
             (Method::Put, "entropy", Some(body)) => parse_put_entropy(body),
             (Method::Put, _, None) => method_to_error(Method::Put),

src/firecracker/src/api_server/request/snapshot.rs

@@ -5,8 +5,8 @@ use serde::de::Error as DeserializeError;
 use vmm::logger::{IncMetric, METRICS};
 use vmm::rpc_interface::VmmAction;
 use vmm::vmm_config::snapshot::{
-    CreateSnapshotParams, LoadSnapshotConfig, LoadSnapshotParams, MemBackendConfig, MemBackendType,
-    Vm, VmState,
+    CreateSnapshotNoMemoryParams, CreateSnapshotParams, LoadSnapshotConfig, LoadSnapshotParams,
+    MemBackendConfig, MemBackendType, Vm, VmState,
 };
 
 use super::super::parsed_request::{ParsedRequest, RequestError};
@@ -42,6 +42,27 @@ pub(crate) fn parse_put_snapshot(
     }
 }
 
+pub(crate) fn parse_put_snapshot_nomemory(
+    body: &Body,
+    request_type_from_path: Option<&str>,
+) -> Result<ParsedRequest, RequestError> {
+    match request_type_from_path {
+        Some(request_type) => match request_type {
+            "create" => Ok(ParsedRequest::new_sync(VmmAction::CreateSnapshotNoMemory(
+                serde_json::from_slice::<CreateSnapshotNoMemoryParams>(body.raw())?,
+            ))),
+            _ => Err(RequestError::InvalidPathMethod(
+                format!("/snapshot-nomemory/{}", request_type),
+                Method::Put,
+            )),
+        },
+        None => Err(RequestError::Generic(
+            StatusCode::BadRequest,
+            "Missing snapshot operation type.".to_string(),
+        )),
+    }
+}
+
 pub(crate) fn parse_patch_vm_state(body: &Body) -> Result<ParsedRequest, RequestError> {
     let vm = serde_json::from_slice::<Vm>(body.raw())?;
 

src/firecracker/swagger/firecracker.yaml

@@ -591,6 +591,32 @@ paths:
           schema:
             $ref: "#/definitions/Error"
 
+  /snapshot-nomemory/create:
+    put:
+      summary: Creates a full snapshot without memory. Post-boot only.
+      description:
+        Creates a snapshot of the microVM state, without memory. The microVM should be
+        in the `Paused` state.
+      operationId: createSnapshotNoMemory
+      parameters:
+        - name: body
+          in: body
+          description: The configuration used for creating a snaphot.
+          required: true
+          schema:
+            $ref: "#/definitions/SnapshotCreateNoMemoryParams"
+      responses:
+        204:
+          description: Snapshot created
+        400:
+          description: Snapshot cannot be created due to bad input
+          schema:
+            $ref: "#/definitions/Error"
+        default:
+          description: Internal server error
+          schema:
+            $ref: "#/definitions/Error"
+
   /snapshot/load:
     put:
       summary: Loads a snapshot. Pre-boot only.
@@ -1203,6 +1229,15 @@ definitions:
           Type of snapshot to create. It is optional and by default, a full
           snapshot is created.
 
+  SnapshotCreateNoMemoryParams:
+    type: object
+    required:
+      - snapshot_path
+    properties:
+      snapshot_path:
+        type: string
+        description: Path to the file that will contain the microVM state.
+
   SnapshotLoadParams:
     type: object
     description:

src/vmm/src/persist.rs

@@ -36,7 +36,8 @@ use crate::vmm_config::boot_source::BootSourceConfig;
 use crate::vmm_config::instance_info::InstanceInfo;
 use crate::vmm_config::machine_config::{HugePageConfig, MachineConfigUpdate, VmConfigError};
 use crate::vmm_config::snapshot::{
-    CreateSnapshotParams, LoadSnapshotParams, MemBackendType, SnapshotType,
+    CreateSnapshotNoMemoryParams, CreateSnapshotParams, LoadSnapshotParams, MemBackendType,
+    SnapshotType,
 };
 use crate::vstate::memory::{
     GuestMemory, GuestMemoryExtension, GuestMemoryMmap, GuestMemoryState, MemoryError,
@@ -153,6 +154,8 @@ pub enum CreateSnapshotError {
     MicrovmState(MicrovmStateError),
     /// Cannot serialize the microVM state: {0}
     SerializeMicrovmState(crate::snapshot::SnapshotError),
+    /// Failed to sync Microvm memory.
+    MemoryMsync(MemoryError),
     /// Cannot perform {0} on the snapshot backing file: {1}
     SnapshotBackingFile(&'static str, io::Error),
     /// Size mismatch when writing diff snapshot on top of base layer: base layer size is {0} but diff layer is size {1}.
@@ -179,6 +182,25 @@ pub fn create_snapshot(
     Ok(())
 }
 
+/// Creates a Microvm snapshot without memory.
+pub fn create_snapshot_nomemory(
+    vmm: &mut Vmm,
+    vm_info: &VmInfo,
+    params: &CreateSnapshotNoMemoryParams,
+) -> std::result::Result<(), CreateSnapshotError> {
+    let microvm_state = vmm
+        .save_state(vm_info)
+        .map_err(CreateSnapshotError::MicrovmState)?;
+
+    snapshot_state_to_file(&microvm_state, &params.snapshot_path)?;
+
+    vmm.guest_memory()
+        .msync()
+        .map_err(CreateSnapshotError::MemoryMsync)?;
+
+    Ok(())
+}
+
 fn snapshot_state_to_file(
     microvm_state: &MicrovmState,
     snapshot_path: &Path,
@@ -501,7 +523,10 @@ fn guest_memory_from_file(
     track_dirty_pages: bool,
     huge_pages: HugePageConfig,
 ) -> Result<GuestMemoryMmap, GuestMemoryFromFileError> {
-    let mem_file = File::open(mem_file_path)?;
+    let mem_file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(mem_file_path)?;
     let guest_mem =
         GuestMemoryMmap::from_state(Some(&mem_file), mem_state, track_dirty_pages, huge_pages)?;
     Ok(guest_mem)

src/vmm/src/rpc_interface.rs

@@ -8,15 +8,15 @@ use seccompiler::BpfThreadMap;
 use serde_json::Value;
 #[cfg(test)]
 use tests::{
-    build_and_boot_microvm, create_snapshot, restore_from_snapshot, MockVmRes as VmResources,
-    MockVmm as Vmm,
+    build_and_boot_microvm, create_snapshot, create_snapshot_nomemory, restore_from_snapshot,
+    MockVmRes as VmResources, MockVmm as Vmm,
 };
 
 use super::VmmError;
 #[cfg(not(test))]
 use super::{
-    builder::build_and_boot_microvm, persist::create_snapshot, persist::restore_from_snapshot,
-    resources::VmResources, Vmm,
+    builder::build_and_boot_microvm, persist::create_snapshot, persist::create_snapshot_nomemory,
+    persist::restore_from_snapshot, resources::VmResources, Vmm,
 };
 use crate::builder::StartMicrovmError;
 use crate::cpu_config::templates::{CustomCpuTemplate, GuestConfigError};
@@ -38,7 +38,9 @@ use crate::vmm_config::mmds::{MmdsConfig, MmdsConfigError};
 use crate::vmm_config::net::{
     NetworkInterfaceConfig, NetworkInterfaceError, NetworkInterfaceUpdateConfig,
 };
-use crate::vmm_config::snapshot::{CreateSnapshotParams, LoadSnapshotParams, SnapshotType};
+use crate::vmm_config::snapshot::{
+    CreateSnapshotNoMemoryParams, CreateSnapshotParams, LoadSnapshotParams, SnapshotType,
+};
 use crate::vmm_config::vsock::{VsockConfigError, VsockDeviceConfig};
 use crate::vmm_config::{self, RateLimiterUpdate};
 use crate::EventManager;
@@ -59,6 +61,9 @@ pub enum VmmAction {
     /// Create a snapshot using as input the `CreateSnapshotParams`. This action can only be called
     /// after the microVM has booted and only when the microVM is in `Paused` state.
     CreateSnapshot(CreateSnapshotParams),
+    /// Create a snapshot without memory using as input the `CreateSnapshotNoMemoryParams`. This action can only be called
+    /// after the microVM has booted and only when the microVM is in `Paused` state.
+    CreateSnapshotNoMemory(CreateSnapshotNoMemoryParams),
     /// Get the balloon device configuration.
     GetBalloonConfig,
     /// Get the ballon device latest statistics.
@@ -443,6 +448,7 @@ impl<'a> PrebootApiController<'a> {
             SetEntropyDevice(config) => self.set_entropy_device(config),
             // Operations not allowed pre-boot.
             CreateSnapshot(_)
+            | CreateSnapshotNoMemory(_)
             | FlushMetrics
             | Pause
             | Resume
@@ -631,6 +637,9 @@ impl RuntimeApiController {
         match request {
             // Supported operations allowed post-boot.
             CreateSnapshot(snapshot_create_cfg) => self.create_snapshot(&snapshot_create_cfg),
+            CreateSnapshotNoMemory(snapshot_create_cfg) => {
+                self.create_snapshot_nomemory(&snapshot_create_cfg)
+            }
             FlushMetrics => self.flush_metrics(),
             GetBalloonConfig => self
                 .vmm
@@ -799,6 +808,30 @@ impl RuntimeApiController {
         Ok(VmmData::Empty)
     }
 
+    fn create_snapshot_nomemory(
+        &mut self,
+        create_params: &CreateSnapshotNoMemoryParams,
+    ) -> Result<VmmData, VmmActionError> {
+        log_dev_preview_warning("Virtual machine snapshots without memory", None);
+
+        let mut locked_vmm = self.vmm.lock().unwrap();
+        let vm_info = VmInfo::from(&self.vm_resources);
+        let create_start_us = utils::time::get_time_us(utils::time::ClockType::Monotonic);
+
+        create_snapshot_nomemory(&mut locked_vmm, &vm_info, create_params)?;
+
+        let elapsed_time_us = update_metric_with_elapsed_time(
+            &METRICS.latencies_us.vmm_full_create_snapshot,
+            create_start_us,
+        );
+        info!(
+            "'create full snapshot without memory' VMM action took {} us.",
+            elapsed_time_us
+        );
+
+        Ok(VmmData::Empty)
+    }
+
     /// Updates block device properties:
     ///  - path of the host file backing the emulated block device, update the disk image on the
     ///    device and its virtio configuration
@@ -1232,6 +1265,16 @@ mod tests {
         Ok(())
     }
 
+    // Need to redefine this since the non-test one uses real Vmm
+    // instead of our mocks.
+    pub fn create_snapshot_nomemory(
+        _: &mut Vmm,
+        _: &VmInfo,
+        _: &CreateSnapshotNoMemoryParams,
+    ) -> Result<(), CreateSnapshotError> {
+        Ok(())
+    }
+
     // Need to redefine this since the non-test one uses real Vmm
     // instead of our mocks.
     pub fn restore_from_snapshot(

src/vmm/src/vmm_config/snapshot.rs

@@ -47,6 +47,14 @@ pub struct CreateSnapshotParams {
     pub mem_file_path: PathBuf,
 }
 
+/// Stores the configuration that will be used for creating a snapshot without memory.
+#[derive(Debug, PartialEq, Eq, Deserialize, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct CreateSnapshotNoMemoryParams {
+    /// Path to the file that will contain the microVM state.
+    pub snapshot_path: PathBuf,
+}
+
 /// Stores the configuration that will be used for loading a snapshot.
 #[derive(Debug, PartialEq, Eq)]
 pub struct LoadSnapshotParams {

src/vmm/src/vstate/memory.rs

@@ -89,6 +89,9 @@ where
         huge_pages: HugePageConfig,
     ) -> Result<Self, MemoryError>;
 
+    /// Flushes memory contents to disk.
+    fn msync(&self) -> std::result::Result<(), MemoryError>;
+
     /// Describes GuestMemoryMmap through a GuestMemoryState struct.
     fn describe(&self) -> GuestMemoryState;
 
@@ -246,7 +249,7 @@ impl GuestMemoryExtension for GuestMemoryMmap {
                     .collect::<Result<Vec<_>, std::io::Error>>()
                     .map_err(MemoryError::FileError)?;
 
-                Self::from_raw_regions_file(regions, track_dirty_pages, false)
+                Self::from_raw_regions_file(regions, track_dirty_pages, true)
             }
             None => {
                 let regions = state
@@ -259,6 +262,20 @@ impl GuestMemoryExtension for GuestMemoryMmap {
         }
     }
 
+    /// Flushes memory contents to disk.
+    fn msync(&self) -> std::result::Result<(), MemoryError> {
+        Ok(self.iter().for_each(|region| {
+            // TODO: Describe safety aspects of this
+            unsafe {
+                libc::msync(
+                    region.as_ptr() as *mut libc::c_void,
+                    region.size(),
+                    libc::MS_SYNC,
+                );
+            }
+        }))
+    }
+
     /// Describes GuestMemoryMmap through a GuestMemoryState struct.
     fn describe(&self) -> GuestMemoryState {
         let mut guest_memory_state = GuestMemoryState::default();