From 7747700a17110d8e15ecf0c3eb4f39b5e80235a9 Mon Sep 17 00:00:00 2001 From: Enes Cakir Date: Wed, 31 Jan 2024 14:25:04 +0300 Subject: [PATCH 1/5] Exit early if the image already exists It's just a structural refactor. No need extra indentation. --- rhizome/host/lib/vm_setup.rb | 68 ++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/rhizome/host/lib/vm_setup.rb b/rhizome/host/lib/vm_setup.rb index 83db0a2c..ce9501bb 100644 --- a/rhizome/host/lib/vm_setup.rb +++ b/rhizome/host/lib/vm_setup.rb @@ -470,43 +470,43 @@ def download_boot_image(boot_image, custom_url: nil, ca_path: nil) download = urls.fetch(boot_image) || custom_url image_path = vp.image_path(boot_image) - unless File.exist?(image_path) - fail "Must provide custom_url for #{boot_image} image" if download.nil? - FileUtils.mkdir_p vp.image_root - - # If image URL has query parameter such as SAS token, File.extname returns - # it too. We need to remove them and only get extension. - image_ext = File.extname(URI.parse(download).path) - initial_format = case image_ext - when ".qcow2", ".img" - "qcow2" - when ".vhd" - "vpc" - when ".raw" - "raw" - else - fail "Unsupported boot_image format: #{image_ext}" - end - - # Use of File::EXCL provokes a crash rather than a race - # condition if two VMs are lazily getting their images at the - # same time. - temp_path = File.join(vp.image_root, boot_image + image_ext + ".tmp") - ca_arg = ca_path ? " --cacert #{ca_path.shellescape}" : "" - File.open(temp_path, File::RDWR | File::CREAT | File::EXCL, 0o644) do - r "curl -f -L10 -o #{temp_path.shellescape} #{download.shellescape}#{ca_arg}" - end + return if File.exist?(image_path) + + fail "Must provide custom_url for #{boot_image} image" if download.nil? + FileUtils.mkdir_p vp.image_root + + # If image URL has query parameter such as SAS token, File.extname returns + # it too. We need to remove them and only get extension. + image_ext = File.extname(URI.parse(download).path) + initial_format = case image_ext + when ".qcow2", ".img" + "qcow2" + when ".vhd" + "vpc" + when ".raw" + "raw" + else + fail "Unsupported boot_image format: #{image_ext}" + end - if initial_format == "raw" - File.rename(temp_path, image_path) - else - # Images are presumed to be atomically renamed into the path, - # i.e. no partial images will be passed to qemu-image. - r "qemu-img convert -p -f #{initial_format.shellescape} -O raw #{temp_path.shellescape} #{image_path.shellescape}" - end + # Use of File::EXCL provokes a crash rather than a race + # condition if two VMs are lazily getting their images at the + # same time. + temp_path = File.join(vp.image_root, boot_image + image_ext + ".tmp") + ca_arg = ca_path ? " --cacert #{ca_path.shellescape}" : "" + File.open(temp_path, File::RDWR | File::CREAT | File::EXCL, 0o644) do + r "curl -f -L10 -o #{temp_path.shellescape} #{download.shellescape}#{ca_arg}" + end - rm_if_exists(temp_path) + if initial_format == "raw" + File.rename(temp_path, image_path) + else + # Images are presumed to be atomically renamed into the path, + # i.e. no partial images will be passed to qemu-image. + r "qemu-img convert -p -f #{initial_format.shellescape} -O raw #{temp_path.shellescape} #{image_path.shellescape}" end + + rm_if_exists(temp_path) end # Unnecessary if host has this set before creating the netns, but From 1a4cd407d302e86274200b2b9be32ebdc0e85777 Mon Sep 17 00:00:00 2001 From: Enes Cakir Date: Wed, 31 Jan 2024 14:37:18 +0300 Subject: [PATCH 2/5] Finalize the image download with File.rename At present, we download the image to a temporary path and then move it to the final location using the `qemu-img convert` command. However, this process is not atomic and can be time-consuming. It would be more efficient to use the `File.rename` command to move the image directly to the final location, as this is an atomic operation. This method also allows other virtual machines to use the old image right up until the last moment. --- rhizome/host/lib/vm_setup.rb | 17 ++++++++--------- rhizome/host/spec/vm_setup_spec.rb | 7 ++++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/rhizome/host/lib/vm_setup.rb b/rhizome/host/lib/vm_setup.rb index ce9501bb..34524a65 100644 --- a/rhizome/host/lib/vm_setup.rb +++ b/rhizome/host/lib/vm_setup.rb @@ -492,21 +492,20 @@ def download_boot_image(boot_image, custom_url: nil, ca_path: nil) # Use of File::EXCL provokes a crash rather than a race # condition if two VMs are lazily getting their images at the # same time. - temp_path = File.join(vp.image_root, boot_image + image_ext + ".tmp") + download_path = File.join(vp.image_root, boot_image + image_ext + ".tmp") ca_arg = ca_path ? " --cacert #{ca_path.shellescape}" : "" - File.open(temp_path, File::RDWR | File::CREAT | File::EXCL, 0o644) do - r "curl -f -L10 -o #{temp_path.shellescape} #{download.shellescape}#{ca_arg}" + File.open(download_path, File::RDWR | File::CREAT | File::EXCL, 0o644) do + r "curl -f -L10 -o #{download_path.shellescape} #{download.shellescape}#{ca_arg}" end - if initial_format == "raw" - File.rename(temp_path, image_path) - else + temp_path = File.join(vp.image_root, boot_image + ".raw.tmp") + if initial_format != "raw" # Images are presumed to be atomically renamed into the path, # i.e. no partial images will be passed to qemu-image. - r "qemu-img convert -p -f #{initial_format.shellescape} -O raw #{temp_path.shellescape} #{image_path.shellescape}" + r "qemu-img convert -p -f #{initial_format.shellescape} -O raw #{download_path.shellescape} #{temp_path.shellescape}" + rm_if_exists(download_path) end - - rm_if_exists(temp_path) + File.rename(temp_path, image_path) end # Unnecessary if host has this set before creating the netns, but diff --git a/rhizome/host/spec/vm_setup_spec.rb b/rhizome/host/spec/vm_setup_spec.rb index 73d1879f..f2216914 100644 --- a/rhizome/host/spec/vm_setup_spec.rb +++ b/rhizome/host/spec/vm_setup_spec.rb @@ -62,8 +62,9 @@ def key_wrapping_secrets expect(FileUtils).to receive(:mkdir_p).with("/var/storage/images/") expect(Arch).to receive(:render).and_return("amd64").at_least(:once) expect(vs).to receive(:r).with("curl -f -L10 -o /var/storage/images/ubuntu-jammy.img.tmp https://cloud-images.ubuntu.com/releases/jammy/release-20231010/ubuntu-22.04-server-cloudimg-amd64.img") - expect(vs).to receive(:r).with("qemu-img convert -p -f qcow2 -O raw /var/storage/images/ubuntu-jammy.img.tmp /var/storage/images/ubuntu-jammy.raw") + expect(vs).to receive(:r).with("qemu-img convert -p -f qcow2 -O raw /var/storage/images/ubuntu-jammy.img.tmp /var/storage/images/ubuntu-jammy.raw.tmp") expect(FileUtils).to receive(:rm_r).with("/var/storage/images/ubuntu-jammy.img.tmp") + expect(File).to receive(:rename).with("/var/storage/images/ubuntu-jammy.raw.tmp", "/var/storage/images/ubuntu-jammy.raw") vs.download_boot_image("ubuntu-jammy") end @@ -75,8 +76,9 @@ def key_wrapping_secrets end.and_yield expect(FileUtils).to receive(:mkdir_p).with("/var/storage/images/") expect(vs).to receive(:r).with("curl -f -L10 -o /var/storage/images/github-ubuntu-2204.vhd.tmp http://minio.ubicloud.com:9000/ubicloud-images/ubuntu-22.04-x64.vhd\\?X-Amz-Algorithm\\=AWS4-HMAC-SHA256\\&X-Amz-Credential\\=user\\%2F20240112\\%2Fus-east-1\\%2Fs3\\%2Faws4_request\\&X-Amz-Date\\=20240112T132931Z\\&X-Amz-Expires\\=3600\\&X-Amz-SignedHeaders\\=host\\&X-Amz-Signature\\=aabbcc") - expect(vs).to receive(:r).with("qemu-img convert -p -f vpc -O raw /var/storage/images/github-ubuntu-2204.vhd.tmp /var/storage/images/github-ubuntu-2204.raw") + expect(vs).to receive(:r).with("qemu-img convert -p -f vpc -O raw /var/storage/images/github-ubuntu-2204.vhd.tmp /var/storage/images/github-ubuntu-2204.raw.tmp") expect(FileUtils).to receive(:rm_r).with("/var/storage/images/github-ubuntu-2204.vhd.tmp") + expect(File).to receive(:rename).with("/var/storage/images/github-ubuntu-2204.raw.tmp", "/var/storage/images/github-ubuntu-2204.raw") vs.download_boot_image("github-ubuntu-2204", custom_url: "http://minio.ubicloud.com:9000/ubicloud-images/ubuntu-22.04-x64.vhd?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=user%2F20240112%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240112T132931Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=aabbcc") end @@ -89,7 +91,6 @@ def key_wrapping_secrets expect(FileUtils).to receive(:mkdir_p).with("/var/storage/images/") expect(vs).to receive(:r).with("curl -f -L10 -o /var/storage/images/github-ubuntu-2204.raw.tmp http://minio.ubicloud.com:9000/ubicloud-images/ubuntu-22.04-x64.raw\\?X-Amz-Algorithm\\=AWS4-HMAC-SHA256\\&X-Amz-Credential\\=user\\%2F20240112\\%2Fus-east-1\\%2Fs3\\%2Faws4_request\\&X-Amz-Date\\=20240112T132931Z\\&X-Amz-Expires\\=3600\\&X-Amz-SignedHeaders\\=host\\&X-Amz-Signature\\=aabbcc") expect(File).to receive(:rename).with("/var/storage/images/github-ubuntu-2204.raw.tmp", "/var/storage/images/github-ubuntu-2204.raw") - expect(FileUtils).to receive(:rm_r).with("/var/storage/images/github-ubuntu-2204.raw.tmp") vs.download_boot_image("github-ubuntu-2204", custom_url: "http://minio.ubicloud.com:9000/ubicloud-images/ubuntu-22.04-x64.raw?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=user%2F20240112%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240112T132931Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=aabbcc") end From 7d4fc8a1c28bf48c03589cbeb6b6dea56f939fa7 Mon Sep 17 00:00:00 2001 From: Enes Cakir Date: Fri, 2 Feb 2024 13:30:18 +0300 Subject: [PATCH 3/5] Add force flag to download image even if exists If the boot image already exists, the virtual machine uses it. However, in certain situations like updating GitHub runner images, we need to download the latest version. If the force flag is set to true, the VM host will download the image regardless of its existence. --- rhizome/host/lib/vm_setup.rb | 4 ++-- rhizome/host/spec/vm_setup_spec.rb | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/rhizome/host/lib/vm_setup.rb b/rhizome/host/lib/vm_setup.rb index 34524a65..baf6092f 100644 --- a/rhizome/host/lib/vm_setup.rb +++ b/rhizome/host/lib/vm_setup.rb @@ -459,7 +459,7 @@ def storage(storage_params, storage_secrets, prep) } end - def download_boot_image(boot_image, custom_url: nil, ca_path: nil) + def download_boot_image(boot_image, force: false, custom_url: nil, ca_path: nil) urls = { "ubuntu-jammy" => "https://cloud-images.ubuntu.com/releases/jammy/release-20231010/ubuntu-22.04-server-cloudimg-#{Arch.render(x64: "amd64")}.img", "almalinux-9.1" => Arch.render(x64: "x86_64", arm64: "aarch64").yield_self { "https://repo.almalinux.org/almalinux/9/cloud/#{_1}/images/AlmaLinux-9-GenericCloud-latest.#{_1}.qcow2" }, @@ -470,7 +470,7 @@ def download_boot_image(boot_image, custom_url: nil, ca_path: nil) download = urls.fetch(boot_image) || custom_url image_path = vp.image_path(boot_image) - return if File.exist?(image_path) + return if File.exist?(image_path) && !force fail "Must provide custom_url for #{boot_image} image" if download.nil? FileUtils.mkdir_p vp.image_root diff --git a/rhizome/host/spec/vm_setup_spec.rb b/rhizome/host/spec/vm_setup_spec.rb index f2216914..5092f5fe 100644 --- a/rhizome/host/spec/vm_setup_spec.rb +++ b/rhizome/host/spec/vm_setup_spec.rb @@ -95,6 +95,21 @@ def key_wrapping_secrets vs.download_boot_image("github-ubuntu-2204", custom_url: "http://minio.ubicloud.com:9000/ubicloud-images/ubuntu-22.04-x64.raw?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=user%2F20240112%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240112T132931Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=aabbcc") end + it "can download the image with force even if it exists" do + expect(File).to receive(:exist?).with("/var/storage/images/ubuntu-jammy.raw").and_return(true) + expect(File).to receive(:open) do |path, *_args| + expect(path).to eq("/var/storage/images/ubuntu-jammy.img.tmp") + end.and_yield + expect(FileUtils).to receive(:mkdir_p).with("/var/storage/images/") + expect(Arch).to receive(:render).and_return("amd64").at_least(:once) + expect(vs).to receive(:r).with("curl -f -L10 -o /var/storage/images/ubuntu-jammy.img.tmp https://cloud-images.ubuntu.com/releases/jammy/release-20231010/ubuntu-22.04-server-cloudimg-amd64.img") + expect(vs).to receive(:r).with("qemu-img convert -p -f qcow2 -O raw /var/storage/images/ubuntu-jammy.img.tmp /var/storage/images/ubuntu-jammy.raw.tmp") + expect(FileUtils).to receive(:rm_r).with("/var/storage/images/ubuntu-jammy.img.tmp") + expect(File).to receive(:rename).with("/var/storage/images/ubuntu-jammy.raw.tmp", "/var/storage/images/ubuntu-jammy.raw") + + vs.download_boot_image("ubuntu-jammy", force: true) + end + it "can use an image that's already downloaded" do expect(File).to receive(:exist?).with("/var/storage/images/almalinux-9.1.raw").and_return(true) vs.download_boot_image("almalinux-9.1") From 9e3702b1526d0fb7736667dd711d14cc672f2a51 Mon Sep 17 00:00:00 2001 From: Enes Cakir Date: Wed, 31 Jan 2024 14:53:34 +0300 Subject: [PATCH 4/5] Allow only one concurrent image download with flock Right now, we use the "File::CREAT | File::EXCL" flags to open a temporary file for image downloads, which should stop multiple downloads happening at once. These flags only let one process create the file, and if the file is already there, it fails. However, if a download gets interrupted, the file sticks around and the next download attempt fails. It's tricky to figure out if the file is incomplete, so we have to delete it manually. Using flock to lock a lock file is a more effective way to stop multiple downloads idempotently at the same time. --- rhizome/host/lib/vm_setup.rb | 31 +++++++++++++++--------------- rhizome/host/spec/vm_setup_spec.rb | 13 +++++++++++++ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/rhizome/host/lib/vm_setup.rb b/rhizome/host/lib/vm_setup.rb index baf6092f..a3a3cd52 100644 --- a/rhizome/host/lib/vm_setup.rb +++ b/rhizome/host/lib/vm_setup.rb @@ -489,23 +489,24 @@ def download_boot_image(boot_image, force: false, custom_url: nil, ca_path: nil) fail "Unsupported boot_image format: #{image_ext}" end - # Use of File::EXCL provokes a crash rather than a race - # condition if two VMs are lazily getting their images at the - # same time. - download_path = File.join(vp.image_root, boot_image + image_ext + ".tmp") - ca_arg = ca_path ? " --cacert #{ca_path.shellescape}" : "" - File.open(download_path, File::RDWR | File::CREAT | File::EXCL, 0o644) do - r "curl -f -L10 -o #{download_path.shellescape} #{download.shellescape}#{ca_arg}" - end + File.open(File.join(vp.image_root, boot_image + ".lock"), File::RDWR | File::CREAT) do |lock| + fail "Another vm is downloading #{boot_image}" unless lock.flock(File::LOCK_EX | File::LOCK_NB) + + download_path = File.join(vp.image_root, boot_image + image_ext + ".tmp") + ca_arg = ca_path ? " --cacert #{ca_path.shellescape}" : "" + File.open(download_path, File::RDWR | File::CREAT, 0o644) do + r "curl -f -L10 -o #{download_path.shellescape} #{download.shellescape}#{ca_arg}" + end - temp_path = File.join(vp.image_root, boot_image + ".raw.tmp") - if initial_format != "raw" - # Images are presumed to be atomically renamed into the path, - # i.e. no partial images will be passed to qemu-image. - r "qemu-img convert -p -f #{initial_format.shellescape} -O raw #{download_path.shellescape} #{temp_path.shellescape}" - rm_if_exists(download_path) + temp_path = File.join(vp.image_root, boot_image + ".raw.tmp") + if initial_format != "raw" + # Images are presumed to be atomically renamed into the path, + # i.e. no partial images will be passed to qemu-image. + r "qemu-img convert -p -f #{initial_format.shellescape} -O raw #{download_path.shellescape} #{temp_path.shellescape}" + rm_if_exists(download_path) + end + File.rename(temp_path, image_path) end - File.rename(temp_path, image_path) end # Unnecessary if host has this set before creating the netns, but diff --git a/rhizome/host/spec/vm_setup_spec.rb b/rhizome/host/spec/vm_setup_spec.rb index 5092f5fe..5248118b 100644 --- a/rhizome/host/spec/vm_setup_spec.rb +++ b/rhizome/host/spec/vm_setup_spec.rb @@ -56,6 +56,7 @@ def key_wrapping_secrets describe "#download_boot_image" do it "can download an image" do expect(File).to receive(:exist?).with("/var/storage/images/ubuntu-jammy.raw").and_return(false) + expect(File).to receive(:open).with("/var/storage/images/ubuntu-jammy.lock", File::RDWR | File::CREAT).and_yield(instance_double(File, flock: true)) expect(File).to receive(:open) do |path, *_args| expect(path).to eq("/var/storage/images/ubuntu-jammy.img.tmp") end.and_yield @@ -71,6 +72,7 @@ def key_wrapping_secrets it "can download vhd image with custom URL that has query params using curl" do expect(File).to receive(:exist?).with("/var/storage/images/github-ubuntu-2204.raw").and_return(false) + expect(File).to receive(:open).with("/var/storage/images/github-ubuntu-2204.lock", File::RDWR | File::CREAT).and_yield(instance_double(File, flock: true)) expect(File).to receive(:open) do |path, *_args| expect(path).to eq("/var/storage/images/github-ubuntu-2204.vhd.tmp") end.and_yield @@ -85,6 +87,7 @@ def key_wrapping_secrets it "does not convert image if it's in raw format already" do expect(File).to receive(:exist?).with("/var/storage/images/github-ubuntu-2204.raw").and_return(false) + expect(File).to receive(:open).with("/var/storage/images/github-ubuntu-2204.lock", File::RDWR | File::CREAT).and_yield(instance_double(File, flock: true)) expect(File).to receive(:open) do |path, *_args| expect(path).to eq("/var/storage/images/github-ubuntu-2204.raw.tmp") end.and_yield @@ -97,6 +100,7 @@ def key_wrapping_secrets it "can download the image with force even if it exists" do expect(File).to receive(:exist?).with("/var/storage/images/ubuntu-jammy.raw").and_return(true) + expect(File).to receive(:open).with("/var/storage/images/ubuntu-jammy.lock", File::RDWR | File::CREAT).and_yield(instance_double(File, flock: true)) expect(File).to receive(:open) do |path, *_args| expect(path).to eq("/var/storage/images/ubuntu-jammy.img.tmp") end.and_yield @@ -125,6 +129,15 @@ def key_wrapping_secrets expect(FileUtils).to receive(:mkdir_p).with("/var/storage/images/") expect { vs.download_boot_image("github-ubuntu-2204", custom_url: "https://example.com/ubuntu.iso") }.to raise_error RuntimeError, "Unsupported boot_image format: .iso" end + + it "fails if another vm is already downloading the image" do + expect(File).to receive(:exist?).with("/var/storage/images/ubuntu-jammy.raw").and_return(false) + expect(FileUtils).to receive(:mkdir_p).with("/var/storage/images/") + expect(Arch).to receive(:render).and_return("amd64").at_least(:once) + expect(File).to receive(:open).with("/var/storage/images/ubuntu-jammy.lock", File::RDWR | File::CREAT).and_yield(instance_double(File, flock: false)) + + expect { vs.download_boot_image("ubuntu-jammy") }.to raise_error RuntimeError, "Another vm is downloading ubuntu-jammy" + end end describe "#purge_storage" do From b48eb697a4fa588cc12c5fdd0acf48e7b24bbf6f Mon Sep 17 00:00:00 2001 From: Enes Cakir Date: Wed, 31 Jan 2024 15:34:34 +0300 Subject: [PATCH 5/5] Use force flag in the boot image download prog The previous version of the `download_boot_image` method fails when the file already exists. I've incorporated a `force` flag into the method to overwrite any pre-existing file. This way, the program doesn't need to delete the existing file before downloading the boot image. --- prog/download_boot_image.rb | 11 ++++++----- rhizome/host/bin/download-boot-image | 2 +- spec/prog/download_boot_image_spec.rb | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/prog/download_boot_image.rb b/prog/download_boot_image.rb index 05c5b57f..2e8f8da5 100644 --- a/prog/download_boot_image.rb +++ b/prog/download_boot_image.rb @@ -26,11 +26,9 @@ def blob_storage_client end label def wait_draining - unless vm_host.vms.empty? - nap 15 - end - sshable.cmd("sudo rm -f /var/storage/images/#{image_name.shellescape}.raw") - hop_download + hop_download if vm_host.vms.empty? + + nap 15 end label def download @@ -73,6 +71,9 @@ def blob_storage_client end label def activate_host + # Verify that the image was downloaded + sshable.cmd("ls /var/storage/images/#{image_name}.raw") + vm_host.update(allocation_state: "accepting") pop "#{image_name} downloaded" diff --git a/rhizome/host/bin/download-boot-image b/rhizome/host/bin/download-boot-image index 8ad9fa19..7468e71c 100755 --- a/rhizome/host/bin/download-boot-image +++ b/rhizome/host/bin/download-boot-image @@ -14,4 +14,4 @@ require_relative "../lib/vm_setup" certs = $stdin.read ca_path = "/usr/lib/ssl/certs/ubicloud_images_blob_storage_certs.crt" safe_write_to_file(ca_path, certs) -VmSetup.new("").download_boot_image(boot_image, custom_url: custom_url, ca_path: ca_path) +VmSetup.new("").download_boot_image(boot_image, force: true, custom_url: custom_url, ca_path: ca_path) diff --git a/spec/prog/download_boot_image_spec.rb b/spec/prog/download_boot_image_spec.rb index 28c05e2f..ba5b948d 100644 --- a/spec/prog/download_boot_image_spec.rb +++ b/spec/prog/download_boot_image_spec.rb @@ -28,7 +28,6 @@ it "hops if it's drained" do expect(vm_host).to receive(:vms).and_return([]) - expect(sshable).to receive(:cmd).with("sudo rm -f /var/storage/images/my-image.raw") expect { dbi.wait_draining }.to hop("download") end end @@ -99,6 +98,7 @@ describe "#activate_host" do it "activates vm host again" do + expect(sshable).to receive(:cmd).with("ls /var/storage/images/my-image.raw") expect { expect { dbi.activate_host }.to exit({"msg" => "my-image downloaded"}) }.to change { vm_host.reload.allocation_state }.from("unprepared").to("accepting")