From 5ae2e578673f895a2b17baf4ee24238d4db456d6 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Tue, 4 Apr 2023 23:41:46 +0000 Subject: [PATCH] Assorted tools around Bazel images --- Cargo.lock | 18 ++ Cargo.toml | 1 + ic-os/defs.bzl | 2 + ic-os/guestos/BUILD.bazel | 27 +++ ic-os/scripts/get-artifacts.sh | 27 ++- rs/ic_os/launch-single-vm/BUILD.bazel | 35 ++++ rs/ic_os/launch-single-vm/Cargo.toml | 21 +++ rs/ic_os/launch-single-vm/src/main.rs | 251 ++++++++++++++++++++++++++ 8 files changed, 376 insertions(+), 6 deletions(-) create mode 100644 rs/ic_os/launch-single-vm/BUILD.bazel create mode 100644 rs/ic_os/launch-single-vm/Cargo.toml create mode 100644 rs/ic_os/launch-single-vm/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 4684113619a..2f45a205d74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10682,6 +10682,24 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" +[[package]] +name = "launch-single-vm" +version = "0.1.0" +dependencies = [ + "clap 3.2.23", + "ic-prep", + "ic-registry-subnet-type", + "ic-types", + "reqwest", + "serde", + "slog", + "slog-async", + "slog-term", + "tempfile", + "tests", + "url", +] + [[package]] name = "lazy-regex" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 83e01f4cb3a..03301ac3194 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,7 @@ members = [ "rs/http_endpoints/public", "rs/http_endpoints/metrics", "rs/http_utils", + "rs/ic_os/launch-single-vm", "rs/ic_os/sev", "rs/ic_os/sev_interfaces", "rs/ic_os/sevctl", diff --git a/ic-os/defs.bzl b/ic-os/defs.bzl index 87fcde080f4..202d479aaf0 100644 --- a/ic-os/defs.bzl +++ b/ic-os/defs.bzl @@ -274,6 +274,7 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False, sha256sum( name = "disk-img.tar.zst.sha256", srcs = [":disk-img.tar.zst"], + visibility = ["//visibility:public"], ) gzip_compress( @@ -396,6 +397,7 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False, name = "disk-img-url", target = ":upload_disk-img", basenames = ["upload_disk-img_disk-img.tar.zst.url"], + visibility = ["//visibility:public"], tags = ["manual"], ) diff --git a/ic-os/guestos/BUILD.bazel b/ic-os/guestos/BUILD.bazel index 3960a44ed28..c903c1e5e9c 100644 --- a/ic-os/guestos/BUILD.bazel +++ b/ic-os/guestos/BUILD.bazel @@ -34,6 +34,33 @@ sh_test( data = [":rootfs/opt/ic/share/ic.json5.template"], ) +genrule( + name = "launch-single-vm", + srcs = [ + "//rs/ic_os/launch-single-vm", + "//ic-os/guestos/envs/dev:disk-img-url", + "//ic-os/guestos/envs/dev:disk-img.tar.zst.sha256", + "//ic-os/guestos:scripts/build-bootstrap-config-image.sh", + "//bazel:version.txt", + ], + outs = ["launch_single_vm_script"], + cmd = """ +BIN="$(location //rs/ic_os/launch-single-vm:launch-single-vm)" +VERSION="$$(cat $(location //bazel:version.txt))" +URL="$$(cat $(location //ic-os/guestos/envs/dev:disk-img-url))" +SHA="$$(cat $(location //ic-os/guestos/envs/dev:disk-img.tar.zst.sha256))" +SCRIPT="$(location //ic-os/guestos:scripts/build-bootstrap-config-image.sh)" +cat < $@ +#!/usr/bin/env bash +set -euo pipefail +cd "\\$$BUILD_WORKSPACE_DIRECTORY" +$$BIN --version "$$VERSION" --url "$$URL" --sha256 "$$SHA" --build-bootstrap-script "$$SCRIPT" +EOF + """, + executable = True, + tags = ["manual"], +) + # All guest-os targets are named the same, just stay in different submodules. # To build or run specific target: # diff --git a/ic-os/scripts/get-artifacts.sh b/ic-os/scripts/get-artifacts.sh index 51f250c6f8b..a35476cd887 100755 --- a/ic-os/scripts/get-artifacts.sh +++ b/ic-os/scripts/get-artifacts.sh @@ -2,6 +2,10 @@ GET_GUEST_OS_DEFAULT=1 GET_GUEST_OS=${GET_GUEST_OS-$GET_GUEST_OS_DEFAULT} +GET_HOST_OS_DEFAULT=0 +GET_HOST_OS=${GET_HOST_OS-$GET_HOST_OS_DEFAULT} +GET_SETUP_OS_DEFAULT=0 +GET_SETUP_OS=${GET_SETUP_OS-$GET_SETUP_OS_DEFAULT} DEFAULT=$(git rev-parse HEAD) GIT=${GIT-$DEFAULT} @@ -32,17 +36,28 @@ if [[ $GET_GUEST_OS -eq 1 ]]; then cd artifacts/guest-os/disk-img for f in *.tar; do tar --sparse -xf $f; done ) +fi + +if [[ $GET_HOST_OS -eq 1 ]]; then + ./gitlab-ci/src/artifacts/rclone_download.py --git-rev $GIT --out=artifacts/host-os --remote-path host-os --latest-to + ( + cd artifacts/host-os/disk-img + for f in *.gz; do gunzip -f $f; done + ) + ( + cd artifacts/host-os/disk-img + for f in *.tar; do tar --sparse -xf $f; done + ) +fi - DEFAULT=$(git rev-parse HEAD) - GIT_REV=${GIT_REV-$DEFAULT} - echo "➡️ Downloading artifacts for revision $GIT_REV (as upgrade target)" - ./gitlab-ci/src/artifacts/rclone_download.py --git-rev $GIT_REV --out=artifacts/guest-os-master --remote-path guest-os --latest-to +if [[ $GET_SETUP_OS -eq 1 ]]; then + ./gitlab-ci/src/artifacts/rclone_download.py --git-rev $GIT --out=artifacts/setup-os --remote-path setup-os --latest-to ( - cd artifacts/guest-os-master/disk-img + cd artifacts/setup-os/disk-img for f in *.gz; do gunzip -f $f; done ) ( - cd artifacts/guest-os-master/disk-img + cd artifacts/setup-os/disk-img for f in *.tar; do tar --sparse -xf $f; done ) fi diff --git a/rs/ic_os/launch-single-vm/BUILD.bazel b/rs/ic_os/launch-single-vm/BUILD.bazel new file mode 100644 index 00000000000..ff9f33a8d5e --- /dev/null +++ b/rs/ic_os/launch-single-vm/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary") + +package(default_visibility = ["//visibility:public"]) + +DEPENDENCIES = [ + "@crate_index//:clap", + "@crate_index//:reqwest", + "@crate_index//:serde", + "@crate_index//:slog-async", + "@crate_index//:slog-term", + "@crate_index//:slog", + "@crate_index//:tempfile", + "@crate_index//:url", + "//rs/tests", + "//rs/prep", + "//rs/registry/subnet_type", + "//rs/types/types", +] + +MACRO_DEPENDENCIES = [] + +ALIASES = {} + +rust_binary( + name = "launch-single-vm", + srcs = glob(["src/*.rs"]), + aliases = ALIASES, + crate_name = "launch_single_vm", + edition = "2021", + proc_macro_deps = MACRO_DEPENDENCIES, + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = DEPENDENCIES, +) diff --git a/rs/ic_os/launch-single-vm/Cargo.toml b/rs/ic_os/launch-single-vm/Cargo.toml new file mode 100644 index 00000000000..4620d18002e --- /dev/null +++ b/rs/ic_os/launch-single-vm/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "launch-single-vm" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tests = { path = "../../tests"} +ic-prep = { path = "../../prep"} +ic-registry-subnet-type = { path = "../../registry/subnet_type"} +ic-types = { path = "../../types/types" } + +clap = { version = "3.1.6", features = ["derive"] } +reqwest = { version = "0.11.1", features = ["blocking", "multipart", "stream"] } +serde = { version = "1.0.115", features = ["derive"] } +slog-async = { version = "2.5", features = ["nested-values"] } +slog-term = "2.6.0" +slog = { version = "2.5.2", features = ["release_max_level_trace"] } +tempfile = "3.1.0" +url = "2.1.1" diff --git a/rs/ic_os/launch-single-vm/src/main.rs b/rs/ic_os/launch-single-vm/src/main.rs new file mode 100644 index 00000000000..0840f36536c --- /dev/null +++ b/rs/ic_os/launch-single-vm/src/main.rs @@ -0,0 +1,251 @@ +use std::collections::BTreeMap; +use std::net::{IpAddr, SocketAddr}; +use std::path::PathBuf; +use std::process::Command; + +use clap::Parser; +use reqwest::blocking::{Client, Response}; +use serde::Serialize; +use slog::{o, Drain}; +use tempfile::tempdir; +use url::Url; + +use ic_prep_lib::{ + internet_computer::{IcConfig, TopologyConfig}, + node::NodeConfiguration, + subnet_configuration::SubnetConfig, +}; +use ic_registry_subnet_type::SubnetType; +use ic_tests::driver::{ + farm::{CreateVmRequest, Farm, GroupSpec, ImageLocation, VmType}, + ic::{Subnet, VmAllocationStrategy}, +}; +use ic_types::ReplicaVersion; + +const FARM_BASE_URL: &str = "https://farm.dfinity.systems"; + +/// Deploy a single ICOS VM to Farm +#[derive(Parser)] +struct Args { + /// Version to deploy + #[clap(long)] + version: String, + /// Image URL + #[clap(long)] + url: Url, + /// Image SHA256SUM + #[clap(long)] + sha256: String, + /// Path to `build-bootstrap-config-image.sh` script + #[clap(long)] + build_bootstrap_script: PathBuf, + /// Key to be used for `admin` SSH + #[clap(long)] + ssh_key_path: Option, +} + +fn main() { + let decorator = slog_term::TermDecorator::new().build(); + let drain = slog_term::FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + let logger = slog::Logger::root(drain, o!()); + + let tempdir = tempdir().unwrap(); + + let farm = Farm::new(Url::parse(FARM_BASE_URL).unwrap(), logger.clone()); + + // Arguments + let args = Args::parse(); + let version = ReplicaVersion::try_from(args.version).unwrap(); + let url = args.url; + let sha256 = args.sha256; + let build_bootstrap_script = args.build_bootstrap_script; + let ssh_key_path = args.ssh_key_path; + + let group_name = if let Ok(user) = std::env::var("USER") { + format!("testvm-{user}-{version}") + } else { + format!("testvm-{version}") + }; + + // Delete any old groups + let _res = delete_group(&group_name); + + // Create a new group + create_group( + &group_name, + CreateGroupRequest { + ttl: 3600, + spec: GroupSpec { + vm_allocation: Some(VmAllocationStrategy::DistributeAcrossDcs), + required_host_features: Vec::new(), + preferred_network: None, + metadata: None, + }, + }, + ); + + // Allocate new VM on Farm + let vm_name = "main"; + let request = CreateVmRequest::new( + vm_name.to_string(), + VmType::Production, + 2.into(), // 2 vCPUs + 25165824.into(), // 24 GibiBytes RAM + Vec::new(), + ImageLocation::IcOsImageViaUrl { url, sha256 }, + Some(100.into()), // 100 GibiByte image + false, + None, + Vec::new(), + ); + let created_vm = farm.create_vm(&group_name, request).unwrap(); + let ipv6_addr = IpAddr::V6(created_vm.ipv6); + + // Build Initial IC State + let prep_dir = tempdir.as_ref().join("prep"); + std::fs::create_dir(&prep_dir).unwrap(); + + let nodes = BTreeMap::from([( + 0, + NodeConfiguration { + xnet_api: vec![SocketAddr::new(ipv6_addr, 2497).into()], + public_api: vec![SocketAddr::new(ipv6_addr, 8080).into()], + private_api: vec![], + p2p_addr: format!( + "org.internetcomputer.p2p1://{}", + SocketAddr::new(ipv6_addr, 4100) + ) + .parse() + .unwrap(), + prometheus_metrics: vec![SocketAddr::new(ipv6_addr, 9090).into()], + node_operator_principal_id: None, + secret_key_store: None, + }, + )]); + + let mut ic_topology = TopologyConfig::default(); + // Build a "Farm" subnet for the convenient defaults + let subnet = Subnet::fast_single_node(SubnetType::System); + ic_topology.insert_subnet(0, subnet_to_subnet_config(subnet, version.clone(), nodes)); + + let ic_config = IcConfig::new( + &prep_dir, + ic_topology, + Some(version), + true, + Some(0), + None, + None, + None, + None, + None, + Vec::new(), + None, + ); + let initialized_ic = ic_config.initialize().unwrap(); + // The first node from the first subnet will be our only node. + let node = initialized_ic + .initialized_topology + .values() + .next() + .unwrap() + .initialized_nodes + .values() + .next() + .unwrap(); + + // Constrcut SSH Key Directory + let keys_dir = tempdir.as_ref().join("ssh_authorized_keys"); + std::fs::create_dir(&keys_dir).unwrap(); + if let Some(key) = ssh_key_path { + std::fs::copy(key, keys_dir.join("admin")).unwrap(); + } + + // Build config image + let filename = "config.tar.gz"; + let config_path = tempdir.as_ref().join(filename); + let local_store = prep_dir.join("ic_registry_local_store"); + Command::new(build_bootstrap_script) + .arg(&config_path) + .arg("--nns_url") + .arg(ipv6_addr.to_string()) + .arg("--ic_crypto") + .arg(node.crypto_path()) + .arg("--ic_registry_local_store") + .arg(&local_store) + .arg("--accounts_ssh_authorized_keys") + .arg(&keys_dir) + .status() + .unwrap(); + + // Upload config image + let image_id = farm.upload_file(&config_path, filename).unwrap(); + + // Attatch image + farm.attach_disk_images(&group_name, vm_name, "usb-storage", vec![image_id]) + .unwrap(); + + // Start VM + farm.start_vm(&group_name, vm_name).unwrap(); +} + +/// Convert from a Farm `Subnet` to the prep `SubnetConfig` +fn subnet_to_subnet_config( + subnet: Subnet, + version: ReplicaVersion, + nodes: BTreeMap, +) -> SubnetConfig { + SubnetConfig::new( + 0, + nodes, + Some(version), + subnet.ingress_bytes_per_block_soft_cap, + subnet.max_ingress_bytes_per_message, + subnet.max_ingress_messages_per_block, + subnet.max_block_payload_size, + subnet.unit_delay, + subnet.initial_notary_delay, + subnet.dkg_interval_length, + subnet.dkg_dealings_per_block, + subnet.subnet_type, + subnet.max_instructions_per_message, + subnet.max_instructions_per_round, + subnet.max_instructions_per_install_code, + subnet.features.map(|f| f.into()), + None, + subnet.max_number_of_canisters, + subnet.ssh_readonly_access, + subnet.ssh_backup_access, + ) +} + +/// Delete a group from Farm, without retries +fn delete_group(group_name: &str) -> Response { + let client = Client::new(); + + client + .delete(format!("{FARM_BASE_URL}/group/{group_name}")) + .send() + .unwrap() +} + +// Need to create our own, as the one from Farm is private +#[derive(Serialize)] +struct CreateGroupRequest { + ttl: u32, + spec: GroupSpec, +} + +/// Create a new group in farm, without needing a `TestEnv` +fn create_group(group_name: &str, body: CreateGroupRequest) { + let client = Client::new(); + + client + .post(format!("{FARM_BASE_URL}/group/{group_name}")) + .header("Accept", "application/json") + .header("Content-Type", "application/json") + .json(&body) + .send() + .unwrap(); +}