Merge branch 'basvandijk/run-systest-in-uvm' into 'master'

rs/tests: have a UVM config image bazel target per systest

This adds a bazel target per system-test that builds a UVM config image that contains a docker image that when run runs the system-test. This enables system-tests that spawn UVMs that run other system-tests. 

This is both useful for load-testing (like needed for the SNS load tests) and co-locating the system-test driver with the IC under test.

This MR also adds an example system-test `remote_replicable_mock_test` that spawns a UVM that runs the `replicable_mock_test`. 

See merge request dfinity-lab/public/ic!10177

rs/tests/BUILD.bazel

@@ -6,6 +6,7 @@ package(default_visibility = ["//visibility:public"])
 exports_files([
     "src/cow_safety.wasm",
     "src/message.wasm",
+    "activate-systest-uvm-config",
 ])
 
 DEPENDENCIES = [
@@ -857,3 +858,15 @@ system_test(
     target_compatible_with = ["@platforms//os:linux"],  # requires libssh that does not build on Mac OS
     deps = DEPENDENCIES + [":tests"],
 )
+
+system_test(
+    name = "remote_replicable_mock_test",
+    flaky = False,  # remove after when PFOPS-3148 is resolved
+    proc_macro_deps = MACRO_DEPENDENCIES,
+    tags = [
+        "manual",
+    ],
+    target_compatible_with = ["@platforms//os:linux"],  # requires libssh that does not build on Mac OS
+    runtime_deps = UNIVERSAL_VM_RUNTIME_DEPS + [":replicable_mock_test_uvm_config_image"],
+    deps = DEPENDENCIES + [":tests"],
+)

rs/tests/Cargo.toml

@@ -314,6 +314,10 @@ path = "bin/backup_manager_test.rs"
 name = "ic-systest-replicable-mock-test"
 path = "bin/replicable_mock_test.rs"
 
+[[bin]]
+name = "ic-systest-remote-replicable-mock-test"
+path = "bin/remote_replicable_mock_test.rs"
+
 [[bin]]
 name = "ic-systest-mainnet"
 path = "bin/mainnet_test.rs"

rs/tests/activate-systest-uvm-config

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+find /config -name *.tar -exec docker load -i {} \;

rs/tests/bin/remote_replicable_mock_test.rs

@@ -0,0 +1,56 @@
+#[rustfmt::skip]
+
+use anyhow::Result;
+
+use ic_tests::driver::farm::HostFeature;
+use ic_tests::driver::new::group::SystemTestGroup;
+use ic_tests::driver::test_env::TestEnv;
+use ic_tests::driver::test_env_api::{HasDependencies, HasGroupSetup, SshSession, ADMIN};
+use ic_tests::driver::universal_vm::{UniversalVm, UniversalVms};
+use ic_tests::systest;
+use slog::info;
+
+const UNIVERSAL_VM_NAME: &str = "systest-runner";
+
+fn main() -> Result<()> {
+    SystemTestGroup::new()
+        .with_setup(setup)
+        .add_test(systest!(test))
+        .execute_from_args()?;
+    Ok(())
+}
+
+pub fn setup(env: TestEnv) {
+    env.ensure_group_setup_created();
+    UniversalVm::new(String::from(UNIVERSAL_VM_NAME))
+        .with_config_img(
+            env.get_dependency_path("rs/tests/replicable_mock_test_uvm_config_image.zst"),
+        )
+        .disable_ipv4()
+        .with_required_host_features(vec![HostFeature::Host(
+            "fr1-dll07.fr1.dfinity.network".to_string(),
+        )])
+        .start(&env)
+        .expect("failed to setup universal VM");
+}
+
+pub fn test(env: TestEnv) {
+    let logger = env.logger();
+    let universal_vm = env.get_deployed_universal_vm(UNIVERSAL_VM_NAME).unwrap();
+    let session = universal_vm.block_on_ssh_session(ADMIN).unwrap();
+
+    info!(
+        logger,
+        "Starting: docker run bazel/rs/tests:replicable_mock_test_image"
+    );
+    universal_vm
+        .block_on_bash_script_from_session(
+            &session,
+            "docker run bazel/rs/tests:replicable_mock_test_image",
+        )
+        .unwrap();
+    info!(
+        logger,
+        "Finished: docker run bazel/rs/tests:replicable_mock_test_image"
+    );
+}

rs/tests/create-universal-vm-config-image.sh

@@ -76,4 +76,4 @@ echo "image size: $size"
 truncate -s $size "$tmp"
 mkfs.vfat -n "$LABEL" "$tmp"
 mcopy -i "$tmp" -sQ "$INPUT_DIR"/* ::
-zstd -i "$tmp" -o "$OUTPUT_FILE" --force
+zstd --threads=0 -10 -i "$tmp" -o "$OUTPUT_FILE" --force

rs/tests/system_tests.bzl

@@ -76,8 +76,10 @@ def system_test(name, runtime_deps = [], tags = [], test_timeout = "long", flaky
         **kwargs
     )
 
+    container_name = name + "_image"
+
     container_image(
-        name = name + "_image",
+        name = container_name,
         base = "//rs/tests/replicated_tests:test_driver_image_base",
         directory = "/home/root/root_env/dependencies",
         data_path = "/",
@@ -90,6 +92,24 @@ def system_test(name, runtime_deps = [], tags = [], test_timeout = "long", flaky
         workdir = "/home/root",
     )
 
+    uvm_config_image_name = name + "_uvm_config_image"
+
+    uvm_config_image(
+        name = uvm_config_image_name,
+        srcs = [
+            ":" + container_name + ".tar",
+            ":activate-systest-uvm-config",
+        ],
+        remap_paths = {
+            "/activate-systest-uvm-config": "/activate",
+        },
+        mode = "664",
+        modes = {
+            "activate": "775",
+        },
+        tags = ["manual"],  # this target will be built if required as a dependency of another target
+    )
+
     run_system_test(
         name = name,
         src = bin_name,
@@ -120,7 +140,7 @@ symlink_dir = rule(
     },
 )
 
-def _uvm_config_img_impl(ctx):
+def _uvm_config_image_impl(ctx):
     out = ctx.actions.declare_file(ctx.label.name + ".zst")
 
     input_tar = ctx.attr.input_tar[DefaultInfo].files.to_list()[0]
@@ -139,8 +159,8 @@ def _uvm_config_img_impl(ctx):
         ),
     ]
 
-uvm_config_img_impl = rule(
-    implementation = _uvm_config_img_impl,
+uvm_config_image_impl = rule(
+    implementation = _uvm_config_image_impl,
     attrs = {
         "input_tar": attr.label(),
         "_create_universal_vm_config_image_from_tar": attr.label(
@@ -156,16 +176,16 @@ uvm_config_img_impl = rule(
     },
 )
 
-def uvm_config_img(name, **kws):
+def uvm_config_image(name, **kws):
     tar = name + "_tar"
 
     pkg_tar(
         name = tar,
         **kws
     )
 
-    uvm_config_img_impl(
+    uvm_config_image_impl(
         name = name,
         input_tar = ":" + tar,
-        target_compatible_with = ["@platforms//os:linux"],  # we expect the coreutils from linux
+        tags = ["manual"],
     )