chore: [IDX-3077] Pass IPv6 prefixes via test driver

dfinity · Jan 30, 2024 · 058672d · 058672d
1 parent d9850d0
commit 058672d
Show file tree

Hide file tree

Showing 14 changed files with 142 additions and 82 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/ic-os/defs.bzl b/ic-os/defs.bzl
@@ -482,6 +482,7 @@ def icos_build(
             ":disk-img.tar.zst.cas-url",
             ":disk-img.tar.zst.sha256",
             "//ic-os:scripts/build-bootstrap-config-image.sh",
+            "//rs/tests:src/ipv6_prefixes.json",
             ":version.txt",
         ],
         outs = ["launch_remote_vm_script"],
@@ -491,11 +492,12 @@ def icos_build(
         URL="$$(cat $(location :disk-img.tar.zst.cas-url))"
         SHA="$$(cat $(location :disk-img.tar.zst.sha256))"
         SCRIPT="$(location //ic-os:scripts/build-bootstrap-config-image.sh)"
+        IPV6_PREFIXES="$(location //rs/tests:src/ipv6_prefixes.json)"
         cat <<EOF > $@
 #!/usr/bin/env bash
 set -euo pipefail
 cd "\\$$BUILD_WORKSPACE_DIRECTORY"
-$$BIN --version "$$VERSION" --url "$$URL" --sha256 "$$SHA" --build-bootstrap-script "$$SCRIPT"
+$$BIN --version "$$VERSION" --url "$$URL" --sha256 "$$SHA" --build-bootstrap-script "$$SCRIPT" --ipv6-prefixes-path "$$IPV6_PREFIXES"
 EOF
         """,
         executable = True,

diff --git a/ic-os/guestos/rootfs/etc/systemd/system/ic-crypto-csp.service b/ic-os/guestos/rootfs/etc/systemd/system/ic-crypto-csp.service
@@ -20,7 +20,7 @@ Environment=RUST_BACKTRACE=1
 # only on ExecStartPre by ic-replica.service. As a temporary workaround, we are generating
 # an ic-csp-service-specific version of ic.json5 of which only the parts related to crypto
 # and logging are relevant.
-ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic-crypto-csp.json5
+ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -n /boot/config/network.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic-crypto-csp.json5
 ExecStart=/opt/ic/bin/ic-crypto-csp --replica-config-file /run/ic-node/config/ic-crypto-csp.json5
 NotifyAccess=main
 Restart=always

diff --git a/ic-os/guestos/rootfs/opt/ic/bin/generate-replica-config.sh b/ic-os/guestos/rootfs/opt/ic/bin/generate-replica-config.sh
@@ -75,6 +75,7 @@ function read_network_variables() {
             "ipv6_address") ipv6_address="${value}" ;;
             "ipv6_gateway") ipv6_gateway="${value}" ;;
             "name_servers") name_servers="${value}" ;;
+            "ipv6_prefixes") ipv6_prefixes="${value}" ;;
             "ipv4_address") ipv4_address="${value}" ;;
             "ipv4_gateway") ipv4_gateway="${value}" ;;
             "domain") domain="${value}" ;;
@@ -202,6 +203,7 @@ fi
 INTERFACE=($(find /sys/class/net -type l -not -lname '*virtual*' -exec basename '{}' ';'))
 IPV6_ADDRESS="${ipv6_address%/*}"
 IPV6_ADDRESS="${IPV6_ADDRESS:-$(get_if_address_retries 6 ${INTERFACE} 12)}"
+IPV6_PREFIXES="${ipv6_prefixes:-}"
 IPV4_ADDRESS="${ipv4_address:-}"
 IPV4_GATEWAY="${ipv4_gateway:-}"
 DOMAIN="${domain:-}"
@@ -238,6 +240,7 @@ if [ "${hostname}" == "" ]; then
 fi
 
 sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \
+    -e "s@{{ ipv6_prefixes }}@${IPV6_PREFIXES}@" \
     -e "s@{{ ipv4_address }}@${IPV4_ADDRESS}@" \
     -e "s@{{ ipv4_gateway }}@${IPV4_GATEWAY}@" \
     -e "s@{{ domain }}@${DOMAIN}@" \

diff --git a/ic-os/guestos/rootfs/opt/ic/share/ic.json5.template b/ic-os/guestos/rootfs/opt/ic/share/ic.json5.template
@@ -260,81 +260,7 @@ table ip6 filter {\n\
         ipv6_user_output_rule_template: "meta skuid <<USER>> ip6 daddr {<<IPv6_PREFIXES>>} ct state { new } tcp dport {<<PORTS>>} <<ACTION>> # <<COMMENT>>",
         default_rules: [{
           ipv4_prefixes: [],
-          ipv6_prefixes: [
-            "2001:438:fffd:11c::/64",
-            "2001:470:1:c76::/64",
-            "2001:4d78:400:10a::/64",
-            "2001:4d78:40d::/48",
-            "2602:fb2b:110::/48",
-            "2001:920:401a:1706::/64",
-            "2001:920:401a:1708::/64",
-            "2001:920:401a:1710::/64",
-            "2401:3f00:1000:22::/64",
-            "2401:3f00:1000:23::/64",
-            "2401:3f00:1000:24::/64",
-            "2600:2c01:21::/64",
-            "2600:3000:1300:1300::/64",
-            "2600:3000:6100:200::/64",
-            "2600:3004:1200:1200::/56",
-            "2600:3006:1400:1500::/64",
-            "2600:c00:2:100::/64",
-            "2600:c02:b002:15::/64",
-            "2600:c0d:3002:4::/64",
-            "2602:ffe4:801:16::/64",
-            "2602:ffe4:801:17::/64",
-            "2602:ffe4:801:18::/64",
-            "2604:1380:4091:3000::/64",
-            "2604:1380:40e1:4700::/64",
-            "2604:1380:40f1:1700::/64",
-            "2604:1380:45d1:bf00::/64",
-            "2604:1380:45e1:a600::/64",
-            "2604:1380:45f1:9400::/64",
-            "2604:1380:4601:6200::/64",
-            "2604:1380:4601:6201::/64",
-            "2604:1380:4601:6202::/64",
-            "2604:1380:4641:6101::/64",
-            "2604:1380:4641:6102::/64",
-            "2604:1380:4091:3001::/64",
-            "2604:1380:4091:3002::/64",
-            "2604:1380:45e1:a601::/64",
-            "2604:1380:45e1:a602::/64",
-            "2604:1380:4641:6100::/64",
-            "2604:3fc0:2001::/48",
-            "2604:3fc0:3002::/48",
-            "2604:6800:258:1::/64",
-            "2604:7e00:30:3::/64",
-            "2604:7e00:50::/64",
-            "2604:b900:4001:76::/64",
-            "2607:f1d0:10:1::/64",
-            "2607:f6f0:3004::/48",
-            "2602:fb2b:120::/48",
-            "2607:f758:1220::/64",
-            "2607:f758:c300::/64",
-            "2607:fb58:9005::/48",
-            "2602:fb2b:100::/48",
-            "2607:ff70:3:2::/64",
-            "2610:190:6000:1::/64",
-            "2610:190:df01:5::/64",
-            "2a00:fa0:3::/48",
-            "2a00:fb01:400:100::/56",
-            "2a00:fb01:400::/56",
-            "2a00:fc0:5000:300::/64",
-            "2a01:138:900a::/48",
-            "2a01:2a8:a13c:1::/64",
-            "2a01:2a8:a13d:1::/64",
-            "2a01:2a8:a13e:1::/64",
-            "2a02:418:3002:0::/64",
-            "2a02:41b:300e::/48",
-            "2a02:800:2:2003::/64",
-            "2a04:9dc0:0:108::/64",
-            "2a05:d014:939:bf00::/56",
-            "2a05:d01c:d9:2b00::/56",
-            "2a05:d01c:e2c:a700::/56",
-            "2a0b:21c0:4003:2::/64",
-            "2a0b:21c0:b002:2::/64",
-            "2a0f:cd00:0002::/56",
-            "fd00:2:1:1::/64",
-          ],
+          ipv6_prefixes: {{ ipv6_prefixes }},
           ports: [22, 2497, 4100, 7070, 8080, 9090, 9091, 9100, 19100, 19531],
           action: 1,
           comment: "Default rule from template",

diff --git a/ic-os/scripts/build-bootstrap-config-image.sh b/ic-os/scripts/build-bootstrap-config-image.sh
@@ -33,6 +33,9 @@ options may be specified:
     script, e.g. --ipv6_name_servers "2606:4700:4700::1111
     2606:4700:4700::1001").
 
+  --ipv6_prefixes prefixes
+    Comma separated list of IPv6 networks that are whitelisted in the guestOS' firewall on orchestrator startup.
+
   --ipv4_address a.b.c.d/n
     (optional) The IPv4 address to assign. Must include prefix length (e.g.
     18.208.190.35/28).
@@ -126,7 +129,7 @@ function build_ic_bootstrap_tar() {
     local OUT_FILE="$1"
     shift
 
-    local IPV6_ADDRESS IPV6_GATEWAY IPV6_NAME_SERVERS DOMAIN HOSTNAME
+    local IPV6_ADDRESS IPV6_GATEWAY IPV6_NAME_SERVERS IVP6_PREFIXES DOMAIN HOSTNAME
     local IC_CRYPTO IC_REGISTRY_LOCAL_STORE
     local NNS_URL NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY
     local BACKUP_RETENTION_TIME_SECS BACKUP_PURGING_INTERVAL_SECS
@@ -151,6 +154,9 @@ function build_ic_bootstrap_tar() {
             --ipv6_name_servers)
                 IPV6_NAME_SERVERS="$2"
                 ;;
+            --ipv6_prefixes)
+                IPV6_PREFIXES="$2"
+                ;;
             --ipv4_address)
                 IPV4_ADDRESS="$2"
                 ;;
@@ -226,6 +232,7 @@ function build_ic_bootstrap_tar() {
 ${IPV6_ADDRESS:+ipv6_address=$IPV6_ADDRESS}
 ${IPV6_GATEWAY:+ipv6_gateway=$IPV6_GATEWAY}
 name_servers=$IPV6_NAME_SERVERS
+ipv6_prefixes=$IPV6_PREFIXES
 hostname=$HOSTNAME
 ${IPV4_ADDRESS:+ipv4_address=$IPV4_ADDRESS}
 ${IPV4_GATEWAY:+ipv4_gateway=$IPV4_GATEWAY}

diff --git a/rs/ic_os/launch-single-vm/BUILD.bazel b/rs/ic_os/launch-single-vm/BUILD.bazel
@@ -6,6 +6,7 @@ DEPENDENCIES = [
     "@crate_index//:clap",
     "@crate_index//:reqwest",
     "@crate_index//:serde",
+    "@crate_index//:serde_json",
     "@crate_index//:slog-async",
     "@crate_index//:slog-term",
     "@crate_index//:slog",

diff --git a/rs/ic_os/launch-single-vm/Cargo.toml b/rs/ic_os/launch-single-vm/Cargo.toml
@@ -6,14 +6,15 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-tests = { path = "../../tests"}
-ic-prep = { path = "../../prep"}
-ic-registry-subnet-type = { path = "../../registry/subnet_type"}
+tests = { path = "../../tests" }
+ic-prep = { path = "../../prep" }
+ic-registry-subnet-type = { path = "../../registry/subnet_type" }
 ic-types = { path = "../../types/types" }
 
 clap = { workspace = true }
 reqwest = { workspace = true }
 serde = { workspace = true }
+serde_json = { workspace = true }
 slog-async = { version = "2.5", features = ["nested-values"] }
 slog-term = "2.6.0"
 slog = { workspace = true }

diff --git a/rs/ic_os/launch-single-vm/src/main.rs b/rs/ic_os/launch-single-vm/src/main.rs
@@ -7,6 +7,7 @@ use clap::Parser;
 use reqwest::blocking::Client;
 use serde::Serialize;
 use slog::{o, Drain};
+use std::fs::File;
 use tempfile::tempdir;
 use url::Url;
 
@@ -39,6 +40,9 @@ struct Args {
     /// Path to `build-bootstrap-config-image.sh` script
     #[clap(long)]
     build_bootstrap_script: PathBuf,
+    /// Path to ipv6_prefixes.json
+    #[clap(long)]
+    ipv6_prefixes_path: PathBuf,
     /// Key to be used for `admin` SSH
     #[clap(long)]
     ssh_key_path: Option<PathBuf>,
@@ -60,6 +64,7 @@ fn main() {
     let url = args.url;
     let sha256 = args.sha256;
     let build_bootstrap_script = args.build_bootstrap_script;
+    let ipv6_prefixes_path = args.ipv6_prefixes_path;
     let ssh_key_path = args.ssh_key_path;
 
     let test_name = "test_single_vm";
@@ -173,6 +178,16 @@ fn main() {
     let filename = "config.tar.gz";
     let config_path = tempdir.as_ref().join(filename);
     let local_store = prep_dir.join("ic_registry_local_store");
+    let ipv6_prefixes: Vec<String> =
+        serde_json::from_reader(File::open(ipv6_prefixes_path).unwrap()).unwrap();
+    let ipv6_prefixes = format!(
+        "[{}]",
+        ipv6_prefixes
+            .iter()
+            .map(|s| format!("\"{s}\""))
+            .collect::<Vec<_>>()
+            .join(", "),
+    );
     Command::new(build_bootstrap_script)
         .arg(&config_path)
         .arg("--nns_url")
@@ -183,6 +198,8 @@ fn main() {
         .arg(&local_store)
         .arg("--accounts_ssh_authorized_keys")
         .arg(&keys_dir)
+        .arg("--ipv6_prefixes")
+        .arg(ipv6_prefixes)
         .status()
         .unwrap();
 

diff --git a/rs/tests/BUILD.bazel b/rs/tests/BUILD.bazel
@@ -95,6 +95,10 @@ exports_files([
     "create-universal-vm-config-image.sh",
 ])
 
+exports_files([
+    "src/ipv6_prefixes.json",
+])
+
 filegroup(
     name = "grafana_dashboards",
     srcs = glob(["dashboards/**/*"]),

diff --git a/rs/tests/src/driver/bootstrap.rs b/rs/tests/src/driver/bootstrap.rs
@@ -436,6 +436,18 @@ pub fn create_config_disk_image(
         .prep_dir(ic_name)
         .expect("no no-name IC")
         .registry_local_store_path();
+    let ipv6_prefixes: Vec<String> = test_env
+        .read_json_object("dependencies/rs/tests/src/ipv6_prefixes.json")
+        .unwrap();
+    let ipv6_prefixes = format!(
+        "[{}]",
+        ipv6_prefixes
+            .iter()
+            .map(|s| format!("\"{s}\""))
+            .collect::<Vec<_>>()
+            .join(", "),
+    );
+
     cmd.arg(img_path.clone())
         .arg("--hostname")
         .arg(node.node_id.to_string())
@@ -444,7 +456,9 @@ pub fn create_config_disk_image(
         .arg("--ic_crypto")
         .arg(node.crypto_path())
         .arg("--elasticsearch_tags")
-        .arg(format!("system_test {}", group_name));
+        .arg(format!("system_test {}", group_name))
+        .arg("--ipv6_prefixes")
+        .arg(ipv6_prefixes);
 
     // We've seen k8s nodes fail to pick up RA correctly, so we specify their
     // addresses directly. Ideally, all nodes should do this, to match mainnet.

diff --git a/rs/tests/src/ipv6_prefixes.json b/rs/tests/src/ipv6_prefixes.json
@@ -0,0 +1,76 @@
+[
+    "2001:438:fffd:11c::/64",
+    "2001:470:1:c76::/64",
+    "2001:4d78:400:10a::/64",
+    "2001:4d78:40d::/48",
+    "2602:fb2b:110::/48",
+    "2001:920:401a:1706::/64",
+    "2001:920:401a:1708::/64",
+    "2001:920:401a:1710::/64",
+    "2401:3f00:1000:22::/64",
+    "2401:3f00:1000:23::/64",
+    "2401:3f00:1000:24::/64",
+    "2600:2c01:21::/64",
+    "2600:3000:1300:1300::/64",
+    "2600:3000:6100:200::/64",
+    "2600:3004:1200:1200::/56",
+    "2600:3006:1400:1500::/64",
+    "2600:c00:2:100::/64",
+    "2600:c02:b002:15::/64",
+    "2600:c0d:3002:4::/64",
+    "2602:ffe4:801:16::/64",
+    "2602:ffe4:801:17::/64",
+    "2602:ffe4:801:18::/64",
+    "2604:1380:4091:3000::/64",
+    "2604:1380:40e1:4700::/64",
+    "2604:1380:40f1:1700::/64",
+    "2604:1380:45d1:bf00::/64",
+    "2604:1380:45e1:a600::/64",
+    "2604:1380:45f1:9400::/64",
+    "2604:1380:4601:6200::/64",
+    "2604:1380:4601:6201::/64",
+    "2604:1380:4601:6202::/64",
+    "2604:1380:4641:6101::/64",
+    "2604:1380:4641:6102::/64",
+    "2604:1380:4091:3001::/64",
+    "2604:1380:4091:3002::/64",
+    "2604:1380:45e1:a601::/64",
+    "2604:1380:45e1:a602::/64",
+    "2604:1380:4641:6100::/64",
+    "2604:3fc0:2001::/48",
+    "2604:3fc0:3002::/48",
+    "2604:6800:258:1::/64",
+    "2604:7e00:30:3::/64",
+    "2604:7e00:50::/64",
+    "2604:b900:4001:76::/64",
+    "2607:f1d0:10:1::/64",
+    "2607:f6f0:3004::/48",
+    "2602:fb2b:120::/48",
+    "2607:f758:1220::/64",
+    "2607:f758:c300::/64",
+    "2607:fb58:9005::/48",
+    "2602:fb2b:100::/48",
+    "2607:ff70:3:2::/64",
+    "2610:190:6000:1::/64",
+    "2610:190:df01:5::/64",
+    "2a00:fa0:3::/48",
+    "2a00:fb01:400:100::/56",
+    "2a00:fb01:400::/56",
+    "2a00:fc0:5000:300::/64",
+    "2a01:138:900a::/48",
+    "2a01:2a8:a13c:1::/64",
+    "2a01:2a8:a13d:1::/64",
+    "2a01:2a8:a13e:1::/64",
+    "2a02:418:3002:0::/64",
+    "2a02:41b:300e::/48",
+    "2a02:800:2:2003::/64",
+    "2a04:9dc0:0:108::/64",
+    "2a05:d014:939:bf00::/56",
+    "2a05:d01c:d9:2b00::/56",
+    "2a05:d01c:e2c:a700::/56",
+    "2a0b:21c0:4003:2::/64",
+    "2a0b:21c0:b002:2::/64",
+    "2a0f:cd00:0002::/56",
+    "fd00:2:1:1::/64",
+    "fda6:8d22:43e1::/48"
+]
diff --git a/rs/tests/system_tests.bzl b/rs/tests/system_tests.bzl
@@ -154,6 +154,9 @@ def system_test(
         **kwargs
     )
 
+    # Every system test has this dependency
+    runtime_deps = runtime_deps + ["//rs/tests:src/ipv6_prefixes.json"]
+
     # Automatically detect system tests that use guestos dev for back compatibility.
     for _d in runtime_deps:
         if _d == GUESTOS_DEV_VERSION: