diff --git a/ic-os/defs.bzl b/ic-os/defs.bzl index bafc77744ac..c413401bd5c 100644 --- a/ic-os/defs.bzl +++ b/ic-os/defs.bzl @@ -480,6 +480,7 @@ def icos_build( ":disk-img.tar.zst.cas-url", ":disk-img.tar.zst.sha256", "//ic-os:scripts/build-bootstrap-config-image.sh", + "//rs/tests:src/default_firewall_whitelist.conf", ":version.txt", ], outs = ["launch_remote_vm_script"], @@ -489,11 +490,12 @@ def icos_build( URL="$$(cat $(location :disk-img.tar.zst.cas-url))" SHA="$$(cat $(location :disk-img.tar.zst.sha256))" SCRIPT="$(location //ic-os:scripts/build-bootstrap-config-image.sh)" + DEFAULT_FIREWALL_WHITELIST="$(location //rs/tests:src/default_firewall_whitelist.conf)" cat < $@ #!/usr/bin/env bash set -euo pipefail cd "\\$$BUILD_WORKSPACE_DIRECTORY" -$$BIN --version "$$VERSION" --url "$$URL" --sha256 "$$SHA" --build-bootstrap-script "$$SCRIPT" +$$BIN --version "$$VERSION" --url "$$URL" --sha256 "$$SHA" --build-bootstrap-script "$$SCRIPT" --default-firewall-whitelist "$$DEFAULT_FIREWALL_WHITELIST" EOF """, executable = True, diff --git a/ic-os/guestos/rootfs/etc/systemd/system/ic-replica.service b/ic-os/guestos/rootfs/etc/systemd/system/ic-replica.service index c6b38ee6b6c..a59210d6310 100644 --- a/ic-os/guestos/rootfs/etc/systemd/system/ic-replica.service +++ b/ic-os/guestos/rootfs/etc/systemd/system/ic-replica.service @@ -28,7 +28,7 @@ Environment=RUST_BACKTRACE=1 # Remember to update 'rs/default.nix' for nix-shell users # Remember to update 'src/dfx/src/actors/replica.rs' in the sdk repo for dfx users Environment=RUST_MIN_STACK=8192000 -ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -n /boot/config/network.conf -c /boot/config/nns.conf -b /boot/config/backup.conf -l /boot/config/log.conf -m /boot/config/malicious_behavior.conf -q /boot/config/query_stats.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 +ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -n /boot/config/network.conf -c /boot/config/nns.conf -b /boot/config/backup.conf -l /boot/config/log.conf -m /boot/config/malicious_behavior.conf -q /boot/config/query_stats.conf -w /boot/config/default_firewall_whitelist.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5 ExecStart=/opt/ic/bin/orchestrator --replica-binary-dir /var/lib/ic/data/images --cup-dir /var/lib/ic/data/cups --replica-config-file /run/ic-node/config/ic.json5 --enable-provisional-registration --ic-binary-directory /opt/ic/bin --orchestrator-data-directory /var/lib/ic/data/orchestrator --version-file /opt/ic/share/version.txt LimitNOFILE=1048576 Restart=always diff --git a/ic-os/guestos/rootfs/opt/ic/bin/bootstrap-ic-node.sh b/ic-os/guestos/rootfs/opt/ic/bin/bootstrap-ic-node.sh index 65c9cc9825a..ff333eb4136 100755 --- a/ic-os/guestos/rootfs/opt/ic/bin/bootstrap-ic-node.sh +++ b/ic-os/guestos/rootfs/opt/ic/bin/bootstrap-ic-node.sh @@ -127,7 +127,7 @@ function process_bootstrap() { # stash the following configuration files to config store # note: keep this list in sync with configurations supported in build-bootstrap-config-image.sh - for FILE in filebeat.conf network.conf nns.conf backup.conf log.conf malicious_behavior.conf query_stats.conf bitcoind_addr.conf socks_proxy.conf; do + for FILE in filebeat.conf network.conf nns.conf backup.conf log.conf malicious_behavior.conf query_stats.conf bitcoind_addr.conf socks_proxy.conf default_firewall_whitelist.conf; do if [ -e "${TMPDIR}/${FILE}" ]; then echo "Setting up ${FILE}" cp "${TMPDIR}/${FILE}" "${CONFIG_ROOT}/${FILE}" diff --git a/ic-os/guestos/rootfs/opt/ic/bin/generate-replica-config.sh b/ic-os/guestos/rootfs/opt/ic/bin/generate-replica-config.sh index 42f91b027de..2d0c175aa77 100755 --- a/ic-os/guestos/rootfs/opt/ic/bin/generate-replica-config.sh +++ b/ic-os/guestos/rootfs/opt/ic/bin/generate-replica-config.sh @@ -6,7 +6,7 @@ function usage() { cat <"${OUT_FILE}" # umask for service is set to be restricted, but this file needs to be diff --git a/ic-os/guestos/rootfs/opt/ic/share/ic.json5.template b/ic-os/guestos/rootfs/opt/ic/share/ic.json5.template index 63747fbb6bc..1f2ffa785b1 100644 --- a/ic-os/guestos/rootfs/opt/ic/share/ic.json5.template +++ b/ic-os/guestos/rootfs/opt/ic/share/ic.json5.template @@ -267,81 +267,7 @@ table ip6 filter {\n\ ipv6_user_output_rule_template: "meta skuid <> ip6 daddr {<>} ct state { new } tcp dport {<>} <> # <>", default_rules: [{ ipv4_prefixes: [], - ipv6_prefixes: [ - "2001:438:fffd:11c::/64", - "2001:470:1:c76::/64", - "2001:4d78:400:10a::/64", - "2001:4d78:40d::/48", - "2602:fb2b:110::/48", - "2001:920:401a:1706::/64", - "2001:920:401a:1708::/64", - "2001:920:401a:1710::/64", - "2401:3f00:1000:22::/64", - "2401:3f00:1000:23::/64", - "2401:3f00:1000:24::/64", - "2600:2c01:21::/64", - "2600:3000:1300:1300::/64", - "2600:3000:6100:200::/64", - "2600:3004:1200:1200::/56", - "2600:3006:1400:1500::/64", - "2600:c00:2:100::/64", - "2600:c02:b002:15::/64", - "2600:c0d:3002:4::/64", - "2602:ffe4:801:16::/64", - "2602:ffe4:801:17::/64", - "2602:ffe4:801:18::/64", - "2604:1380:4091:3000::/64", - "2604:1380:40e1:4700::/64", - "2604:1380:40f1:1700::/64", - "2604:1380:45d1:bf00::/64", - "2604:1380:45e1:a600::/64", - "2604:1380:45f1:9400::/64", - "2604:1380:4601:6200::/64", - "2604:1380:4601:6201::/64", - "2604:1380:4601:6202::/64", - "2604:1380:4641:6101::/64", - "2604:1380:4641:6102::/64", - "2604:1380:4091:3001::/64", - "2604:1380:4091:3002::/64", - "2604:1380:45e1:a601::/64", - "2604:1380:45e1:a602::/64", - "2604:1380:4641:6100::/64", - "2604:3fc0:2001::/48", - "2604:3fc0:3002::/48", - "2604:6800:258:1::/64", - "2604:7e00:30:3::/64", - "2604:7e00:50::/64", - "2604:b900:4001:76::/64", - "2607:f1d0:10:1::/64", - "2607:f6f0:3004::/48", - "2602:fb2b:120::/48", - "2607:f758:1220::/64", - "2607:f758:c300::/64", - "2607:fb58:9005::/48", - "2602:fb2b:100::/48", - "2607:ff70:3:2::/64", - "2610:190:6000:1::/64", - "2610:190:df01:5::/64", - "2a00:fa0:3::/48", - "2a00:fb01:400:100::/56", - "2a00:fb01:400::/56", - "2a00:fc0:5000:300::/64", - "2a01:138:900a::/48", - "2a01:2a8:a13c:1::/64", - "2a01:2a8:a13d:1::/64", - "2a01:2a8:a13e:1::/64", - "2a02:418:3002:0::/64", - "2a02:41b:300e::/48", - "2a02:800:2:2003::/64", - "2a04:9dc0:0:108::/64", - "2a05:d014:939:bf00::/56", - "2a05:d01c:d9:2b00::/56", - "2a05:d01c:e2c:a700::/56", - "2a0b:21c0:4003:2::/64", - "2a0b:21c0:b002:2::/64", - "2a0f:cd00:0002::/56", - "fd00:2:1:1::/64", - ], + ipv6_prefixes: [{{ ipv6_whitelist }}], ports: [22, 2497, 4100, 7070, 8080, 9090, 9091, 9100, 19100, 19531], action: 1, comment: "Default rule from template", diff --git a/ic-os/hostos/rootfs/opt/ic/bin/generate-guestos-config.sh b/ic-os/hostos/rootfs/opt/ic/bin/generate-guestos-config.sh index d72baf421ff..ea5615111bb 100755 --- a/ic-os/hostos/rootfs/opt/ic/bin/generate-guestos-config.sh +++ b/ic-os/hostos/rootfs/opt/ic/bin/generate-guestos-config.sh @@ -114,6 +114,9 @@ function assemble_config_media() { if [ -f "/boot/config/node_operator_private_key.pem" ]; then cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem") fi + if [ -f "/boot/config/default_firewall_whitelist.conf" ]; then + cmd+=(--default_firewall_whitelist "/boot/config/default_firewall_whitelist.conf") + fi # Run the above command "${cmd[@]}" diff --git a/ic-os/scripts/build-bootstrap-config-image.sh b/ic-os/scripts/build-bootstrap-config-image.sh index c35fd02c8e6..98ae3498a87 100755 --- a/ic-os/scripts/build-bootstrap-config-image.sh +++ b/ic-os/scripts/build-bootstrap-config-image.sh @@ -27,6 +27,10 @@ options may be specified: --ipv6_gateway a:b::c Default IPv6 gateway. + --default_firewall_whitelist path + File containing the default whitelist configuration for GuestOS on + orchestrator startup. + --ipv4_address a.b.c.d/n (optional) The IPv4 address to assign. Must include prefix length (e.g. 18.208.190.35/28). @@ -134,6 +138,7 @@ function build_ic_bootstrap_tar() { local MALICIOUS_BEHAVIOR local QUERY_STATS_EPOCH_LENGTH local BITCOIND_ADDR + local DEFAULT_FIREWALL_WHITELIST while true; do if [ $# == 0 ]; then @@ -147,6 +152,9 @@ function build_ic_bootstrap_tar() { --ipv6_gateway) IPV6_GATEWAY="$2" ;; + --default_firewall_whitelist) + DEFAULT_FIREWALL_WHITELIST="$2" + ;; --ipv4_address) IPV4_ADDRESS="$2" ;; @@ -272,6 +280,9 @@ EOF if [ "${NODE_OPERATOR_PRIVATE_KEY}" != "" ]; then cp "${NODE_OPERATOR_PRIVATE_KEY}" "${BOOTSTRAP_TMPDIR}/node_operator_private_key.pem" fi + if [ "${DEFAULT_FIREWALL_WHITELIST}" != "" ]; then + cp "${DEFAULT_FIREWALL_WHITELIST}" "${BOOTSTRAP_TMPDIR}/default_firewall_whitelist.conf" + fi tar cf "${OUT_FILE}" \ --sort=name \ diff --git a/ic-os/setupos/rootfs/opt/ic/bin/config.sh b/ic-os/setupos/rootfs/opt/ic/bin/config.sh index 97326544d7f..cb0d2455dff 100755 --- a/ic-os/setupos/rootfs/opt/ic/bin/config.sh +++ b/ic-os/setupos/rootfs/opt/ic/bin/config.sh @@ -48,6 +48,11 @@ function clone_config() { log_and_reboot_on_error "${?}" "Unable to copy 'node_operator_private_key.pem' configuration file." fi + if [ -f "${CONFIG_DIR}/default_firewall_whitelist.conf" ]; then + cp ${CONFIG_DIR}/default_firewall_whitelist.conf ${CONFIG_TMP}/default_firewall_whitelist.conf + log_and_reboot_on_error "${?}" "Unable to copy 'default_firewall_whitelist.conf' configuration file." + fi + if [ -d "${SSH_AUTHORIZED_KEYS}" ]; then cp -r "${SSH_AUTHORIZED_KEYS}" "${CONFIG_TMP}" log_and_reboot_on_error "${?}" "Unable to copy 'ssh_authorized_keys' directory." diff --git a/ic-os/setupos/rootfs/opt/ic/bin/devices.sh b/ic-os/setupos/rootfs/opt/ic/bin/devices.sh index ddfb235d26c..ced1d0640fa 100755 --- a/ic-os/setupos/rootfs/opt/ic/bin/devices.sh +++ b/ic-os/setupos/rootfs/opt/ic/bin/devices.sh @@ -42,6 +42,11 @@ function copy_config_files() { echo "node_operator_private_key.pem does not exist, requiring HSM." fi + if [ -f "${CONFIG_DIR}/default_firewall_whitelist.conf" ]; then + cp ${CONFIG_DIR}/default_firewall_whitelist.conf /media/ + log_and_reboot_on_error "${?}" "Unable to copy default_firewall_whitelist.conf to hostOS config partition." + fi + echo "* Copying deployment.json to config partition..." cp /data/deployment.json /media/ log_and_reboot_on_error "${?}" "Unable to copy deployment.json to hostOS config partition." diff --git a/rs/ic_os/launch-single-vm/Cargo.toml b/rs/ic_os/launch-single-vm/Cargo.toml index 93e765145ff..a8817e9a7bb 100644 --- a/rs/ic_os/launch-single-vm/Cargo.toml +++ b/rs/ic_os/launch-single-vm/Cargo.toml @@ -6,9 +6,9 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tests = { path = "../../tests"} -ic-prep = { path = "../../prep"} -ic-registry-subnet-type = { path = "../../registry/subnet_type"} +tests = { path = "../../tests" } +ic-prep = { path = "../../prep" } +ic-registry-subnet-type = { path = "../../registry/subnet_type" } ic-types = { path = "../../types/types" } clap = { workspace = true } diff --git a/rs/ic_os/launch-single-vm/src/main.rs b/rs/ic_os/launch-single-vm/src/main.rs index 4458870f8d4..137ac711dd0 100644 --- a/rs/ic_os/launch-single-vm/src/main.rs +++ b/rs/ic_os/launch-single-vm/src/main.rs @@ -39,6 +39,9 @@ struct Args { /// Path to `build-bootstrap-config-image.sh` script #[clap(long)] build_bootstrap_script: PathBuf, + /// Path to `default_firewall_whitelist.conf` file + #[clap(long)] + default_firewall_whitelist: PathBuf, /// Key to be used for `admin` SSH #[clap(long)] ssh_key_path: Option, @@ -60,6 +63,7 @@ fn main() { let url = args.url; let sha256 = args.sha256; let build_bootstrap_script = args.build_bootstrap_script; + let default_firewall_whitelist = args.default_firewall_whitelist; let ssh_key_path = args.ssh_key_path; let test_name = "test_single_vm"; @@ -183,6 +187,8 @@ fn main() { .arg(&local_store) .arg("--accounts_ssh_authorized_keys") .arg(&keys_dir) + .arg("--default_firewall_whitelist") + .arg(default_firewall_whitelist) .status() .unwrap(); diff --git a/rs/ic_os/setupos-inject-configuration/src/main.rs b/rs/ic_os/setupos-inject-configuration/src/main.rs index 582570c39fa..dd040d3c6c7 100644 --- a/rs/ic_os/setupos-inject-configuration/src/main.rs +++ b/rs/ic_os/setupos-inject-configuration/src/main.rs @@ -34,6 +34,9 @@ struct Cli { #[command(flatten)] deployment: DeploymentConfig, + + #[arg(long)] + default_firewall_whitelist_path: Option, } #[derive(Args)] @@ -132,6 +135,17 @@ async fn main() -> Result<(), Error> { .context("failed to copy public keys")?; } + // Update default firewall whitelist + if let Some(default_firewall_whitelist_path) = cli.default_firewall_whitelist_path { + config + .write_file( + &default_firewall_whitelist_path, + Path::new("/default_firewall_whitelist.conf"), + ) + .await + .context("failed to copy default firewall config")?; + } + // Close config partition config.close().await?; diff --git a/rs/prep/src/internet_computer.rs b/rs/prep/src/internet_computer.rs index 85841400356..f7c00ac180d 100644 --- a/rs/prep/src/internet_computer.rs +++ b/rs/prep/src/internet_computer.rs @@ -388,14 +388,15 @@ impl IcConfig { let mut mutations = self.initial_mutations.clone(); if let Some(prefixes) = self.whitelisted_prefixes { - let ports = if let Some(ports) = self.whitelisted_ports { - ports + // Port 8080 is always included. + let mut ports = vec![8080]; + + if let Some(additional_ports) = self.whitelisted_ports { + ports = additional_ports .split(',') - .map(|port| port.parse::().unwrap()) - .chain(std::iter::once(8080)) + .map(|port| port.trim().parse::().unwrap()) + .chain(ports) .collect() - } else { - vec![8080] }; mutations.extend(vec![insert( @@ -403,7 +404,7 @@ impl IcConfig { encode_or_panic(&FirewallRuleSet { entries: vec![FirewallRule { ipv4_prefixes: Vec::new(), - ipv6_prefixes: prefixes.split(',').map(|v| v.to_string()).collect(), + ipv6_prefixes: prefixes.split(',').map(|v| v.trim().to_string()).collect(), ports, action: FirewallAction::Allow as i32, comment: "Globally allow provided prefixes for testing".to_string(), diff --git a/rs/tests/BUILD.bazel b/rs/tests/BUILD.bazel index c2f22c00ec0..4fb4ba03460 100644 --- a/rs/tests/BUILD.bazel +++ b/rs/tests/BUILD.bazel @@ -95,6 +95,10 @@ exports_files([ "create-universal-vm-config-image.sh", ]) +exports_files([ + "src/default_firewall_whitelist.conf", +]) + filegroup( name = "grafana_dashboards", srcs = glob(["dashboards/**/*"]), diff --git a/rs/tests/common.bzl b/rs/tests/common.bzl index e36b9684cf4..49fa45e468f 100644 --- a/rs/tests/common.bzl +++ b/rs/tests/common.bzl @@ -181,6 +181,7 @@ GUESTOS_DEV_VERSION = "//ic-os/guestos/envs/dev:version.txt" GUESTOS_RUNTIME_DEPS = [ GUESTOS_DEV_VERSION, "//ic-os:scripts/build-bootstrap-config-image.sh", + "//rs/tests:src/default_firewall_whitelist.conf", ] MAINNET_REVISION_RUNTIME_DEPS = ["//testnet:mainnet_nns_revision"] diff --git a/rs/tests/consensus/BUILD.bazel b/rs/tests/consensus/BUILD.bazel index 7bcaf808df0..e3922d033f3 100644 --- a/rs/tests/consensus/BUILD.bazel +++ b/rs/tests/consensus/BUILD.bazel @@ -33,7 +33,7 @@ system_test( ], target_compatible_with = ["@platforms//os:linux"], # requires libssh that does not build on Mac OS uses_guestos_dev = True, - runtime_deps = ["//ic-os:scripts/build-bootstrap-config-image.sh"] + GRAFANA_RUNTIME_DEPS, + runtime_deps = GUESTOS_RUNTIME_DEPS + GRAFANA_RUNTIME_DEPS, deps = DEPENDENCIES + ["//rs/tests"], ) @@ -47,7 +47,7 @@ system_test( ], target_compatible_with = ["@platforms//os:linux"], # requires libssh that does not build on Mac OS uses_guestos_dev = True, - runtime_deps = ["//ic-os:scripts/build-bootstrap-config-image.sh"] + GRAFANA_RUNTIME_DEPS, + runtime_deps = GUESTOS_RUNTIME_DEPS + GRAFANA_RUNTIME_DEPS, deps = DEPENDENCIES + ["//rs/tests"], ) @@ -77,7 +77,7 @@ system_test( target_compatible_with = ["@platforms//os:linux"], # requires libssh that does not build on Mac OS uses_guestos_dev = True, uses_guestos_dev_test = True, - runtime_deps = ["//ic-os:scripts/build-bootstrap-config-image.sh"], + runtime_deps = GUESTOS_RUNTIME_DEPS, deps = DEPENDENCIES + ["//rs/tests"], ) @@ -134,7 +134,7 @@ system_test( target_compatible_with = ["@platforms//os:linux"], # requires libssh that does not build on Mac OS uses_guestos_dev = True, uses_guestos_dev_test = True, - runtime_deps = ["//ic-os:scripts/build-bootstrap-config-image.sh"], + runtime_deps = GUESTOS_RUNTIME_DEPS, deps = DEPENDENCIES + ["//rs/tests"], ) @@ -146,7 +146,7 @@ system_test( target_compatible_with = ["@platforms//os:linux"], # requires libssh that does not build on Mac OS uses_guestos_dev = True, uses_guestos_dev_test = True, - runtime_deps = ["//ic-os:scripts/build-bootstrap-config-image.sh"], + runtime_deps = GUESTOS_RUNTIME_DEPS, deps = DEPENDENCIES + ["//rs/tests"], ) @@ -158,7 +158,7 @@ system_test( target_compatible_with = ["@platforms//os:linux"], # requires libssh that does not build on Mac OS uses_guestos_dev = True, uses_guestos_dev_test = True, - runtime_deps = ["//ic-os:scripts/build-bootstrap-config-image.sh"], + runtime_deps = GUESTOS_RUNTIME_DEPS, deps = DEPENDENCIES + ["//rs/tests"], ) diff --git a/rs/tests/consensus/tecdsa/BUILD.bazel b/rs/tests/consensus/tecdsa/BUILD.bazel index 23a88702872..3f4353db69b 100644 --- a/rs/tests/consensus/tecdsa/BUILD.bazel +++ b/rs/tests/consensus/tecdsa/BUILD.bazel @@ -11,7 +11,7 @@ system_test( target_compatible_with = ["@platforms//os:linux"], # requires libssh that does not build on Mac OS uses_guestos_dev = True, uses_guestos_dev_test = True, - runtime_deps = ["//ic-os:scripts/build-bootstrap-config-image.sh"] + NNS_CANISTER_RUNTIME_DEPS, + runtime_deps = GUESTOS_RUNTIME_DEPS + NNS_CANISTER_RUNTIME_DEPS, deps = DEPENDENCIES + ["//rs/tests"], ) diff --git a/rs/tests/message_routing/BUILD.bazel b/rs/tests/message_routing/BUILD.bazel index 49fcd03deaf..c812e2b318a 100644 --- a/rs/tests/message_routing/BUILD.bazel +++ b/rs/tests/message_routing/BUILD.bazel @@ -49,6 +49,6 @@ system_test( ], target_compatible_with = ["@platforms//os:linux"], # requires libssh that does not build on Mac OS test_timeout = "eternal", - runtime_deps = ["//ic-os:scripts/build-bootstrap-config-image.sh"] + GRAFANA_RUNTIME_DEPS + NNS_CANISTER_RUNTIME_DEPS + STATESYNC_TEST_CANISTER_RUNTIME_DEPS, + runtime_deps = GUESTOS_RUNTIME_DEPS + GRAFANA_RUNTIME_DEPS + NNS_CANISTER_RUNTIME_DEPS + STATESYNC_TEST_CANISTER_RUNTIME_DEPS, deps = DEPENDENCIES + ["//rs/tests"], ) diff --git a/rs/tests/message_routing/xnet/BUILD.bazel b/rs/tests/message_routing/xnet/BUILD.bazel index 37912c1684a..c8361a36106 100644 --- a/rs/tests/message_routing/xnet/BUILD.bazel +++ b/rs/tests/message_routing/xnet/BUILD.bazel @@ -65,7 +65,7 @@ system_test( ], target_compatible_with = ["@platforms//os:linux"], # requires libssh that does not build on Mac OS test_timeout = "long", - runtime_deps = ["//ic-os:scripts/build-bootstrap-config-image.sh"] + GRAFANA_RUNTIME_DEPS + XNET_TEST_CANISTER_RUNTIME_DEPS, + runtime_deps = GUESTOS_RUNTIME_DEPS + GRAFANA_RUNTIME_DEPS + XNET_TEST_CANISTER_RUNTIME_DEPS, deps = DEPENDENCIES + ["//rs/tests"], ) diff --git a/rs/tests/src/default_firewall_whitelist.conf b/rs/tests/src/default_firewall_whitelist.conf new file mode 100644 index 00000000000..8dcc06809d0 --- /dev/null +++ b/rs/tests/src/default_firewall_whitelist.conf @@ -0,0 +1 @@ +ipv6_whitelist="2001:438:fffd:11c::/64", "2001:470:1:c76::/64", "2001:4d78:400:10a::/64", "2001:4d78:40d::/48", "2602:fb2b:110::/48", "2001:920:401a:1706::/64", "2001:920:401a:1708::/64", "2001:920:401a:1710::/64", "2401:3f00:1000:22::/64", "2401:3f00:1000:23::/64", "2401:3f00:1000:24::/64", "2600:2c01:21::/64", "2600:3000:1300:1300::/64", "2600:3000:6100:200::/64", "2600:3004:1200:1200::/56", "2600:3006:1400:1500::/64", "2600:c00:2:100::/64", "2600:c02:b002:15::/64", "2600:c0d:3002:4::/64", "2602:ffe4:801:16::/64", "2602:ffe4:801:17::/64", "2602:ffe4:801:18::/64", "2604:1380:4091:3000::/64", "2604:1380:40e1:4700::/64", "2604:1380:40f1:1700::/64", "2604:1380:45d1:bf00::/64", "2604:1380:45e1:a600::/64", "2604:1380:45f1:9400::/64", "2604:1380:4601:6200::/64", "2604:1380:4601:6201::/64", "2604:1380:4601:6202::/64", "2604:1380:4641:6101::/64", "2604:1380:4641:6102::/64", "2604:1380:4091:3001::/64", "2604:1380:4091:3002::/64", "2604:1380:45e1:a601::/64", "2604:1380:45e1:a602::/64", "2604:1380:4641:6100::/64", "2604:3fc0:2001::/48", "2604:3fc0:3002::/48", "2604:6800:258:1::/64", "2604:7e00:30:3::/64", "2604:7e00:50::/64", "2604:b900:4001:76::/64", "2607:f1d0:10:1::/64", "2607:f6f0:3004::/48", "2602:fb2b:120::/48", "2607:f758:1220::/64", "2607:f758:c300::/64", "2607:fb58:9005::/48", "2602:fb2b:100::/48", "2607:ff70:3:2::/64", "2610:190:6000:1::/64", "2610:190:df01:5::/64", "2a00:fa0:3::/48", "2a00:fb01:400:100::/56", "2a00:fb01:400::/56", "2a00:fc0:5000:300::/64", "2a01:138:900a::/48", "2a01:2a8:a13c:1::/64", "2a01:2a8:a13d:1::/64", "2a01:2a8:a13e:1::/64", "2a02:418:3002:0::/64", "2a02:41b:300e::/48", "2a02:800:2:2003::/64", "2a04:9dc0:0:108::/64", "2a05:d014:939:bf00::/56", "2a05:d01c:d9:2b00::/56", "2a05:d01c:e2c:a700::/56", "2a0b:21c0:4003:2::/64", "2a0b:21c0:b002:2::/64", "2a0f:cd00:0002::/56", "fd00:2:1:1::/64", "fda6:8d22:43e1::/48" diff --git a/rs/tests/src/driver/bootstrap.rs b/rs/tests/src/driver/bootstrap.rs index 46c317e2019..f1158ac154f 100644 --- a/rs/tests/src/driver/bootstrap.rs +++ b/rs/tests/src/driver/bootstrap.rs @@ -214,6 +214,28 @@ pub fn init_ic( )); } + // Due to a change in how default firewall rules are supplied, they are + // not preserved across the transitional upgrade. We temporarily stash + // the whitelist in the registry for the time being. + // THIS PATH SHOULD BE REMOVED. + if ic.with_forced_default_firewall { + let default_firewall_whitelist_path = + test_env.get_dependency_path("rs/tests/src/default_firewall_whitelist.conf"); + let default_firewall_whitelist = std::fs::read_to_string(default_firewall_whitelist_path) + .expect("Could not read default whitelist"); + let ipv6_whitelist = default_firewall_whitelist + .lines() + .find_map(|v| v.strip_prefix("ipv6_whitelist=")) + .expect("Could not find ipv6_whitelist") + .replace('"', "") + .to_string(); + + ic_config.set_whitelisted_prefixes(Some(ipv6_whitelist)); + ic_config.set_whitelisted_ports(Some( + "22,2497,4100,7070,8080,9090,9091,9100,19100,19531".to_string(), + )); + } + info!(test_env.logger(), "Initializing via {:?}", &ic_config); Ok(ic_config.initialize()?) @@ -400,6 +422,9 @@ fn create_config_disk_image( .prep_dir(ic_name) .expect("no no-name IC") .registry_local_store_path(); + let default_firewall_whitelist_path = + test_env.get_dependency_path("rs/tests/src/default_firewall_whitelist.conf"); + cmd.arg(img_path.clone()) .arg("--hostname") .arg(node.node_id.to_string()) @@ -408,7 +433,9 @@ fn create_config_disk_image( .arg("--ic_crypto") .arg(node.crypto_path()) .arg("--elasticsearch_tags") - .arg(format!("system_test {}", group_name)); + .arg(format!("system_test {}", group_name)) + .arg("--default_firewall_whitelist") + .arg(default_firewall_whitelist_path); // We've seen k8s nodes fail to pick up RA correctly, so we specify their // addresses directly. Ideally, all nodes should do this, to match mainnet. @@ -586,6 +613,9 @@ fn configure_setupos_image( .map(|v| v.lines().map(|v| v.to_owned()).collect()) .unwrap_or_default(); + let default_firewall_whitelist = + env.get_dependency_path("rs/tests/src/default_firewall_whitelist.conf"); + // TODO: We transform the IPv6 to get this information, but it could be // passed natively. let old_ip = nested_vm.get_vm()?.ipv6; @@ -633,6 +663,8 @@ fn configure_setupos_image( .arg(&uncompressed_image) .arg("--mgmt-mac") .arg(&mac) + .arg("--default-firewall-whitelist-path") + .arg(&default_firewall_whitelist) .arg("--ipv6-prefix") .arg(&prefix) .arg("--ipv6-gateway") diff --git a/rs/tests/src/driver/ic.rs b/rs/tests/src/driver/ic.rs index fa0be39b0f2..50a63ffacec 100644 --- a/rs/tests/src/driver/ic.rs +++ b/rs/tests/src/driver/ic.rs @@ -49,6 +49,11 @@ pub struct InternetComputer { /// Indicates whether this `InternetComputer` instance should be installed with /// GuestOS disk images of the latest-deployed mainnet version. pub with_mainnet_config: bool, + // Due to a change in how default firewall rules are supplied, they are + // not preserved across the transitional upgrade. We temporarily stash + // the whitelist in the registry for the time being. + // THIS PATH SHOULD BE REMOVED. + pub with_forced_default_firewall: bool, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] @@ -193,6 +198,15 @@ impl InternetComputer { self } + // Due to a change in how default firewall rules are supplied, they are + // not preserved across the transitional upgrade. We temporarily stash + // the whitelist in the registry for the time being. + // THIS PATH SHOULD BE REMOVED. + pub fn with_forced_default_firewall(mut self) -> Self { + self.with_forced_default_firewall = true; + self + } + pub fn setup_and_start(&mut self, env: &TestEnv) -> Result<()> { // propagate required host features and resource settings to all vms let farm = Farm::from_test_env(env, "Internet Computer"); diff --git a/rs/tests/src/message_routing/compatibility.rs b/rs/tests/src/message_routing/compatibility.rs index 3ddbaf2b916..001f6f7ec7b 100644 --- a/rs/tests/src/message_routing/compatibility.rs +++ b/rs/tests/src/message_routing/compatibility.rs @@ -80,6 +80,13 @@ fn setup(env: TestEnv, config: Config) { .expect("failed to start prometheus VM"); } let ic = InternetComputer::new().with_mainnet_config(); + + // Due to a change in how default firewall rules are supplied, they are + // not preserved across the transitional upgrade. We temporarily stash + // the whitelist in the registry for the time being. + // THIS PATH SHOULD BE REMOVED. + let ic = ic.with_forced_default_firewall(); + ic.add_subnet(subnet(SubnetType::System, None)) .add_subnet(subnet(SubnetType::Application, Some(DKG_INTERVAL))) .add_subnet(subnet(SubnetType::Application, Some(DKG_INTERVAL))) diff --git a/rs/tests/src/networking/firewall_priority.rs b/rs/tests/src/networking/firewall_priority.rs index 05d45a4c9a1..cfaed006eb8 100644 --- a/rs/tests/src/networking/firewall_priority.rs +++ b/rs/tests/src/networking/firewall_priority.rs @@ -28,8 +28,8 @@ end::catalog[] */ use crate::driver::ic::{InternetComputer, Subnet}; use crate::driver::test_env::TestEnv; use crate::driver::test_env_api::{ - HasPublicApiUrl, HasTopologySnapshot, IcNodeContainer, IcNodeSnapshot, NnsInstallationBuilder, - SshSession, + HasDependencies, HasPublicApiUrl, HasTopologySnapshot, IcNodeContainer, IcNodeSnapshot, + NnsInstallationBuilder, SshSession, }; use crate::nns::{ await_proposal_execution, submit_external_proposal_with_test_id, @@ -111,16 +111,20 @@ pub fn override_firewall_rules_with_priority(env: TestEnv) { info!( log, - "Firewall priority test is ready. Setting default rules in the registry..." + "Firewall priority test is ready. Setting base rules in the registry..." ); - // Set the default rules in the registry for the first time - block_on(set_default_registry_rules(&log, &nns_node)); + // Grab the prefixes from the test whitelist + let ipv6_prefixes = read_firewall_whitelist_config(&env); - info!( - log, - "Default rules set. Testing connectivity with backoff..." - ); + // Set the base rules in the registry for the first time + block_on(set_base_registry_rules( + &log, + &nns_node, + ipv6_prefixes.clone(), + )); + + info!(log, "Base rules set. Testing connectivity with backoff..."); assert!(await_rule_execution_with_backoff( &log, @@ -140,12 +144,10 @@ pub fn override_firewall_rules_with_priority(env: TestEnv) { ); // add a firewall rule that disables 9090 on the first node - let firewall_config = util::get_config().firewall.unwrap(); - let deny_port = FirewallAction::Deny; let mut node_rules = vec![FirewallRule { ipv4_prefixes: vec![], - ipv6_prefixes: firewall_config.default_rules[0].ipv6_prefixes.clone(), + ipv6_prefixes, ports: vec![9090], action: deny_port.into(), comment: "Test rule".to_string(), @@ -336,13 +338,38 @@ pub fn override_firewall_rules_with_priority(env: TestEnv) { info!(log, "Firewall priority tests has succeeded.") } -async fn set_default_registry_rules(log: &Logger, nns_node: &IcNodeSnapshot) { - let firewall_config = util::get_config().firewall.unwrap(); - let default_rules = firewall_config.default_rules.clone(); +fn read_firewall_whitelist_config(env: &TestEnv) -> Vec { + let default_firewall_whitelist_path = + env.get_dependency_path("rs/tests/src/default_firewall_whitelist.conf"); + let default_firewall_whitelist = std::fs::read_to_string(default_firewall_whitelist_path) + .expect("Could not read default whitelist"); + let ipv6_whitelist = default_firewall_whitelist + .lines() + .find_map(|v| v.strip_prefix("ipv6_whitelist=")) + .expect("Could not find ipv6_whitelist"); + + serde_json::from_str(&format!("[{}]", ipv6_whitelist)).expect("Unable to read ipv6_whitelist") +} + +async fn set_base_registry_rules( + log: &Logger, + nns_node: &IcNodeSnapshot, + ipv6_prefixes: Vec, +) { + let base_rules = vec![FirewallRule { + ipv4_prefixes: vec![], + ipv6_prefixes, + ports: vec![22, 2497, 4100, 7070, 8080, 9090, 9091, 9100, 19100, 19531], + action: FirewallAction::Allow as i32, + comment: "Base rule for test".to_string(), + user: None, + direction: Some(FirewallRuleDirection::Inbound as i32), + }]; + let proposal = prepare_add_rules_proposal( FirewallRulesScope::ReplicaNodes, - default_rules.clone(), - (0..default_rules.len()).map(|u| u as i32).collect(), + base_rules.clone(), + (0..base_rules.len()).map(|u| u as i32).collect(), vec![], ); execute_proposal( diff --git a/rs/tests/src/orchestrator/upgrade_downgrade.rs b/rs/tests/src/orchestrator/upgrade_downgrade.rs index 17196bd717f..dc6f83a8cad 100644 --- a/rs/tests/src/orchestrator/upgrade_downgrade.rs +++ b/rs/tests/src/orchestrator/upgrade_downgrade.rs @@ -51,6 +51,12 @@ pub fn config(env: TestEnv, subnet_type: SubnetType, mainnet_version: bool) { let mut ic = InternetComputer::new(); if mainnet_version { ic = ic.with_mainnet_config(); + + // Due to a change in how default firewall rules are supplied, they are + // not preserved across the transitional upgrade. We temporarily stash + // the whitelist in the registry for the time being. + // THIS PATH SHOULD BE REMOVED. + ic = ic.with_forced_default_firewall(); } let mut subnet_under_test = Subnet::new(subnet_type) diff --git a/rs/tests/src/util.rs b/rs/tests/src/util.rs index 6a1f1de857f..e205f3dfecf 100644 --- a/rs/tests/src/util.rs +++ b/rs/tests/src/util.rs @@ -1295,7 +1295,9 @@ pub fn get_config() -> ConfigOptional { .replace("{{ nns_url }}", "http://www.fakeurl.com/") .replace("{{ malicious_behavior }}", "null") .replace("{{ query_stats_aggregation }}", "\"Disabled\"") - .replace("{{ query_stats_epoch_length }}", "1800"); + .replace("{{ query_stats_epoch_length }}", "1800") + // Confirm that config is valid when no prefixes are specified. + .replace("{{ ipv6_whitelist }}", ""); json5::from_str::(&cfg).expect("Could not parse json5") } diff --git a/scalability/BUILD.bazel b/scalability/BUILD.bazel index 47d8243b03a..c54520d7b59 100644 --- a/scalability/BUILD.bazel +++ b/scalability/BUILD.bazel @@ -248,6 +248,7 @@ sh_test( ":e2e-scalability-tests", "//bazel:version_file_path", "//ic-os:scripts/build-bootstrap-config-image.sh", + "//rs/tests:src/default_firewall_whitelist.conf", IC_OS_ENV + ":disk-img.tar.zst.cas-url", IC_OS_ENV + ":disk-img.tar.zst.sha256", IC_OS_ENV + ":version.txt", diff --git a/scalability/common/ictools.py b/scalability/common/ictools.py index f02e5fbaf0d..f50a1c821d7 100644 --- a/scalability/common/ictools.py +++ b/scalability/common/ictools.py @@ -50,10 +50,13 @@ def build_bootstrap_config_image(name, **kwargs): p = Path(__file__).parents[2] bootstrap_script = os.path.join(p, "ic-os/scripts/build-bootstrap-config-image.sh") + default_firewall_whitelist = os.path.join(p, "rs/tests/src/default_firewall_whitelist.conf") args = [bootstrap_script, config_image] for key, value in kwargs.items(): args.append("--" + key) args.append(value) + args.append("--default_firewall_whitelist") + args.append(default_firewall_whitelist) subprocess.run(args, stdout=subprocess.DEVNULL, check=True) return config_image