Update firewall configuration through SetupOS and HostOS

Author: Bownairo

Cargo.lock

@@ -482,7 +482,7 @@ def icos_build(
             ":disk-img.tar.zst.cas-url",
             ":disk-img.tar.zst.sha256",
             "//ic-os:scripts/build-bootstrap-config-image.sh",
-            "//rs/tests:src/ipv6_prefixes.json",
+            "//rs/tests:src/default_firewall_whitelist.conf",
             ":version.txt",
         ],
         outs = ["launch_remote_vm_script"],
@@ -492,12 +492,12 @@ def icos_build(
         URL="$$(cat $(location :disk-img.tar.zst.cas-url))"
         SHA="$$(cat $(location :disk-img.tar.zst.sha256))"
         SCRIPT="$(location //ic-os:scripts/build-bootstrap-config-image.sh)"
-        IPV6_PREFIXES="$(location //rs/tests:src/ipv6_prefixes.json)"
+        DEFAULT_FIREWALL_WHITELIST="$(location //rs/tests:src/default_firewall_whitelist.conf)"
         cat <<EOF > $@
 #!/usr/bin/env bash
 set -euo pipefail
 cd "\\$$BUILD_WORKSPACE_DIRECTORY"
-$$BIN --version "$$VERSION" --url "$$URL" --sha256 "$$SHA" --build-bootstrap-script "$$SCRIPT" --ipv6-prefixes-path "$$IPV6_PREFIXES"
+$$BIN --version "$$VERSION" --url "$$URL" --sha256 "$$SHA" --build-bootstrap-script "$$SCRIPT" --default-firewall-whitelist "$$DEFAULT_FIREWALL_WHITELIST"
 EOF
         """,
         executable = True,

ic-os/defs.bzl

@@ -20,7 +20,7 @@ Environment=RUST_BACKTRACE=1
 # only on ExecStartPre by ic-replica.service. As a temporary workaround, we are generating
 # an ic-csp-service-specific version of ic.json5 of which only the parts related to crypto
 # and logging are relevant.
-ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -n /boot/config/network.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic-crypto-csp.json5
+ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic-crypto-csp.json5
 ExecStart=/opt/ic/bin/ic-crypto-csp --replica-config-file /run/ic-node/config/ic-crypto-csp.json5
 NotifyAccess=main
 Restart=always

ic-os/guestos/rootfs/etc/systemd/system/ic-crypto-csp.service

@@ -28,7 +28,7 @@ Environment=RUST_BACKTRACE=1
 # Remember to update 'rs/default.nix' for nix-shell users
 # Remember to update 'src/dfx/src/actors/replica.rs' in the sdk repo for dfx users
 Environment=RUST_MIN_STACK=8192000
-ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -n /boot/config/network.conf -c /boot/config/nns.conf -b /boot/config/backup.conf -l /boot/config/log.conf -m /boot/config/malicious_behavior.conf -q /boot/config/query_stats.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5
+ExecStartPre=+/opt/ic/bin/generate-replica-config.sh -n /boot/config/network.conf -c /boot/config/nns.conf -b /boot/config/backup.conf -l /boot/config/log.conf -m /boot/config/malicious_behavior.conf -q /boot/config/query_stats.conf -w /boot/config/default_firewall_whitelist.conf -i /opt/ic/share/ic.json5.template -o /run/ic-node/config/ic.json5
 ExecStart=/opt/ic/bin/orchestrator --replica-binary-dir /var/lib/ic/data/images --cup-dir /var/lib/ic/data/cups --replica-config-file /run/ic-node/config/ic.json5 --enable-provisional-registration --ic-binary-directory /opt/ic/bin --orchestrator-data-directory /var/lib/ic/data/orchestrator --version-file /opt/ic/share/version.txt
 LimitNOFILE=1048576
 Restart=always

ic-os/guestos/rootfs/etc/systemd/system/ic-replica.service

@@ -127,7 +127,7 @@ function process_bootstrap() {
 
     # stash the following configuration files to config store
     # note: keep this list in sync with configurations supported in build-bootstrap-config-image.sh
-    for FILE in filebeat.conf network.conf nns.conf backup.conf log.conf malicious_behavior.conf query_stats.conf bitcoind_addr.conf socks_proxy.conf; do
+    for FILE in filebeat.conf network.conf nns.conf backup.conf log.conf malicious_behavior.conf query_stats.conf bitcoind_addr.conf socks_proxy.conf default_firewall_whitelist.conf; do
         if [ -e "${TMPDIR}/${FILE}" ]; then
             echo "Setting up ${FILE}"
             cp "${TMPDIR}/${FILE}" "${CONFIG_ROOT}/${FILE}"

ic-os/guestos/rootfs/opt/ic/bin/bootstrap-ic-node.sh

@@ -6,7 +6,7 @@
 function usage() {
     cat <<EOF
 Usage:
-  generate-replica-config [-n network.conf] [-c nns.conf] [-b backup.conf] [-l log.conf] [-m malicious_behavior.conf] [-q query_stats.conf] -i ic.json5.template -o ic.json5
+  generate-replica-config [-n network.conf] [-c nns.conf] [-b backup.conf] [-l log.conf] [-m malicious_behavior.conf] [-q query_stats.conf] [-w default_firewall_whitelist.conf] -i ic.json5.template -o ic.json5
 
   Generate replica config from template file.
 
@@ -16,6 +16,7 @@ Usage:
   -l log.conf: Optional, logging parameters of the node software
   -m malicious_behavior.conf: Optional, malicious behavior parameters
   -q query_stats.conf: Optional, query statistics epoch length configuration
+  -w default_firewall_whitelist.conf: Optional, default firewall whitelist configuration
   -i infile: input ic.json5.template file
   -o outfile: output ic.json5 file
 EOF
@@ -76,7 +77,6 @@ function read_network_variables() {
             "ipv6_address") ipv6_address="${value}" ;;
             "ipv6_gateway") ipv6_gateway="${value}" ;;
             "name_servers") name_servers="${value}" ;;
-            "ipv6_prefixes") ipv6_prefixes="${value}" ;;
             "ipv4_address") ipv4_address="${value}" ;;
             "ipv4_gateway") ipv4_gateway="${value}" ;;
             "domain") domain="${value}" ;;
@@ -158,7 +158,23 @@ function read_query_stats_variables() {
     done <"$1"
 }
 
-while getopts "l:m:q:n:c:i:o:b:" OPT; do
+# Read default firewall whitelist config variables from file. The file must be of the
+# form "key=value" for each line with a specific set of keys permissible (see
+# code below).
+#
+# Arguments:
+# - $1: Name of the file to be read.
+function read_default_firewall_whitelist_variables() {
+    # Read limited set of keys. Be extra-careful quoting values as it could
+    # otherwise lead to executing arbitrary shell code!
+    while IFS="=" read -r key value; do
+        case "$key" in
+            "ipv6_whitelist") ipv6_whitelist="${value}" ;;
+        esac
+    done <"$1"
+}
+
+while getopts "l:m:q:n:c:i:o:b:w:" OPT; do
     case "${OPT}" in
         n)
             NETWORK_CONFIG_FILE="${OPTARG}"
@@ -178,6 +194,9 @@ while getopts "l:m:q:n:c:i:o:b:" OPT; do
         q)
             QUERY_STATS_CONFIG_FILE="${OPTARG}"
             ;;
+        w)
+            DEFAULT_FIREWALL_WHITELIST_CONFIG_FILE="${OPTARG}"
+            ;;
         i)
             IN_FILE="${OPTARG}"
             ;;
@@ -220,10 +239,13 @@ if [ "${QUERY_STATS_CONFIG_FILE}" != "" -a -e "${QUERY_STATS_CONFIG_FILE}" ]; th
     read_query_stats_variables "${QUERY_STATS_CONFIG_FILE}"
 fi
 
+if [ "${DEFAULT_FIREWALL_WHITELIST_CONFIG_FILE}" != "" -a -e "${DEFAULT_FIREWALL_WHITELIST_CONFIG_FILE}" ]; then
+    read_default_firewall_whitelist_variables "${DEFAULT_FIREWALL_WHITELIST_CONFIG_FILE}"
+fi
+
 INTERFACE=($(find /sys/class/net -type l -not -lname '*virtual*' -exec basename '{}' ';'))
 IPV6_ADDRESS="${ipv6_address%/*}"
 IPV6_ADDRESS="${IPV6_ADDRESS:-$(get_if_address_retries 6 ${INTERFACE} 12)}"
-IPV6_PREFIXES="${ipv6_prefixes:-}"
 IPV4_ADDRESS="${ipv4_address:-}"
 IPV4_GATEWAY="${ipv4_gateway:-}"
 DOMAIN="${domain:-}"
@@ -241,6 +263,8 @@ MALICIOUS_BEHAVIOR="${malicious_behavior:-null}"
 QUERY_STATS_AGGREGATION="${query_stats_aggregation:-\"Disabled\"}"
 # Default is 1800 blocks i.e. around 30min
 QUERY_STATS_EPOCH_LENGTH="${query_stats_epoch_length:-1800}"
+# Default value is none
+IPV6_WHITELIST="${ipv6_whitelist:-}"
 
 if [ "${IPV6_ADDRESS}" == "" ]; then
     echo "Cannot determine an IPv6 address, aborting"
@@ -264,7 +288,6 @@ if [ "${hostname}" == "" ]; then
 fi
 
 sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \
-    -e "s@{{ ipv6_prefixes }}@${IPV6_PREFIXES}@" \
     -e "s@{{ ipv4_address }}@${IPV4_ADDRESS}@" \
     -e "s@{{ ipv4_gateway }}@${IPV4_GATEWAY}@" \
     -e "s@{{ domain }}@${DOMAIN}@" \
@@ -276,6 +299,7 @@ sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \
     -e "s@{{ malicious_behavior }}@${MALICIOUS_BEHAVIOR}@" \
     -e "s@{{ query_stats_aggregation }}@${QUERY_STATS_AGGREGATION}@" \
     -e "s@{{ query_stats_epoch_length }}@${QUERY_STATS_EPOCH_LENGTH}@" \
+    -e "s@{{ ipv6_whitelist }}@${IPV6_WHITELIST}@" \
     "${IN_FILE}" >"${OUT_FILE}"
 
 # umask for service is set to be restricted, but this file needs to be

ic-os/guestos/rootfs/opt/ic/bin/generate-replica-config.sh

@@ -267,7 +267,7 @@ table ip6 filter {\n\
         ipv6_user_output_rule_template: "meta skuid <<USER>> ip6 daddr {<<IPv6_PREFIXES>>} ct state { new } tcp dport {<<PORTS>>} <<ACTION>> # <<COMMENT>>",
         default_rules: [{
           ipv4_prefixes: [],
-          ipv6_prefixes: {{ ipv6_prefixes }},
+          ipv6_prefixes: [{{ ipv6_whitelist }}],
           ports: [22, 2497, 4100, 7070, 8080, 9090, 9091, 9100, 19100, 19531],
           action: 1,
           comment: "Default rule from template",

ic-os/guestos/rootfs/opt/ic/share/ic.json5.template

@@ -115,6 +115,9 @@ function assemble_config_media() {
     if [ -f "/boot/config/node_operator_private_key.pem" ]; then
         cmd+=(--node_operator_private_key "/boot/config/node_operator_private_key.pem")
     fi
+    if [ -f "/boot/config/default_firewall_whitelist.conf" ]; then
+        cmd+=(--default_firewall_whitelist "/boot/config/default_firewall_whitelist.conf")
+    fi
 
     # Run the above command
     "${cmd[@]}"

ic-os/hostos/rootfs/opt/ic/bin/generate-guestos-config.sh

@@ -33,8 +33,9 @@ options may be specified:
     script, e.g. --ipv6_name_servers "2606:4700:4700::1111
     2606:4700:4700::1001").
 
-  --ipv6_prefixes prefixes
-    Comma separated list of IPv6 networks that are whitelisted in the guestOS' firewall on orchestrator startup.
+  --default_firewall_whitelist path
+    File containing the default whitelist configuration for GuestOS on
+    orchestrator startup.
 
   --ipv4_address a.b.c.d/n
     (optional) The IPv4 address to assign. Must include prefix length (e.g.
@@ -133,7 +134,7 @@ function build_ic_bootstrap_tar() {
     local OUT_FILE="$1"
     shift
 
-    local IPV6_ADDRESS IPV6_GATEWAY IPV6_NAME_SERVERS IVP6_PREFIXES DOMAIN HOSTNAME
+    local IPV6_ADDRESS IPV6_GATEWAY IPV6_NAME_SERVERS DOMAIN HOSTNAME
     local IC_CRYPTO IC_REGISTRY_LOCAL_STORE
     local NNS_URL NNS_PUBLIC_KEY NODE_OPERATOR_PRIVATE_KEY
     local BACKUP_RETENTION_TIME_SECS BACKUP_PURGING_INTERVAL_SECS
@@ -143,6 +144,7 @@ function build_ic_bootstrap_tar() {
     local MALICIOUS_BEHAVIOR
     local QUERY_STATS_EPOCH_LENGTH
     local BITCOIND_ADDR
+    local DEFAULT_FIREWALL_WHITELIST
 
     while true; do
         if [ $# == 0 ]; then
@@ -159,8 +161,8 @@ function build_ic_bootstrap_tar() {
             --ipv6_name_servers)
                 IPV6_NAME_SERVERS="$2"
                 ;;
-            --ipv6_prefixes)
-                IPV6_PREFIXES="$2"
+            --default_firewall_whitelist)
+                DEFAULT_FIREWALL_WHITELIST="$2"
                 ;;
             --ipv4_address)
                 IPV4_ADDRESS="$2"
@@ -240,7 +242,6 @@ function build_ic_bootstrap_tar() {
 ${IPV6_ADDRESS:+ipv6_address=$IPV6_ADDRESS}
 ${IPV6_GATEWAY:+ipv6_gateway=$IPV6_GATEWAY}
 name_servers=$IPV6_NAME_SERVERS
-ipv6_prefixes=$IPV6_PREFIXES
 hostname=$HOSTNAME
 ${IPV4_ADDRESS:+ipv4_address=$IPV4_ADDRESS}
 ${IPV4_GATEWAY:+ipv4_gateway=$IPV4_GATEWAY}
@@ -289,6 +290,9 @@ EOF
     if [ "${NODE_OPERATOR_PRIVATE_KEY}" != "" ]; then
         cp "${NODE_OPERATOR_PRIVATE_KEY}" "${BOOTSTRAP_TMPDIR}/node_operator_private_key.pem"
     fi
+    if [ "${DEFAULT_FIREWALL_WHITELIST}" != "" ]; then
+        cp "${DEFAULT_FIREWALL_WHITELIST}" "${BOOTSTRAP_TMPDIR}/default_firewall_whitelist.conf"
+    fi
 
     tar cf "${OUT_FILE}" \
         --sort=name \

ic-os/scripts/build-bootstrap-config-image.sh

@@ -48,6 +48,11 @@ function clone_config() {
         log_and_reboot_on_error "${?}" "Unable to copy 'node_operator_private_key.pem' configuration file."
     fi
 
+    if [ -f "${CONFIG_DIR}/default_firewall_whitelist.conf" ]; then
+        cp ${CONFIG_DIR}/default_firewall_whitelist.conf ${CONFIG_TMP}/default_firewall_whitelist.conf
+        log_and_reboot_on_error "${?}" "Unable to copy 'default_firewall_whitelist.conf' configuration file."
+    fi
+
     if [ -d "${SSH_AUTHORIZED_KEYS}" ]; then
         cp -r "${SSH_AUTHORIZED_KEYS}" "${CONFIG_TMP}"
         log_and_reboot_on_error "${?}" "Unable to copy 'ssh_authorized_keys' directory."

ic-os/setupos/rootfs/opt/ic/bin/config.sh

@@ -42,6 +42,11 @@ function copy_config_files() {
         echo "node_operator_private_key.pem does not exist, requiring HSM."
     fi
 
+    if [ -f "${CONFIG_DIR}/default_firewall_whitelist.conf" ]; then
+        cp ${CONFIG_DIR}/default_firewall_whitelist.conf /media/
+        log_and_reboot_on_error "${?}" "Unable to copy default_firewall_whitelist.conf to hostOS config partition."
+    fi
+
     echo "* Copying deployment.json to config partition..."
     cp /data/deployment.json /media/
     log_and_reboot_on_error "${?}" "Unable to copy deployment.json to hostOS config partition."

ic-os/setupos/rootfs/opt/ic/bin/devices.sh

@@ -6,7 +6,6 @@ DEPENDENCIES = [
     "@crate_index//:clap",
     "@crate_index//:reqwest",
     "@crate_index//:serde",
-    "@crate_index//:serde_json",
     "@crate_index//:slog-async",
     "@crate_index//:slog-term",
     "@crate_index//:slog",

rs/ic_os/launch-single-vm/BUILD.bazel

@@ -14,7 +14,6 @@ ic-types = { path = "../../types/types" }
 clap = { workspace = true }
 reqwest = { workspace = true }
 serde = { workspace = true }
-serde_json = { workspace = true }
 slog-async = { version = "2.5", features = ["nested-values"] }
 slog-term = "2.6.0"
 slog = { workspace = true }

rs/ic_os/launch-single-vm/Cargo.toml

@@ -7,7 +7,6 @@ use clap::Parser;
 use reqwest::blocking::Client;
 use serde::Serialize;
 use slog::{o, Drain};
-use std::fs::File;
 use tempfile::tempdir;
 use url::Url;
 
@@ -40,9 +39,9 @@ struct Args {
     /// Path to `build-bootstrap-config-image.sh` script
     #[clap(long)]
     build_bootstrap_script: PathBuf,
-    /// Path to ipv6_prefixes.json
+    /// Path to `default_firewall_whitelist.conf` file
     #[clap(long)]
-    ipv6_prefixes_path: PathBuf,
+    default_firewall_whitelist: PathBuf,
     /// Key to be used for `admin` SSH
     #[clap(long)]
     ssh_key_path: Option<PathBuf>,
@@ -64,7 +63,7 @@ fn main() {
     let url = args.url;
     let sha256 = args.sha256;
     let build_bootstrap_script = args.build_bootstrap_script;
-    let ipv6_prefixes_path = args.ipv6_prefixes_path;
+    let default_firewall_whitelist = args.default_firewall_whitelist;
     let ssh_key_path = args.ssh_key_path;
 
     let test_name = "test_single_vm";
@@ -178,16 +177,6 @@ fn main() {
     let filename = "config.tar.gz";
     let config_path = tempdir.as_ref().join(filename);
     let local_store = prep_dir.join("ic_registry_local_store");
-    let ipv6_prefixes: Vec<String> =
-        serde_json::from_reader(File::open(ipv6_prefixes_path).unwrap()).unwrap();
-    let ipv6_prefixes = format!(
-        "[{}]",
-        ipv6_prefixes
-            .iter()
-            .map(|s| format!("\"{s}\""))
-            .collect::<Vec<_>>()
-            .join(", "),
-    );
     Command::new(build_bootstrap_script)
         .arg(&config_path)
         .arg("--nns_url")
@@ -198,8 +187,8 @@ fn main() {
         .arg(&local_store)
         .arg("--accounts_ssh_authorized_keys")
         .arg(&keys_dir)
-        .arg("--ipv6_prefixes")
-        .arg(ipv6_prefixes)
+        .arg("--default_firewall_whitelist")
+        .arg(default_firewall_whitelist)
         .status()
         .unwrap();
 

rs/ic_os/launch-single-vm/src/main.rs

@@ -34,6 +34,9 @@ struct Cli {
 
     #[command(flatten)]
     deployment: DeploymentConfig,
+
+    #[arg(long)]
+    default_firewall_whitelist_path: Option<PathBuf>,
 }
 
 #[derive(Args)]
@@ -120,6 +123,17 @@ async fn main() -> Result<(), Error> {
             .context("failed to copy public keys")?;
     }
 
+    // Update default firewall whitelist
+    if let Some(default_firewall_whitelist_path) = cli.default_firewall_whitelist_path {
+        config
+            .write_file(
+                &default_firewall_whitelist_path,
+                Path::new("/default_firewall_whitelist.conf"),
+            )
+            .await
+            .context("failed to copy default firewall config")?;
+    }
+
     // Close config partition
     config.close().await?;
 

rs/ic_os/setupos-inject-configuration/src/main.rs

@@ -388,22 +388,23 @@ impl IcConfig {
         let mut mutations = self.initial_mutations.clone();
 
         if let Some(prefixes) = self.whitelisted_prefixes {
-            let ports = if let Some(ports) = self.whitelisted_ports {
-                ports
+            // Port 8080 is always included.
+            let mut ports = vec![8080];
+
+            if let Some(additional_ports) = self.whitelisted_ports {
+                ports = additional_ports
                     .split(',')
-                    .map(|port| port.parse::<u32>().unwrap())
-                    .chain(std::iter::once(8080))
+                    .map(|port| port.trim().parse::<u32>().unwrap())
+                    .chain(ports)
                     .collect()
-            } else {
-                vec![8080]
             };
 
             mutations.extend(vec![insert(
                 make_firewall_rules_record_key(&FirewallRulesScope::Global),
                 encode_or_panic(&FirewallRuleSet {
                     entries: vec![FirewallRule {
                         ipv4_prefixes: Vec::new(),
-                        ipv6_prefixes: prefixes.split(',').map(|v| v.to_string()).collect(),
+                        ipv6_prefixes: prefixes.split(',').map(|v| v.trim().to_string()).collect(),
                         ports,
                         action: FirewallAction::Allow as i32,
                         comment: "Globally allow provided prefixes for testing".to_string(),