feat(cli): Support for a stand-alone CLI binary, part 1

rs/cli/BUILD.bazel

@@ -6,6 +6,7 @@ DEPS = [
     "//rs/ic-canisters",
     "//rs/decentralization",
     "//rs/ic-management-types",
+    "//rs/ic-management-backend:ic-management-backend-lib",
     ":build_script",
 ]
 

rs/cli/Cargo.toml

@@ -26,7 +26,7 @@ anyhow = "1.0.44"
 futures = "0.3.21"
 colored = "2.0.0"
 log = "0.4.14"
-pretty_env_logger = "0.4.0"
+pretty_env_logger = "0.5.0"
 regex = "1.5.4"
 ic-base-types = { git = "https://github.com/dfinity/ic.git", rev = "abac636d8a281a1e7600d876aaed10f7ce7e5040" }
 ic-nns-constants = { git = "https://github.com/dfinity/ic.git", rev = "abac636d8a281a1e7600d876aaed10f7ce7e5040" }
@@ -46,6 +46,7 @@ flate2 = "1.0.22"
 dirs = "5.0.1"
 decentralization = { path = "../decentralization" }
 ic-management-types = { path = "../ic-management-types" }
+ic-management-backend = { path = "../ic-management-backend" }
 dialoguer = "0.10.0"
 itertools = "0.11.0"
 async-trait = "0.1.53"
@@ -58,6 +59,12 @@ sha2 = "0.10.6"
 edit = "0.1.4"
 tabular = "0.2"
 ic-agent = "0.27.0"
+actix-web = { version = "4.2.1", default-features = false, features = [
+    "compress-gzip",
+    "macros",
+] }
+dotenv = "0.15.0"
+socket2 = "0.5.4"
 
 [dev-dependencies]
 tempfile = "3.3.0"

rs/cli/src/cli.rs

@@ -32,8 +32,7 @@ pub struct Opts {
     #[clap(long, env = "VERBOSE", global = true)]
     pub(crate) verbose: bool,
 
-    // Specify the target network. Should be either "mainnet" (default) or "staging".
-    // If you want to use the cli, use the --nns-url
+    // Specify the target network: "mainnet" (default), "staging", or NNS URL
     #[clap(long, env = "NETWORK", default_value = "mainnet")]
     pub(crate) network: Network,
 

rs/cli/src/clients.rs

@@ -37,6 +37,12 @@ impl DashboardBackendClient {
         }
     }
 
+    pub fn new_with_network_url(url: String) -> Self {
+        Self {
+            url: reqwest::Url::parse(&url).unwrap(),
+        }
+    }
+
     pub async fn subnet_pending_action(&self, subnet: PrincipalId) -> anyhow::Result<Option<TopologyProposal>> {
         reqwest::Client::new()
             .get(

rs/cli/src/ic_admin.rs

@@ -177,7 +177,7 @@ impl Cli {
 
         // User wants to proceed but does not have neuron configuration. Bail out.
         if self.neuron.is_none() {
-            return Err(anyhow::anyhow!("Submitting this proposal requires a neuron, which was not detected -- and will cause ic-admin to fail during submition. Please look through your scroll buffer for specific error messages about your HSM and address the issue that prevents your neuron from being detected."));
+            return Err(anyhow::anyhow!("Submitting this proposal requires a neuron, which was not detected -- and would cause ic-admin to fail during submition. Please look through your scroll buffer for specific error messages about your HSM and address the issue that prevents your neuron from being detected."));
         }
 
         if Confirm::new()

rs/cli/src/main.rs

@@ -1,11 +1,16 @@
 use crate::cli::version::Commands::Update;
 use clap::{error::ErrorKind, CommandFactory, Parser};
+use dotenv::dotenv;
 use ic_canisters::governance_canister_version;
+use ic_management_backend::endpoints;
 use ic_management_types::requests::NodesRemoveRequest;
 use ic_management_types::{MinNakamotoCoefficients, Network, NodeFeature};
 use itertools::Itertools;
+use log::info;
 use std::collections::BTreeMap;
 use std::str::FromStr;
+use std::sync::mpsc;
+use std::thread;
 
 mod cli;
 mod clients;
@@ -18,14 +23,32 @@ const STAGING_NEURON_ID: u64 = 49;
 
 #[tokio::main]
 async fn main() -> Result<(), anyhow::Error> {
+    dotenv().ok();
     init_logger();
+    info!("Running version {}", env!("GIT_HASH"));
+
     let mut cli_opts = cli::Opts::parse();
     let mut cmd = cli::Opts::command();
 
     let governance_canister_v = governance_canister_version(cli_opts.network.get_url()).await?;
-    let governance_canister_build = governance_canister_v.stringified_hash;
+    let governance_canister_version = governance_canister_v.stringified_hash;
+
+    let target_network = cli_opts.network.clone();
+    let (tx, rx) = mpsc::channel();
+
+    let backend_port = local_unused_port();
+    thread::spawn(move || {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        rt.block_on(async move {
+            endpoints::run_backend(target_network, "127.0.0.1", backend_port, true, Some(tx))
+                .await
+                .expect("failed")
+        });
+    });
+    let srv = rx.recv().unwrap();
+
+    ic_admin::with_ic_admin(governance_canister_version.into(), async {
 
-    ic_admin::with_ic_admin(governance_canister_build.into(), async {
         // Start of actually doing stuff with commands.
         if cli_opts.network == Network::Staging {
             cli_opts.private_key_pem = Some(std::env::var("HOME").expect("Please set HOME env var") + "/.config/dfx/identity/bootstrap-super-leader/identity.pem");
@@ -42,7 +65,6 @@ async fn main() -> Result<(), anyhow::Error> {
             }
 
             cli::Commands::Subnet(subnet) => {
-                let runner = runner::Runner::from_opts(&cli_opts).await?;
                 match &subnet.subcommand {
                     cli::subnet::Commands::Deploy { .. } | cli::subnet::Commands::Resize { .. } => {
                         if subnet.id.is_none() {
@@ -72,7 +94,10 @@ async fn main() -> Result<(), anyhow::Error> {
                 }
 
                 match &subnet.subcommand {
-                    cli::subnet::Commands::Deploy { version } => runner.deploy(&subnet.id.unwrap(), version, simulate),
+                    cli::subnet::Commands::Deploy { version } => {
+                        let runner = runner::Runner::new_with_network_url(ic_admin::Cli::from_opts(&cli_opts, false).await?, backend_port).await?;
+                        runner.deploy(&subnet.id.unwrap(), version, simulate)
+                    },
                     cli::subnet::Commands::Replace {
                         nodes,
                         no_heal,
@@ -85,7 +110,7 @@ async fn main() -> Result<(), anyhow::Error> {
                         verbose,
                     } => {
                         let min_nakamoto_coefficients = parse_min_nakamoto_coefficients(&mut cmd, min_nakamoto_coefficients);
-
+                            let runner = runner::Runner::new_with_network_url(ic_admin::Cli::from_opts(&cli_opts, true).await?, backend_port).await?;
                             runner
                                 .membership_replace(ic_management_types::requests::MembershipReplaceRequest {
                                     target: match &subnet.id {
@@ -118,6 +143,7 @@ async fn main() -> Result<(), anyhow::Error> {
                     }
                     cli::subnet::Commands::Resize { add, remove, include, only, exclude, motivation, verbose, } => {
                         if let Some(motivation) = motivation.clone() {
+                            let runner = runner::Runner::new_with_network_url(ic_admin::Cli::from_opts(&cli_opts, true).await?, backend_port).await?;
                             runner.subnet_resize(ic_management_types::requests::SubnetResizeRequest {
                                 subnet: subnet.id.unwrap(),
                                 add: *add,
@@ -137,6 +163,7 @@ async fn main() -> Result<(), anyhow::Error> {
                     cli::subnet::Commands::Create { size, min_nakamoto_coefficients, exclude, only, include, motivation, verbose, replica_version } => {
                         let min_nakamoto_coefficients = parse_min_nakamoto_coefficients(&mut cmd, min_nakamoto_coefficients);
                         if let Some(motivation) = motivation.clone() {
+                            let runner = runner::Runner::new_with_network_url(ic_admin::Cli::from_opts(&cli_opts, true).await?, backend_port).await?;
                             runner.subnet_create(ic_management_types::requests::SubnetCreateRequest {
                                 size: *size,
                                 min_nakamoto_coefficients,
@@ -168,6 +195,7 @@ async fn main() -> Result<(), anyhow::Error> {
             cli::Commands::Version(cmd) => {
                 match &cmd.subcommand {
                     Update { version, release_tag} => {
+                        // FIXME: backend needs gitlab access to figure out RCs and versions to retire
                         let runner = runner::Runner::from_opts(&cli_opts).await?;
                         let (_, retire_versions) = runner.prepare_versions_to_retire(false).await?;
                         let ic_admin = ic_admin::Cli::from_opts(&cli_opts, true).await?;
@@ -207,7 +235,7 @@ async fn main() -> Result<(), anyhow::Error> {
                             )
                             .exit();
                         }
-                        let runner = runner::Runner::from_opts(&cli_opts).await?;
+                        let runner = runner::Runner::new_with_network_url(ic_admin::Cli::from_opts(&cli_opts, true).await?, backend_port).await?;
                         runner.remove_nodes(NodesRemoveRequest {
                             extra_nodes_filter: extra_nodes_filter.clone(),
                             no_auto: *no_auto,
@@ -219,7 +247,11 @@ async fn main() -> Result<(), anyhow::Error> {
             },
         }
     })
-    .await
+    .await?;
+
+    srv.stop(false).await;
+
+    Ok(())
 }
 
 // Construct MinNakamotoCoefficients from an array (slice) of ["key=value"], and
@@ -299,14 +331,30 @@ fn parse_min_nakamoto_coefficients(
     })
 }
 
+/// Get a localhost socket address with random, unused port.
+fn local_unused_port() -> u16 {
+    let addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
+    let socket = socket2::Socket::new(
+        socket2::Domain::IPV4,
+        socket2::Type::STREAM,
+        Some(socket2::Protocol::TCP),
+    )
+    .unwrap();
+    socket.bind(&addr.into()).unwrap();
+    socket.set_reuse_address(true).unwrap();
+    let tcp = std::net::TcpListener::from(socket);
+    tcp.local_addr().unwrap().port()
+}
+
 fn init_logger() {
     match std::env::var("RUST_LOG") {
         Ok(val) => std::env::set_var("LOG_LEVEL", val),
         Err(_) => {
             if std::env::var("LOG_LEVEL").is_err() {
-                // Set a default logging level: info, if nothing else specified in environment
-                // variables RUST_LOG or LOG_LEVEL
-                std::env::set_var("LOG_LEVEL", "info")
+                // Default logging level is: info generally, warn for mio and actix_server
+                // You can override defaults by setting environment variables
+                // RUST_LOG or LOG_LEVEL
+                std::env::set_var("LOG_LEVEL", "info,mio::=warn,actix_server::=warn")
             }
         }
     }

rs/cli/src/runner.rs

@@ -1,8 +1,7 @@
-use crate::cli::Opts;
-use crate::clients;
 use crate::ic_admin;
 use crate::ic_admin::ProposeOptions;
 use crate::ops_subnet_node_replace;
+use crate::{cli::Opts, clients::DashboardBackendClient};
 use decentralization::SubnetChangeResponse;
 use ic_base_types::PrincipalId;
 use ic_management_types::requests::NodesRemoveRequest;
@@ -13,7 +12,7 @@ use log::{info, warn};
 #[derive(Clone)]
 pub struct Runner {
     ic_admin: ic_admin::Cli,
-    dashboard_backend_client: clients::DashboardBackendClient,
+    dashboard_backend_client: DashboardBackendClient,
 }
 
 impl Runner {
@@ -175,8 +174,17 @@ impl Runner {
 
     pub async fn from_opts(cli_opts: &Opts) -> anyhow::Result<Self> {
         Ok(Self {
-            ic_admin: ic_admin::Cli::from_opts(cli_opts, true).await?,
-            dashboard_backend_client: clients::DashboardBackendClient::new(cli_opts.network.clone(), cli_opts.dev),
+            ic_admin: ic_admin::Cli::from_opts(cli_opts, false).await?,
+            dashboard_backend_client: DashboardBackendClient::new(cli_opts.network.clone(), cli_opts.dev),
+        })
+    }
+
+    pub async fn new_with_network_url(ic_admin: ic_admin::Cli, backend_port: u16) -> anyhow::Result<Self> {
+        let dashboard_backend_client =
+            DashboardBackendClient::new_with_network_url(format!("http://localhost:{}/", backend_port));
+        Ok(Self {
+            ic_admin,
+            dashboard_backend_client,
         })
     }
 

rs/ic-management-backend/src/config.rs

@@ -7,10 +7,23 @@ pub fn target_network() -> Network {
         .expect("Invalid network")
 }
 
-pub fn nns_url() -> String {
-    std::env::var("NNS_URL").expect("NNS_URL environment variable not provided")
+pub fn get_nns_url_string_from_target_network(target_network: &Network) -> String {
+    match std::env::var("NNS_URL") {
+        Ok(nns_url) => nns_url,
+        Err(_) => match target_network {
+            Network::Mainnet => "https://ic0.app".to_string(),
+            Network::Staging => "http://[2600:3004:1200:1200:5000:11ff:fe37:c55d]:8080".to_string(),
+            _ => panic!(
+                "Cannot get NNS URL for target network {}. Please set NNS_URL environment variable",
+                target_network
+            ),
+        },
+    }
 }
 
-pub fn nns_nodes_urls() -> Vec<Url> {
-    vec![Url::parse(&nns_url()).expect("Cannot parse NNS_URL environment variable as a valid URL")]
+pub fn get_nns_url_vec_from_target_network(target_network: &Network) -> Vec<Url> {
+    get_nns_url_string_from_target_network(target_network)
+        .split(',')
+        .map(|s| Url::parse(s).unwrap_or_else(|_| panic!("Cannot parse {} as a valid NNS URL", s)))
+        .collect()
 }

rs/ic-management-backend/src/endpoints/governance_canister.rs

@@ -1,10 +1,11 @@
-use super::super::config::nns_nodes_urls;
 use super::*;
+use crate::config::{get_nns_url_vec_from_target_network, target_network};
 use ic_canisters::governance_canister_version;
 
 #[get("/canisters/governance/version")]
 async fn governance_canister_version_endpoint() -> Result<HttpResponse, Error> {
-    let u = nns_nodes_urls()[0].clone();
+    let network = target_network();
+    let u = get_nns_url_vec_from_target_network(&network)[0].clone();
     let g = governance_canister_version(u).await;
     response_from_result(g)
 }