Skip to content

Commit

Permalink
Chore(NODE-1241): Move IPv4 connectivity check to be a separate service
Browse files Browse the repository at this point in the history
  • Loading branch information
andrewbattat committed Jan 24, 2024
1 parent 16c08f4 commit b5859f9
Show file tree
Hide file tree
Showing 5 changed files with 50 additions and 68 deletions.
@@ -0,0 +1,9 @@
[Unit]
Description=Check IPv4 connectivity

[Install]
WantedBy=multi-user.target

[Service]
Type=oneshot
ExecStart=/opt/ic/bin/ipv4-connectivity-check.sh
@@ -0,0 +1,10 @@
[Unit]
Description=IPv4 connectivity timer

[Timer]
OnBootSec=60s
OnUnitActiveSec=60s
Unit=ipv4-connectivity-check.service

[Install]
WantedBy=timers.target
29 changes: 29 additions & 0 deletions ic-os/guestos/rootfs/opt/ic/bin/ipv4-connectivity-check.sh
@@ -0,0 +1,29 @@
#!/bin/bash

set -o nounset
set -o pipefail

METRICS_DIR="/run/node_exporter/collector_textfile"

write_metric() {
local name=$1
local value=$2
local help=$3
local type=$4

echo -e "# HELP ${name} ${help}\n# TYPE ${type}\n${name} ${value}" >"${METRICS_DIR}/${name}.prom"
}

endpoints=("1.1.1.1" "8.8.8.8" "9.9.9.9")

connectivity_status=0

for endpoint in "${endpoints[@]}"; do
# Using curl instead of ping as it requires less permissions
if curl --connect-timeout 10 "${endpoint}" &>/dev/null; then
connectivity_status=1
break
fi
done

write_metric "ipv4_connectivity_status" "${connectivity_status}" "Status of IPv4 connectivity" "gauge"
34 changes: 1 addition & 33 deletions rs/ic_os/guestos_tool/src/main.rs
Expand Up @@ -7,9 +7,8 @@ mod node_gen;
use node_gen::get_node_gen_metric;

mod prometheus_metric;
use prometheus_metric::{write_single_metric, LabelPair, MetricType};
use prometheus_metric::write_single_metric;

use network::interfaces::has_ipv4_connectivity;
mod generate_network_config;
use generate_network_config::{
generate_networkd_config, validate_and_construct_ipv4_address_info,
Expand All @@ -18,10 +17,6 @@ use generate_network_config::{

use network::systemd::{restart_systemd_networkd, DEFAULT_SYSTEMD_NETWORK_DIR};

use crate::prometheus_metric::PrometheusMetric;

static NODE_EXPORTER_METRIC_PATH: &str = "/run/node_exporter/collector_textfile";

#[derive(Subcommand)]
pub enum Commands {
/// Generate systemd network configuration files.
Expand Down Expand Up @@ -117,33 +112,6 @@ pub fn main() -> Result<()> {
eprintln!("Restarting systemd networkd");
restart_systemd_networkd();

if ipv4_gateway.is_some() {
let connectivity_metric_value = if has_ipv4_connectivity() {
eprintln!("Ipv4 configuration successful!");
1.0
} else {
eprintln!("ERROR: Ipv4 configuration failed!");
0.0
};

let prometheus_metric = PrometheusMetric {
name: "ipv4_connectivity_status".into(),
help: "Status of IPv4 connectivity".into(),
metric_type: MetricType::Gauge,
labels: [LabelPair {
label: "target".into(),
value: "ipv4_connectivity".into(),
}]
.to_vec(),
value: connectivity_metric_value,
};

let output_path = format!("{}/ipv4_connectivity.prom", NODE_EXPORTER_METRIC_PATH);
let output_path = Path::new(&output_path);

write_single_metric(&prometheus_metric, output_path)?;
}

Ok(())
}
None => Ok(()),
Expand Down
36 changes: 1 addition & 35 deletions rs/ic_os/network/src/interfaces.rs
@@ -1,5 +1,5 @@
use std::fs;
use std::net::{IpAddr, Ipv6Addr, TcpStream};
use std::net::{IpAddr, Ipv6Addr};
use std::path::PathBuf;
use std::thread::sleep;
use std::time::Duration;
Expand Down Expand Up @@ -79,40 +79,6 @@ pub fn get_interface_name(interface_path: &PathBuf) -> Result<String> {
))
}

pub fn has_ipv4_connectivity() -> bool {
let wait_time: Duration = Duration::from_secs(2);
let dns_servers = ["1.1.1.1", "1.0.0.1"];
let result = retry(
10,
|| {
eprintln!(
"Waiting for {} seconds before attempting a TCP connection with the following DNS servers: {:?}",
wait_time.as_secs(),
dns_servers
);

for &server in &dns_servers {
let connection_target_with_port = format!("{}:80", server);
match TcpStream::connect(&connection_target_with_port) {
Ok(_) => {
eprintln!("Successfully connected to {}", connection_target_with_port);
return Ok(true);
}
Err(e) => {
eprintln!(
"Failed to connect to {}: {}",
connection_target_with_port, e
);
}
}
}
Err(anyhow::Error::msg("All connection attempts failed"))
},
wait_time,
);
matches!(result, Ok(true))
}

fn qualify_and_generate_interface(interface_name: &str) -> Result<Option<Interface>> {
let ethtool_output = get_command_stdout("ethtool", [interface_name])?;
let link_is_up = is_link_up_from_ethool_output(&ethtool_output)?;
Expand Down

0 comments on commit b5859f9

Please sign in to comment.