Skip to content

Commit

Permalink
feat(msd): metrics (#171)
Browse files Browse the repository at this point in the history
* started work on metrics

* adding definition metrics

* fixing counter for msd

* adding scraping errors metrics

* repinning

* adding correct suffixes

* Revert "adding correct suffixes"

This reverts commit 1509277.

* fixing naming of UpDownCounter

* Revert "fixing naming of UpDownCounter"

This reverts commit 8e74c5c.

* migrating metric from total to a labeled metric with the name of the network for easier alert creation

* rustfmt

* implement two more status metrics

* moving to observable metrics

* migrate counters to observable gauges

---------

Co-authored-by: pietrodimarco-dfinity <124565147+pietrodimarco-dfinity@users.noreply.github.com>
  • Loading branch information
NikolaMilosa and pietrodimarco-dfinity committed Feb 12, 2024
1 parent 4adbc7d commit a480f59
Show file tree
Hide file tree
Showing 24 changed files with 1,105 additions and 248 deletions.
457 changes: 456 additions & 1 deletion Cargo.Bazel.lock

Large diffs are not rendered by default.

96 changes: 96 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

42 changes: 9 additions & 33 deletions rs/ic-observability/config-writer-common/src/config_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,7 @@ pub struct ConfigWriter {
}

impl ConfigWriter {
pub fn new<P: AsRef<Path>>(
write_path: P,
filters: Arc<dyn TargetGroupFilter>,
log: Logger,
) -> Self {
pub fn new<P: AsRef<Path>>(write_path: P, filters: Arc<dyn TargetGroupFilter>, log: Logger) -> Self {
ConfigWriter {
base_directory: PathBuf::from(write_path.as_ref()),
last_targets: Default::default(),
Expand All @@ -48,16 +44,10 @@ impl ConfigWriter {
) -> std::io::Result<()> {
let last_job_targets = self.last_targets.entry(job.to_string()).or_default();
if last_job_targets == &target_groups {
debug!(
self.log,
"Targets didn't change, skipped regenerating config"
);
debug!(self.log, "Targets didn't change, skipped regenerating config");
return Ok(());
}
debug!(
self.log,
"Targets changed, proceeding with regenerating config"
);
debug!(self.log, "Targets changed, proceeding with regenerating config");
let target_path = self.base_directory.join(format!("{}.json", job));

let filtered_target_groups: BTreeSet<TargetGroup> = target_groups
Expand All @@ -69,12 +59,8 @@ impl ConfigWriter {
let vector_config = vector_config_builder.build(filtered_target_groups, job);

ic_utils::fs::write_atomically(target_path.as_path(), |f| {
serde_json::to_writer_pretty(f, &vector_config).map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::Other,
format!("Serialization error: {:?}", e),
)
})
serde_json::to_writer_pretty(f, &vector_config)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, format!("Serialization error: {:?}", e)))
})?;
self.last_targets.insert(job.to_string(), target_groups);
Ok(())
Expand All @@ -84,25 +70,15 @@ impl ConfigWriter {
impl ConfigUpdater for ConfigWriter {
fn update(&self, config: &dyn Config) -> Result<(), Box<dyn Error>> {
if !config.updated() {
debug!(
self.log,
"Targets didn't change, skipped regenerating config"
);
debug!(self.log, "Targets didn't change, skipped regenerating config");
return Ok(());
}
debug!(
self.log,
"Targets changed, proceeding with regenerating config"
);
debug!(self.log, "Targets changed, proceeding with regenerating config");
let target_path = self.base_directory.join(format!("{}.json", config.name()));

ic_utils::fs::write_atomically(target_path.as_path(), |f| {
serde_json::to_writer_pretty(f, &config).map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::Other,
format!("Serialization error: {:?}", e),
)
})
serde_json::to_writer_pretty(f, &config)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, format!("Serialization error: {:?}", e)))
})?;
Ok(())
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ pub fn config_writer_loop(
metrics: Metrics,
) -> impl FnMut() {
move || {
let mut config_writer =
ConfigWriter::new(vector_config_dir.clone(), filters.clone(), log.clone());
let mut config_writer = ConfigWriter::new(vector_config_dir.clone(), filters.clone(), log.clone());
loop {
for job in &jobs {
let targets = match discovery.get_target_groups(*job, log.clone()) {
Expand All @@ -42,10 +41,7 @@ pub fn config_writer_loop(
.with_label_values(&[job.to_string().as_str()])
.set(targets.len().try_into().unwrap());
if let Err(e) = config_writer.write_config(*job, targets, &vector_config_builder) {
warn!(
log,
"Failed to write config for targets for job {}: {:?}", job, e
);
warn!(log, "Failed to write config for targets for job {}: {:?}", job, e);
};
}
select! {
Expand Down
24 changes: 5 additions & 19 deletions rs/ic-observability/config-writer-common/src/filters.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,7 @@ mod tests {

fn create_dummy_target_group(ipv6: &str) -> TargetGroup {
let mut targets = BTreeSet::new();
targets.insert(std::net::SocketAddr::V6(
SocketAddrV6::from_str(ipv6).unwrap(),
));
targets.insert(std::net::SocketAddr::V6(SocketAddrV6::from_str(ipv6).unwrap()));
TargetGroup {
node_id: NodeId::from(PrincipalId::new_anonymous()),
ic_name: "mercury".into(),
Expand All @@ -82,10 +80,7 @@ mod tests {

let accepted_tg = TargetGroup {
node_id: NodeId::from(
PrincipalId::from_str(
"iylgr-zpxwq-kqgmf-4srtx-o4eey-d6bln-smmq6-we7px-ibdea-nondy-eae",
)
.unwrap(),
PrincipalId::from_str("iylgr-zpxwq-kqgmf-4srtx-o4eey-d6bln-smmq6-we7px-ibdea-nondy-eae").unwrap(),
),
ic_name: "mercury".into(),
targets: BTreeSet::new(),
Expand All @@ -98,10 +93,7 @@ mod tests {

let rejected_tg = TargetGroup {
node_id: NodeId::from(
PrincipalId::from_str(
"x33ed-h457x-bsgyx-oqxqf-6pzwv-wkhzr-rm2j3-npodi-purzm-n66cg-gae",
)
.unwrap(),
PrincipalId::from_str("x33ed-h457x-bsgyx-oqxqf-6pzwv-wkhzr-rm2j3-npodi-purzm-n66cg-gae").unwrap(),
),
ic_name: "mercury".into(),
targets: BTreeSet::new(),
Expand All @@ -121,10 +113,7 @@ mod tests {

let accepted_tg = TargetGroup {
node_id: NodeId::from(
PrincipalId::from_str(
"iylgr-zpxwq-kqgmf-4srtx-o4eey-d6bln-smmq6-we7px-ibdea-nondy-eae",
)
.unwrap(),
PrincipalId::from_str("iylgr-zpxwq-kqgmf-4srtx-o4eey-d6bln-smmq6-we7px-ibdea-nondy-eae").unwrap(),
),
ic_name: "mercury".into(),
targets: BTreeSet::new(),
Expand All @@ -137,10 +126,7 @@ mod tests {

let rejected_tg_1 = TargetGroup {
node_id: NodeId::from(
PrincipalId::from_str(
"x33ed-h457x-bsgyx-oqxqf-6pzwv-wkhzr-rm2j3-npodi-purzm-n66cg-gae",
)
.unwrap(),
PrincipalId::from_str("x33ed-h457x-bsgyx-oqxqf-6pzwv-wkhzr-rm2j3-npodi-purzm-n66cg-gae").unwrap(),
),
ic_name: "mercury".into(),
targets: BTreeSet::new(),
Expand Down
17 changes: 3 additions & 14 deletions rs/ic-observability/multiservice-discovery-downloader/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,7 @@ fn main() {

info!(logger, "Starting downloader loop"; "cli_args" => ?cli_args);

let downloader_handle = rt.spawn(run_downloader_loop(
logger.clone(),
cli_args,
stop_signal_rcv,
));
let downloader_handle = rt.spawn(run_downloader_loop(logger.clone(), cli_args, stop_signal_rcv));

rt.block_on(shutdown_signal);
info!(logger, "Received shutdown signal, shutting down ...");
Expand Down Expand Up @@ -127,11 +123,7 @@ pub mod log_subtype {
use super::*;
#[derive(Parser, Clone, Debug)]
pub struct LogSubtype {
#[clap(
long = "port",
help = "Custom port for standard nodes",
default_value = "19531"
)]
#[clap(long = "port", help = "Custom port for standard nodes", default_value = "19531")]
pub port: u64,
#[clap(
long = "boundary-nodes-port",
Expand All @@ -158,10 +150,7 @@ pub mod log_subtype {
#[clap(long = "journals-folder", help = "Path to the root journals folder")]
journals_folder: String,

#[clap(
long = "worker-cursor-folder",
help = "Path for the root worker cursors folder"
)]
#[clap(long = "worker-cursor-folder", help = "Path for the root worker cursors folder")]
worker_cursor_folder: String,

#[clap(long = "data-folder", help = "Path for the data folder")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,7 @@ impl TargetGroupFilter for TargetGroupFilterList {
if self.filters.is_empty() {
true
} else {
self.filters
.iter()
.map(|f| f.filter(target_group))
.all(|status| status)
self.filters.iter().map(|f| f.filter(target_group)).all(|status| status)
}
}
}
2 changes: 2 additions & 0 deletions rs/ic-observability/multiservice-discovery/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ tokio = { workspace = true }
url = { workspace = true }
futures.workspace = true
axum = "0.7.4"
axum-otel-metrics = "0.8.0"
opentelemetry = { version = "0.21.0", features = ["metrics"] }
retry = { workspace = true }

[dev-dependencies]
Expand Down
Loading

0 comments on commit a480f59

Please sign in to comment.