fix: gracefully handle prom failure for pb topo

packages/core/services/cluster/src/ops/datacenter/topology_get/mod.rs

@@ -486,7 +486,7 @@ async fn fetch_server_metrics(
 								}}
 								[5m]
 							)
-							[3h]
+							[3h:]
 						)
 					)
 				)
@@ -518,7 +518,7 @@ async fn fetch_server_metrics(
 									device=~"(eth0|eth1)"
 								}}[1m]
 							)
-						) [3h])
+						) [3h:])
 					)
 				)
 				# Convert from B/s to Kb/s

packages/core/services/cluster/src/ops/datacenter/topology_get/pegboard.rs

@@ -58,7 +58,17 @@ pub async fn pegboard_client_usage_get(ctx: &OperationCtx, input: &Input) -> Glo
 		)
 		.to_string(),
 	)
-	.await?;
+	.await;
+
+	// Gracefully handle prometheus failure, fallback to no data
+	let prom_res = match prom_res {
+		Ok(x) => x,
+		Err(err) => {
+			tracing::error!(?err, "failed to fetch pegboard prometheus metrics");
+
+			Default::default()
+		},
+	};
 
 	let mut stats_by_client_id = HashMap::new();
 

packages/core/services/cluster/src/workflows/datacenter/scale.rs

@@ -207,6 +207,14 @@ async fn calculate_diff(
 		}),
 	)?;
 
+	if !topology_res.prometheus_fetched {
+		tracing::error!("prometheus could not be fetched, not scaling");
+
+		return Ok(CalculateDiffOutput {
+			actions: Vec::new(),
+		});
+	}
+
 	let dc = unwrap!(datacenter_res.datacenters.first());
 
 	// Build hashmap from topos for sorting