fix spurious metrics test failures

nexus/test-utils/src/lib.rs

@@ -57,6 +57,7 @@ pub const TEST_PHYSICAL_RAM: u64 = 32 * (1 << 30);
 pub const TEST_SUITE_PASSWORD: &str = "oxide";
 
 pub struct ControlPlaneTestContext<N> {
+    pub start_time: chrono::DateTime<chrono::Utc>,
     pub external_client: ClientTestContext,
     pub internal_client: ClientTestContext,
     pub server: N,
@@ -157,6 +158,7 @@ pub async fn test_setup_with_config<N: NexusServer>(
     config: &mut omicron_common::nexus_config::Config,
     sim_mode: sim::SimMode,
 ) -> ControlPlaneTestContext<N> {
+    let start_time = chrono::Utc::now();
     let logctx = LogContext::new(test_name, &config.pkg.log);
     let log = &logctx.log;
 
@@ -360,6 +362,7 @@ pub async fn test_setup_with_config<N: NexusServer>(
     register_test_producer(&producer).unwrap();
 
     ControlPlaneTestContext {
+        start_time,
         server,
         external_client: testctx_external,
         internal_client: testctx_internal,

nexus/tests/integration_tests/disks.rs

@@ -4,9 +4,7 @@
 
 //! Tests basic disk support in the API
 
-use super::metrics::{
-    query_for_latest_metric, query_for_metrics_until_they_exist,
-};
+use super::metrics::{query_for_latest_metric, query_for_metrics};
 
 use chrono::Utc;
 use crucible_agent_client::types::State as RegionState;
@@ -1360,23 +1358,24 @@ async fn test_disk_metrics(cptestctx: &ControlPlaneTestContext) {
     let disk = create_disk(&client, PROJECT_NAME, DISK_NAME).await;
     oximeter.force_collect().await;
 
-    // Whenever we grab this URL, get the surrounding few seconds of metrics.
+    // When grabbing a metric, we look for data points going back to the
+    // start of this test all the way up to the current time.
     let metric_url = |metric: &str| {
         format!(
             "/v1/disks/{}/metrics/{}?start_time={:?}&end_time={:?}&project={}",
             DISK_NAME,
             metric,
-            Utc::now() - chrono::Duration::seconds(10),
-            Utc::now() + chrono::Duration::seconds(10),
+            cptestctx.start_time,
+            Utc::now(),
             PROJECT_NAME,
         )
     };
     // Check the utilization info for the whole project too.
     let utilization_url = |id: Uuid| {
         format!(
             "/v1/system/metrics/virtual_disk_space_provisioned?start_time={:?}&end_time={:?}&id={:?}",
-            Utc::now() - chrono::Duration::seconds(10),
-            Utc::now() + chrono::Duration::seconds(10),
+            cptestctx.start_time,
+            Utc::now(),
             id,
         )
     };
@@ -1400,9 +1399,7 @@ async fn test_disk_metrics(cptestctx: &ControlPlaneTestContext) {
     oximeter.force_collect().await;
 
     for metric in &ALL_METRICS {
-        let measurements =
-            query_for_metrics_until_they_exist(client, &metric_url(metric))
-                .await;
+        let measurements = query_for_metrics(client, &metric_url(metric)).await;
 
         assert!(!measurements.items.is_empty());
         for item in &measurements.items {
@@ -1442,8 +1439,8 @@ async fn test_disk_metrics_paginated(cptestctx: &ControlPlaneTestContext) {
         );
         let initial_params = format!(
             "start_time={:?}&end_time={:?}",
-            Utc::now() - chrono::Duration::seconds(2),
-            Utc::now() + chrono::Duration::seconds(2),
+            cptestctx.start_time,
+            Utc::now(),
         );
 
         objects_list_page_authz::<Measurement>(

nexus/tests/integration_tests/instances.rs

@@ -519,8 +519,8 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) {
     let metric_url = |metric_type: &str, id: Uuid| {
         format!(
             "/v1/system/metrics/{metric_type}?start_time={:?}&end_time={:?}&id={id}",
-            Utc::now() - chrono::Duration::seconds(30),
-            Utc::now() + chrono::Duration::seconds(30),
+            cptestctx.start_time,
+            Utc::now(),
         )
     };
     oximeter.force_collect().await;

nexus/tests/integration_tests/metrics.rs

@@ -8,19 +8,14 @@ use nexus_test_utils::resource_helpers::objects_list_page_authz;
 use oximeter::types::Datum;
 use oximeter::types::Measurement;
 
-pub async fn query_for_metrics_until_they_exist(
+pub async fn query_for_metrics(
     client: &ClientTestContext,
     path: &str,
 ) -> ResultsPage<Measurement> {
-    loop {
-        let measurements: ResultsPage<Measurement> =
-            objects_list_page_authz(client, path).await;
-
-        if !measurements.items.is_empty() {
-            return measurements;
-        }
-        tokio::time::sleep(tokio::time::Duration::from_millis(10)).await;
-    }
+    let measurements: ResultsPage<Measurement> =
+        objects_list_page_authz(client, path).await;
+    assert!(!measurements.items.is_empty());
+    measurements
 }
 
 pub async fn query_for_latest_metric(