test failed in CI: oximeter-db client::tests::test_replicated #10052
Description
This test failed on a CI run on #10050:
https://github.com/oxidecomputer/omicron/pull/10050/checks?check_run_id=66875697907
Log showing the specific test failure:
Excerpt from the log showing the failure:
There is an enormous amount of output from the test, but it ends with:
18551 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Bool([true]), data_type: Bool }} }
18552 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Bool([true]), data_type: Bool }} }
18553 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: UInt8([1]), data_type: UInt8 }} }
18554 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: UInt8([1]), data_type: UInt8 }} }
18555 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Int8([1]), data_type: Int8 }} }
18556 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Int8([1]), data_type: Int8 }} }
18557 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: UInt16([1]), data_type: UInt16 }} }
18558 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: UInt16([1]), data_type: UInt16 }} }
18559 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Int16([1]), data_type: Int16 }} }
18560 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Int16([1]), data_type: Int16 }} }
18561 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: UInt32([1]), data_type: UInt32 }} }
18562 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: UInt32([1]), data_type: UInt32 }} }
18563 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Int32([1]), data_type: Int32 }} }
18564 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Int32([1]), data_type: Int32 }} }
18565 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: UInt64([1]), data_type: UInt64 }} }
18566 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: UInt64([1]), data_type: UInt64 }} }
18567 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Int64([1]), data_type: Int64 }} }
18568 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Int64([1]), data_type: Int64 }} }
18569 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: String(["foo"]), data_type: String }} }
18570 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: String(["foo"]), data_type: String }} }
18571 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Ipv6([::1]), data_type: Ipv6 }} }
18572 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Ipv6([::1]), data_type: Ipv6 }} }
18573 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Uuid([9fb9b4fc-f9a9-4ea4-a583-4bd2d07b45da]), data_type: Uuid }} }
18574 2026-03-12T22:38:44.618Z Block { name: "", info: BlockInfo { is_overflows: false, bucket_num: -1 }, columns: {"timeseries_name": Column { values: String(["foo:bar"]), data_type: String }, "timeseries_key": Column { values: UInt64([101]), data_type: UInt64 }, "field_name": Column { values: String(["baz"]), data_type: String }, "field_value": Column { values: Uuid([9fb9b4fc-f9a9-4ea4-a583-4bd2d07b45da]), data_type: Uuid }} }
18575 2026-03-12T22:38:44.618Z test client::tests::test_replicated ... FAILED
18576 2026-03-12T22:38:44.618Z
18577 2026-03-12T22:38:44.618Z failures:
18578 2026-03-12T22:38:44.618Z
18579 2026-03-12T22:38:44.618Z failures:
18580 2026-03-12T22:38:44.618Z client::tests::test_replicated
18581 2026-03-12T22:38:44.618Z
18582 2026-03-12T22:38:44.623Z test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 246 filtered out; finished in 682.96s
18583 2026-03-12T22:38:44.623Z
18584 2026-03-12T22:38:44.623Z stderr ───
18585 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_replicated.60765.0.log
18586 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_replicated.60765.0.log"
18587 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_is_oximeter_cluster_replicated.60765.6.log
18588 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_is_oximeter_cluster_replicated.60765.6.log"
18589 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_insert_samples_replicated.60765.7.log
18590 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_insert_samples_replicated.60765.7.log"
18591 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_schema_mismatch_replicated.60765.8.log
18592 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_schema_mismatch_replicated.60765.8.log"
18593 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_schema_updated_replicated.60765.9.log
18594 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_schema_updated_replicated.60765.9.log"
18595 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_client_select_timeseries_one_replicated.60765.10.log
18596 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_client_select_timeseries_one_replicated.60765.10.log"
18597 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_field_record_count_replicated.60765.11.log
18598 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_field_record_count_replicated.60765.11.log"
18599 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_differentiate_by_timeseries_name_replicated.60765.12.log
18600 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_differentiate_by_timeseries_name_replicated.60765.12.log"
18601 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_select_one_replicated.60765.13.log
18602 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_select_one_replicated.60765.13.log"
18603 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_select_one_field_with_multiple_values_replicated.60765.14.log
18604 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_select_one_field_with_multiple_values_replicated.60765.14.log"
18605 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_select_multiple_fields_with_multiple_values_replicated.60765.15.log
18606 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_select_multiple_fields_with_multiple_values_replicated.60765.15.log"
18607 2026-03-12T22:38:44.623Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_all_replicated.60765.16.log
18608 2026-03-12T22:38:44.623Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_all_replicated.60765.16.log"
18609 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_start_time_replicated.60765.17.log
18610 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_start_time_replicated.60765.17.log"
18611 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_limit_replicated.60765.18.log
18612 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_limit_replicated.60765.18.log"
18613 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_order_replicated.60765.19.log
18614 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_timeseries_with_order_replicated.60765.19.log"
18615 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_get_schema_no_new_values_replicated.60765.20.log
18616 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_get_schema_no_new_values_replicated.60765.20.log"
18617 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_timeseries_schema_list_replicated.60765.21.log
18618 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_timeseries_schema_list_replicated.60765.21.log"
18619 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_list_timeseries_replicated.60765.22.log
18620 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_list_timeseries_replicated.60765.22.log"
18621 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_list_timeseries_replicated.60765.23.log
18622 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_list_timeseries_replicated.60765.23.log"
18623 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_database_version_update_idempotent_replicated.60765.24.log
18624 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_database_version_update_idempotent_replicated.60765.24.log"
18625 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_database_version_will_not_downgrade_replicated.60765.25.log
18626 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_database_version_will_not_downgrade_replicated.60765.25.log"
18627 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_database_version_will_not_upgrade_replicated.60765.26.log
18628 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_database_version_will_not_upgrade_replicated.60765.26.log"
18629 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_update_schema_cache_on_new_sample_replicated.60765.27.log
18630 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_update_schema_cache_on_new_sample_replicated.60765.27.log"
18631 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_all_datum_types_replicated.60765.28.log
18632 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_select_all_datum_types_replicated.60765.28.log"
18633 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_new_schema_removed_when_not_inserted_replicated.60765.29.log
18634 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_new_schema_removed_when_not_inserted_replicated.60765.29.log"
18635 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_recall_of_all_fields_replicated.60765.30.log
18636 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_recall_of_all_fields_replicated.60765.30.log"
18637 2026-03-12T22:38:44.624Z log file: /var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_expunge_timeseries_by_name_replicated.60765.31.log
18638 2026-03-12T22:38:44.624Z note: configured to log to "/var/tmp/omicron_tmp/oximeter_db-344f7c3c30ba1776-test_expunge_timeseries_by_name_replicated.60765.31.log"
18639 2026-03-12T22:38:44.624Z
18640 2026-03-12T22:38:44.624Z thread 'client::tests::test_replicated' (60766) panicked at oximeter/db/src/client/mod.rs:4865:13:
18641 2026-03-12T22:38:44.624Z assertion `left == right` failed: Should not have any rows associated with the deleted timeseries 'first_target:shared_metric', but found 1 records in table timeseries_schema_local
18642 2026-03-12T22:38:44.624Z left: 1
18643 2026-03-12T22:38:44.624Z right: 0
18644 2026-03-12T22:38:44.624Z stack backtrace:
18645 2026-03-12T22:38:44.624Z 0: __rustc::rust_begin_unwind
18646 2026-03-12T22:38:44.624Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:698:5
18647 2026-03-12T22:38:44.624Z 1: core::panicking::panic_fmt
18648 2026-03-12T22:38:44.625Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:75:14
18649 2026-03-12T22:38:44.625Z 2: core::panicking::assert_failed_inner
18650 2026-03-12T22:38:44.625Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:434:23
18651 2026-03-12T22:38:44.625Z 3: core::panicking::assert_failed
18652 2026-03-12T22:38:44.625Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:394:5
18653 2026-03-12T22:38:44.625Z 4: {async_fn#0}
18654 2026-03-12T22:38:44.625Z at ./src/client/mod.rs:4865:13
18655 2026-03-12T22:38:44.625Z 5: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>
18656 2026-03-12T22:38:44.625Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9
18657 2026-03-12T22:38:44.625Z 6: {async_block#0}
18658 2026-03-12T22:38:44.625Z at ./src/client/mod.rs:1816:63
18659 2026-03-12T22:38:44.625Z 7: poll<&mut dyn core::future::future::Future<Output=()>>
18660 2026-03-12T22:38:44.625Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9
18661 2026-03-12T22:38:44.625Z 8: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
18662 2026-03-12T22:38:44.625Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9
18663 2026-03-12T22:38:44.625Z 9: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
18664 2026-03-12T22:38:44.625Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:70
18665 2026-03-12T22:38:44.625Z 10: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
18666 2026-03-12T22:38:44.625Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:167:5
18667 2026-03-12T22:38:44.625Z 11: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
18668 2026-03-12T22:38:44.625Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/task/coop/mod.rs:133:5
18669 2026-03-12T22:38:44.625Z 12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
18670 2026-03-12T22:38:44.625Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:753:25
18671 2026-03-12T22:38:44.625Z 13: tokio::runtime::scheduler::current_thread::Context::enter
18672 2026-03-12T22:38:44.625Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:442:19
18673 2026-03-12T22:38:44.625Z 14: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
18674 2026-03-12T22:38:44.625Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:752:44
18675 2026-03-12T22:38:44.625Z 15: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}}
18676 2026-03-12T22:38:44.625Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:68
18677 2026-03-12T22:38:44.625Z 16: tokio::runtime::context::scoped::Scoped<T>::set
18678 2026-03-12T22:38:44.625Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/scoped.rs:40:9
18679 2026-03-12T22:38:44.625Z 17: tokio::runtime::context::set_scheduler::{{closure}}
18680 2026-03-12T22:38:44.625Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:38
18681 2026-03-12T22:38:44.625Z 18: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#9}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#9}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)>
18682 2026-03-12T22:38:44.629Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:315:12
18683 2026-03-12T22:38:44.629Z 19: std::thread::local::LocalKey<T>::with
18684 2026-03-12T22:38:44.629Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:279:20
18685 2026-03-12T22:38:44.629Z 20: tokio::runtime::context::set_scheduler
18686 2026-03-12T22:38:44.629Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context.rs:176:17
18687 2026-03-12T22:38:44.629Z 21: tokio::runtime::scheduler::current_thread::CoreGuard::enter
18688 2026-03-12T22:38:44.629Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:840:27
18689 2026-03-12T22:38:44.629Z 22: tokio::runtime::scheduler::current_thread::CoreGuard::block_on
18690 2026-03-12T22:38:44.629Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:740:24
18691 2026-03-12T22:38:44.629Z 23: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
18692 2026-03-12T22:38:44.629Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:200:33
18693 2026-03-12T22:38:44.629Z 24: tokio::runtime::context::runtime::enter_runtime
18694 2026-03-12T22:38:44.629Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/context/runtime.rs:65:16
18695 2026-03-12T22:38:44.629Z 25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
18696 2026-03-12T22:38:44.629Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/scheduler/current_thread/mod.rs:188:9
18697 2026-03-12T22:38:44.629Z 26: tokio::runtime::runtime::Runtime::block_on_inner
18698 2026-03-12T22:38:44.630Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:368:52
18699 2026-03-12T22:38:44.630Z 27: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
18700 2026-03-12T22:38:44.630Z at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.49.0/src/runtime/runtime.rs:342:18
18701 2026-03-12T22:38:44.630Z 28: test_replicated
18702 2026-03-12T22:38:44.630Z at ./src/client/mod.rs:1821:36
18703 2026-03-12T22:38:44.630Z 29: oximeter_db::client::tests::test_replicated::{{closure}}
18704 2026-03-12T22:38:44.630Z at ./src/client/mod.rs:1634:31
18705 2026-03-12T22:38:44.630Z 30: core::ops::function::FnOnce::call_once
18706 2026-03-12T22:38:44.630Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5
18707 2026-03-12T22:38:44.630Z 31: core::ops::function::FnOnce::call_once
18708 2026-03-12T22:38:44.630Z at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5