Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions crates/core/tests/connectivity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,22 @@ async fn test_three_node_network_connectivity(ctx: &mut TestContext) -> TestResu
}

if !mesh_established {
tracing::error!(
gateway_peers = %last_snapshot.0,
peer1_peers = %last_snapshot.1,
peer2_peers = %last_snapshot.2,
"Connectivity check failed; dumping last snapshot"
);

if let Ok(aggregator) = ctx.aggregate_events().await {
if let Ok(events) = aggregator.get_all_events().await {
tracing::error!(total_events = events.len(), "Aggregated events at timeout");
for event in events.iter().rev().take(10).rev() {
tracing::error!(?event.kind, peer=%event.peer_id, ts=%event.datetime, "Recent event");
}
}
}

bail!(
"Failed to establish minimum connectivity after {} attempts. Gateway peers: {}; peer1 peers: {}; peer2 peers: {}",
MAX_RETRIES,
Expand Down
24 changes: 24 additions & 0 deletions crates/core/tests/operations.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,27 @@ use tokio::select;
use tokio::time::timeout;
use tokio_tungstenite::connect_async;

async fn log_recent_events(ctx: &TestContext, label: &str) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is a dump of all events in a file from the aggregator for debugging, we should include this in CI as an artifact somehow probably: #2076

if let Ok(aggregator) = ctx.aggregate_events().await {
if let Ok(events) = aggregator.get_all_events().await {
tracing::error!(
label,
total_events = events.len(),
"Aggregated events at failure"
);
for event in events.iter().rev().take(10).rev() {
tracing::error!(
label,
?event.kind,
peer = %event.peer_id,
ts = %event.datetime,
"Recent event"
);
}
}
}
}

static RNG: LazyLock<Mutex<rand::rngs::StdRng>> = LazyLock::new(|| {
Mutex::new(rand::rngs::StdRng::from_seed(
*b"0102030405060708090a0b0c0d0e0f10",
Expand Down Expand Up @@ -1950,12 +1971,15 @@ async fn test_put_contract_three_hop_returns_response(ctx: &mut TestContext) ->
assert_eq!(key, contract_key);
}
Ok(Ok(other)) => {
log_recent_events(ctx, "put-unexpected").await;
bail!("Unexpected response while waiting for put: {:?}", other);
}
Ok(Err(e)) => {
log_recent_events(ctx, "put-error").await;
bail!("Error receiving put response: {}", e);
}
Err(_) => {
log_recent_events(ctx, "put-timeout").await;
bail!("Timeout waiting for put response after 120 seconds");
}
}
Expand Down
Loading