fix: backport rusoto timeout change to v2 (#3872)

Backport of
#3283

Applies the fix in
#2384 everywhere
an `HttpClient` is constructed via rusoto.

It lowers the S3 timeout to 15s based on tips in [this
thread](hyperium/hyper#2136 (comment)),
to avoid `Error during dispatch: connection closed before message
completed` errors. Note that we'll probably still run into these issues,
but less frequently

([source](rusoto/rusoto#1766 (comment))).

rust/hyperlane-base/src/settings/signers.rs

@@ -1,16 +1,15 @@
-use std::time::Duration;
-
 use async_trait::async_trait;
 use ed25519_dalek::SecretKey;
 use ethers::prelude::{AwsSigner, LocalWallet};
 use eyre::{bail, Context, Report};
 use hyperlane_core::H256;
 use hyperlane_sealevel::Keypair;
-use rusoto_core::{HttpClient, HttpConfig, Region};
+use rusoto_core::Region;
 use rusoto_kms::KmsClient;
 use tracing::instrument;
 
 use super::aws_credentials::AwsChainCredentialsProvider;
+use crate::types::utils;
 
 /// Signer types
 #[derive(Default, Debug, Clone)]
@@ -59,13 +58,10 @@ impl BuildableWithSignerConf for hyperlane_ethereum::Signers {
                 ),
             )),
             SignerConf::Aws { id, region } => {
-                let mut config = HttpConfig::new();
-                // see https://github.com/hyperium/hyper/issues/2136#issuecomment-589345238
-                config.pool_idle_timeout(Duration::from_secs(20));
                 let client = KmsClient::new_with_client(
                     rusoto_core::Client::new_with(
                         AwsChainCredentialsProvider::new(),
-                        HttpClient::new_with_config(config).unwrap(),
+                        utils::http_client_with_timeout().unwrap(),
                     ),
                     region.clone(),
                 );

rust/hyperlane-base/src/types/mod.rs

@@ -2,6 +2,8 @@ mod local_storage;
 mod multisig;
 mod s3_storage;
 
+/// Reusable logic for working with storage backends.
+pub mod utils;
 pub use local_storage::*;
 pub use multisig::*;
 pub use s3_storage::*;

rust/hyperlane-base/src/types/s3_storage.rs

@@ -8,11 +8,12 @@ use hyperlane_core::{SignedAnnouncement, SignedCheckpoint, SignedCheckpointWithM
 use prometheus::IntGauge;
 use rusoto_core::{
     credential::{Anonymous, AwsCredentials, StaticProvider},
-    HttpClient, Region, RusotoError,
+    Region, RusotoError,
 };
 use rusoto_s3::{GetObjectError, GetObjectRequest, PutObjectRequest, S3Client, S3};
 use tokio::time::timeout;
 
+use crate::types::utils;
 use crate::{settings::aws_credentials::AwsChainCredentialsProvider, CheckpointSyncer};
 
 /// The timeout for S3 requests. Rusoto doesn't offer timeout configuration
@@ -93,7 +94,7 @@ impl S3Storage {
     fn authenticated_client(&self) -> &S3Client {
         self.authenticated_client.get_or_init(|| {
             S3Client::new_with(
-                HttpClient::new().unwrap(),
+                utils::http_client_with_timeout().unwrap(),
                 AwsChainCredentialsProvider::new(),
                 self.region.clone(),
             )
@@ -113,7 +114,7 @@ impl S3Storage {
             assert!(credentials.is_anonymous(), "AWS credentials not anonymous");
 
             S3Client::new_with(
-                HttpClient::new().unwrap(),
+                utils::http_client_with_timeout().unwrap(),
                 StaticProvider::from(credentials),
                 self.region.clone(),
             )

rust/hyperlane-base/src/types/utils.rs

@@ -0,0 +1,15 @@
+use std::time::Duration;
+
+use eyre::Result;
+use rusoto_core::{HttpClient, HttpConfig};
+
+/// See https://github.com/hyperium/hyper/issues/2136#issuecomment-589488526
+pub const HYPER_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(15);
+
+/// Create a new HTTP client with a timeout for the connection pool.
+/// This is a workaround for https://github.com/hyperium/hyper/issues/2136#issuecomment-589345238
+pub fn http_client_with_timeout() -> Result<HttpClient> {
+    let mut config = HttpConfig::new();
+    config.pool_idle_timeout(HYPER_POOL_IDLE_TIMEOUT);
+    Ok(HttpClient::new_with_config(config)?)
+}

typescript/infra/config/environments/mainnet2/agent.ts

@@ -141,14 +141,14 @@ const hyperlane: RootAgentConfig = {
   validators: {
     docker: {
       repo,
-      tag: 'ed7569d-20230725-171222',
+      tag: 'c30f471-20240530-150953',
     },
     chainDockerOverrides: {
       [chainMetadata.solana.name]: {
-        tag: '475bd1c-20240416-105206',
+        tag: 'c30f471-20240530-150953',
       },
       [chainMetadata.nautilus.name]: {
-        tag: '3b0685f-20230815-110725',
+        tag: 'c30f471-20240530-150953',
       },
     },
     connectionType: AgentConnectionType.HttpQuorum,