Reduce hyper client pool idle timeout to 20s (#2384)

### Description

- configures hyper http client to close connections after 20s
- makes singleton signer retry (5) times before bubbling error up

### Backward compatibility

Yes

### Testing

Local binary testing against cloud resources

![Screen Shot 2023-06-13 at 5 45 47
PM](https://github.com/hyperlane-xyz/hyperlane-monorepo/assets/3020995/de47a03c-b40f-49fd-a32d-c8da6ecffa6e)

rust/chains/hyperlane-ethereum/src/singleton_signer.rs

@@ -19,6 +19,7 @@ type SignTask = (H256, Callback);
 /// made at a time. Mostly useful for the AWS signers.
 pub struct SingletonSigner {
     inner: Signers,
+    retries: usize,
     rx: mpsc::UnboundedReceiver<SignTask>,
 }
 
@@ -62,13 +63,38 @@ impl SingletonSigner {
     pub fn new(inner: Signers) -> (Self, SingletonSignerHandle) {
         let (tx, rx) = mpsc::unbounded_channel::<SignTask>();
         let address = inner.eth_address();
-        (Self { inner, rx }, SingletonSignerHandle { address, tx })
+        (
+            Self {
+                inner,
+                rx,
+                retries: 5,
+            },
+            SingletonSignerHandle { address, tx },
+        )
+    }
+
+    /// Change default (5) retries for signing
+    pub fn config_retries(&mut self, retries: usize) {
+        self.retries = retries;
     }
 
     /// Run this signer's event loop.
     pub async fn run(mut self) {
         while let Some((hash, tx)) = self.rx.recv().await {
-            if tx.send(self.inner.sign_hash(&hash).await).is_err() {
+            let mut retries = self.retries;
+            let res = loop {
+                match self.inner.sign_hash(&hash).await {
+                    Ok(res) => break Ok(res),
+                    Err(err) => {
+                        warn!("Error signing hash: {}", err);
+                        if retries == 0 {
+                            break Err(err);
+                        }
+                        retries -= 1;
+                    }
+                }
+            };
+            if tx.send(res).is_err() {
                 warn!(
                     "Failed to send signature back to the signer handle because the channel was closed"
                 );

rust/hyperlane-base/src/settings/signers.rs

@@ -1,7 +1,9 @@
+use std::time::Duration;
+
 use async_trait::async_trait;
 use ethers::prelude::{AwsSigner, LocalWallet};
 use eyre::{bail, eyre, Context, Report};
-use rusoto_core::{HttpClient, Region};
+use rusoto_core::{HttpClient, HttpConfig, Region};
 use rusoto_kms::KmsClient;
 use serde::Deserialize;
 use tracing::instrument;
@@ -104,10 +106,13 @@ impl BuildableWithSignerConf for hyperlane_ethereum::Signers {
                 ),
             )),
             SignerConf::Aws { id, region } => {
+                let mut config = HttpConfig::new();
+                // see https://github.com/hyperium/hyper/issues/2136#issuecomment-589345238
+                config.pool_idle_timeout(Duration::from_secs(20));
                 let client = KmsClient::new_with_client(
                     rusoto_core::Client::new_with(
                         AwsChainCredentialsProvider::new(),
-                        HttpClient::new().unwrap(),
+                        HttpClient::new_with_config(config).unwrap(),
                     ),
                     region.clone(),
                 );