more stable request handling

hyperium/hyper#2136 (comment)

src/crawler/common.rs

@@ -83,6 +83,15 @@ pub fn extract_link_size(body: &str) -> Vec<(String, i64)> {
     res
 }
 
+pub(crate) async fn fetch_body(url: &str) -> Result<String, BrokerError> {
+    let client = reqwest::Client::builder()
+        .user_agent("bgpkit-broker/3")
+        .pool_max_idle_per_host(0)
+        .build()?;
+    let body = client.get(url).send().await?.text().await?;
+    Ok(body)
+}
+
 /// Remove trailing slash from a string.
 ///
 /// # Arguments

src/crawler/riperis.rs

@@ -1,4 +1,6 @@
-use crate::crawler::common::{crawl_months_list, extract_link_size, remove_trailing_slash};
+use crate::crawler::common::{
+    crawl_months_list, extract_link_size, fetch_body, remove_trailing_slash,
+};
 use crate::crawler::Collector;
 use crate::{BrokerError, BrokerItem};
 use chrono::{NaiveDate, NaiveDateTime};
@@ -44,7 +46,7 @@ pub async fn crawl_ripe_ris(
 async fn crawl_month(url: String, collector_id: String) -> Result<Vec<BrokerItem>, BrokerError> {
     let url = remove_trailing_slash(url.as_str());
     debug!("crawling data for {} ...", url.as_str());
-    let body = reqwest::get(url.as_str()).await?.text().await?;
+    let body = fetch_body(url.as_str()).await?;
     debug!("    download for {} finished ", url.as_str());
 
     let new_url = url.to_string();

src/crawler/routeviews.rs

@@ -1,4 +1,6 @@
-use crate::crawler::common::{crawl_months_list, extract_link_size, remove_trailing_slash};
+use crate::crawler::common::{
+    crawl_months_list, extract_link_size, fetch_body, remove_trailing_slash,
+};
 use crate::crawler::Collector;
 use crate::{BrokerError, BrokerItem};
 use chrono::{NaiveDate, NaiveDateTime};
@@ -51,7 +53,7 @@ async fn crawl_month(url: String, collector_id: String) -> Result<Vec<BrokerItem
     // RIBS
     for subdir in ["RIBS", "UPDATES"] {
         let url = format!("{}/{}", &root_url, subdir);
-        let body = reqwest::get(url.as_str()).await?.text().await?;
+        let body = fetch_body(url.as_str()).await?;
         let collector_id_clone = collector_id.clone();
         let data_items: Vec<BrokerItem> = tokio::task::spawn_blocking(move || {
             let items = extract_link_size(body.as_str());