Fix pagination, add retry

rgardner · Mar 10, 2020 · d5adbb1 · d5adbb1
1 parent 74fbef7
commit d5adbb1
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 20 deletions.
diff --git a/app/pocket_cleaner/src/lib.rs b/app/pocket_cleaner/src/lib.rs
@@ -18,7 +18,7 @@ pub mod pocket;
 pub mod trends;
 pub mod view;
 
-const ITEMS_PER_PAGE: i32 = 500;
+const ITEMS_PER_PAGE: u32 = 100;
 
 pub struct SavedItemMediator<'a> {
     pocket: &'a UserPocketManager,
@@ -64,8 +64,9 @@ impl<'a> SavedItemMediator<'a> {
                 .collect();
             self.saved_item_store.upsert_items(&store_items)?;
             log::debug!("Synced {} items to DB (page {})", store_items.len(), page);
-            offset += store_items.len() as i32;
-            if store_items.is_empty() {
+            let num_stored_items = store_items.len() as u32;
+            offset += num_stored_items;
+            if num_stored_items < ITEMS_PER_PAGE {
                 break since;
             }
         };

diff --git a/app/pocket_cleaner/src/pocket.rs b/app/pocket_cleaner/src/pocket.rs
@@ -62,16 +62,14 @@ impl UserPocketManager {
             PocketRetrieveItemList::Map(items) => {
                 Ok(items.values().cloned().map(PocketItem::from).collect())
             }
-            PocketRetrieveItemList::List(_) => Err(PocketCleanerError::Unknown(
-                "Invalid response from Pocket: received list instead of object".into(),
-            )),
+            PocketRetrieveItemList::List(_) => Ok(Vec::new()),
         }
     }
 
     pub async fn get_items_paginated(
         &self,
-        count: i32,
-        offset: i32,
+        count: u32,
+        offset: u32,
         since: Option<i64>,
     ) -> Result<PocketPage> {
         let client = Client::default();
@@ -90,11 +88,7 @@ impl UserPocketManager {
                 .cloned()
                 .map(PocketItem::from)
                 .collect::<Vec<_>>(),
-            PocketRetrieveItemList::List(_) => {
-                return Err(PocketCleanerError::Unknown(
-                    "Invalid response from Pocket: received list instead of object".into(),
-                ))
-            }
+            PocketRetrieveItemList::List(_) => Vec::new(),
         };
         Ok(PocketPage {
             items,
@@ -139,8 +133,8 @@ struct PocketRetrieveItemRequest {
     user_access_token: String,
     search: Option<String>,
     since: Option<i64>,
-    count: Option<i32>,
-    offset: Option<i32>,
+    count: Option<u32>,
+    offset: Option<u32>,
 }
 
 #[derive(Deserialize, PartialEq, Eq, Hash, Clone, Debug)]
@@ -201,11 +195,30 @@ async fn send_pocket_retrieve_request(
     req: &PocketRetrieveItemRequest,
 ) -> Result<PocketRetrieveItemResponse> {
     let url = build_pocket_retrieve_url(req)?;
-    let mut response = client
-        .get(url)
-        .send()
-        .await
-        .map_err(|e| PocketCleanerError::Unknown(e.to_string()))?;
+
+    let mut num_attempts = 0;
+    let mut response = loop {
+        if num_attempts == 3 {
+            return Err(PocketCleanerError::Unknown(format!(
+                "failed to connect to or receive a response from Pocket after {} attempts",
+                num_attempts
+            )));
+        }
+        let response = client.get(&url).send().await;
+        num_attempts += 1;
+        match response {
+            Ok(resp) => break resp,
+            Err(actix_web::client::SendRequestError::Connect(_))
+            | Err(actix_web::client::SendRequestError::Timeout) => continue,
+            Err(e) => {
+                return Err(PocketCleanerError::Unknown(format!(
+                    "failed to send 'pocket retrieve' request: {}",
+                    e
+                )))
+            }
+        }
+    };
+
     let body = response
         .body()
         .await