Skip to content

Commit

Permalink
Fix pagination, add retry
Browse files Browse the repository at this point in the history
  • Loading branch information
rgardner committed Mar 10, 2020
1 parent 74fbef7 commit d5adbb1
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 20 deletions.
7 changes: 4 additions & 3 deletions app/pocket_cleaner/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ pub mod pocket;
pub mod trends;
pub mod view;

const ITEMS_PER_PAGE: i32 = 500;
const ITEMS_PER_PAGE: u32 = 100;

pub struct SavedItemMediator<'a> {
pocket: &'a UserPocketManager,
Expand Down Expand Up @@ -64,8 +64,9 @@ impl<'a> SavedItemMediator<'a> {
.collect();
self.saved_item_store.upsert_items(&store_items)?;
log::debug!("Synced {} items to DB (page {})", store_items.len(), page);
offset += store_items.len() as i32;
if store_items.is_empty() {
let num_stored_items = store_items.len() as u32;
offset += num_stored_items;
if num_stored_items < ITEMS_PER_PAGE {
break since;
}
};
Expand Down
47 changes: 30 additions & 17 deletions app/pocket_cleaner/src/pocket.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,16 +62,14 @@ impl UserPocketManager {
PocketRetrieveItemList::Map(items) => {
Ok(items.values().cloned().map(PocketItem::from).collect())
}
PocketRetrieveItemList::List(_) => Err(PocketCleanerError::Unknown(
"Invalid response from Pocket: received list instead of object".into(),
)),
PocketRetrieveItemList::List(_) => Ok(Vec::new()),
}
}

pub async fn get_items_paginated(
&self,
count: i32,
offset: i32,
count: u32,
offset: u32,
since: Option<i64>,
) -> Result<PocketPage> {
let client = Client::default();
Expand All @@ -90,11 +88,7 @@ impl UserPocketManager {
.cloned()
.map(PocketItem::from)
.collect::<Vec<_>>(),
PocketRetrieveItemList::List(_) => {
return Err(PocketCleanerError::Unknown(
"Invalid response from Pocket: received list instead of object".into(),
))
}
PocketRetrieveItemList::List(_) => Vec::new(),
};
Ok(PocketPage {
items,
Expand Down Expand Up @@ -139,8 +133,8 @@ struct PocketRetrieveItemRequest {
user_access_token: String,
search: Option<String>,
since: Option<i64>,
count: Option<i32>,
offset: Option<i32>,
count: Option<u32>,
offset: Option<u32>,
}

#[derive(Deserialize, PartialEq, Eq, Hash, Clone, Debug)]
Expand Down Expand Up @@ -201,11 +195,30 @@ async fn send_pocket_retrieve_request(
req: &PocketRetrieveItemRequest,
) -> Result<PocketRetrieveItemResponse> {
let url = build_pocket_retrieve_url(req)?;
let mut response = client
.get(url)
.send()
.await
.map_err(|e| PocketCleanerError::Unknown(e.to_string()))?;

let mut num_attempts = 0;
let mut response = loop {
if num_attempts == 3 {
return Err(PocketCleanerError::Unknown(format!(
"failed to connect to or receive a response from Pocket after {} attempts",
num_attempts
)));
}
let response = client.get(&url).send().await;
num_attempts += 1;
match response {
Ok(resp) => break resp,
Err(actix_web::client::SendRequestError::Connect(_))
| Err(actix_web::client::SendRequestError::Timeout) => continue,
Err(e) => {
return Err(PocketCleanerError::Unknown(format!(
"failed to send 'pocket retrieve' request: {}",
e
)))
}
}
};

let body = response
.body()
.await
Expand Down

0 comments on commit d5adbb1

Please sign in to comment.