From d86d10503fa43692fdb33c3b4d7a8bf3f6e40e41 Mon Sep 17 00:00:00 2001 From: Bob Gardner Date: Sun, 1 Mar 2020 18:36:56 -0500 Subject: [PATCH] Finish command-line MVP --- app/pocket_cleaner/Cargo.lock | 80 ++++++++++++++++++ app/pocket_cleaner/Cargo.toml | 11 ++- app/pocket_cleaner/README.md | 35 ++++++++ app/pocket_cleaner/src/error.rs | 13 +++ app/pocket_cleaner/src/main.rs | 61 ++++++++++++-- app/pocket_cleaner/src/pocket.rs | 134 +++++++++++++++++++++++++++++++ app/pocket_cleaner/src/trends.rs | 31 ++++--- 7 files changed, 346 insertions(+), 19 deletions(-) create mode 100644 app/pocket_cleaner/README.md create mode 100644 app/pocket_cleaner/src/error.rs create mode 100644 app/pocket_cleaner/src/pocket.rs diff --git a/app/pocket_cleaner/Cargo.lock b/app/pocket_cleaner/Cargo.lock index 7de5dfc..d94c002 100644 --- a/app/pocket_cleaner/Cargo.lock +++ b/app/pocket_cleaner/Cargo.lock @@ -278,6 +278,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "anyhow" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c" + [[package]] name = "arc-swap" version = "0.4.4" @@ -295,6 +301,17 @@ dependencies = [ "syn", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.8", +] + [[package]] name = "autocfg" version = "1.0.0" @@ -500,6 +517,19 @@ dependencies = [ "syn", ] +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "failure" version = "0.1.6" @@ -746,6 +776,15 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + [[package]] name = "idna" version = "0.2.0" @@ -1064,10 +1103,13 @@ version = "0.1.0" dependencies = [ "actix-rt", "actix-web", + "anyhow", "chrono", + "env_logger", "log", "serde", "serde_json", + "thiserror", "url", ] @@ -1317,6 +1359,35 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "termcolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee14bf8e6767ab4c687c9e8bc003879e042a96fd67a3ba5934eadb6536bef4db" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b51e1fbc44b5a0840be594fbc0f960be09050f2617e61e6aa43bef97cd3ef4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.0.1" @@ -1517,6 +1588,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ccfbf554c6ad11084fb7517daca16cfdcaccbdadba4fc336f032a8b12c2ad80" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/app/pocket_cleaner/Cargo.toml b/app/pocket_cleaner/Cargo.toml index 896a125..9af2ca4 100644 --- a/app/pocket_cleaner/Cargo.toml +++ b/app/pocket_cleaner/Cargo.toml @@ -7,10 +7,13 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -chrono = "0.4.10" +actix-rt = "1.0.0" actix-web = { version = "2.0.0", features = ["openssl"] } -url = "2.1.1" -serde = "1.0.104" +anyhow = "1.0.26" +chrono = "0.4.10" +env_logger = "0.7.1" log = "0.4.8" +serde = "1.0.104" serde_json = "1.0.48" -actix-rt = "1.0.0" +thiserror = "1.0.11" +url = "2.1.1" diff --git a/app/pocket_cleaner/README.md b/app/pocket_cleaner/README.md new file mode 100644 index 0000000..9f93d82 --- /dev/null +++ b/app/pocket_cleaner/README.md @@ -0,0 +1,35 @@ +# Pocket Cleaner + +Finds items from your Pocket library that are relevant to trending news. + +```sh +$ pocket_cleaner +1. amirrajan/survivingtheappstore (Why: Real Madrid) +2. CppCon 2017: Nicolas Guillemot “Design Patterns for Low-Level Real-Time Rendering” (Why: Real Madrid) +3. I Am Legend author Richard Matheson dies (Why: Mikaela Spielberg) +4. Record and share your terminal sessions, the right way. (Why: Mikaela Spielberg) +5. Firefox (1982) (Why: Carrie Symonds) +6. Navy Drone Lands on Aircraft Carrier (Why: Carrie Symonds) +7. Hillary Clinton on the Sanctity of Protecting Classified Information (Why: Drake) +8. EFF’s Game Plan for Ending Global Mass Surveillance (Why: Drake) +9. Drawing with Ants: Generative Art with Ant Colony Optimization Algorithms (Why: El Clasico 2020) +10. All 50+ Adobe apps explained in 10 minutes (Why: El Clasico 2020) +``` + +## Getting Started + +Set the following environment variables: + +- `POCKET_CLEANER_CONSUMER_KEY` + - Create a Pocket app on the [Pocket Developer + Portal](https://getpocket.com/developer/apps/) +- `POCKET_TEMP_USER_ACCESS_TOKEN` + - This will go away soon, but for now, manually use the [Pocket Authentication API](https://getpocket.com/developer/docs/authentication) to obtain your user access token. + +```sh +export POCKET_CLEANER_CONSUMER_KEY="" +export POCKET_TEMP_USER_ACCESS_TOKEN="" +``` + +Then, run `cargo run` to build and run Pocket Cleaner to obtain +items from your Pocket list that are relevant to trending news. diff --git a/app/pocket_cleaner/src/error.rs b/app/pocket_cleaner/src/error.rs new file mode 100644 index 0000000..e061947 --- /dev/null +++ b/app/pocket_cleaner/src/error.rs @@ -0,0 +1,13 @@ +//! A module for working with Pocket Cleaner errors. + +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum PocketCleanerError { + #[error("faulty logic: {0}")] + Logic(String), + #[error("unknown error: {0}")] + Unknown(String), +} + +pub type Result = std::result::Result; diff --git a/app/pocket_cleaner/src/main.rs b/app/pocket_cleaner/src/main.rs index ea6b623..6b8499b 100644 --- a/app/pocket_cleaner/src/main.rs +++ b/app/pocket_cleaner/src/main.rs @@ -1,15 +1,66 @@ //! Surfaces items from your [Pocket](https://getpocket.com) library based on //! trending headlines. -use crate::trends::{Geo, TrendFinder}; +#![deny( + clippy::all, + missing_debug_implementations, + missing_copy_implementations, + trivial_casts, + trivial_numeric_casts, + unsafe_code, + unused_import_braces, + unused_qualifications +)] +use std::env; + +use anyhow::{Context, Result}; +use env_logger::Env; + +use crate::{ + pocket::PocketManager, + trends::{Geo, TrendFinder}, +}; + +mod error; +mod pocket; mod trends; +static POCKET_CONSUMER_KEY_ENV_VAR: &str = "POCKET_CLEANER_CONSUMER_KEY"; +static POCKET_USER_ACCESS_TOKEN: &str = "POCKET_TEMP_USER_ACCESS_TOKEN"; + +fn get_pocket_consumer_key() -> Result { + let key = POCKET_CONSUMER_KEY_ENV_VAR; + let value = env::var(key).with_context(|| format!("missing app config env var: {}", key))?; + Ok(value) +} + +async fn try_main() -> Result<()> { + env_logger::from_env(Env::default().default_filter_or("warn")).init(); + + let trend_finder = TrendFinder::new(); + let trends = trend_finder.daily_trends(&Geo("US".into())).await?; + + let pocket_consumer_key = get_pocket_consumer_key()?; + let pocket_manager = PocketManager::new(pocket_consumer_key); + let user_pocket = pocket_manager.for_user(&env::var(POCKET_USER_ACCESS_TOKEN)?); + + let mut items = Vec::new(); + for trend in trends[..5].iter() { + let mut relevant_items = user_pocket.get_items(&trend.name()).await?; + items.extend(relevant_items.drain(..5).map(|i| (trend.name(), i))); + } + + for (i, item) in items.iter().enumerate() { + println!("{} {} (Why: {})", i, item.1.title(), item.0); + } + + Ok(()) +} + #[actix_rt::main] async fn main() { - let trend_finder = TrendFinder::new(); - let trends = trend_finder.daily_trends(&Geo("US".into())).await.unwrap(); - for (i, trend) in trends.iter().enumerate() { - println!("{}. {}", i, trend.name()); + if let Err(e) = try_main().await { + eprintln!("{}", e); } } diff --git a/app/pocket_cleaner/src/pocket.rs b/app/pocket_cleaner/src/pocket.rs new file mode 100644 index 0000000..013ffee --- /dev/null +++ b/app/pocket_cleaner/src/pocket.rs @@ -0,0 +1,134 @@ +//! A module for working with a user's [Pocket](https://getpocket.com) library. + +use std::collections::HashMap; + +use actix_web::{ + client::Client, + http::{uri::Uri, PathAndQuery}, +}; +use serde::Deserialize; +use url::form_urlencoded; + +use crate::error::{PocketCleanerError, Result}; + +pub struct PocketManager { + consumer_key: String, +} + +pub struct UserPocketManager { + consumer_key: String, + user_access_token: String, +} + +impl PocketManager { + pub fn new(consumer_key: String) -> Self { + PocketManager { consumer_key } + } + + pub fn for_user(&self, user_access_token: &str) -> UserPocketManager { + UserPocketManager { + consumer_key: self.consumer_key.clone(), + user_access_token: user_access_token.into(), + } + } +} + +#[derive(Clone, Debug)] +pub struct PocketItem { + title: String, +} + +impl UserPocketManager { + pub async fn get_items(&self, keyword: &str) -> Result> { + let client = Client::default(); + let req = PocketRetrieveItemRequest { + consumer_key: self.consumer_key.clone(), + user_access_token: self.user_access_token.clone(), + search: Some(keyword.into()), + }; + let resp = send_pocket_retrieve_request(&client, &req).await?; + Ok(resp.list.values().cloned().map(PocketItem::from).collect()) + } +} + +impl PocketItem { + pub fn title(&self) -> String { + self.title.clone() + } +} + +impl From for PocketItem { + fn from(remote: RemotePocketItem) -> Self { + Self { + title: remote.resolved_title, + } + } +} + +struct PocketRetrieveItemRequest { + consumer_key: String, + user_access_token: String, + search: Option, +} + +#[derive(Deserialize, PartialEq, Eq, Hash, Clone, Debug)] +struct RemotePocketItemId(String); + +#[derive(Deserialize, Debug)] +struct PocketRetrieveItemResponse { + list: HashMap, +} + +#[derive(Clone, Deserialize, Debug)] +struct RemotePocketItem { + item_id: RemotePocketItemId, + resolved_title: String, +} + +fn build_pocket_retrieve_url(req: &PocketRetrieveItemRequest) -> Result { + let mut query_builder = form_urlencoded::Serializer::new(String::new()); + query_builder.append_pair("consumer_key", &req.consumer_key); + query_builder.append_pair("access_token", &req.user_access_token); + if let Some(search) = &req.search { + query_builder.append_pair("search", &search); + } + + let encoded: String = query_builder.finish(); + + let path_and_query: PathAndQuery = format!("/v3/get?{}", encoded).parse().unwrap(); + Ok(Uri::builder() + .scheme("https") + .authority("getpocket.com") + .path_and_query(path_and_query) + .build() + .map_err(|e| PocketCleanerError::Logic(e.to_string()))?) +} + +async fn send_pocket_retrieve_request( + client: &Client, + req: &PocketRetrieveItemRequest, +) -> Result { + let url = build_pocket_retrieve_url(req)?; + let mut response = client + .get(url) + .send() + .await + .map_err(|e| PocketCleanerError::Unknown(e.to_string()))?; + let body = response + .body() + .await + .map_err(|e| PocketCleanerError::Unknown(e.to_string()))?; + let body = + std::str::from_utf8(&body).map_err(|e| PocketCleanerError::Unknown(e.to_string()))?; + + let data: Result = + serde_json::from_str(body).map_err(|e| PocketCleanerError::Unknown(e.to_string())); + + match data { + Ok(data) => Ok(data), + Err(e) => { + log::error!("failed to deserialize payload: {}", body); + Err(e) + } + } +} diff --git a/app/pocket_cleaner/src/trends.rs b/app/pocket_cleaner/src/trends.rs index 1e2de18..958e340 100644 --- a/app/pocket_cleaner/src/trends.rs +++ b/app/pocket_cleaner/src/trends.rs @@ -7,6 +7,8 @@ use actix_web::{ use serde::Deserialize; use url::form_urlencoded; +use crate::error::{PocketCleanerError, Result}; + pub struct TrendFinder; #[derive(Clone, Debug)] @@ -21,7 +23,7 @@ impl TrendFinder { TrendFinder {} } - pub async fn daily_trends(&self, geo: &Geo) -> actix_web::Result> { + pub async fn daily_trends(&self, geo: &Geo) -> Result> { let client = Client::default(); let req = DailyTrendsRequest::new(geo.clone()); let mut raw_trends = send_daily_trends_request(&client, &req).await?; @@ -83,10 +85,9 @@ struct TrendingSearchTitle { query: String, } -fn build_daily_trends_url(req: &DailyTrendsRequest) -> actix_web::Result { +fn build_daily_trends_url(req: &DailyTrendsRequest) -> Result { let mut query_builder = form_urlencoded::Serializer::new(String::new()); query_builder.append_pair("geo", &req.geo.0); - let encoded: String = query_builder.finish(); let path_and_query: PathAndQuery = format!("/trends/api/dailytrends?{}", encoded) @@ -96,27 +97,37 @@ fn build_daily_trends_url(req: &DailyTrendsRequest) -> actix_web::Result { .scheme("https") .authority("trends.google.com") .path_and_query(path_and_query) - .build()?) + .build() + .map_err(|e| PocketCleanerError::Logic(e.to_string()))?) } async fn send_daily_trends_request( client: &Client, req: &DailyTrendsRequest, -) -> actix_web::Result { +) -> Result { let url = build_daily_trends_url(req)?; - let mut response = client.get(url).send().await?; - let body = response.body().await?; - let body = std::str::from_utf8(&body)?; + let mut response = client + .get(url) + .send() + .await + .map_err(|e| PocketCleanerError::Unknown(e.to_string()))?; + let body = response + .body() + .await + .map_err(|e| PocketCleanerError::Unknown(e.to_string()))?; + let body = + std::str::from_utf8(&body).map_err(|e| PocketCleanerError::Unknown(e.to_string()))?; // For some reason, Google Trends prepends 5 characters at the start of the // response that makes this invalid JSON, specifically: ")]}'," - let data: Result = serde_json::from_str(&body[5..]); + let data: Result = + serde_json::from_str(&body[5..]).map_err(|e| PocketCleanerError::Unknown(e.to_string())); match data { Ok(data) => Ok(data), Err(e) => { log::error!("failed to deserialize payload: {}", body); - Err(e.into()) + Err(e) } } }