Skip to content

Commit

Permalink
Merge pull request #6 from philippeitis/cache-searches
Browse files Browse the repository at this point in the history 
Cache searches
  • Loading branch information
@TheMayoras
TheMayoras committed Jan 19, 2021
2 parents d4dd895 + 39f9b64 commit 4d532ea
Show file tree
Hide file tree
Showing 5 changed files with 181 additions and 61 deletions.
1 change: 1 addition & 0 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,4 @@ structopt = "0.3.21"
default = ["clipboard"]

no-copy = []

108 changes: 53 additions & 55 deletions src/app.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ use tui::{
#[cfg(not(feature = "no-copy"))]
use clipboard::{ClipboardContext, ClipboardProvider};

#[cfg(not(feature = "no-copy"))]
use crate::crates_io::CrateSearch;

use crate::{
crates_io::{CrateSearchResponse, CrateSearcher, CratesSort},
ceil_div,
crates_io::{CrateSearcher, CratesSort},
input::InputEvent,
widgets::{CrateWidget, InputWidget, SortingWidget},
};
Expand Down Expand Up @@ -55,10 +55,10 @@ pub enum AppMode {
pub struct App {
input_rx: Receiver<InputEvent>,
client: CrateSearcher,
pub crates: Option<CrateSearchResponse>,
pub quit: bool,
inpt: Option<String>,
page: u32,
items_per_page: u32,
sort: CratesSort,
mode: AppMode,
selection: Option<usize>,
Expand All @@ -69,10 +69,10 @@ impl App {
Self {
input_rx,
client: CrateSearcher::new().unwrap(),
crates: None,
quit: false,
inpt: Some("".to_string()),
page: 1,
items_per_page: 5,
mode: AppMode::Input("".to_string()),
sort: CratesSort::Relevance,
selection: None,
Expand Down Expand Up @@ -117,20 +117,11 @@ impl App {
let bot = splits[1];
let area = splits[0];

if let Some(ref crates) = self.crates {
let message = Paragraph::new(format!(
"Page {} of {}",
self.page,
(crates.meta.total + 5) / 5
));
if let Some((total, crates)) = self.get_cached_crates() {
let message =
Paragraph::new(format!("Page {} of {}", self.page, self.num_pages(total)));
f.render_widget(message, bot);
}

if let Some(CrateSearchResponse {
ref crates,
meta: ref _meta,
}) = self.crates
{
let mut widgets = Vec::new();
for (i, crte) in crates.iter().enumerate() {
if let Some(selection) = self.selection {
Expand All @@ -140,17 +131,19 @@ impl App {
}
}

let mut raw = vec![100u16 / (self.items_per_page as u16); self.items_per_page as usize];
let sum_diff = 100 - raw.iter().sum::<u16>();
if sum_diff != 0 {
for item in raw[..sum_diff as usize].iter_mut() {
*item += 1;
}
}
let splits = Layout::default()
.horizontal_margin(1)
.constraints(
[
Constraint::Percentage(20),
Constraint::Percentage(20),
Constraint::Percentage(20),
Constraint::Percentage(20),
Constraint::Percentage(20),
]
.as_ref(),
raw.into_iter()
.map(Constraint::Percentage)
.collect::<Vec<_>>(),
)
.split(area);
widgets
Expand Down Expand Up @@ -179,8 +172,8 @@ impl App {

fn next_item(&mut self) {
if let Some(selection) = self.selection {
if let Some(cratex) = self.crates.as_ref() {
if selection + 1 >= cratex.crates.len() {
if let Some((_, crates)) = self.get_cached_crates() {
if selection + 1 >= crates.len() {
self.next_page();
} else {
self.selection = Some(selection + 1);
Expand All @@ -193,8 +186,8 @@ impl App {
if let Some(selection) = self.selection {
self.selection = if selection == 0 && self.page != 1 {
self.prev_page();
if let Some(cratex) = self.crates.as_ref() {
Some(cratex.crates.len().saturating_sub(1))
if let Some((_, crates)) = self.get_cached_crates() {
Some(crates.len().saturating_sub(1))
} else {
None
}
Expand All @@ -205,8 +198,8 @@ impl App {
}

fn next_page(&mut self) {
if let Some(cratex) = self.crates.as_ref() {
if self.page as usize * 5 < cratex.meta.total {
if let Some((total, _)) = self.get_cached_crates() {
if self.page * self.items_per_page < total {
self.selection = Some(0);
self.page += 1;
self.do_search();
Expand All @@ -230,15 +223,19 @@ impl App {
}
}

fn num_pages(&self, num_items: u32) -> u32 {
ceil_div(num_items, self.items_per_page)
}

fn end(&mut self) {
if let Some(cratex) = self.crates.as_ref() {
if self.page as usize * 5 < cratex.meta.total {
self.page = (cratex.meta.total as u32 + 5) / 5;
if let Some((total, _)) = self.get_cached_crates() {
if self.page * self.items_per_page < total {
self.page = self.num_pages(total);
self.do_search();
}
}
if let Some(cratex) = self.crates.as_ref() {
self.selection = cratex.crates.len().checked_sub(1);
if let Some((_, crates)) = self.get_cached_crates() {
self.selection = crates.len().checked_sub(1);
}
}

Expand Down Expand Up @@ -343,36 +340,37 @@ impl App {
}
}

fn get_cached_crates(&self) -> Option<(u32, Vec<&CrateSearch>)> {
let search = self.inpt.as_ref();
self.client.search_sorted_cached(
search.unwrap(),
self.page,
self.items_per_page,
&self.sort,
)
}

fn do_search(&mut self) {
let search = self.inpt.as_ref();
let resp = self
let (_, crates) = self
.client
.search_sorted(search.unwrap(), self.page, &self.sort);
match resp {
Ok(crates) => self.crates = Some(crates),
Err(_) => self.crates = None,
}

self.selection = if let Some(ref crates) = self.crates {
if crates.crates.len() > 0 {
Some(0)
} else {
None
}
} else {
None
}
.search_sorted_with_cache(search.unwrap(), self.page, self.items_per_page, &self.sort)
.unwrap_or((0, vec![]));
self.selection = if crates.is_empty() { None } else { Some(0) }
}

#[cfg(not(feature = "no-copy"))]
fn copy_selection(&self) {
if let Some(selection) = self.selection {
if let Some(ref crates) = self.crates {
let crte = crates.crates.get(selection);
let toml = crte.map(CrateSearch::get_toml_str);
if let Some((_, crates)) = self.get_cached_crates() {
let toml = crates
.get(selection)
.map(|x| CrateSearch::get_toml_str(x.to_owned()));
let mut clipboard: ClipboardContext = ClipboardProvider::new().unwrap();

toml.map(|toml| clipboard.set_contents(toml).unwrap());
if let Some(toml) = toml {
clipboard.set_contents(toml).unwrap()
}
}
}
}
Expand Down
115 changes: 113 additions & 2 deletions src/crates_io.rs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
use std::collections::HashMap;

use crate::ceil_div;
use std::str::FromStr;

use chrono::{DateTime, Local};
Expand Down Expand Up @@ -72,7 +75,7 @@ pub struct CrateSearchResponse {

#[derive(Serialize, Deserialize, Debug)]
pub struct CrateSearchResponseMeta {
pub total: usize,
pub total: u32,
pub next_page: Option<String>,
pub prev_page: Option<String>,
}
Expand Down Expand Up @@ -114,6 +117,7 @@ pub struct CrateSearchLinks {
/// A struct that will be used to search crates.io
pub struct CrateSearcher {
client: Client,
search_cache: HashMap<(String, String), (u32, HashMap<u32, CrateSearch>)>,
}

impl CrateSearcher {
Expand All @@ -122,18 +126,125 @@ impl CrateSearcher {
client: Client::builder()
.user_agent("craters-tui-searcher")
.build()?,
search_cache: HashMap::new(),
})
}
}
fn get_all_items(
page_cache: &HashMap<u32, CrateSearch>,
page: u32,
items_per_page: u32,
total_num: u32,
) -> Option<Vec<&CrateSearch>> {
let start = (page - 1) * items_per_page;
let end = page * items_per_page;
let mut res = Vec::with_capacity(items_per_page as usize);
for index in start..end.min(total_num) {
res.push(page_cache.get(&index)?);
}
Some(res)
}

impl CrateSearcher {
/// Adds the search query results to the internal cache.
pub fn search_and_add_to_cache<T: AsRef<str>>(
&mut self,
term: T,
page: u32,
items_per_page: u32,
sort: &CratesSort,
) -> Result<(), reqwest::Error> {
let key = (term.as_ref().to_string(), sort.to_sort_string());
let resp = self.search_sorted(term, page, items_per_page, sort)?;
let start = (page - 1) * items_per_page;
let (total, page_cache) = self.search_cache.entry(key).or_insert((0, HashMap::new()));
for (ind, item) in resp.crates.into_iter().enumerate() {
page_cache.insert(start + ind as u32, item);
}
*total = resp.meta.total;
Ok(())
}

/// Searches the query, defaulting to data available in the cache. If not cached, cache is updated
/// to include new values. May fetch more items at once to prevent excessive API requests.
pub fn search_sorted_with_cache<T: AsRef<str>>(
&mut self,
term: T,
page: u32,
items_per_page: u32,
sort: &CratesSort,
) -> Result<(u32, Vec<&CrateSearch>), reqwest::Error> {
let key = (term.as_ref().to_string(), sort.to_sort_string());
if !self.search_cache.contains_key(&key) {
self.search_and_add_to_cache(
term.as_ref(),
ceil_div(page, 10),
10 * items_per_page,
sort,
)?;
return Ok(self
.search_sorted_cached(term.as_ref(), page, items_per_page, sort)
.unwrap());
}

if self.check_page_cached(term.as_ref(), page, items_per_page, sort) {
return Ok(self
.search_sorted_cached(term.as_ref(), page, items_per_page, sort)
.unwrap());
}

// This thrashes the cache if the page size changes between calls.
self.search_and_add_to_cache(term.as_ref(), ceil_div(page, 10), 10 * items_per_page, sort)?;
let (total_num, page_cache) = self.search_cache.get(&key).unwrap();
let res = get_all_items(page_cache, page, items_per_page, *total_num).unwrap();
Ok((*total_num, res))
}

/// Checks if the given page is cached.
fn check_page_cached<T: AsRef<str>>(
&self,
term: T,
page: u32,
items_per_page: u32,
sort: &CratesSort,
) -> bool {
let key = (term.as_ref().to_string(), sort.to_sort_string());
if let Some((total_num, page_cache)) = self.search_cache.get(&key) {
let start = (page - 1) * items_per_page;
let end = page * items_per_page;
(start..end.min(*total_num))
.map(|x| page_cache.get(&x).is_some())
.all(|x| x)
} else {
false
}
}

/// Searches the query and associated pages from the internal cache.
pub fn search_sorted_cached<T: AsRef<str>>(
&self,
term: T,
page: u32,
items_per_page: u32,
sort: &CratesSort,
) -> Option<(u32, Vec<&CrateSearch>)> {
let key = (term.as_ref().to_string(), sort.to_sort_string());
let (total_num, page_cache) = self.search_cache.get(&key)?;
Some((
*total_num,
get_all_items(page_cache, page, items_per_page, *total_num)?,
))
}

/// Searches the query without any caching.
pub fn search_sorted<T: AsRef<str>>(
&self,
term: T,
page: u32,
items_per_page: u32,
sort: &CratesSort,
) -> Result<CrateSearchResponse, reqwest::Error> {
self.search_sorted_count(term, page, 5, sort)
self.search_sorted_count(term, page, items_per_page, sort)
}

pub fn search_sorted_count<T: AsRef<str>>(
Expand Down
10 changes: 10 additions & 0 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,16 @@ mod crates_io;
mod input;
mod widgets;

pub(crate) fn ceil_div(a: u32, b: u32) -> u32 {
if b == 0 {
panic!("attempt to divide by zero");
} else if a == 0 {
0
} else {
(a + b - 1) / b
}
}

#[derive(Debug, StructOpt)]
#[structopt(name = "Cratuity", about = "A simple TUI for searching Crates.io")]
/// A TUI for searching crates.io in the terminal.
Expand Down

0 comments on commit 4d532ea

Please sign in to comment.