From 3ecad150a20f4326a981563d43517bef53874a09 Mon Sep 17 00:00:00 2001 From: arctic_hen7 Date: Mon, 10 Jan 2022 16:33:15 +1100 Subject: [PATCH] feat: added external request caching --- docs/next/en-US/server-communication.md | 4 + examples/fetching/src/index.rs | 13 +- packages/perseus/src/cache_res.rs | 159 ++++++++++++++++++++++++ packages/perseus/src/lib.rs | 2 + 4 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 packages/perseus/src/cache_res.rs diff --git a/docs/next/en-US/server-communication.md b/docs/next/en-US/server-communication.md index 68f2813f05..2932083a21 100644 --- a/docs/next/en-US/server-communication.md +++ b/docs/next/en-US/server-communication.md @@ -29,6 +29,10 @@ In the last few years, a new technology has sprung up that allows you to run ind It's fairly trivial to communicate with a server at build-time in Perseus, which allows you to fetch data when you build your app, and then your users don't have to do as much work. You can also use other strategies to fetch data [at request-time](:strategies/request-state) if needed. Right now, it's best to use a blocking API to make requests on the server, which you can do with libraries like [`ureq`](https://docs.rs/ureq). +One of the problems with fetching data at build time (or request-time, etc.) in development is that it often adds a significant delay to building your project, which slows you down. To solve this, Perseus provides two functions `cache_res` and `cache_fallible_res` that can be used to wrap your request code. They'll run it the first time, and then they'll cache the result to `.perseus/cache/`, which will be used in all future requests. These functions take a name (for the cache file), the function to run (which must be async), and a boolean that can be set to `true` if you want to temporarily disable caching. Usefully, these functions don't have to be removed for production, because they'll automatically avoid caching there. You can find an example of using these in [this example](https://github.com/arctic-hen7/perseus/tree/main/examples/fetching). + +Incidentally, you can also use those functions to work in an offline environment, even if your app includes calls to external APIs at build time. As long as you've called your app's build process once so that Perseus can cache all the requests, it won't make any more network requests in development unless you tell it to explicitly or delete `.perseus/cache/`. + ### In the Browser In some cases, it's just not possible to fetch the data you need on the server, and the client needs to fetch it themselves. This is often the case in [exported](:exporting) apps. This is typically done with the browser's inbuilt Fetch API, which is conveniently wrapped by [`reqwasm`](https://docs.rs/reqwasm). Note that you'll need to do this in a `Future`, which you can spawn using [`wasm_bindgen_futures::spawn_local`](https://docs.rs/wasm-bindgen-futures/latest/wasm_bindgen_futures/fn.spawn_local.html), conveniently re-exported from Perseus as `perseus::spawn_local`. You can then modify a `Signal` in there that holds the data you want. It's common practice in web development today to show a page skeleton (those pulsating bars instead of text) while data are still being loaded. diff --git a/examples/fetching/src/index.rs b/examples/fetching/src/index.rs index a633dec871..3cc48b4aed 100644 --- a/examples/fetching/src/index.rs +++ b/examples/fetching/src/index.rs @@ -52,7 +52,16 @@ pub async fn get_build_state( _path: String, _locale: String, ) -> RenderFnResultWithCause { - // This just gets the IP address of the machine that built the app - let body: String = ureq::get("https://api.ipify.org").call()?.into_string()?; + // We'll cache the result with `try_cache_res`, which means we only make the request once, and future builds will use the cached result (speeds up development) + let body: String = perseus::cache_fallible_res( + "ipify", + || async { + // This just gets the IP address of the machine that built the app + let res = ureq::get("https://api.ipify.org").call()?.into_string()?; + Ok::(res) + }, + false, + ) + .await?; Ok(IndexProps { ip: body }) } diff --git a/packages/perseus/src/cache_res.rs b/packages/perseus/src/cache_res.rs new file mode 100644 index 0000000000..3ab2da7ea6 --- /dev/null +++ b/packages/perseus/src/cache_res.rs @@ -0,0 +1,159 @@ +use std::any::Any; +use std::convert::Infallible; + +use futures::Future; +use serde::{Deserialize, Serialize}; +use tokio::fs::{create_dir_all, File}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +/// Runs the given function once and then caches the result to the filesystem for future execution. Think of this as filesystem-level memoizing. In future, this will be broken out into +/// its own crate and wrapped by Perseus. The second parameter to this allows forcing the function to re-fetch data every time, which is useful if you want to revalidate data or test +/// your fetching logic again. Note that a change to the logic will not trigger a reload unless you make it do so. For this reason, it's recommended to only use this wrapper once +/// you've tested your fetching logic. +/// +/// When running automated tests, you may wish to set `force_run` to the result of an environment variable check that you'll use when testing. +/// +/// This function expects to be run in the context of `.perseus/`, or any directory in which a folder `cache/` is available. If you're using Perseus without the CLI and you don't want +/// that directory to exist, you shouldn't use this function. +/// +/// # Panics +/// If this filesystem operations fail, this function will panic. It can't return a graceful error since it's expected to return the type you requested. +pub async fn cache_res(name: &str, f: F, force_run: bool) -> D +where + // By making this `Any`, we can downcast it to manage errors intuitively + D: Serialize + for<'de> Deserialize<'de> + Any, + F: Fn() -> Ft, + Ft: Future, +{ + let f_res = || async { Ok::(f().await) }; + // This can't fail, we just invented an error type for an infallible function + cache_fallible_res(name, f_res, force_run).await.unwrap() +} + +/// Same as `cache_res`, but takes a function that returns a `Result`, allowing you to use `?` and the like inside your logic. +pub async fn cache_fallible_res(name: &str, f: F, force_run: bool) -> Result +where + // By making this `Any`, we can downcast it to manage errors intuitively + D: Serialize + for<'de> Deserialize<'de>, + E: std::error::Error, + F: Fn() -> Ft, + Ft: Future>, +{ + // Replace any slashes with dashes to keep a flat directory structure + let name = name.replace("/", "-"); + // In production, we'll just run the function directly + if cfg!(debug_assertions) { + // Check if the cache file exists + let filename = format!("cache/{}.json", &name); + match File::open(&filename).await { + Ok(mut file) => { + if force_run { + let res = f().await?; + // Now cache the result + let str_res = serde_json::to_string(&res).unwrap_or_else(|err| { + panic!( + "couldn't serialize result of entry '{}' for caching: {}", + &filename, err + ) + }); + let mut file = File::create(&filename).await.unwrap_or_else(|err| { + panic!( + "couldn't create cache file for entry '{}': {}", + &filename, err + ) + }); + file.write_all(str_res.as_bytes()) + .await + .unwrap_or_else(|err| { + panic!( + "couldn't write cache to file for entry '{}': {}", + &filename, err + ) + }); + + Ok(res) + } else { + let mut contents = String::new(); + file.read_to_string(&mut contents) + .await + .unwrap_or_else(|err| { + panic!( + "couldn't read cache from file for entry '{}': {}", + &filename, err + ) + }); + let res: D = match serde_json::from_str(&contents) { + Ok(cached_res) => cached_res, + // If the stuff in the cache can't be deserialized, we'll force a recreation (we don't recurse because that requires boxing the future) + Err(_) => { + let res = f().await?; + // Now cache the result + let str_res = serde_json::to_string(&res).unwrap_or_else(|err| { + panic!( + "couldn't serialize result of entry '{}' for caching: {}", + &filename, err + ) + }); + let mut file = File::create(&filename).await.unwrap_or_else(|err| { + panic!( + "couldn't create cache file for entry '{}': {}", + &filename, err + ) + }); + file.write_all(str_res.as_bytes()) + .await + .unwrap_or_else(|err| { + panic!( + "couldn't write cache to file for entry '{}': {}", + &filename, err + ) + }); + + res + } + }; + + Ok(res) + } + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + // The file doesn't exist yet, create the parent cache directory + create_dir_all("cache") + .await + .unwrap_or_else(|err| panic!("couldn't create cache directory: {}", err)); + // We have no cache, so we'll have to run the function + let res = f().await?; + // Now cache the result + let str_res = serde_json::to_string(&res).unwrap_or_else(|err| { + panic!( + "couldn't serialize result of entry '{}' for caching: {}", + &filename, err + ) + }); + let mut file = File::create(&filename).await.unwrap_or_else(|err| { + panic!( + "couldn't create cache file for entry '{}': {}", + &filename, err + ) + }); + file.write_all(str_res.as_bytes()) + .await + .unwrap_or_else(|err| { + panic!( + "couldn't write cache to file for entry '{}': {}", + &filename, err + ) + }); + + Ok(res) + } + // Any other filesystem errors are unacceptable + Err(err) => panic!( + "filesystem error occurred while trying to read cache file for entry '{}': {}", + &filename, err + ), + } + } else { + f().await + } +} diff --git a/packages/perseus/src/lib.rs b/packages/perseus/src/lib.rs index 54e76c4b7c..a27e4617bc 100644 --- a/packages/perseus/src/lib.rs +++ b/packages/perseus/src/lib.rs @@ -42,6 +42,7 @@ pub mod plugins; pub mod stores; mod build; +mod cache_res; mod client_translations_manager; mod decode_time_str; mod default_headers; @@ -74,6 +75,7 @@ pub use sycamore::{generic_node::Html, DomNode, HydrateNode, SsrNode}; pub use sycamore_router::{navigate, Route}; // Items that should be available at the root (this should be nearly everything used in a typical Perseus app) +pub use crate::cache_res::{cache_fallible_res, cache_res}; pub use crate::error_pages::ErrorPages; pub use crate::errors::{ErrorCause, GenericErrorWithCause}; pub use crate::plugins::{Plugin, PluginAction, Plugins};