Skip to content

Commit

Permalink
Allow load_data to take a literal (#1794) (#1794)
Browse files Browse the repository at this point in the history 
* Add `literal` as a new entry for `data source`, to be used by the `load_data` function

* Add tests to the module for plain text, json, xml, toml, and csv

* Update error messaging to include literal as a potential choice

* Update site documentation to include instructions for using `load_data` with a literal
  • Loading branch information
@asimpletune
asimpletune authored Mar 21, 2022
1 parent 336a271 commit a13d41b
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 5 deletions.
112 changes: 110 additions & 2 deletions components/templates/src/global_fns/load_data.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use utils::fs::{get_file_time, read_file};
use crate::global_fns::helpers::search_for_file;

static GET_DATA_ARGUMENT_ERROR_MESSAGE: &str =
"`load_data`: requires EITHER a `path` or `url` argument";
"`load_data`: requires EITHER a `path`, `url`, or `literal` argument";

#[derive(Debug, PartialEq, Clone, Copy, Hash)]
enum Method {
Expand Down Expand Up @@ -82,6 +82,7 @@ impl OutputFormat {
enum DataSource {
Url(Url),
Path(PathBuf),
Literal(String),
}

impl DataSource {
Expand All @@ -93,11 +94,16 @@ impl DataSource {
fn from_args(
path_arg: Option<String>,
url_arg: Option<String>,
literal_arg: Option<String>,
base_path: &Path,
theme: &Option<String>,
output_path: &Path,
) -> Result<Option<Self>> {
if path_arg.is_some() && url_arg.is_some() {
// only one of `path`, `url`, or `literal` can be specified
if (path_arg.is_some() && url_arg.is_some())
|| (path_arg.is_some() && literal_arg.is_some())
|| (url_arg.is_some() && literal_arg.is_some())
{
return Err(GET_DATA_ARGUMENT_ERROR_MESSAGE.into());
}

Expand All @@ -117,6 +123,10 @@ impl DataSource {
.map_err(|e| format!("`load_data`: Failed to parse {} as url: {}", url, e).into());
}

if let Some(string_literal) = literal_arg {
return Ok(Some(DataSource::Literal(string_literal)));
}

Err(GET_DATA_ARGUMENT_ERROR_MESSAGE.into())
}

Expand Down Expand Up @@ -147,6 +157,8 @@ impl Hash for DataSource {
path.hash(state);
get_file_time(path).expect("get file time").hash(state);
}
// TODO: double check expectations here
DataSource::Literal(string_literal) => string_literal.hash(state),
};
}
}
Expand Down Expand Up @@ -223,6 +235,8 @@ impl TeraFn for LoadData {
// Either a local path or a URL
let path_arg = optional_arg!(String, args.get("path"), GET_DATA_ARGUMENT_ERROR_MESSAGE);
let url_arg = optional_arg!(String, args.get("url"), GET_DATA_ARGUMENT_ERROR_MESSAGE);
let literal_arg =
optional_arg!(String, args.get("literal"), GET_DATA_ARGUMENT_ERROR_MESSAGE);
// Optional general params
let format_arg = optional_arg!(
String,
Expand Down Expand Up @@ -267,6 +281,7 @@ impl TeraFn for LoadData {
DataSource::from_args(
path_arg.clone(),
url_arg,
literal_arg,
&self.base_path,
&self.theme,
&self.output_path,
Expand Down Expand Up @@ -364,6 +379,7 @@ impl TeraFn for LoadData {
}
}
}
DataSource::Literal(string_literal) => Ok(string_literal),
}?;

let result_value: Result<Value> = match file_format {
Expand Down Expand Up @@ -1217,4 +1233,96 @@ mod tests {

_mjson.assert();
}

#[test]
fn can_load_plain_literal() {
let static_fn = LoadData::new(PathBuf::from("../utils"), None, PathBuf::new());
let mut args = HashMap::new();
let plain_str = "abc 123";
args.insert("literal".to_string(), to_value(plain_str).unwrap());

let result = static_fn.call(&args.clone()).unwrap();

assert_eq!(result, plain_str);
}

#[test]
fn can_load_json_literal() {
let static_fn = LoadData::new(PathBuf::from("../utils"), None, PathBuf::new());
let mut args = HashMap::new();
let json_str = r#"{
"key": "value",
"array": [1, 2, 3],
"subpackage": {
"subkey": 5
}
}"#;
args.insert("literal".to_string(), to_value(json_str).unwrap());
args.insert("format".to_string(), to_value("json").unwrap());

let result = static_fn.call(&args.clone()).unwrap();

assert_eq!(
result,
json!({
"key": "value",
"array": [1, 2, 3],
"subpackage": {
"subkey": 5
}
})
);
}

#[test]
fn can_load_toml_literal() {
let static_fn = LoadData::new(PathBuf::from("../utils"), None, PathBuf::new());
let mut args = HashMap::new();
let toml_str = r#"
[category]
key = "value"
date = 1979-05-27T07:32:00Z
lt1 = 07:32:00
"#;
args.insert("literal".to_string(), to_value(toml_str).unwrap());
args.insert("format".to_string(), to_value("toml").unwrap());

let result = static_fn.call(&args.clone()).unwrap();

// TOML does not load in order
assert_eq!(
result,
json!({
"category": {
"date": "1979-05-27T07:32:00Z",
"lt1": "07:32:00",
"key": "value"
},
})
);
}

#[test]
fn can_load_csv_literal() {
let static_fn = LoadData::new(PathBuf::from("../utils"), None, PathBuf::new());
let mut args = HashMap::new();
let csv_str = r#"Number,Title
1,Gutenberg
2,Printing"#;
args.insert("literal".to_string(), to_value(csv_str).unwrap());
args.insert("format".to_string(), to_value("csv").unwrap());

let result = static_fn.call(&args.clone()).unwrap();

assert_eq!(
result,
json!({
"headers": ["Number", "Title"],
"records": [
["1", "Gutenberg"],
["2", "Printing"]
],
})
)
}
}
16 changes: 13 additions & 3 deletions docs/content/documentation/templates/overview.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,9 @@ The method returns a map containing `width`, `height` and `format` (the lowercas
```

### `load_data`
Loads data from a file or URL. Supported file types include *toml*, *json*, *csv*, *bibtex* and *xml* and only supports UTF-8 encoding.

Loads data from a file, URL, or string literal. Supported file types include *toml*, *json*, *csv*, *bibtex* and *xml* and only supports UTF-8 encoding.

Any other file type will be loaded as plain text.

The `path` argument specifies the path to a local data file, according to the [File Searching Logic](@/documentation/templates/overview.md#file-searching-logic).
Expand All @@ -273,6 +275,15 @@ Alternatively, the `url` argument specifies the location of a remote URL to load
{% set data = load_data(url="https://en.wikipedia.org/wiki/Commune_of_Paris") %}
```

Alternatively, the `literal` argument specifies an object literal. Note: if the `format` argument is not specified, then plain text will be what is assumed.

```jinja2
{% set data = load_data(literal='{"name": "bob"}', format="json") %}
{{ data["name"] }}
```

*Note: the `required` parameter has no effect when used in combination with the `literal` argument.*

The optional `required` boolean argument can be set to false so that missing data (HTTP error or local file not found) does not produce an error, but returns a null value instead. However, permission issues with a local file and invalid data that could not be parsed to the requested data format will still produce an error even with `required=false`.

The snippet below outputs the HTML from a Wikipedia page, or "No data found" if the page was not reachable, or did not return a successful HTTP code:
Expand All @@ -282,8 +293,7 @@ The snippet below outputs the HTML from a Wikipedia page, or "No data found" if
{% if data %}{{ data | safe }}{% else %}No data found{% endif %}
```

The optional `format` argument allows you to specify and override which data type is contained
within the specified file or URL. Valid entries are `toml`, `json`, `csv`, `bibtex`, `xml` or `plain`. If the `format` argument isn't specified, then the path extension is used.
The optional `format` argument allows you to specify and override which data type is contained within the specified file or URL. Valid entries are `toml`, `json`, `csv`, `bibtex`, `xml` or `plain`. If the `format` argument isn't specified, then the path extension is used. In the case of a literal, `plain` is assumed if `format` is unspecified.


```jinja2
Expand Down

0 comments on commit a13d41b

Please sign in to comment.