implement preprocessors

src/main.rs

@@ -5,6 +5,7 @@ mod create;
 mod files;
 mod lock;
 mod parse;
+mod preprocess;
 mod print;
 mod util;
 
@@ -351,6 +352,8 @@ fn process_inputs_forward(
         .into());
     }
 
+    let documents = preprocess::pre_process(config, documents)?;
+
     for (path, doc) in documents.iter() {
         compile::compile(config, &doc, &path, &mut track_code_files)?;
     }

src/preprocess.rs

@@ -0,0 +1,53 @@
+use crate::util::Fallible;
+use std::collections::HashMap;
+use std::error::Error;
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::{Command, Stdio};
+use yarner_lib::config::Config;
+use yarner_lib::document::Document;
+
+pub fn pre_process(
+    config: &Config,
+    documents: HashMap<PathBuf, Document>,
+) -> Fallible<HashMap<PathBuf, Document>> {
+    let mut docs = documents;
+    for (name, proc) in &config.preprocessor {
+        let json = yarner_lib::to_json(proc, &docs)?;
+        let mut child = Command::new(format!("yarner-{}", name))
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .spawn()
+            .map_err(|err| format_error(err.into(), name))?;
+
+        {
+            let stdin = child
+                .stdin
+                .as_mut()
+                .ok_or("Unable to access child process stdin.")
+                .map_err(|err| format_error(err.into(), name))?;
+            stdin
+                .write_all(json.as_bytes())
+                .map_err(|err| format_error(err.into(), name))?;
+        }
+
+        let output = child
+            .wait_with_output()
+            .map_err(|err| format_error(err.into(), name))?;
+        let out_json =
+            String::from_utf8(output.stdout).map_err(|err| format_error(err.into(), name))?;
+
+        let (_, new_docs) =
+            yarner_lib::from_json(&out_json).map_err(|err| format_error(err.into(), name))?;
+        docs = new_docs;
+    }
+    Ok(docs)
+}
+
+fn format_error(err: Box<dyn Error>, name: &str) -> String {
+    format!(
+        "Failed to run command 'yarner-{}': {}",
+        name,
+        err.to_string()
+    )
+}

yarner-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "yarner-lib"
-version = "0.1.0"
+version = "0.4.1"
 authors = ["Martin Lange <martin_lange_@gmx.net>"]
 edition = "2018"
 
@@ -9,4 +9,4 @@ once_cell = "1.5"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 regex = "1.4"
-toml = "0.5"
+toml = { version = "0.5", features = ["preserve_order"] }

yarner-lib/src/config.rs

@@ -6,6 +6,7 @@ use once_cell::sync::Lazy;
 use regex::Regex;
 use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer};
 use std::error::Error;
+use toml::value::Table;
 
 pub const LINK_PATTERN: &str = r"\[([^\[\]]*)\]\((.*?)\)";
 pub static LINK_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(LINK_PATTERN).unwrap());
@@ -24,6 +25,8 @@ pub struct Config {
     /// Programming language specific settings
     #[serde(default)]
     pub language: HashMap<String, LanguageSettings>,
+    #[serde(default)]
+    pub preprocessor: Table,
 }
 
 impl Config {
@@ -278,6 +281,7 @@ pub fn default_config() -> Config {
             entrypoint: None,
         },
         language: HashMap::new(),
+        preprocessor: Table::new(),
     }
 }
 

yarner-lib/src/document.rs

@@ -127,7 +127,7 @@ impl Document {
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct TextBlock {
     /// The source text
-    text: Vec<String>,
+    pub text: Vec<String>,
 }
 
 impl TextBlock {
@@ -327,7 +327,8 @@ mod tests {
                 doc_paths: None,
                 entrypoint: None,
             },
-            language: HashMap::new(),
+            language: Default::default(),
+            preprocessor: Default::default(),
         }
     }
 }

yarner-lib/src/lib.rs

@@ -1,12 +1,23 @@
-use crate::config::Config;
 use crate::document::Document;
 use std::collections::HashMap;
+use std::io::Read;
 use std::path::PathBuf;
+use toml::Value;
 
 pub mod config;
 pub mod document;
 
-#[allow(dead_code)]
-fn to_json(config: &Config, documents: &HashMap<PathBuf, Document>) -> serde_json::Result<String> {
+pub fn to_json(
+    config: &Value,
+    documents: &HashMap<PathBuf, Document>,
+) -> serde_json::Result<String> {
     serde_json::to_string_pretty(&(config, documents))
 }
+
+pub fn from_json(json: &str) -> serde_json::Result<(Value, HashMap<PathBuf, Document>)> {
+    serde_json::from_str(json)
+}
+
+pub fn parse_input<R: Read>(reader: R) -> serde_json::Result<(Value, HashMap<PathBuf, Document>)> {
+    serde_json::from_reader(reader)
+}