Merge pull request #464 from filecoin-project/features/parampublish-456

parampublish/paramfetch - cli tools to help with param distribution

filecoin-proofs/Cargo.toml

@@ -32,8 +32,10 @@ itertools = "0.7.3"
 serde_cbor = "0.9.0"
 serde = { version = "1", features = ["rc"] }
 serde_derive = "1.0"
+serde_json = "1.0"
 blake2 = "0.8"
 slog = { version = "2.4.1", features = ["max_level_trace", "release_max_level_trace"] }
+regex = "1"
 
 [dev-dependencies]
 gperftools = { git = "https://github.com/dignifiedquire/rust-gperftools" }

filecoin-proofs/src/bin/paramfetch.rs

@@ -0,0 +1,69 @@
+use clap::{App, AppSettings, Arg, SubCommand};
+
+use filecoin_proofs::param::*;
+use storage_proofs::parameter_cache::PARAMETER_CACHE_DIR;
+
+pub fn main() {
+    let matches = App::new("paramfetch")
+        .setting(AppSettings::ArgRequiredElseHelp)
+        .version("1.0")
+        .about("")
+        .about(
+            &format!(
+                "
+Set $FILECOIN_PARAMETER_CACHE to specify parameter directory. Defaults to '{}'
+",
+                PARAMETER_CACHE_DIR
+            )[..],
+        )
+        .subcommand(
+            SubCommand::with_name("fetch")
+                .arg(
+                    Arg::with_name("all")
+                        .short("a")
+                        .long("all")
+                        .help("Download all available parameters"),
+                )
+                .about("Download parameters through IPFS"),
+        )
+        .subcommand(
+            SubCommand::with_name("check").about("Check which mapped parameters have been fetched"),
+        )
+        .get_matches();
+
+    if let Some(matches) = matches.subcommand_matches("fetch") {
+        let parameters = if matches.is_present("all") {
+            get_mapped_parameters()
+        } else {
+            choose_mapped_parameters()
+        }
+        .expect(ERROR_PARAMETERS_MAPPED);
+
+        if parameters.len() > 0 {
+            println!("fetching parameters:");
+
+            parameters.iter().for_each(|p| {
+                println!("{}...", p);
+
+                match fetch_parameter_file(p.to_string()) {
+                    Ok(_) => println!("ok"),
+                    Err(_) => println!("error"),
+                }
+            });
+        } else {
+            println!("nothing to fetch");
+        }
+    }
+
+    if let Some(_) = matches.subcommand_matches("check") {
+        let mapped_parameters = get_mapped_parameters().expect(ERROR_PARAMETERS_MAPPED);
+        let local_parameters = get_local_parameters().expect(ERROR_PARAMETERS_LOCAL);
+
+        mapped_parameters.iter().for_each(|p| {
+            let local = local_parameters.contains(p);
+            let check = if local { "☑" } else { "☐" };
+
+            println!("{} {}", check, p);
+        });
+    }
+}

filecoin-proofs/src/bin/parampublish.rs

@@ -0,0 +1,76 @@
+use clap::{App, AppSettings, Arg, SubCommand};
+use std::collections::HashMap;
+
+use filecoin_proofs::param::*;
+use storage_proofs::parameter_cache::PARAMETER_CACHE_DIR;
+
+pub fn main() {
+    let matches = App::new("parampublish")
+        .setting(AppSettings::ArgRequiredElseHelp)
+        .version("1.0")
+        .about(
+            &format!(
+                "
+Set $FILECOIN_PARAMETER_CACHE to specify parameter directory. Defaults to '{}'
+",
+                PARAMETER_CACHE_DIR
+            )[..],
+        )
+        .subcommand(
+            SubCommand::with_name("publish")
+                .arg(
+                    Arg::with_name("all")
+                        .short("a")
+                        .long("all")
+                        .help("Publish all local parameters"),
+                )
+                .about("Publish local parameters through IPFS"),
+        )
+        .subcommand(
+            SubCommand::with_name("check")
+                .about("Check which local parameters have been published"),
+        )
+        .get_matches();
+
+    if let Some(matches) = matches.subcommand_matches("publish") {
+        let mut map = HashMap::new();
+        let parameters = if matches.is_present("all") {
+            get_local_parameters()
+        } else {
+            choose_local_parameters()
+        }
+        .expect(ERROR_PARAMETERS_LOCAL);
+
+        if parameters.len() > 0 {
+            println!("publishing parameters:");
+
+            parameters.iter().for_each(|p| {
+                print!("{}... ", p);
+
+                match publish_parameter_file(p.to_string()) {
+                    Ok(cid) => {
+                        map.insert(p.to_string(), cid);
+                        println!("ok");
+                    }
+                    Err(_) => println!("error"),
+                }
+            });
+
+            save_parameter_map(map).expect(ERROR_PARAMETER_MAP_SAVE);
+        } else {
+            println!("nothing to publish");
+        }
+    }
+
+    if let Some(_) = matches.subcommand_matches("check") {
+        let mapped_parameters = get_mapped_parameters().expect(ERROR_PARAMETERS_MAPPED);
+        let local_parameters = get_local_parameters().expect(ERROR_PARAMETERS_LOCAL);
+
+        local_parameters.iter().for_each(|p| {
+            let mapped = mapped_parameters.contains(p);
+            let check = if mapped { "☑" } else { "☐" };
+
+            println!("{} {}", check, p);
+        });
+    }
+}

filecoin-proofs/src/lib.rs

@@ -28,6 +28,7 @@ extern crate slog;
 
 pub mod api;
 pub mod error;
+pub mod param;
 pub mod serde_big_array;
 
 use logging_toolkit::make_logger;

filecoin-proofs/src/param.rs

@@ -0,0 +1,167 @@
+use failure::{err_msg, Error};
+use regex::Regex;
+use std::collections::HashMap;
+use std::fs::{read_dir, File};
+use std::io::{stdin, stdout, BufReader, BufWriter, Write};
+use std::path::PathBuf;
+use std::process::Command;
+use storage_proofs::parameter_cache::parameter_cache_dir;
+
+const PARAMETER_JSON_PATH: &str = "./parameters.json";
+
+pub const ERROR_CURL_COMMAND: &str = "failed to run curl";
+pub const ERROR_CURL_FETCH: &str = "failed to fetch via curl";
+pub const ERROR_IPFS_COMMAND: &str = "failed to run ipfs";
+pub const ERROR_IPFS_OUTPUT: &str = "failed to capture ipfs output";
+pub const ERROR_IPFS_PARSE: &str = "failed to parse ipfs output";
+pub const ERROR_IPFS_PUBLISH: &str = "failed to publish via ipfs";
+pub const ERROR_PARAMETERS_LOCAL: &str = "failed to load local parameters";
+pub const ERROR_PARAMETERS_MAPPED: &str = "failed to load mapped parameters";
+pub const ERROR_PARAMETER_ID: &str = "failed to find parameter in map";
+pub const ERROR_PARAMETER_MAP_LOAD: &str = "failed to load parameter map";
+pub const ERROR_PARAMETER_MAP_SAVE: &str = "failed to save parameter map";
+pub const ERROR_STRING: &str = "invalid string";
+
+pub type Result<T> = ::std::result::Result<T, Error>;
+
+pub type ParameterMap = HashMap<String, String>;
+
+pub fn get_local_parameters() -> Result<Vec<String>> {
+    let path = parameter_cache_dir();
+
+    if path.exists() {
+        Ok(read_dir(path)?
+            .map(|f| f.unwrap().path())
+            .filter(|p| p.is_file())
+            .map(|p| {
+                p.as_path()
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .to_string()
+            })
+            .collect())
+    } else {
+        println!(
+            "parameter directory '{}' does not exist",
+            path.as_path().to_str().unwrap()
+        );
+
+        Ok(Vec::new())
+    }
+}
+
+pub fn get_mapped_parameters() -> Result<Vec<String>> {
+    Ok(load_parameter_map()?.into_iter().map(|(k, _)| k).collect())
+}
+
+pub fn load_parameter_map() -> Result<ParameterMap> {
+    let path = PathBuf::from(PARAMETER_JSON_PATH);
+
+    if path.exists() {
+        let file = File::open(path)?;
+        let reader = BufReader::new(file);
+        let map = serde_json::from_reader(reader)?;
+
+        Ok(map)
+    } else {
+        println!(
+            "parameter manifest '{}' does not exist",
+            path.as_path().to_str().unwrap()
+        );
+
+        Ok(HashMap::new())
+    }
+}
+
+pub fn save_parameter_map(map: ParameterMap) -> Result<()> {
+    let file = File::create(PARAMETER_JSON_PATH)?;
+    let writer = BufWriter::new(file);
+    serde_json::to_writer_pretty(writer, &map)?;
+
+    Ok(())
+}
+
+pub fn publish_parameter_file(parameter: String) -> Result<String> {
+    let mut path = parameter_cache_dir();
+    path.push(parameter);
+
+    let output = Command::new("ipfs")
+        .arg("add")
+        .arg(path.as_path().to_str().unwrap().to_string())
+        .output()
+        .expect(ERROR_IPFS_COMMAND);
+
+    if !output.status.success() {
+        Err(err_msg(ERROR_IPFS_PUBLISH))
+    } else {
+        let pattern = Regex::new("added ([^ ]+) ")?;
+        let string = String::from_utf8(output.stdout)?;
+        let captures = pattern.captures(string.as_str()).expect(ERROR_IPFS_OUTPUT);
+        let cid = captures.get(1).expect(ERROR_IPFS_PARSE);
+
+        Ok(cid.as_str().to_string())
+    }
+}
+
+pub fn fetch_parameter_file(parameter: String) -> Result<()> {
+    let map = load_parameter_map()?;
+    let cid = map.get(&parameter).expect(ERROR_PARAMETER_ID);
+
+    let mut path = parameter_cache_dir();
+    path.push(parameter);
+
+    if path.exists() {
+        println!(
+            "parameter file '{}' already exists",
+            path.as_path().to_str().unwrap()
+        );
+
+        Ok(())
+    } else {
+        let output = Command::new("curl")
+            .arg("-o")
+            .arg(path.as_path().to_str().unwrap().to_string())
+            .arg(format!("https://ipfs.io/ipfs/{}", cid))
+            .output()
+            .expect(ERROR_CURL_COMMAND);
+
+        if !output.status.success() {
+            Err(err_msg(ERROR_CURL_FETCH))
+        } else {
+            Ok(())
+        }
+    }
+}
+
+pub fn choose(message: String) -> bool {
+    loop {
+        print!("{} [y/n]: ", message);
+
+        let _ = stdout().flush();
+        let mut s = String::new();
+        stdin().read_line(&mut s).expect(ERROR_STRING);
+
+        match s.trim().to_uppercase().as_str() {
+            "Y" => return true,
+            "N" => return false,
+            _ => {}
+        }
+    }
+}
+
+pub fn choose_from(vector: Vec<String>) -> Result<Vec<String>> {
+    Ok(vector
+        .into_iter()
+        .filter(|i| choose(i.to_string()))
+        .collect())
+}
+
+pub fn choose_local_parameters() -> Result<Vec<String>> {
+    choose_from(get_local_parameters()?)
+}
+
+pub fn choose_mapped_parameters() -> Result<Vec<String>> {
+    choose_from(get_mapped_parameters()?)
+}

parameters.json

@@ -0,0 +1,4 @@
+{
+  "v9-zigzag-proof-of-replication-f8b6b5b4f1015da3984944b4aef229b63ce950f65c7f41055a995718a452204d": "QmY8UR7sPGgfeoVB381mrvcdVK4QF28qV5cdoyZFfPuUfM",
+  "v9-zigzag-proof-of-replication-52431242c129794fe51d373ae29953f2ff52abd94c78756e318ce45f3e4946d8": "QmP7yBN63YRmnQt7p7ZTQWgahwniNgSoJL5KD1b8eKmGWr"
+}

scripts/publish-release.sh

@@ -24,8 +24,11 @@ mkdir $RELEASE_PATH
 mkdir $RELEASE_PATH/bin
 mkdir $RELEASE_PATH/include
 mkdir -p $RELEASE_PATH/lib/pkgconfig
+mkdir $RELEASE_PATH/misc
 
+cp parameters.json $RELEASE_PATH/misc/
 cp target/release/paramcache $RELEASE_PATH/bin/
+cp target/release/paramfetch $RELEASE_PATH/bin/
 cp target/release/libfilecoin_proofs.h $RELEASE_PATH/include/
 cp target/release/libfilecoin_proofs.a $RELEASE_PATH/lib/
 cp target/release/libfilecoin_proofs.pc $RELEASE_PATH/lib/pkgconfig

storage-proofs/src/parameter_cache.rs

@@ -23,7 +23,7 @@ pub const PARAMETER_CACHE_DIR: &str = "/tmp/filecoin-proof-parameters/";
 /// If this changes, parameters generated under different conditions may vary. Don't change it.
 pub const PARAMETER_RNG_SEED: [u32; 4] = [0x3dbe6259, 0x8d313d76, 0x3237db17, 0xe5bc0654];
 
-fn parameter_cache_dir_name() -> String {
+pub fn parameter_cache_dir_name() -> String {
     match env::var("FILECOIN_PARAMETER_CACHE") {
         Ok(dir) => dir,
         Err(_) => String::from(PARAMETER_CACHE_DIR),