Calculate challenges per partition, ensuring total minimum is met.

filecoin-proofs/src/api/internal.rs

@@ -47,8 +47,6 @@ pub type ChallengeSeed = Fr32Ary;
 /// FrSafe is an array of the largest whole number of bytes guaranteed not to overflow the field.
 type FrSafe = [u8; 31];
 
-const POREP_MINIMUM_CHALLENGES: usize = 1; // FIXME: 8,000
-
 lazy_static! {
     pub static ref ENGINE_PARAMS: JubjubBls12 = JubjubBls12::new();
 }
@@ -122,7 +120,10 @@ where
 ////////////////////////////////////////////////////////////////////////////////
 
 fn get_zigzag_params(porep_config: PoRepConfig) -> error::Result<Arc<groth16::Parameters<Bls12>>> {
-    let public_params = public_params(PaddedBytesAmount::from(porep_config));
+    let public_params = public_params(
+        PaddedBytesAmount::from(porep_config),
+        usize::from(PoRepProofPartitions::from(porep_config)),
+    );
 
     let get_params =
         || ZigZagCompound::groth_params(&public_params, &ENGINE_PARAMS).map_err(Into::into);
@@ -158,7 +159,10 @@ fn get_post_params(post_config: PoStConfig) -> error::Result<Arc<groth16::Parame
 }
 
 fn get_zigzag_verifying_key(porep_config: PoRepConfig) -> error::Result<Arc<Bls12VerifyingKey>> {
-    let public_params = public_params(PaddedBytesAmount::from(porep_config));
+    let public_params = public_params(
+        PaddedBytesAmount::from(porep_config),
+        usize::from(PoRepProofPartitions::from(porep_config)),
+    );
 
     let get_verifying_key =
         || ZigZagCompound::verifying_key(&public_params, &ENGINE_PARAMS).map_err(Into::into);
@@ -199,17 +203,50 @@ const SLOTH_ITER: usize = 0;
 const LAYERS: usize = 4; // TODO: 10;
 const TAPER_LAYERS: usize = 2; // TODO: 7
 const TAPER: f64 = 1.0 / 3.0;
-const CHALLENGE_COUNT: usize = 2;
+const POREP_MINIMUM_CHALLENGES: usize = 12; // FIXME: 8,000
+
 const DRG_SEED: [u32; 7] = [1, 2, 3, 4, 5, 6, 7]; // Arbitrary, need a theory for how to vary this over time.
 
-lazy_static! {
-    static ref CHALLENGES: LayerChallenges =
-        LayerChallenges::new_tapered(LAYERS, CHALLENGE_COUNT, TAPER_LAYERS, TAPER);
+fn select_challenges(
+    partitions: usize,
+    minimum_total_challenges: usize,
+    layers: usize,
+    taper_layers: usize,
+    taper: f64,
+) -> LayerChallenges {
+    let mut count = 1;
+    let mut guess = LayerChallenges::Tapered {
+        count,
+        layers,
+        taper,
+        taper_layers,
+    };
+    while partitions * guess.total_challenges() < minimum_total_challenges {
+        count += 1;
+        guess = LayerChallenges::Tapered {
+            count,
+            layers,
+            taper,
+            taper_layers,
+        };
+    }
+    guess
 }
 
-fn setup_params(sector_bytes: PaddedBytesAmount) -> layered_drgporep::SetupParams {
+fn setup_params(
+    sector_bytes: PaddedBytesAmount,
+    partitions: usize,
+) -> layered_drgporep::SetupParams {
     let sector_bytes = usize::from(sector_bytes);
 
+    let challenges = select_challenges(
+        partitions,
+        POREP_MINIMUM_CHALLENGES,
+        LAYERS,
+        TAPER_LAYERS,
+        TAPER,
+    );
+
     assert!(
         sector_bytes % 32 == 0,
         "sector_bytes ({}) must be a multiple of 32",
@@ -224,14 +261,15 @@ fn setup_params(sector_bytes: PaddedBytesAmount) -> layered_drgporep::SetupParam
             seed: DRG_SEED,
         },
         sloth_iter: SLOTH_ITER,
-        layer_challenges: CHALLENGES.clone(),
+        layer_challenges: challenges,
     }
 }
 
 pub fn public_params(
     sector_bytes: PaddedBytesAmount,
+    partitions: usize,
 ) -> layered_drgporep::PublicParams<DefaultTreeHasher, ZigZagBucketGraph<DefaultTreeHasher>> {
-    ZigZagDrgPoRep::<DefaultTreeHasher>::setup(&setup_params(sector_bytes)).unwrap()
+    ZigZagDrgPoRep::<DefaultTreeHasher>::setup(&setup_params(sector_bytes, partitions)).unwrap()
 }
 
 type PostSetupParams = vdf_post::SetupParams<PedersenDomain, vdf_sloth::Sloth>;
@@ -447,7 +485,7 @@ fn make_merkle_tree<T: Into<PathBuf> + AsRef<Path>>(
     let mut data = Vec::new();
     f_in.read_to_end(&mut data)?;
 
-    public_params(bytes).graph.merkle_tree(&data)
+    public_params(bytes, 1).graph.merkle_tree(&data)
 }
 
 #[derive(Clone, Debug)]
@@ -508,7 +546,10 @@ pub fn seal<T: Into<PathBuf> + AsRef<Path>>(
     let replica_id = replica_id::<DefaultTreeHasher>(prover_id, sector_id);
 
     let compound_setup_params = compound_proof::SetupParams {
-        vanilla_params: &setup_params(PaddedBytesAmount::from(porep_config)),
+        vanilla_params: &setup_params(
+            PaddedBytesAmount::from(porep_config),
+            usize::from(PoRepProofPartitions::from(porep_config)),
+        ),
         engine_params: &(*ENGINE_PARAMS),
         partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))),
     };
@@ -604,7 +645,10 @@ pub fn get_unsealed_range<T: Into<PathBuf> + AsRef<Path>>(
     let mut buf_writer = BufWriter::new(f_out);
 
     let unsealed = ZigZagDrgPoRep::extract_all(
-        &public_params(PaddedBytesAmount::from(porep_config)),
+        &public_params(
+            PaddedBytesAmount::from(porep_config),
+            usize::from(PoRepProofPartitions::from(porep_config)),
+        ),
         &replica_id,
         &data,
     )?;
@@ -638,7 +682,10 @@ pub fn verify_seal(
     let comm_r_star = bytes_into_fr::<Bls12>(&comm_r_star)?;
 
     let compound_setup_params = compound_proof::SetupParams {
-        vanilla_params: &setup_params(PaddedBytesAmount::from(porep_config)),
+        vanilla_params: &setup_params(
+            PaddedBytesAmount::from(porep_config),
+            usize::from(PoRepProofPartitions::from(porep_config)),
+        ),
         engine_params: &(*ENGINE_PARAMS),
         partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))),
     };
@@ -1142,4 +1189,24 @@ mod tests {
     fn post_verify_test() {
         post_verify_aux(SectorClass::Test, BytesAmount::Max);
     }
+
+    #[test]
+    fn partition_layer_challenges_test() {
+        let f = |partitions| {
+            select_challenges(
+                partitions,
+                POREP_MINIMUM_CHALLENGES,
+                LAYERS,
+                TAPER_LAYERS,
+                TAPER,
+            )
+            .all_challenges()
+        };
+        // Update to ensure all supported PoRepProofPartitions options are represented here.
+        assert_eq!(vec![1, 1, 2, 2], f(usize::from(PoRepProofPartitions::Two)));
+
+        assert_eq!(vec![3, 3, 4, 5], f(1));
+        assert_eq!(vec![1, 1, 2, 2], f(2));
+        assert_eq!(vec![1, 1, 1, 1], f(4));
+    }
 }

filecoin-proofs/src/bin/paramcache.rs

@@ -7,6 +7,8 @@ use filecoin_proofs::api::internal;
 use filecoin_proofs::FCP_LOG;
 use pairing::bls12_381::Bls12;
 use sector_base::api::bytes_amount::PaddedBytesAmount;
+use sector_base::api::porep_config::POREP_PROOF_PARTITION_CHOICES;
+use sector_base::api::sector_size::SECTOR_SIZE_CHOICES;
 use storage_proofs::circuit::vdf_post::{VDFPoStCircuit, VDFPostCompound};
 use storage_proofs::circuit::zigzag::ZigZagCompound;
 use storage_proofs::compound_proof::CompoundProof;
@@ -27,7 +29,10 @@ fn cache_porep_params(porep_config: PoRepConfig) {
     let n = u64::from(PaddedBytesAmount::from(porep_config));
     info!(FCP_LOG, "begin PoRep parameter-cache check/populate routine for {}-byte sectors", n; "target" => "paramcache");
 
-    let public_params = internal::public_params(PaddedBytesAmount::from(porep_config));
+    let public_params = internal::public_params(
+        PaddedBytesAmount::from(porep_config),
+        usize::from(PoRepProofPartitions::from(porep_config)),
+    );
     {
         let circuit = ZigZagCompound::blank_circuit(&public_params, &internal::ENGINE_PARAMS);
 
@@ -84,10 +89,11 @@ pub fn main() {
     cache_post_params(PoStConfig::Test);
 
     if !test_only {
-        cache_porep_params(PoRepConfig::Live(
-            SectorSize::TwoHundredFiftySixMiB,
-            PoRepProofPartitions::Two,
-        ));
+        for p in &POREP_PROOF_PARTITION_CHOICES {
+            for s in &SECTOR_SIZE_CHOICES {
+                cache_porep_params(PoRepConfig::Live(*s, *p));
+            }
+        }
         cache_post_params(PoStConfig::Live(
             SectorSize::TwoHundredFiftySixMiB,
             PoStProofPartitions::One,

filecoin-proofs/src/bin/paramgen.rs

@@ -10,7 +10,7 @@ use std::fs::File;
 
 use filecoin_proofs::api::internal;
 use sector_base::api::bytes_amount::PaddedBytesAmount;
-use sector_base::api::porep_config::PoRepConfig;
+use sector_base::api::porep_config::{PoRepConfig, PoRepProofPartitions};
 use storage_proofs::circuit::zigzag::ZigZagCompound;
 use storage_proofs::compound_proof::CompoundProof;
 
@@ -19,7 +19,10 @@ pub fn main() {
     let args: Vec<String> = env::args().collect();
     let out_file = &args[1];
 
-    let public_params = internal::public_params(PaddedBytesAmount::from(PoRepConfig::Test));
+    let public_params = internal::public_params(
+        PaddedBytesAmount::from(PoRepConfig::Test),
+        usize::from(PoRepProofPartitions::from(PoRepConfig::Test)),
+    );
 
     let circuit = ZigZagCompound::blank_circuit(&public_params, &internal::ENGINE_PARAMS);
     let mut params = phase2::MPCParameters::new(circuit).unwrap();

filecoin-proofs/src/bin/parampublish.rs

@@ -1,6 +1,6 @@
 use clap::{App, Arg, ArgMatches};
 use filecoin_proofs::param::*;
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::io;
 use std::io::prelude::*;
 use std::path::PathBuf;
@@ -53,7 +53,7 @@ fn publish(matches: &ArgMatches) -> Result<()> {
     };
 
     let json = PathBuf::from(matches.value_of("json").unwrap_or("./parameters.json"));
-    let mut parameter_map: ParameterMap = HashMap::new();
+    let mut parameter_map: ParameterMap = BTreeMap::new();
 
     if !parameter_ids.is_empty() {
         println!("publishing {} parameters...", parameter_ids.len());

filecoin-proofs/src/param.rs

@@ -2,7 +2,7 @@ use blake2b_simd::State as Blake2b;
 use failure::{err_msg, Error};
 use regex::Regex;
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::fs::{create_dir_all, read_dir, rename, File};
 use std::io::{stdin, stdout, BufReader, BufWriter, Write};
 use std::path::PathBuf;
@@ -19,7 +19,7 @@ const ERROR_PARAMETER_ID: &str = "failed to find parameter in map";
 const ERROR_STRING: &str = "invalid string";
 
 pub type Result<T> = ::std::result::Result<T, Error>;
-pub type ParameterMap = HashMap<String, ParameterData>;
+pub type ParameterMap = BTreeMap<String, ParameterData>;
 
 #[derive(Deserialize, Serialize)]
 pub struct ParameterData {

parameters.json

@@ -1,34 +1,34 @@
 {
-  "v10-vdf-post-c8ddc8e74f9c2ed6a4bc83df171a9e9e84e242cf6e48e20f71585106fa8a3ce6.vk": {
-    "cid": "QmTKL1MiWSWFUMEECZPaBX6ddoZy9HCAA4mdaW65BKDoEp",
-    "digest": "2684b87a68d3a02118584e524a3f073a"
-  },
-  "v10-zigzag-proof-of-replication-6648cd509799c8b76f944bbc971692c69d1480fa60c569bfc1284f0bd8ead988": {
-    "cid": "QmeAwbo2CM4t58kV7byr1X2beA7pzbGvStCVSDSG4ydu2E",
-    "digest": "a30003a7b2b2512cd5e66cfefdd9fe7b"
-  },
   "v10-vdf-post-912e5a392c09cbfa0bf369ac77f939a45e0f2aec968cf3846cbe356a19b76685": {
     "cid": "QmP1PkagfiC9cUEh3r7NEWaJfnBGEnqfABxXxh3PSoiMtX",
     "digest": "ff7fa52e9bf2a3b5261f6d5740156f79"
   },
+  "v10-vdf-post-912e5a392c09cbfa0bf369ac77f939a45e0f2aec968cf3846cbe356a19b76685.vk": {
+    "cid": "QmaNXg87KFw4pKxa9mXThprpDns7aJZCN5k9WCtneMBkoY",
+    "digest": "3e97aff1d9d2d712163befee897c8b5d"
+  },
   "v10-vdf-post-c8ddc8e74f9c2ed6a4bc83df171a9e9e84e242cf6e48e20f71585106fa8a3ce6": {
     "cid": "QmPuZ1fZGVDBTjsARTWuMesKjaXqUeNENsQrWBLh7PAjEU",
     "digest": "71f6673c73376cca102390150174bbe4"
   },
+  "v10-vdf-post-c8ddc8e74f9c2ed6a4bc83df171a9e9e84e242cf6e48e20f71585106fa8a3ce6.vk": {
+    "cid": "QmTKL1MiWSWFUMEECZPaBX6ddoZy9HCAA4mdaW65BKDoEp",
+    "digest": "2684b87a68d3a02118584e524a3f073a"
+  },
   "v10-zigzag-proof-of-replication-56360c4554216174de15d2c76acbd0f360e0fabf07dd470ca50f7d39676e7874": {
     "cid": "QmbPHbry667qc7JEhYDWpvs6mj6rD9UQRi9ZpQb6q6pcoq",
     "digest": "9268d86f4e3c23eb8483d65834e27c0d"
   },
-  "v10-zigzag-proof-of-replication-6648cd509799c8b76f944bbc971692c69d1480fa60c569bfc1284f0bd8ead988.vk": {
-    "cid": "QmeubWLDsLvL5ULFote1Gce1RfNKcHicCkcwsNyBkhtWuv",
-    "digest": "61eb4d1b2862feb1149284108c29386b"
-  },
   "v10-zigzag-proof-of-replication-56360c4554216174de15d2c76acbd0f360e0fabf07dd470ca50f7d39676e7874.vk": {
     "cid": "QmRkiXN7RTWjTgTz3DTjCjDaVNS5vNd8z174UrZLyMowmS",
     "digest": "d9f8f2c3a74c45cce4dfa26b65ccc3c9"
   },
-  "v10-vdf-post-912e5a392c09cbfa0bf369ac77f939a45e0f2aec968cf3846cbe356a19b76685.vk": {
-    "cid": "QmaNXg87KFw4pKxa9mXThprpDns7aJZCN5k9WCtneMBkoY",
-    "digest": "3e97aff1d9d2d712163befee897c8b5d"
+  "v10-zigzag-proof-of-replication-6648cd509799c8b76f944bbc971692c69d1480fa60c569bfc1284f0bd8ead988": {
+    "cid": "QmeAwbo2CM4t58kV7byr1X2beA7pzbGvStCVSDSG4ydu2E",
+    "digest": "a30003a7b2b2512cd5e66cfefdd9fe7b"
+  },
+  "v10-zigzag-proof-of-replication-6648cd509799c8b76f944bbc971692c69d1480fa60c569bfc1284f0bd8ead988.vk": {
+    "cid": "QmeubWLDsLvL5ULFote1Gce1RfNKcHicCkcwsNyBkhtWuv",
+    "digest": "61eb4d1b2862feb1149284108c29386b"
   }
 }

sector-base/src/api/porep_config.rs

@@ -9,11 +9,14 @@ pub enum PoRepConfig {
     Test,
 }
 
+// When modifying, update internal::tests::partition_layer_challenges_test to reflect supported PoRepProofPartitions.
 #[derive(Clone, Copy, Debug)]
 pub enum PoRepProofPartitions {
     Two,
 }
 
+pub const POREP_PROOF_PARTITION_CHOICES: [PoRepProofPartitions; 1] = [PoRepProofPartitions::Two];
+
 impl Default for PoRepConfig {
     fn default() -> Self {
         PoRepConfig::Live(SectorSize::TwoHundredFiftySixMiB, PoRepProofPartitions::Two)

sector-base/src/api/sector_size.rs

@@ -10,6 +10,9 @@ pub enum SectorSize {
     TwoHundredFiftySixMiB,
 }
 
+pub const SECTOR_SIZE_CHOICES: [SectorSize; 2] =
+    [SectorSize::OneKiB, SectorSize::TwoHundredFiftySixMiB];
+
 impl From<SectorSize> for UnpaddedBytesAmount {
     fn from(x: SectorSize) -> Self {
         match x {

storage-proofs/src/settings.rs

@@ -21,7 +21,7 @@ impl Default for Settings {
         Settings {
             maximize_caching: false,
             merkle_tree_path: "/tmp/merkle-trees".into(),
-            num_proving_threads: 4,
+            num_proving_threads: 1,
         }
     }
 }