diff --git a/drop-struct-macro-derive/README.md b/drop-struct-macro-derive/README.md index a87135166..8da912d35 100644 --- a/drop-struct-macro-derive/README.md +++ b/drop-struct-macro-derive/README.md @@ -50,9 +50,9 @@ $ cargo expand --lib api::responses Finished dev [unoptimized + debuginfo] target(s) in 0.70s pub mod responses { - use crate::api::sector_builder::errors::SectorBuilderErr; - use crate::api::sector_builder::SectorBuilder; - use crate::api::API_POREP_PROOF_BYTES; + use crate::sector_builder::errors::SectorBuilderErr; + use crate::sector_builder::SectorBuilder; + use crate::API_POREP_PROOF_BYTES; use drop_struct_macro_derive::DropStructMacro; use failure::Error; … diff --git a/filecoin-proofs/examples/beacon-post.rs b/filecoin-proofs/examples/beacon-post.rs index 104139839..a42c7431e 100644 --- a/filecoin-proofs/examples/beacon-post.rs +++ b/filecoin-proofs/examples/beacon-post.rs @@ -12,7 +12,7 @@ use paired::bls12_381::Bls12; use rand::{Rng, SeedableRng, XorShiftRng}; use std::time::{Duration, Instant}; -use filecoin_proofs::FCP_LOG; +use filecoin_proofs::singletons::FCP_LOG; use storage_proofs::beacon_post::*; use storage_proofs::drgraph::*; use storage_proofs::example_helper::prettyb; diff --git a/filecoin-proofs/examples/drgporep-vanilla-disk.rs b/filecoin-proofs/examples/drgporep-vanilla-disk.rs index 853d4a0e7..b5373e00f 100644 --- a/filecoin-proofs/examples/drgporep-vanilla-disk.rs +++ b/filecoin-proofs/examples/drgporep-vanilla-disk.rs @@ -21,7 +21,7 @@ use memmap::MmapOptions; use std::fs::File; use std::io::Write; -use filecoin_proofs::FCP_LOG; +use filecoin_proofs::singletons::FCP_LOG; fn file_backed_mmap_from_random_bytes(n: usize) -> MmapMut { let rng = &mut XorShiftRng::from_seed([0x3dbe6259, 0x8d313d76, 0x3237db17, 0xe5bc0654]); diff --git a/filecoin-proofs/examples/drgporep-vanilla.rs b/filecoin-proofs/examples/drgporep-vanilla.rs index e76e4b7d5..da454e388 100644 --- a/filecoin-proofs/examples/drgporep-vanilla.rs +++ b/filecoin-proofs/examples/drgporep-vanilla.rs @@ -21,7 +21,7 @@ use storage_proofs::hasher::{Blake2sHasher, Hasher, PedersenHasher, Sha256Hasher use storage_proofs::porep::PoRep; use storage_proofs::proof::ProofScheme; -use filecoin_proofs::FCP_LOG; +use filecoin_proofs::singletons::FCP_LOG; #[cfg(feature = "cpu-profile")] #[inline(always)] diff --git a/filecoin-proofs/examples/encoding.rs b/filecoin-proofs/examples/encoding.rs index f8dbb60fe..096b54228 100644 --- a/filecoin-proofs/examples/encoding.rs +++ b/filecoin-proofs/examples/encoding.rs @@ -26,7 +26,7 @@ use storage_proofs::proof::ProofScheme; use storage_proofs::vde; use storage_proofs::zigzag_drgporep::*; -use filecoin_proofs::FCP_LOG; +use filecoin_proofs::singletons::FCP_LOG; #[cfg(feature = "cpu-profile")] #[inline(always)] diff --git a/filecoin-proofs/examples/zigzag.rs b/filecoin-proofs/examples/zigzag.rs index 089f693a6..04b77149b 100644 --- a/filecoin-proofs/examples/zigzag.rs +++ b/filecoin-proofs/examples/zigzag.rs @@ -36,7 +36,7 @@ use storage_proofs::porep::PoRep; use storage_proofs::proof::ProofScheme; use storage_proofs::zigzag_drgporep::*; -use filecoin_proofs::FCP_LOG; +use filecoin_proofs::singletons::FCP_LOG; // We can only one of the profilers at a time, either CPU (`profile`) // or memory (`heap-profile`), duplicating the function so they won't diff --git a/filecoin-proofs/src/bin/paramcache.rs b/filecoin-proofs/src/bin/paramcache.rs index 7ad6b5c2c..80d1b6dde 100644 --- a/filecoin-proofs/src/bin/paramcache.rs +++ b/filecoin-proofs/src/bin/paramcache.rs @@ -1,9 +1,10 @@ use clap::{App, Arg}; +use paired::bls12_381::Bls12; use slog::*; -use filecoin_proofs::api::internal; -use filecoin_proofs::FCP_LOG; -use paired::bls12_381::Bls12; +use filecoin_proofs::parameters::{post_public_params, public_params}; +use filecoin_proofs::singletons::ENGINE_PARAMS; +use filecoin_proofs::singletons::FCP_LOG; use sector_base::api::bytes_amount::PaddedBytesAmount; use sector_base::api::disk_backed_storage::LIVE_SECTOR_SIZE; use sector_base::api::disk_backed_storage::TEST_SECTOR_SIZE; @@ -26,17 +27,17 @@ fn cache_porep_params(porep_config: PoRepConfig) { let n = u64::from(PaddedBytesAmount::from(porep_config)); info!(FCP_LOG, "begin PoRep parameter-cache check/populate routine for {}-byte sectors", n; "target" => "paramcache"); - let public_params = internal::public_params( + let public_params = public_params( PaddedBytesAmount::from(porep_config), usize::from(PoRepProofPartitions::from(porep_config)), ); { - let circuit = ZigZagCompound::blank_circuit(&public_params, &internal::ENGINE_PARAMS); + let circuit = ZigZagCompound::blank_circuit(&public_params, &ENGINE_PARAMS); let _ = ZigZagCompound::get_groth_params(circuit, &public_params); } { - let circuit = ZigZagCompound::blank_circuit(&public_params, &internal::ENGINE_PARAMS); + let circuit = ZigZagCompound::blank_circuit(&public_params, &ENGINE_PARAMS); let _ = ZigZagCompound::get_verifying_key(circuit, &public_params); } } @@ -45,14 +46,14 @@ fn cache_post_params(post_config: PoStConfig) { let n = u64::from(PaddedBytesAmount::from(post_config)); info!(FCP_LOG, "begin PoSt parameter-cache check/populate routine for {}-byte sectors", n; "target" => "paramcache"); - let post_public_params = internal::post_public_params(post_config); + let post_public_params = post_public_params(post_config); { let post_circuit: VDFPoStCircuit = , VDFPoStCircuit, - >>::blank_circuit(&post_public_params, &internal::ENGINE_PARAMS); + >>::blank_circuit(&post_public_params, &ENGINE_PARAMS); let _ = VDFPostCompound::get_groth_params(post_circuit, &post_public_params); } { @@ -61,7 +62,7 @@ fn cache_post_params(post_config: PoStConfig) { Bls12, VDFPoSt, VDFPoStCircuit, - >>::blank_circuit(&post_public_params, &internal::ENGINE_PARAMS); + >>::blank_circuit(&post_public_params, &ENGINE_PARAMS); let _ = VDFPostCompound::get_verifying_key(post_circuit, &post_public_params); } diff --git a/filecoin-proofs/src/bin/paramgen.rs b/filecoin-proofs/src/bin/paramgen.rs index c251f6e15..f19bc35d3 100644 --- a/filecoin-proofs/src/bin/paramgen.rs +++ b/filecoin-proofs/src/bin/paramgen.rs @@ -1,8 +1,10 @@ -use rand::OsRng; use std::env; use std::fs::File; -use filecoin_proofs::api::internal; +use rand::OsRng; + +use filecoin_proofs::parameters::public_params; +use filecoin_proofs::singletons::ENGINE_PARAMS; use sector_base::api::bytes_amount::PaddedBytesAmount; use sector_base::api::disk_backed_storage::TEST_SECTOR_SIZE; use sector_base::api::porep_proof_partitions::PoRepProofPartitions; @@ -15,19 +17,19 @@ pub fn main() { let args: Vec = env::args().collect(); let out_file = &args[1]; - let public_params = internal::public_params( + let public_params = public_params( PaddedBytesAmount::from(SectorSize(TEST_SECTOR_SIZE)), usize::from(PoRepProofPartitions(2)), ); - let circuit = ZigZagCompound::blank_circuit(&public_params, &internal::ENGINE_PARAMS); + let circuit = ZigZagCompound::blank_circuit(&public_params, &ENGINE_PARAMS); let mut params = phase21::MPCParameters::new(circuit).unwrap(); let rng = &mut OsRng::new().unwrap(); let hash = params.contribute(rng); { - let circuit = ZigZagCompound::blank_circuit(&public_params, &internal::ENGINE_PARAMS); + let circuit = ZigZagCompound::blank_circuit(&public_params, &ENGINE_PARAMS); let contributions = params.verify(circuit).expect("parameters should be valid!"); // We need to check the `contributions` to see if our `hash` diff --git a/filecoin-proofs/src/caches.rs b/filecoin-proofs/src/caches.rs new file mode 100644 index 000000000..625b9e3c3 --- /dev/null +++ b/filecoin-proofs/src/caches.rs @@ -0,0 +1,171 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::Mutex; + +use bellperson::groth16; +use paired::bls12_381::Bls12; + +use sector_base::api::bytes_amount::PaddedBytesAmount; +use sector_base::api::porep_config::PoRepConfig; +use sector_base::api::porep_proof_partitions::PoRepProofPartitions; +use sector_base::api::post_config::PoStConfig; +use storage_proofs::circuit::vdf_post::VDFPoStCircuit; +use storage_proofs::circuit::vdf_post::VDFPostCompound; +use storage_proofs::circuit::zigzag::ZigZagCompound; +use storage_proofs::compound_proof::CompoundProof; +use storage_proofs::hasher::PedersenHasher; +use storage_proofs::vdf_post::VDFPoSt; +use storage_proofs::vdf_sloth::Sloth; + +use crate::error; +use crate::parameters::{post_public_params, public_params}; +use crate::singletons::ENGINE_PARAMS; +use crate::singletons::FCP_LOG; + +type Bls12GrothParams = groth16::Parameters; +pub type Bls12VerifyingKey = groth16::VerifyingKey; + +type Cache = HashMap>; +type GrothMemCache = Cache; +type VerifyingKeyMemCache = Cache; + +lazy_static! { + static ref GROTH_PARAM_MEMORY_CACHE: Mutex = Default::default(); + static ref VERIFYING_KEY_MEMORY_CACHE: Mutex = Default::default(); +} + +pub fn cache_lookup( + cache_ref: &Mutex>, + identifier: String, + generator: F, +) -> error::Result> +where + F: FnOnce() -> error::Result, + G: Send + Sync, +{ + info!(FCP_LOG, "trying parameters memory cache for: {}", &identifier; "target" => "params"); + { + let cache = (*cache_ref).lock().unwrap(); + + if let Some(entry) = cache.get(&identifier) { + info!(FCP_LOG, "found params in memory cache for {}", &identifier; "target" => "params"); + return Ok(entry.clone()); + } + } + + info!(FCP_LOG, "no params in memory cache for {}", &identifier; "target" => "params"); + + let new_entry = Arc::new(generator()?); + let res = new_entry.clone(); + { + let cache = &mut (*cache_ref).lock().unwrap(); + cache.insert(identifier, new_entry); + } + + Ok(res) +} + +#[inline] +pub fn lookup_groth_params( + identifier: String, + generator: F, +) -> error::Result> +where + F: FnOnce() -> error::Result, +{ + cache_lookup(&*GROTH_PARAM_MEMORY_CACHE, identifier, generator) +} + +#[inline] +pub fn lookup_verifying_key( + identifier: String, + generator: F, +) -> error::Result> +where + F: FnOnce() -> error::Result, +{ + let vk_identifier = format!("{}-verifying-key", &identifier); + cache_lookup(&*VERIFYING_KEY_MEMORY_CACHE, vk_identifier, generator) +} + +pub fn get_zigzag_params( + porep_config: PoRepConfig, +) -> error::Result>> { + let public_params = public_params( + PaddedBytesAmount::from(porep_config), + usize::from(PoRepProofPartitions::from(porep_config)), + ); + + let parameters_generator = + || ZigZagCompound::groth_params(&public_params, &ENGINE_PARAMS).map_err(Into::into); + + Ok(lookup_groth_params( + format!( + "ZIGZAG[{}]", + usize::from(PaddedBytesAmount::from(porep_config)) + ), + parameters_generator, + )?) +} + +pub fn get_post_params(post_config: PoStConfig) -> error::Result>> { + let post_public_params = post_public_params(post_config); + + let parameters_generator = || { + , + VDFPoStCircuit, + >>::groth_params(&post_public_params, &ENGINE_PARAMS) + .map_err(Into::into) + }; + + Ok(lookup_groth_params( + format!( + "POST[{}]", + usize::from(PaddedBytesAmount::from(post_config)) + ), + parameters_generator, + )?) +} + +pub fn get_zigzag_verifying_key( + porep_config: PoRepConfig, +) -> error::Result> { + let public_params = public_params( + PaddedBytesAmount::from(porep_config), + usize::from(PoRepProofPartitions::from(porep_config)), + ); + + let vk_generator = + || ZigZagCompound::verifying_key(&public_params, &ENGINE_PARAMS).map_err(Into::into); + + Ok(lookup_verifying_key( + format!( + "ZIGZAG[{}]", + usize::from(PaddedBytesAmount::from(porep_config)) + ), + vk_generator, + )?) +} + +pub fn get_post_verifying_key(post_config: PoStConfig) -> error::Result> { + let post_public_params = post_public_params(post_config); + + let vk_generator = || { + , + VDFPoStCircuit, + >>::verifying_key(&post_public_params, &ENGINE_PARAMS) + .map_err(Into::into) + }; + + Ok(lookup_verifying_key( + format!( + "POST[{}]", + usize::from(PaddedBytesAmount::from(post_config)) + ), + vk_generator, + )?) +} diff --git a/filecoin-proofs/src/constants.rs b/filecoin-proofs/src/constants.rs new file mode 100644 index 000000000..90644252c --- /dev/null +++ b/filecoin-proofs/src/constants.rs @@ -0,0 +1,2 @@ +pub const POST_SECTORS_COUNT: usize = 2; +pub const POREP_MINIMUM_CHALLENGES: usize = 12; // FIXME: 8,000 diff --git a/filecoin-proofs/src/error.rs b/filecoin-proofs/src/error.rs index 58efbab42..322aead99 100644 --- a/filecoin-proofs/src/error.rs +++ b/filecoin-proofs/src/error.rs @@ -1,4 +1,4 @@ -use crate::FCP_LOG; +use crate::singletons::FCP_LOG; use failure::{Backtrace, Error}; use slog::*; diff --git a/filecoin-proofs/src/api/mod.rs b/filecoin-proofs/src/ffi_sector_builder.rs similarity index 70% rename from filecoin-proofs/src/api/mod.rs rename to filecoin-proofs/src/ffi_sector_builder.rs index f173c3550..24fe6ca8e 100644 --- a/filecoin-proofs/src/api/mod.rs +++ b/filecoin-proofs/src/ffi_sector_builder.rs @@ -1,94 +1,30 @@ -use libc; -use slog::*; use std::mem; use std::ptr; use std::slice::from_raw_parts; -use crate::api::post_adapter::*; -use crate::api::responses::err_code_and_msg; -use crate::api::responses::FCPResponseStatus; -use crate::api::responses::FFIPieceMetadata; -use crate::api::responses::FFISealStatus; -use crate::api::sector_builder::metadata::SealStatus; -use crate::api::sector_builder::SectorBuilder; -use crate::FCP_LOG; +use libc; +use slog::*; + use ffi_toolkit::rust_str_to_c_str; use ffi_toolkit::{c_str_to_rust_str, raw_ptr}; -use sector_base::api::bytes_amount::UnpaddedBytesAmount; -use sector_base::api::porep_config::PoRepConfig; -use sector_base::api::porep_proof_partitions; use sector_base::api::porep_proof_partitions::PoRepProofPartitions; use sector_base::api::post_config::PoStConfig; use sector_base::api::post_proof_partitions::PoStProofPartitions; use sector_base::api::sector_class::SectorClass; use sector_base::api::sector_size::SectorSize; -use sector_base::api::SINGLE_PARTITION_PROOF_LEN; - -pub mod internal; -pub mod post_adapter; -pub mod responses; -mod sector_builder; - -/// Verifies the output of seal. -/// -#[allow(dead_code)] -#[no_mangle] -pub unsafe extern "C" fn verify_seal( - sector_size: u64, - comm_r: &[u8; 32], - comm_d: &[u8; 32], - comm_r_star: &[u8; 32], - prover_id: &[u8; 31], - sector_id: &[u8; 31], - proof_ptr: *const u8, - proof_len: libc::size_t, -) -> *mut responses::VerifySealResponse { - info!(FCP_LOG, "verify_seal: {}", "start"; "target" => "FFI"); - - let porep_bytes = try_into_porep_proof_bytes(proof_ptr, proof_len); - - let result = porep_bytes.and_then(|bs| { - porep_proof_partitions::try_from_bytes(&bs).and_then(|ppp| { - let cfg = PoRepConfig(SectorSize(sector_size), ppp); - - internal::verify_seal( - cfg, - *comm_r, - *comm_d, - *comm_r_star, - prover_id, - sector_id, - &bs, - ) - }) - }); - - let mut response = responses::VerifySealResponse::default(); - - match result { - Ok(true) => { - response.status_code = FCPResponseStatus::FCPNoError; - response.is_valid = true; - } - Ok(false) => { - response.status_code = FCPResponseStatus::FCPNoError; - response.is_valid = false; - } - Err(err) => { - let (code, ptr) = err_code_and_msg(&err); - response.status_code = code; - response.error_msg = ptr; - } - }; - info!(FCP_LOG, "verify_seal: {}", "finish"; "target" => "FFI"); - - raw_ptr(response) -} +use crate::post_adapter::*; +use crate::responses; +use crate::responses::err_code_and_msg; +use crate::responses::FCPResponseStatus; +use crate::responses::FFIPieceMetadata; +use crate::responses::FFISealStatus; +use crate::sector_builder::metadata::SealStatus; +use crate::sector_builder::SectorBuilder; +use crate::singletons::FCP_LOG; /// Generates a proof-of-spacetime for the given replica commitments. /// -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn generate_post( ptr: *mut SectorBuilder, @@ -135,63 +71,8 @@ pub unsafe extern "C" fn generate_post( raw_ptr(response) } -/// Verifies that a proof-of-spacetime is valid. -/// -#[allow(dead_code)] -#[no_mangle] -pub unsafe extern "C" fn verify_post( - sector_size: u64, - proof_partitions: u8, - flattened_comm_rs_ptr: *const u8, - flattened_comm_rs_len: libc::size_t, - challenge_seed: &[u8; 32], - flattened_proofs_ptr: *const u8, - flattened_proofs_len: libc::size_t, - faults_ptr: *const u64, - faults_len: libc::size_t, -) -> *mut responses::VerifyPoSTResponse { - info!(FCP_LOG, "verify_post: {}", "start"; "target" => "FFI"); - - let post_bytes = - try_into_post_proofs_bytes(proof_partitions, flattened_proofs_ptr, flattened_proofs_len); - - let result = post_bytes.and_then(|bs| { - let cfg = PoStConfig( - SectorSize(sector_size), - PoStProofPartitions(proof_partitions), - ); - - internal::verify_post(VerifyPoStDynamicSectorsCountInput { - post_config: cfg, - comm_rs: into_commitments(flattened_comm_rs_ptr, flattened_comm_rs_len), - challenge_seed: into_safe_challenge_seed(challenge_seed), - proofs: bs, - faults: from_raw_parts(faults_ptr, faults_len).to_vec(), - }) - }); - - let mut response = responses::VerifyPoSTResponse::default(); - - match result { - Ok(dynamic) => { - response.status_code = FCPResponseStatus::FCPNoError; - response.is_valid = dynamic.is_valid; - } - Err(err) => { - let (code, ptr) = err_code_and_msg(&err); - response.status_code = code; - response.error_msg = ptr; - } - } - - info!(FCP_LOG, "verify_post: {}", "finish"; "target" => "FFI"); - - raw_ptr(response) -} - /// Initializes and returns a SectorBuilder. /// -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn init_sector_builder( sector_class: FFISectorClass, @@ -231,24 +112,14 @@ pub unsafe extern "C" fn init_sector_builder( /// Destroys a SectorBuilder. /// -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_sector_builder(ptr: *mut SectorBuilder) { let _ = Box::from_raw(ptr); } -/// Returns the number of user bytes that will fit into a staged sector. -/// -#[allow(dead_code)] -#[no_mangle] -pub unsafe extern "C" fn get_max_user_bytes_per_staged_sector(sector_size: u64) -> u64 { - u64::from(UnpaddedBytesAmount::from(SectorSize(sector_size))) -} - /// Writes user piece-bytes to a staged sector and returns the id of the sector /// to which the bytes were written. /// -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn add_piece( ptr: *mut SectorBuilder, @@ -282,7 +153,6 @@ pub unsafe extern "C" fn add_piece( /// Unseals and returns the bytes associated with the provided piece key. /// -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn read_piece_from_sealed_sector( ptr: *mut SectorBuilder, @@ -311,7 +181,6 @@ pub unsafe extern "C" fn read_piece_from_sealed_sector( /// For demo purposes. Seals all staged sectors. /// -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn seal_all_staged_sectors( ptr: *mut SectorBuilder, @@ -335,7 +204,6 @@ pub unsafe extern "C" fn seal_all_staged_sectors( /// Returns sector sealing status for the provided sector id if it exists. If /// we don't know about the provided sector id, produce an error. /// -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn get_seal_status( ptr: *mut SectorBuilder, @@ -396,7 +264,6 @@ pub unsafe extern "C" fn get_seal_status( raw_ptr(response) } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn get_sealed_sectors( ptr: *mut SectorBuilder, @@ -455,7 +322,6 @@ pub unsafe extern "C" fn get_sealed_sectors( raw_ptr(response) } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn get_staged_sectors( ptr: *mut SectorBuilder, @@ -524,59 +390,6 @@ pub unsafe extern "C" fn get_staged_sectors( raw_ptr(response) } -unsafe fn try_into_post_proofs_bytes( - proof_partitions: u8, - flattened_proofs_ptr: *const u8, - flattened_proofs_len: libc::size_t, -) -> crate::error::Result>> { - let chunk_size = proof_partitions as usize * SINGLE_PARTITION_PROOF_LEN; - - ensure!( - flattened_proofs_len % chunk_size == 0, - "proofs array len={:?} incompatible with partitions={:?}", - flattened_proofs_len, - proof_partitions - ); - - Ok(into_proof_vecs( - chunk_size, - flattened_proofs_ptr, - flattened_proofs_len, - )) -} - -unsafe fn try_into_porep_proof_bytes( - proof_ptr: *const u8, - proof_len: libc::size_t, -) -> crate::error::Result> { - into_proof_vecs(proof_len, proof_ptr, proof_len) - .first() - .map(Vec::clone) - .ok_or_else(|| format_err!("no proofs in chunked vec")) -} - -unsafe fn into_proof_vecs( - proof_chunk: usize, - flattened_proofs_ptr: *const u8, - flattened_proofs_len: libc::size_t, -) -> Vec> { - from_raw_parts(flattened_proofs_ptr, flattened_proofs_len) - .iter() - .step_by(proof_chunk) - .fold(Default::default(), |mut acc: Vec>, item| { - let sliced = from_raw_parts(item, proof_chunk); - acc.push(sliced.to_vec()); - acc - }) -} - -fn into_safe_challenge_seed(challenge_seed: &[u8; 32]) -> [u8; 32] { - let mut cs = [0; 32]; - cs.copy_from_slice(challenge_seed); - cs[31] &= 0b0011_1111; - cs -} - unsafe fn into_commitments( flattened_comms_ptr: *const u8, flattened_comms_len: libc::size_t, diff --git a/filecoin-proofs/src/ffi_stateless.rs b/filecoin-proofs/src/ffi_stateless.rs new file mode 100644 index 000000000..5675b5034 --- /dev/null +++ b/filecoin-proofs/src/ffi_stateless.rs @@ -0,0 +1,227 @@ +use std::slice::from_raw_parts; + +use libc; +use slog::*; + +use ffi_toolkit::raw_ptr; +use sector_base::api::bytes_amount::UnpaddedBytesAmount; +use sector_base::api::porep_config::PoRepConfig; +use sector_base::api::porep_proof_partitions; +use sector_base::api::porep_proof_partitions::PoRepProofPartitions; +use sector_base::api::post_config::PoStConfig; +use sector_base::api::post_proof_partitions::PoStProofPartitions; +use sector_base::api::sector_class::SectorClass; +use sector_base::api::sector_size::SectorSize; +use sector_base::api::SINGLE_PARTITION_PROOF_LEN; + +use crate::responses; +use crate::responses::err_code_and_msg; +use crate::responses::FCPResponseStatus; +use crate::safe; +use crate::singletons::FCP_LOG; + +/// Verifies the output of seal. +/// +#[no_mangle] +pub unsafe extern "C" fn verify_seal( + sector_size: u64, + comm_r: &[u8; 32], + comm_d: &[u8; 32], + comm_r_star: &[u8; 32], + prover_id: &[u8; 31], + sector_id: &[u8; 31], + proof_ptr: *const u8, + proof_len: libc::size_t, +) -> *mut responses::VerifySealResponse { + info!(FCP_LOG, "verify_seal: {}", "start"; "target" => "FFI"); + + let porep_bytes = try_into_porep_proof_bytes(proof_ptr, proof_len); + + let result = porep_bytes.and_then(|bs| { + porep_proof_partitions::try_from_bytes(&bs).and_then(|ppp| { + let cfg = PoRepConfig(SectorSize(sector_size), ppp); + + safe::verify_seal( + cfg, + *comm_r, + *comm_d, + *comm_r_star, + prover_id, + sector_id, + &bs, + ) + }) + }); + + let mut response = responses::VerifySealResponse::default(); + + match result { + Ok(true) => { + response.status_code = FCPResponseStatus::FCPNoError; + response.is_valid = true; + } + Ok(false) => { + response.status_code = FCPResponseStatus::FCPNoError; + response.is_valid = false; + } + Err(err) => { + let (code, ptr) = err_code_and_msg(&err); + response.status_code = code; + response.error_msg = ptr; + } + }; + + info!(FCP_LOG, "verify_seal: {}", "finish"; "target" => "FFI"); + + raw_ptr(response) +} + +/// Verifies that a proof-of-spacetime is valid. +/// +#[no_mangle] +pub unsafe extern "C" fn verify_post( + sector_size: u64, + proof_partitions: u8, + flattened_comm_rs_ptr: *const u8, + flattened_comm_rs_len: libc::size_t, + challenge_seed: &[u8; 32], + flattened_proofs_ptr: *const u8, + flattened_proofs_len: libc::size_t, + faults_ptr: *const u64, + faults_len: libc::size_t, +) -> *mut responses::VerifyPoSTResponse { + info!(FCP_LOG, "verify_post: {}", "start"; "target" => "FFI"); + + let post_bytes = + try_into_post_proofs_bytes(proof_partitions, flattened_proofs_ptr, flattened_proofs_len); + + let result = post_bytes.and_then(|bs| { + let cfg = PoStConfig( + SectorSize(sector_size), + PoStProofPartitions(proof_partitions), + ); + + safe::verify_post( + cfg, + into_commitments(flattened_comm_rs_ptr, flattened_comm_rs_len), + into_safe_challenge_seed(challenge_seed), + bs, + from_raw_parts(faults_ptr, faults_len).to_vec(), + ) + }); + + let mut response = responses::VerifyPoSTResponse::default(); + + match result { + Ok(dynamic) => { + response.status_code = FCPResponseStatus::FCPNoError; + response.is_valid = dynamic.is_valid; + } + Err(err) => { + let (code, ptr) = err_code_and_msg(&err); + response.status_code = code; + response.error_msg = ptr; + } + } + + info!(FCP_LOG, "verify_post: {}", "finish"; "target" => "FFI"); + + raw_ptr(response) +} + +/// Returns the number of user bytes that will fit into a staged sector. +/// +#[no_mangle] +pub unsafe extern "C" fn get_max_user_bytes_per_staged_sector(sector_size: u64) -> u64 { + u64::from(UnpaddedBytesAmount::from(SectorSize(sector_size))) +} + +unsafe fn try_into_post_proofs_bytes( + proof_partitions: u8, + flattened_proofs_ptr: *const u8, + flattened_proofs_len: libc::size_t, +) -> crate::error::Result>> { + let chunk_size = proof_partitions as usize * SINGLE_PARTITION_PROOF_LEN; + + ensure!( + flattened_proofs_len % chunk_size == 0, + "proofs array len={:?} incompatible with partitions={:?}", + flattened_proofs_len, + proof_partitions + ); + + Ok(into_proof_vecs( + chunk_size, + flattened_proofs_ptr, + flattened_proofs_len, + )) +} + +unsafe fn try_into_porep_proof_bytes( + proof_ptr: *const u8, + proof_len: libc::size_t, +) -> crate::error::Result> { + into_proof_vecs(proof_len, proof_ptr, proof_len) + .first() + .map(Vec::clone) + .ok_or_else(|| format_err!("no proofs in chunked vec")) +} + +unsafe fn into_proof_vecs( + proof_chunk: usize, + flattened_proofs_ptr: *const u8, + flattened_proofs_len: libc::size_t, +) -> Vec> { + from_raw_parts(flattened_proofs_ptr, flattened_proofs_len) + .iter() + .step_by(proof_chunk) + .fold(Default::default(), |mut acc: Vec>, item| { + let sliced = from_raw_parts(item, proof_chunk); + acc.push(sliced.to_vec()); + acc + }) +} + +fn into_safe_challenge_seed(challenge_seed: &[u8; 32]) -> [u8; 32] { + let mut cs = [0; 32]; + cs.copy_from_slice(challenge_seed); + cs[31] &= 0b0011_1111; + cs +} + +unsafe fn into_commitments( + flattened_comms_ptr: *const u8, + flattened_comms_len: libc::size_t, +) -> Vec<[u8; 32]> { + from_raw_parts(flattened_comms_ptr, flattened_comms_len) + .iter() + .step_by(32) + .fold(Default::default(), |mut acc: Vec<[u8; 32]>, item| { + let sliced = from_raw_parts(item, 32); + let mut x: [u8; 32] = Default::default(); + x.copy_from_slice(&sliced[..32]); + acc.push(x); + acc + }) +} + +#[repr(C)] +pub struct FFISectorClass { + sector_size: u64, + porep_proof_partitions: u8, + post_proof_partitions: u8, +} + +pub fn from_ffi_sector_class(fsc: FFISectorClass) -> SectorClass { + match fsc { + FFISectorClass { + sector_size, + porep_proof_partitions, + post_proof_partitions, + } => SectorClass( + SectorSize(sector_size), + PoRepProofPartitions(porep_proof_partitions), + PoStProofPartitions(post_proof_partitions), + ), + } +} diff --git a/filecoin-proofs/src/file_cleanup.rs b/filecoin-proofs/src/file_cleanup.rs new file mode 100644 index 000000000..d1f42eea2 --- /dev/null +++ b/filecoin-proofs/src/file_cleanup.rs @@ -0,0 +1,25 @@ +use std::fs::remove_file; +use std::path::Path; + +impl<'a, T: AsRef> FileCleanup { + pub fn new(path: T) -> FileCleanup { + FileCleanup { + path, + success: false, + } + } +} + +impl> Drop for FileCleanup { + fn drop(&mut self) { + if !self.success { + let _ = remove_file(&self.path); + } + } +} + +/// Minimal support for cleaning (deleting) a file unless it was successfully populated. +pub struct FileCleanup> { + path: T, + pub success: bool, +} diff --git a/filecoin-proofs/src/lib.rs b/filecoin-proofs/src/lib.rs index 2a5169d38..c3b631534 100644 --- a/filecoin-proofs/src/lib.rs +++ b/filecoin-proofs/src/lib.rs @@ -9,14 +9,18 @@ extern crate serde; #[macro_use] extern crate slog; -pub mod api; +mod caches; +mod constants; +mod file_cleanup; +mod post_adapter; +mod responses; +mod safe; +mod sector_builder; + pub mod error; +pub mod ffi_sector_builder; +pub mod ffi_stateless; pub mod param; +pub mod parameters; pub mod serde_big_array; - -use logging_toolkit::make_logger; -use slog::Logger; - -lazy_static! { - pub static ref FCP_LOG: Logger = make_logger("filecoin-proofs"); -} +pub mod singletons; diff --git a/filecoin-proofs/src/parameters.rs b/filecoin-proofs/src/parameters.rs new file mode 100644 index 000000000..df58a519b --- /dev/null +++ b/filecoin-proofs/src/parameters.rs @@ -0,0 +1,146 @@ +use sector_base::api::bytes_amount::PaddedBytesAmount; +use sector_base::api::post_config::PoStConfig; +use storage_proofs::drgporep::DrgParams; +use storage_proofs::drgraph::DefaultTreeHasher; +use storage_proofs::hasher::pedersen::PedersenDomain; +use storage_proofs::hasher::PedersenHasher; +use storage_proofs::layered_drgporep; +use storage_proofs::layered_drgporep::LayerChallenges; +use storage_proofs::proof::ProofScheme; +use storage_proofs::vdf_post; +use storage_proofs::vdf_post::VDFPoSt; +use storage_proofs::vdf_sloth; +use storage_proofs::zigzag_drgporep::ZigZagDrgPoRep; +use storage_proofs::zigzag_graph::ZigZagBucketGraph; + +use crate::constants::POREP_MINIMUM_CHALLENGES; +use crate::constants::POST_SECTORS_COUNT; +use crate::singletons::POST_VDF_KEY; + +const POST_CHALLENGE_COUNT: usize = 30; +const POST_EPOCHS: usize = 3; +const POST_VDF_ROUNDS: usize = 1; + +const DEGREE: usize = 5; +const EXPANSION_DEGREE: usize = 8; +const SLOTH_ITER: usize = 0; +const LAYERS: usize = 4; // TODO: 10; +const TAPER_LAYERS: usize = 2; // TODO: 7 +const TAPER: f64 = 1.0 / 3.0; + +const DRG_SEED: [u32; 7] = [1, 2, 3, 4, 5, 6, 7]; // Arbitrary, need a theory for how to vary this over time. + +type PostSetupParams = vdf_post::SetupParams; +pub type PostPublicParams = vdf_post::PublicParams; + +pub fn public_params( + sector_bytes: PaddedBytesAmount, + partitions: usize, +) -> layered_drgporep::PublicParams> { + ZigZagDrgPoRep::::setup(&setup_params(sector_bytes, partitions)).unwrap() +} + +pub fn post_public_params(post_config: PoStConfig) -> PostPublicParams { + VDFPoSt::::setup(&post_setup_params(post_config)).unwrap() +} + +pub fn post_setup_params(post_config: PoStConfig) -> PostSetupParams { + let size = PaddedBytesAmount::from(post_config); + + vdf_post::SetupParams:: { + challenge_count: POST_CHALLENGE_COUNT, + sector_size: size.into(), + post_epochs: POST_EPOCHS, + setup_params_vdf: vdf_sloth::SetupParams { + key: *POST_VDF_KEY, + rounds: POST_VDF_ROUNDS, + }, + sectors_count: POST_SECTORS_COUNT, + } +} + +pub fn setup_params( + sector_bytes: PaddedBytesAmount, + partitions: usize, +) -> layered_drgporep::SetupParams { + let sector_bytes = usize::from(sector_bytes); + + let challenges = select_challenges( + partitions, + POREP_MINIMUM_CHALLENGES, + LAYERS, + TAPER_LAYERS, + TAPER, + ); + + assert!( + sector_bytes % 32 == 0, + "sector_bytes ({}) must be a multiple of 32", + sector_bytes, + ); + let nodes = sector_bytes / 32; + layered_drgporep::SetupParams { + drg: DrgParams { + nodes, + degree: DEGREE, + expansion_degree: EXPANSION_DEGREE, + seed: DRG_SEED, + }, + sloth_iter: SLOTH_ITER, + layer_challenges: challenges, + } +} + +fn select_challenges( + partitions: usize, + minimum_total_challenges: usize, + layers: usize, + taper_layers: usize, + taper: f64, +) -> LayerChallenges { + let mut count = 1; + let mut guess = LayerChallenges::Tapered { + count, + layers, + taper, + taper_layers, + }; + while partitions * guess.total_challenges() < minimum_total_challenges { + count += 1; + guess = LayerChallenges::Tapered { + count, + layers, + taper, + taper_layers, + }; + } + guess +} + +#[cfg(test)] +mod tests { + use sector_base::api::porep_proof_partitions::PoRepProofPartitions; + + use crate::constants::POREP_MINIMUM_CHALLENGES; + use crate::parameters::*; + + #[test] + fn partition_layer_challenges_test() { + let f = |partitions| { + select_challenges( + partitions, + POREP_MINIMUM_CHALLENGES, + LAYERS, + TAPER_LAYERS, + TAPER, + ) + .all_challenges() + }; + // Update to ensure all supported PoRepProofPartitions options are represented here. + assert_eq!(vec![1, 1, 2, 2], f(usize::from(PoRepProofPartitions(2)))); + + assert_eq!(vec![3, 3, 4, 5], f(1)); + assert_eq!(vec![1, 1, 2, 2], f(2)); + assert_eq!(vec![1, 1, 1, 1], f(4)); + } +} diff --git a/filecoin-proofs/src/api/post_adapter.rs b/filecoin-proofs/src/post_adapter.rs similarity index 99% rename from filecoin-proofs/src/api/post_adapter.rs rename to filecoin-proofs/src/post_adapter.rs index 792121a83..e63edba96 100644 --- a/filecoin-proofs/src/api/post_adapter.rs +++ b/filecoin-proofs/src/post_adapter.rs @@ -1,12 +1,13 @@ use std::iter::repeat; use std::iter::repeat_with; -use crate::api::internal::ChallengeSeed; -use crate::api::internal::Commitment; -use crate::api::internal::POST_SECTORS_COUNT; -use crate::error; use sector_base::api::post_config::PoStConfig; +use crate::constants::POST_SECTORS_COUNT; +use crate::error; +use crate::safe::ChallengeSeed; +use crate::safe::Commitment; + pub struct GeneratePoStDynamicSectorsCountInput { pub post_config: PoStConfig, pub challenge_seed: ChallengeSeed, @@ -318,11 +319,12 @@ pub fn verify_post_collect_output( #[cfg(test)] mod tests { - use super::*; use sector_base::api::disk_backed_storage::TEST_SECTOR_SIZE; use sector_base::api::post_proof_partitions::PoStProofPartitions; use sector_base::api::sector_size::SectorSize; + use super::*; + const TEST_CONFIG: PoStConfig = PoStConfig(SectorSize(TEST_SECTOR_SIZE), PoStProofPartitions(1)); diff --git a/filecoin-proofs/src/api/responses.rs b/filecoin-proofs/src/responses.rs similarity index 97% rename from filecoin-proofs/src/api/responses.rs rename to filecoin-proofs/src/responses.rs index bbaccd124..6c8a86e82 100644 --- a/filecoin-proofs/src/api/responses.rs +++ b/filecoin-proofs/src/responses.rs @@ -1,5 +1,5 @@ -use crate::api::sector_builder::errors::SectorBuilderErr; -use crate::api::sector_builder::SectorBuilder; +use crate::sector_builder::errors::SectorBuilderErr; +use crate::sector_builder::SectorBuilder; use drop_struct_macro_derive::DropStructMacro; use failure::Error; use ffi_toolkit::free_c_str; @@ -50,7 +50,6 @@ impl Default for VerifySealResponse { } } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_verify_seal_response(ptr: *mut VerifySealResponse) { let _ = Box::from_raw(ptr); @@ -86,7 +85,6 @@ impl Default for GeneratePoStResponse { } } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_generate_post_response(ptr: *mut GeneratePoStResponse) { let _ = Box::from_raw(ptr); @@ -114,7 +112,6 @@ impl Default for VerifyPoSTResponse { } } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_verify_post_response(ptr: *mut VerifyPoSTResponse) { let _ = Box::from_raw(ptr); @@ -124,7 +121,7 @@ pub unsafe extern "C" fn destroy_verify_post_response(ptr: *mut VerifyPoSTRespon // status code and a pointer to a C string, both of which can be used to set // fields in a response struct to be returned from an FFI call. pub fn err_code_and_msg(err: &Error) -> (FCPResponseStatus, *const libc::c_char) { - use crate::api::responses::FCPResponseStatus::*; + use crate::responses::FCPResponseStatus::*; let msg = CString::new(format!("{}", err)).unwrap(); let ptr = msg.as_ptr(); @@ -170,7 +167,6 @@ impl Default for InitSectorBuilderResponse { } } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_init_sector_builder_response(ptr: *mut InitSectorBuilderResponse) { let _ = Box::from_raw(ptr); @@ -198,7 +194,6 @@ impl Default for AddPieceResponse { } } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_add_piece_response(ptr: *mut AddPieceResponse) { let _ = Box::from_raw(ptr); @@ -228,7 +223,6 @@ impl Default for ReadPieceFromSealedSectorResponse { } } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_read_piece_from_sealed_sector_response( ptr: *mut ReadPieceFromSealedSectorResponse, @@ -256,7 +250,6 @@ impl Default for SealAllStagedSectorsResponse { } } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_seal_all_staged_sectors_response( ptr: *mut SealAllStagedSectorsResponse, @@ -318,7 +311,6 @@ impl Default for GetSealStatusResponse { } } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_get_seal_status_response(ptr: *mut GetSealStatusResponse) { let _ = Box::from_raw(ptr); @@ -386,7 +378,6 @@ impl Default for GetSealedSectorsResponse { } } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_get_sealed_sectors_response(ptr: *mut GetSealedSectorsResponse) { let _ = Box::from_raw(ptr); @@ -417,7 +408,6 @@ impl Default for GetStagedSectorsResponse { } } -#[allow(dead_code)] #[no_mangle] pub unsafe extern "C" fn destroy_get_staged_sectors_response(ptr: *mut GetStagedSectorsResponse) { let _ = Box::from_raw(ptr); diff --git a/filecoin-proofs/src/api/internal.rs b/filecoin-proofs/src/safe.rs similarity index 74% rename from filecoin-proofs/src/api/internal.rs rename to filecoin-proofs/src/safe.rs index b495fd306..725a93762 100644 --- a/filecoin-proofs/src/api/internal.rs +++ b/filecoin-proofs/src/safe.rs @@ -1,20 +1,12 @@ -use std::collections::HashMap; -use std::fs::{copy, remove_file, File, OpenOptions}; +use std::fs::{copy, File, OpenOptions}; use std::io::{BufWriter, Read}; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; -use bellperson::groth16; use ff::PrimeField; -use fil_sapling_crypto::jubjub::JubjubBls12; use memmap::MmapOptions; -use paired::bls12_381::{Bls12, Fr}; +use paired::bls12_381::Bls12; use paired::Engine; -use crate::api::post_adapter::*; -use crate::error; -use crate::error::ExpectWithBacktrace; -use crate::FCP_LOG; use sector_base::api::bytes_amount::{PaddedBytesAmount, UnpaddedByteIndex, UnpaddedBytesAmount}; use sector_base::api::porep_config::PoRepConfig; use sector_base::api::porep_proof_partitions::PoRepProofPartitions; @@ -23,301 +15,307 @@ use sector_base::api::post_proof_partitions::PoStProofPartitions; use sector_base::api::SINGLE_PARTITION_PROOF_LEN; use sector_base::io::fr32::write_unpadded; use storage_proofs::circuit::multi_proof::MultiProof; -use storage_proofs::circuit::vdf_post::{VDFPoStCircuit, VDFPostCompound}; +use storage_proofs::circuit::vdf_post::VDFPostCompound; use storage_proofs::circuit::zigzag::ZigZagCompound; use storage_proofs::compound_proof::{self, CompoundProof}; -use storage_proofs::drgporep::DrgParams; use storage_proofs::drgraph::{DefaultTreeHasher, Graph}; use storage_proofs::fr32::{bytes_into_fr, fr_into_bytes, Fr32Ary}; use storage_proofs::hasher::pedersen::{PedersenDomain, PedersenHasher}; use storage_proofs::hasher::{Domain, Hasher}; -use storage_proofs::layered_drgporep::{self, ChallengeRequirements, LayerChallenges}; +use storage_proofs::layered_drgporep::{self, ChallengeRequirements}; use storage_proofs::merkle::MerkleTree; use storage_proofs::porep::{replica_id, PoRep, Tau}; -use storage_proofs::proof::{NoRequirements, ProofScheme}; -use storage_proofs::vdf_post::{self, VDFPoSt}; -use storage_proofs::vdf_sloth::{self, Sloth}; +use storage_proofs::proof::NoRequirements; +use storage_proofs::vdf_post; +use storage_proofs::vdf_sloth; use storage_proofs::zigzag_drgporep::ZigZagDrgPoRep; -use storage_proofs::zigzag_graph::ZigZagBucketGraph; -pub type Commitment = Fr32Ary; -pub type ChallengeSeed = Fr32Ary; +use crate::caches::{ + get_post_params, get_post_verifying_key, get_zigzag_params, get_zigzag_verifying_key, +}; +use crate::constants::POREP_MINIMUM_CHALLENGES; +use crate::error; +use crate::error::ExpectWithBacktrace; +use crate::file_cleanup::FileCleanup; +use crate::parameters::{post_setup_params, public_params, setup_params}; +use crate::post_adapter::*; +use crate::singletons::ENGINE_PARAMS; +use crate::singletons::FCP_LOG; /// FrSafe is an array of the largest whole number of bytes guaranteed not to overflow the field. type FrSafe = [u8; 31]; -lazy_static! { - pub static ref ENGINE_PARAMS: JubjubBls12 = JubjubBls12::new(); -} - -//////////////////////////////////////////////////////////////////////////////// -/// Groth Params/Verifying-keys Memory Cache +pub type Commitment = Fr32Ary; +pub type ChallengeSeed = Fr32Ary; +type Tree = MerkleTree::Function>; -type Bls12GrothParams = groth16::Parameters; -type Bls12VerifyingKey = groth16::VerifyingKey; +#[derive(Clone, Debug)] +pub struct SealOutput { + pub comm_r: Commitment, + pub comm_r_star: Commitment, + pub comm_d: Commitment, + pub proof: Vec, +} -type Cache = HashMap>; -type GrothMemCache = Cache; -type VerifyingKeyMemCache = Cache; +/// Generates a proof-of-spacetime, returning and detected storage faults. +/// Accepts as input a challenge seed, configuration struct, and a vector of +/// sealed sector file-path plus CommR tuples. +/// +pub fn generate_post( + post_config: PoStConfig, + challenge_seed: ChallengeSeed, + input_parts: Vec<(Option, Commitment)>, +) -> error::Result { + generate_post_dynamic(GeneratePoStDynamicSectorsCountInput { + post_config, + challenge_seed, + input_parts, + }) +} -lazy_static! { - static ref GROTH_PARAM_MEMORY_CACHE: Mutex = Default::default(); - static ref VERIFYING_KEY_MEMORY_CACHE: Mutex = Default::default(); +/// Verifies a proof-of-spacetime. +/// +pub fn verify_post( + post_config: PoStConfig, + comm_rs: Vec, + challenge_seed: ChallengeSeed, + proofs: Vec>, + faults: Vec, +) -> error::Result { + verify_post_dynamic(VerifyPoStDynamicSectorsCountInput { + post_config, + comm_rs, + challenge_seed, + proofs, + faults, + }) } -fn cache_lookup( - cache_ref: &Mutex>, - identifier: String, - generator: F, -) -> error::Result> -where - F: FnOnce() -> error::Result, - G: Send + Sync, -{ - info!(FCP_LOG, "trying parameters memory cache for: {}", &identifier; "target" => "params"); - { - let cache = (*cache_ref).lock().unwrap(); - - if let Some(entry) = cache.get(&identifier) { - info!(FCP_LOG, "found params in memory cache for {}", &identifier; "target" => "params"); - return Ok(entry.clone()); - } - } +/// Seals the staged sector at `in_path` in place, saving the resulting replica +/// to `out_path`. +/// +pub fn seal + AsRef>( + porep_config: PoRepConfig, + in_path: T, + out_path: T, + prover_id_in: &FrSafe, + sector_id_in: &FrSafe, +) -> error::Result { + let sector_bytes = usize::from(PaddedBytesAmount::from(porep_config)); - info!(FCP_LOG, "no params in memory cache for {}", &identifier; "target" => "params"); + let mut cleanup = FileCleanup::new(&out_path); - let new_entry = Arc::new(generator()?); - let res = new_entry.clone(); - { - let cache = &mut (*cache_ref).lock().unwrap(); - cache.insert(identifier, new_entry); - } + // Copy unsealed data to output location, where it will be sealed in place. + copy(&in_path, &out_path)?; + let f_data = OpenOptions::new().read(true).write(true).open(&out_path)?; - Ok(res) -} + // Zero-pad the data to the requested size by extending the underlying file if needed. + f_data.set_len(sector_bytes as u64)?; -#[inline] -fn lookup_groth_params(identifier: String, generator: F) -> error::Result> -where - F: FnOnce() -> error::Result, -{ - cache_lookup(&*GROTH_PARAM_MEMORY_CACHE, identifier, generator) -} + let mut data = unsafe { MmapOptions::new().map_mut(&f_data).unwrap() }; -#[inline] -fn lookup_verifying_key( - identifier: String, - generator: F, -) -> error::Result> -where - F: FnOnce() -> error::Result, -{ - let vk_identifier = format!("{}-verifying-key", &identifier); - cache_lookup(&*VERIFYING_KEY_MEMORY_CACHE, vk_identifier, generator) -} + // Zero-pad the prover_id to 32 bytes (and therefore Fr32). + let prover_id = pad_safe_fr(prover_id_in); + // Zero-pad the sector_id to 32 bytes (and therefore Fr32). + let sector_id = pad_safe_fr(sector_id_in); + let replica_id = replica_id::(prover_id, sector_id); -//////////////////////////////////////////////////////////////////////////////// + let compound_setup_params = compound_proof::SetupParams { + vanilla_params: &setup_params( + PaddedBytesAmount::from(porep_config), + usize::from(PoRepProofPartitions::from(porep_config)), + ), + engine_params: &(*ENGINE_PARAMS), + partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))), + }; -fn get_zigzag_params(porep_config: PoRepConfig) -> error::Result>> { - let public_params = public_params( - PaddedBytesAmount::from(porep_config), - usize::from(PoRepProofPartitions::from(porep_config)), - ); + let compound_public_params = ZigZagCompound::setup(&compound_setup_params)?; + + let (tau, aux) = ZigZagDrgPoRep::replicate( + &compound_public_params.vanilla_params, + &replica_id, + &mut data, + None, + )?; - let get_params = - || ZigZagCompound::groth_params(&public_params, &ENGINE_PARAMS).map_err(Into::into); + // If we succeeded in replicating, flush the data and protect output from being cleaned up. + data.flush()?; + cleanup.success = true; - Ok(lookup_groth_params( - format!( - "ZIGZAG[{}]", - usize::from(PaddedBytesAmount::from(porep_config)) - ), - get_params, - )?) -} + let public_tau = tau.simplify(); -fn get_post_params(post_config: PoStConfig) -> error::Result>> { - let post_public_params = post_public_params(post_config); + let public_inputs = layered_drgporep::PublicInputs { + replica_id, + tau: Some(public_tau), + comm_r_star: tau.comm_r_star, + k: None, + }; - let get_params = || { - , - VDFPoStCircuit, - >>::groth_params(&post_public_params, &ENGINE_PARAMS) - .map_err(Into::into) + let private_inputs = layered_drgporep::PrivateInputs:: { + aux, + tau: tau.layer_taus, }; - Ok(lookup_groth_params( - format!( - "POST[{}]", - usize::from(PaddedBytesAmount::from(post_config)) - ), - get_params, - )?) -} + let groth_params = get_zigzag_params(porep_config)?; + + info!(FCP_LOG, "got groth params ({}) while sealing", u64::from(PaddedBytesAmount::from(porep_config)); "target" => "params"); + + let proof = ZigZagCompound::prove( + &compound_public_params, + &public_inputs, + &private_inputs, + &groth_params, + )?; -fn get_zigzag_verifying_key(porep_config: PoRepConfig) -> error::Result> { - let public_params = public_params( - PaddedBytesAmount::from(porep_config), - usize::from(PoRepProofPartitions::from(porep_config)), + let mut buf = Vec::with_capacity( + SINGLE_PARTITION_PROOF_LEN * usize::from(PoRepProofPartitions::from(porep_config)), ); - let get_verifying_key = - || ZigZagCompound::verifying_key(&public_params, &ENGINE_PARAMS).map_err(Into::into); + proof.write(&mut buf)?; - Ok(lookup_verifying_key( - format!( - "ZIGZAG[{}]", - usize::from(PaddedBytesAmount::from(porep_config)) - ), - get_verifying_key, - )?) + let comm_r = commitment_from_fr::(public_tau.comm_r.into()); + let comm_d = commitment_from_fr::(public_tau.comm_d.into()); + let comm_r_star = commitment_from_fr::(tau.comm_r_star.into()); + + // Verification is cheap when parameters are cached, + // and it is never correct to return a proof which does not verify. + verify_seal( + porep_config, + comm_r, + comm_d, + comm_r_star, + prover_id_in, + sector_id_in, + &buf, + ) + .expect("post-seal verification sanity check failed"); + + Ok(SealOutput { + comm_r, + comm_r_star, + comm_d, + proof: buf, + }) } -fn get_post_verifying_key(post_config: PoStConfig) -> error::Result> { - let post_public_params = post_public_params(post_config); +/// Verifies the output of some previously-run seal operation. +/// +pub fn verify_seal( + porep_config: PoRepConfig, + comm_r: Commitment, + comm_d: Commitment, + comm_r_star: Commitment, + prover_id_in: &FrSafe, + sector_id_in: &FrSafe, + proof_vec: &[u8], +) -> error::Result { + let sector_bytes = PaddedBytesAmount::from(porep_config); + let prover_id = pad_safe_fr(prover_id_in); + let sector_id = pad_safe_fr(sector_id_in); + let replica_id = replica_id::(prover_id, sector_id); - let get_verifying_key = || { - , - VDFPoStCircuit, - >>::verifying_key(&post_public_params, &ENGINE_PARAMS) - .map_err(Into::into) - }; + let comm_r = bytes_into_fr::(&comm_r)?; + let comm_d = bytes_into_fr::(&comm_d)?; + let comm_r_star = bytes_into_fr::(&comm_r_star)?; - Ok(lookup_verifying_key( - format!( - "POST[{}]", - usize::from(PaddedBytesAmount::from(post_config)) + let compound_setup_params = compound_proof::SetupParams { + vanilla_params: &setup_params( + PaddedBytesAmount::from(porep_config), + usize::from(PoRepProofPartitions::from(porep_config)), ), - get_verifying_key, - )?) -} + engine_params: &(*ENGINE_PARAMS), + partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))), + }; + + let compound_public_params: compound_proof::PublicParams< + '_, + Bls12, + ZigZagDrgPoRep<'_, DefaultTreeHasher>, + > = ZigZagCompound::setup(&compound_setup_params)?; -const DEGREE: usize = 5; -const EXPANSION_DEGREE: usize = 8; -const SLOTH_ITER: usize = 0; -const LAYERS: usize = 4; // TODO: 10; -const TAPER_LAYERS: usize = 2; // TODO: 7 -const TAPER: f64 = 1.0 / 3.0; -const POREP_MINIMUM_CHALLENGES: usize = 12; // FIXME: 8,000 - -const DRG_SEED: [u32; 7] = [1, 2, 3, 4, 5, 6, 7]; // Arbitrary, need a theory for how to vary this over time. - -fn select_challenges( - partitions: usize, - minimum_total_challenges: usize, - layers: usize, - taper_layers: usize, - taper: f64, -) -> LayerChallenges { - let mut count = 1; - let mut guess = LayerChallenges::Tapered { - count, - layers, - taper, - taper_layers, + let public_inputs = layered_drgporep::PublicInputs::<::Domain> { + replica_id, + tau: Some(Tau { + comm_r: comm_r.into(), + comm_d: comm_d.into(), + }), + comm_r_star: comm_r_star.into(), + k: None, }; - while partitions * guess.total_challenges() < minimum_total_challenges { - count += 1; - guess = LayerChallenges::Tapered { - count, - layers, - taper, - taper_layers, - }; - } - guess -} -fn setup_params( - sector_bytes: PaddedBytesAmount, - partitions: usize, -) -> layered_drgporep::SetupParams { - let sector_bytes = usize::from(sector_bytes); - - let challenges = select_challenges( - partitions, - POREP_MINIMUM_CHALLENGES, - LAYERS, - TAPER_LAYERS, - TAPER, - ); + let verifying_key = get_zigzag_verifying_key(porep_config)?; - assert!( - sector_bytes % 32 == 0, - "sector_bytes ({}) must be a multiple of 32", - sector_bytes, - ); - let nodes = sector_bytes / 32; - layered_drgporep::SetupParams { - drg: DrgParams { - nodes, - degree: DEGREE, - expansion_degree: EXPANSION_DEGREE, - seed: DRG_SEED, - }, - sloth_iter: SLOTH_ITER, - layer_challenges: challenges, - } -} + info!(FCP_LOG, "got verifying key ({}) while verifying seal", u64::from(sector_bytes); "target" => "params"); -pub fn public_params( - sector_bytes: PaddedBytesAmount, - partitions: usize, -) -> layered_drgporep::PublicParams> { - ZigZagDrgPoRep::::setup(&setup_params(sector_bytes, partitions)).unwrap() + let proof = MultiProof::new_from_reader( + Some(usize::from(PoRepProofPartitions::from(porep_config))), + proof_vec, + &verifying_key, + )?; + + ZigZagCompound::verify( + &compound_public_params, + &public_inputs, + &proof, + &ChallengeRequirements { + minimum_challenges: POREP_MINIMUM_CHALLENGES, + }, + ) + .map_err(Into::into) } -type PostSetupParams = vdf_post::SetupParams; -pub type PostPublicParams = vdf_post::PublicParams; +/// Unseals the sector at `sealed_path` and returns the bytes for a piece +/// whose first (unpadded) byte begins at `offset` and ends at `offset` plus +/// `num_bytes`, inclusive. Note that the entire sector is unsealed each time +/// this function is called. +/// +pub fn get_unsealed_range + AsRef>( + porep_config: PoRepConfig, + sealed_path: T, + output_path: T, + prover_id_in: &FrSafe, + sector_id_in: &FrSafe, + offset: UnpaddedByteIndex, + num_bytes: UnpaddedBytesAmount, +) -> error::Result<(UnpaddedBytesAmount)> { + let prover_id = pad_safe_fr(prover_id_in); + let sector_id = pad_safe_fr(sector_id_in); + let replica_id = replica_id::(prover_id, sector_id); -const POST_CHALLENGE_COUNT: usize = 30; -const POST_EPOCHS: usize = 3; -pub const POST_SECTORS_COUNT: usize = 2; -const POST_VDF_ROUNDS: usize = 1; + let f_in = File::open(sealed_path)?; + let mut data = Vec::new(); + f_in.take(u64::from(PaddedBytesAmount::from(porep_config))) + .read_to_end(&mut data)?; -lazy_static! { - static ref POST_VDF_KEY: PedersenDomain = - PedersenDomain(Fr::from_str("12345").unwrap().into_repr()); -} + let f_out = File::create(output_path)?; + let mut buf_writer = BufWriter::new(f_out); -fn post_setup_params(post_config: PoStConfig) -> PostSetupParams { - let size = PaddedBytesAmount::from(post_config); + let unsealed = ZigZagDrgPoRep::extract_all( + &public_params( + PaddedBytesAmount::from(porep_config), + usize::from(PoRepProofPartitions::from(porep_config)), + ), + &replica_id, + &data, + )?; - vdf_post::SetupParams:: { - challenge_count: POST_CHALLENGE_COUNT, - sector_size: size.into(), - post_epochs: POST_EPOCHS, - setup_params_vdf: vdf_sloth::SetupParams { - key: *POST_VDF_KEY, - rounds: POST_VDF_ROUNDS, - }, - sectors_count: POST_SECTORS_COUNT, - } -} + let written = write_unpadded(&unsealed, &mut buf_writer, offset.into(), num_bytes.into())?; -pub fn post_public_params(post_config: PoStConfig) -> PostPublicParams { - VDFPoSt::::setup(&post_setup_params(post_config)).unwrap() + Ok(UnpaddedBytesAmount(written as u64)) } -fn commitment_from_fr(fr: E::Fr) -> Commitment { - let mut commitment = [0; 32]; - for (i, b) in fr_into_bytes::(&fr).iter().enumerate() { - commitment[i] = *b; - } - commitment -} +fn verify_post_dynamic( + dynamic: VerifyPoStDynamicSectorsCountInput, +) -> error::Result { + let fixed = verify_post_spread_input(dynamic)? + .iter() + .map(verify_post_fixed_sectors_count) + .collect(); -fn pad_safe_fr(unpadded: &FrSafe) -> Fr32Ary { - let mut res = [0; 32]; - res[0..31].copy_from_slice(unpadded); - res + verify_post_collect_output(fixed) } -pub fn generate_post( +fn generate_post_dynamic( dynamic: GeneratePoStDynamicSectorsCountInput, ) -> error::Result { let n = { dynamic.input_parts.len() }; @@ -330,18 +328,7 @@ pub fn generate_post( generate_post_collect_output(n, fixed_output) } -pub fn verify_post( - dynamic: VerifyPoStDynamicSectorsCountInput, -) -> error::Result { - let fixed = verify_post_spread_input(dynamic)? - .iter() - .map(verify_post_fixed_sectors_count) - .collect(); - - verify_post_collect_output(fixed) -} - -pub fn generate_post_fixed_sectors_count( +fn generate_post_fixed_sectors_count( fixed: &GeneratePoStFixedSectorsCountInput, ) -> error::Result { let faults: Vec = Vec::new(); @@ -471,7 +458,6 @@ fn verify_post_fixed_sectors_count( Ok(VerifyPoStFixedSectorsCountOutput { is_valid }) } -type Tree = MerkleTree::Function>; fn make_merkle_tree + AsRef>( sealed_path: T, bytes: PaddedBytesAmount, @@ -483,250 +469,22 @@ fn make_merkle_tree + AsRef>( public_params(bytes, 1).graph.merkle_tree(&data) } -#[derive(Clone, Debug)] -pub struct SealOutput { - pub comm_r: Commitment, - pub comm_r_star: Commitment, - pub comm_d: Commitment, - pub proof: Vec, -} - -/// Minimal support for cleaning (deleting) a file unless it was successfully populated. -struct FileCleanup> { - path: T, - success: bool, -} - -impl<'a, T: AsRef> FileCleanup { - fn new(path: T) -> FileCleanup { - FileCleanup { - path, - success: false, - } - } -} - -impl> Drop for FileCleanup { - fn drop(&mut self) { - if !self.success { - let _ = remove_file(&self.path); - } +fn commitment_from_fr(fr: E::Fr) -> Commitment { + let mut commitment = [0; 32]; + for (i, b) in fr_into_bytes::(&fr).iter().enumerate() { + commitment[i] = *b; } + commitment } -pub fn seal + AsRef>( - porep_config: PoRepConfig, - in_path: T, - out_path: T, - prover_id_in: &FrSafe, - sector_id_in: &FrSafe, -) -> error::Result { - let sector_bytes = usize::from(PaddedBytesAmount::from(porep_config)); - - let mut cleanup = FileCleanup::new(&out_path); - - // Copy unsealed data to output location, where it will be sealed in place. - copy(&in_path, &out_path)?; - let f_data = OpenOptions::new().read(true).write(true).open(&out_path)?; - - // Zero-pad the data to the requested size by extending the underlying file if needed. - f_data.set_len(sector_bytes as u64)?; - - let mut data = unsafe { MmapOptions::new().map_mut(&f_data).unwrap() }; - - // Zero-pad the prover_id to 32 bytes (and therefore Fr32). - let prover_id = pad_safe_fr(prover_id_in); - // Zero-pad the sector_id to 32 bytes (and therefore Fr32). - let sector_id = pad_safe_fr(sector_id_in); - let replica_id = replica_id::(prover_id, sector_id); - - let compound_setup_params = compound_proof::SetupParams { - vanilla_params: &setup_params( - PaddedBytesAmount::from(porep_config), - usize::from(PoRepProofPartitions::from(porep_config)), - ), - engine_params: &(*ENGINE_PARAMS), - partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))), - }; - - let compound_public_params = ZigZagCompound::setup(&compound_setup_params)?; - - let (tau, aux) = ZigZagDrgPoRep::replicate( - &compound_public_params.vanilla_params, - &replica_id, - &mut data, - None, - )?; - - // If we succeeded in replicating, flush the data and protect output from being cleaned up. - data.flush()?; - cleanup.success = true; - - let public_tau = tau.simplify(); - - let public_inputs = layered_drgporep::PublicInputs { - replica_id, - tau: Some(public_tau), - comm_r_star: tau.comm_r_star, - k: None, - }; - - let private_inputs = layered_drgporep::PrivateInputs:: { - aux, - tau: tau.layer_taus, - }; - - let groth_params = get_zigzag_params(porep_config)?; - - info!(FCP_LOG, "got groth params ({}) while sealing", u64::from(PaddedBytesAmount::from(porep_config)); "target" => "params"); - - let proof = ZigZagCompound::prove( - &compound_public_params, - &public_inputs, - &private_inputs, - &groth_params, - )?; - - let mut buf = Vec::with_capacity( - SINGLE_PARTITION_PROOF_LEN * usize::from(PoRepProofPartitions::from(porep_config)), - ); - - proof.write(&mut buf)?; - - let comm_r = commitment_from_fr::(public_tau.comm_r.into()); - let comm_d = commitment_from_fr::(public_tau.comm_d.into()); - let comm_r_star = commitment_from_fr::(tau.comm_r_star.into()); - - // Verification is cheap when parameters are cached, - // and it is never correct to return a proof which does not verify. - verify_seal( - porep_config, - comm_r, - comm_d, - comm_r_star, - prover_id_in, - sector_id_in, - &buf, - ) - .expect("post-seal verification sanity check failed"); - - Ok(SealOutput { - comm_r, - comm_r_star, - comm_d, - proof: buf, - }) -} - -pub fn get_unsealed_range + AsRef>( - porep_config: PoRepConfig, - sealed_path: T, - output_path: T, - prover_id_in: &FrSafe, - sector_id_in: &FrSafe, - offset: UnpaddedByteIndex, - num_bytes: UnpaddedBytesAmount, -) -> error::Result<(UnpaddedBytesAmount)> { - let prover_id = pad_safe_fr(prover_id_in); - let sector_id = pad_safe_fr(sector_id_in); - let replica_id = replica_id::(prover_id, sector_id); - - let f_in = File::open(sealed_path)?; - let mut data = Vec::new(); - f_in.take(u64::from(PaddedBytesAmount::from(porep_config))) - .read_to_end(&mut data)?; - - let f_out = File::create(output_path)?; - let mut buf_writer = BufWriter::new(f_out); - - let unsealed = ZigZagDrgPoRep::extract_all( - &public_params( - PaddedBytesAmount::from(porep_config), - usize::from(PoRepProofPartitions::from(porep_config)), - ), - &replica_id, - &data, - )?; - - let written = write_unpadded(&unsealed, &mut buf_writer, offset.into(), num_bytes.into())?; - - Ok(UnpaddedBytesAmount(written as u64)) -} - -pub fn verify_seal( - porep_config: PoRepConfig, - comm_r: Commitment, - comm_d: Commitment, - comm_r_star: Commitment, - prover_id_in: &FrSafe, - sector_id_in: &FrSafe, - proof_vec: &[u8], -) -> error::Result { - let sector_bytes = PaddedBytesAmount::from(porep_config); - let prover_id = pad_safe_fr(prover_id_in); - let sector_id = pad_safe_fr(sector_id_in); - let replica_id = replica_id::(prover_id, sector_id); - - let comm_r = bytes_into_fr::(&comm_r)?; - let comm_d = bytes_into_fr::(&comm_d)?; - let comm_r_star = bytes_into_fr::(&comm_r_star)?; - - let compound_setup_params = compound_proof::SetupParams { - vanilla_params: &setup_params( - PaddedBytesAmount::from(porep_config), - usize::from(PoRepProofPartitions::from(porep_config)), - ), - engine_params: &(*ENGINE_PARAMS), - partitions: Some(usize::from(PoRepProofPartitions::from(porep_config))), - }; - - let compound_public_params: compound_proof::PublicParams< - '_, - Bls12, - ZigZagDrgPoRep<'_, DefaultTreeHasher>, - > = ZigZagCompound::setup(&compound_setup_params)?; - - let public_inputs = layered_drgporep::PublicInputs::<::Domain> { - replica_id, - tau: Some(Tau { - comm_r: comm_r.into(), - comm_d: comm_d.into(), - }), - comm_r_star: comm_r_star.into(), - k: None, - }; - - let verifying_key = get_zigzag_verifying_key(porep_config)?; - - info!(FCP_LOG, "got verifying key ({}) while verifying seal", u64::from(sector_bytes); "target" => "params"); - - let proof = MultiProof::new_from_reader( - Some(usize::from(PoRepProofPartitions::from(porep_config))), - proof_vec, - &verifying_key, - )?; - - ZigZagCompound::verify( - &compound_public_params, - &public_inputs, - &proof, - &ChallengeRequirements { - minimum_challenges: POREP_MINIMUM_CHALLENGES, - }, - ) - .map_err(Into::into) +fn pad_safe_fr(unpadded: &FrSafe) -> Fr32Ary { + let mut res = [0; 32]; + res[0..31].copy_from_slice(unpadded); + res } #[cfg(test)] mod tests { - use super::*; - - use rand::{thread_rng, Rng}; - use sector_base::api::disk_backed_storage::new_sector_store; - use sector_base::api::disk_backed_storage::TEST_SECTOR_SIZE; - use sector_base::api::sector_class::SectorClass; - use sector_base::api::sector_size::SectorSize; - use sector_base::api::sector_store::SectorStore; use std::fs::create_dir_all; use std::fs::File; use std::io::Read; @@ -734,8 +492,18 @@ mod tests { use std::io::SeekFrom; use std::io::Write; use std::thread; + + use rand::{thread_rng, Rng}; use tempfile::NamedTempFile; + use sector_base::api::disk_backed_storage::new_sector_store; + use sector_base::api::disk_backed_storage::TEST_SECTOR_SIZE; + use sector_base::api::sector_class::SectorClass; + use sector_base::api::sector_size::SectorSize; + use sector_base::api::sector_store::SectorStore; + + use super::*; + const TEST_CLASS: SectorClass = SectorClass( SectorSize(TEST_SECTOR_SIZE), PoRepProofPartitions(2), @@ -924,23 +692,23 @@ mod tests { let comm_rs = vec![comm_r, comm_r]; let challenge_seed = rng.gen(); - let post_output = generate_post(GeneratePoStDynamicSectorsCountInput { - post_config: h.store.proofs_config().post_config(), + let post_output = generate_post( + h.store.proofs_config().post_config(), challenge_seed, - input_parts: vec![ + vec![ (Some(h.sealed_access.clone()), comm_r), (Some(h.sealed_access.clone()), comm_r), ], - }) + ) .expect("PoSt generation failed"); - let result = verify_post(VerifyPoStDynamicSectorsCountInput { - post_config: h.store.proofs_config().post_config(), + let result = verify_post( + h.store.proofs_config().post_config(), comm_rs, challenge_seed, - proofs: post_output.proofs, - faults: post_output.faults, - }) + post_output.proofs, + post_output.faults, + ) .expect("failed to run verify_post"); assert!(result.is_valid, "verification of valid proof failed"); @@ -1185,24 +953,4 @@ mod tests { fn post_verify_test() { post_verify_aux(TEST_CLASS, BytesAmount::Max); } - - #[test] - fn partition_layer_challenges_test() { - let f = |partitions| { - select_challenges( - partitions, - POREP_MINIMUM_CHALLENGES, - LAYERS, - TAPER_LAYERS, - TAPER, - ) - .all_challenges() - }; - // Update to ensure all supported PoRepProofPartitions options are represented here. - assert_eq!(vec![1, 1, 2, 2], f(usize::from(PoRepProofPartitions(2)))); - - assert_eq!(vec![3, 3, 4, 5], f(1)); - assert_eq!(vec![1, 1, 2, 2], f(2)); - assert_eq!(vec![1, 1, 1, 1], f(4)); - } } diff --git a/filecoin-proofs/src/api/sector_builder/errors.rs b/filecoin-proofs/src/sector_builder/errors.rs similarity index 100% rename from filecoin-proofs/src/api/sector_builder/errors.rs rename to filecoin-proofs/src/sector_builder/errors.rs diff --git a/filecoin-proofs/src/api/sector_builder/helpers/add_piece.rs b/filecoin-proofs/src/sector_builder/helpers/add_piece.rs similarity index 95% rename from filecoin-proofs/src/api/sector_builder/helpers/add_piece.rs rename to filecoin-proofs/src/sector_builder/helpers/add_piece.rs index 0d2c35334..47ac54c33 100644 --- a/filecoin-proofs/src/api/sector_builder/helpers/add_piece.rs +++ b/filecoin-proofs/src/sector_builder/helpers/add_piece.rs @@ -2,14 +2,14 @@ use std::fs::File; use std::iter::Iterator; use std::sync::Arc; -use crate::api::sector_builder::errors::*; -use crate::api::sector_builder::metadata::StagedSectorMetadata; -use crate::api::sector_builder::pieces::{ +use crate::error; +use crate::sector_builder::errors::*; +use crate::sector_builder::metadata::StagedSectorMetadata; +use crate::sector_builder::pieces::{ get_aligned_source, get_piece_alignment, sum_piece_bytes_with_alignment, PieceAlignment, }; -use crate::api::sector_builder::state::StagedState; -use crate::api::sector_builder::*; -use crate::error; +use crate::sector_builder::state::StagedState; +use crate::sector_builder::*; use sector_base::api::bytes_amount::UnpaddedBytesAmount; use sector_base::api::sector_store::SectorManager; @@ -133,7 +133,7 @@ fn provision_new_staged_sector( #[cfg(test)] mod tests { use super::*; - use crate::api::sector_builder::metadata::PieceMetadata; + use crate::sector_builder::metadata::PieceMetadata; #[test] fn test_alpha() { diff --git a/filecoin-proofs/src/api/sector_builder/helpers/get_seal_status.rs b/filecoin-proofs/src/sector_builder/helpers/get_seal_status.rs similarity index 84% rename from filecoin-proofs/src/api/sector_builder/helpers/get_seal_status.rs rename to filecoin-proofs/src/sector_builder/helpers/get_seal_status.rs index 0bd5b77db..799f4ce08 100644 --- a/filecoin-proofs/src/api/sector_builder/helpers/get_seal_status.rs +++ b/filecoin-proofs/src/sector_builder/helpers/get_seal_status.rs @@ -1,9 +1,9 @@ -use crate::api::sector_builder::errors::err_unrecov; -use crate::api::sector_builder::metadata::SealStatus; -use crate::api::sector_builder::state::SealedState; -use crate::api::sector_builder::state::StagedState; -use crate::api::sector_builder::SectorId; use crate::error; +use crate::sector_builder::errors::err_unrecov; +use crate::sector_builder::metadata::SealStatus; +use crate::sector_builder::state::SealedState; +use crate::sector_builder::state::StagedState; +use crate::sector_builder::SectorId; pub fn get_seal_status( staged_state: &StagedState, @@ -26,10 +26,10 @@ pub fn get_seal_status( #[cfg(test)] mod tests { use super::*; - use crate::api::sector_builder::metadata::{SealedSectorMetadata, StagedSectorMetadata}; - use crate::api::sector_builder::state::SealedState; - use crate::api::sector_builder::state::SectorBuilderState; - use crate::api::sector_builder::state::StagedState; + use crate::sector_builder::metadata::{SealedSectorMetadata, StagedSectorMetadata}; + use crate::sector_builder::state::SealedState; + use crate::sector_builder::state::SectorBuilderState; + use crate::sector_builder::state::StagedState; use std::collections::HashMap; fn setup() -> SectorBuilderState { diff --git a/filecoin-proofs/src/api/sector_builder/helpers/get_sectors_ready_for_sealing.rs b/filecoin-proofs/src/sector_builder/helpers/get_sectors_ready_for_sealing.rs similarity index 90% rename from filecoin-proofs/src/api/sector_builder/helpers/get_sectors_ready_for_sealing.rs rename to filecoin-proofs/src/sector_builder/helpers/get_sectors_ready_for_sealing.rs index 0563e59cf..9653cefc4 100644 --- a/filecoin-proofs/src/api/sector_builder/helpers/get_sectors_ready_for_sealing.rs +++ b/filecoin-proofs/src/sector_builder/helpers/get_sectors_ready_for_sealing.rs @@ -1,8 +1,8 @@ -use crate::api::sector_builder::metadata::SealStatus; -use crate::api::sector_builder::metadata::StagedSectorMetadata; -use crate::api::sector_builder::pieces::sum_piece_bytes_with_alignment; -use crate::api::sector_builder::state::StagedState; -use crate::api::sector_builder::SectorId; +use crate::sector_builder::metadata::SealStatus; +use crate::sector_builder::metadata::StagedSectorMetadata; +use crate::sector_builder::pieces::sum_piece_bytes_with_alignment; +use crate::sector_builder::state::StagedState; +use crate::sector_builder::SectorId; use itertools::chain; use sector_base::api::bytes_amount::UnpaddedBytesAmount; use std::cmp::Reverse; @@ -38,10 +38,10 @@ pub fn get_sectors_ready_for_sealing( #[cfg(test)] mod tests { use super::*; - use crate::api::sector_builder::metadata::PieceMetadata; - use crate::api::sector_builder::metadata::StagedSectorMetadata; - use crate::api::sector_builder::state::StagedState; - use crate::api::sector_builder::SectorId; + use crate::sector_builder::metadata::PieceMetadata; + use crate::sector_builder::metadata::StagedSectorMetadata; + use crate::sector_builder::state::StagedState; + use crate::sector_builder::SectorId; use std::collections::HashMap; fn make_meta( diff --git a/filecoin-proofs/src/api/sector_builder/helpers/mod.rs b/filecoin-proofs/src/sector_builder/helpers/mod.rs similarity index 100% rename from filecoin-proofs/src/api/sector_builder/helpers/mod.rs rename to filecoin-proofs/src/sector_builder/helpers/mod.rs diff --git a/filecoin-proofs/src/api/sector_builder/helpers/retrieve_piece.rs b/filecoin-proofs/src/sector_builder/helpers/retrieve_piece.rs similarity index 85% rename from filecoin-proofs/src/api/sector_builder/helpers/retrieve_piece.rs rename to filecoin-proofs/src/sector_builder/helpers/retrieve_piece.rs index e15b413cf..56ae0878b 100644 --- a/filecoin-proofs/src/api/sector_builder/helpers/retrieve_piece.rs +++ b/filecoin-proofs/src/sector_builder/helpers/retrieve_piece.rs @@ -1,10 +1,10 @@ -use crate::api::internal; -use crate::api::sector_builder::errors::err_unrecov; -use crate::api::sector_builder::metadata::sector_id_as_bytes; -use crate::api::sector_builder::metadata::SealedSectorMetadata; -use crate::api::sector_builder::pieces::{get_piece_by_key, get_piece_start_byte}; -use crate::api::sector_builder::WrappedSectorStore; use crate::error; +use crate::safe; +use crate::sector_builder::errors::err_unrecov; +use crate::sector_builder::metadata::sector_id_as_bytes; +use crate::sector_builder::metadata::SealedSectorMetadata; +use crate::sector_builder::pieces::{get_piece_by_key, get_piece_start_byte}; +use crate::sector_builder::WrappedSectorStore; use sector_base::api::bytes_amount::UnpaddedBytesAmount; use std::path::PathBuf; use std::sync::Arc; @@ -58,7 +58,7 @@ fn retrieve_piece_aux<'a>( err_unrecov(msg) })?; - let num_bytes_unsealed = internal::get_unsealed_range( + let num_bytes_unsealed = safe::get_unsealed_range( (*sector_store.inner).proofs_config().porep_config(), &PathBuf::from(sealed_sector.sector_access.clone()), &PathBuf::from(staging_sector_access), diff --git a/filecoin-proofs/src/api/sector_builder/helpers/seal.rs b/filecoin-proofs/src/sector_builder/helpers/seal.rs similarity index 78% rename from filecoin-proofs/src/api/sector_builder/helpers/seal.rs rename to filecoin-proofs/src/sector_builder/helpers/seal.rs index e22cf5ff8..0da04645d 100644 --- a/filecoin-proofs/src/api/sector_builder/helpers/seal.rs +++ b/filecoin-proofs/src/sector_builder/helpers/seal.rs @@ -1,10 +1,10 @@ -use crate::api::internal::seal as seal_internal; -use crate::api::internal::SealOutput; -use crate::api::sector_builder::metadata::sector_id_as_bytes; -use crate::api::sector_builder::metadata::SealedSectorMetadata; -use crate::api::sector_builder::metadata::StagedSectorMetadata; -use crate::api::sector_builder::WrappedSectorStore; use crate::error; +use crate::safe::seal as seal_internal; +use crate::safe::SealOutput; +use crate::sector_builder::metadata::sector_id_as_bytes; +use crate::sector_builder::metadata::SealedSectorMetadata; +use crate::sector_builder::metadata::StagedSectorMetadata; +use crate::sector_builder::WrappedSectorStore; use std::path::PathBuf; use std::sync::Arc; diff --git a/filecoin-proofs/src/api/sector_builder/helpers/snapshots.rs b/filecoin-proofs/src/sector_builder/helpers/snapshots.rs similarity index 80% rename from filecoin-proofs/src/api/sector_builder/helpers/snapshots.rs rename to filecoin-proofs/src/sector_builder/helpers/snapshots.rs index 005c8c908..879d47a32 100644 --- a/filecoin-proofs/src/api/sector_builder/helpers/snapshots.rs +++ b/filecoin-proofs/src/sector_builder/helpers/snapshots.rs @@ -1,9 +1,9 @@ use std::sync::Arc; -use crate::api::sector_builder::kv_store::KeyValueStore; -use crate::api::sector_builder::state::*; -use crate::api::sector_builder::WrappedKeyValueStore; use crate::error::Result; +use crate::sector_builder::kv_store::KeyValueStore; +use crate::sector_builder::state::*; +use crate::sector_builder::WrappedKeyValueStore; pub fn load_snapshot( kv_store: &Arc>, @@ -48,13 +48,13 @@ pub fn make_snapshot( #[cfg(test)] mod tests { - use crate::api::sector_builder::helpers::snapshots::*; - use crate::api::sector_builder::kv_store::SledKvs; - use crate::api::sector_builder::metadata::StagedSectorMetadata; - use crate::api::sector_builder::state::SealedState; - use crate::api::sector_builder::state::StagedState; - use crate::api::sector_builder::SectorId; - use crate::api::sector_builder::WrappedKeyValueStore; + use crate::sector_builder::helpers::snapshots::*; + use crate::sector_builder::kv_store::SledKvs; + use crate::sector_builder::metadata::StagedSectorMetadata; + use crate::sector_builder::state::SealedState; + use crate::sector_builder::state::StagedState; + use crate::sector_builder::SectorId; + use crate::sector_builder::WrappedKeyValueStore; use std::collections::HashMap; use std::sync::Arc; use std::sync::Mutex; diff --git a/filecoin-proofs/src/api/sector_builder/kv_store/fs.rs b/filecoin-proofs/src/sector_builder/kv_store/fs.rs similarity index 97% rename from filecoin-proofs/src/api/sector_builder/kv_store/fs.rs rename to filecoin-proofs/src/sector_builder/kv_store/fs.rs index c30b20bf5..3f4ca08d9 100644 --- a/filecoin-proofs/src/api/sector_builder/kv_store/fs.rs +++ b/filecoin-proofs/src/sector_builder/kv_store/fs.rs @@ -1,5 +1,5 @@ -use crate::api::sector_builder::kv_store::KeyValueStore; use crate::error::Result; +use crate::sector_builder::kv_store::KeyValueStore; use blake2b_simd::State as Blake2b; use std::fs::{self, File, OpenOptions}; use std::io::{ErrorKind, Read, Write}; diff --git a/filecoin-proofs/src/api/sector_builder/kv_store/mod.rs b/filecoin-proofs/src/sector_builder/kv_store/mod.rs similarity index 100% rename from filecoin-proofs/src/api/sector_builder/kv_store/mod.rs rename to filecoin-proofs/src/sector_builder/kv_store/mod.rs diff --git a/filecoin-proofs/src/api/sector_builder/kv_store/sled.rs b/filecoin-proofs/src/sector_builder/kv_store/sled.rs similarity index 94% rename from filecoin-proofs/src/api/sector_builder/kv_store/sled.rs rename to filecoin-proofs/src/sector_builder/kv_store/sled.rs index e9ba8c14f..9fdf9119a 100644 --- a/filecoin-proofs/src/api/sector_builder/kv_store/sled.rs +++ b/filecoin-proofs/src/sector_builder/kv_store/sled.rs @@ -2,8 +2,8 @@ use std::path::Path; use sled::Db; -use crate::api::sector_builder::kv_store::KeyValueStore; use crate::error::Result; +use crate::sector_builder::kv_store::KeyValueStore; pub struct SledKvs { db: Db, diff --git a/filecoin-proofs/src/api/sector_builder/metadata.rs b/filecoin-proofs/src/sector_builder/metadata.rs similarity index 98% rename from filecoin-proofs/src/api/sector_builder/metadata.rs rename to filecoin-proofs/src/sector_builder/metadata.rs index 6350d0c2d..1ef14cec2 100644 --- a/filecoin-proofs/src/api/sector_builder/metadata.rs +++ b/filecoin-proofs/src/sector_builder/metadata.rs @@ -1,5 +1,5 @@ -use crate::api::sector_builder::SectorId; use crate::error; +use crate::sector_builder::SectorId; use byteorder::LittleEndian; use byteorder::WriteBytesExt; use sector_base::api::bytes_amount::UnpaddedBytesAmount; diff --git a/filecoin-proofs/src/api/sector_builder/mod.rs b/filecoin-proofs/src/sector_builder/mod.rs similarity index 95% rename from filecoin-proofs/src/api/sector_builder/mod.rs rename to filecoin-proofs/src/sector_builder/mod.rs index e30d97753..470060805 100644 --- a/filecoin-proofs/src/api/sector_builder/mod.rs +++ b/filecoin-proofs/src/sector_builder/mod.rs @@ -1,16 +1,16 @@ use slog::*; use std::sync::{mpsc, Arc, Mutex}; -use crate::api::post_adapter::*; -use crate::api::sector_builder::errors::SectorBuilderErr; -use crate::api::sector_builder::kv_store::{KeyValueStore, SledKvs}; -use crate::api::sector_builder::metadata::*; -use crate::api::sector_builder::scheduler::Request; -use crate::api::sector_builder::scheduler::Scheduler; -use crate::api::sector_builder::sealer::*; use crate::error::ExpectWithBacktrace; use crate::error::Result; -use crate::FCP_LOG; +use crate::post_adapter::*; +use crate::sector_builder::errors::SectorBuilderErr; +use crate::sector_builder::kv_store::{KeyValueStore, SledKvs}; +use crate::sector_builder::metadata::*; +use crate::sector_builder::scheduler::Request; +use crate::sector_builder::scheduler::Scheduler; +use crate::sector_builder::sealer::*; +use crate::singletons::FCP_LOG; use sector_base::api::disk_backed_storage::new_sector_store; use sector_base::api::sector_class::SectorClass; use sector_base::api::sector_store::SectorStore; diff --git a/filecoin-proofs/src/api/sector_builder/pieces.rs b/filecoin-proofs/src/sector_builder/pieces.rs similarity index 99% rename from filecoin-proofs/src/api/sector_builder/pieces.rs rename to filecoin-proofs/src/sector_builder/pieces.rs index 5a99244f1..4484fb740 100644 --- a/filecoin-proofs/src/api/sector_builder/pieces.rs +++ b/filecoin-proofs/src/sector_builder/pieces.rs @@ -1,4 +1,4 @@ -use crate::api::sector_builder::metadata::PieceMetadata; +use crate::sector_builder::metadata::PieceMetadata; use sector_base::api::bytes_amount::{UnpaddedByteIndex, UnpaddedBytesAmount}; use std::io::Cursor; use std::io::Read; diff --git a/filecoin-proofs/src/api/sector_builder/scheduler.rs b/filecoin-proofs/src/sector_builder/scheduler.rs similarity index 89% rename from filecoin-proofs/src/api/sector_builder/scheduler.rs rename to filecoin-proofs/src/sector_builder/scheduler.rs index 01d1c35f8..63c0de97b 100644 --- a/filecoin-proofs/src/api/sector_builder/scheduler.rs +++ b/filecoin-proofs/src/sector_builder/scheduler.rs @@ -1,24 +1,24 @@ -use crate::api::internal; -use crate::api::post_adapter::*; -use crate::api::sector_builder::errors::err_piecenotfound; -use crate::api::sector_builder::errors::err_unrecov; -use crate::api::sector_builder::helpers::add_piece::add_piece; -use crate::api::sector_builder::helpers::get_seal_status::get_seal_status; -use crate::api::sector_builder::helpers::get_sectors_ready_for_sealing::get_sectors_ready_for_sealing; -use crate::api::sector_builder::helpers::snapshots::load_snapshot; -use crate::api::sector_builder::helpers::snapshots::make_snapshot; -use crate::api::sector_builder::helpers::snapshots::persist_snapshot; -use crate::api::sector_builder::kv_store::KeyValueStore; -use crate::api::sector_builder::metadata::SealStatus; -use crate::api::sector_builder::metadata::SealedSectorMetadata; -use crate::api::sector_builder::metadata::StagedSectorMetadata; -use crate::api::sector_builder::sealer::SealerInput; -use crate::api::sector_builder::state::SectorBuilderState; -use crate::api::sector_builder::state::StagedState; -use crate::api::sector_builder::SectorId; -use crate::api::sector_builder::{WrappedKeyValueStore, WrappedSectorStore}; use crate::error::ExpectWithBacktrace; use crate::error::Result; +use crate::post_adapter::*; +use crate::safe; +use crate::sector_builder::errors::err_piecenotfound; +use crate::sector_builder::errors::err_unrecov; +use crate::sector_builder::helpers::add_piece::add_piece; +use crate::sector_builder::helpers::get_seal_status::get_seal_status; +use crate::sector_builder::helpers::get_sectors_ready_for_sealing::get_sectors_ready_for_sealing; +use crate::sector_builder::helpers::snapshots::load_snapshot; +use crate::sector_builder::helpers::snapshots::make_snapshot; +use crate::sector_builder::helpers::snapshots::persist_snapshot; +use crate::sector_builder::kv_store::KeyValueStore; +use crate::sector_builder::metadata::SealStatus; +use crate::sector_builder::metadata::SealedSectorMetadata; +use crate::sector_builder::metadata::StagedSectorMetadata; +use crate::sector_builder::sealer::SealerInput; +use crate::sector_builder::state::SectorBuilderState; +use crate::sector_builder::state::StagedState; +use crate::sector_builder::SectorId; +use crate::sector_builder::{WrappedKeyValueStore, WrappedSectorStore}; use sector_base::api::bytes_amount::UnpaddedBytesAmount; use std::collections::HashMap; @@ -186,11 +186,11 @@ impl SectorMetadataManager { let mut seed = [0; 32]; seed.copy_from_slice(challenge_seed); - let output = internal::generate_post(GeneratePoStDynamicSectorsCountInput { - post_config: self.sector_store.inner.proofs_config().post_config(), - challenge_seed: seed, + let output = safe::generate_post( + self.sector_store.inner.proofs_config().post_config(), + seed, input_parts, - }); + ); // TODO: Where should this work be scheduled? New worker type? return_channel.send(output).expects(FATAL_HUNGUP); diff --git a/filecoin-proofs/src/api/sector_builder/sealer.rs b/filecoin-proofs/src/sector_builder/sealer.rs similarity index 86% rename from filecoin-proofs/src/api/sector_builder/sealer.rs rename to filecoin-proofs/src/sector_builder/sealer.rs index f5f3def55..5892a2c75 100644 --- a/filecoin-proofs/src/api/sector_builder/sealer.rs +++ b/filecoin-proofs/src/sector_builder/sealer.rs @@ -1,11 +1,11 @@ -use crate::api::sector_builder::helpers::retrieve_piece::retrieve_piece; -use crate::api::sector_builder::helpers::seal::seal; -use crate::api::sector_builder::metadata::SealedSectorMetadata; -use crate::api::sector_builder::metadata::StagedSectorMetadata; -use crate::api::sector_builder::scheduler::Request; -use crate::api::sector_builder::WrappedSectorStore; use crate::error::ExpectWithBacktrace; use crate::error::Result; +use crate::sector_builder::helpers::retrieve_piece::retrieve_piece; +use crate::sector_builder::helpers::seal::seal; +use crate::sector_builder::metadata::SealedSectorMetadata; +use crate::sector_builder::metadata::StagedSectorMetadata; +use crate::sector_builder::scheduler::Request; +use crate::sector_builder::WrappedSectorStore; use std::sync::mpsc; use std::sync::{Arc, Mutex}; use std::thread; diff --git a/filecoin-proofs/src/api/sector_builder/state.rs b/filecoin-proofs/src/sector_builder/state.rs similarity index 87% rename from filecoin-proofs/src/api/sector_builder/state.rs rename to filecoin-proofs/src/sector_builder/state.rs index cfccc6429..57955dce2 100644 --- a/filecoin-proofs/src/api/sector_builder/state.rs +++ b/filecoin-proofs/src/sector_builder/state.rs @@ -1,5 +1,5 @@ -use crate::api::sector_builder::metadata::{SealedSectorMetadata, StagedSectorMetadata}; -use crate::api::sector_builder::SectorId; +use crate::sector_builder::metadata::{SealedSectorMetadata, StagedSectorMetadata}; +use crate::sector_builder::SectorId; use std::collections::HashMap; #[derive(Default, Serialize, Deserialize, Debug, PartialEq)] diff --git a/filecoin-proofs/src/singletons.rs b/filecoin-proofs/src/singletons.rs new file mode 100644 index 000000000..6670adf2e --- /dev/null +++ b/filecoin-proofs/src/singletons.rs @@ -0,0 +1,20 @@ +use ff::PrimeField; +use fil_sapling_crypto::jubjub::JubjubBls12; +use paired::bls12_381::Fr; +use slog::Logger; + +use logging_toolkit::make_logger; +use storage_proofs::hasher::pedersen::PedersenDomain; + +lazy_static! { + pub static ref ENGINE_PARAMS: JubjubBls12 = JubjubBls12::new(); +} + +lazy_static! { + pub static ref POST_VDF_KEY: PedersenDomain = + PedersenDomain(Fr::from_str("12345").unwrap().into_repr()); +} + +lazy_static! { + pub static ref FCP_LOG: Logger = make_logger("filecoin-proofs"); +}