From b90a38671effad5c78f7f142c5899c8dcc40dd71 Mon Sep 17 00:00:00 2001 From: Georg Wiese Date: Wed, 10 Jul 2024 11:58:57 +0200 Subject: [PATCH] Move VariablySizedColumns to executor crate --- backend/src/composite/mod.rs | 7 ++- backend/src/estark/mod.rs | 7 ++- backend/src/estark/polygon_wrapper.rs | 7 ++- backend/src/estark/starky_wrapper.rs | 7 +-- backend/src/halo2/mod.rs | 3 +- backend/src/lib.rs | 4 +- backend/src/plonky3/mod.rs | 7 +-- executor/Cargo.toml | 1 + .../src/constant_evaluator/data_structures.rs | 48 ++++++++++++++++++ executor/src/constant_evaluator/mod.rs | 8 ++- executor/src/witgen/block_processor.rs | 4 +- executor/src/witgen/global_constraints.rs | 4 +- executor/src/witgen/mod.rs | 4 +- number/src/lib.rs | 50 ------------------- pipeline/src/pipeline.rs | 7 +-- pipeline/src/util.rs | 3 +- pipeline/tests/asm.rs | 3 +- plonky3/src/stark.rs | 3 +- 18 files changed, 98 insertions(+), 79 deletions(-) create mode 100644 executor/src/constant_evaluator/data_structures.rs diff --git a/backend/src/composite/mod.rs b/backend/src/composite/mod.rs index 34a255289..51013c63f 100644 --- a/backend/src/composite/mod.rs +++ b/backend/src/composite/mod.rs @@ -1,8 +1,11 @@ use std::{collections::BTreeMap, io, marker::PhantomData, path::PathBuf, sync::Arc}; use powdr_ast::analyzed::Analyzed; -use powdr_executor::witgen::WitgenCallback; -use powdr_number::{get_only_size_cloned, DegreeType, FieldElement, VariablySizedColumn}; +use powdr_executor::{ + constant_evaluator::{get_only_size_cloned, VariablySizedColumn}, + witgen::WitgenCallback, +}; +use powdr_number::{DegreeType, FieldElement}; use serde::{Deserialize, Serialize}; use split::select_machine_columns; diff --git a/backend/src/estark/mod.rs b/backend/src/estark/mod.rs index cfa71ed02..b772a2ffd 100644 --- a/backend/src/estark/mod.rs +++ b/backend/src/estark/mod.rs @@ -15,8 +15,11 @@ use std::{ use crate::{Backend, BackendFactory, BackendOptions, Error, Proof}; use powdr_ast::analyzed::Analyzed; -use powdr_executor::witgen::WitgenCallback; -use powdr_number::{get_only_size_cloned, DegreeType, FieldElement, VariablySizedColumn}; +use powdr_executor::{ + constant_evaluator::{get_only_size_cloned, VariablySizedColumn}, + witgen::WitgenCallback, +}; +use powdr_number::{DegreeType, FieldElement}; use serde::Serialize; use starky::types::{StarkStruct, Step, PIL}; diff --git a/backend/src/estark/polygon_wrapper.rs b/backend/src/estark/polygon_wrapper.rs index a2635e031..78d4ac32a 100644 --- a/backend/src/estark/polygon_wrapper.rs +++ b/backend/src/estark/polygon_wrapper.rs @@ -1,8 +1,11 @@ use std::{fs, path::PathBuf, sync::Arc}; use powdr_ast::analyzed::Analyzed; -use powdr_executor::witgen::WitgenCallback; -use powdr_number::{get_only_size_cloned, FieldElement, VariablySizedColumn}; +use powdr_executor::{ + constant_evaluator::{get_only_size_cloned, VariablySizedColumn}, + witgen::WitgenCallback, +}; +use powdr_number::FieldElement; use crate::{Backend, BackendFactory, BackendOptions, Error, Proof}; diff --git a/backend/src/estark/starky_wrapper.rs b/backend/src/estark/starky_wrapper.rs index e7ba4a83b..c98532236 100644 --- a/backend/src/estark/starky_wrapper.rs +++ b/backend/src/estark/starky_wrapper.rs @@ -5,10 +5,11 @@ use std::time::Instant; use crate::{Backend, BackendFactory, BackendOptions, Error}; use powdr_ast::analyzed::Analyzed; -use powdr_executor::witgen::WitgenCallback; -use powdr_number::{ - get_only_size_cloned, FieldElement, GoldilocksField, LargeInt, VariablySizedColumn, +use powdr_executor::{ + constant_evaluator::{get_only_size_cloned, VariablySizedColumn}, + witgen::WitgenCallback, }; +use powdr_number::{FieldElement, GoldilocksField, LargeInt}; use starky::{ merklehash::MerkleTreeGL, diff --git a/backend/src/halo2/mod.rs b/backend/src/halo2/mod.rs index 7bef444a7..28ec9a726 100644 --- a/backend/src/halo2/mod.rs +++ b/backend/src/halo2/mod.rs @@ -6,8 +6,9 @@ use std::sync::Arc; use crate::{Backend, BackendFactory, BackendOptions, Error, Proof}; use powdr_ast::analyzed::Analyzed; +use powdr_executor::constant_evaluator::{get_only_size_cloned, VariablySizedColumn}; use powdr_executor::witgen::WitgenCallback; -use powdr_number::{get_only_size_cloned, DegreeType, FieldElement, VariablySizedColumn}; +use powdr_number::{DegreeType, FieldElement}; use prover::{generate_setup, Halo2Prover}; use serde::de::{self, Deserializer}; diff --git a/backend/src/lib.rs b/backend/src/lib.rs index 28d270dc2..f5ca148ea 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -9,8 +9,8 @@ mod plonky3; mod composite; use powdr_ast::analyzed::Analyzed; -use powdr_executor::witgen::WitgenCallback; -use powdr_number::{DegreeType, FieldElement, VariablySizedColumn}; +use powdr_executor::{constant_evaluator::VariablySizedColumn, witgen::WitgenCallback}; +use powdr_number::{DegreeType, FieldElement}; use std::{io, path::PathBuf, sync::Arc}; use strum::{Display, EnumString, EnumVariantNames}; diff --git a/backend/src/plonky3/mod.rs b/backend/src/plonky3/mod.rs index fabaf2113..60e6cb619 100644 --- a/backend/src/plonky3/mod.rs +++ b/backend/src/plonky3/mod.rs @@ -1,10 +1,11 @@ use std::{io, path::PathBuf, sync::Arc}; use powdr_ast::analyzed::Analyzed; -use powdr_executor::witgen::WitgenCallback; -use powdr_number::{ - get_only_size_cloned, FieldElement, GoldilocksField, LargeInt, VariablySizedColumn, +use powdr_executor::{ + constant_evaluator::{get_only_size_cloned, VariablySizedColumn}, + witgen::WitgenCallback, }; +use powdr_number::{FieldElement, GoldilocksField, LargeInt}; use powdr_plonky3::Plonky3Prover; use crate::{Backend, BackendFactory, BackendOptions, Error, Proof}; diff --git a/executor/Cargo.toml b/executor/Cargo.toml index 83f2f9219..0079cce9a 100644 --- a/executor/Cargo.toml +++ b/executor/Cargo.toml @@ -20,6 +20,7 @@ bit-vec = "0.6.3" num-traits = "0.2.15" lazy_static = "1.4.0" indicatif = "0.17.7" +serde = { version = "1.0", default-features = false, features = ["alloc", "derive", "rc"] } [dev-dependencies] test-log = "0.2.12" diff --git a/executor/src/constant_evaluator/data_structures.rs b/executor/src/constant_evaluator/data_structures.rs new file mode 100644 index 000000000..2e0a7a958 --- /dev/null +++ b/executor/src/constant_evaluator/data_structures.rs @@ -0,0 +1,48 @@ +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Serialize, Deserialize)] +pub struct VariablySizedColumn { + column_by_size: BTreeMap>, +} + +#[derive(Debug)] +pub struct HasMultipleSizesError; + +impl VariablySizedColumn { + /// Create a view where each column has a single size. Fails if any column has multiple sizes. + pub fn get_only_size(&self) -> Result<&Vec, HasMultipleSizesError> { + if self.column_by_size.len() != 1 { + return Err(HasMultipleSizesError); + } + Ok(self.column_by_size.values().next().unwrap()) + } +} + +pub fn get_only_size( + column: &[(String, VariablySizedColumn)], +) -> Result)>, HasMultipleSizesError> { + column + .iter() + .map(|(name, column)| Ok((name.clone(), column.get_only_size()?))) + .collect() +} + +pub fn get_only_size_cloned( + column: &[(String, VariablySizedColumn)], +) -> Result)>, HasMultipleSizesError> { + get_only_size(column).map(|column| { + column + .into_iter() + .map(|(name, column)| (name, column.clone())) + .collect() + }) +} + +impl From> for VariablySizedColumn { + fn from(column: Vec) -> Self { + VariablySizedColumn { + column_by_size: [(column.len(), column)].into_iter().collect(), + } + } +} diff --git a/executor/src/constant_evaluator/mod.rs b/executor/src/constant_evaluator/mod.rs index 45003b0e1..1faf28cc8 100644 --- a/executor/src/constant_evaluator/mod.rs +++ b/executor/src/constant_evaluator/mod.rs @@ -3,6 +3,7 @@ use std::{ sync::{Arc, RwLock}, }; +pub use data_structures::{get_only_size, get_only_size_cloned, VariablySizedColumn}; use itertools::Itertools; use powdr_ast::{ analyzed::{Analyzed, FunctionValueDefinition, Symbol, TypedExpression}, @@ -11,10 +12,12 @@ use powdr_ast::{ IndexAccess, }, }; -use powdr_number::{BigInt, BigUint, DegreeType, FieldElement, VariablySizedColumn}; +use powdr_number::{BigInt, BigUint, DegreeType, FieldElement}; use powdr_pil_analyzer::evaluator::{self, Definitions, SymbolLookup, Value}; use rayon::prelude::{IntoParallelIterator, ParallelIterator}; +mod data_structures; + /// Generates the fixed column values for all fixed columns that are defined /// (and not just declared). /// @returns the names (in source order) and the values for the columns. @@ -169,7 +172,8 @@ impl<'a, T: FieldElement> SymbolLookup<'a, T> for CachedSymbols<'a, T> { #[cfg(test)] mod test { - use powdr_number::{get_only_size, GoldilocksField}; + use data_structures::get_only_size; + use powdr_number::GoldilocksField; use powdr_pil_analyzer::analyze_string; use pretty_assertions::assert_eq; use test_log::test; diff --git a/executor/src/witgen/block_processor.rs b/executor/src/witgen/block_processor.rs index 222df650b..2803f99c2 100644 --- a/executor/src/witgen/block_processor.rs +++ b/executor/src/witgen/block_processor.rs @@ -117,11 +117,11 @@ mod tests { use std::collections::BTreeMap; use powdr_ast::analyzed::{PolyID, PolynomialType}; - use powdr_number::{get_only_size, FieldElement, GoldilocksField}; + use powdr_number::{FieldElement, GoldilocksField}; use powdr_pil_analyzer::analyze_string; use crate::{ - constant_evaluator::generate, + constant_evaluator::{generate, get_only_size}, witgen::{ data_structures::finalizable_data::FinalizableData, identity_processor::Machines, diff --git a/executor/src/witgen/global_constraints.rs b/executor/src/witgen/global_constraints.rs index 0104e23b6..22ed5582d 100644 --- a/executor/src/witgen/global_constraints.rs +++ b/executor/src/witgen/global_constraints.rs @@ -361,10 +361,12 @@ mod test { use std::collections::BTreeMap; use powdr_ast::analyzed::{PolyID, PolynomialType}; - use powdr_number::{get_only_size, GoldilocksField}; + use powdr_number::GoldilocksField; use pretty_assertions::assert_eq; use test_log::test; + use crate::constant_evaluator::get_only_size; + use super::*; #[test] diff --git a/executor/src/witgen/mod.rs b/executor/src/witgen/mod.rs index b314be967..327b131fa 100644 --- a/executor/src/witgen/mod.rs +++ b/executor/src/witgen/mod.rs @@ -8,7 +8,9 @@ use powdr_ast::analyzed::{ }; use powdr_ast::parsed::visitor::ExpressionVisitable; use powdr_ast::parsed::{FunctionKind, LambdaExpression}; -use powdr_number::{get_only_size, DegreeType, FieldElement, VariablySizedColumn}; +use powdr_number::{DegreeType, FieldElement}; + +use crate::constant_evaluator::{get_only_size, VariablySizedColumn}; use self::data_structures::column_map::{FixedColumnMap, WitnessColumnMap}; pub use self::eval_result::{ diff --git a/number/src/lib.rs b/number/src/lib.rs index c6b0bf61f..8842281c6 100644 --- a/number/src/lib.rs +++ b/number/src/lib.rs @@ -8,10 +8,6 @@ mod bn254; mod goldilocks; mod serialize; mod traits; - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; pub use serialize::{ buffered_write_file, read_polys_csv_file, write_polys_csv_file, CsvRenderMode, ReadWrite, }; @@ -33,52 +29,6 @@ pub fn log2_exact(n: BigUint) -> Option { .filter(|zeros| n == (BigUint::from(1u32) << zeros)) } -#[derive(Serialize, Deserialize)] -pub struct VariablySizedColumn { - column_by_size: BTreeMap>, -} - -#[derive(Debug)] -pub struct HasMultipleSizesError; - -impl VariablySizedColumn { - /// Create a view where each column has a single size. Fails if any column has multiple sizes. - pub fn get_only_size(&self) -> Result<&Vec, HasMultipleSizesError> { - if self.column_by_size.len() != 1 { - return Err(HasMultipleSizesError); - } - Ok(self.column_by_size.values().next().unwrap()) - } -} - -pub fn get_only_size( - column: &[(String, VariablySizedColumn)], -) -> Result)>, HasMultipleSizesError> { - column - .iter() - .map(|(name, column)| Ok((name.clone(), column.get_only_size()?))) - .collect() -} - -pub fn get_only_size_cloned( - column: &[(String, VariablySizedColumn)], -) -> Result)>, HasMultipleSizesError> { - get_only_size(column).map(|column| { - column - .into_iter() - .map(|(name, column)| (name, column.clone())) - .collect() - }) -} - -impl From> for VariablySizedColumn { - fn from(column: Vec) -> Self { - VariablySizedColumn { - column_by_size: [(column.len(), column)].into_iter().collect(), - } - } -} - #[cfg(test)] mod test { use super::*; diff --git a/pipeline/src/pipeline.rs b/pipeline/src/pipeline.rs index 83fb902f4..6387295c9 100644 --- a/pipeline/src/pipeline.rs +++ b/pipeline/src/pipeline.rs @@ -18,16 +18,13 @@ use powdr_ast::{ }; use powdr_backend::{BackendOptions, BackendType, Proof}; use powdr_executor::{ - constant_evaluator, + constant_evaluator::{self, get_only_size_cloned, VariablySizedColumn}, witgen::{ chain_callbacks, extract_publics, unused_query_callback, QueryCallback, WitgenCallback, WitnessGenerator, }, }; -use powdr_number::{ - get_only_size_cloned, write_polys_csv_file, CsvRenderMode, FieldElement, ReadWrite, - VariablySizedColumn, -}; +use powdr_number::{write_polys_csv_file, CsvRenderMode, FieldElement, ReadWrite}; use powdr_schemas::SerializedAnalyzed; use crate::{ diff --git a/pipeline/src/util.rs b/pipeline/src/util.rs index 3564929eb..c6415f387 100644 --- a/pipeline/src/util.rs +++ b/pipeline/src/util.rs @@ -1,5 +1,6 @@ use powdr_ast::analyzed::{Analyzed, FunctionValueDefinition, Symbol}; -use powdr_number::{ReadWrite, VariablySizedColumn}; +use powdr_executor::constant_evaluator::VariablySizedColumn; +use powdr_number::ReadWrite; use serde::{de::DeserializeOwned, Serialize}; use std::{fs::File, io::BufReader, marker::PhantomData, path::Path}; diff --git a/pipeline/tests/asm.rs b/pipeline/tests/asm.rs index bd8c24890..1a716c0b5 100644 --- a/pipeline/tests/asm.rs +++ b/pipeline/tests/asm.rs @@ -1,5 +1,6 @@ use powdr_backend::BackendType; -use powdr_number::{get_only_size, Bn254Field, FieldElement, GoldilocksField}; +use powdr_executor::constant_evaluator::get_only_size; +use powdr_number::{Bn254Field, FieldElement, GoldilocksField}; use powdr_pipeline::{ test_util::{gen_estark_proof, resolve_test_file, test_halo2, verify_test_file}, util::{FixedPolySet, PolySet, WitnessPolySet}, diff --git a/plonky3/src/stark.rs b/plonky3/src/stark.rs index f04ed87c5..695bdcf0f 100644 --- a/plonky3/src/stark.rs +++ b/plonky3/src/stark.rs @@ -205,7 +205,8 @@ impl Plonky3Prover { mod tests { use std::sync::Arc; - use powdr_number::{get_only_size_cloned, GoldilocksField}; + use powdr_executor::constant_evaluator::get_only_size_cloned; + use powdr_number::GoldilocksField; use powdr_pipeline::Pipeline; use test_log::test;