From 259c2c85991dc3eb1af118ec89a4aa453e71e7c5 Mon Sep 17 00:00:00 2001 From: ok <59588824+jku20@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:29:32 -0400 Subject: [PATCH] [fud2] update path finding to support multi state input and output (#2134) This PR begins progress towards addressing #1958. It updates the algorithm used by `fud2` for finding paths from input files to output files to take in operations which have many inputs and outputs. The behavior concurs with the current `fud2` in some but not all cases and does not maintain `fud2`'s always finding the shortest path of operations. Specifying operations themselves and emitting the Ninja file still assume single state inputs and outputs. This will be changed in further PRs. --- fud2/fud-core/src/cli.rs | 107 ++++-- fud2/fud-core/src/exec/data.rs | 6 +- fud2/fud-core/src/exec/driver.rs | 341 ++++++++++-------- fud2/fud-core/src/exec/mod.rs | 4 +- fud2/fud-core/src/exec/planner.rs | 171 +++++++++ fud2/fud-core/src/exec/request.rs | 15 +- fud2/fud-core/src/run.rs | 128 ++++--- fud2/fud-core/src/script/exec_scripts.rs | 10 +- fud2/fud-core/src/script/plugin.rs | 9 +- fud2/fud-core/tests/tests.rs | 1 + fud2/src/lib.rs | 97 ++--- .../snapshots/tests__test@calyx_debug.snap | 4 +- ...sts__test@calyx_firrtl_verilog-refmem.snap | 12 +- .../tests__test@calyx_icarus_dat.snap | 12 +- .../tests__test@calyx_icarus_vcd.snap | 12 +- .../tests__test@calyx_interp_dat.snap | 6 +- .../tests__test@calyx_verilator_dat.snap | 14 +- .../tests__test@calyx_verilator_vcd.snap | 14 +- .../snapshots/tests__test@calyx_verilog.snap | 4 +- .../tests__test@calyx_xrt-trace_vcd.snap | 14 +- .../snapshots/tests__test@calyx_xrt_dat.snap | 14 +- .../snapshots/tests__test@dahlia_calyx.snap | 4 +- .../snapshots/tests__test@mrxl_calyx.snap | 4 +- fud2/tests/tests.rs | 38 +- 24 files changed, 673 insertions(+), 368 deletions(-) create mode 100644 fud2/fud-core/src/exec/planner.rs create mode 100644 fud2/fud-core/tests/tests.rs diff --git a/fud2/fud-core/src/cli.rs b/fud2/fud-core/src/cli.rs index 2a3de2fd64..ef64f808da 100644 --- a/fud2/fud-core/src/cli.rs +++ b/fud2/fud-core/src/cli.rs @@ -1,5 +1,7 @@ use crate::config; -use crate::exec::{Driver, Request, StateRef}; +use crate::exec::{ + Driver, EnumeratePlanner, Request, SingleOpOutputPlanner, StateRef, +}; use crate::run::Run; use anyhow::{anyhow, bail}; use argh::FromArgs; @@ -91,19 +93,23 @@ struct FakeArgs { /// the input file #[argh(positional)] - input: Option, + input: Vec, /// the output file #[argh(option, short = 'o')] - output: Option, + output: Vec, - /// the state to start from + /// the states to start from. + /// The ith state is applied to the ith input file. + /// If more states are specified than files, files for these states are read from stdin. #[argh(option)] - from: Option, + from: Vec, /// the state to produce + /// The ith state is applied to the ith output file. + /// If more states are specified than files, files for these states are written to stdout #[argh(option)] - to: Option, + to: Vec, /// execution mode (run, plan, emit, gen, dot) #[argh(option, short = 'm', default = "Mode::Run")] @@ -132,34 +138,63 @@ struct FakeArgs { /// log level for debugging fud internal #[argh(option, long = "log", default = "log::LevelFilter::Warn")] pub log_level: log::LevelFilter, + + /// use new enumeration algorithm for finding operation sequences + #[argh(switch)] + planner: bool, } -fn from_state(driver: &Driver, args: &FakeArgs) -> anyhow::Result { - match &args.from { - Some(name) => driver - .get_state(name) - .ok_or(anyhow!("unknown --from state")), - None => match args.input { - Some(ref input) => driver - .guess_state(input) - .ok_or(anyhow!("could not infer input state")), - None => bail!("specify an input file or use --from"), - }, +fn get_states_with_errors( + driver: &Driver, + explicit_states: &[String], + files: &[Utf8PathBuf], + unknown_state: &str, + uninferable_file: &str, + no_states: &str, +) -> anyhow::Result> { + let explicit_states = explicit_states.iter().map(|state_str| { + driver + .get_state(state_str) + .ok_or(anyhow!("{unknown_state}")) + }); + let inferred_states = + files.iter().skip(explicit_states.len()).map(|input_str| { + driver + .guess_state(input_str) + .ok_or(anyhow!("{uninferable_file}")) + }); + let states = explicit_states + .chain(inferred_states) + .collect::, _>>()?; + if states.is_empty() { + bail!("{no_states}"); } + Ok(states) } -fn to_state(driver: &Driver, args: &FakeArgs) -> anyhow::Result { - match &args.to { - Some(name) => { - driver.get_state(name).ok_or(anyhow!("unknown --to state")) - } - None => match &args.output { - Some(out) => driver - .guess_state(out) - .ok_or(anyhow!("could not infer output state")), - None => Err(anyhow!("specify an output file or use --to")), - }, - } +fn from_states( + driver: &Driver, + args: &FakeArgs, +) -> anyhow::Result> { + get_states_with_errors( + driver, + &args.from, + &args.input, + "unknown --from state", + "could not infer input state", + "specify and input file or use --from", + ) +} + +fn to_state(driver: &Driver, args: &FakeArgs) -> anyhow::Result> { + get_states_with_errors( + driver, + &args.to, + &args.output, + "unknown --to state", + "could no infer output state", + "specify an output file or use --to", + ) } fn get_request(driver: &Driver, args: &FakeArgs) -> anyhow::Result { @@ -185,14 +220,18 @@ fn get_request(driver: &Driver, args: &FakeArgs) -> anyhow::Result { .ok_or(anyhow!("unknown --through op {}", s)) }) .collect(); - Ok(Request { - start_file: args.input.clone(), - start_state: from_state(driver, args)?, - end_file: args.output.clone(), - end_state: to_state(driver, args)?, + start_files: args.input.clone(), + start_states: from_states(driver, args)?, + end_files: args.output.clone(), + end_states: to_state(driver, args)?, through: through?, workdir, + planner: if args.planner { + Box::new(EnumeratePlanner {}) + } else { + Box::new(SingleOpOutputPlanner {}) + }, }) } diff --git a/fud2/fud-core/src/exec/data.rs b/fud2/fud-core/src/exec/data.rs index 8dbda63860..22e7193ef7 100644 --- a/fud2/fud-core/src/exec/data.rs +++ b/fud2/fud-core/src/exec/data.rs @@ -24,7 +24,7 @@ impl State { self.extensions.iter().any(|e| e == ext) } - /// Is this a "pseudo-state": doesn't correspond to an actual file, and must be an output state? + /// Is this a "pseudo-state": doesn't correspond to an actual file pub fn is_pseudo(&self) -> bool { self.extensions.is_empty() } @@ -38,8 +38,8 @@ entity_impl!(StateRef, "state"); /// An Operation transforms files from one State to another. pub struct Operation { pub name: String, - pub input: StateRef, - pub output: StateRef, + pub input: Vec, + pub output: Vec, pub setups: Vec, pub emit: Box, /// Describes where this operation was defined. diff --git a/fud2/fud-core/src/exec/driver.rs b/fud2/fud-core/src/exec/driver.rs index 94581989a5..da7c1a1bd3 100644 --- a/fud2/fud-core/src/exec/driver.rs +++ b/fud2/fud-core/src/exec/driver.rs @@ -1,16 +1,10 @@ use super::{OpRef, Operation, Request, Setup, SetupRef, State, StateRef}; use crate::{config, run, script, utils}; use camino::{Utf8Path, Utf8PathBuf}; -use cranelift_entity::{PrimaryMap, SecondaryMap}; +use cranelift_entity::PrimaryMap; use rand::distributions::{Alphanumeric, DistString}; use std::{collections::HashMap, error::Error, ffi::OsStr, fmt::Display}; -#[derive(PartialEq)] -enum Destination { - State(StateRef), - Op(OpRef), -} - type FileData = HashMap<&'static str, &'static [u8]>; /// A Driver encapsulates a set of States and the Operations that can transform between them. It @@ -25,104 +19,129 @@ pub struct Driver { } impl Driver { - /// Find a chain of Operations from the `start` state to the `end`, which may be a state or the - /// final operation in the chain. - fn find_path_segment( + /// Generate a filename with an extension appropriate for the given State, `state_ref` relative + /// to `workdir`. + /// + /// If `used` is false, the state is neither an output to the user, or used as input an op. In + /// this case, the filename associated with the state will be prefixed by `_unused_`. + fn gen_name( &self, - start: StateRef, - end: Destination, - ) -> Option> { - // Our start state is the input. - let mut visited = SecondaryMap::::new(); - visited[start] = true; - - // Build the incoming edges for each vertex. - let mut breadcrumbs = SecondaryMap::>::new(); - - // Breadth-first search. - let mut state_queue: Vec = vec![start]; - while !state_queue.is_empty() { - let cur_state = state_queue.remove(0); - - // Finish when we reach the goal vertex. - if end == Destination::State(cur_state) { - break; - } + state_ref: StateRef, + used: bool, + workdir: &Utf8PathBuf, + ) -> IO { + let state = &self.states[state_ref]; + + let prefix = if !used { "_unused_" } else { "" }; + let extension = if !state.extensions.is_empty() { + &state.extensions[0] + } else { + "" + }; - // Traverse any edge from the current state to an unvisited state. - for (op_ref, op) in self.ops.iter() { - if op.input == cur_state && !visited[op.output] { - state_queue.push(op.output); - visited[op.output] = true; - breadcrumbs[op.output] = Some(op_ref); - } - - // Finish when we reach the goal edge. - if end == Destination::Op(op_ref) { - break; - } - } - } + IO::File(if state.is_pseudo() { + utils::relative_path( + &Utf8PathBuf::from(format!("{}pseudo_{}", prefix, state.name)), + workdir, + ) + } else { + // TODO avoid collisions in case of reused extensions... + utils::relative_path( + &Utf8PathBuf::from(format!("{}{}", prefix, state.name)) + .with_extension(extension), + workdir, + ) + }) + } - // Traverse the breadcrumbs backward to build up the path back from output to input. - let mut op_path: Vec = vec![]; - let mut cur_state = match end { - Destination::State(state) => state, - Destination::Op(op) => { - op_path.push(op); - self.ops[op].input - } + /// Generates a filename for a state tagged with if the file should be read from StdIO and path + /// name relative to `workdir`. + /// + /// The state is searched for in `states`. If it is found, the name at the same index in `files` is + /// returned, else `stdio_name` is returned. + /// + /// If the state is not in states, new name is generated. + /// This name will be prefixed by `_unused_` if unused is `true`. This signifies the file is + /// neither requested as an output by the user nor used as input to any op. + fn gen_name_or_use_given( + &self, + state_ref: StateRef, + states: &[StateRef], + files: &[Utf8PathBuf], + stdio_name: &str, + used: bool, + workdir: &Utf8PathBuf, + ) -> IO { + let state = &self.states[state_ref]; + let extension = if !state.extensions.is_empty() { + &state.extensions[0] + } else { + "" }; - while cur_state != start { - match breadcrumbs[cur_state] { - Some(op) => { - op_path.push(op); - cur_state = self.ops[op].input; - } - None => return None, + + if let Some(idx) = states.iter().position(|&s| s == state_ref) { + if let Some(filename) = files.get(idx) { + IO::File(utils::relative_path(&filename.clone(), workdir)) + } else { + IO::StdIO(utils::relative_path( + &Utf8PathBuf::from(stdio_name).with_extension(extension), + workdir, + )) } + } else { + self.gen_name(state_ref, used, workdir) } - op_path.reverse(); - - Some(op_path) } - /// Find a chain of operations from the `start` state to the `end` state, passing through each - /// `through` operation in order. - pub fn find_path( + /// Generates a vector contianing filenames for files of each state in `states`. + /// + /// `req` is used to generate filenames as inputs and outputs may want to takes names from + /// `req.end_files` or `req.start_files`. + /// + /// `input` is true if all states in `states` are an input to an op. + /// `used` is the states in `states` which are an input to another op or in `req.end_states`. + fn gen_names( &self, - start: StateRef, - end: StateRef, - through: &[OpRef], - ) -> Option> { - let mut cur_state = start; - let mut op_path: Vec = vec![]; - - // Build path segments through each through required operation. - for op in through { - let segment = - self.find_path_segment(cur_state, Destination::Op(*op))?; - op_path.extend(segment); - cur_state = self.ops[*op].output; - } - - // Build the final path segment to the destination state. - let segment = - self.find_path_segment(cur_state, Destination::State(end))?; - op_path.extend(segment); - - Some(op_path) - } + states: &[StateRef], + req: &Request, + input: bool, + used: &[StateRef], + ) -> Vec { + // Inputs cannot be results, so look at starting states, else look at ending states. + let req_states = if input { + &req.start_states + } else { + &req.end_states + }; - /// Generate a filename with an extension appropriate for the given State. - fn gen_name(&self, stem: &str, state: StateRef) -> Utf8PathBuf { - let state = &self.states[state]; - if state.is_pseudo() { - Utf8PathBuf::from(format!("_pseudo_{}", state.name)) + // Inputs cannot be results, so look at starting files, else look at ending files. + let req_files = if input { + &req.start_files } else { - // TODO avoid collisions in case we reuse extensions... - Utf8PathBuf::from(stem).with_extension(&state.extensions[0]) - } + &req.end_files + }; + // The above lists can't be the concatination of the two branches because start and end + // states are not necessarily disjoint, but they could still have different files assigned + // to each state. + + states + .iter() + .map(|&state| { + let stdio_name = if input { + format!("_from_stdin_{}", self.states[state].name) + } else { + format!("_to_stdout_{}", self.states[state].name) + }; + self.gen_name_or_use_given( + state, + req_states, + req_files, + &stdio_name, + input || used.contains(&state), + &req.workdir, + ) + }) + .collect() } /// Concoct a plan to carry out the requested build. @@ -130,42 +149,37 @@ impl Driver { /// This works by searching for a path through the available operations from the input state /// to the output state. If no such path exists in the operation graph, we return None. pub fn plan(&self, req: Request) -> Option { - // Find a path through the states. - let path = - self.find_path(req.start_state, req.end_state, &req.through)?; - - let mut steps: Vec<(OpRef, Utf8PathBuf)> = vec![]; - - // Get the initial input filename and the stem to use to generate all intermediate filenames. - let (stdin, start_file) = match req.start_file { - Some(path) => (false, utils::relative_path(&path, &req.workdir)), - None => (true, "stdin".into()), - }; - let stem = start_file.file_stem().unwrap(); + // Find a plan through the states. + let path = req.planner.find_plan( + &req.start_states, + &req.end_states, + &req.through, + &self.ops, + )?; // Generate filenames for each step. - steps.extend(path.into_iter().map(|op| { - let filename = self.gen_name(stem, self.ops[op].output); - (op, filename) - })); - - // If we have a specified output filename, use that instead of the generated one. - let stdout = if let Some(end_file) = req.end_file { - // TODO Can we just avoid generating the unused filename in the first place? - let last_step = steps.last_mut().expect("no steps"); - last_step.1 = utils::relative_path(&end_file, &req.workdir); - false - } else { - // Print to stdout if the last state is a real (non-pseudo) state. - !self.states[req.end_state].is_pseudo() - }; + let steps = path + .into_iter() + .map(|(op, used)| { + let input_filenames = + self.gen_names(&self.ops[op].input, &req, true, &used); + let output_filenames = + self.gen_names(&self.ops[op].output, &req, false, &used); + (op, input_filenames, output_filenames) + }) + .collect::>(); + + // Collect filenames of inputs and outputs + let results = + self.gen_names(&req.end_states, &req, false, &req.end_states); + let inputs = + self.gen_names(&req.start_states, &req, true, &req.start_states); Some(Plan { - start: start_file, steps, + inputs, + results, workdir: req.workdir, - stdin, - stdout, }) } @@ -240,8 +254,8 @@ impl Driver { println!( " {}: {} -> {}{}", op.name, - self.states[op.input].name, - self.states[op.output].name, + self.states[op.input[0]].name, + self.states[op.output[0]].name, dev_info ); } @@ -341,15 +355,15 @@ impl DriverBuilder { &mut self, name: &str, setups: &[SetupRef], - input: StateRef, - output: StateRef, + input: &[StateRef], + output: &[StateRef], emit: T, ) -> OpRef { self.ops.push(Operation { name: name.into(), setups: setups.into(), - input, - output, + input: input.into(), + output: output.into(), emit: Box::new(emit), source: None, }) @@ -363,7 +377,7 @@ impl DriverBuilder { output: StateRef, build: run::EmitBuildFn, ) -> OpRef { - self.add_op(name, setups, input, output, build) + self.add_op(name, setups, &[input], &[output], build) } pub fn op_source(&mut self, op: OpRef, src: S) { @@ -380,8 +394,8 @@ impl DriverBuilder { self.add_op( rule_name, setups, - input, - output, + &[input], + &[output], run::EmitRuleBuild { rule_name: rule_name.to_string(), }, @@ -455,29 +469,52 @@ impl DriverBuilder { } } +/// A file tagged with its input source. +#[derive(Debug, Clone)] +pub enum IO { + /// A file at a given path which is to be read from stdin or output to stdout. + StdIO(Utf8PathBuf), + /// A file at a given path which need not be read from stdin or output ot stdout. + File(Utf8PathBuf), +} + +impl IO { + /// Returns the filename of the file `self` represents + pub fn filename(&self) -> &Utf8PathBuf { + match self { + Self::StdIO(p) => p, + Self::File(p) => p, + } + } + + /// Returns if `self` is a `StdIO`. + pub fn is_from_stdio(&self) -> bool { + match self { + Self::StdIO(_) => true, + Self::File(_) => false, + } + } +} + +impl Display for IO { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.filename()) + } +} + #[derive(Debug)] pub struct Plan { - /// The input to the first step. - pub start: Utf8PathBuf, + /// The chain of operations to run and each step's input and output files. + pub steps: Vec<(OpRef, Vec, Vec)>, - /// The chain of operations to run and each step's output file. - pub steps: Vec<(OpRef, Utf8PathBuf)>, + /// The inputs used to generate the results. + /// Earlier elements of inputs should be read before later ones. + pub inputs: Vec, + + /// The resulting files of the plan. + /// Earlier elements of inputs should be written before later ones. + pub results: Vec, /// The directory that the build will happen in. pub workdir: Utf8PathBuf, - - /// Read the first input from stdin. - pub stdin: bool, - - /// Write the final output to stdout. - pub stdout: bool, -} - -impl Plan { - pub fn end(&self) -> &Utf8Path { - match self.steps.last() { - Some((_, path)) => path, - None => &self.start, - } - } } diff --git a/fud2/fud-core/src/exec/mod.rs b/fud2/fud-core/src/exec/mod.rs index 48895a4b17..8a17549929 100644 --- a/fud2/fud-core/src/exec/mod.rs +++ b/fud2/fud-core/src/exec/mod.rs @@ -1,8 +1,10 @@ mod data; mod driver; +mod planner; mod request; pub use data::{OpRef, SetupRef, StateRef}; pub(super) use data::{Operation, Setup, State}; -pub use driver::{Driver, DriverBuilder, Plan}; +pub use driver::{Driver, DriverBuilder, Plan, IO}; +pub use planner::{EnumeratePlanner, FindPlan, SingleOpOutputPlanner}; pub use request::Request; diff --git a/fud2/fud-core/src/exec/planner.rs b/fud2/fud-core/src/exec/planner.rs new file mode 100644 index 0000000000..49adc1dada --- /dev/null +++ b/fud2/fud-core/src/exec/planner.rs @@ -0,0 +1,171 @@ +use cranelift_entity::{PrimaryMap, SecondaryMap}; + +use super::{OpRef, Operation, StateRef}; + +/// `Step` is an op paired with its used outputs. +type Step = (OpRef, Vec); + +/// A reified function for finding a sequence of operations taking a start set of states to an end +/// set of states while guaranteing a set of "though" operations is used in the sequence. +pub trait FindPlan: std::fmt::Debug { + /// Returns a sequence of `Step`s to transform `start` to `end`. The `Step`s are guaranteed to + /// contain all ops in `through`. If no such sequence exists, `None` is returned. + /// + /// `ops` is a complete list of operations. + fn find_plan( + &self, + start: &[StateRef], + end: &[StateRef], + through: &[OpRef], + ops: &PrimaryMap, + ) -> Option>; +} + +#[derive(Debug, Default)] +pub struct EnumeratePlanner {} +impl EnumeratePlanner { + pub fn new() -> Self { + EnumeratePlanner {} + } + /// Returns a sequence of `Step`s to transform `start` to `end`. The `Step`s are guaranteed to + /// contain all ops in `through`. If no such sequence exists, `None` is returned. + /// + /// `ops` is a complete list of operations. + fn find_plan( + _start: &[StateRef], + _end: &[StateRef], + _through: &[OpRef], + _ops: &PrimaryMap, + ) -> Option> { + todo!() + } +} + +impl FindPlan for EnumeratePlanner { + fn find_plan( + &self, + start: &[StateRef], + end: &[StateRef], + through: &[OpRef], + ops: &PrimaryMap, + ) -> Option> { + Self::find_plan(start, end, through, ops) + } +} + +#[derive(PartialEq)] +enum Destination { + State(StateRef), + Op(OpRef), +} + +#[derive(Debug, Default)] +pub struct SingleOpOutputPlanner {} +impl SingleOpOutputPlanner { + /// Find a chain of Operations from the `start` state to the `end`, which may be a state or the + /// final operation in the chain. + fn find_path_segment( + start: StateRef, + end: Destination, + ops: &PrimaryMap, + ) -> Option> { + // Our start state is the input. + let mut visited = SecondaryMap::::new(); + visited[start] = true; + + // Build the incoming edges for each vertex. + let mut breadcrumbs = SecondaryMap::>::new(); + + // Breadth-first search. + let mut state_queue: Vec = vec![start]; + while !state_queue.is_empty() { + let cur_state = state_queue.remove(0); + + // Finish when we reach the goal vertex. + if end == Destination::State(cur_state) { + break; + } + + // Traverse any edge from the current state to an unvisited state. + for (op_ref, op) in ops.iter() { + if op.input[0] == cur_state && !visited[op.output[0]] { + state_queue.push(op.output[0]); + visited[op.output[0]] = true; + breadcrumbs[op.output[0]] = Some(op_ref); + } + + // Finish when we reach the goal edge. + if end == Destination::Op(op_ref) { + break; + } + } + } + + // Traverse the breadcrumbs backward to build up the path back from output to input. + let mut op_path: Vec = vec![]; + let mut cur_state = match end { + Destination::State(state) => state, + Destination::Op(op) => { + op_path.push(op); + ops[op].input[0] + } + }; + while cur_state != start { + match breadcrumbs[cur_state] { + Some(op) => { + op_path.push(op); + cur_state = ops[op].input[0]; + } + None => return None, + } + } + op_path.reverse(); + + Some( + op_path + .iter() + .map(|&op| (op, vec![ops[op].output[0]])) + .collect::>(), + ) + } + + /// Find a chain of operations from the `start` state to the `end` state, passing through each + /// `through` operation in order. + pub fn find_plan( + start: StateRef, + end: StateRef, + through: &[OpRef], + ops: &PrimaryMap, + ) -> Option> { + let mut cur_state = start; + let mut op_path = vec![]; + + // Build path segments through each through required operation. + for op in through { + let segment = + Self::find_path_segment(cur_state, Destination::Op(*op), ops)?; + op_path.extend(segment); + cur_state = ops[*op].output[0]; + } + + // Build the final path segment to the destination state. + let segment = + Self::find_path_segment(cur_state, Destination::State(end), ops)?; + op_path.extend(segment); + + Some(op_path) + } +} + +impl FindPlan for SingleOpOutputPlanner { + fn find_plan( + &self, + start: &[StateRef], + end: &[StateRef], + through: &[OpRef], + ops: &PrimaryMap, + ) -> Option> { + assert!(start.len() == 1 && end.len() == 1); + Self::find_plan(start[0], end[0], through, ops) + } +} diff --git a/fud2/fud-core/src/exec/request.rs b/fud2/fud-core/src/exec/request.rs index ebde6047eb..1a1db20075 100644 --- a/fud2/fud-core/src/exec/request.rs +++ b/fud2/fud-core/src/exec/request.rs @@ -1,24 +1,29 @@ -use super::{OpRef, StateRef}; +use super::{planner::FindPlan, OpRef, StateRef}; use camino::Utf8PathBuf; /// A request to the Driver directing it what to build. #[derive(Debug)] pub struct Request { /// The input format. - pub start_state: StateRef, + /// Invariant: start_states.len() >= start_files.len() + pub start_states: Vec, /// The output format to produce. - pub end_state: StateRef, + /// Invariant: end_states.len() >= end_files.len() + pub end_states: Vec, /// The filename to read the input from, or None to read from stdin. - pub start_file: Option, + pub start_files: Vec, /// The filename to write the output to, or None to print to stdout. - pub end_file: Option, + pub end_files: Vec, /// A sequence of operators to route the conversion through. pub through: Vec, /// The working directory for the build. pub workdir: Utf8PathBuf, + + /// The algorithm used to find a plan to turn start states into end states + pub planner: Box, } diff --git a/fud2/fud-core/src/run.rs b/fud2/fud-core/src/run.rs index d42acb652d..afb44e34b4 100644 --- a/fud2/fud-core/src/run.rs +++ b/fud2/fud-core/src/run.rs @@ -69,19 +69,19 @@ pub trait EmitBuild { fn build( &self, emitter: &mut StreamEmitter, - input: &str, - output: &str, + input: &[&str], + output: &[&str], ) -> EmitResult; } -pub type EmitBuildFn = fn(&mut StreamEmitter, &str, &str) -> EmitResult; +pub type EmitBuildFn = fn(&mut StreamEmitter, &[&str], &[&str]) -> EmitResult; impl EmitBuild for EmitBuildFn { fn build( &self, emitter: &mut StreamEmitter, - input: &str, - output: &str, + input: &[&str], + output: &[&str], ) -> EmitResult { self(emitter, input, output) } @@ -97,10 +97,10 @@ impl EmitBuild for EmitRuleBuild { fn build( &self, emitter: &mut StreamEmitter, - input: &str, - output: &str, + input: &[&str], + output: &[&str], ) -> EmitResult { - emitter.build(&self.rule_name, input, output)?; + emitter.build_cmd(output, &self.rule_name, input, &[])?; Ok(()) } } @@ -148,16 +148,21 @@ impl<'a> Run<'a> { /// Just print the plan for debugging purposes. pub fn show(self) { - if self.plan.stdin { - println!("(stdin) -> {}", self.plan.start); - } else { - println!("start: {}", self.plan.start); - } - for (op, file) in self.plan.steps { - println!("{}: {} -> {}", op, self.driver.ops[op].name, file); - } - if self.plan.stdout { - println!("-> (stdout)"); + for (op, files_in, files_out) in self.plan.steps { + println!( + "{}: {} -> {}", + self.driver.ops[op].name, + files_in + .into_iter() + .map(|f| f.to_string()) + .collect::>() + .join(", "), + files_out + .into_iter() + .map(|f| f.to_string()) + .collect::>() + .join(", "), + ); } } @@ -170,14 +175,17 @@ impl<'a> Run<'a> { // Record the states and ops that are actually used in the plan. let mut states: HashMap = HashMap::new(); let mut ops: HashSet = HashSet::new(); - let first_op = self.plan.steps[0].0; - states.insert( - self.driver.ops[first_op].input, - self.plan.start.to_string(), - ); - for (op, file) in &self.plan.steps { - states.insert(self.driver.ops[*op].output, file.to_string()); - ops.insert(*op); + for (op_ref, files_in, files_out) in &self.plan.steps { + let op = &self.driver.ops[*op_ref]; + for (s, f) in op.input.iter().zip(files_in.iter()) { + let filename = f.to_string(); + states.insert(*s, filename.to_string()); + } + for (s, f) in op.output.iter().zip(files_out.iter()) { + let filename = format!("{f}"); + states.insert(*s, filename.to_string()); + } + ops.insert(*op_ref); } // Show all states. @@ -196,7 +204,10 @@ impl<'a> Run<'a> { // Show all operations. for (op_ref, op) in self.driver.ops.iter() { - print!(" {} -> {} [label=\"{}\"", op.input, op.output, op.name); + print!( + " {} -> {} [label=\"{}\"", + op.input[0], op.output[0], op.name + ); if ops.contains(&op_ref) { print!(" penwidth=3"); } @@ -228,10 +239,15 @@ impl<'a> Run<'a> { let dir = self.emit_to_dir(dir)?; // Capture stdin. - if self.plan.stdin { - let stdin_file = std::fs::File::create( - self.plan.workdir.join(&self.plan.start), - )?; + for filename in self.plan.inputs.iter().filter_map(|f| { + if f.is_from_stdio() { + Some(f.filename()) + } else { + None + } + }) { + let stdin_file = + std::fs::File::create(self.plan.workdir.join(filename))?; std::io::copy( &mut std::io::stdin(), &mut std::io::BufWriter::new(stdin_file), @@ -253,17 +269,23 @@ impl<'a> Run<'a> { cmd.stdout(std::io::stderr()); // Send Ninja's stdout to our stderr. let status = cmd.status()?; - // Emit stdout, only when Ninja succeeded. - if status.success() && self.plan.stdout { - let stdout_file = - std::fs::File::open(self.plan.workdir.join(self.plan.end()))?; - std::io::copy( - &mut std::io::BufReader::new(stdout_file), - &mut std::io::stdout(), - )?; - } - + // Emit to stdout, only when Ninja succeeded. if status.success() { + // Outputs results to stdio if tagged as such. + for filename in self.plan.results.iter().filter_map(|f| { + if f.is_from_stdio() { + Some(f.filename()) + } else { + None + } + }) { + let stdout_files = + std::fs::File::open(self.plan.workdir.join(filename))?; + std::io::copy( + &mut std::io::BufReader::new(stdout_files), + &mut std::io::stdout(), + )?; + } Ok(()) } else { Err(RunError::NinjaFailed(status)) @@ -284,7 +306,7 @@ impl<'a> Run<'a> { // Emit the setup for each operation used in the plan, only once. let mut done_setups = HashSet::::new(); - for (op, _) in &self.plan.steps { + for (op, _, _) in &self.plan.steps { for setup in &self.driver.ops[*op].setups { if done_setups.insert(*setup) { let setup = &self.driver.setups[*setup]; @@ -297,20 +319,28 @@ impl<'a> Run<'a> { // Emit the build commands for each step in the plan. emitter.comment("build targets")?; - let mut last_file = &self.plan.start; - for (op, out_file) in &self.plan.steps { + for (op, in_files, out_files) in &self.plan.steps { let op = &self.driver.ops[*op]; op.emit.build( &mut emitter, - last_file.as_str(), - out_file.as_str(), + in_files + .iter() + .map(|io| io.filename().as_str()) + .collect::>() + .as_slice(), + out_files + .iter() + .map(|io| io.filename().as_str()) + .collect::>() + .as_slice(), )?; - last_file = out_file; } writeln!(emitter.out)?; - // Mark the last file as the default target. - writeln!(emitter.out, "default {}", last_file)?; + // Mark the last file as the default targets. + for result in &self.plan.results { + writeln!(emitter.out, "default {}", result.filename())?; + } Ok(()) } diff --git a/fud2/fud-core/src/script/exec_scripts.rs b/fud2/fud-core/src/script/exec_scripts.rs index 03c6101392..3d7e7c65be 100644 --- a/fud2/fud-core/src/script/exec_scripts.rs +++ b/fud2/fud-core/src/script/exec_scripts.rs @@ -229,8 +229,8 @@ impl EmitBuild for RhaiSetupCtx { fn build( &self, emitter: &mut StreamEmitter, - input: &str, - output: &str, + input: &[&str], + output: &[&str], ) -> EmitResult { RhaiEmitter::with(emitter, |rhai_emit| { EMIT_ENGINE.with(|e| { @@ -238,7 +238,11 @@ impl EmitBuild for RhaiSetupCtx { &mut rhai::Scope::new(), &self.ast, &self.name, - (rhai_emit.clone(), input.to_string(), output.to_string()), + ( + rhai_emit.clone(), + input[0].to_string(), + output[0].to_string(), + ), ) .report(self.path.as_ref()) }); diff --git a/fud2/fud-core/src/script/plugin.rs b/fud2/fud-core/src/script/plugin.rs index 2570b48b29..4c2df65b72 100644 --- a/fud2/fud-core/src/script/plugin.rs +++ b/fud2/fud-core/src/script/plugin.rs @@ -222,8 +222,13 @@ impl ScriptRunner { ast: Rc::new(sctx.ast.clone_functions_only()), name: build.fn_name().to_string(), }; - let op = - bld.borrow_mut().add_op(name, &setups, input, output, rctx); + let op = bld.borrow_mut().add_op( + name, + &setups, + &[input], + &[output], + rctx, + ); // try to set op source #[cfg(debug_assertions)] diff --git a/fud2/fud-core/tests/tests.rs b/fud2/fud-core/tests/tests.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/fud2/fud-core/tests/tests.rs @@ -0,0 +1 @@ + diff --git a/fud2/src/lib.rs b/fud2/src/lib.rs index 5bac0cac26..788fe93f43 100644 --- a/fud2/src/lib.rs +++ b/fud2/src/lib.rs @@ -44,7 +44,7 @@ fn setup_calyx( calyx, verilog, |e, input, output| { - e.build_cmd(&[output], "calyx", &[input], &[])?; + e.build_cmd(&[output[0]], "calyx", &[input[0]], &[])?; e.arg("backend", "verilog")?; Ok(()) }, @@ -135,11 +135,11 @@ pub fn build_driver(bld: &mut DriverBuilder) { simulator, dat, |e, input, output| { - e.build_cmd(&["sim.log"], "sim-run", &[input, "$datadir"], &[])?; - e.arg("bin", input)?; + e.build_cmd(&["sim.log"], "sim-run", &[input[0], "$datadir"], &[])?; + e.arg("bin", input[0])?; e.arg("args", "+NOTRACE=1")?; e.build_cmd( - &[output], + &[output[0]], "json-data", &["$datadir", "sim.log"], &["json-dat.py"], @@ -149,13 +149,13 @@ pub fn build_driver(bld: &mut DriverBuilder) { ); bld.op("trace", &[sim_setup], simulator, vcd, |e, input, output| { e.build_cmd( - &["sim.log", output], + &["sim.log", output[0]], "sim-run", - &[input, "$datadir"], + &[input[0], "$datadir"], &[], )?; - e.arg("bin", input)?; - e.arg("args", &format!("+NOTRACE=0 +OUT={}", output))?; + e.arg("bin", input[0])?; + e.arg("args", &format!("+NOTRACE=0 +OUT={}", output[0]))?; Ok(()) }); @@ -218,7 +218,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { verilog_noverify, |e, input, output| { // Icarus requires a special --disable-verify version of Calyx code. - e.build_cmd(&[output], "calyx", &[input], &[])?; + e.build_cmd(&[output[0]], "calyx", &[input[0]], &[])?; e.arg("backend", "verilog")?; e.arg("args", "--disable-verify")?; Ok(()) @@ -232,9 +232,9 @@ pub fn build_driver(bld: &mut DriverBuilder) { simulator, |e, input, output| { e.build_cmd( - &[output], + &[output[0]], "icarus-compile-standalone-tb", - &[input], + &[input[0]], &["tb.sv"], )?; Ok(()) @@ -247,9 +247,9 @@ pub fn build_driver(bld: &mut DriverBuilder) { simulator, |e, input, output| { e.build_cmd( - &[output], + &[output[0]], "icarus-compile-custom-tb", - &[input], + &[input[0]], &["tb.sv", "memories.sv"], )?; Ok(()) @@ -355,7 +355,9 @@ pub fn build_driver(bld: &mut DriverBuilder) { &[calyx_setup, custom_testbench_setup], calyx, firrtl, - |e, input, output| calyx_to_firrtl_helper(e, input, output, false), + |e, input, output| { + calyx_to_firrtl_helper(e, input[0], output[0], false) + }, ); bld.op( @@ -363,7 +365,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { &[calyx_setup, firrtl_primitives_setup, custom_testbench_setup], calyx, firrtl_with_primitives, - |e, input, output| calyx_to_firrtl_helper(e, input, output, true), + |e, input, output| calyx_to_firrtl_helper(e, input[0], output[0], true), ); // The FIRRTL compiler. @@ -407,7 +409,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { &[firrtl_setup], firrtl, verilog_refmem, - |e, input, output| firrtl_compile_helper(e, input, output, false), + |e, input, output| firrtl_compile_helper(e, input[0], output[0], false), ); // FIRRTL --> Verilog compilation using Verilog primitive implementations for Icarus // This is a bit of a hack, but the Icarus-friendly "noverify" state is identical for this path @@ -417,7 +419,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { &[firrtl_setup], firrtl, verilog_refmem_noverify, - |e, input, output| firrtl_compile_helper(e, input, output, false), + |e, input, output| firrtl_compile_helper(e, input[0], output[0], false), ); // FIRRTL --> Verilog compilation using FIRRTL primitive implementations for Verilator bld.op( @@ -425,7 +427,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { &[firrtl_setup], firrtl_with_primitives, verilog_refmem, - |e, input, output| firrtl_compile_helper(e, input, output, true), + |e, input, output| firrtl_compile_helper(e, input[0], output[0], true), ); // FIRRTL --> Verilog compilation using FIRRTL primitive implementations for Icarus bld.op( @@ -433,7 +435,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { &[firrtl_setup], firrtl_with_primitives, verilog_refmem_noverify, - |e, input, output| firrtl_compile_helper(e, input, output, true), + |e, input, output| firrtl_compile_helper(e, input[0], output[0], true), ); // primitive-uses backend @@ -444,7 +446,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { calyx, primitive_uses_json, |e, input, output| { - e.build_cmd(&[output], "calyx", &[input], &[])?; + e.build_cmd(&[output[0]], "calyx", &[input[0]], &[])?; e.arg("backend", "primitive-uses")?; Ok(()) }, @@ -498,7 +500,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { &[sim_setup, standalone_testbench_setup, verilator_setup], verilog, simulator, - |e, input, output| verilator_build(e, input, output, true), + |e, input, output| verilator_build(e, input[0], output[0], true), ); bld.op( @@ -506,7 +508,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { &[sim_setup, custom_testbench_setup, verilator_setup], verilog_refmem, simulator, - |e, input, output| verilator_build(e, input, output, false), + |e, input, output| verilator_build(e, input[0], output[0], false), ); // Interpreter. @@ -575,7 +577,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { e.build_cmd( &["cider-input.futil"], "calyx-with-flags", - &[input], + input, &[], )?; Ok(()) @@ -594,9 +596,9 @@ pub fn build_driver(bld: &mut DriverBuilder) { dat, |e, input, output| { let out_file = "interp_out.json"; - e.build_cmd(&[out_file], "cider", &[input], &["data.json"])?; + e.build_cmd(&[out_file], "cider", &[input[0]], &["data.json"])?; e.build_cmd( - &[output], + &[output[0]], "interp-to-dat", &[out_file], &["$sim_data", "interp-dat.py"], @@ -618,7 +620,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { &["data.dump"], )?; e.build_cmd( - &[output], + &[output[0]], "interp-to-dump", &[out_file], &["$sim_data", "$cider-converter"], @@ -637,7 +639,12 @@ pub fn build_driver(bld: &mut DriverBuilder) { calyx, debug, |e, input, output| { - e.build_cmd(&[output], "cider-debug", &[input], &["data.json"])?; + e.build_cmd( + &[output[0]], + "cider-debug", + &[input[0]], + &["data.json"], + )?; Ok(()) }, ); @@ -672,19 +679,19 @@ pub fn build_driver(bld: &mut DriverBuilder) { xo, |e, input, output| { // Emit the Verilog itself in "synthesis mode." - e.build_cmd(&["main.sv"], "calyx", &[input], &[])?; + e.build_cmd(&["main.sv"], "calyx", &[input[0]], &[])?; e.arg("backend", "verilog")?; e.arg("args", "--synthesis -p external")?; // Extra ingredients for the `.xo` package. - e.build_cmd(&["toplevel.v"], "calyx", &[input], &[])?; + e.build_cmd(&["toplevel.v"], "calyx", &[input[0]], &[])?; e.arg("backend", "xilinx")?; - e.build_cmd(&["kernel.xml"], "calyx", &[input], &[])?; + e.build_cmd(&["kernel.xml"], "calyx", &[input[0]], &[])?; e.arg("backend", "xilinx-xml")?; // Package the `.xo`. e.build_cmd( - &[output], + &[output[0]], "gen-xo", &[], &[ @@ -699,7 +706,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { }, ); bld.op("xclbin", &[xilinx_setup], xo, xclbin, |e, input, output| { - e.build_cmd(&[output], "compile-xclbin", &[input], &[])?; + e.build_cmd(&[output[0]], "compile-xclbin", &[input[0]], &[])?; Ok(()) }); @@ -737,9 +744,9 @@ pub fn build_driver(bld: &mut DriverBuilder) { |e, input, output| { e.rsrc("xrt.ini")?; e.build_cmd( - &[output], + &[output[0]], "xclrun", - &[input, "$sim_data"], + &[input[0], "$sim_data"], &["emconfig.json", "xrt.ini"], )?; e.arg("xrt_ini", "xrt.ini")?; @@ -759,9 +766,9 @@ pub fn build_driver(bld: &mut DriverBuilder) { |e, input, output| { e.rsrc("xrt_trace.ini")?; e.build_cmd( - &[output], // TODO not the VCD, yet... + &[output[0]], // TODO not the VCD, yet... "xclrun", - &[input, "$sim_data"], + &[input[0], "$sim_data"], &[ "emconfig.json", "pre_sim.tcl", @@ -787,7 +794,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { calyx, yxi, |e, input, output| { - e.build_cmd(&[output], "yxi", &[input], &[])?; + e.build_cmd(output, "yxi", input, &[])?; Ok(()) }, ); @@ -826,15 +833,15 @@ pub fn build_driver(bld: &mut DriverBuilder) { |e, input, output| { // Generate the YXI file. // no extension - let file_name = basename(input); + let file_name = basename(input[0]); // Get yxi file from main compute program. let tmp_yxi = format!("{}.yxi", file_name); - e.build_cmd(&[&tmp_yxi], "yxi", &[input], &[])?; + e.build_cmd(&[&tmp_yxi], "yxi", input, &[])?; // Generate the AXI wrapper. let refified_calyx = format!("refified_{}.futil", file_name); - e.build_cmd(&[&refified_calyx], "calyx-pass", &[input], &[])?; + e.build_cmd(&[&refified_calyx], "calyx-pass", &[input[0]], &[])?; e.arg("pass", "external-to-ref")?; let axi_wrapper = "axi_wrapper.futil"; @@ -851,7 +858,7 @@ pub fn build_driver(bld: &mut DriverBuilder) { // Combine the original Calyx and the wrapper. e.build_cmd( - &[output], + &[output[0]], "combine", &[axi_wrapper, &no_imports_calyx], &[], @@ -922,11 +929,11 @@ e.var("cocotb-args", if waves {"WAVES=1"} else {""})?; let waves = FromStr::from_str(&waves) .expect("The 'waves' flag should be either 'true' or 'false'."); - let vcd_file_name = format!("{}.fst", basename(input)); - let mut make_in = input; + let vcd_file_name = format!("{}.fst", basename(input[0])); + let mut make_in = input[0]; if waves { make_in = "dumpvars.v"; - e.build_cmd(&[make_in], "iverilog-fst-sed", &[input], &[])?; + e.build_cmd(&[make_in], "iverilog-fst-sed", input, &[])?; e.arg("fst_file_name", &vcd_file_name)?; } e.build_cmd( @@ -935,7 +942,7 @@ e.var("cocotb-args", if waves {"WAVES=1"} else {""})?; &[make_in], &["Makefile", "axi_test.py", "run_axi_test.py"], )?; - e.build_cmd(&[output], "cleanup-cocotb", &["tmp.dat"], &[])?; + e.build_cmd(output, "cleanup-cocotb", &["tmp.dat"], &[])?; Ok(()) }, diff --git a/fud2/tests/snapshots/tests__test@calyx_debug.snap b/fud2/tests/snapshots/tests__test@calyx_debug.snap index 2bc1b3b9f9..8c65bdb7e6 100644 --- a/fud2/tests/snapshots/tests__test@calyx_debug.snap +++ b/fud2/tests/snapshots/tests__test@calyx_debug.snap @@ -54,6 +54,6 @@ rule interp-to-dump command = $cider-converter --to json $in > $out build data.dump: dump-to-interp $sim_data | $cider-converter -build _pseudo_debug: cider-debug stdin | data.json +build _to_stdout_debug: cider-debug _from_stdin_calyx.futil | data.json -default _pseudo_debug +default _to_stdout_debug diff --git a/fud2/tests/snapshots/tests__test@calyx_firrtl_verilog-refmem.snap b/fud2/tests/snapshots/tests__test@calyx_firrtl_verilog-refmem.snap index f8c2132df3..5b22cb0ebf 100644 --- a/fud2/tests/snapshots/tests__test@calyx_firrtl_verilog-refmem.snap +++ b/fud2/tests/snapshots/tests__test@calyx_firrtl_verilog-refmem.snap @@ -35,15 +35,15 @@ build primitives-for-firrtl.sv: get-rsrc rule add-verilog-primitives command = cat primitives-for-firrtl.sv $in > $out -build external.futil: ref-to-external stdin -build ref.futil: external-to-ref stdin +build external.futil: ref-to-external _from_stdin_calyx.futil +build ref.futil: external-to-ref _from_stdin_calyx.futil build memory-info.json: yxi external.futil build tb.sv: generate-refmem-testbench memory-info.json build tmp-out.fir: calyx ref.futil backend = firrtl args = --emit-primitive-extmodules -build stdin.fir: dummy tmp-out.fir tb.sv -build partial.sv: firrtl stdin.fir -build stdin.sv: add-verilog-primitives partial.sv | primitives-for-firrtl.sv +build firrtl.fir: dummy tmp-out.fir tb.sv +build partial.sv: firrtl firrtl.fir +build _to_stdout_verilog-refmem.sv: add-verilog-primitives partial.sv | primitives-for-firrtl.sv -default stdin.sv +default _to_stdout_verilog-refmem.sv diff --git a/fud2/tests/snapshots/tests__test@calyx_icarus_dat.snap b/fud2/tests/snapshots/tests__test@calyx_icarus_dat.snap index 28436d7c5c..2a1a35ed98 100644 --- a/fud2/tests/snapshots/tests__test@calyx_icarus_dat.snap +++ b/fud2/tests/snapshots/tests__test@calyx_icarus_dat.snap @@ -38,13 +38,13 @@ rule icarus-compile-standalone-tb rule icarus-compile-custom-tb command = $iverilog -g2012 -o $out tb.sv memories.sv $in -build stdin.sv: calyx stdin +build verilog-noverify.sv: calyx _from_stdin_calyx.futil backend = verilog args = --disable-verify -build stdin.exe: icarus-compile-standalone-tb stdin.sv | tb.sv -build sim.log: sim-run stdin.exe $datadir - bin = stdin.exe +build sim.exe: icarus-compile-standalone-tb verilog-noverify.sv | tb.sv +build sim.log: sim-run sim.exe $datadir + bin = sim.exe args = +NOTRACE=1 -build stdin.json: json-data $datadir sim.log | json-dat.py +build _to_stdout_dat.json: json-data $datadir sim.log | json-dat.py -default stdin.json +default _to_stdout_dat.json diff --git a/fud2/tests/snapshots/tests__test@calyx_icarus_vcd.snap b/fud2/tests/snapshots/tests__test@calyx_icarus_vcd.snap index 81173c4956..85827059d6 100644 --- a/fud2/tests/snapshots/tests__test@calyx_icarus_vcd.snap +++ b/fud2/tests/snapshots/tests__test@calyx_icarus_vcd.snap @@ -38,12 +38,12 @@ rule icarus-compile-standalone-tb rule icarus-compile-custom-tb command = $iverilog -g2012 -o $out tb.sv memories.sv $in -build stdin.sv: calyx stdin +build verilog-noverify.sv: calyx _from_stdin_calyx.futil backend = verilog args = --disable-verify -build stdin.exe: icarus-compile-standalone-tb stdin.sv | tb.sv -build sim.log stdin.vcd: sim-run stdin.exe $datadir - bin = stdin.exe - args = +NOTRACE=0 +OUT=stdin.vcd +build sim.exe: icarus-compile-standalone-tb verilog-noverify.sv | tb.sv +build sim.log _to_stdout_vcd.vcd: sim-run sim.exe $datadir + bin = sim.exe + args = +NOTRACE=0 +OUT=_to_stdout_vcd.vcd -default stdin.vcd +default _to_stdout_vcd.vcd diff --git a/fud2/tests/snapshots/tests__test@calyx_interp_dat.snap b/fud2/tests/snapshots/tests__test@calyx_interp_dat.snap index e6ef5b4935..713d53f60d 100644 --- a/fud2/tests/snapshots/tests__test@calyx_interp_dat.snap +++ b/fud2/tests/snapshots/tests__test@calyx_interp_dat.snap @@ -54,7 +54,7 @@ rule interp-to-dump command = $cider-converter --to json $in > $out build data.dump: dump-to-interp $sim_data | $cider-converter -build interp_out.json: cider stdin | data.json -build stdin.json: interp-to-dat interp_out.json | $sim_data interp-dat.py +build interp_out.json: cider _from_stdin_calyx.futil | data.json +build _to_stdout_dat.json: interp-to-dat interp_out.json | $sim_data interp-dat.py -default stdin.json +default _to_stdout_dat.json diff --git a/fud2/tests/snapshots/tests__test@calyx_verilator_dat.snap b/fud2/tests/snapshots/tests__test@calyx_verilator_dat.snap index bf66e8d417..184bb8a24f 100644 --- a/fud2/tests/snapshots/tests__test@calyx_verilator_dat.snap +++ b/fud2/tests/snapshots/tests__test@calyx_verilator_dat.snap @@ -41,14 +41,14 @@ rule verilator-compile-custom-tb rule cp command = cp $in $out -build stdin.sv: calyx stdin +build verilog.sv: calyx _from_stdin_calyx.futil backend = verilog -build verilator-out/VTOP: verilator-compile-standalone-tb stdin.sv | tb.sv +build verilator-out/VTOP: verilator-compile-standalone-tb verilog.sv | tb.sv out-dir = verilator-out -build stdin.exe: cp verilator-out/VTOP -build sim.log: sim-run stdin.exe $datadir - bin = stdin.exe +build sim.exe: cp verilator-out/VTOP +build sim.log: sim-run sim.exe $datadir + bin = sim.exe args = +NOTRACE=1 -build stdin.json: json-data $datadir sim.log | json-dat.py +build _to_stdout_dat.json: json-data $datadir sim.log | json-dat.py -default stdin.json +default _to_stdout_dat.json diff --git a/fud2/tests/snapshots/tests__test@calyx_verilator_vcd.snap b/fud2/tests/snapshots/tests__test@calyx_verilator_vcd.snap index f0289263cf..07ab0b6544 100644 --- a/fud2/tests/snapshots/tests__test@calyx_verilator_vcd.snap +++ b/fud2/tests/snapshots/tests__test@calyx_verilator_vcd.snap @@ -41,13 +41,13 @@ rule verilator-compile-custom-tb rule cp command = cp $in $out -build stdin.sv: calyx stdin +build verilog.sv: calyx _from_stdin_calyx.futil backend = verilog -build verilator-out/VTOP: verilator-compile-standalone-tb stdin.sv | tb.sv +build verilator-out/VTOP: verilator-compile-standalone-tb verilog.sv | tb.sv out-dir = verilator-out -build stdin.exe: cp verilator-out/VTOP -build sim.log stdin.vcd: sim-run stdin.exe $datadir - bin = stdin.exe - args = +NOTRACE=0 +OUT=stdin.vcd +build sim.exe: cp verilator-out/VTOP +build sim.log _to_stdout_vcd.vcd: sim-run sim.exe $datadir + bin = sim.exe + args = +NOTRACE=0 +OUT=_to_stdout_vcd.vcd -default stdin.vcd +default _to_stdout_vcd.vcd diff --git a/fud2/tests/snapshots/tests__test@calyx_verilog.snap b/fud2/tests/snapshots/tests__test@calyx_verilog.snap index dd36be84b1..af413d8098 100644 --- a/fud2/tests/snapshots/tests__test@calyx_verilog.snap +++ b/fud2/tests/snapshots/tests__test@calyx_verilog.snap @@ -17,7 +17,7 @@ flags = -p none rule calyx-with-flags command = $calyx-exe -l $calyx-base $flags $args $in > $out -build stdin.sv: calyx stdin +build _to_stdout_verilog.sv: calyx _from_stdin_calyx.futil backend = verilog -default stdin.sv +default _to_stdout_verilog.sv diff --git a/fud2/tests/snapshots/tests__test@calyx_xrt-trace_vcd.snap b/fud2/tests/snapshots/tests__test@calyx_xrt-trace_vcd.snap index 28c0170690..6531aaf4f2 100644 --- a/fud2/tests/snapshots/tests__test@calyx_xrt-trace_vcd.snap +++ b/fud2/tests/snapshots/tests__test@calyx_xrt-trace_vcd.snap @@ -60,17 +60,17 @@ build pre_sim.tcl: echo | build post_sim.tcl: echo | contents = close_vcd\\n -build main.sv: calyx stdin +build main.sv: calyx _from_stdin_calyx.futil backend = verilog args = --synthesis -p external -build toplevel.v: calyx stdin +build toplevel.v: calyx _from_stdin_calyx.futil backend = xilinx -build kernel.xml: calyx stdin +build kernel.xml: calyx _from_stdin_calyx.futil backend = xilinx-xml -build stdin.xo: gen-xo | main.sv toplevel.v kernel.xml gen_xo.tcl get-ports.py -build stdin.xclbin: compile-xclbin stdin.xo +build xo.xo: gen-xo | main.sv toplevel.v kernel.xml gen_xo.tcl get-ports.py +build xclbin.xclbin: compile-xclbin xo.xo build xrt_trace.ini: get-rsrc -build stdin.vcd: xclrun stdin.xclbin $sim_data | emconfig.json pre_sim.tcl post_sim.tcl xrt_trace.ini +build _to_stdout_vcd.vcd: xclrun xclbin.xclbin $sim_data | emconfig.json pre_sim.tcl post_sim.tcl xrt_trace.ini xrt_ini = xrt_trace.ini -default stdin.vcd +default _to_stdout_vcd.vcd diff --git a/fud2/tests/snapshots/tests__test@calyx_xrt_dat.snap b/fud2/tests/snapshots/tests__test@calyx_xrt_dat.snap index 5f68ee5341..5aa1580b4b 100644 --- a/fud2/tests/snapshots/tests__test@calyx_xrt_dat.snap +++ b/fud2/tests/snapshots/tests__test@calyx_xrt_dat.snap @@ -60,17 +60,17 @@ build pre_sim.tcl: echo | build post_sim.tcl: echo | contents = close_vcd\\n -build main.sv: calyx stdin +build main.sv: calyx _from_stdin_calyx.futil backend = verilog args = --synthesis -p external -build toplevel.v: calyx stdin +build toplevel.v: calyx _from_stdin_calyx.futil backend = xilinx -build kernel.xml: calyx stdin +build kernel.xml: calyx _from_stdin_calyx.futil backend = xilinx-xml -build stdin.xo: gen-xo | main.sv toplevel.v kernel.xml gen_xo.tcl get-ports.py -build stdin.xclbin: compile-xclbin stdin.xo +build xo.xo: gen-xo | main.sv toplevel.v kernel.xml gen_xo.tcl get-ports.py +build xclbin.xclbin: compile-xclbin xo.xo build xrt.ini: get-rsrc -build stdin.json: xclrun stdin.xclbin $sim_data | emconfig.json xrt.ini +build _to_stdout_dat.json: xclrun xclbin.xclbin $sim_data | emconfig.json xrt.ini xrt_ini = xrt.ini -default stdin.json +default _to_stdout_dat.json diff --git a/fud2/tests/snapshots/tests__test@dahlia_calyx.snap b/fud2/tests/snapshots/tests__test@dahlia_calyx.snap index b4fb83da93..9906ddd097 100644 --- a/fud2/tests/snapshots/tests__test@dahlia_calyx.snap +++ b/fud2/tests/snapshots/tests__test@dahlia_calyx.snap @@ -10,6 +10,6 @@ dahlia-exe = /test/bin/dahlia rule dahlia-to-calyx command = $dahlia-exe -b calyx --lower -l error $in -o $out -build stdin.futil: dahlia-to-calyx stdin +build _to_stdout_calyx.futil: dahlia-to-calyx _from_stdin_dahlia.fuse -default stdin.futil +default _to_stdout_calyx.futil diff --git a/fud2/tests/snapshots/tests__test@mrxl_calyx.snap b/fud2/tests/snapshots/tests__test@mrxl_calyx.snap index 6516c15f8f..fbd9d680b1 100644 --- a/fud2/tests/snapshots/tests__test@mrxl_calyx.snap +++ b/fud2/tests/snapshots/tests__test@mrxl_calyx.snap @@ -10,6 +10,6 @@ mrxl-exe = mrxl rule mrxl-to-calyx command = $mrxl-exe $in > $out -build stdin.futil: mrxl-to-calyx stdin +build _to_stdout_calyx.futil: mrxl-to-calyx _from_stdin_mrxl.mrxl -default stdin.futil +default _to_stdout_calyx.futil diff --git a/fud2/tests/tests.rs b/fud2/tests/tests.rs index f39b422643..126f9345a6 100644 --- a/fud2/tests/tests.rs +++ b/fud2/tests/tests.rs @@ -1,6 +1,6 @@ use fud_core::{ config::default_config, - exec::{Plan, Request}, + exec::{Plan, Request, SingleOpOutputPlanner, IO}, run::Run, Driver, DriverBuilder, }; @@ -50,7 +50,7 @@ impl InstaTest for Plan { let ops = self .steps .iter() - .map(|(opref, _path)| driver.ops[*opref].name.to_string()) + .map(|(opref, _, _)| driver.ops[*opref].name.to_string()) .collect_vec() .join(" -> "); format!("emit plan: {ops}") @@ -60,7 +60,7 @@ impl InstaTest for Plan { let ops = self .steps .iter() - .map(|(opref, _path)| driver.ops[*opref].name.to_string()) + .map(|(opref, _, _)| driver.ops[*opref].name.to_string()) .collect_vec() .join("_"); format!("plan_{ops}") @@ -93,8 +93,8 @@ impl InstaTest for Request { fn desc(&self, driver: &Driver) -> String { let mut desc = format!( "emit request: {} -> {}", - driver.states[self.start_state].name, - driver.states[self.end_state].name + driver.states[self.start_states[0]].name, + driver.states[self.end_states[0]].name ); if !self.through.is_empty() { desc.push_str(" through"); @@ -107,13 +107,13 @@ impl InstaTest for Request { } fn slug(&self, driver: &Driver) -> String { - let mut desc = driver.states[self.start_state].name.to_string(); + let mut desc = driver.states[self.start_states[0]].name.to_string(); for op in &self.through { desc.push('_'); desc.push_str(&driver.ops[*op].name); } desc.push('_'); - desc.push_str(&driver.states[self.end_state].name); + desc.push_str(&driver.states[self.end_states[0]].name); desc } @@ -130,12 +130,13 @@ fn request( through: &[&str], ) -> Request { fud_core::exec::Request { - start_file: None, - start_state: driver.get_state(start).unwrap(), - end_file: None, - end_state: driver.get_state(end).unwrap(), + start_files: vec![], + start_states: vec![driver.get_state(start).unwrap()], + end_files: vec![], + end_states: vec![driver.get_state(end).unwrap()], through: through.iter().map(|s| driver.get_op(s).unwrap()).collect(), workdir: ".".into(), + planner: Box::new(SingleOpOutputPlanner {}), } } @@ -144,11 +145,14 @@ fn all_ops() { let driver = test_driver(); for op in driver.ops.keys() { let plan = Plan { - start: "/input.ext".into(), - steps: vec![(op, "/output.ext".into())], + steps: vec![( + op, + vec![IO::File("/input.ext".into())], + vec![IO::File("/output.ext".into())], + )], workdir: ".".into(), - stdin: false, - stdout: false, + inputs: vec![IO::File("/input.ext".into())], + results: vec![IO::File("/output.ext".into())], }; plan.test(&driver); } @@ -179,8 +183,8 @@ fn list_ops() { .map(|op| { ( &op.name, - &driver.states[op.input].name, - &driver.states[op.output].name, + &driver.states[op.input[0]].name, + &driver.states[op.output[0]].name, ) }) .sorted()