Introduce reporting and some quality of life stuff

Cargo.toml

@@ -22,4 +22,5 @@ jsonschema = "0.16.0"
 log = "0.4.17"
 serde = "1.0.145"
 serde_json = "1.0.86"
+tabled = "0.9.0"
 users = "0.11.0"

outputs/build/snailtracer2/SnailTracer2.abi

@@ -0,0 +1 @@
+[{"constant":true,"inputs":[],"name":"Benchmark","outputs":[{"name":"r","type":"bytes1"},{"name":"g","type":"bytes1"},{"name":"b","type":"bytes1"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"y","type":"int256"},{"name":"spp","type":"int256"}],"name":"TraceScanline","outputs":[{"name":"","type":"bytes"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"x","type":"int256"},{"name":"y","type":"int256"},{"name":"spp","type":"int256"}],"name":"TracePixel","outputs":[{"name":"r","type":"bytes1"},{"name":"g","type":"bytes1"},{"name":"b","type":"bytes1"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"spp","type":"int256"}],"name":"TraceImage","outputs":[{"name":"","type":"bytes"}],"payable":false,"stateMutability":"view","type":"function"}]

src/build.rs

@@ -1,6 +1,7 @@
 use std::{
     collections::HashSet,
     error,
+    fs::create_dir_all,
     path::{Path, PathBuf},
     process::Command,
 };
@@ -47,6 +48,8 @@ fn build_benchmark(
     let docker_contract_path = PathBuf::from("/benchmark").join(&contract_name);
     let docker_build_path = PathBuf::from("/build");
 
+    create_dir_all(&build_context.build_path)?;
+
     let out = Command::new(&build_context.docker_executable)
         .arg("run")
         .args([

src/exec.rs

@@ -1,9 +1,13 @@
 use std::{
+    error,
     path::{Path, PathBuf},
-    process::{exit, Command},
+    process::Command,
 };
 
-pub fn validate_executable_or_exit(name: &str, executable: &Path) -> PathBuf {
+pub fn validate_executable(
+    name: &str,
+    executable: &Path,
+) -> Result<PathBuf, Box<dyn error::Error>> {
     log::trace!("validating executable {} ({name})", executable.display());
     match Command::new(&executable).arg("--version").output() {
         Ok(out) => {
@@ -14,17 +18,13 @@ pub fn validate_executable_or_exit(name: &str, executable: &Path) -> PathBuf {
                     .expect("could not decode program stdout")
                     .trim_end_matches("\n")
             );
-            executable.to_path_buf()
+            Ok(executable.to_path_buf())
         }
         Err(e) => match e.kind() {
             std::io::ErrorKind::NotFound => {
-                log::error!("{name} not found, tried {}", executable.display());
-                exit(-1);
-            }
-            _ => {
-                log::error!("unknown error: {e}");
-                exit(-1);
+                Err(format!("{name} not found, tried {}", executable.display()).into())
             }
+            _ => Err(format!("unknown error: {e}").into()),
         },
     }
 }

src/main.rs

@@ -1,9 +1,9 @@
-use std::{path::PathBuf, process::exit};
+use std::{error, path::PathBuf, process::exit};
 
 extern crate glob;
 
 use clap::Parser;
-use results::record_results;
+use results::{print_results, record_results};
 
 mod build;
 mod exec;
@@ -13,7 +13,7 @@ mod run;
 
 use crate::{
     build::build_benchmarks,
-    exec::validate_executable_or_exit,
+    exec::validate_executable,
     metadata::{find_benchmarks, find_runners, BenchmarkDefaults},
     run::run_benchmarks_on_runners,
 };
@@ -77,56 +77,47 @@ fn main() {
 
     let args = Args::parse();
 
-    let docker_executable = validate_executable_or_exit("docker", &args.docker_executable);
-    let _ = validate_executable_or_exit("cargo", &PathBuf::from("cargo"));
-
-    let benchmarks_path = args.benchmark_search_path.canonicalize().unwrap();
-    let benchmarks = find_benchmarks(
-        &args.benchmark_metadata_name,
-        &args.benchmark_metadata_schema,
-        &benchmarks_path,
-        BenchmarkDefaults {
-            solc_version: args.default_solc_version,
-            num_runs: args.default_num_runs,
-            calldata: hex::decode(args.default_calldata_str.to_string())
-                .expect("error parsing default calldata"),
-        },
-    )
+    (|| -> Result<(), Box<dyn error::Error>> {
+        let docker_executable = validate_executable("docker", &args.docker_executable)?;
+        let _ = validate_executable("cargo", &PathBuf::from("cargo"))?;
+
+        let default_calldata = hex::decode(args.default_calldata_str.to_string())?;
+
+        let benchmarks_path = args.benchmark_search_path.canonicalize()?;
+        let benchmarks = find_benchmarks(
+            &args.benchmark_metadata_name,
+            &args.benchmark_metadata_schema,
+            &benchmarks_path,
+            BenchmarkDefaults {
+                solc_version: args.default_solc_version,
+                num_runs: args.default_num_runs,
+                calldata: default_calldata,
+            },
+        )?;
+
+        let runners_path = args.runner_search_path.canonicalize()?;
+        let runners = find_runners(
+            &args.runner_metadata_name,
+            &args.runner_metadata_schema,
+            &runners_path,
+            (),
+        )?;
+
+        let outputs_path = args.output_path.canonicalize()?;
+
+        let builds_path = outputs_path.join("build");
+        let built_benchmarks = build_benchmarks(&benchmarks, &docker_executable, &builds_path)?;
+
+        let results = run_benchmarks_on_runners(&built_benchmarks, &runners)?;
+
+        let results_path = outputs_path.join("results");
+        let result_file_path = record_results(&results_path, args.output_file_name, &results)?;
+        print_results(&result_file_path)?;
+
+        Ok(())
+    })()
     .unwrap_or_else(|e| {
-        log::error!("could not find benchmarks: {e}");
+        log::error!("{e}");
         exit(-1);
     });
-
-    let runners_path = args.runner_search_path.canonicalize().unwrap();
-    let runners = find_runners(
-        &args.runner_metadata_name,
-        &args.runner_metadata_schema,
-        &runners_path,
-        (),
-    )
-    .unwrap_or_else(|e| {
-        log::error!("could not find runners: {e}");
-        exit(-1);
-    });
-
-    let outputs_path = args.output_path.canonicalize().unwrap();
-
-    let builds_path = outputs_path.join("build");
-    let built_benchmarks = build_benchmarks(&benchmarks, &docker_executable, &builds_path)
-        .unwrap_or_else(|e| {
-            log::error!("could not build benchmarks: {e}");
-            exit(-1);
-        });
-
-    let results = run_benchmarks_on_runners(&built_benchmarks, &runners).unwrap_or_else(|e| {
-        log::error!("could not run benchmarks: {e}");
-        exit(-1);
-    });
-
-    let results_path = outputs_path.join("results");
-    let _result_file_path = record_results(&results_path, args.output_file_name, &results)
-        .unwrap_or_else(|e| {
-            log::error!("could not record results: {e}");
-            exit(-1);
-        });
 }

src/metadata.rs

@@ -5,7 +5,7 @@ use std::{
 };
 
 use glob::glob;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 
 pub trait MetadataParser
 where
@@ -55,7 +55,7 @@ where
     ) -> Result<Self, Box<dyn error::Error>>;
 }
 
-#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize)]
+#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
 pub struct Benchmark {
     pub name: String,
     pub solc_version: String,
@@ -123,7 +123,7 @@ impl MetadataParser for Benchmark {
     }
 }
 
-#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize)]
+#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
 pub struct Runner {
     pub name: String,
     pub entry: PathBuf,