From ea26bc045868578e17ae91b78da427c40f660086 Mon Sep 17 00:00:00 2001 From: Ziyad Edher Date: Fri, 14 Oct 2022 00:39:50 -0700 Subject: [PATCH] Use native Python and update results (#19) --- README.md | 20 +++++----- .../snailtracer/benchmark.evm-bench.json | 2 +- runners/py-evm/cpython/entry.sh | 2 +- runners/py-evm/poetry.lock | 4 +- runners/py-evm/pyproject.toml | 2 +- runners/py-evm/runner.py | 6 +-- src/results.rs | 37 +++++++++++++++---- 7 files changed, 47 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 94ef9185..5e6212c8 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,16 @@ evm-bench makes it easy to compare EVM performance in a scalable, standardized, and portable way. -| | py-evm.cpython | py-evm.pypy | pyrevm | revm | -| ----------------------- | -------------- | ----------- | ------- | ------ | -| erc20.approval-transfer | 1.9362s | 424.2ms | 16.8ms | 9.8ms | -| erc20.mint | 1.8968s | 374ms | 15ms | 5.6ms | -| erc20.transfer | 2.7296s | 482.6ms | 23.4ms | 11.4ms | -| snailtracer | 37.861s | 7.409s | 131.7ms | 60.7ms | -| ten-thousand-hashes | 4.1496s | 632ms | 17ms | 6.8ms | -| | | | | | -| **sum** | 48.6s | 9.32s | 203ms | 94.3ms | -| **relative** | 515x | 98.8x | 2.15x | 1.00x | +| | py-evm.cpython | py-evm.pypy | pyrevm | revm | +| ----------------------- | -------------- | ----------- | ------ | ------ | +| erc20.approval-transfer | 1.4258s | 377.2ms | 16.8ms | 10ms | +| erc20.mint | 1.4092s | 347.8ms | 14.4ms | 5.6ms | +| erc20.transfer | 1.9988s | 420.8ms | 22.6ms | 11.4ms | +| snailtracer | 17.857s | 5.17s | 138ms | 58ms | +| ten-thousand-hashes | 3.4344s | 665.6ms | 18.2ms | 7ms | +| | | | | | +| **sum** | 26.1252s | 6.9814s | 210ms | 92ms | +| **relative** | 284x | 75.9x | 2.28x | 1.00x | To reproduce these results, check out [usage with the evm-bench suite below](#with-the-evm-bench-suite). diff --git a/benchmarks/snailtracer/benchmark.evm-bench.json b/benchmarks/snailtracer/benchmark.evm-bench.json index cde81102..eb577b16 100644 --- a/benchmarks/snailtracer/benchmark.evm-bench.json +++ b/benchmarks/snailtracer/benchmark.evm-bench.json @@ -1,7 +1,7 @@ { "$schema": "../schema.json", "name": "snailtracer", - "num-runs": 3, + "num-runs": 1, "solc-version": "0.4.26", "contract": "SnailTracer.sol", "calldata": "30627b7c" diff --git a/runners/py-evm/cpython/entry.sh b/runners/py-evm/cpython/entry.sh index 38a009bb..1b24322a 100755 --- a/runners/py-evm/cpython/entry.sh +++ b/runners/py-evm/cpython/entry.sh @@ -4,7 +4,7 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR -poetry env use python3.9 >&2 +poetry env use python3 >&2 poetry install >&2 poetry update >&2 poetry run python ../runner.py $@ diff --git a/runners/py-evm/poetry.lock b/runners/py-evm/poetry.lock index 6009fe6c..59fe2e23 100644 --- a/runners/py-evm/poetry.lock +++ b/runners/py-evm/poetry.lock @@ -358,8 +358,8 @@ python-versions = ">=3.7" [metadata] lock-version = "1.1" -python-versions = "^3.9,<3.10" -content-hash = "76aafa3a281e0ea848ca6a5c27c320253a70a6ad565d7267c081f54496a4f767" +python-versions = "^3.9" +content-hash = "197ec2cb8f9c7d7c04cff2e822e388fb151daa3bc0e786e0e90e3eb425c0326b" [metadata.files] black = [] diff --git a/runners/py-evm/pyproject.toml b/runners/py-evm/pyproject.toml index d0ac2bbd..890dce33 100644 --- a/runners/py-evm/pyproject.toml +++ b/runners/py-evm/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = ["Ziyad Edher "] [tool.poetry.dependencies] -python = "^3.9,<3.10" +python = "^3.9" py-evm = "^0.6.0-alpha.1" [tool.poetry.dev-dependencies] diff --git a/runners/py-evm/runner.py b/runners/py-evm/runner.py index 728fc81d..45688821 100755 --- a/runners/py-evm/runner.py +++ b/runners/py-evm/runner.py @@ -1,6 +1,6 @@ -import argparse -from typing import Final, cast +from typing import cast, Final +import argparse import pathlib import time @@ -44,7 +44,7 @@ def _construct_chain() -> eth.chains.base.MiningChain: chain = chain_class.from_genesis( eth.db.atomic.AtomicDB(), genesis_params={ - "difficulty": 100, + "difficulty": 1, "gas_limit": 2 * GAS_LIMIT, }, ) diff --git a/src/results.rs b/src/results.rs index c03b03a6..81dfe34f 100644 --- a/src/results.rs +++ b/src/results.rs @@ -101,31 +101,52 @@ pub fn print_results(results_file_path: &Path) -> Result<(), Box>(); runs.sort_by_key(|(b, _)| b.clone()); + let mut runner_times = HashMap::>::new(); + let mut builder = Builder::default(); for (benchmark_name, benchmark_runs) in runs.iter() { let vals = runner_names.iter().map(|runner_name| { let run = benchmark_runs.get(runner_name)?; - Some( - run.run_times - .iter() - .fold(Duration::ZERO, |a, v| a + v.clone()) - .div_f64(run.run_times.len() as f64), - ) + let avg_run_time = run + .run_times + .iter() + .fold(Duration::ZERO, |a, v| a + v.clone()) + .div_f64(run.run_times.len() as f64); + runner_times + .entry(runner_name.clone()) + .or_default() + .push(avg_run_time); + Some(avg_run_time) }); + let mut record = vec![benchmark_name.clone()]; record.extend( vals.map(|val| Some(format!("{:?}", val?))) - .map(|s| s.unwrap_or("".into())), + .map(|s| s.unwrap_or_default()), ); builder.add_record(record); } + let average_runner_times = runner_times + .into_iter() + .map(|(name, times)| (name, times.iter().sum::())) + .collect::>(); + let mut record = vec!["sum".to_string()]; + record.extend( + runner_names + .iter() + .map(|runner_name| average_runner_times.get(runner_name)) + .map(|val| Some(format!("{:?}", val?))) + .map(|s| s.unwrap_or_default()), + ); + builder.add_record(record); + let mut columns = vec!["".to_owned()]; columns.extend(runner_names); builder.set_columns(columns); let mut table = builder.build(); - table.with(Style::rounded()); + table.with(Style::markdown()); println!("{}", table); Ok(())