From a74507af3ff8aaa803044efa65a7200521101360 Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Thu, 20 Sep 2018 22:40:55 +0900 Subject: [PATCH 1/3] feat: add benchmark tools --- .gitignore | 5 ++- .travis.yml | 14 ++++++++- tools/benchmark.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++ tools/format.py | 2 +- website/app.js | 26 ++++++++++++++++ website/index.html | 14 +++++++++ 6 files changed, 136 insertions(+), 3 deletions(-) create mode 100755 tools/benchmark.py create mode 100644 website/app.js create mode 100644 website/index.html diff --git a/.gitignore b/.gitignore index 51eb81ca8bb14..89bdcc37ee14c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,10 @@ Cargo.lock yarn.lock # npm deps node_modules +# editor files .idea - # RLS generated files /target/ +# benchmark temp files +/benchmark.json +/gh-pages diff --git a/.travis.yml b/.travis.yml index 27f50544eb41d..d4a8085505eb6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,11 +37,23 @@ script: - bash -c "sleep 2100; pkill ninja" & - "./tools/build.py -j2" - "./tools/test.py $DENO_BUILD_PATH" +after_success: +# benchmark +- cargo install hyperfine +- ./tools/benchmark.py $DENO_BUILD_PATH +- cp website/*.* gh-pages/ before_deploy: | # gzip and name release to denote platform gzip -c $DENO_BUILD_PATH/deno > $DENO_BUILD_PATH/deno_${TRAVIS_OS_NAME}_x64.gz deploy: - provider: releases +- provider: pages + local-dir: gh-pages + skip-cleanup: true + github-token: $GITHUB_TOKEN + keep-history: true + on: + branch: master +- provider: releases api_key: secure: RIwv515oDcPAlEvt7uG8FeSFi6Tz6ODJUOXcFj6FYUPszxJ7Cg1kBLKln+fNW5OeOc52VsaZb/vPZ85skyEM6zk2ijL9FcSnnfNEm548w77iH6G0sk09NgBTy6KRXES6NZHD9jN1YTWYkT2G1NQi7mLqxR8a8pnWTbeK5HhtSWGsZPtXqf5iQbvnWsmKA0/w+FIgKupU0xe/qsYjh0eMLYpZDUWoKO0VxBKJ/ix5Uz91aJTjMIcHHij+ALg4pk+FkDotdyx39XB9b25KDxGuaI7NxWjSPzDxs/ZBHP6QYDLO0ti93ftvLAxRoBKPFoZrXqAu3KG9anr9WvxE40DO9OdV0VX2ZUatMUQm3DpSheN8ml2sErFqjIInqlpkdOVDYORz7FikPxkb9DKt+iuyFfxPRa4YWJv2tg8+Hy/nRCQw69OoKqrSNJ8KJDB3OjYbRBtdHz79RLJhTsGZla6RiyXfM7crR7CbFjbwdbW3Pt60t24fhvXQ0SwR0QTgzS/ieYEQHq/9GtSQA/Tn4kdIkyN6BdOMrQd/aUtgKmNdqbSlfmWGNyNZIxHdB+3RrTNT1tagkRI4UHEUfEujpIdYKwLjv0Xmi/VtTM+zOSkzHsIWGPfHBmIGnXfAItUHqivQYJ15E+dzg3T1CEbBxkDQtvwien9Fa8/pBsMkyovl8ps= file: "$DENO_BUILD_PATH/deno_${TRAVIS_OS_NAME}_x64.gz" diff --git a/tools/benchmark.py b/tools/benchmark.py new file mode 100755 index 0000000000000..63eefb1b0a9c6 --- /dev/null +++ b/tools/benchmark.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# Performs benchmark, and append data to //gh-pages/data.json. + +import os +import sys +import json +import time +from util import run, run_output, root_path, build_path + +benchmark_types = ["hello", "relative_import"] +benchmark_files = ["tests/002_hello.ts", "tests/003_relative_import.ts"] + +data_file = "gh-pages/data.json" +benchmark_file = "benchmark.json" + + +def read_json(filename): + with open(filename) as json_file: + return json.load(json_file) + + +def write_json(filename, data): + with open(filename, 'w') as outfile: + json.dump(data, outfile) + + +def prepare_gh_pages_dir(): + if os.path.exists("gh-pages"): + return + try: + run([ + "git", "clone", "--depth", "1", "-b", "gh-pages", + "https://github.com/denoland/deno.git", "gh-pages" + ]) + except: + os.mkdir("gh-pages") + with open("gh-pages/data.json", "w") as f: + f.write("[]") # writes empty json data + + +def main(argv): + if len(argv) == 2: + build_dir = sys.argv[1] + elif len(argv) == 1: + build_dir = build_path() + else: + print "Usage: tools/benchmark.py [build_dir]" + sys.exit(1) + + os.chdir(root_path) + prepare_gh_pages_dir() + run(["hyperfine", "--export-json", benchmark_file, "--warmup", "3"] + [ + os.path.join(build_dir, "deno") + " " + file + for file in benchmark_files + ]) + all_data = read_json(data_file) + benchmark_data = read_json(benchmark_file) + sha1 = run_output(["git", "rev-parse", "HEAD"]).strip() + new_data = { + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ"), + "sha1": sha1, + "benchmark": {} + } + for type, data in zip(benchmark_types, benchmark_data["results"]): + new_data["benchmark"][type] = { + "mean": data["mean"], + "stddev": data["stddev"], + "user": data["user"], + "system": data["system"], + "min": data["min"], + "max": data["max"] + } + all_data.append(new_data) + write_json(data_file, all_data) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/tools/format.py b/tools/format.py index 47774c44c1c1c..b91e75635a655 100755 --- a/tools/format.py +++ b/tools/format.py @@ -29,7 +29,7 @@ run(["yapf", "-i"] + glob("tools/*.py") + find_exts("build_extra", ".py")) run(["node", prettier, "--write"] + find_exts("js/", ".js", ".ts") + - find_exts("tests/", ".js", ".ts") + + find_exts("tests/", ".js", ".ts") + find_exts("website/", ".js", ".ts") + ["rollup.config.js", "tsconfig.json", "tslint.json"]) # Requires rustfmt 0.8.2 (flags were different in previous versions) diff --git a/website/app.js b/website/app.js new file mode 100644 index 0000000000000..e65d05e7a4127 --- /dev/null +++ b/website/app.js @@ -0,0 +1,26 @@ +const benchmarkTypes = ["hello", "relative_import"]; + +(async () => { + const data = await (await fetch("./data.json")).json(); + + const benchmarkColumns = benchmarkTypes.map(type => [ + type, + ...data.map(d => { + const benchmark = d.benchmark[type]; + return benchmark ? benchmark.mean : 0; + }) + ]); + + const sha1List = data.map(d => d.sha1); + + c3.generate({ + bindto: "#benchmark-chart", + data: { columns: benchmarkColumns }, + axis: { + x: { + type: "category", + categories: sha1List + } + } + }); +})(); diff --git a/website/index.html b/website/index.html new file mode 100644 index 0000000000000..b15a9c9498a17 --- /dev/null +++ b/website/index.html @@ -0,0 +1,14 @@ + + + + deno benchmark + + + +
+ + + + + + From 37c3abf288eaa544d48786b4aae54f62ac4f38e7 Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Fri, 21 Sep 2018 15:50:35 +0900 Subject: [PATCH 2/3] fix: address feedbacks in #777 --- .travis.yml | 2 +- tools/benchmark.py | 23 +++++++++++++---------- website/app.js | 8 ++++---- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4a8085505eb6..0e16af2e4ea95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ script: - "./tools/build.py -j2" - "./tools/test.py $DENO_BUILD_PATH" after_success: -# benchmark +# TODO: Include hyperfine in //third_party - cargo install hyperfine - ./tools/benchmark.py $DENO_BUILD_PATH - cp website/*.* gh-pages/ diff --git a/tools/benchmark.py b/tools/benchmark.py index 63eefb1b0a9c6..3011f27afe737 100755 --- a/tools/benchmark.py +++ b/tools/benchmark.py @@ -7,11 +7,12 @@ import time from util import run, run_output, root_path, build_path -benchmark_types = ["hello", "relative_import"] -benchmark_files = ["tests/002_hello.ts", "tests/003_relative_import.ts"] +# The list of the tuples of the benchmark name and arguments +benchmarks = [("hello", ["tests/002_hello.ts", "--reload"]), + ("relative_import", ["tests/003_relative_import.ts", + "--reload"])] data_file = "gh-pages/data.json" -benchmark_file = "benchmark.json" def read_json(filename): @@ -34,7 +35,7 @@ def prepare_gh_pages_dir(): ]) except: os.mkdir("gh-pages") - with open("gh-pages/data.json", "w") as f: + with open(data_file, "w") as f: f.write("[]") # writes empty json data @@ -47,12 +48,14 @@ def main(argv): print "Usage: tools/benchmark.py [build_dir]" sys.exit(1) + deno_path = os.path.join(build_dir, "deno") + benchmark_file = os.path.join(build_dir, "benchmark.json") + os.chdir(root_path) prepare_gh_pages_dir() - run(["hyperfine", "--export-json", benchmark_file, "--warmup", "3"] + [ - os.path.join(build_dir, "deno") + " " + file - for file in benchmark_files - ]) + # TODO: Use hyperfine in //third_party + run(["hyperfine", "--export-json", benchmark_file, "--warmup", "3"] + + [deno_path + " " + " ".join(args) for [_, args] in benchmarks]) all_data = read_json(data_file) benchmark_data = read_json(benchmark_file) sha1 = run_output(["git", "rev-parse", "HEAD"]).strip() @@ -61,8 +64,8 @@ def main(argv): "sha1": sha1, "benchmark": {} } - for type, data in zip(benchmark_types, benchmark_data["results"]): - new_data["benchmark"][type] = { + for [[name, _], data] in zip(benchmarks, benchmark_data["results"]): + new_data["benchmark"][name] = { "mean": data["mean"], "stddev": data["stddev"], "user": data["user"], diff --git a/website/app.js b/website/app.js index e65d05e7a4127..7a6566a61b27d 100644 --- a/website/app.js +++ b/website/app.js @@ -1,12 +1,12 @@ -const benchmarkTypes = ["hello", "relative_import"]; +const benchmarkNames = ["hello", "relative_import"]; (async () => { const data = await (await fetch("./data.json")).json(); - const benchmarkColumns = benchmarkTypes.map(type => [ - type, + const benchmarkColumns = benchmarkNames.map(name => [ + name, ...data.map(d => { - const benchmark = d.benchmark[type]; + const benchmark = d.benchmark[name]; return benchmark ? benchmark.mean : 0; }) ]); From fd4867735dd500b87108136b9740a683bfcbb839 Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Fri, 21 Sep 2018 19:50:40 +0900 Subject: [PATCH 3/3] chore: change json locations --- .gitignore | 5 +++-- .travis.yml | 1 + tools/benchmark.py | 20 ++++++++++++-------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 89bdcc37ee14c..bd850eb16d75f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ node_modules .idea # RLS generated files /target/ -# benchmark temp files -/benchmark.json +# export dir for gh-pages /gh-pages +# temp benchmark data +/website/data.json diff --git a/.travis.yml b/.travis.yml index 0e16af2e4ea95..ae8d3c4b1868a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,6 +41,7 @@ after_success: # TODO: Include hyperfine in //third_party - cargo install hyperfine - ./tools/benchmark.py $DENO_BUILD_PATH +# export website assets for deployment - cp website/*.* gh-pages/ before_deploy: | # gzip and name release to denote platform diff --git a/tools/benchmark.py b/tools/benchmark.py index 3011f27afe737..f0f9e4ac62782 100755 --- a/tools/benchmark.py +++ b/tools/benchmark.py @@ -1,10 +1,14 @@ #!/usr/bin/env python -# Performs benchmark, and append data to //gh-pages/data.json. +# Performs benchmark and append data to //website/data.json. +# If //website/data.json doesn't exist, this script tries to import it from gh-pages branch. +# To view the results locally run ./tools/http_server.py and visit +# http://localhost:4545/website import os import sys import json import time +import shutil from util import run, run_output, root_path, build_path # The list of the tuples of the benchmark name and arguments @@ -12,7 +16,8 @@ ("relative_import", ["tests/003_relative_import.ts", "--reload"])] -data_file = "gh-pages/data.json" +gh_pages_data_file = "gh-pages/data.json" +data_file = "website/data.json" def read_json(filename): @@ -25,18 +30,17 @@ def write_json(filename, data): json.dump(data, outfile) -def prepare_gh_pages_dir(): - if os.path.exists("gh-pages"): +def import_data_from_gh_pages(): + if os.path.exists(data_file): return try: run([ "git", "clone", "--depth", "1", "-b", "gh-pages", "https://github.com/denoland/deno.git", "gh-pages" ]) + shutil.copy(gh_pages_data_file, data_file) except: - os.mkdir("gh-pages") - with open(data_file, "w") as f: - f.write("[]") # writes empty json data + write_json(data_file, []) # writes empty json data def main(argv): @@ -52,7 +56,7 @@ def main(argv): benchmark_file = os.path.join(build_dir, "benchmark.json") os.chdir(root_path) - prepare_gh_pages_dir() + import_data_from_gh_pages() # TODO: Use hyperfine in //third_party run(["hyperfine", "--export-json", benchmark_file, "--warmup", "3"] + [deno_path + " " + " ".join(args) for [_, args] in benchmarks])