From 1a728d0ebd656f0c46403d79f8b2936122dd06eb Mon Sep 17 00:00:00 2001 From: Valdis Date: Sun, 12 May 2019 23:09:12 +0200 Subject: [PATCH 01/32] autodiscover all configs add all the latest configs up to 0.4.0-SNAPSHOT --- .../build.sbt | 6 +++ .../compile | 1 + .../plugins.sbt | 1 + confs/scala-native-0.3.9-SNAPSHOT-commix/run | 1 + .../build.sbt | 6 +++ .../compile | 1 + .../plugins.sbt | 1 + .../scala-native-0.3.9-SNAPSHOT-full-lto/run | 1 + .../build.sbt | 6 +++ .../compile | 1 + .../plugins.sbt | 1 + confs/scala-native-0.3.9-SNAPSHOT-no-lto/run | 1 + confs/scala-native-0.3.9-SNAPSHOT/build.sbt | 6 +++ confs/scala-native-0.3.9-SNAPSHOT/compile | 1 + confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt | 1 + confs/scala-native-0.3.9-SNAPSHOT/run | 1 + confs/scala-native-0.3.9/build.sbt | 1 + .../build.sbt | 6 +++ .../compile | 1 + .../plugins.sbt | 1 + confs/scala-native-0.4.0-SNAPSHOT-commix/run | 1 + confs/scala-native-0.4.0-SNAPSHOT/build.sbt | 6 +++ confs/scala-native-0.4.0-SNAPSHOT/compile | 1 + confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt | 1 + confs/scala-native-0.4.0-SNAPSHOT/run | 1 + scripts/run.py | 42 +++++++++++++++---- 26 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-commix/compile create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-commix/run create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-full-lto/run create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt create mode 100644 confs/scala-native-0.3.9-SNAPSHOT-no-lto/run create mode 100644 confs/scala-native-0.3.9-SNAPSHOT/build.sbt create mode 100644 confs/scala-native-0.3.9-SNAPSHOT/compile create mode 100644 confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt create mode 100644 confs/scala-native-0.3.9-SNAPSHOT/run create mode 100644 confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt create mode 100644 confs/scala-native-0.4.0-SNAPSHOT-commix/compile create mode 100644 confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt create mode 100644 confs/scala-native-0.4.0-SNAPSHOT-commix/run create mode 100644 confs/scala-native-0.4.0-SNAPSHOT/build.sbt create mode 100644 confs/scala-native-0.4.0-SNAPSHOT/compile create mode 100644 confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt create mode 100644 confs/scala-native-0.4.0-SNAPSHOT/run diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt new file mode 100644 index 0000000..2104b7a --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt @@ -0,0 +1,6 @@ +scalaVersion := "2.11.12" +enablePlugins(ScalaNativePlugin) +nativeLinkStubs := true +nativeGC := "commix" +nativeMode := "release" +nativeLTO := "thin" diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/compile b/confs/scala-native-0.3.9-SNAPSHOT-commix/compile new file mode 100644 index 0000000..2f3f09f --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-commix/compile @@ -0,0 +1 @@ +nativeLink diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt new file mode 100644 index 0000000..c1423b6 --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT") diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/run b/confs/scala-native-0.3.9-SNAPSHOT-commix/run new file mode 100644 index 0000000..ae89e34 --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-commix/run @@ -0,0 +1 @@ +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt new file mode 100644 index 0000000..fae449a --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt @@ -0,0 +1,6 @@ +scalaVersion := "2.11.12" +enablePlugins(ScalaNativePlugin) +nativeLinkStubs := true +nativeGC := "immix" +nativeMode := "release" +nativeLTO := "full" diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile new file mode 100644 index 0000000..2f3f09f --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile @@ -0,0 +1 @@ +nativeLink diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt new file mode 100644 index 0000000..c1423b6 --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT") diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/run b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/run new file mode 100644 index 0000000..ae89e34 --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/run @@ -0,0 +1 @@ +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt new file mode 100644 index 0000000..2fc1873 --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt @@ -0,0 +1,6 @@ +scalaVersion := "2.11.12" +enablePlugins(ScalaNativePlugin) +nativeLinkStubs := true +nativeGC := "immix" +nativeMode := "release" +nativeLTO := "none" diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile new file mode 100644 index 0000000..2f3f09f --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile @@ -0,0 +1 @@ +nativeLink diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt new file mode 100644 index 0000000..c1423b6 --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT") diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/run b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/run new file mode 100644 index 0000000..ae89e34 --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/run @@ -0,0 +1 @@ +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.3.9-SNAPSHOT/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT/build.sbt new file mode 100644 index 0000000..ae87f31 --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT/build.sbt @@ -0,0 +1,6 @@ +scalaVersion := "2.11.12" +enablePlugins(ScalaNativePlugin) +nativeLinkStubs := true +nativeGC := "immix" +nativeMode := "release" +nativeLTO := "thin" diff --git a/confs/scala-native-0.3.9-SNAPSHOT/compile b/confs/scala-native-0.3.9-SNAPSHOT/compile new file mode 100644 index 0000000..2f3f09f --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT/compile @@ -0,0 +1 @@ +nativeLink diff --git a/confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt new file mode 100644 index 0000000..c1423b6 --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT") diff --git a/confs/scala-native-0.3.9-SNAPSHOT/run b/confs/scala-native-0.3.9-SNAPSHOT/run new file mode 100644 index 0000000..ae89e34 --- /dev/null +++ b/confs/scala-native-0.3.9-SNAPSHOT/run @@ -0,0 +1 @@ +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.3.9/build.sbt b/confs/scala-native-0.3.9/build.sbt index b4a5690..ae87f31 100644 --- a/confs/scala-native-0.3.9/build.sbt +++ b/confs/scala-native-0.3.9/build.sbt @@ -3,3 +3,4 @@ enablePlugins(ScalaNativePlugin) nativeLinkStubs := true nativeGC := "immix" nativeMode := "release" +nativeLTO := "thin" diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt b/confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt new file mode 100644 index 0000000..2104b7a --- /dev/null +++ b/confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt @@ -0,0 +1,6 @@ +scalaVersion := "2.11.12" +enablePlugins(ScalaNativePlugin) +nativeLinkStubs := true +nativeGC := "commix" +nativeMode := "release" +nativeLTO := "thin" diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/compile b/confs/scala-native-0.4.0-SNAPSHOT-commix/compile new file mode 100644 index 0000000..2f3f09f --- /dev/null +++ b/confs/scala-native-0.4.0-SNAPSHOT-commix/compile @@ -0,0 +1 @@ +nativeLink diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt b/confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt new file mode 100644 index 0000000..2a63bf0 --- /dev/null +++ b/confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-SNAPSHOT") diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/run b/confs/scala-native-0.4.0-SNAPSHOT-commix/run new file mode 100644 index 0000000..ae89e34 --- /dev/null +++ b/confs/scala-native-0.4.0-SNAPSHOT-commix/run @@ -0,0 +1 @@ +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-SNAPSHOT/build.sbt b/confs/scala-native-0.4.0-SNAPSHOT/build.sbt new file mode 100644 index 0000000..ae87f31 --- /dev/null +++ b/confs/scala-native-0.4.0-SNAPSHOT/build.sbt @@ -0,0 +1,6 @@ +scalaVersion := "2.11.12" +enablePlugins(ScalaNativePlugin) +nativeLinkStubs := true +nativeGC := "immix" +nativeMode := "release" +nativeLTO := "thin" diff --git a/confs/scala-native-0.4.0-SNAPSHOT/compile b/confs/scala-native-0.4.0-SNAPSHOT/compile new file mode 100644 index 0000000..2f3f09f --- /dev/null +++ b/confs/scala-native-0.4.0-SNAPSHOT/compile @@ -0,0 +1 @@ +nativeLink diff --git a/confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt b/confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt new file mode 100644 index 0000000..2a63bf0 --- /dev/null +++ b/confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-SNAPSHOT") diff --git a/confs/scala-native-0.4.0-SNAPSHOT/run b/confs/scala-native-0.4.0-SNAPSHOT/run new file mode 100644 index 0000000..ae89e34 --- /dev/null +++ b/confs/scala-native-0.4.0-SNAPSHOT/run @@ -0,0 +1 @@ +target/scala-2.11/scala-native-benchmarks-out diff --git a/scripts/run.py b/scripts/run.py index 3dc2eda..56f2d2e 100755 --- a/scripts/run.py +++ b/scripts/run.py @@ -5,6 +5,7 @@ import subprocess as subp import shutil as sh import time +import argparse def mkdir(path): try: @@ -66,15 +67,15 @@ def compile(bench, compilecmd): 'histogram.Histogram', ] -configurations = [ - 'scala-native-0.3.9', - 'scala-native-0.4.0-M2', - 'scala-native-0.4.0-2.11', - 'scala-native-0.4.0-2.12', -] +stable = 'scala-native-0.4.0' +latest = 'scala-native-0.4.1-SNAPSHOT' + +confs_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + "/confs" + +all_configs = next(os.walk(confs_path))[1] if 'GRAALVM_HOME' in os.environ: - configurations += [ + all_configs += [ 'native-image', 'native-image-pgo', ] @@ -83,7 +84,34 @@ def compile(bench, compilecmd): batches = 2000 batch_size = 1 + +def expand_wild_cards(arg): + if arg == None: + return arg + elif arg.startswith("latest"): + return latest + arg[len("latest"):] + elif arg.startswith("stable"): + return stable + arg[len("stable"):] + else: + return arg + + if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("set", nargs='*', default=[None]) + args = parser.parse_args() + print args + + configurations = [] + for choice in args.set: + expanded = expand_wild_cards(choice) + if expanded == None: + configurations = [stable, latest] + else: + configurations += [expanded] + + print "configurations:", configurations + for conf in configurations: for bench in benchmarks: print('--- conf: {}, bench: {}'.format(conf, bench)) From 47e0591ad2a881b440062216333448aadf074c3c Mon Sep 17 00:00:00 2001 From: Valdis Date: Sun, 12 May 2019 23:23:54 +0200 Subject: [PATCH 02/32] explicitly set project names --- confs/jvm/build.sbt | 1 + confs/native-image-pgo/build.sbt | 1 + confs/native-image/build.sbt | 1 + confs/scala-native-0.3.7/build.sbt | 1 + confs/scala-native-0.3.8/build.sbt | 1 + confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt | 1 + confs/scala-native-0.3.9-SNAPSHOT/build.sbt | 1 + confs/scala-native-0.3.9/build.sbt | 1 + confs/scala-native-0.4.0-SNAPSHOT/build.sbt | 1 + 9 files changed, 9 insertions(+) diff --git a/confs/jvm/build.sbt b/confs/jvm/build.sbt index 9874e27..ef3680d 100644 --- a/confs/jvm/build.sbt +++ b/confs/jvm/build.sbt @@ -1 +1,2 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" diff --git a/confs/native-image-pgo/build.sbt b/confs/native-image-pgo/build.sbt index 65e12d6..c622b0a 100644 --- a/confs/native-image-pgo/build.sbt +++ b/confs/native-image-pgo/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" val instrumentedRuns = settingKey[Int]("The number of instrumented runs.") diff --git a/confs/native-image/build.sbt b/confs/native-image/build.sbt index 065cf09..4073552 100644 --- a/confs/native-image/build.sbt +++ b/confs/native-image/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" val nativeImage = taskKey[Unit]("Builds a native image of a benchmark.") diff --git a/confs/scala-native-0.3.7/build.sbt b/confs/scala-native-0.3.7/build.sbt index b4a5690..702aa9d 100644 --- a/confs/scala-native-0.3.7/build.sbt +++ b/confs/scala-native-0.3.7/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/confs/scala-native-0.3.8/build.sbt b/confs/scala-native-0.3.8/build.sbt index b4a5690..702aa9d 100644 --- a/confs/scala-native-0.3.8/build.sbt +++ b/confs/scala-native-0.3.8/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt index fae449a..2bcb9cc 100644 --- a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt +++ b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/confs/scala-native-0.3.9-SNAPSHOT/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT/build.sbt index ae87f31..441c07f 100644 --- a/confs/scala-native-0.3.9-SNAPSHOT/build.sbt +++ b/confs/scala-native-0.3.9-SNAPSHOT/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/confs/scala-native-0.3.9/build.sbt b/confs/scala-native-0.3.9/build.sbt index ae87f31..441c07f 100644 --- a/confs/scala-native-0.3.9/build.sbt +++ b/confs/scala-native-0.3.9/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/confs/scala-native-0.4.0-SNAPSHOT/build.sbt b/confs/scala-native-0.4.0-SNAPSHOT/build.sbt index ae87f31..441c07f 100644 --- a/confs/scala-native-0.4.0-SNAPSHOT/build.sbt +++ b/confs/scala-native-0.4.0-SNAPSHOT/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true From 54e03ea9c58fb12bae107684de99366f55c90a5b Mon Sep 17 00:00:00 2001 From: Valdis Date: Sat, 18 May 2019 10:09:36 +0200 Subject: [PATCH 03/32] 0.3.9 release did not have nativeLTO --- confs/scala-native-0.3.9/build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/confs/scala-native-0.3.9/build.sbt b/confs/scala-native-0.3.9/build.sbt index 441c07f..702aa9d 100644 --- a/confs/scala-native-0.3.9/build.sbt +++ b/confs/scala-native-0.3.9/build.sbt @@ -4,4 +4,3 @@ enablePlugins(ScalaNativePlugin) nativeLinkStubs := true nativeGC := "immix" nativeMode := "release" -nativeLTO := "thin" From c43fbcd04d52747182fe19ca59b4cf3b257eb86f Mon Sep 17 00:00:00 2001 From: Valdis Date: Sat, 18 May 2019 10:05:11 +0200 Subject: [PATCH 04/32] Ignore build.sbt and plugins.sbt files because they keep getting changed by the runner. --- .gitignore | 2 ++ confs/jvm/plugins.sbt | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 confs/jvm/plugins.sbt diff --git a/.gitignore b/.gitignore index 528a077..ec7b92a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ bin/.scalafmt* results/ *.iprof .idea +build.sbt +project/plugins.sbt \ No newline at end of file diff --git a/confs/jvm/plugins.sbt b/confs/jvm/plugins.sbt deleted file mode 100644 index 8b13789..0000000 --- a/confs/jvm/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ - From 90be66078ce8c5992f1d9784945100f8d83f7539 Mon Sep 17 00:00:00 2001 From: Valdis Date: Sat, 18 May 2019 10:02:49 +0200 Subject: [PATCH 05/32] 0.3.9 release did not have nativeLTO --- confs/scala-native-0.3.9-SNAPSHOT/build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/confs/scala-native-0.3.9-SNAPSHOT/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT/build.sbt index 441c07f..702aa9d 100644 --- a/confs/scala-native-0.3.9-SNAPSHOT/build.sbt +++ b/confs/scala-native-0.3.9-SNAPSHOT/build.sbt @@ -4,4 +4,3 @@ enablePlugins(ScalaNativePlugin) nativeLinkStubs := true nativeGC := "immix" nativeMode := "release" -nativeLTO := "thin" From 92ea36f4cd5963ebff92805d484c98bf76d7c3de Mon Sep 17 00:00:00 2001 From: Valdis Date: Wed, 15 May 2019 23:44:04 +0200 Subject: [PATCH 06/32] refactor run.py add interactive mode --- confs/scala-native-0.4.0-SNAPSHOT/build.sbt | 1 - scripts/run.py | 156 +++++--------------- scripts/shared/__init__.py | 0 scripts/shared/benchmarks.py | 31 ++++ scripts/shared/configurations.py | 60 ++++++++ scripts/shared/file_utils.py | 39 +++++ 6 files changed, 166 insertions(+), 121 deletions(-) create mode 100644 scripts/shared/__init__.py create mode 100644 scripts/shared/benchmarks.py create mode 100644 scripts/shared/configurations.py create mode 100644 scripts/shared/file_utils.py diff --git a/confs/scala-native-0.4.0-SNAPSHOT/build.sbt b/confs/scala-native-0.4.0-SNAPSHOT/build.sbt index 441c07f..ae87f31 100644 --- a/confs/scala-native-0.4.0-SNAPSHOT/build.sbt +++ b/confs/scala-native-0.4.0-SNAPSHOT/build.sbt @@ -1,4 +1,3 @@ -name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/scripts/run.py b/scripts/run.py index 56f2d2e..92cda54 100755 --- a/scripts/run.py +++ b/scripts/run.py @@ -1,51 +1,9 @@ #!/usr/bin/env python -import sys -import os -import errno -import subprocess as subp -import shutil as sh -import time import argparse +import os -def mkdir(path): - try: - os.makedirs(path) - except OSError as exc: # Python >2.5 - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - -def slurp(path): - with open(path) as f: - return f.read().strip() - -def where(cmd): - if os.path.isfile(cmd): - return cmd - else: - paths = os.environ['PATH'].split(os.pathsep) - for p in paths: - f = os.path.join(p, cmd) - if os.path.isfile(f): - return f - else: - return None - -def run(cmd): - print(">>> " + str(cmd)) - return subp.check_output(cmd) - -def compile(bench, compilecmd): - start = time.time_ns() - cmd = [sbt, '-J-Xmx6G', 'clean'] - cmd.append('set mainClass in Compile := Some("{}")'.format(bench)) - cmd.append(compilecmd) - run(cmd) - end = time.time_ns() - return end - start - -sbt = where('sbt') +from shared.benchmarks import Benchmark +from shared.configurations import Configuration benchmarks = [ 'bounce.BounceBenchmark', @@ -70,23 +28,9 @@ def compile(bench, compilecmd): stable = 'scala-native-0.4.0' latest = 'scala-native-0.4.1-SNAPSHOT' -confs_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + "/confs" - -all_configs = next(os.walk(confs_path))[1] - -if 'GRAALVM_HOME' in os.environ: - all_configs += [ - 'native-image', - 'native-image-pgo', - ] - -runs = 20 -batches = 2000 -batch_size = 1 - def expand_wild_cards(arg): - if arg == None: + if arg is None: return arg elif arg.startswith("latest"): return latest + arg[len("latest"):] @@ -98,67 +42,39 @@ def expand_wild_cards(arg): if __name__ == "__main__": parser = argparse.ArgumentParser() + parser.add_argument("-i", help="interactive mode", action="store_true") parser.add_argument("set", nargs='*', default=[None]) args = parser.parse_args() print args - configurations = [] - for choice in args.set: - expanded = expand_wild_cards(choice) - if expanded == None: - configurations = [stable, latest] - else: - configurations += [expanded] - - print "configurations:", configurations - - for conf in configurations: - for bench in benchmarks: - print('--- conf: {}, bench: {}'.format(conf, bench)) - - resultsdir = os.path.join('results', conf, bench) - mkdir(resultsdir) - - bindir = os.path.join("binaries", conf, bench) - mkdir(bindir) - - binfile = os.path.join(bindir, bench) - - input = slurp(os.path.join('input', bench)) - output = slurp(os.path.join('output', bench)) - compilecmd = slurp(os.path.join('confs', conf, 'compile')) - runcmd = slurp(os.path.join('confs', conf, 'run')).replace('$BENCH', bench).replace('$HOME', os.environ['HOME']).split(' ') - - if not os.path.exists(binfile): - if os.path.exists(os.path.join('confs', conf, 'build.sbt')): - sh.copyfile(os.path.join('confs', conf, 'build.sbt'), 'build.sbt') - else: - os.remove('build.sbt') - - if os.path.exists(os.path.join('confs', conf, 'build.properties')): - sh.copyfile(os.path.join('confs', conf, 'build.properties'), 'project/build.properties') - else: - os.remove('project/build.properties') - - if os.path.exists(os.path.join('confs', conf, 'plugins.sbt')): - sh.copyfile(os.path.join('confs', conf, 'plugins.sbt'), 'project/plugins.sbt') - else: - os.remove('project/plugins.sbt') - - compile_time = compile(bench, compilecmd) - - with open(os.path.join(resultsdir, "compile_time"), 'w') as f: - f.write(str(compile_time)) - - sh.move(runcmd[0], binfile) - - for n in range(runs): - print('--- run {}/{}'.format(n, runs)) - - cmd = [binfile] - cmd.extend(runcmd[1:]) - cmd.extend([str(batches), str(batch_size), input, output]) - out = run(cmd) - with open(os.path.join(resultsdir, str(n)), 'wb') as resultfile: - resultfile.write(out) - + if args.i: + from IPython import embed + embed() + else: + configurations = [] + for choice in args.set: + expanded = expand_wild_cards(choice) + if expanded is None: + configurations = [stable, latest] + else: + configurations += [expanded] + + print "configurations:", configurations + + for conf_name in configurations: + conf = Configuration(conf_name) + conf.make_active() + for bench_name in benchmarks: + bench = Benchmark(bench_name) + print('--- conf: {}, bench: {}'.format(conf, bench)) + + bench.compile(conf) + resultsdir = bench.ensure_results_dir(conf) + + runs = conf.runs + for n in xrange(runs): + print('--- run {}/{}'.format(n, runs)) + + out = bench.run(conf) + with open(os.path.join(resultsdir, str(n)), 'wb') as resultfile: + resultfile.write(out) diff --git a/scripts/shared/__init__.py b/scripts/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/shared/benchmarks.py b/scripts/shared/benchmarks.py new file mode 100644 index 0000000..6a90a01 --- /dev/null +++ b/scripts/shared/benchmarks.py @@ -0,0 +1,31 @@ +import os + +from shared.file_utils import sbt, run, slurp, mkdir + + +class Benchmark: + + def __init__(self, name): + self.name = name + + def compile(self, conf): + cmd = [sbt, '-J-Xmx6G', 'clean'] + cmd.append('set mainClass in Compile := Some("{}")'.format(self.name)) + cmd.append(conf.compile_cmd()) + return run(cmd) + + def run(self, conf): + run_cmd = conf.run_cmd() + input = slurp(os.path.join('input', self.name)) + output = slurp(os.path.join('output', self.name)) + cmd = [] + cmd.extend(run_cmd) + cmd.extend([str(conf.batches), str(conf.batch_size), input, output]) + + def ensure_results_dir(self, conf): + dir = os.path.join(conf.ensure_results_dir(), self.name) + mkdir(dir) + return dir + + def __str__(self): + return self.name diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py new file mode 100644 index 0000000..b0a8677 --- /dev/null +++ b/scripts/shared/configurations.py @@ -0,0 +1,60 @@ +import os +import shutil as sh + +from shared.file_utils import slurp, mkdir + +confs_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + "/confs" + +default_batches = 4000 +default_runs = 20 + + +class Configuration: + + def __init__(self, name, batches=default_batches, runs=default_runs): + self.name = name + self.conf_dir = os.path.join(confs_path, self.name) + self.results_dir = os.path.join('results', self.name) + self.batches = batches + self.batch_size = 1 + self.runs = runs + print self.conf_dir + + def make_active(self): + build_sbt_src = os.path.join(self.conf_dir, 'build.sbt') + if os.path.exists(build_sbt_src): + sh.copyfile(build_sbt_src, 'build.sbt') + else: + os.remove('build.sbt') + + plugins_sbt_src = os.path.join(self.conf_dir, 'plugins.sbt') + if os.path.exists(plugins_sbt_src): + sh.copyfile(plugins_sbt_src, 'project/plugins.sbt') + else: + os.remove('project/plugins.sbt') + + def run_cmd(self, bench): + return slurp(os.path.join(self.conf_dir, 'run')) \ + .replace('$BENCH', bench) \ + .replace('$HOME', os.environ['HOME']) \ + .split(' ') + + def compile_cmd(self): + return slurp(os.path.join(self.conf_dir, 'compile')) + + def ensure_results_dir(self): + dir = self.results_dir + mkdir(dir) + return dir + + def __str__(self): + return self.name + + +all_configs = next(os.walk(confs_path))[1] + +if 'GRAALVM_HOME' in os.environ: + all_configs += [ + 'native-image', + 'native-image-pgo', + ] diff --git a/scripts/shared/file_utils.py b/scripts/shared/file_utils.py new file mode 100644 index 0000000..90d7968 --- /dev/null +++ b/scripts/shared/file_utils.py @@ -0,0 +1,39 @@ +import os +import errno +import subprocess as subp + + +def mkdir(path): + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + +def slurp(path): + with open(path) as f: + return f.read().strip() + + +def where(cmd): + if os.path.isfile(cmd): + return cmd + else: + paths = os.environ['PATH'].split(os.pathsep) + for p in paths: + f = os.path.join(p, cmd) + if os.path.isfile(f): + return f + else: + return None + + +def run(cmd): + print(">>> " + str(cmd)) + return subp.check_output(cmd) + + +sbt = where('sbt') From 5e0c1f3371f02921aed64fbb69cd9bc326d392f8 Mon Sep 17 00:00:00 2001 From: Valdis Date: Thu, 16 May 2019 17:22:16 +0200 Subject: [PATCH 07/32] extract even more --- scripts/run.py | 45 ++++---------------------------- scripts/shared/benchmarks.py | 21 ++++++++++++++- scripts/shared/configurations.py | 23 +++++++++++++++- 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/scripts/run.py b/scripts/run.py index 92cda54..0a431f7 100755 --- a/scripts/run.py +++ b/scripts/run.py @@ -1,32 +1,11 @@ #!/usr/bin/env python import argparse -import os -from shared.benchmarks import Benchmark +from shared.benchmarks import all_benchmarks from shared.configurations import Configuration -benchmarks = [ - 'bounce.BounceBenchmark', - 'list.ListBenchmark', - 'queens.QueensBenchmark', - 'richards.RichardsBenchmark', - 'permute.PermuteBenchmark', - 'deltablue.DeltaBlueBenchmark', - 'tracer.TracerBenchmark', - 'json.JsonBenchmark', - 'sudoku.SudokuBenchmark', - 'brainfuck.BrainfuckBenchmark', - 'cd.CDBenchmark', - 'kmeans.KmeansBenchmark', - 'nbody.NbodyBenchmark', - 'rsc.RscBenchmark', - 'gcbench.GCBenchBenchmark', - 'mandelbrot.MandelbrotBenchmark', - 'histogram.Histogram', -] - -stable = 'scala-native-0.4.0' -latest = 'scala-native-0.4.1-SNAPSHOT' +stable = 'scala-native-0.3.9' +latest = 'scala-native-0.4.0-SNAPSHOT' def expand_wild_cards(arg): @@ -41,6 +20,7 @@ def expand_wild_cards(arg): if __name__ == "__main__": + parser = argparse.ArgumentParser() parser.add_argument("-i", help="interactive mode", action="store_true") parser.add_argument("set", nargs='*', default=[None]) @@ -62,19 +42,4 @@ def expand_wild_cards(arg): print "configurations:", configurations for conf_name in configurations: - conf = Configuration(conf_name) - conf.make_active() - for bench_name in benchmarks: - bench = Benchmark(bench_name) - print('--- conf: {}, bench: {}'.format(conf, bench)) - - bench.compile(conf) - resultsdir = bench.ensure_results_dir(conf) - - runs = conf.runs - for n in xrange(runs): - print('--- run {}/{}'.format(n, runs)) - - out = bench.run(conf) - with open(os.path.join(resultsdir, str(n)), 'wb') as resultfile: - resultfile.write(out) + Configuration(conf_name).run_benchmarks(all_benchmarks) diff --git a/scripts/shared/benchmarks.py b/scripts/shared/benchmarks.py index 6a90a01..23df95e 100644 --- a/scripts/shared/benchmarks.py +++ b/scripts/shared/benchmarks.py @@ -1,6 +1,25 @@ import os -from shared.file_utils import sbt, run, slurp, mkdir +from file_utils import sbt, run, slurp, mkdir + +all_benchmarks = [ + 'bounce.BounceBenchmark', + 'list.ListBenchmark', + 'queens.QueensBenchmark', + 'richards.RichardsBenchmark', + 'permute.PermuteBenchmark', + 'deltablue.DeltaBlueBenchmark', + 'tracer.TracerBenchmark', + 'json.JsonBenchmark', + 'sudoku.SudokuBenchmark', + 'brainfuck.BrainfuckBenchmark', + 'cd.CDBenchmark', + 'kmeans.KmeansBenchmark', + 'nbody.NbodyBenchmark', + 'rsc.RscBenchmark', + 'gcbench.GCBenchBenchmark', + 'mandelbrot.MandelbrotBenchmark', +] class Benchmark: diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py index b0a8677..aad9410 100644 --- a/scripts/shared/configurations.py +++ b/scripts/shared/configurations.py @@ -1,7 +1,8 @@ import os import shutil as sh -from shared.file_utils import slurp, mkdir +from benchmarks import Benchmark +from file_utils import slurp, mkdir confs_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + "/confs" @@ -33,6 +34,26 @@ def make_active(self): else: os.remove('project/plugins.sbt') + def run_benchmarks(self, benchmarks): + self.make_active() + for item in benchmarks: + if isinstance(item, basestring): + bench = Benchmark(item) + else: + bench = item + print('--- conf: {}, bench: {}'.format(self.name, bench)) + + bench.compile(self) + resultsdir = bench.ensure_results_dir(self) + + runs = self.runs + for n in xrange(runs): + print('--- run {}/{}'.format(n, runs)) + + out = bench.run(self) + with open(os.path.join(resultsdir, str(n)), 'w+') as resultfile: + resultfile.write(out) + def run_cmd(self, bench): return slurp(os.path.join(self.conf_dir, 'run')) \ .replace('$BENCH', bench) \ From 7a8b1591f3242a069d66be4c674048aaef893b61 Mon Sep 17 00:00:00 2001 From: Valdis Date: Thu, 16 May 2019 17:26:22 +0200 Subject: [PATCH 08/32] fix summary.py --- scripts/summary.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/summary.py b/scripts/summary.py index ede68c4..03b97f6 100644 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -1,10 +1,11 @@ -from run import benchmarks, runs, configurations +from shared.configurations import default_runs, all_configs +from shared.benchmarks import all_benchmarks import numpy as np def config_data(bench, conf): out = [] - for run in range(runs): + for run in xrange(default_runs): try: points = [] with open('results/{}/{}/{}'.format(conf, bench, run)) as data: @@ -23,9 +24,9 @@ def config_data(bench, conf): def peak_performance(): out = [] - for bench in benchmarks: + for bench in all_benchmarks: res = [] - for conf in configurations: + for conf in all_configs: try: res.append(np.percentile(config_data(bench, conf), 10)) except IndexError: @@ -35,9 +36,9 @@ def peak_performance(): if __name__ == '__main__': leading = ['name'] - for conf in configurations: + for conf in all_configs: leading.append(conf) print(','.join(leading)) - for bench, res in zip(benchmarks, peak_performance()): + for bench, res in zip(all_benchmarks, peak_performance()): print(','.join([bench] + list(map(str, res)))) From 757338047fc1248cfff12416bbc9d0fc37971b65 Mon Sep 17 00:00:00 2001 From: Valdis Date: Thu, 16 May 2019 17:46:19 +0200 Subject: [PATCH 09/32] configurable batch sizes and run count --- scripts/run.py | 6 ++++-- scripts/shared/configurations.py | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/run.py b/scripts/run.py index 0a431f7..2a5af1b 100755 --- a/scripts/run.py +++ b/scripts/run.py @@ -2,7 +2,7 @@ import argparse from shared.benchmarks import all_benchmarks -from shared.configurations import Configuration +from shared.configurations import Configuration, default_runs, default_batches stable = 'scala-native-0.3.9' latest = 'scala-native-0.4.0-SNAPSHOT' @@ -23,6 +23,8 @@ def expand_wild_cards(arg): parser = argparse.ArgumentParser() parser.add_argument("-i", help="interactive mode", action="store_true") + parser.add_argument("--runs", help="number of runs", type=int, default=default_runs) + parser.add_argument("--batches", help="number of batches per run", type=int, default=default_batches) parser.add_argument("set", nargs='*', default=[None]) args = parser.parse_args() print args @@ -42,4 +44,4 @@ def expand_wild_cards(arg): print "configurations:", configurations for conf_name in configurations: - Configuration(conf_name).run_benchmarks(all_benchmarks) + Configuration(conf_name, batches=args.batches, runs=args.runs).run_benchmarks(all_benchmarks) diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py index aad9410..c0fa4b5 100644 --- a/scripts/shared/configurations.py +++ b/scripts/shared/configurations.py @@ -15,10 +15,17 @@ class Configuration: def __init__(self, name, batches=default_batches, runs=default_runs): self.name = name self.conf_dir = os.path.join(confs_path, self.name) - self.results_dir = os.path.join('results', self.name) self.batches = batches self.batch_size = 1 self.runs = runs + + suffix = "" + if runs != default_runs: + suffix += "-r" + str(runs) + if batches != default_batches: + suffix += "-b" + str(batches) + + self.results_dir = os.path.join('results', self.name + suffix) print self.conf_dir def make_active(self): From 9a5559d9807b7d11aee699f7ed1dd316232f82e5 Mon Sep 17 00:00:00 2001 From: Valdis Date: Sat, 18 May 2019 09:50:57 +0200 Subject: [PATCH 10/32] persist settings in the results folders --- scripts/shared/configurations.py | 38 +++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py index c0fa4b5..4ce3f3a 100644 --- a/scripts/shared/configurations.py +++ b/scripts/shared/configurations.py @@ -5,6 +5,7 @@ from file_utils import slurp, mkdir confs_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + "/confs" +results_path = 'results' default_batches = 4000 default_runs = 20 @@ -25,9 +26,33 @@ def __init__(self, name, batches=default_batches, runs=default_runs): if batches != default_batches: suffix += "-b" + str(batches) - self.results_dir = os.path.join('results', self.name + suffix) + self.results_dir = os.path.join(results_path, self.name + suffix) print self.conf_dir + @classmethod + def from_results(cls, full_name): + return cls.from_results_dir(os.path.join(results_path, full_name)) + + @classmethod + def from_results_dir(cls, full_path): + name = None + batches = None + runs = None + with open(os.path.join(full_path, 'settings.properties')) as settings: + for line in settings.readlines(): + key, value = line.split('=') + if key == 'name': + name = value + elif key == 'batches': + batches = value + elif key == 'runs': + runs = value + if batches is None: + batches = default_batches + if runs is None: + runs = default_runs + return cls(name, batches=batches, runs=runs) + def make_active(self): build_sbt_src = os.path.join(self.conf_dir, 'build.sbt') if os.path.exists(build_sbt_src): @@ -51,14 +76,14 @@ def run_benchmarks(self, benchmarks): print('--- conf: {}, bench: {}'.format(self.name, bench)) bench.compile(self) - resultsdir = bench.ensure_results_dir(self) + results_dir = bench.ensure_results_dir(self) runs = self.runs for n in xrange(runs): print('--- run {}/{}'.format(n, runs)) out = bench.run(self) - with open(os.path.join(resultsdir, str(n)), 'w+') as resultfile: + with open(os.path.join(results_dir, str(n)), 'w+') as resultfile: resultfile.write(out) def run_cmd(self, bench): @@ -70,6 +95,13 @@ def run_cmd(self, bench): def compile_cmd(self): return slurp(os.path.join(self.conf_dir, 'compile')) + def write_settings(self): + results_dir = self.results_dir + with open(os.path.join(results_dir, 'settings.properties'), 'w+') as settings: + settings.write('name={}\n'.format(self.name)) + settings.write('batches={}\n'.format(self.batches)) + settings.write('runs={}\n'.format(self.runs)) + def ensure_results_dir(self): dir = self.results_dir mkdir(dir) From df5930b12ca96e895b924e6d3d8166a07c8e7b76 Mon Sep 17 00:00:00 2001 From: Valdis Date: Sat, 18 May 2019 11:44:28 +0200 Subject: [PATCH 11/32] misc fixes --- scripts/shared/benchmarks.py | 3 ++- scripts/shared/configurations.py | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/shared/benchmarks.py b/scripts/shared/benchmarks.py index 23df95e..29a5496 100644 --- a/scripts/shared/benchmarks.py +++ b/scripts/shared/benchmarks.py @@ -34,12 +34,13 @@ def compile(self, conf): return run(cmd) def run(self, conf): - run_cmd = conf.run_cmd() + run_cmd = conf.run_cmd(self) input = slurp(os.path.join('input', self.name)) output = slurp(os.path.join('output', self.name)) cmd = [] cmd.extend(run_cmd) cmd.extend([str(conf.batches), str(conf.batch_size), input, output]) + return run(cmd) def ensure_results_dir(self, conf): dir = os.path.join(conf.ensure_results_dir(), self.name) diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py index 4ce3f3a..375a798 100644 --- a/scripts/shared/configurations.py +++ b/scripts/shared/configurations.py @@ -27,7 +27,6 @@ def __init__(self, name, batches=default_batches, runs=default_runs): suffix += "-b" + str(batches) self.results_dir = os.path.join(results_path, self.name + suffix) - print self.conf_dir @classmethod def from_results(cls, full_name): @@ -40,13 +39,14 @@ def from_results_dir(cls, full_path): runs = None with open(os.path.join(full_path, 'settings.properties')) as settings: for line in settings.readlines(): - key, value = line.split('=') + key, raw_value = line.split('=') + value = raw_value.strip() if key == 'name': name = value elif key == 'batches': - batches = value + batches = int(value) elif key == 'runs': - runs = value + runs = int(value) if batches is None: batches = default_batches if runs is None: @@ -88,7 +88,7 @@ def run_benchmarks(self, benchmarks): def run_cmd(self, bench): return slurp(os.path.join(self.conf_dir, 'run')) \ - .replace('$BENCH', bench) \ + .replace('$BENCH', bench.name) \ .replace('$HOME', os.environ['HOME']) \ .split(' ') @@ -105,6 +105,7 @@ def write_settings(self): def ensure_results_dir(self): dir = self.results_dir mkdir(dir) + self.write_settings() return dir def __str__(self): From 1d041ec8abc85abf685d659760231a8975a1c500 Mon Sep 17 00:00:00 2001 From: Valdis Date: Sat, 18 May 2019 13:46:40 +0200 Subject: [PATCH 12/32] move common command line parsing to shared module refactor summary --- scripts/run.py | 31 +++---------------- scripts/shared/benchmarks.py | 19 ++++++++++++ scripts/shared/cmdline.py | 24 +++++++++++++++ scripts/shared/comparison.py | 47 ++++++++++++++++++++++++++++ scripts/shared/configurations.py | 39 ++++++++++++++++++++--- scripts/shared/parser.py | 19 ++++++++++++ scripts/summary.py | 53 ++++++++------------------------ 7 files changed, 160 insertions(+), 72 deletions(-) create mode 100644 scripts/shared/cmdline.py create mode 100644 scripts/shared/comparison.py create mode 100644 scripts/shared/parser.py mode change 100644 => 100755 scripts/summary.py diff --git a/scripts/run.py b/scripts/run.py index 2a5af1b..cd989df 100755 --- a/scripts/run.py +++ b/scripts/run.py @@ -1,31 +1,17 @@ #!/usr/bin/env python import argparse +from shared.cmdline import expand_all from shared.benchmarks import all_benchmarks from shared.configurations import Configuration, default_runs, default_batches -stable = 'scala-native-0.3.9' -latest = 'scala-native-0.4.0-SNAPSHOT' - - -def expand_wild_cards(arg): - if arg is None: - return arg - elif arg.startswith("latest"): - return latest + arg[len("latest"):] - elif arg.startswith("stable"): - return stable + arg[len("stable"):] - else: - return arg - - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-i", help="interactive mode", action="store_true") parser.add_argument("--runs", help="number of runs", type=int, default=default_runs) parser.add_argument("--batches", help="number of batches per run", type=int, default=default_batches) - parser.add_argument("set", nargs='*', default=[None]) + parser.add_argument("configurations", nargs='*', default=[None]) args = parser.parse_args() print args @@ -33,15 +19,8 @@ def expand_wild_cards(arg): from IPython import embed embed() else: - configurations = [] - for choice in args.set: - expanded = expand_wild_cards(choice) - if expanded is None: - configurations = [stable, latest] - else: - configurations += [expanded] - - print "configurations:", configurations + configuration_names = expand_all(args.configurations) + print "configurations:", configuration_names - for conf_name in configurations: + for conf_name in configuration_names: Configuration(conf_name, batches=args.batches, runs=args.runs).run_benchmarks(all_benchmarks) diff --git a/scripts/shared/benchmarks.py b/scripts/shared/benchmarks.py index 29a5496..41fa37b 100644 --- a/scripts/shared/benchmarks.py +++ b/scripts/shared/benchmarks.py @@ -26,6 +26,7 @@ class Benchmark: def __init__(self, name): self.name = name + self.short_name = name.split(".")[0] def compile(self, conf): cmd = [sbt, '-J-Xmx6G', 'clean'] @@ -47,5 +48,23 @@ def ensure_results_dir(self, conf): mkdir(dir) return dir + def results_dir(self, conf): + return os.path.join(conf.results_dir, self.name) + + def __eq__(self, other): + if isinstance(other, Benchmark): + return self.name == other.name + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.name) + def __str__(self): return self.name + + def __repr__(self): + return 'Benchmark(\'{}\')'.format(self.name) diff --git a/scripts/shared/cmdline.py b/scripts/shared/cmdline.py new file mode 100644 index 0000000..bc3b048 --- /dev/null +++ b/scripts/shared/cmdline.py @@ -0,0 +1,24 @@ +stable = 'scala-native-0.3.9' +latest = 'scala-native-0.4.0-SNAPSHOT' + + +def expand_all(arg): + confs = [] + for choice in arg: + expanded = expand_wild_cards(choice) + if expanded is None: + confs = [stable, latest] + else: + confs += [expanded] + return confs + + +def expand_wild_cards(arg): + if arg is None: + return arg + elif arg.startswith("latest"): + return latest + arg[len("latest"):] + elif arg.startswith("stable"): + return stable + arg[len("stable"):] + else: + return arg \ No newline at end of file diff --git a/scripts/shared/comparison.py b/scripts/shared/comparison.py new file mode 100644 index 0000000..32bf6e5 --- /dev/null +++ b/scripts/shared/comparison.py @@ -0,0 +1,47 @@ +import numpy as np +from configurations import Configuration +from parser import config_data + +default_warmup = 2000 + + +class Comparison: + def __init__(self, confs, warmup=default_warmup): + assert len(confs) > 0 + + configurations = [] + for item in confs: + if isinstance(item, basestring): + configurations.append(Configuration.from_results(item)) + else: + configurations.append(item) + self.configurations = configurations + + common_benchmarks = configurations[0].finished_benchmarks() + print common_benchmarks + for other_conf in configurations[1:]: + present = set(other_conf.finished_benchmarks()) + common_benchmarks = filter(lambda b: b in present, common_benchmarks) + print common_benchmarks + self.common_benchmarks = common_benchmarks + self.warmup = warmup + + def peak_performances(self): + out = [] + for bench in self.common_benchmarks: + res = [] + for conf in self.configurations: + try: + res.append(np.percentile(config_data(bench, conf, self.warmup), 50)) + except IndexError: + res.append(0) + out.append(res) + return out + + def simple_report(self): + leading = ['name'] + for conf in self.configurations: + leading.append(conf.full_name) + print ','.join(leading) + for bench, res in zip(self.common_benchmarks, self.peak_performances()): + print ','.join([str(bench)] + list(map(str, res))) \ No newline at end of file diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py index 375a798..d5a7973 100644 --- a/scripts/shared/configurations.py +++ b/scripts/shared/configurations.py @@ -26,7 +26,8 @@ def __init__(self, name, batches=default_batches, runs=default_runs): if batches != default_batches: suffix += "-b" + str(batches) - self.results_dir = os.path.join(results_path, self.name + suffix) + self.full_name = self.name + suffix + self.results_dir = os.path.join(results_path, self.full_name) @classmethod def from_results(cls, full_name): @@ -103,13 +104,41 @@ def write_settings(self): settings.write('runs={}\n'.format(self.runs)) def ensure_results_dir(self): - dir = self.results_dir - mkdir(dir) + results_dir = self.results_dir + mkdir(results_dir) self.write_settings() - return dir + return results_dir + + def finished_benchmarks(self): + benchmarks = [] + results_dir = self.results_dir + all_subdirs = next(os.walk(results_dir))[1] + for subdir in all_subdirs: + all_runs_present = True + for run in xrange(self.runs): + if not os.path.isfile(os.path.join(results_dir, subdir, str(run))): + all_runs_present = False + break + if all_runs_present: + benchmarks.append(Benchmark(subdir)) + + return benchmarks + + def benchmark_result_files(self, benchmark): + benchmark_dir = benchmark.results_dir(self) + for run in xrange(self.runs): + yield os.path.join(benchmark_dir, str(run)) def __str__(self): - return self.name + s = '{}('.format(self.name) + s += 'runs={}'.format(str(self.runs)) + if self.runs == default_runs: + s += '[default]' + s += ', batches={}'.format(str(self.batches)) + if self.batches == default_batches: + s += '[default]' + s += ') = {}'.format(self.full_name) + return s all_configs = next(os.walk(confs_path))[1] diff --git a/scripts/shared/parser.py b/scripts/shared/parser.py new file mode 100644 index 0000000..c7fd96e --- /dev/null +++ b/scripts/shared/parser.py @@ -0,0 +1,19 @@ +import numpy as np + + +def config_data(bench, conf, warmup): + out = [] + for result_file in conf.benchmark_result_files(bench): + try: + raw_points = [] + with open(result_file) as data: + for line in data.readlines(): + try: + # in ms + raw_points.append(float(line) / 1000000) + except Exception as e: + print e + out += raw_points[warmup:] + except IOError: + pass + return np.array(out) \ No newline at end of file diff --git a/scripts/summary.py b/scripts/summary.py old mode 100644 new mode 100755 index 03b97f6..94a008e --- a/scripts/summary.py +++ b/scripts/summary.py @@ -1,44 +1,15 @@ -from shared.configurations import default_runs, all_configs -from shared.benchmarks import all_benchmarks +#!/usr/bin/env python +import argparse -import numpy as np - -def config_data(bench, conf): - out = [] - for run in xrange(default_runs): - try: - points = [] - with open('results/{}/{}/{}'.format(conf, bench, run)) as data: - for line in data.readlines(): - points.append(float(line)) - # take only last 2000 to account for startup - points = points[-2000:] - # filter out 1% worst measurements as outliers - pmax = np.percentile(points, 99) - for point in points: - if point <= pmax: - out.append(point) - except IOError: - pass - return np.array(out) - -def peak_performance(): - out = [] - for bench in all_benchmarks: - res = [] - for conf in all_configs: - try: - res.append(np.percentile(config_data(bench, conf), 10)) - except IndexError: - res.append(0) - out.append(res) - return out +from shared.cmdline import expand_all +from shared.comparison import Comparison, default_warmup if __name__ == '__main__': - leading = ['name'] - for conf in all_configs: - leading.append(conf) - print(','.join(leading)) - for bench, res in zip(all_benchmarks, peak_performance()): - print(','.join([bench] + list(map(str, res)))) - + parser = argparse.ArgumentParser() + parser.add_argument("--warmup", help="number of iterations to skip before calculating percentiles", type=int, default=default_warmup) + parser.add_argument("configurations", nargs='*', default=[None]) + args = parser.parse_args() + + configuration_names = expand_all(args.configurations) + comparison = Comparison(configuration_names, warmup=args.warmup) + comparison.simple_report() From dc0129e67c20936e1de75e3517a81520cd812461 Mon Sep 17 00:00:00 2001 From: Valdis Date: Sat, 18 May 2019 20:05:10 +0200 Subject: [PATCH 13/32] interactive mode as its own file --- scripts/interactive.py | 15 +++++++++++++++ scripts/run.py | 13 ++++--------- scripts/shared/benchmarks.py | 6 +++--- 3 files changed, 22 insertions(+), 12 deletions(-) create mode 100755 scripts/interactive.py diff --git a/scripts/interactive.py b/scripts/interactive.py new file mode 100755 index 0000000..cfdd798 --- /dev/null +++ b/scripts/interactive.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +# import for interactive mode +# noinspection PyUnresolvedReferences +from shared.benchmarks import * +# noinspection PyUnresolvedReferences +from shared.configurations import * +# noinspection PyUnresolvedReferences +from shared.comparison import * +# noinspection PyUnresolvedReferences +from shared.cmdline import stable,latest + +if __name__ == '__main__': + from IPython import embed + embed() \ No newline at end of file diff --git a/scripts/run.py b/scripts/run.py index cd989df..6702948 100755 --- a/scripts/run.py +++ b/scripts/run.py @@ -8,19 +8,14 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("-i", help="interactive mode", action="store_true") parser.add_argument("--runs", help="number of runs", type=int, default=default_runs) parser.add_argument("--batches", help="number of batches per run", type=int, default=default_batches) parser.add_argument("configurations", nargs='*', default=[None]) args = parser.parse_args() print args - if args.i: - from IPython import embed - embed() - else: - configuration_names = expand_all(args.configurations) - print "configurations:", configuration_names + configuration_names = expand_all(args.configurations) + print "configurations:", configuration_names - for conf_name in configuration_names: - Configuration(conf_name, batches=args.batches, runs=args.runs).run_benchmarks(all_benchmarks) + for conf_name in configuration_names: + Configuration(conf_name, batches=args.batches, runs=args.runs).run_benchmarks(all_benchmarks) diff --git a/scripts/shared/benchmarks.py b/scripts/shared/benchmarks.py index 41fa37b..59fb67d 100644 --- a/scripts/shared/benchmarks.py +++ b/scripts/shared/benchmarks.py @@ -29,9 +29,9 @@ def __init__(self, name): self.short_name = name.split(".")[0] def compile(self, conf): - cmd = [sbt, '-J-Xmx6G', 'clean'] - cmd.append('set mainClass in Compile := Some("{}")'.format(self.name)) - cmd.append(conf.compile_cmd()) + cmd = [sbt, '-J-Xmx6G', 'clean', + 'set mainClass in Compile := Some("{}")'.format(self.name), + conf.compile_cmd()] return run(cmd) def run(self, conf): From e677dbf998864bc70fcab4a1fb47b0188fc61a7b Mon Sep 17 00:00:00 2001 From: Valdis Date: Sat, 18 May 2019 20:09:24 +0200 Subject: [PATCH 14/32] cleanup --- scripts/shared/comparison.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/shared/comparison.py b/scripts/shared/comparison.py index 32bf6e5..4af7dd8 100644 --- a/scripts/shared/comparison.py +++ b/scripts/shared/comparison.py @@ -18,11 +18,9 @@ def __init__(self, confs, warmup=default_warmup): self.configurations = configurations common_benchmarks = configurations[0].finished_benchmarks() - print common_benchmarks for other_conf in configurations[1:]: present = set(other_conf.finished_benchmarks()) common_benchmarks = filter(lambda b: b in present, common_benchmarks) - print common_benchmarks self.common_benchmarks = common_benchmarks self.warmup = warmup From 190c6df960d938980f0e3a69f4d6b3952573a251 Mon Sep 17 00:00:00 2001 From: Valdis Date: Sat, 18 May 2019 20:16:58 +0200 Subject: [PATCH 15/32] more general percentile methods --- scripts/shared/comparison.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/shared/comparison.py b/scripts/shared/comparison.py index 4af7dd8..f1ac17b 100644 --- a/scripts/shared/comparison.py +++ b/scripts/shared/comparison.py @@ -24,22 +24,25 @@ def __init__(self, confs, warmup=default_warmup): self.common_benchmarks = common_benchmarks self.warmup = warmup - def peak_performances(self): + def percentile(self, p): out = [] for bench in self.common_benchmarks: - res = [] - for conf in self.configurations: - try: - res.append(np.percentile(config_data(bench, conf, self.warmup), 50)) - except IndexError: - res.append(0) - out.append(res) + out.append(self.percentile_bench(bench, p)) return out + def percentile_bench(self, bench, p): + res = [] + for conf in self.configurations: + try: + res.append(np.percentile(config_data(bench, conf, self.warmup), p)) + except IndexError: + res.append(0) + return res + def simple_report(self): leading = ['name'] for conf in self.configurations: leading.append(conf.full_name) print ','.join(leading) - for bench, res in zip(self.common_benchmarks, self.peak_performances()): + for bench, res in zip(self.common_benchmarks, self.percentile(50)): print ','.join([str(bench)] + list(map(str, res))) \ No newline at end of file From d6375846bfc77044d13d19ad2b1ee91455fb5415 Mon Sep 17 00:00:00 2001 From: Valdis Date: Tue, 21 May 2019 16:30:31 +0200 Subject: [PATCH 16/32] writing csv files, separate reports class --- scripts/shared/comparison.py | 19 +++++++++++++------ scripts/shared/reports.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 scripts/shared/reports.py diff --git a/scripts/shared/comparison.py b/scripts/shared/comparison.py index f1ac17b..efc45f5 100644 --- a/scripts/shared/comparison.py +++ b/scripts/shared/comparison.py @@ -1,6 +1,7 @@ import numpy as np from configurations import Configuration from parser import config_data +from reports import Report default_warmup = 2000 @@ -39,10 +40,16 @@ def percentile_bench(self, bench, p): res.append(0) return res + def csv_file(self, data, path): + with open(path, 'w+') as resultfile: + leading = ['name'] + for conf in self.configurations: + leading.append(conf.full_name) + resultfile.write(','.join(leading) + '\n') + for bench, res in zip(self.common_benchmarks, data): + resultfile.write(','.join([str(bench)] + list(map(str, res))) + '\n') + def simple_report(self): - leading = ['name'] - for conf in self.configurations: - leading.append(conf.full_name) - print ','.join(leading) - for bench, res in zip(self.common_benchmarks, self.percentile(50)): - print ','.join([str(bench)] + list(map(str, res))) \ No newline at end of file + report = Report(self) + report.generate() + return report diff --git a/scripts/shared/reports.py b/scripts/shared/reports.py new file mode 100644 index 0000000..31cbe67 --- /dev/null +++ b/scripts/shared/reports.py @@ -0,0 +1,30 @@ +import os +from datetime import datetime, time +from file_utils import mkdir + +reports_path = 'reports' + + +class Report: + def __init__(self, comparison, date=None, comment=None): + self.comparison = comparison + if comment is None: + comment = '_vs_'.join(map(lambda c: c.full_name, comparison.configurations)).replace(os.sep, "_") + self.comment = comment + if date is None: + date = datetime.now() + self.date = date + self.full_name = 'summary_' + time.strftime(self.date, '%Y%m%d_%H%M%S') + '_' + comment + self.results_dir = os.path.join(reports_path, self.full_name) + + def generate(self): + c = self.comparison + results_dir = self.ensure_results_dir() + for p in [50,90,99]: + path = os.path.join(results_dir, 'percentile{}.csv'.format(str(p))) + c.csv_file(c.percentile(p), path) + + def ensure_results_dir(self): + results_dir = self.results_dir + mkdir(results_dir) + return results_dir \ No newline at end of file From c521e78856aa61b1c804b28cae56c820d4c2dfd1 Mon Sep 17 00:00:00 2001 From: Valdis Date: Sun, 26 May 2019 10:37:20 +0200 Subject: [PATCH 17/32] refactor use dictionaries as intermediate representation --- scripts/shared/configurations.py | 40 ++++++++++++++------------------ scripts/shared/file_utils.py | 16 +++++++++++++ 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py index d5a7973..a81ff98 100644 --- a/scripts/shared/configurations.py +++ b/scripts/shared/configurations.py @@ -2,7 +2,7 @@ import shutil as sh from benchmarks import Benchmark -from file_utils import slurp, mkdir +from file_utils import slurp, mkdir, dict_from_file, dict_to_file confs_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + "/confs" results_path = 'results' @@ -28,6 +28,7 @@ def __init__(self, name, batches=default_batches, runs=default_runs): self.full_name = self.name + suffix self.results_dir = os.path.join(results_path, self.full_name) + self.settings_file = os.path.join(self.results_dir, 'settings.properties') @classmethod def from_results(cls, full_name): @@ -35,23 +36,15 @@ def from_results(cls, full_name): @classmethod def from_results_dir(cls, full_path): - name = None - batches = None - runs = None - with open(os.path.join(full_path, 'settings.properties')) as settings: - for line in settings.readlines(): - key, raw_value = line.split('=') - value = raw_value.strip() - if key == 'name': - name = value - elif key == 'batches': - batches = int(value) - elif key == 'runs': - runs = int(value) - if batches is None: - batches = default_batches - if runs is None: - runs = default_runs + settings_file = os.path.join(full_path, 'settings.properties') + kv = dict_from_file(settings_file) + return cls.from_dict(kv) + + @classmethod + def from_dict(cls, kv): + batches = int(kv.get('batches', default_batches)) + runs = int(kv.get('runs', default_runs)) + name = kv.get('name') return cls(name, batches=batches, runs=runs) def make_active(self): @@ -96,12 +89,13 @@ def run_cmd(self, bench): def compile_cmd(self): return slurp(os.path.join(self.conf_dir, 'compile')) + def to_dict(self): + return {'name': self.name, 'batches': self.batches, 'runs': self.runs} + def write_settings(self): - results_dir = self.results_dir - with open(os.path.join(results_dir, 'settings.properties'), 'w+') as settings: - settings.write('name={}\n'.format(self.name)) - settings.write('batches={}\n'.format(self.batches)) - settings.write('runs={}\n'.format(self.runs)) + settings_file = self.settings_file + kv = self.to_dict() + dict_to_file(settings_file, kv) def ensure_results_dir(self): results_dir = self.results_dir diff --git a/scripts/shared/file_utils.py b/scripts/shared/file_utils.py index 90d7968..328992c 100644 --- a/scripts/shared/file_utils.py +++ b/scripts/shared/file_utils.py @@ -36,4 +36,20 @@ def run(cmd): return subp.check_output(cmd) +def dict_from_file(settings_file): + kv = {} + with open(settings_file) as settings: + for line in settings.readlines(): + key, raw_value = line.split('=') + value = raw_value.strip() + kv[key] = value + return kv + + +def dict_to_file(settings_file, kv): + with open(settings_file, 'w+') as settings: + for k, v in kv.iteritems(): + settings.write('{}={}'.format(k, v)) + + sbt = where('sbt') From a21eace6939970041f16fc8cb76f03763302e057 Mon Sep 17 00:00:00 2001 From: Valdis Date: Sun, 26 May 2019 11:29:48 +0200 Subject: [PATCH 18/32] convert between report and dictionary --- scripts/shared/comparison.py | 14 +++++++++++ scripts/shared/misc_utils.py | 20 ++++++++++++++++ scripts/shared/reports.py | 46 ++++++++++++++++++++++++++++++++---- 3 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 scripts/shared/misc_utils.py diff --git a/scripts/shared/comparison.py b/scripts/shared/comparison.py index efc45f5..822ee3e 100644 --- a/scripts/shared/comparison.py +++ b/scripts/shared/comparison.py @@ -2,6 +2,7 @@ from configurations import Configuration from parser import config_data from reports import Report +from misc_utils import dict_write_arr, dict_get_arr default_warmup = 2000 @@ -53,3 +54,16 @@ def simple_report(self): report = Report(self) report.generate() return report + + @classmethod + def from_dict(cls, kv): + warmup = int(kv.get('warmup', default_warmup)) + confs = map(Configuration.from_dict, dict_get_arr(kv, 'confs')) + return cls(confs, warmup) + + def to_dict(self): + kv = {'warmup': self.warmup} + conf_kvs = map(Configuration.to_dict, self.configurations) + dict_write_arr(kv, 'confs', conf_kvs) + + return kv diff --git a/scripts/shared/misc_utils.py b/scripts/shared/misc_utils.py new file mode 100644 index 0000000..695274a --- /dev/null +++ b/scripts/shared/misc_utils.py @@ -0,0 +1,20 @@ +def dict_write_arr(kv, arr_key, conf_kvs): + for index, inner_kv in enumerate(conf_kvs): + for k, v in inner_kv.iteritems(): + kv['{}.{}.{}'.format(arr_key, index, k)] = v + + +def dict_get_arr(kv, arr_key): + arr = [] + arr_kv = {} + max_index = -1 + for k, v in kv.iteritems(): + if k.startswith(arr_key + '.'): + _, raw_index, key = k.split('.') + index = int(raw_index) + max_index = max(index, max_index) + arr_kv.get(index, {})[key] = v + for index in xrange(max_index + 1): + arr.append(arr_kv.get(index)) + + return arr \ No newline at end of file diff --git a/scripts/shared/reports.py b/scripts/shared/reports.py index 31cbe67..810a86c 100644 --- a/scripts/shared/reports.py +++ b/scripts/shared/reports.py @@ -1,6 +1,9 @@ import os from datetime import datetime, time -from file_utils import mkdir +from file_utils import mkdir, dict_to_file, dict_from_file +from comparison import Comparison + +date_format = '%Y%m%d_%H%M%S' reports_path = 'reports' @@ -14,17 +17,50 @@ def __init__(self, comparison, date=None, comment=None): if date is None: date = datetime.now() self.date = date - self.full_name = 'summary_' + time.strftime(self.date, '%Y%m%d_%H%M%S') + '_' + comment + self.full_name = 'summary_' + datetime.strftime(self.date, date_format) + '_' + comment self.results_dir = os.path.join(reports_path, self.full_name) + self.settings_file = os.path.join(self.results_dir, 'settings.properties') def generate(self): c = self.comparison - results_dir = self.ensure_results_dir() + results_dir = self.ensure_dir() for p in [50,90,99]: path = os.path.join(results_dir, 'percentile{}.csv'.format(str(p))) c.csv_file(c.percentile(p), path) - def ensure_results_dir(self): + def write_settings(self): + settings_file = self.settings_file + kv = self.to_dict() + dict_to_file(settings_file, kv) + + @classmethod + def from_full_name(cls, full_name): + return cls.from_dir(os.path.join(reports_path, full_name)) + + @classmethod + def from_dir(cls, full_path): + settings_file = os.path.join(full_path, 'settings.properties') + kv = dict_from_file(settings_file) + return cls.from_dict(kv) + + def ensure_dir(self): results_dir = self.results_dir mkdir(results_dir) - return results_dir \ No newline at end of file + self.write_settings() + return results_dir + + @classmethod + def from_dict(cls, kv): + comment = kv.get('comment') + raw_date = kv.get('date') + if raw_date is None: + date = None + else: + date = datetime.strptime(raw_date, date_format) + return cls(Comparison.from_dict(kv), date, comment) + + def to_dict(self): + kv = self.comparison.to_dict() + kv['comment'] = self.comment + kv['date'] = datetime.strftime(self.date, date_format) + return kv \ No newline at end of file From e67c4690e024bf2390bb04bbb23afdd2c81554ea Mon Sep 17 00:00:00 2001 From: Valdis Date: Sun, 26 May 2019 12:04:23 +0200 Subject: [PATCH 19/32] do not overwrite existing report --- scripts/shared/file_utils.py | 3 +++ scripts/shared/reports.py | 31 +++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/scripts/shared/file_utils.py b/scripts/shared/file_utils.py index 328992c..5e7bb39 100644 --- a/scripts/shared/file_utils.py +++ b/scripts/shared/file_utils.py @@ -13,6 +13,9 @@ def mkdir(path): raise +def touch(path): + open(path, 'w+').close() + def slurp(path): with open(path) as f: return f.read().strip() diff --git a/scripts/shared/reports.py b/scripts/shared/reports.py index 810a86c..36fa8d0 100644 --- a/scripts/shared/reports.py +++ b/scripts/shared/reports.py @@ -1,7 +1,8 @@ import os -from datetime import datetime, time -from file_utils import mkdir, dict_to_file, dict_from_file +from datetime import datetime + from comparison import Comparison +from file_utils import mkdir, dict_to_file, dict_from_file, touch date_format = '%Y%m%d_%H%M%S' @@ -20,13 +21,27 @@ def __init__(self, comparison, date=None, comment=None): self.full_name = 'summary_' + datetime.strftime(self.date, date_format) + '_' + comment self.results_dir = os.path.join(reports_path, self.full_name) self.settings_file = os.path.join(self.results_dir, 'settings.properties') + self.done_file = os.path.join(self.results_dir, '.done') def generate(self): - c = self.comparison - results_dir = self.ensure_dir() - for p in [50,90,99]: - path = os.path.join(results_dir, 'percentile{}.csv'.format(str(p))) - c.csv_file(c.percentile(p), path) + if self.is_done(): + copy = Report(self.comparison, date=None, comment=self.comment) + copy.generate() + return copy + else: + c = self.comparison + results_dir = self.ensure_dir() + for p in [50, 90, 99]: + path = os.path.join(results_dir, 'percentile{}.csv'.format(str(p))) + c.csv_file(c.percentile(p), path) + self.mark_done() + return self + + def mark_done(self): + touch(self.done_file) + + def is_done(self): + return os.path.isfile(self.done_file) def write_settings(self): settings_file = self.settings_file @@ -63,4 +78,4 @@ def to_dict(self): kv = self.comparison.to_dict() kv['comment'] = self.comment kv['date'] = datetime.strftime(self.date, date_format) - return kv \ No newline at end of file + return kv From edd67bb0112d33b24712f9dda51336ed868bb18b Mon Sep 17 00:00:00 2001 From: Valdis Date: Sun, 26 May 2019 12:39:45 +0200 Subject: [PATCH 20/32] fixes --- .gitignore | 1 + scripts/shared/file_utils.py | 2 +- scripts/shared/reports.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ec7b92a..41a2088 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ target/ bin/.coursier bin/.scalafmt* results/ +reports/ *.iprof .idea build.sbt diff --git a/scripts/shared/file_utils.py b/scripts/shared/file_utils.py index 5e7bb39..d7d0809 100644 --- a/scripts/shared/file_utils.py +++ b/scripts/shared/file_utils.py @@ -52,7 +52,7 @@ def dict_from_file(settings_file): def dict_to_file(settings_file, kv): with open(settings_file, 'w+') as settings: for k, v in kv.iteritems(): - settings.write('{}={}'.format(k, v)) + settings.write('{}={}\n'.format(k, v)) sbt = where('sbt') diff --git a/scripts/shared/reports.py b/scripts/shared/reports.py index 36fa8d0..cf9692e 100644 --- a/scripts/shared/reports.py +++ b/scripts/shared/reports.py @@ -1,7 +1,6 @@ import os from datetime import datetime -from comparison import Comparison from file_utils import mkdir, dict_to_file, dict_from_file, touch date_format = '%Y%m%d_%H%M%S' @@ -72,6 +71,7 @@ def from_dict(cls, kv): date = None else: date = datetime.strptime(raw_date, date_format) + from comparison import Comparison return cls(Comparison.from_dict(kv), date, comment) def to_dict(self): From 783c892c284a4cab03175a6b614cf7e161e5a673 Mon Sep 17 00:00:00 2001 From: Valdis Date: Sun, 26 May 2019 12:42:34 +0200 Subject: [PATCH 21/32] ability to specify the comment --- scripts/shared/comparison.py | 4 ++-- scripts/summary.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/shared/comparison.py b/scripts/shared/comparison.py index 822ee3e..fef5e17 100644 --- a/scripts/shared/comparison.py +++ b/scripts/shared/comparison.py @@ -50,8 +50,8 @@ def csv_file(self, data, path): for bench, res in zip(self.common_benchmarks, data): resultfile.write(','.join([str(bench)] + list(map(str, res))) + '\n') - def simple_report(self): - report = Report(self) + def simple_report(self, comment=None): + report = Report(self, comment=comment) report.generate() return report diff --git a/scripts/summary.py b/scripts/summary.py index 94a008e..2cf8a08 100755 --- a/scripts/summary.py +++ b/scripts/summary.py @@ -6,10 +6,11 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() + parser.add_argument("--comment", help="comment at the suffix of the report name", default=None) parser.add_argument("--warmup", help="number of iterations to skip before calculating percentiles", type=int, default=default_warmup) parser.add_argument("configurations", nargs='*', default=[None]) args = parser.parse_args() configuration_names = expand_all(args.configurations) comparison = Comparison(configuration_names, warmup=args.warmup) - comparison.simple_report() + comparison.simple_report(comment=args.comment) From d8b44ac7480a796394c6eb93f2c4600998ef5a2c Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 16 Feb 2021 21:20:14 +0100 Subject: [PATCH 22/32] Remove spurious configs --- confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt | 6 ------ confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt | 1 - confs/scala-native-0.3.9-SNAPSHOT-commix/run | 1 - confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt | 7 ------- confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile | 1 - confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt | 1 - confs/scala-native-0.3.9-SNAPSHOT-full-lto/run | 1 - confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt | 6 ------ confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile | 1 - confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt | 1 - confs/scala-native-0.3.9-SNAPSHOT-no-lto/run | 1 - confs/scala-native-0.3.9-SNAPSHOT/build.sbt | 6 ------ confs/scala-native-0.3.9-SNAPSHOT/compile | 1 - confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt | 1 - confs/scala-native-0.3.9-SNAPSHOT/run | 1 - confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt | 6 ------ confs/scala-native-0.4.0-SNAPSHOT-commix/compile | 1 - confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt | 1 - confs/scala-native-0.4.0-SNAPSHOT-commix/run | 1 - confs/scala-native-0.4.0-SNAPSHOT/build.sbt | 6 ------ confs/scala-native-0.4.0-SNAPSHOT/compile | 1 - confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt | 1 - confs/scala-native-0.4.0-SNAPSHOT/run | 1 - 23 files changed, 54 deletions(-) delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-commix/run delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-full-lto/run delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT-no-lto/run delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT/build.sbt delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT/compile delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt delete mode 100644 confs/scala-native-0.3.9-SNAPSHOT/run delete mode 100644 confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt delete mode 100644 confs/scala-native-0.4.0-SNAPSHOT-commix/compile delete mode 100644 confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt delete mode 100644 confs/scala-native-0.4.0-SNAPSHOT-commix/run delete mode 100644 confs/scala-native-0.4.0-SNAPSHOT/build.sbt delete mode 100644 confs/scala-native-0.4.0-SNAPSHOT/compile delete mode 100644 confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt delete mode 100644 confs/scala-native-0.4.0-SNAPSHOT/run diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt deleted file mode 100644 index 2104b7a..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-commix/build.sbt +++ /dev/null @@ -1,6 +0,0 @@ -scalaVersion := "2.11.12" -enablePlugins(ScalaNativePlugin) -nativeLinkStubs := true -nativeGC := "commix" -nativeMode := "release" -nativeLTO := "thin" diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt deleted file mode 100644 index c1423b6..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-commix/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT") diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/run b/confs/scala-native-0.3.9-SNAPSHOT-commix/run deleted file mode 100644 index ae89e34..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-commix/run +++ /dev/null @@ -1 +0,0 @@ -target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt deleted file mode 100644 index 2bcb9cc..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/build.sbt +++ /dev/null @@ -1,7 +0,0 @@ -name := "scala-native-benchmarks" -scalaVersion := "2.11.12" -enablePlugins(ScalaNativePlugin) -nativeLinkStubs := true -nativeGC := "immix" -nativeMode := "release" -nativeLTO := "full" diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile deleted file mode 100644 index 2f3f09f..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/compile +++ /dev/null @@ -1 +0,0 @@ -nativeLink diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt deleted file mode 100644 index c1423b6..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT") diff --git a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/run b/confs/scala-native-0.3.9-SNAPSHOT-full-lto/run deleted file mode 100644 index ae89e34..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-full-lto/run +++ /dev/null @@ -1 +0,0 @@ -target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt deleted file mode 100644 index 2fc1873..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/build.sbt +++ /dev/null @@ -1,6 +0,0 @@ -scalaVersion := "2.11.12" -enablePlugins(ScalaNativePlugin) -nativeLinkStubs := true -nativeGC := "immix" -nativeMode := "release" -nativeLTO := "none" diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile deleted file mode 100644 index 2f3f09f..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/compile +++ /dev/null @@ -1 +0,0 @@ -nativeLink diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt deleted file mode 100644 index c1423b6..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT") diff --git a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/run b/confs/scala-native-0.3.9-SNAPSHOT-no-lto/run deleted file mode 100644 index ae89e34..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT-no-lto/run +++ /dev/null @@ -1 +0,0 @@ -target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.3.9-SNAPSHOT/build.sbt b/confs/scala-native-0.3.9-SNAPSHOT/build.sbt deleted file mode 100644 index 702aa9d..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT/build.sbt +++ /dev/null @@ -1,6 +0,0 @@ -name := "scala-native-benchmarks" -scalaVersion := "2.11.12" -enablePlugins(ScalaNativePlugin) -nativeLinkStubs := true -nativeGC := "immix" -nativeMode := "release" diff --git a/confs/scala-native-0.3.9-SNAPSHOT/compile b/confs/scala-native-0.3.9-SNAPSHOT/compile deleted file mode 100644 index 2f3f09f..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT/compile +++ /dev/null @@ -1 +0,0 @@ -nativeLink diff --git a/confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt b/confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt deleted file mode 100644 index c1423b6..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9-SNAPSHOT") diff --git a/confs/scala-native-0.3.9-SNAPSHOT/run b/confs/scala-native-0.3.9-SNAPSHOT/run deleted file mode 100644 index ae89e34..0000000 --- a/confs/scala-native-0.3.9-SNAPSHOT/run +++ /dev/null @@ -1 +0,0 @@ -target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt b/confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt deleted file mode 100644 index 2104b7a..0000000 --- a/confs/scala-native-0.4.0-SNAPSHOT-commix/build.sbt +++ /dev/null @@ -1,6 +0,0 @@ -scalaVersion := "2.11.12" -enablePlugins(ScalaNativePlugin) -nativeLinkStubs := true -nativeGC := "commix" -nativeMode := "release" -nativeLTO := "thin" diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/compile b/confs/scala-native-0.4.0-SNAPSHOT-commix/compile deleted file mode 100644 index 2f3f09f..0000000 --- a/confs/scala-native-0.4.0-SNAPSHOT-commix/compile +++ /dev/null @@ -1 +0,0 @@ -nativeLink diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt b/confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt deleted file mode 100644 index 2a63bf0..0000000 --- a/confs/scala-native-0.4.0-SNAPSHOT-commix/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-SNAPSHOT") diff --git a/confs/scala-native-0.4.0-SNAPSHOT-commix/run b/confs/scala-native-0.4.0-SNAPSHOT-commix/run deleted file mode 100644 index ae89e34..0000000 --- a/confs/scala-native-0.4.0-SNAPSHOT-commix/run +++ /dev/null @@ -1 +0,0 @@ -target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-SNAPSHOT/build.sbt b/confs/scala-native-0.4.0-SNAPSHOT/build.sbt deleted file mode 100644 index ae87f31..0000000 --- a/confs/scala-native-0.4.0-SNAPSHOT/build.sbt +++ /dev/null @@ -1,6 +0,0 @@ -scalaVersion := "2.11.12" -enablePlugins(ScalaNativePlugin) -nativeLinkStubs := true -nativeGC := "immix" -nativeMode := "release" -nativeLTO := "thin" diff --git a/confs/scala-native-0.4.0-SNAPSHOT/compile b/confs/scala-native-0.4.0-SNAPSHOT/compile deleted file mode 100644 index 2f3f09f..0000000 --- a/confs/scala-native-0.4.0-SNAPSHOT/compile +++ /dev/null @@ -1 +0,0 @@ -nativeLink diff --git a/confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt b/confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt deleted file mode 100644 index 2a63bf0..0000000 --- a/confs/scala-native-0.4.0-SNAPSHOT/plugins.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.0-SNAPSHOT") diff --git a/confs/scala-native-0.4.0-SNAPSHOT/run b/confs/scala-native-0.4.0-SNAPSHOT/run deleted file mode 100644 index ae89e34..0000000 --- a/confs/scala-native-0.4.0-SNAPSHOT/run +++ /dev/null @@ -1 +0,0 @@ -target/scala-2.11/scala-native-benchmarks-out From b2e254466570435d55f108a821ea00c0799063a0 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 16 Feb 2021 21:47:20 +0100 Subject: [PATCH 23/32] Rebase onto current master, remove Python2 syntax, --- .gitignore | 7 +- .../notebook-checkpoint.ipynb | 195 ------------------ scripts/notebook.ipynb | 23 +-- scripts/run.py | 7 +- scripts/shared/benchmarks.py | 13 +- scripts/shared/cmdline.py | 14 +- scripts/shared/comparison.py | 13 +- scripts/shared/configurations.py | 61 ++++-- scripts/shared/file_utils.py | 2 +- scripts/shared/misc_utils.py | 6 +- scripts/shared/parser.py | 2 +- scripts/shared/reports.py | 2 +- scripts/size-summary.py | 9 +- 13 files changed, 88 insertions(+), 266 deletions(-) delete mode 100644 scripts/.ipynb_checkpoints/notebook-checkpoint.ipynb diff --git a/.gitignore b/.gitignore index 41a2088..97d29fc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,9 @@ reports/ *.iprof .idea build.sbt -project/plugins.sbt \ No newline at end of file +project/plugins.sbt +/scripts/.ipynb_checkpoints/ +/.bloop/ +/.bsp/ +/.metals/ +/.vscode/ diff --git a/scripts/.ipynb_checkpoints/notebook-checkpoint.ipynb b/scripts/.ipynb_checkpoints/notebook-checkpoint.ipynb deleted file mode 100644 index cd185bf..0000000 --- a/scripts/.ipynb_checkpoints/notebook-checkpoint.ipynb +++ /dev/null @@ -1,195 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'matplotlib'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'matplotlib inline'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mmagic\u001b[0;34m(self, arg_s)\u001b[0m\n\u001b[1;32m 2156\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marg_s\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2157\u001b[0m \u001b[0mmagic_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprefilter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mESC_MAGIC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2158\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2160\u001b[0m \u001b[0;31m#-------------------------------------------------------------------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_line_magic\u001b[0;34m(self, magic_name, line)\u001b[0m\n\u001b[1;32m 2077\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'local_ns'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_locals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2078\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2079\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2080\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2081\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mmatplotlib\u001b[0;34m(self, line)\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/magics/pylab.py\u001b[0m in \u001b[0;36mmatplotlib\u001b[0;34m(self, line)\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Available matplotlib backends: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mbackends_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m \u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_matplotlib\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 101\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_show_matplotlib_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36menable_matplotlib\u001b[0;34m(self, gui)\u001b[0m\n\u001b[1;32m 2933\u001b[0m \"\"\"\n\u001b[1;32m 2934\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpylabtools\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2935\u001b[0;31m \u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_gui_and_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpylab_gui_select\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2936\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2937\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'inline'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/pylabtools.py\u001b[0m in \u001b[0;36mfind_gui_and_backend\u001b[0;34m(gui, gui_select)\u001b[0m\n\u001b[1;32m 257\u001b[0m \"\"\"\n\u001b[1;32m 258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 259\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'auto'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'matplotlib'" - ] - } - ], - "source": [ - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import matplotlib\n", - "import matplotlib.pyplot as plt\n", - "matplotlib.rcParams['figure.figsize'] = (20.0, 15.0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "rawpoints = []\n", - "with open(\"results/bounce.BounceBenchmark/1\") as data:\n", - " for line in data.readlines():\n", - " rawpoints.append(float(line)/1000000)\n", - "raw = np.array(rawpoints)\n", - "filtered = []\n", - "pmax = np.percentile(raw[500:], 99)\n", - "for point in rawpoints:\n", - " if point < pmax:\n", - " filtered.append(point)\n", - "y = np.array(filtered)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "len(y)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": false - }, - "outputs": [], - "source": [ - "#plt.axis([0, len(y), 0.5, 5])\n", - "plt.plot(y)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": false - }, - "outputs": [], - "source": [ - "plt.hist(y[:200], bins = 10)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "percentiles = np.arange(0, 100)\n", - "percvalue = np.array([np.percentile(y[:500], perc) for perc in percentiles])\n", - "plt.plot(percentiles, percvalue)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "np.percentile(y, 5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "np.percentile(y, 50)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "np.percentile(y, 95)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "np.mean(y)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "y.std()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/scripts/notebook.ipynb b/scripts/notebook.ipynb index 75c9347..7e0c576 100644 --- a/scripts/notebook.ipynb +++ b/scripts/notebook.ipynb @@ -7,26 +7,7 @@ "collapsed": false, "scrolled": true }, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'matplotlib'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'matplotlib inline'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mmagic\u001b[0;34m(self, arg_s)\u001b[0m\n\u001b[1;32m 2156\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marg_s\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2157\u001b[0m \u001b[0mmagic_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprefilter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mESC_MAGIC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2158\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2160\u001b[0m \u001b[0;31m#-------------------------------------------------------------------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_line_magic\u001b[0;34m(self, magic_name, line)\u001b[0m\n\u001b[1;32m 2077\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'local_ns'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_locals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2078\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2079\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2080\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2081\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mmatplotlib\u001b[0;34m(self, line)\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/magics/pylab.py\u001b[0m in \u001b[0;36mmatplotlib\u001b[0;34m(self, line)\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Available matplotlib backends: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mbackends_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m \u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_matplotlib\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 101\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_show_matplotlib_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36menable_matplotlib\u001b[0;34m(self, gui)\u001b[0m\n\u001b[1;32m 2933\u001b[0m \"\"\"\n\u001b[1;32m 2934\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpylabtools\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2935\u001b[0;31m \u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_gui_and_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgui\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpylab_gui_select\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2936\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2937\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'inline'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3/dist-packages/IPython/core/pylabtools.py\u001b[0m in \u001b[0;36mfind_gui_and_backend\u001b[0;34m(gui, gui_select)\u001b[0m\n\u001b[1;32m 257\u001b[0m \"\"\"\n\u001b[1;32m 258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 259\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mgui\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'auto'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'matplotlib'" - ] - } - ], + "outputs": [], "source": [ "%matplotlib inline" ] @@ -188,7 +169,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.3" + "version": "3.8.5" } }, "nbformat": 4, diff --git a/scripts/run.py b/scripts/run.py index 6702948..bd30754 100755 --- a/scripts/run.py +++ b/scripts/run.py @@ -2,7 +2,7 @@ import argparse from shared.cmdline import expand_all -from shared.benchmarks import all_benchmarks +from shared.benchmarks import benchmarks from shared.configurations import Configuration, default_runs, default_batches if __name__ == "__main__": @@ -12,10 +12,9 @@ parser.add_argument("--batches", help="number of batches per run", type=int, default=default_batches) parser.add_argument("configurations", nargs='*', default=[None]) args = parser.parse_args() - print args configuration_names = expand_all(args.configurations) - print "configurations:", configuration_names + print("configurations:", configuration_names) for conf_name in configuration_names: - Configuration(conf_name, batches=args.batches, runs=args.runs).run_benchmarks(all_benchmarks) + Configuration(conf_name, batches=args.batches, runs=args.runs).run_benchmarks(benchmarks) diff --git a/scripts/shared/benchmarks.py b/scripts/shared/benchmarks.py index 59fb67d..97635cb 100644 --- a/scripts/shared/benchmarks.py +++ b/scripts/shared/benchmarks.py @@ -1,8 +1,11 @@ import os -from file_utils import sbt, run, slurp, mkdir +from shared.file_utils import sbt, run, slurp, mkdir +import time +import subprocess +import sys -all_benchmarks = [ +benchmarks = [ 'bounce.BounceBenchmark', 'list.ListBenchmark', 'queens.QueensBenchmark', @@ -19,6 +22,7 @@ 'rsc.RscBenchmark', 'gcbench.GCBenchBenchmark', 'mandelbrot.MandelbrotBenchmark', + 'histogram.Histogram' ] @@ -32,7 +36,10 @@ def compile(self, conf): cmd = [sbt, '-J-Xmx6G', 'clean', 'set mainClass in Compile := Some("{}")'.format(self.name), conf.compile_cmd()] - return run(cmd) + start = time.time_ns() + subprocess.run(cmd, stdout=sys.stdout, stderr=subprocess.STDOUT) + end = time.time_ns() + return end - start def run(self, conf): run_cmd = conf.run_cmd(self) diff --git a/scripts/shared/cmdline.py b/scripts/shared/cmdline.py index bc3b048..a4ab164 100644 --- a/scripts/shared/cmdline.py +++ b/scripts/shared/cmdline.py @@ -1,24 +1,26 @@ -stable = 'scala-native-0.3.9' -latest = 'scala-native-0.4.0-SNAPSHOT' - +stable = 'scala-native-0.4.0' +latest = 'scala-native-0.4.1-SNAPSHOT' +default_scala_version = "2.12" def expand_all(arg): confs = [] for choice in arg: expanded = expand_wild_cards(choice) if expanded is None: - confs = [stable, latest] + confs = list(map(add_version_suffix, [stable, latest])) else: confs += [expanded] return confs +def add_version_suffix(arg): + return arg + "-" + default_scala_version def expand_wild_cards(arg): if arg is None: return arg elif arg.startswith("latest"): - return latest + arg[len("latest"):] + return add_version_suffix(latest) + arg[len("latest"):] elif arg.startswith("stable"): - return stable + arg[len("stable"):] + return add_version_suffix(stable) + arg[len("stable"):] else: return arg \ No newline at end of file diff --git a/scripts/shared/comparison.py b/scripts/shared/comparison.py index fef5e17..be6c1e5 100644 --- a/scripts/shared/comparison.py +++ b/scripts/shared/comparison.py @@ -1,19 +1,18 @@ import numpy as np -from configurations import Configuration -from parser import config_data -from reports import Report -from misc_utils import dict_write_arr, dict_get_arr +from shared.configurations import Configuration +from shared.parser import config_data +from shared.reports import Report +from shared.misc_utils import dict_write_arr, dict_get_arr default_warmup = 2000 - class Comparison: def __init__(self, confs, warmup=default_warmup): assert len(confs) > 0 configurations = [] for item in confs: - if isinstance(item, basestring): + if isinstance(item, str): configurations.append(Configuration.from_results(item)) else: configurations.append(item) @@ -37,7 +36,7 @@ def percentile_bench(self, bench, p): for conf in self.configurations: try: res.append(np.percentile(config_data(bench, conf, self.warmup), p)) - except IndexError: + except (IndexError, FileNotFoundError): res.append(0) return res diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py index a81ff98..c3fd05e 100644 --- a/scripts/shared/configurations.py +++ b/scripts/shared/configurations.py @@ -1,13 +1,15 @@ import os import shutil as sh -from benchmarks import Benchmark -from file_utils import slurp, mkdir, dict_from_file, dict_to_file +from shared.benchmarks import Benchmark +from shared.file_utils import slurp, mkdir, dict_from_file, dict_to_file -confs_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + "/confs" -results_path = 'results' +base_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) +confs_path = base_path + "/confs" +binaries_path = base_path + "/binaries" +results_path = base_path + '/results' -default_batches = 4000 +default_batches = 2000 default_runs = 20 @@ -16,9 +18,11 @@ class Configuration: def __init__(self, name, batches=default_batches, runs=default_runs): self.name = name self.conf_dir = os.path.join(confs_path, self.name) + self.bin_dir = os.path.join(binaries_path, self.name) self.batches = batches self.batch_size = 1 self.runs = runs + self.isJVM = self.name.lower().startswith("jvm") suffix = "" if runs != default_runs: @@ -37,8 +41,13 @@ def from_results(cls, full_name): @classmethod def from_results_dir(cls, full_path): settings_file = os.path.join(full_path, 'settings.properties') - kv = dict_from_file(settings_file) - return cls.from_dict(kv) + try: + kv = dict_from_file(settings_file) + return cls.from_dict(kv) + except FileNotFoundError: + kv = {} + kv['name'] = os.path.basename(full_path) + return cls.from_dict(kv) @classmethod def from_dict(cls, kv): @@ -60,24 +69,38 @@ def make_active(self): else: os.remove('project/plugins.sbt') + properties_sbt_src = os.path.join(self.conf_dir, 'build.properties') + if os.path.exists(properties_sbt_src): + sh.copyfile(properties_sbt_src, 'project/build.properties') + else: + os.remove('project/build.properties') + def run_benchmarks(self, benchmarks): - self.make_active() + for item in benchmarks: - if isinstance(item, basestring): + if isinstance(item, str): bench = Benchmark(item) else: bench = item print('--- conf: {}, bench: {}'.format(self.name, bench)) - - bench.compile(self) results_dir = bench.ensure_results_dir(self) + bin_file = os.path.join(self.bin_dir, bench.name) + if not os.path.exists(self.bin_dir): + mkdir(self.bin_dir) + if not os.path.exists(bin_file): + self.make_active() + compile_time = bench.compile(self) + with open(os.path.join(results_dir, "compile_time"), 'w') as f: + f.write(str(compile_time)) + + sh.copyfile(self.run_cmd(bench)[0], bin_file) + runs = self.runs - for n in xrange(runs): + for n in range(runs): print('--- run {}/{}'.format(n, runs)) - out = bench.run(self) - with open(os.path.join(results_dir, str(n)), 'w+') as resultfile: + with open(os.path.join(results_dir, str(n)), 'wb') as resultfile: resultfile.write(out) def run_cmd(self, bench): @@ -106,10 +129,10 @@ def ensure_results_dir(self): def finished_benchmarks(self): benchmarks = [] results_dir = self.results_dir - all_subdirs = next(os.walk(results_dir))[1] + all_subdirs = next(os.walk(results_dir), [results_dir, []])[1] for subdir in all_subdirs: all_runs_present = True - for run in xrange(self.runs): + for run in range(self.runs): if not os.path.isfile(os.path.join(results_dir, subdir, str(run))): all_runs_present = False break @@ -120,7 +143,7 @@ def finished_benchmarks(self): def benchmark_result_files(self, benchmark): benchmark_dir = benchmark.results_dir(self) - for run in xrange(self.runs): + for run in range(self.runs): yield os.path.join(benchmark_dir, str(run)) def __str__(self): @@ -135,10 +158,10 @@ def __str__(self): return s -all_configs = next(os.walk(confs_path))[1] +configs = next(os.walk(confs_path))[1] if 'GRAALVM_HOME' in os.environ: - all_configs += [ + configs += [ 'native-image', 'native-image-pgo', ] diff --git a/scripts/shared/file_utils.py b/scripts/shared/file_utils.py index d7d0809..e4146f0 100644 --- a/scripts/shared/file_utils.py +++ b/scripts/shared/file_utils.py @@ -51,7 +51,7 @@ def dict_from_file(settings_file): def dict_to_file(settings_file, kv): with open(settings_file, 'w+') as settings: - for k, v in kv.iteritems(): + for k, v in kv.items(): settings.write('{}={}\n'.format(k, v)) diff --git a/scripts/shared/misc_utils.py b/scripts/shared/misc_utils.py index 695274a..3a95ab2 100644 --- a/scripts/shared/misc_utils.py +++ b/scripts/shared/misc_utils.py @@ -1,6 +1,6 @@ def dict_write_arr(kv, arr_key, conf_kvs): for index, inner_kv in enumerate(conf_kvs): - for k, v in inner_kv.iteritems(): + for k, v in inner_kv.items(): kv['{}.{}.{}'.format(arr_key, index, k)] = v @@ -8,13 +8,13 @@ def dict_get_arr(kv, arr_key): arr = [] arr_kv = {} max_index = -1 - for k, v in kv.iteritems(): + for k, v in kv.items(): if k.startswith(arr_key + '.'): _, raw_index, key = k.split('.') index = int(raw_index) max_index = max(index, max_index) arr_kv.get(index, {})[key] = v - for index in xrange(max_index + 1): + for index in range(max_index + 1): arr.append(arr_kv.get(index)) return arr \ No newline at end of file diff --git a/scripts/shared/parser.py b/scripts/shared/parser.py index c7fd96e..e72e9f0 100644 --- a/scripts/shared/parser.py +++ b/scripts/shared/parser.py @@ -12,7 +12,7 @@ def config_data(bench, conf, warmup): # in ms raw_points.append(float(line) / 1000000) except Exception as e: - print e + print(e) out += raw_points[warmup:] except IOError: pass diff --git a/scripts/shared/reports.py b/scripts/shared/reports.py index cf9692e..056dcbb 100644 --- a/scripts/shared/reports.py +++ b/scripts/shared/reports.py @@ -1,7 +1,7 @@ import os from datetime import datetime -from file_utils import mkdir, dict_to_file, dict_from_file, touch +from shared.file_utils import mkdir, dict_to_file, dict_from_file, touch date_format = '%Y%m%d_%H%M%S' diff --git a/scripts/size-summary.py b/scripts/size-summary.py index 65783c9..161dd94 100644 --- a/scripts/size-summary.py +++ b/scripts/size-summary.py @@ -1,10 +1,11 @@ -from run import benchmarks, runs, configurations +from shared.benchmarks import benchmarks +from shared.configurations import configs, default_runs import os import numpy as np def config_data(bench, conf): try: - return len(open('binaries/{}/{}/{}'.format(conf, bench, bench), 'rb').read()) + return len(open('binaries/{}/{}'.format(conf, bench), 'rb').read()) except IOError: return 0 @@ -12,7 +13,7 @@ def binary_size(): out = [] for bench in benchmarks: res = [] - for conf in configurations: + for conf in configs: try: res.append(np.percentile(config_data(bench, conf), 10)) except IndexError: @@ -22,7 +23,7 @@ def binary_size(): if __name__ == '__main__': leading = ['name'] - for conf in configurations: + for conf in configs: leading.append(conf) print(','.join(leading)) for bench, res in zip(benchmarks, binary_size()): From 9862766dbe2d20a447306772e563cb8b2a1470fe Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 16 Feb 2021 21:47:51 +0100 Subject: [PATCH 24/32] Add 0.4.1-SNAPSHOT config --- confs/scala-native-0.4.1-SNAPSHOT-2.12/build.properties | 1 + .../compile | 0 confs/scala-native-0.4.1-SNAPSHOT-2.12/plugins.sbt | 1 + confs/scala-native-0.4.1-SNAPSHOT-2.12/run | 1 + 4 files changed, 3 insertions(+) create mode 100644 confs/scala-native-0.4.1-SNAPSHOT-2.12/build.properties rename confs/{scala-native-0.3.9-SNAPSHOT-commix => scala-native-0.4.1-SNAPSHOT-2.12}/compile (100%) create mode 100644 confs/scala-native-0.4.1-SNAPSHOT-2.12/plugins.sbt create mode 100644 confs/scala-native-0.4.1-SNAPSHOT-2.12/run diff --git a/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.properties b/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.properties new file mode 100644 index 0000000..d18a12e --- /dev/null +++ b/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.properties @@ -0,0 +1 @@ +sbt.version = 1.4.7 diff --git a/confs/scala-native-0.3.9-SNAPSHOT-commix/compile b/confs/scala-native-0.4.1-SNAPSHOT-2.12/compile similarity index 100% rename from confs/scala-native-0.3.9-SNAPSHOT-commix/compile rename to confs/scala-native-0.4.1-SNAPSHOT-2.12/compile diff --git a/confs/scala-native-0.4.1-SNAPSHOT-2.12/plugins.sbt b/confs/scala-native-0.4.1-SNAPSHOT-2.12/plugins.sbt new file mode 100644 index 0000000..05fa43c --- /dev/null +++ b/confs/scala-native-0.4.1-SNAPSHOT-2.12/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.1-SNAPSHOT") diff --git a/confs/scala-native-0.4.1-SNAPSHOT-2.12/run b/confs/scala-native-0.4.1-SNAPSHOT-2.12/run new file mode 100644 index 0000000..f21e891 --- /dev/null +++ b/confs/scala-native-0.4.1-SNAPSHOT-2.12/run @@ -0,0 +1 @@ +target/scala-2.12/benchmarks-out From 96269c5dd9bd38e0124766263e455800901253f6 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 16 Feb 2021 21:49:01 +0100 Subject: [PATCH 25/32] Remove accidentally create files --- confs/scala-native-0.4.0-M2/build.propert | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 confs/scala-native-0.4.0-M2/build.propert diff --git a/confs/scala-native-0.4.0-M2/build.propert b/confs/scala-native-0.4.0-M2/build.propert deleted file mode 100644 index e69de29..0000000 From dc48d4f50ea0e40c667cd7b2841a9be1fc5d37a1 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 16 Feb 2021 22:02:14 +0100 Subject: [PATCH 26/32] Update default interactive notepad --- scripts/notebook.ipynb | 57 ++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/scripts/notebook.ipynb b/scripts/notebook.ipynb index 7e0c576..472285a 100644 --- a/scripts/notebook.ipynb +++ b/scripts/notebook.ipynb @@ -2,9 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { - "collapsed": false, "scrolled": true }, "outputs": [], @@ -15,9 +14,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -29,13 +26,14 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "rawpoints = []\n", - "with open(\"results/bounce.BounceBenchmark/1\") as data:\n", + "conf = \"scala-native-0.4.0-2.12\"\n", + "bench = \"bounce.BounceBenchmark\"\n", + "run = 1\n", + "with open(\"../results/{}/{}/{}\".format(conf, bench, run)) as data:\n", " for line in data.readlines():\n", " rawpoints.append(float(line)/1000000)\n", "raw = np.array(rawpoints)\n", @@ -50,9 +48,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "len(y)" @@ -62,7 +58,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false, "scrolled": false }, "outputs": [], @@ -75,7 +70,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false, "scrolled": false }, "outputs": [], @@ -86,9 +80,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "percentiles = np.arange(0, 100)\n", @@ -99,9 +91,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "np.percentile(y, 5)" @@ -111,7 +101,6 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false, "scrolled": true }, "outputs": [], @@ -122,9 +111,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "np.percentile(y, 95)" @@ -133,9 +120,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "np.mean(y)" @@ -144,13 +129,25 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "y.std()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From e8138d38c135b0be51715f4ad50c37274af12634 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 16 Feb 2021 22:05:07 +0100 Subject: [PATCH 27/32] Revert adding default scala version to stable and last version --- scripts/shared/cmdline.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/shared/cmdline.py b/scripts/shared/cmdline.py index a4ab164..9a0494c 100644 --- a/scripts/shared/cmdline.py +++ b/scripts/shared/cmdline.py @@ -1,26 +1,22 @@ -stable = 'scala-native-0.4.0' -latest = 'scala-native-0.4.1-SNAPSHOT' -default_scala_version = "2.12" +stable = 'scala-native-0.4.0-2.12' +latest = 'scala-native-0.4.1-SNAPSHOT-2.12' def expand_all(arg): confs = [] for choice in arg: expanded = expand_wild_cards(choice) if expanded is None: - confs = list(map(add_version_suffix, [stable, latest])) + confs = [stable, latest] else: confs += [expanded] return confs -def add_version_suffix(arg): - return arg + "-" + default_scala_version - def expand_wild_cards(arg): if arg is None: return arg elif arg.startswith("latest"): - return add_version_suffix(latest) + arg[len("latest"):] + return latest + arg[len("latest"):] elif arg.startswith("stable"): - return add_version_suffix(stable) + arg[len("stable"):] + return stable + arg[len("stable"):] else: return arg \ No newline at end of file From a1a7e43be66f0742ddbca65b59d628c2e6827c35 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 16 Feb 2021 23:08:16 +0100 Subject: [PATCH 28/32] Allow for running jvm benchmarks --- scripts/shared/configurations.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py index c3fd05e..188aa24 100644 --- a/scripts/shared/configurations.py +++ b/scripts/shared/configurations.py @@ -84,17 +84,17 @@ def run_benchmarks(self, benchmarks): bench = item print('--- conf: {}, bench: {}'.format(self.name, bench)) results_dir = bench.ensure_results_dir(self) - - bin_file = os.path.join(self.bin_dir, bench.name) - if not os.path.exists(self.bin_dir): - mkdir(self.bin_dir) - if not os.path.exists(bin_file): + if self.isJVM or not os.path.exists(self.__bin_file(bench)): self.make_active() compile_time = bench.compile(self) with open(os.path.join(results_dir, "compile_time"), 'w') as f: f.write(str(compile_time)) - sh.copyfile(self.run_cmd(bench)[0], bin_file) + if not self.isJVM: + if not os.path.exists(self.bin_dir): + mkdir(self.bin_dir) + bin_file = self.__bin_file(bench) + sh.move(self.__native_binary_path(), bin_file) runs = self.runs for n in range(runs): @@ -103,12 +103,25 @@ def run_benchmarks(self, benchmarks): with open(os.path.join(results_dir, str(n)), 'wb') as resultfile: resultfile.write(out) + def __bin_file(self, bench): + assert not self.isJVM + return os.path.join(self.bin_dir, bench.name) + + def __native_binary_path(self): + assert not self.isJVM + return slurp(os.path.join(self.conf_dir, 'run')).split(' ')[0] + def run_cmd(self, bench): - return slurp(os.path.join(self.conf_dir, 'run')) \ + cmd = slurp(os.path.join(self.conf_dir, 'run')) \ .replace('$BENCH', bench.name) \ .replace('$HOME', os.environ['HOME']) \ .split(' ') + if self.isJVM: + return cmd + else: + return [ self.__bin_file(bench) ] + cmd[1:] + def compile_cmd(self): return slurp(os.path.join(self.conf_dir, 'compile')) From 5ab30afe07eea8cf3362e4905afc9793b5b03a64 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 16 Feb 2021 23:20:10 +0100 Subject: [PATCH 29/32] Styling fixes --- scripts/shared/configurations.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py index 188aa24..0343c36 100644 --- a/scripts/shared/configurations.py +++ b/scripts/shared/configurations.py @@ -5,14 +5,13 @@ from shared.file_utils import slurp, mkdir, dict_from_file, dict_to_file base_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) -confs_path = base_path + "/confs" -binaries_path = base_path + "/binaries" -results_path = base_path + '/results' +confs_path = base_path + '/confs' +binaries_path = base_path + '/binaries' +results_path = base_path + '/results' default_batches = 2000 default_runs = 20 - class Configuration: def __init__(self, name, batches=default_batches, runs=default_runs): From 81bde8e0554dda2bf8a29fd01cf9a247a7805fe5 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 16 Feb 2021 23:37:05 +0100 Subject: [PATCH 30/32] Fix bug in comparsion, lower default_warmup to 25% of default iterations --- scripts/shared/comparison.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/shared/comparison.py b/scripts/shared/comparison.py index be6c1e5..036148a 100644 --- a/scripts/shared/comparison.py +++ b/scripts/shared/comparison.py @@ -4,7 +4,7 @@ from shared.reports import Report from shared.misc_utils import dict_write_arr, dict_get_arr -default_warmup = 2000 +default_warmup = 500 class Comparison: def __init__(self, confs, warmup=default_warmup): @@ -21,7 +21,7 @@ def __init__(self, confs, warmup=default_warmup): common_benchmarks = configurations[0].finished_benchmarks() for other_conf in configurations[1:]: present = set(other_conf.finished_benchmarks()) - common_benchmarks = filter(lambda b: b in present, common_benchmarks) + common_benchmarks = list(filter(lambda b: b in present, common_benchmarks)) self.common_benchmarks = common_benchmarks self.warmup = warmup From c96ac4ad807d0dfbe5b7596ceddf3ce908d5c6e1 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Wed, 17 Feb 2021 11:33:15 +0100 Subject: [PATCH 31/32] Normalize names of modules and run commands in case if local repository name differs from scala-native-benchmarks --- confs/scala-native-0.3.9/run | 2 +- confs/scala-native-0.4.0-2.11/build.sbt | 5 +++-- confs/scala-native-0.4.0-2.11/run | 2 +- confs/scala-native-0.4.0-2.12/build.sbt | 5 +++-- confs/scala-native-0.4.0-2.12/run | 2 +- confs/scala-native-0.4.0-M2/build.sbt | 1 + confs/scala-native-0.4.0-M2/run | 2 +- confs/scala-native-0.4.1-SNAPSHOT-2.12/build.sbt | 9 +++++++++ confs/scala-native-0.4.1-SNAPSHOT-2.12/run | 2 +- 9 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 confs/scala-native-0.4.1-SNAPSHOT-2.12/build.sbt diff --git a/confs/scala-native-0.3.9/run b/confs/scala-native-0.3.9/run index 8fedce3..ae89e34 100644 --- a/confs/scala-native-0.3.9/run +++ b/confs/scala-native-0.3.9/run @@ -1 +1 @@ -target/scala-2.11/benchmarks-out +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-2.11/build.sbt b/confs/scala-native-0.4.0-2.11/build.sbt index 80c509f..a5ecbdd 100644 --- a/confs/scala-native-0.4.0-2.11/build.sbt +++ b/confs/scala-native-0.4.0-2.11/build.sbt @@ -1,8 +1,9 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) import scala.scalanative.build nativeConfig ~= { _.withGC(build.GC.immix) - .withMode(build.Mode.releaseFull) - .withLTO(build.LTO.thin) + .withMode(build.Mode.releaseFull) + .withLTO(build.LTO.thin) } diff --git a/confs/scala-native-0.4.0-2.11/run b/confs/scala-native-0.4.0-2.11/run index 8fedce3..ae89e34 100644 --- a/confs/scala-native-0.4.0-2.11/run +++ b/confs/scala-native-0.4.0-2.11/run @@ -1 +1 @@ -target/scala-2.11/benchmarks-out +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-2.12/build.sbt b/confs/scala-native-0.4.0-2.12/build.sbt index fca72b4..6f30aa6 100644 --- a/confs/scala-native-0.4.0-2.12/build.sbt +++ b/confs/scala-native-0.4.0-2.12/build.sbt @@ -1,8 +1,9 @@ +name := "scala-native-benchmarks" scalaVersion := "2.12.13" enablePlugins(ScalaNativePlugin) import scala.scalanative.build nativeConfig ~= { _.withGC(build.GC.immix) - .withMode(build.Mode.releaseFull) - .withLTO(build.LTO.thin) + .withMode(build.Mode.releaseFull) + .withLTO(build.LTO.thin) } diff --git a/confs/scala-native-0.4.0-2.12/run b/confs/scala-native-0.4.0-2.12/run index f21e891..4c1cad0 100644 --- a/confs/scala-native-0.4.0-2.12/run +++ b/confs/scala-native-0.4.0-2.12/run @@ -1 +1 @@ -target/scala-2.12/benchmarks-out +target/scala-2.12/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.0-M2/build.sbt b/confs/scala-native-0.4.0-M2/build.sbt index 5df1bb4..ffde723 100644 --- a/confs/scala-native-0.4.0-M2/build.sbt +++ b/confs/scala-native-0.4.0-M2/build.sbt @@ -1,3 +1,4 @@ +name := "scala-native-benchmarks" scalaVersion := "2.11.12" enablePlugins(ScalaNativePlugin) nativeLinkStubs := true diff --git a/confs/scala-native-0.4.0-M2/run b/confs/scala-native-0.4.0-M2/run index 8fedce3..ae89e34 100644 --- a/confs/scala-native-0.4.0-M2/run +++ b/confs/scala-native-0.4.0-M2/run @@ -1 +1 @@ -target/scala-2.11/benchmarks-out +target/scala-2.11/scala-native-benchmarks-out diff --git a/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.sbt b/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.sbt new file mode 100644 index 0000000..6f30aa6 --- /dev/null +++ b/confs/scala-native-0.4.1-SNAPSHOT-2.12/build.sbt @@ -0,0 +1,9 @@ +name := "scala-native-benchmarks" +scalaVersion := "2.12.13" +enablePlugins(ScalaNativePlugin) +import scala.scalanative.build +nativeConfig ~= { + _.withGC(build.GC.immix) + .withMode(build.Mode.releaseFull) + .withLTO(build.LTO.thin) +} diff --git a/confs/scala-native-0.4.1-SNAPSHOT-2.12/run b/confs/scala-native-0.4.1-SNAPSHOT-2.12/run index f21e891..4c1cad0 100644 --- a/confs/scala-native-0.4.1-SNAPSHOT-2.12/run +++ b/confs/scala-native-0.4.1-SNAPSHOT-2.12/run @@ -1 +1 @@ -target/scala-2.12/benchmarks-out +target/scala-2.12/scala-native-benchmarks-out From ed42f512e5a71ba5e9c9ba9951090fb9c82c96b6 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Fri, 19 Feb 2021 12:33:37 +0100 Subject: [PATCH 32/32] Handle missing required and optional config file --- scripts/shared/configurations.py | 36 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/shared/configurations.py b/scripts/shared/configurations.py index 0343c36..361c46c 100644 --- a/scripts/shared/configurations.py +++ b/scripts/shared/configurations.py @@ -56,24 +56,24 @@ def from_dict(cls, kv): return cls(name, batches=batches, runs=runs) def make_active(self): - build_sbt_src = os.path.join(self.conf_dir, 'build.sbt') - if os.path.exists(build_sbt_src): - sh.copyfile(build_sbt_src, 'build.sbt') - else: - os.remove('build.sbt') - - plugins_sbt_src = os.path.join(self.conf_dir, 'plugins.sbt') - if os.path.exists(plugins_sbt_src): - sh.copyfile(plugins_sbt_src, 'project/plugins.sbt') - else: - os.remove('project/plugins.sbt') - - properties_sbt_src = os.path.join(self.conf_dir, 'build.properties') - if os.path.exists(properties_sbt_src): - sh.copyfile(properties_sbt_src, 'project/build.properties') - else: - os.remove('project/build.properties') - + required = True + optional = False + + files = [ + ('build.sbt', 'build.sbt', required), + ('plugins.sbt', 'project/plugins.sbt', optional), + ('build.properties', 'project/build.properties', optional) + ] + for filename, destName, isRequired in files: + src_path = os.path.join(self.conf_dir, filename) + if os.path.exists(src_path): + sh.copyfile(src_path, destName) + elif isRequired: + raise Exception("Required file {} missing for config {}".format(filename, self.name)) + elif os.path.exists(destName): + os.remove(destName) + + def run_benchmarks(self, benchmarks): for item in benchmarks: