Updated all benchmarks to Play 2.3.6 and Scala 2.11.4.

Updated Ebean benchmarks to include missing tests as well as include HikariCP version. Moved `/plaintext` tests from JPA/Ebean benchmarks to the `play2-java` benchmark as per TechEmpower#1149 (comment) Added BoneCP qualifier as per TechEmpower#1149 (comment) and changed names accordingly.
donovanmuller · Nov 3, 2014 · 0a57b1d · 0a57b1d
1 parent 9f295a5
commit 0a57b1d
Show file tree

Hide file tree

Showing 155 changed files with 3,399 additions and 1,118 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -123,6 +123,7 @@ env:
     - "TESTDIR=PHP/php-yaf"
     - "TESTDIR=PHP/php-yii2"
     - "TESTDIR=PHP/php-zend-framework"
+    - "TESTDIR=PHP/php-zend-framework1"
     - "TESTDIR=PHP/phreeze"
     - "TESTDIR=Python/bottle"
     - "TESTDIR=Python/django"

diff --git a/benchmark.cfg.example b/benchmark.cfg.example
@@ -12,21 +12,18 @@ exclude=None
 install=server
 install_error_action=continue
 install_strategy=unified
-install_only=False
-list_test_metadata=False
-list_tests=False
-max_concurrency=256
-max_queries=20
-max_threads=8
+#install_only=True
+#list_test_metadata=True
+#list_tests=True
+concurrency_levels=[8, 16, 32, 64, 128, 256]
+query_levels=[1, 5,10,15,20]
+threads=8
 mode=benchmark
 name=ec2
 os=linux
-parse=None
 password_prompt=False
-query_interval=5
 server_host=localhost
 sleep=60
-starting_concurrency=8
 test=None
 type=all
 verbose=True
diff --git a/frameworks/C/lwan/benchmark_config b/frameworks/C/lwan/benchmark_config
@@ -1,7 +1,7 @@
 {
   "framework": "lwan",
   "tests": [{
-    "raw": {
+    "sqlite": {
       "setup_file": "setup",
       "db_url": "/db",
       "query_url": "/queries?queries=",
@@ -22,6 +22,28 @@
       "display_name": "Lwan",
       "notes": "",
       "versus": ""
+    },
+    "mysql": {
+      "setup_file": "setup",
+      "db_url": "/db",
+      "query_url": "/queries?queries=",
+      "fortune_url": "/fortunes",
+      "plaintext_url": "/plaintext",
+      "json_url": "/json",
+      "port": 8080,
+      "approach": "Realistic",
+      "classification": "Platform",
+      "database": "MySQL",
+      "framework": "lwan",
+      "language": "C",
+      "orm": "Raw",
+      "platform": "Lwan",
+      "webserver": "Lwan",
+      "os": "Linux",
+      "database_os": "Linux",
+      "display_name": "Lwan",
+      "notes": "",
+      "versus": ""
     }
   }]
 }
diff --git a/frameworks/C/lwan/install.sh b/frameworks/C/lwan/install.sh
@@ -1,14 +1,16 @@
 #!/bin/bash
 
-REV='aa6c15fbdf63d9db722ddc72bd736b23381331be'
+REV='eb96604657dd940ecb70b56fef4279077e3f9c21'
 
 INSTALLED_FILE="${IROOT}/lwan-${REV}.installed"
 RETCODE=$(fw_exists ${INSTALLED_FILE})
 [ ! "$RETCODE" == 0 ] || { return 0; }
 
+[ ! -e ${INSTALLED_FILE} -a -d ${IROOT}/lwan ] && rm -rf ${IROOT}/lwan
+
 # Lwan is only built during installation as a dependency sanity check.
 sudo apt-get update && \
-	sudo apt-get install libjemalloc-dev && \
+	sudo apt-get install -qqy libjemalloc-dev libmysqlclient-dev libsqlite3-dev && \
 	git clone git://github.com/lpereira/lwan.git && \
         cd lwan && \
         git checkout ${REV} && \

diff --git a/frameworks/C/lwan/setup.py b/frameworks/C/lwan/setup.py
@@ -2,6 +2,17 @@
 import sys
 import os
 
+def get_env_for_database(args):
+  if args.database == 'MySQL':
+    return {
+      'USE_MYSQL': '1',
+      'MYSQL_USER': 'benchmarkdbuser',
+      'MYSQL_PASS': 'benchmarkdbpass',
+      'MYSQL_HOST': args.database_host,
+      'MYSQL_DB': 'hello_world'
+    }
+  return None
+
 def start(args, logfile, errfile):
   subprocess.call('rm -rf ${LWAN_BUILD}', shell=True, stderr=errfile, stdout=logfile)
   subprocess.call('mkdir -p ${LWAN_BUILD}', shell=True, stderr=errfile, stdout=logfile)
@@ -10,7 +21,9 @@ def start(args, logfile, errfile):
 
   db_dir = os.path.join(os.environ['LWAN_ROOT'], 'techempower')
   exe_path = os.path.join(os.environ['LWAN_BUILD'], 'techempower', 'techempower')
-  subprocess.Popen(exe_path, cwd=db_dir, stderr=errfile, stdout=logfile, shell=True)
+
+  subprocess.Popen(exe_path, cwd=db_dir, stderr=errfile, stdout=logfile, shell=True,
+      env = get_env_for_database(args))
 
   return 0
 

diff --git a/frameworks/Dart/dart-start/server.dart b/frameworks/Dart/dart-start/server.dart
@@ -288,7 +288,7 @@ main(List<String> args) {
 
           _fortuneCollection.find().toList().then((fortunes) {
             fortunes = fortunes.map((fortune) {
-              return new Fortune(fortune["_id"], fortune["message"]);
+              return new Fortune(fortune["_id"].toInt(), fortune["message"]);
             }).toList();
             fortunes.add(new Fortune(0, 'Additional fortune added at request time.'));
             fortunes.sort();

diff --git a/frameworks/Dart/dart-stream/server.dart b/frameworks/Dart/dart-stream/server.dart
@@ -243,7 +243,7 @@ _fortunesMongoTest(HttpConnect connect) {
 
   return _fortuneCollection.find().toList().then((fortunes) {
     fortunes = fortunes.map((fortune) {
-      return new Fortune(fortune["_id"], fortune["message"]);
+      return new Fortune(fortune["_id"].toInt(), fortune["message"]);
     }).toList();
     fortunes.add(new Fortune(0, 'Additional fortune added at request time.'));
     fortunes.sort();

diff --git a/frameworks/Elixir/WeberFramework/benchmark_config b/frameworks/Elixir/WeberFramework/benchmark_config
@@ -18,7 +18,6 @@
       "database_os": "Linux",
       "display_name": "weber",
       "notes": "",
-      "versus": "",
-      "skip": "false"
+      "versus": ""
   }}]
 }
diff --git a/frameworks/Erlang/cowboy/rebar.config b/frameworks/Erlang/cowboy/rebar.config
@@ -1,6 +1,6 @@
 {deps,
  [
-  {jiffy, ".*", {git, "https://github.com/davisp/jiffy.git", {tag, "0.8.3"}}},
+  {jiffy, ".*", {git, "https://github.com/davisp/jiffy.git", {tag, "0.8.5"}}},
   {mimetypes, ".*", {git, "http://github.com/spawngrid/mimetypes.git", {branch, master}}},
   {emysql, ".*", {git, "https://github.com/Eonblast/Emysql.git"}},
   {cowboy, ".*", {git, "https://github.com/extend/cowboy.git", {tag, "0.8.3"}}}

diff --git a/frameworks/Erlang/cowboy/setup_erlang.py b/frameworks/Erlang/cowboy/setup_erlang.py
@@ -6,6 +6,7 @@ def start(args, logfile, errfile):
   setup_util.replace_text("cowboy/src/hello_world_app.erl", "\"benchmarkdbpass\", \".*\", 3306", "\"benchmarkdbpass\", \"" + args.database_host + "\", 3306")
 
   try:
+    subprocess.check_call("rm -rf deps/*", shell=True, cwd="cowboy", stderr=errfile, stdout=logfile)
     subprocess.check_call("./rebar get-deps", shell=True, cwd="cowboy", stderr=errfile, stdout=logfile)
     subprocess.check_call("./rebar compile", shell=True, cwd="cowboy", stderr=errfile, stdout=logfile)
     subprocess.check_call("erl -pa ebin deps/*/ebin +sbwt very_long +swt very_low -s hello_world -noshell -detached", shell=True, cwd="cowboy", stderr=errfile, stdout=logfile)

diff --git a/frameworks/Go/revel-jet/benchmark_config b/frameworks/Go/revel-jet/benchmark_config
@@ -20,8 +20,7 @@
       "database_os": "Linux",
       "display_name": "revel-jet",
       "notes": "",
-      "versus": "go",
-      "skip": "true"
+      "versus": "go"
     }
   }]
 }
diff --git a/frameworks/Java/play2-java/benchmark_config b/frameworks/Java/play2-java/benchmark_config
@@ -18,11 +18,12 @@
         "notes": "", 
         "versus": "netty", 
         "port": "9000", 
-        "json_url": "/json"
+        "json_url": "/json", 
+        "plaintext_url": "/plaintext"
       }, 
-      "java-ebean": {
-        "display_name": "play2-java-ebean", 
-        "setup_file": "setup_java_ebean", 
+      "java-ebean-bonecp": {
+        "display_name": "play2-java-ebean-bonecp", 
+        "setup_file": "setup_java_ebean_bonecp", 
         "framework": "play2", 
         "language": "Java", 
         "orm": "Full", 
@@ -37,11 +38,13 @@
         "versus": "netty", 
         "port": "9000", 
         "db_url": "/db", 
-        "query_url": "/queries?queries="
+        "query_url": "/queries?queries=", 
+        "fortune_url": "/fortunes", 
+        "update_url": "/update?queries="
       }, 
-      "java-jpa": {
-        "display_name": "play2-java-jpa", 
-        "setup_file": "setup_java_jpa", 
+      "java-ebean-hikaricp": {
+        "display_name": "play2-java-ebean-hikaricp", 
+        "setup_file": "setup_java_ebean_hikaricp", 
         "framework": "play2", 
         "language": "Java", 
         "orm": "Full", 
@@ -56,8 +59,52 @@
         "versus": "netty", 
         "port": "9000", 
         "db_url": "/db", 
-        "query_url": "/queries?queries="
+        "query_url": "/queries?queries=", 
+        "fortune_url": "/fortunes", 
+        "update_url": "/update?queries="
+      }, 
+      "java-jpa-bonecp": {
+        "display_name": "play2-java-jpa-bonecp", 
+        "setup_file": "setup_java_jpa_bonecp", 
+        "framework": "play2", 
+        "language": "Java", 
+        "orm": "Full", 
+        "os": "Linux", 
+        "database": "MySQL", 
+        "approach": "Realistic", 
+        "classification": "Fullstack", 
+        "platform": "Netty", 
+        "webserver": "None", 
+        "database_os": "Linux", 
+        "notes": "", 
+        "versus": "netty", 
+        "port": "9000", 
+        "db_url": "/db", 
+        "query_url": "/queries?queries=", 
+        "fortune_url": "/fortunes", 
+        "update_url": "/update?queries="
+      }, 
+      "java-jpa-hikaricp": {
+        "display_name": "play2-java-jpa-hikaricp", 
+        "setup_file": "setup_java_jpa_hikaricp", 
+        "framework": "play2", 
+        "language": "Java", 
+        "orm": "Full", 
+        "os": "Linux", 
+        "database": "MySQL", 
+        "approach": "Realistic", 
+        "classification": "Fullstack", 
+        "platform": "Netty", 
+        "webserver": "None", 
+        "database_os": "Linux", 
+        "notes": "", 
+        "versus": "netty", 
+        "port": "9000", 
+        "db_url": "/db", 
+        "query_url": "/queries?queries=", 
+        "fortune_url": "/fortunes", 
+        "update_url": "/update?queries="
       }
     }
   ]
-}
+}
diff --git a/frameworks/Java/play2-java/generate_config.py b/frameworks/Java/play2-java/generate_config.py
@@ -9,12 +9,14 @@
 # Format is: (language, orm, (opsys, ...), (test, ...))
 # See the dir_name logic below to see the directory name for each test application.
 configurations = [
-  ('Java',  None,    ['Linux'],            ['json']),
-  ('Java',  'Ebean', ['Linux'],            ['db', 'query']),
-  ('Java',  'JPA',   ['Linux'],            ['db', 'query']),
-  ('Scala', None,    ['Linux'],            ['json']),
-  ('Scala', 'Anorm', ['Linux', 'Windows'], ['db', 'query', 'fortune', 'update']),
-  ('Scala', 'Slick', ['Linux'],            ['db', 'query', 'fortune', 'update']),
+  ('Java',  None,    			['Linux'],            ['json', 'plaintext']),
+  ('Java',  'Ebean BoneCP', 	['Linux'],            ['db', 'query', 'fortune', 'update']),
+  ('Java',  'Ebean HikariCP', 	['Linux'],            ['db', 'query', 'fortune', 'update']),
+  ('Java',  'JPA BoneCP',		['Linux'],            ['db', 'query', 'fortune', 'update']),
+  ('Java',  'JPA HikariCP',		['Linux'],            ['db', 'query', 'fortune', 'update']),
+  ('Scala', None,    			['Linux'],            ['json']),
+  ('Scala', 'Anorm', 			['Linux', 'Windows'], ['db', 'query', 'fortune', 'update']),
+  ('Scala', 'Slick', 			['Linux'],            ['db', 'query', 'fortune', 'update']),
 ]
 
 # All play2 test applications must use the same URLs.
@@ -24,6 +26,7 @@
   'query': '/queries?queries=',
   'fortune': '/fortunes',
   'update': '/update?queries=',
+  'plaintext': '/plaintext',
 }
 
 langs = {
@@ -41,8 +44,8 @@ def frameworksPath():
   lang_test_configs[lang] = collections.OrderedDict()
 
 for lang, orm, opsyses, tests in configurations:
-  dir_name = 'play2-' + lang.lower() + (('-'+orm.lower()) if orm else '')
-  setup_name = 'setup_' + lang.lower() + (('_'+orm.lower()) if orm else '')
+  dir_name = 'play2-' + lang.lower() + (('-'+orm.replace(' ', '-').lower()) if orm else '')
+  setup_name = 'setup_' + lang.lower() + (('_'+orm.replace(' ', '_').lower()) if orm else '')
 
   setup_path = os.path.join(pathForLang(lang), setup_name+'.py')
   print 'Generating', setup_path
@@ -57,9 +60,9 @@ def frameworksPath():
 
   for opsys in opsyses:
     if len(opsyses) == 1:
-      test_name = lang.lower() + (('-'+orm.lower()) if orm else '')
+      test_name = lang.lower() + (('-'+orm.replace(' ', '-').lower()) if orm else '')
     else:
-      test_name = lang.lower() + (('-'+orm.lower()) if orm else '') + '-'+opsys.lower()
+      test_name = lang.lower() + (('-'+orm.replace(' ', '-').lower()) if orm else '') + '-'+opsys.lower()
     test_config_json = collections.OrderedDict([
       ('display_name', 'play2-'+test_name),
       ('setup_file', setup_name),

diff --git a/...va/play2-java/play2-java-ebean/.gitignore → ...2-java/play2-java-ebean-bonecp/.gitignore b/...va/play2-java/play2-java-ebean/.gitignore → ...2-java/play2-java-ebean-bonecp/.gitignore
diff --git a/frameworks/Java/play2-java/play2-java-ebean-bonecp/README.md b/frameworks/Java/play2-java/play2-java-ebean-bonecp/README.md
@@ -0,0 +1,35 @@
+#Play Benchmarking Test
+
+This is the Play portion of a [benchmarking test suite](../) comparing a variety of web development platforms.
+
+### JSON Encoding Test
+
+* [JSON test controller](app/controllers/Application.java)
+
+### Data-Store/Database Mapping Test
+
+* [Database test controller](app/controllers/Application.java)
+* [Database World test model](app/models/World.java)
+* [Database Fortune test model](app/models/Fortune.java)
+
+### Plain Text Test
+
+* [Plain text test controller](app/controllers/Application.java)
+
+## Infrastructure Software Versions
+The tests were run with:
+
+* [Java OpenJDK 1.7](http://openjdk.java.net/)
+* [Play 2.3.6](http://http://www.playframework.com/)
+
+## Test URLs
+### JSON Encoding Test
+
+* http://localhost/json
+
+### Data-Store/Database Mapping Test
+
+* http://localhost/db
+* http://localhost/queries?queries=10
+* http://localhost/fortunes
+* http://localhost/update?queries=10