Implemented json schema

benchpress/__init__.py

@@ -27,3 +27,10 @@
 except DistributionNotFound:
     # package is not installed
     pass
+
+# We expose the suite schema as the dict `suite_schema`
+def _suite_schema():
+    from os.path import join, realpath, dirname
+    import json
+    return json.load(open(join(dirname(realpath(__file__)), "suite_schema.json"), "r"))
+suite_schema = _suite_schema()

benchpress/suite_schema.json

@@ -0,0 +1,72 @@
+{
+    "id": "https://raw.githubusercontent.com/bh107/benchpress/master/benchpress/suite_schema.json",
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "Benchpress Suite file that contains the commands to execute and their results",
+    "type": "object",
+    "properties": {
+        "creation_date_utc": {
+            "description": "The creation time of the suite file in UTC",
+            "type": "string"
+        },
+        "cmd_list": {
+            "description": "List of the commands that makes up this benchmark suite",
+            "type": "array",
+            "properties": {
+                "cmd": {
+                    "description": "A command to execute",
+                    "type": "string"
+                },
+                "env": {
+                    "description": "The environment variables to define",
+                    "type": "object"
+                },
+                "label": {
+                    "description": "The label of the command",
+                    "type": "string"
+                },
+                "jobs": {
+                    "description": "List of scheduled commands",
+                    "type": "array",
+                    "properties": {
+                        "status": {
+                            "description": "The status of the scheduled command",
+                            "type": "string",
+                            "pattern": "finished|failed|pending"
+                        },
+                        "nruns": {
+                            "description": "The number of times to executed the scheduled command",
+                            "type": "number",
+                            "minimum": 1
+                        },
+                        "warmup": {
+                            "description": "Include a warm up run before the recorded runs",
+                            "type": "boolean"
+                        },
+                        "results": {
+                            "description": "List of recorded results",
+                            "type": "array",
+                            "properties": {
+                                "success": {
+                                    "description": "Did the run succeed",
+                                    "type": "boolean"
+                                },
+                                "stderr": {
+                                    "description": "The standard error output",
+                                    "type": "string"
+                                },
+                                "stdout": {
+                                    "description": "The standard output",
+                                    "type": "string"
+                                }
+                            },
+                            "required": ["success"]
+                        }
+                    },
+                    "required": ["status", "nruns"]
+                }
+            },
+            "required": ["cmd", "env", "label"]
+        }
+    },
+    "required": ["creation_date_utc", "cmd_list"]
+}

benchpress/testing.py

@@ -17,6 +17,8 @@
 from os.path import join
 import sys
 import benchpress as bp
+import json
+import jsonschema
 
 
 def create_test_suite(suite_path):
@@ -37,7 +39,31 @@ def create_test_suite(suite_path):
     bp.create_suite(cmd_list, suite_path)
 
 
-class InitNumPy(unittest.TestCase):
+class SuiteSchema(unittest.TestCase):
+
+    def testSchema(self):
+        from . import run
+        from . import suite_schema
+        tmpdir = tempfile.mkdtemp()
+        suite_file = join(tmpdir, "res.json")
+
+        # Check after suite creation
+        create_test_suite(suite_file)
+        with open(suite_file, "r") as f:
+            suite = json.load(f)
+            jsonschema.validate(suite, suite_schema)
+
+        # Check after run
+        old_argv = sys.argv[:]
+        sys.argv[:] = [old_argv[0], suite_file]
+        run.main()
+        sys.argv = old_argv
+        with open(suite_file, "r") as f:
+            suite = json.load(f)
+            jsonschema.validate(suite, suite_schema)
+
+
+class BP(unittest.TestCase):
 
     def setUp(self):
         from . import run
@@ -58,6 +84,18 @@ def testCli(self):
         sys.argv = [old_argv[0], self.suite_file]
         cli.main()
 
+    def testCliCSV(self):
+        from .visualizer import cli
+        old_argv = sys.argv
+        sys.argv = [old_argv[0], self.suite_file, "--csv"]
+        cli.main()
+
+    def testJSON(self):
+        from . import suite_schema
+        with open(self.suite_file, "r") as f:
+            suite = json.load(f)
+            jsonschema.validate(suite, suite_schema)
+
 
 def main():
     unittest.main()

package/pypi/build-n-upload.py

@@ -34,7 +34,6 @@ def main(args):
 
     # Clean up the repos
     if args.git_cleanup:
-        bash_cmd('git reset --hard master', cwd=bp_dir)
         bash_cmd('git clean -xdf', cwd=bp_dir)
 
     # Update the repos

setup.py

@@ -88,7 +88,7 @@ def find_data_files(relative_to, directory, regex_exclude="\.pyc|{0}bin{0}|{0}ob
     # your project is installed. For an analysis of "install_requires" vs pip's
     # requirements files see:
     # https://packaging.python.org/en/latest/requirements.html
-    install_requires=['numpy'],
+    install_requires=['numpy', 'jsonschema'],
 
     # List additional groups of dependencies here (e.g. development
     # dependencies). You can install these using the following syntax,