Log and readme improvements

etc/ci/performance/README.md

@@ -68,3 +68,12 @@ If you want to test the data submission code in `submit_to_perfherder.py` withou
 * Run `jpm xpi` in the `firefox/addon` folder
 * Install the generated `xpi` file to your Firefox Nightly
 
+# Troubleshooting
+
+ If you saw this error message:
+
+```
+venv/bin/activate: line 8: _OLD_VIRTUAL_PATH: unbound variable
+```
+
+That means your `virtualenv` is too old, try run `pip install -U virtualenv` to upgrade (If you installed ubuntu's `python-virtualenv` package, uninstall it first then install it through `pip`)

etc/ci/performance/runner.py

@@ -24,9 +24,6 @@ def parse_manifest(text):
 
 
 def execute_test(url, command, timeout):
-    print("Running test:")
-    print(' '.join(command))
-    print("Timeout:{}".format(timeout))
     try:
         return subprocess.check_output(command, stderr=subprocess.STDOUT, timeout=timeout)
     except subprocess.CalledProcessError as e:
@@ -35,7 +32,7 @@ def execute_test(url, command, timeout):
         print("You may want to re-run the test manually:\n{}"
               .format(' '.join(command)))
     except subprocess.TimeoutExpired:
-        print("Test timeout: {}".format(url))
+        print("Test FAILED due to timeout: {}".format(url))
     return ""
 
 
@@ -247,9 +244,12 @@ def main():
                                                         args.runs,
                                                         testcase))
                 log = execute_test(testcase, command, args.timeout)
+                print("Finished")
                 result = parse_log(log, testcase)
                 # TODO: Record and analyze other performance.timing properties
                 results += result
+            print("To reproduce the above test, run the following command:")
+            print("    {0}\n".format(' '.join(command)))
 
         print(format_result_summary(results))
         save_result_json(results, args.output_file, testcases, args.runs)