Skip to content
This repository
Browse code

A script to run tests against saucelabs (w/ contributions from Leah […

…:klrmn] Klearman and Zach [:zcarter] Carter)
  • Loading branch information...
commit 45bf72866450deb666ad1551d6ba63c567c015ca 1 parent 5cf2bc0
Lloyd Hilaiel authored August 17, 2012
75  automation-tests/lib/convert_results.js
... ...
@@ -0,0 +1,75 @@
  1
+#!/usr/bin/env node
  2
+/*
  3
+ * Converts html reports into nice, machine readable JSON
  4
+ * Run: $ ./convert_result.js result/index.html
  5
+ */
  6
+
  7
+const fs         = require('fs'),
  8
+      path       = require('path'),
  9
+      jsonselect = require('JSONSelect'),
  10
+      htmlparser = require('htmlparser');
  11
+
  12
+
  13
+function main (args) {
  14
+  var file = fs.readFile(path.resolve(args[2]), "utf8", function (err, html) {
  15
+    if (err) throw err;
  16
+    parseReport(html);
  17
+  });
  18
+}
  19
+
  20
+function parseReport (html) {
  21
+  var report = {};
  22
+  var handler = new htmlparser.DefaultHandler(function(err, dom) {
  23
+    if (err) {
  24
+      console.error("Error: " + err);
  25
+    } else {
  26
+      var results = jsonselect.match(':has(:root > .attribs > .id:val("results")) .children :has(:root > .name:val("tr"))', dom);
  27
+
  28
+      // remove header row
  29
+      results.shift();
  30
+
  31
+      results.forEach(function (node, i, array) {
  32
+        var url;
  33
+        var result = node.children[1].attribs.class;
  34
+
  35
+        // skip traceback rows
  36
+        if (!result) return;
  37
+
  38
+        try {
  39
+          url = result === 'error' ?
  40
+                  findJobUrl(array[i+1].children[1].children[1].children) :
  41
+                  node.children[9].children[0].attribs.href;
  42
+        } catch (e) {
  43
+          url = '';
  44
+        }
  45
+
  46
+        var name = node.children[5].children[0].data;
  47
+
  48
+        report[name] = {
  49
+          success: result === 'passed',
  50
+          class: node.children[3].children[0].data,
  51
+          duration: node.children[7].children[0].data,
  52
+          url: url
  53
+        };
  54
+      });
  55
+    }
  56
+  });
  57
+
  58
+  var parser = new htmlparser.Parser(handler);
  59
+  parser.parseComplete(html);
  60
+  return report;
  61
+}
  62
+
  63
+// extract saucelab url from error report
  64
+function findJobUrl (children) {
  65
+  var result;
  66
+  children.forEach(function (node) {
  67
+    var match = node.raw.match(/https:\/\/saucelabs.com\/jobs\/[a-f0-9]+/);
  68
+    if (match) result = match[0];
  69
+  });
  70
+  return result;
  71
+}
  72
+
  73
+exports.parseReport = parseReport;
  74
+
  75
+if (process.argv[1] === __filename) main(process.argv);
364  automation-tests/run_saucelabs
... ...
@@ -0,0 +1,364 @@
  1
+#!/usr/bin/env node
  2
+
  3
+var child_process = require('child_process'),
  4
+    path = require('path'),
  5
+    _ = require('underscore'),
  6
+    fs = require('fs'),
  7
+    glob = require('minimatch'),
  8
+    temp = require('temp'),
  9
+    which = require('which'),
  10
+    parseReport = require('./lib/convert_results.js').parseReport,
  11
+    events = require('events'),
  12
+    util = require('util'),
  13
+    mkdirp = require('mkdirp');
  14
+
  15
+function runCmd(cmd, opts, cb) {
  16
+  if (!cb) {
  17
+    cb = opts;
  18
+    opts = { cwd: path.dirname(__dirname) };
  19
+  }
  20
+  var cp = child_process.exec(cmd, opts, function(err, stdout, stderr) {
  21
+    cb(err, stdout, stderr);
  22
+  });
  23
+}
  24
+
  25
+function TestRunner() {
  26
+  events.EventEmitter.call(this);
  27
+}
  28
+
  29
+util.inherits(TestRunner, events.EventEmitter);
  30
+
  31
+// path to automation_tests
  32
+const testPath = path.join(path.dirname(__dirname), "automation-tests");
  33
+
  34
+// ephemeral files to hold various credentials desired by py.test gunk
  35
+const sauceYAMLPath = temp.path({suffix: '.yaml'}),
  36
+      credentialsYAMLPath = temp.path({suffix: '.yaml'});
  37
+
  38
+// python arguments common to all tests
  39
+var globalPythonArgs = {
  40
+  "-m": "py.test",
  41
+  "--credentials": credentialsYAMLPath,
  42
+  "--saucelabs": sauceYAMLPath,
  43
+  "--webqatimeout": 90,
  44
+  "--destructive": null,
  45
+  "-q": null,
  46
+  '--capabilities': JSON.stringify({ "avoid-proxy":"true"})
  47
+};
  48
+
  49
+// python arguments specific to different test classes
  50
+var testSpecificPythonArgs = {
  51
+  "123done": {
  52
+    "--baseurl": "http://dev.123done.org"
  53
+  },
  54
+  "browserid": {
  55
+    "--baseurl": "http://dev.123done.org"
  56
+  },
  57
+  "myfavoritebeer": {
  58
+    "--baseurl": "http://dev.myfavoritebeer.org"
  59
+  }
  60
+};
  61
+
  62
+var browserSpecificPythonArgs = {
  63
+  "linux_firefox_13": {
  64
+    '--platform': 'LINUX',
  65
+    '--browsername': 'firefox',
  66
+    '--browserver': '13'
  67
+  },
  68
+  "linux_opera_12": {
  69
+    '--platform': 'LINUX',
  70
+    '--browsername': 'opera',
  71
+    '--browserver': '12'
  72
+  },
  73
+  "osx_firefox_14": {
  74
+    '--platform': 'MAC',
  75
+    '--browsername':'firefox',
  76
+    '--browserver':'14'
  77
+  },
  78
+  "vista_chrome": {
  79
+    '--platform':'VISTA',
  80
+    '--browsername':'chrome'
  81
+  },
  82
+  "vista_firefox_13": {
  83
+    '--platform':'VISTA',
  84
+    '--browsername':'firefox',
  85
+    '--browserver':'13'
  86
+  },
  87
+  "vista_ie_9": {
  88
+    '--platform':'VISTA',
  89
+    '--browsername':'internet explorer',
  90
+    '--browserver':'9'
  91
+  },
  92
+  "xp_ie_8": {
  93
+    '--platform':'XP',
  94
+    '--browsername': 'internet explorer',
  95
+    '--browserver':'8'
  96
+  }
  97
+};
  98
+
  99
+function escape(val) {
  100
+  return '"'+val.replace(/(["'$`\\])/g,'\\$1')+'"';
  101
+};
  102
+
  103
+// now write a yaml file with sauce creds
  104
+function writeSauceYAML() {
  105
+  var envVars = {
  106
+    'PERSONA_SAUCE_USER': 'username',
  107
+    'PERSONA_SAUCE_PASSWORD': 'password',
  108
+    'PERSONA_SAUCE_APIKEY': 'api-key'
  109
+  };
  110
+
  111
+  var fileContents = "";
  112
+  Object.keys(envVars).forEach(function(key) {
  113
+    if (!process.env[key]) throw "missing sauce labs creds from environment";
  114
+    fileContents += envVars[key] + ": " + process.env[key] + "\n"
  115
+  });
  116
+  fs.writeFileSync(sauceYAMLPath, fileContents);
  117
+}
  118
+
  119
+// now write a yaml file with sauce creds
  120
+function writeCredsYAML() {
  121
+  var envVars = {
  122
+    'PERSONA_EMAIL': 'email',
  123
+    'PERSONA_PASSWORD': 'password'
  124
+  };
  125
+
  126
+  var fileContents = "default:\n";
  127
+  Object.keys(envVars).forEach(function(key) {
  128
+    if (!process.env[key]) throw "missing exisiting users creds from environment: " + key;
  129
+    fileContents += "    " + envVars[key] + ": " + process.env[key] + "\n"
  130
+  });
  131
+  fs.writeFileSync(credentialsYAMLPath, fileContents);
  132
+}
  133
+
  134
+// setup python testing environment iff required, return path to python
  135
+TestRunner.prototype._setupPythonEnv = function(cb) {
  136
+  var binPath = path.join(testPath, "bid_selenium", "bin");
  137
+
  138
+  function isSetup(lcb) {
  139
+    var pathToPython = path.join(binPath, "python");
  140
+    fs.stat(pathToPython, function(err, r) {
  141
+      if (!err && r.isFile()) lcb(null, { pathToPython: pathToPython });
  142
+      else lcb("not setup");
  143
+    });
  144
+  }
  145
+  function findExecutable(names, cb) {
  146
+    if (!names.length) return cb("not found");
  147
+    var n = names.shift();
  148
+    which(n, function(err, p) {
  149
+      if (err) return findExecutable(names, cb);
  150
+      cb(null, p);
  151
+    });
  152
+  }
  153
+
  154
+  isSetup(function(err, r) {
  155
+    if (!err) return cb(null, r);
  156
+
  157
+    // time to set it up!
  158
+    findExecutable([ 'virtualenv-2.7', 'virtualenv2', 'virtualenv' ], function(err, virtualenv) {
  159
+      if (err) return cb("cannot find virtualenv");
  160
+      runCmd(virtualenv + " " + path.join(testPath, "bid_selenium"), function(err, stdout) {
  161
+        if (err) return cb("cannot run virtualenv: " + err);
  162
+        runCmd(path.join(binPath, "pip") + " install -Ur requirements.txt", { cwd: testPath }, function(err, stdout) {
  163
+          if (err) cb(err);
  164
+          else isSetup(cb);
  165
+        });
  166
+      });
  167
+    });
  168
+  });
  169
+}
  170
+
  171
+TestRunner.prototype.run = function(opts, cb) {
  172
+  var self = this;
  173
+  var processesRunning = 0;
  174
+  var testReports = [];
  175
+  var overallStartTime = new Date();
  176
+
  177
+  // once all tests are complete, crawl through the data and write
  178
+  // summary information
  179
+  function addSummaryInfo(results) {
  180
+    // XXX: calculcate browser and tests summary information
  181
+    return {
  182
+      duration: ((new Date() - overallStartTime) / 1000.0),
  183
+      reports: results
  184
+    };
  185
+  }
  186
+
  187
+  opts = opts || {};
  188
+
  189
+  writeSauceYAML();
  190
+  writeCredsYAML();
  191
+
  192
+  this._setupPythonEnv(function(err, testEnvDetails) {
  193
+    if (err) {
  194
+      console.log("ERROR: couldn't setup python environment");
  195
+      return cb(err);
  196
+    }
  197
+    Object.keys(browserSpecificPythonArgs).forEach(function(browser) {
  198
+      if (opts.browser && !glob(browser, opts.browser.toString())) return;
  199
+      var browserArgs = browserSpecificPythonArgs[browser];
  200
+
  201
+      Object.keys(testSpecificPythonArgs).forEach(function(test) {
  202
+        if (opts['test-group'] && !glob(test, (opts['test-group']).toString())) return;
  203
+
  204
+        var htmlReportPath = temp.path({suffix: '.html'});
  205
+        var testArgs = testSpecificPythonArgs[test];
  206
+
  207
+        // build up the command line arguments
  208
+        var cmdargsObj = {};
  209
+        _.extend(cmdargsObj, globalPythonArgs, testArgs, browserArgs,
  210
+                 { '--webqareport': htmlReportPath });
  211
+        var cmdargs = "";
  212
+        Object.keys(cmdargsObj).forEach(function(flag) {
  213
+          var spc = " ";
  214
+          if (flag.substr(2) === '--') spc = "=";
  215
+          if (null === cmdargsObj[flag]) {
  216
+            cmdargs += flag + " ";
  217
+          } else {
  218
+            cmdargs += flag + spc + escape(cmdargsObj[flag].toString()) + " ";
  219
+          }
  220
+        });
  221
+        if (opts.single) {
  222
+          cmdargs += " " + path.relative(testPath, opts.single);
  223
+        } else {
  224
+          cmdargs += " " + test;
  225
+        }
  226
+        self.emit('started', { browser: browser, test: test });
  227
+        var startTime = new Date();
  228
+        processesRunning++;
  229
+        runCmd(testEnvDetails.pathToPython + " " + cmdargs, { cwd: testPath }, function(err, stdout, stderr) {
  230
+          var report = {
  231
+            browser: browser,
  232
+            test: test,
  233
+            duration: ((new Date() - startTime) / 1000.0),
  234
+            stdout: stdout,
  235
+            stderr: stderr,
  236
+            err: err,
  237
+            passed: !err
  238
+          };
  239
+
  240
+          try {
  241
+            report.htmlReport = fs.readFileSync(htmlReportPath);
  242
+          } catch(e) { }
  243
+
  244
+          // extract key information from the html report and attach it to the report object
  245
+          report.results = parseReport(report.htmlReport);
  246
+
  247
+          self.emit('finished', report);
  248
+
  249
+          testReports.push(report);
  250
+
  251
+          // remove artifacts
  252
+          fs.unlink(htmlReportPath);
  253
+
  254
+          if (--processesRunning === 0) {
  255
+            fs.unlink(sauceYAMLPath);
  256
+            fs.unlink(credentialsYAMLPath);
  257
+            if (cb) cb(null, addSummaryInfo(testReports));
  258
+          }
  259
+        });
  260
+      });
  261
+    });
  262
+  });
  263
+};
  264
+
  265
+module.exports = TestRunner;
  266
+
  267
+// if we're invoked from the command line, do command liney things
  268
+if (process.argv[1] === __filename) {
  269
+  process.on('uncaughtException', function(err) {
  270
+    console.log("OH NOES", err);
  271
+    process.exit(1);
  272
+  });
  273
+
  274
+  var argv = require('optimist')
  275
+    .usage('Run selenium tests via sauce labs.\nUsage: $0')
  276
+    .alias('help', 'h')
  277
+    .describe('help', 'display this usage message')
  278
+    .alias('list-browsers', 'lb')
  279
+    .describe('lb', 'list available browsers to test on')
  280
+    .alias('browser', 'b')
  281
+    .describe('browser', 'specify which browser to run tests on (globs supported)')
  282
+    .alias('list-test-groups', 'lt')
  283
+    .describe('list-test-groups', 'list available groups of tests that can be run')
  284
+    .alias('test-group', 't')
  285
+    .describe('test-group', 'specify which test groups to run (globs supported)')
  286
+    .describe('single', 'run a single test within the specified test group')
  287
+    .alias('single', 's')
  288
+    .default('browser', "linux_firefox_13") // default to a specific browser + test group
  289
+    .default("test-group", "123done");  // lest folks accidentally launch large #s of || jobs
  290
+
  291
+  var args = argv.argv;
  292
+
  293
+  if (args.h) {
  294
+    argv.showHelp();
  295
+    process.exit(0);
  296
+  }
  297
+
  298
+  if (args.lb) {
  299
+    console.log("available browsers:");
  300
+    console.log("   * " + Object.keys(browserSpecificPythonArgs).join("\n   * "));
  301
+    process.exit(0);
  302
+  }
  303
+
  304
+  if (args.lt) {
  305
+    console.log("available tests:");
  306
+    console.log("   * " + Object.keys(testSpecificPythonArgs).join("\n   * "));
  307
+    process.exit(0);
  308
+  }
  309
+
  310
+  // nice file names
  311
+  var startTime = new Date();
  312
+  function testFileName(browser, test) {
  313
+    function pad(n){return n<10 ? '0'+n : n}
  314
+    var d = startTime;
  315
+    var name = "" +
  316
+      d.getFullYear() +
  317
+      "." +  pad(d.getMonth()+1)+'.'
  318
+      + pad(d.getDate())+'-'
  319
+      + pad(d.getHours())+':'
  320
+      + pad(d.getMinutes())+':'
  321
+      + pad(d.getSeconds());
  322
+    name += "-" + browser + "-" + test;
  323
+    return name;
  324
+  }
  325
+
  326
+  var tester = new TestRunner();
  327
+  tester.on('started', function(e) {
  328
+    console.log("STARTED:", e.browser + "/" + e.test);
  329
+  });
  330
+
  331
+  tester.on('finished', function(report) {
  332
+    // save off the report files
  333
+    var basename = testFileName(report.browser, report.test);
  334
+    var resultsPath = path.join(testPath, "results", basename);
  335
+    mkdirp.sync(path.join(testPath, "results"));
  336
+    fs.writeFileSync(resultsPath + ".html", report.htmlReport);
  337
+    if (report.stderr.length) fs.writeFileSync(resultsPath + ".stderr.txt", report.stderr);
  338
+    if (report.stdout.length) fs.writeFileSync(resultsPath + ".stdout.txt", report.stdout);
  339
+
  340
+    var what = report.passed ? "PASSED" : "FAILED";
  341
+    console.log(what + ": " + report.browser + "/" + report.test +
  342
+                " - " + report.duration.toFixed(2) + "s (" +
  343
+               path.relative(process.cwd, resultsPath + ".html") + ")");
  344
+
  345
+    if (!report.passed) {
  346
+      Object.keys(report.results).forEach(function (resultName) {
  347
+        var result = report.results[resultName];
  348
+        if (result.success) return;
  349
+        console.log("   " + result.class + ": " + result.url);
  350
+      });
  351
+      if (report.stderr.length) {
  352
+        console.log("   ERRORS:\n     > " +  report.stderr.split("\n").join("\n     >  "));
  353
+      }
  354
+    }
  355
+  });
  356
+  tester.run(args, function(report) {
  357
+    // We would like to summarize:
  358
+    //   1. total test duration
  359
+    //   2. number of browsers failing
  360
+    //   3. number of tests failing
  361
+    // XXX: after ALL tests complete output a short summary
  362
+//    console.log(JSON.stringify(report, null, "  "));
  363
+  });
  364
+}
5  package.json
@@ -40,7 +40,10 @@
40 40
         "vows": "0.5.13",
41 41
         "awsbox": "0.2.15",
42 42
         "irc": "0.3.3",
43  
-        "jshint": "0.7.1"
  43
+        "jshint": "0.7.1",
  44
+        "minimatch": "0.2.6",
  45
+        "which": "1.0.5",
  46
+        "htmlparser": "1.7.6"
44 47
     },
45 48
     "scripts": {
46 49
         "postinstall": "node ./scripts/generate_ephemeral_keys.js",

0 notes on commit 45bf728

Please sign in to comment.
Something went wrong with that request. Please try again.