content/canvas/test/webgl/test_webgl_conformance_test_suite.html

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
  "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<title>
Mochitest version of the WebGL Conformance Test Suite
</title>
<script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
<link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
<script type="text/javascript" src="resources/webgl-test-harness.js"></script>
<script>

var CONFORMANCE_TEST_VERSION = "1.0.1 (beta)";

var OPTIONS = {
  version: CONFORMANCE_TEST_VERSION
};

/**
 * This is copied from webgl-test-harness.js where it is defined as a private function, not accessible to us (argh!)
 *
 * Loads text from an external file. This function is synchronous.
 * @param {string} url The url of the external file.
 * @return {string} the loaded text if the request is synchronous.
 */
  var loadTextFileSynchronous = function (url) {
    var error = 'loadTextFileSynchronous failed to load url "' + url + '"';
    var request;
    if (window.XMLHttpRequest) {
      request = new XMLHttpRequest();
      if (request.overrideMimeType) {
        request.overrideMimeType('text/plain');
      }
    } else {
      throw 'XMLHttpRequest is disabled';
    }
    request.open('GET', url, false);
    request.send(null);
    if (request.readyState != 4) {
      throw error;
    }
    return request.responseText;
  };

SimpleTest.waitForExplicitFinish();

function detectDriverType() {
  const Cc = SpecialPowers.wrap(Components).classes;
  const Ci = SpecialPowers.wrap(Components).interfaces;
  var doc = Cc["@mozilla.org/xmlextras/domparser;1"].createInstance(Ci.nsIDOMParser).parseFromString("<html/>", "text/html");

  var canvas = doc.createElement("canvas");
  canvas.width = 1;
  canvas.height = 1;

  var type = "";
  var gl;
  try {
    gl = canvas.getContext("experimental-webgl");
  } catch(e) {}

  if (gl) {
    var ext = gl.getExtension("WEBGL_debug_renderer_info");
    // this extension is unconditionally available to chrome. No need to check.
    var webglRenderer = gl.getParameter(ext.UNMASKED_RENDERER_WEBGL);
    var webglVendor = gl.getParameter(ext.UNMASKED_VENDOR_WEBGL);
    if (webglRenderer.contains('llvmpipe')) {
      type = "mesa";
    } else if (webglVendor.contains('NVIDIA')) {
      type = "nvidia";
    }
  }
  return type;
}

function start() {

  var kIsWindows = false;
  var kIsAndroid = false;
  var kIsLinux = false;
  var kIsLinuxNVidia = false;
  var kIsLinuxMesa = false;

  if (navigator.platform.indexOf("Win") == 0)
    kIsWindows = true;
  else if (navigator.appVersion.indexOf("Android") != -1)
    kIsAndroid = true;
  else if (navigator.platform.indexOf("Linux") == 0)  {
    // must be checked after android, as android also has a 'Linux' platform string
    var type = detectDriverType();
    if (type == "mesa") {
      kIsLinuxMesa = true;
    } else if (type == "nvidia") {
      kIsLinuxNVidia = true;
    } else {
      kIsLinux = true;
    }
  }

  // Set kMacVersion to the OS X version for Mac, and 0 otherwise.
  var osxmatch = /Mac OS X (\d+.\d+)/.exec(navigator.userAgent);
  var kMacVersion = osxmatch ? parseFloat(osxmatch[1]) : 0;

  var kIsWindowsVistaOrHigher = false;
  if (kIsWindows) {
    // code borrowed from browser/modules/test/browser_taskbar_preview.js
    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
    var version = Components.classes["@mozilla.org/system-info;1"]
                            .getService(Components.interfaces.nsIPropertyBag2)
                            .getProperty("version");
    kIsWindowsVistaOrHigher = (parseFloat(version) >= 6.0);
  }

  function getEnv(env) {
    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
    var envsvc = Components.classes["@mozilla.org/process/environment;1"].getService(Components.interfaces.nsIEnvironment);
    var val = envsvc.get(env);
    if (val == "")
      return null;
    return val;
  }

  var reportType = WebGLTestHarnessModule.TestHarness.reportType;

  var Page = function(reporter, url) {
    this.reporter = reporter;
    this.url = url;
    this.totalTests = 0;
    this.totalSuccessful = 0;
    this.totalTimeouts = 0;

    var li = reporter.localDoc.createElement('li');
    var div = reporter.localDoc.createElement('div');
    var a = reporter.localDoc.createElement('a');
    a.href = url;
    var node = reporter.localDoc.createTextNode(url);
    a.appendChild(node);
    div.appendChild(a);
    li.setAttribute('class', 'testpage');
    li.appendChild(div);
    var ul = reporter.localDoc.createElement('ul');
    var node = reporter.localDoc.createTextNode('');
    li.appendChild(ul);
    div.appendChild(node);
    this.totalsElem = node;
    this.resultElem = ul;
    this.elem = li;
  };

  /**
   * Indicates whether this test page results are not to be ignored.
   */
  Page.prototype.shouldBeAccountedFor = function() {
    return testsToIgnore.indexOf(this.url) == -1;
  }

  /**
   * Indicates whether all this test page results are expected not to fail,
   * if not ignored.
   */
  Page.prototype.isExpectedToFullyPass = function() {
    return this.shouldBeAccountedFor() &&
           testsExpectedToFail.indexOf(this.url) == -1;
  }

  /**
   * Returns log message with added test page url.
   */
  Page.prototype.logMsg = function(msg) {
    return '[' + this.url + '] ' + msg;
  }

  /**
   * Reports an individual test result of test page.
   */
  Page.prototype.addResult = function(msg, success) {
    ++this.totalTests;
    if (success === undefined) {
      ++this.totalTimeouts;
      var result = "timeout";
      var css = "timeout";
      // only few timeouts are actually caught here --- most are caught in finishPage().
      if (this.isExpectedToFullyPass()) {
        ok(false, this.logMsg('Test timed out'), msg);
      } else {
        todo(false, this.logMsg('Test timed out'), msg);
      }
    } else if (success) {
      ++this.totalSuccessful;
      var result = "success";
      var css = "success";
      if (this.shouldBeAccountedFor()) {
        ok(true, this.logMsg('Test passed'), msg);
      } else {
        todo(false, this.logMsg('Test passed, but is ignored'), msg);
      }
      // Don't report individual success to UI, to keep it light.
      return;
    } else {
      var result = "failed";
      var css = "fail";
      if (this.isExpectedToFullyPass()) {
        ok(false, this.logMsg('Test failed'), msg);
      } else {
        todo(false, this.logMsg('Test failed'), msg);
      }
    }

    var node = this.reporter.localDoc.createTextNode(result + ': ' + msg);
    var li = this.reporter.localDoc.createElement('li');
    li.appendChild(node);
    li.setAttribute('class', css);
    this.resultElem.appendChild(li);
  };

  Page.prototype.startPage = function() {
    this.totalTests = 0;
    this.totalSuccessful = 0;
    this.totalTimeouts = 0;
    // remove previous results.
    while (this.resultElem.hasChildNodes()) {
      this.resultElem.removeChild(this.resultElem.childNodes[0]);
    }
    this.totalsElem.textContent = '';
    return true;
  };

  /**
   * Reports test page result summary.
   */
  Page.prototype.finishPage = function(success) {
    var msg = ' (' + this.totalSuccessful + ' of ' +
              this.totalTests + ' passed)';
    if (success === undefined) {
      var css = 'testpagetimeout';
      msg = '(*timeout*)';
      ++this.totalTests;
      ++this.totalTimeouts;
      // Most timeouts are only caught here --- though a few are (already) caught in addResult().
      if (this.isExpectedToFullyPass()) {
        ok(false, this.logMsg('Timeout in this test page'));
      } else {
        todo(false, this.logMsg('Timeout in this test page'));
      }
    } else if (this.totalSuccessful != this.totalTests) {
      var css = 'testpagefail';
      var totalFailed = this.totalTests - this.totalTimeouts - this.totalSuccessful;
      if (this.isExpectedToFullyPass()) {
        ok(false, this.logMsg("(WebGL test error) " + totalFailed + ' failure(s) and ' + this.totalTimeouts + ' timeout(s)'));
      } else {
        todo(false, this.logMsg("(WebGL test error) " + totalFailed + ' failure(s) and ' + this.totalTimeouts + ' timeout(s)'));
      }
    } else {
      var css = 'testpagesuccess';
      if (this.isExpectedToFullyPass()) {
        ok(true, this.logMsg('All ' + this.totalSuccessful + ' test(s) passed'));
      } else {
        if (this.shouldBeAccountedFor()) {
          todo(true, this.logMsg('Test page expected to fail, but all ' + this.totalSuccessful + ' tests passed'));
        } else {
          todo(false, this.logMsg('All ' + this.totalSuccessful + ' test(s) passed, but test page is ignored'));
        }
      }
    }
    this.elem.setAttribute('class', css);
    this.totalsElem.textContent = msg;
  };

  var Reporter = function() {
    this.localDoc = document;

    this.fullResultsElem = document.getElementById("results-default");

    this.resultElem = document.getElementById("results");
    var node = this.localDoc.createTextNode('');
    this.fullResultsElem.appendChild(node);
    this.fullResultsNode = node;
    this.iframe = document.getElementById("testframe");
    this.currentPageElem = null;
    this.totalPages = 0;
    this.pagesByURL = {};
    this.currentPage = null;
    this.totalTests = 0;
    this.totalSuccessful = 0;
    this.totalTimeouts = 0;
  };

  Reporter.prototype.runTest = function(url) {
    var page = this.pagesByURL[url];
    page.startPage();
    this.currentPage = page;
    this.iframe.src = url;
    return result;
  };

  Reporter.prototype.addPage = function(url) {
    this.currentPage = new Page(this, url, this.resultElem);
    this.resultElem.appendChild(this.currentPage.elem);
    ++this.totalPages;
    this.pagesByURL[url] = this.currentPage;
  };

  Reporter.prototype.startPage = function(url) {
    if (testsToSkip.indexOf(url) != -1) {
      info("[" + url + "] (WebGL mochitest) Skipping test page");
      return false;
    }
    info("[" + url + "] (WebGL mochitest) Starting test page");

    // Calling garbageCollect before each test page fixes intermittent failures with
    // out-of-memory errors, often failing to create a WebGL context.
    // The explanation is that the JS engine keeps unreferenced WebGL contexts around
    // for too long before GCing (bug 617453), so that during this mochitest dozens of unreferenced
    // WebGL contexts can accumulate at a given time.
    SpecialPowers.DOMWindowUtils.garbageCollect();

    var page = this.pagesByURL[url];
    this.currentPage = page;
    statusTextNode.textContent = 'Running URL: ' + url;
    expectedtofailTextNode.textContent = testsExpectedToFail.length +
                                         ' test pages are expected to fail out of ' +
                                         this.totalPages;
    ignoredtestsTextNode.textContent = testsToIgnore.length +
                                         ' test pages have their results ignored';
    return page.startPage();
  };

  Reporter.prototype.displayStats = function() {
    var totalFailed = this.totalTests - this.totalTimeouts - this.totalSuccessful;
    this.fullResultsNode.textContent =
      this.totalSuccessful + ' passed, ' +
      totalFailed + ' failed, ' +
      this.totalTimeouts + ' timed out';
  };

  Reporter.prototype.addResult = function(msg, success) {
    if (this.currentPage != null) {
      this.currentPage.addResult(msg, success);
    }
  };

  Reporter.prototype.finishPage = function(success) {
    if (this.currentPage != null) {
      this.currentPage.finishPage(success); // must call that first, since this is where totalTimeouts is computed
      this.totalTests += this.currentPage.totalTests;
      this.totalSuccessful += this.currentPage.totalSuccessful;
      this.totalTimeouts += this.currentPage.totalTimeouts;
      this.currentPage = null;
      this.displayStats();
    }
  };

  Reporter.prototype.finishedTestSuite = function() {
      statusTextNode.textContent = 'Finished';
      SimpleTest.finish();
  }

  Reporter.prototype.ready = function() {
    statusTextNode.textContent = 'Loaded test lists. Starting tests...';
    window.webglTestHarness.runTests();
  }

  Reporter.prototype.reportFunc = function(type, msg, success) {
    switch (type) {
      case reportType.ADD_PAGE:
        return this.addPage(msg);
      case reportType.READY:
        return this.ready();
      case reportType.START_PAGE:
        return this.startPage(msg);
      case reportType.TEST_RESULT:
        return this.addResult(msg, success);
      case reportType.FINISH_PAGE:
        return this.finishPage(success);
      case reportType.FINISHED_ALL_TESTS:
        this.finishedTestSuite();
        return true;
      default:
        throw 'unhandled';
        break;
    }
  };

  var getURLOptions = function(obj) {
    var s = window.location.href;
    var q = s.indexOf("?");
    var e = s.indexOf("#");
    if (e < 0) {
      e = s.length;
    }
    var query = s.substring(q + 1, e);
    var pairs = query.split("&");
    for (var ii = 0; ii < pairs.length; ++ii) {
      var keyValue = pairs[ii].split("=");
      var key = keyValue[0];
      var value = decodeURIComponent(keyValue[1]);
          obj[key] = value;
    }
  };

  getURLOptions(OPTIONS);

  function runTestSuite() {
    var reporter = new Reporter();

    // try to create a dummy WebGL context, just to catch context creation failures once here,
    // rather than having them result in 100's of failures (one in each test page)
    var ctx;
    try {
      ctx = document.getElementById("webglcheck-default")
                    .getContext("experimental-webgl");
    } catch(e) {}
    if (!ctx) {
        var errmsg = "Can't create a WebGL context";
        reporter.fullResultsNode.textContent = errmsg;
        // Workaround for SeaMonkey tinderboxes which don't support WebGL.
        if (navigator.userAgent.match(/ SeaMonkey\//))
          todo(false, errmsg + " (This is expected on SeaMonkey (tinderboxes).)");
        else
          ok(false, errmsg);
        reporter.finishedTestSuite();
        return;
    }

    statusTextNode.textContent = 'Loading test lists...';
    var iframe = document.getElementById("testframe");
    var testHarness = new WebGLTestHarnessModule.TestHarness(
        iframe,
        '00_test_list.txt',
        function(type, msg, success) {
          return reporter.reportFunc(type, msg, success);
        },
        OPTIONS);
    // Make timeout delay much higher when running under valgrind.
    testHarness.setTimeoutDelay(20000);
    window.webglTestHarness = testHarness;
  }

  SimpleTest.requestLongerTimeout(kIsAndroid ? 6 : 3);

  var statusElem = document.getElementById("status");
  var statusTextNode = document.createTextNode('');
  statusElem.appendChild(statusTextNode);

  var expectedtofailElem = document.getElementById("expectedtofail");
  var expectedtofailTextNode = document.createTextNode('');
  expectedtofailElem.appendChild(expectedtofailTextNode);

  var ignoredtestsElem = document.getElementById("ignoredtests");
  var ignoredtestsTextNode = document.createTextNode('');
  ignoredtestsElem.appendChild(ignoredtestsTextNode);

  // Windows uses the ANGLE library for rendering. Until everything is perfect, this means a different set of
  // failing tests. It's easier to do a platform check for Windows than for ANGLE itself.
  // Moreover, we currently also have different tests failing on Mac and on Linux,
  // presumably due to differences in the drivers.
  var failingTestsFilename;
  if (kIsWindows)
    failingTestsFilename = 'failing_tests_windows.txt';
  else if (kIsLinux)
    failingTestsFilename = 'failing_tests_linux.txt';
  else if (kIsLinuxMesa)
    failingTestsFilename = 'failing_tests_linux_mesa.txt';
  else if (kIsLinuxNVidia)
    failingTestsFilename = 'failing_tests_linux_nvidia.txt';
  else if (kMacVersion == 10.8)
    failingTestsFilename = 'failing_tests_mac_mtnlion.txt';
  else if (kMacVersion)
    failingTestsFilename = 'failing_tests_mac.txt';
  else if (kIsAndroid)
    failingTestsFilename = 'failing_tests_android.txt';

  var testsExpectedToFail = loadTextFileSynchronous(failingTestsFilename)
                            .replace(/\r/g, '') // convert to unix line breaks
                            .split('\n');

  // remove empty filenames
  var indexOfEmptyFilename = testsExpectedToFail.indexOf("");
  while(indexOfEmptyFilename != -1) {
    testsExpectedToFail.splice(indexOfEmptyFilename, 1);
    indexOfEmptyFilename = testsExpectedToFail.indexOf("");
  }

  var testsToIgnore = [];

  var testsToSkip = [];

  if (kIsAndroid) {
    var testsToSkip = loadTextFileSynchronous('skipped_tests_android.txt')
                      .replace(/\r/g, '') // convert to unix line breaks
                      .split('\n');
  }

  if (kIsLinuxMesa) {
    var testsToSkip = loadTextFileSynchronous('skipped_tests_linux_mesa.txt')
                      .replace(/\r/g, '') // convert to unix line breaks
                      .split('\n');
  }

  runTestSuite();
}

</script>
</head>
<body onload="start();">
<p id="display"></p>
<div id="content" style="display: none">

</div>
<table border="2px">
  <tr style="height: 500px;">
    <td style="width: 500px;">
      <iframe id="testframe" scrolling="no" width="500px" height="500px"></iframe>
    </td>
    <td>
      <table>
        <tr>
          <td><h4>WebGL Conformance Test Runner</h4></td>
        </tr>
        <tr>
          <td>
            <div style="border: 1px">
              <b>Status:</b> <div><span id="status"></span></div><br />
              <b>Results:</b>
              <div><span id="results-default"></span></div>
              <br />
              <div><span id="expectedtofail"></span></div>
              <br />
              <div><span id="ignoredtests"></span></div>
            </div>
          </td>
        </tr>
      </table>
    </td>
  </tr>
  <tr>
    <td colspan="2">
      <div style="text-align: left; width: 100%; height: 100%; overflow: auto;">
        <div><ul id="results"></ul></div>
      </div>
    </td>
  </tr>
</table>
<canvas id="webglcheck-default" style="display: none;"></canvas>
</body>
</html>