content/canvas/test/webgl/test_webgl_conformance_test_suite.html

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
  "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<title>
Mochitest version of the WebGL Conformance Test Suite
</title>
<script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
<link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
<script type="text/javascript" src="resources/webgl-test-harness.js"></script>
<script>

var CONFORMANCE_TEST_VERSION = "1.0.1 (beta)";

var OPTIONS = {
  version: CONFORMANCE_TEST_VERSION
};

/**
 * This is copied from webgl-test-harness.js where it is defined as a private function, not accessible to us (argh!)
 *
 * Loads text from an external file. This function is synchronous.
 * @param {string} url The url of the external file.
 * @return {string} the loaded text if the request is synchronous.
 */
  var loadTextFileSynchronous = function (url) {
    var error = 'loadTextFileSynchronous failed to load url "' + url + '"';
    var request;
    if (window.XMLHttpRequest) {
      request = new XMLHttpRequest();
      if (request.overrideMimeType) {
        request.overrideMimeType('text/plain');
      }
    } else {
      throw 'XMLHttpRequest is disabled';
    }
    request.open('GET', url, false);
    request.send(null);
    if (request.readyState != 4) {
      throw error;
    }
    if (request.status >= 400) {
      // Error response, probably a 404.
      throw 'Error: request.status: ' + request.status;
    }
    return request.responseText;
  };

SimpleTest.waitForExplicitFinish();

function detectDriverInfo() {
  const Cc = SpecialPowers.Cc;
  const Ci = SpecialPowers.Ci;
  var doc = Cc["@mozilla.org/xmlextras/domparser;1"].createInstance(Ci.nsIDOMParser).parseFromString("<html/>", "text/html");

  var canvas = doc.createElement("canvas");
  canvas.width = 1;
  canvas.height = 1;

  var type = "";
  var gl;
  try {
    gl = canvas.getContext("experimental-webgl");
  } catch(e) {
    ok(false, "Failed to create a WebGL context for getting driver info.");
    return ["", ""]
  }
  var ext = gl.getExtension("WEBGL_debug_renderer_info");
  // this extension is unconditionally available to chrome. No need to check.
  
  var webglRenderer = gl.getParameter(ext.UNMASKED_RENDERER_WEBGL);
  var webglVendor = gl.getParameter(ext.UNMASKED_VENDOR_WEBGL);
  return [webglVendor, webglRenderer];
}

function start() {
  var OS_WINDOWS = 'windows';
  var OS_MAC     = 'mac';
  var OS_LINUX   = 'linux';
  var OS_ANDROID = 'android';
  
  var GLDRIVER_MESA   = 'mesa';
  var GLDRIVER_NVIDIA = 'nvidia';
  var GLDRIVER_X86EMULATOR = 'android x86 emulator';

  var kOS = null;
  var kOSVersion = null;
  var kGLDriver = null;

  if (navigator.platform.indexOf('Win') == 0) {
    kOS = OS_WINDOWS;
    
    // code borrowed from browser/modules/test/browser_taskbar_preview.js
    var version = SpecialPowers.Services.sysinfo.getProperty('version');
    kOSVersion = parseFloat(version);
    // Version 6.0 is Vista, 6.1 is 7.
  } else if (navigator.platform.indexOf('Mac') == 0) {
    kOS = OS_MAC;
    
    var versionMatch = /Mac OS X (\d+.\d+)/.exec(navigator.userAgent);
    kOSVersion = versionMatch ? parseFloat(versionMatch[1]) : null;
  } else if (navigator.appVersion.indexOf('Android') != -1) {
    kOS = OS_ANDROID;
  } else if (navigator.platform.indexOf('Linux') == 0) {
    // Must be checked after android, as android also has a 'Linux' platform string.
    kOS = OS_LINUX;
  }
  
  var glVendor, glRenderer;
  [glVendor, glRenderer] = detectDriverInfo();
  info('GL vendor: ' + glVendor);
  info('GL renderer: ' + glRenderer);
  
  if (glRenderer.contains('llvmpipe')) {
    kGLDriver = GLDRIVER_MESA;
  } else if (glRenderer.contains('Android Emulator')) {
    kGLDriver = GLDRIVER_X86EMULATOR;
  } else if (glVendor.contains('NVIDIA')) {
    kGLDriver = GLDRIVER_NVIDIA;
  }
  
  if (kOS) {
    info('OS detected as: ' + kOS);
    info('  Version: ' + kOSVersion);
  } else {
    info('OS not detected.');
    info('  `platform`:   ' + navigator.platform);
    info('  `appVersion`: ' + navigator.appVersion);
    info('  `userAgent`:  ' + navigator.userAgent);
  }
  if (kGLDriver) {
    info('GL driver detected as: ' + kGLDriver);
  } else {
    info('GL driver not detected.');
  }
  
  var requestLongerTimeoutLen = 3;
  if (kOS == OS_ANDROID)
    requestLongerTimeoutLen = 6;

  function getEnv(env) {
    var envsvc = SpecialPowers.Cc["@mozilla.org/process/environment;1"].getService(SpecialPowers.Ci.nsIEnvironment);
    var val = envsvc.get(env);
    if (val == "")
      return null;
    return val;
  }

  var reportType = WebGLTestHarnessModule.TestHarness.reportType;

  var Page = function(reporter, url) {
    this.reporter = reporter;
    this.url = url;
    this.totalTests = 0;
    this.totalSuccessful = 0;
    this.totalTimeouts = 0;

    var li = reporter.localDoc.createElement('li');
    var div = reporter.localDoc.createElement('div');
    var a = reporter.localDoc.createElement('a');
    a.href = url;
    var node = reporter.localDoc.createTextNode(url);
    a.appendChild(node);
    div.appendChild(a);
    li.setAttribute('class', 'testpage');
    li.appendChild(div);
    var ul = reporter.localDoc.createElement('ul');
    var node = reporter.localDoc.createTextNode('');
    li.appendChild(ul);
    div.appendChild(node);
    this.totalsElem = node;
    this.resultElem = ul;
    this.elem = li;
  };

  /**
   * Indicates whether this test page results are not to be ignored.
   */
  Page.prototype.shouldBeAccountedFor = function() {
    return testsToIgnore.indexOf(this.url) == -1;
  }

  /**
   * Indicates whether all this test page results are expected not to fail,
   * if not ignored.
   */
  Page.prototype.isExpectedToFullyPass = function() {
    return this.shouldBeAccountedFor() &&
           testsExpectedToFail.indexOf(this.url) == -1;
  }

  /**
   * Returns log message with added test page url.
   */
  Page.prototype.logMsg = function(msg) {
    return '[' + this.url + '] ' + msg;
  }

  /**
   * Reports an individual test result of test page.
   */
  Page.prototype.addResult = function(msg, success) {
    ++this.totalTests;
    if (success === undefined) {
      ++this.totalTimeouts;
      var result = "timeout";
      var css = "timeout";
      // only few timeouts are actually caught here --- most are caught in finishPage().
      if (this.isExpectedToFullyPass()) {
        ok(false, this.logMsg('Test timed out'), msg);
      } else {
        todo(false, this.logMsg('Test timed out'), msg);
      }
    } else if (success) {
      ++this.totalSuccessful;
      var result = "success";
      var css = "success";
      if (this.shouldBeAccountedFor()) {
        ok(true, this.logMsg('Test passed'), msg);
      } else {
        todo(false, this.logMsg('Test passed, but is ignored'), msg);
      }
      // Don't report individual success to UI, to keep it light.
      return;
    } else {
      var result = "failed";
      var css = "fail";
      if (this.isExpectedToFullyPass()) {
        ok(false, this.logMsg('Test failed'), msg);
      } else {
        todo(false, this.logMsg('Test failed'), msg);
      }
    }

    var node = this.reporter.localDoc.createTextNode(result + ': ' + msg);
    var li = this.reporter.localDoc.createElement('li');
    li.appendChild(node);
    li.setAttribute('class', css);
    this.resultElem.appendChild(li);
  };

  Page.prototype.startPage = function() {
    this.totalTests = 0;
    this.totalSuccessful = 0;
    this.totalTimeouts = 0;
    // remove previous results.
    while (this.resultElem.hasChildNodes()) {
      this.resultElem.removeChild(this.resultElem.childNodes[0]);
    }
    this.totalsElem.textContent = '';
    return true;
  };

  /**
   * Reports test page result summary.
   */
  Page.prototype.finishPage = function(success) {
    var msg = ' (' + this.totalSuccessful + ' of ' +
              this.totalTests + ' passed)';
    if (success === undefined) {
      var css = 'testpagetimeout';
      msg = '(*timeout*)';
      ++this.totalTests;
      ++this.totalTimeouts;
      // Most timeouts are only caught here --- though a few are (already) caught in addResult().
      if (this.isExpectedToFullyPass()) {
        ok(false, this.logMsg('Timeout in this test page'));
      } else {
        todo(false, this.logMsg('Timeout in this test page'));
      }
    } else if (this.totalSuccessful != this.totalTests) {
      var css = 'testpagefail';
      var totalFailed = this.totalTests - this.totalTimeouts - this.totalSuccessful;
      if (this.isExpectedToFullyPass()) {
        ok(false, this.logMsg("(WebGL test error) " + totalFailed + ' failure(s) and ' + this.totalTimeouts + ' timeout(s)'));
      } else {
        todo(false, this.logMsg("(WebGL test error) " + totalFailed + ' failure(s) and ' + this.totalTimeouts + ' timeout(s)'));
      }
    } else {
      var css = 'testpagesuccess';
      if (this.isExpectedToFullyPass()) {
        ok(true, this.logMsg('All ' + this.totalSuccessful + ' test(s) passed'));
      } else {
        if (this.shouldBeAccountedFor()) {
          todo(true, this.logMsg('Test page expected to fail, but all ' + this.totalSuccessful + ' tests passed'));
        } else {
          todo(false, this.logMsg('All ' + this.totalSuccessful + ' test(s) passed, but test page is ignored'));
        }
      }
    }
    this.elem.setAttribute('class', css);
    this.totalsElem.textContent = msg;
  };

  var Reporter = function() {
    this.localDoc = document;

    this.fullResultsElem = document.getElementById("results-default");

    this.resultElem = document.getElementById("results");
    var node = this.localDoc.createTextNode('');
    this.fullResultsElem.appendChild(node);
    this.fullResultsNode = node;
    this.iframe = document.getElementById("testframe");
    this.currentPageElem = null;
    this.totalPages = 0;
    this.pagesByURL = {};
    this.currentPage = null;
    this.totalTests = 0;
    this.totalSuccessful = 0;
    this.totalTimeouts = 0;
  };

  Reporter.prototype.runTest = function(url) {
    var page = this.pagesByURL[url];
    page.startPage();
    this.currentPage = page;
    this.iframe.src = url;
    return result;
  };

  Reporter.prototype.addPage = function(url) {
    this.currentPage = new Page(this, url, this.resultElem);
    this.resultElem.appendChild(this.currentPage.elem);
    ++this.totalPages;
    this.pagesByURL[url] = this.currentPage;
  };

  Reporter.prototype.startPage = function(url) {
    if (testsToSkip.indexOf(url) != -1) {
      info("[" + url + "] (WebGL mochitest) Skipping test page");
      return false;
    }
    info("[" + url + "] (WebGL mochitest) Starting test page");

    // Calling garbageCollect before each test page fixes intermittent failures with
    // out-of-memory errors, often failing to create a WebGL context.
    // The explanation is that the JS engine keeps unreferenced WebGL contexts around
    // for too long before GCing (bug 617453), so that during this mochitest dozens of unreferenced
    // WebGL contexts can accumulate at a given time.
    SpecialPowers.DOMWindowUtils.cycleCollect();
    SpecialPowers.DOMWindowUtils.garbageCollect();
    SpecialPowers.DOMWindowUtils.garbageCollect();

    var page = this.pagesByURL[url];
    this.currentPage = page;
    statusTextNode.textContent = 'Running URL: ' + url;
    expectedtofailTextNode.textContent = testsExpectedToFail.length +
                                         ' test pages are expected to fail out of ' +
                                         this.totalPages;
    ignoredtestsTextNode.textContent = testsToIgnore.length +
                                         ' test pages have their results ignored';
    return page.startPage();
  };

  Reporter.prototype.displayStats = function() {
    var totalFailed = this.totalTests - this.totalTimeouts - this.totalSuccessful;
    this.fullResultsNode.textContent =
      this.totalSuccessful + ' passed, ' +
      totalFailed + ' failed, ' +
      this.totalTimeouts + ' timed out';
  };

  Reporter.prototype.addResult = function(msg, success) {
    if (this.currentPage != null) {
      this.currentPage.addResult(msg, success);
    }
  };

  Reporter.prototype.finishPage = function(success) {
    if (this.currentPage != null) {
      this.currentPage.finishPage(success); // must call that first, since this is where totalTimeouts is computed
      this.totalTests += this.currentPage.totalTests;
      this.totalSuccessful += this.currentPage.totalSuccessful;
      this.totalTimeouts += this.currentPage.totalTimeouts;
      this.currentPage = null;
      this.displayStats();
    }
  };

  Reporter.prototype.finishedTestSuite = function() {
      statusTextNode.textContent = 'Finished';
      SimpleTest.finish();
  }

  Reporter.prototype.ready = function() {
    statusTextNode.textContent = 'Loaded test lists. Starting tests...';
    window.webglTestHarness.runTests();
  }

  Reporter.prototype.reportFunc = function(type, msg, success) {
    switch (type) {
      case reportType.ADD_PAGE:
        return this.addPage(msg);
      case reportType.READY:
        return this.ready();
      case reportType.START_PAGE:
        return this.startPage(msg);
      case reportType.TEST_RESULT:
        return this.addResult(msg, success);
      case reportType.FINISH_PAGE:
        return this.finishPage(success);
      case reportType.FINISHED_ALL_TESTS:
        this.finishedTestSuite();
        return true;
      default:
        throw 'unhandled';
        break;
    }
  };

  var getURLOptions = function(obj) {
    var s = window.location.href;
    var q = s.indexOf("?");
    var e = s.indexOf("#");
    if (e < 0) {
      e = s.length;
    }
    var query = s.substring(q + 1, e);
    var pairs = query.split("&");
    for (var ii = 0; ii < pairs.length; ++ii) {
      var keyValue = pairs[ii].split("=");
      var key = keyValue[0];
      var value = decodeURIComponent(keyValue[1]);
          obj[key] = value;
    }
  };

  getURLOptions(OPTIONS);

  function runTestSuite() {
    var reporter = new Reporter();

    // try to create a dummy WebGL context, just to catch context creation failures once here,
    // rather than having them result in 100's of failures (one in each test page)
    var ctx;
    try {
      ctx = document.getElementById("webglcheck-default")
                    .getContext("experimental-webgl");
    } catch(e) {}
    if (!ctx) {
        var errmsg = "Can't create a WebGL context";
        reporter.fullResultsNode.textContent = errmsg;
        // Workaround for SeaMonkey tinderboxes which don't support WebGL.
        if (navigator.userAgent.match(/ SeaMonkey\//))
          todo(false, errmsg + " (This is expected on SeaMonkey (tinderboxes).)");
        else if (SpecialPowers.getBoolPref("webgl.disabled"))
          todo(false, errmsg + " (This is expected on when WebGL is disabled)");
        else
          ok(false, errmsg);
        reporter.finishedTestSuite();
        return;
    }

    statusTextNode.textContent = 'Loading test lists...';
    var iframe = document.getElementById("testframe");
    var testHarness = new WebGLTestHarnessModule.TestHarness(
        iframe,
        '00_test_list.txt',
        function(type, msg, success) {
          return reporter.reportFunc(type, msg, success);
        },
        OPTIONS);
    // Make timeout delay much higher when running under valgrind.
    testHarness.setTimeoutDelay(20000);
    window.webglTestHarness = testHarness;
  }

  SimpleTest.requestLongerTimeout(requestLongerTimeoutLen);

  var statusElem = document.getElementById("status");
  var statusTextNode = document.createTextNode('');
  statusElem.appendChild(statusTextNode);

  var expectedtofailElem = document.getElementById("expectedtofail");
  var expectedtofailTextNode = document.createTextNode('');
  expectedtofailElem.appendChild(expectedtofailTextNode);

  var ignoredtestsElem = document.getElementById("ignoredtests");
  var ignoredtestsTextNode = document.createTextNode('');
  ignoredtestsElem.appendChild(ignoredtestsTextNode);

  // Windows uses the ANGLE library for rendering. Until everything is perfect, this means a different set of
  // failing tests. It's easier to do a platform check for Windows than for ANGLE itself.
  // Moreover, we currently also have different tests failing on Mac and on Linux,
  // presumably due to differences in the drivers.
  var failingTestsFilename = null;
  var skippedTestsFilename = null;
  switch (kOS) {
    case OS_WINDOWS: {
      failingTestsFilename = 'failing_tests_windows.txt';
      
      if (kOSVersion >= 6.0) // 6.0 is Vista
        skippedTestsFilename = 'skipped_tests_win_vista.txt'
      else // XP
        skippedTestsFilename = 'skipped_tests_winxp.txt';
        
      break;
    }
    case OS_MAC: {
      if (kOSVersion == 10.8)
        failingTestsFilename = 'failing_tests_mac_mtnlion.txt';
      else
        failingTestsFilename = 'failing_tests_mac.txt';
        
      break;
    }
    case OS_LINUX: {
      switch (kGLDriver) {
        case GLDRIVER_MESA:
          failingTestsFilename = 'failing_tests_linux_mesa.txt';
          skippedTestsFilename = 'skipped_tests_linux_mesa.txt';
          break;
        case GLDRIVER_NVIDIA:
          failingTestsFilename = 'failing_tests_linux_nvidia.txt';
          break;
        default:
          failingTestsFilename = 'failing_tests_linux.txt';
          skippedTestsFilename = 'skipped_tests_linux.txt';
          break;
      }
      break;
    }
    case OS_ANDROID: {
      switch (kGLDriver) {
        case GLDRIVER_NVIDIA:
          failingTestsFilename = 'failing_tests_android_nvidia.txt';
          skippedTestsFilename = 'skipped_tests_android.txt';
          break;
        case GLDRIVER_X86EMULATOR:
          failingTestsFilename = 'failing_tests_android_x86.txt';
          skippedTestsFilename = 'skipped_tests_android_x86.txt';
          break;
        default:
          failingTestsFilename = 'failing_tests_android.txt';
          skippedTestsFilename = 'skipped_tests_android.txt';
          break;
      }
      break;
    }
  }
  
  info('Failing tests file: ' + failingTestsFilename);
  info('Skipped tests file: ' + skippedTestsFilename);
  
  function LoadNewlineSepFile(filename) {
    var lines;
    try {
      lines = loadTextFileSynchronous(filename)
              .replace(/\r/g, '') // convert to unix line breaks
              .split('\n');
    }
    catch(e) {
      // Request failed for some reason.
      ok(false, 'Loading \'' + filename + '\' failed: ' + e);
      return [];
    }
    
    // Remove comments and trim whitespace.
    var retLines = [];
    for (var i = 0; i < lines.length; i++) {
      var line = lines[i];
      line = line.split('#', 1)[0].trim();
      if (!line.length)
        continue;
      
      retLines.push(line);
    }
    
    return retLines;
  };

  var testsExpectedToFail = [];
  if (failingTestsFilename)
    testsExpectedToFail = LoadNewlineSepFile(failingTestsFilename);
  
  var testsToSkip = [];
  if (skippedTestsFilename)
    testsToSkip = LoadNewlineSepFile(skippedTestsFilename);
    
  var testsToIgnore = [];
  
  info('Tests to fail: ' + testsExpectedToFail.length + (testsExpectedToFail.length ? ':' : ''));
  for (var i = 0; i < testsExpectedToFail.length; i++) {
    var test = testsExpectedToFail[i];
    info('  ' + test);
  }
  info('Tests to skip: ' + testsToSkip.length + (testsToSkip.length ? ':' : ''));
  for (var i = 0; i < testsToSkip.length; i++) {
    var test = testsToSkip[i];
    info('  ' + test);
  }
  info('Tests to ignore: ' + testsToIgnore.length + (testsToIgnore.length ? ':' : ''));
  for (var i = 0; i < testsToIgnore.length; i++) {
    var test = testsToIgnore[i];
    info('  ' + test);
  }

  runTestSuite();
}

</script>
</head>
<body onload="start();">
<p id="display"></p>
<div id="content" style="display: none">

</div>
<table border="2px">
  <tr style="height: 500px;">
    <td style="width: 500px;">
      <iframe id="testframe" scrolling="no" width="500px" height="500px"></iframe>
    </td>
    <td>
      <table>
        <tr>
          <td><h4>WebGL Conformance Test Runner</h4></td>
        </tr>
        <tr>
          <td>
            <div style="border: 1px">
              <b>Status:</b> <div><span id="status"></span></div><br />
              <b>Results:</b>
              <div><span id="results-default"></span></div>
              <br />
              <div><span id="expectedtofail"></span></div>
              <br />
              <div><span id="ignoredtests"></span></div>
            </div>
          </td>
        </tr>
      </table>
    </td>
  </tr>
  <tr>
    <td colspan="2">
      <div style="text-align: left; width: 100%; height: 100%; overflow: auto;">
        <div><ul id="results"></ul></div>
      </div>
    </td>
  </tr>
</table>
<canvas id="webglcheck-default" style="display: none;"></canvas>
</body>
</html>