Fetch: Content-Type parsing

See whatwg/fetch#831 for context.

fetch/content-type/README.md

@@ -0,0 +1,20 @@
+# `resources/content-types.json`
+
+An array of tests. Each test has these fields:
+
+* `contentType`: an array of values for the `Content-Type` header. A harness needs to run the test twice if there are multiple values. One time with the values concatenated with `,` followed by a space and one time with multiple `Content-Type` declarations, each on their own line with one of the values, in order.
+* `encoding`: the expected encoding, null for the default.
+* `mimeType`: the result of extracing a MIME type and serializing it.
+* `documentContentType`: the MIME type expected to be exposed in DOM documents.
+
+(These tests are currently somewhat geared towards browser use, but could be generalized easily enough if someone wanted to contribute tests for MIME types that would cause downloads in the browser or some such.)
+
+# `resources/script-content-types.json`
+
+An array of tests, surprise. Each test has these fields:
+
+* `contentType`: see above.
+* `executes`: whether the script is expected to execute.
+* `encoding`: how the script is expected to be decoded.
+
+These tests are expected to be loaded through `<script src>` and the server is expected to set `X-Content-Type-Options: nosniff`.

fetch/content-type/resources/content-type.py

@@ -0,0 +1,15 @@
+def main(request, response):
+    values = request.GET.get_list("value")
+    content = request.GET.first("content", "<b>hi</b>\n")
+    output =  "HTTP/1.1 200 OK\r\n"
+    output += "X-Content-Type-Options: nosniff\r\n"
+    if "single_header" in request.GET:
+        output += "Content-Type: " + ",".join(values) + "\r\n"
+    else:
+        for value in values:
+            output += "Content-Type: " + value + "\r\n"
+    output += "Content-Length: " + str(len(content)) + "\r\n"
+    output += "\r\n"
+    output += content
+    response.writer.write(output)
+    response.close_connection = True

fetch/content-type/resources/content-types.json

@@ -0,0 +1,122 @@
+[
+  {
+    "contentType": ["", "text/plain"],
+    "encoding": null,
+    "mimeType": "text/plain",
+    "documentContentType": "text/plain"
+  },
+  {
+    "contentType": ["text/plain", ""],
+    "encoding": null,
+    "mimeType": "text/plain",
+    "documentContentType": "text/plain"
+  },
+  {
+    "contentType": ["text/html", "text/plain"],
+    "encoding": null,
+    "mimeType": "text/plain",
+    "documentContentType": "text/plain"
+  },
+  {
+    "contentType": ["text/plain;charset=gbk", "text/html"],
+    "encoding": null,
+    "mimeType": "text/html",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["text/plain;charset=gbk", "text/html;charset=windows-1254"],
+    "encoding": "windows-1254",
+    "mimeType": "text/html;charset=windows-1254",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["text/plain;charset=gbk", "text/plain"],
+    "encoding": "GBK",
+    "mimeType": "text/plain;charset=gbk",
+    "documentContentType": "text/plain"
+  },
+  {
+    "contentType": ["text/plain;charset=gbk", "text/plain;charset=windows-1252"],
+    "encoding": "windows-1252",
+    "mimeType": "text/plain;charset=windows-1252",
+    "documentContentType": "text/plain"
+  },
+  {
+    "contentType": ["text/html;charset=gbk", "text/html;x=\",text/plain"],
+    "encoding": "GBK",
+    "mimeType": "text/html;x=\",text/plain\";charset=gbk",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["text/plain;charset=gbk;x=foo", "text/plain"],
+    "encoding": "GBK",
+    "mimeType": "text/plain;charset=gbk",
+    "documentContentType": "text/plain"
+  },
+  {
+    "contentType": ["text/html;charset=gbk", "text/plain", "text/html"],
+    "encoding": null,
+    "mimeType": "text/html",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["text/plain", "*/*"],
+    "encoding": null,
+    "mimeType": "text/plain",
+    "documentContentType": "text/plain"
+  },
+  {
+    "contentType": ["text/html", "*/*"],
+    "encoding": null,
+    "mimeType": "text/html",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["*/*", "text/html"],
+    "encoding": null,
+    "mimeType": "text/html",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["text/plain", "*/*;charset=gbk"],
+    "encoding": null,
+    "mimeType": "text/plain",
+    "documentContentType": "text/plain"
+  },
+  {
+    "contentType": ["text/html", "*/*;charset=gbk"],
+    "encoding": null,
+    "mimeType": "text/html",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["text/html;x=\"", "text/plain"],
+    "encoding": null,
+    "mimeType": "text/html;x=\", text/plain\"",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["text/html;\"", "text/plain"],
+    "encoding": null,
+    "mimeType": "text/html",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["text/html;\"", "\\\"", "text/plain"],
+    "encoding": null,
+    "mimeType": "text/html",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["text/html;\"", "\\\"", "text/plain", "\";charset=GBK"],
+    "encoding": "GBK",
+    "mimeType": "text/html;charset=GBK",
+    "documentContentType": "text/html"
+  },
+  {
+    "contentType": ["text/html;\"",  "\"", "text/plain"],
+    "encoding": null,
+    "mimeType": "text/plain",
+    "documentContentType": "text/plain"
+  }
+]

fetch/content-type/resources/script-content-types.json

@@ -0,0 +1,92 @@
+[
+  {
+    "contentType": ["text/javascript;charset=windows-1252"],
+    "executes": true,
+    "encoding": "windows-1252"
+  },
+  {
+    "contentType": ["text/javascript;\";charset=windows-1252"],
+    "executes": true,
+    "encoding": "windows-1252"
+  },
+  {
+    "contentType": ["text/javascript\u000C"],
+    "executes": false,
+    "encoding": null
+  },
+  {
+    "contentType": ["\"text/javascript\""],
+    "executes": false,
+    "encoding": null
+  },
+  {
+    "contentType": ["text/ javascript"],
+    "executes": false,
+    "encoding": null
+  },
+  {
+    "contentType": ["text /javascript"],
+    "executes": false,
+    "encoding": null
+  },
+  {
+    "contentType": ["x/x", "text/javascript"],
+    "executes": true,
+    "encoding": null
+  },
+  {
+    "contentType": ["x/x;charset=windows-1252", "text/javascript"],
+    "executes": true,
+    "encoding": null
+  },
+  {
+    "contentType": ["text/javascript", "x/x"],
+    "executes": false,
+    "encoding": null
+  },
+  {
+    "contentType": ["text/javascript; charset=windows-1252", "text/javascript"],
+    "executes": true,
+    "encoding": "windows-1252"
+  },
+  {
+    "contentType": ["text/javascript;\"", "x/x"],
+    "executes": true,
+    "encoding": null
+  },
+  {
+    "contentType": ["text/javascript", ""],
+    "executes": true,
+    "encoding": null
+  },
+  {
+    "contentType": ["text/javascript", "error"],
+    "executes": true,
+    "encoding": null
+  },
+  {
+    "contentType": ["text/javascript;charset=windows-1252", "x/x", "text/javascript"],
+    "executes": true,
+    "encoding": null
+  },
+  {
+    "contentType": ["text/javascript;charset=windows-1252", "error", "text/javascript"],
+    "executes": true,
+    "encoding": "windows-1252"
+  },
+  {
+    "contentType": ["text/javascript;charset=windows-1252", "", "text/javascript"],
+    "executes": true,
+    "encoding": "windows-1252"
+  },
+  {
+    "contentType": ["text/javascript;charset=windows-1252;\"", "\\\"", "x/x"],
+    "executes": true,
+    "encoding": "windows-1252"
+  },
+  {
+    "contentType": ["x/x;\"", "x/y;\\\"", "text/javascript;charset=windows-1252;\"", "text/javascript"],
+    "executes": true,
+    "encoding": null
+  }
+]

fetch/content-type/response.window.js

@@ -0,0 +1,72 @@
+promise_test(() => {
+  return fetch("resources/content-types.json").then(res => res.json()).then(runTests);
+}, "Loading JSON…");
+
+function runTests(tests) {
+  tests.forEach(testUnit => {
+    runFrameTest(testUnit, false);
+    runFrameTest(testUnit, true);
+    runFetchTest(testUnit, false);
+    runFetchTest(testUnit, true);
+    runRequestResponseTest(testUnit, "Request");
+    runRequestResponseTest(testUnit, "Response");
+  });
+}
+
+function runFrameTest(testUnit, singleHeader) {
+  // Note: window.js is always UTF-8
+  const encoding = testUnit.encoding !== null ? testUnit.encoding : "UTF-8";
+  async_test(t => {
+    const frame = document.body.appendChild(document.createElement("iframe"));
+    t.add_cleanup(() => frame.remove());
+    frame.src = getURL(testUnit.contentType, singleHeader);
+    frame.onload = t.step_func_done(() => {
+      // Edge requires toUpperCase()
+      const doc = frame.contentDocument;
+      assert_equals(doc.characterSet.toUpperCase(), encoding.toUpperCase());
+      if (testUnit.documentContentType === "text/plain") {
+        assert_equals(doc.body.textContent, "<b>hi</b>\n");
+      } else if (testUnit.documentContentType === "text/html") {
+        assert_equals(doc.body.firstChild.localName, "b");
+        assert_equals(doc.body.firstChild.textContent, "hi");
+      }
+      assert_equals(doc.contentType, testUnit.documentContentType);
+    });
+  }, getDesc("<iframe>", testUnit.contentType, singleHeader));
+}
+
+function getDesc(type, input, singleHeader) {
+  return type + ": " + (singleHeader ? "combined" : "separate") + " response Content-Type: " + input.join(" ");
+}
+
+function getURL(input, singleHeader) {
+  // Edge does not support URLSearchParams
+  let url = "resources/content-type.py?"
+  if (singleHeader) {
+    url += "single_header&"
+  }
+  input.forEach(val => {
+    url += "value=" + encodeURIComponent(val) + "&";
+  });
+  return url;
+}
+
+function runFetchTest(testUnit, singleHeader) {
+  promise_test(async t => {
+    const blob = await (await fetch(getURL(testUnit.contentType, singleHeader))).blob();
+    assert_equals(blob.type, testUnit.mimeType);
+  }, getDesc("fetch()", testUnit.contentType, singleHeader));
+}
+
+function runRequestResponseTest(testUnit, stringConstructor) {
+  promise_test(async t => {
+    // Cannot give Response a body as that will set Content-Type, but Request needs a URL
+    const constructorArgument = stringConstructor === "Request" ? "about:blank" : undefined;
+    const r = new self[stringConstructor](constructorArgument);
+    testUnit.contentType.forEach(val => {
+      r.headers.append("Content-Type", val);
+    });
+    const blob = await r.blob();
+    assert_equals(blob.type, testUnit.mimeType);
+  }, getDesc(stringConstructor, testUnit.contentType, true));
+}

fetch/content-type/script.window.js

@@ -0,0 +1,48 @@
+promise_test(() => {
+  return fetch("resources/script-content-types.json").then(res => res.json()).then(runTests);
+}, "Loading JSON…");
+
+self.stringFromExecutedScript = undefined;
+
+function runTests(allTestData) {
+  allTestData.forEach(testData => {
+    runScriptTest(testData, false);
+    if (testData.contentType.length > 1) {
+      runScriptTest(testData, true);
+    }
+  });
+}
+
+function runScriptTest(testData, singleHeader) {
+  async_test(t => {
+    const script = document.createElement("script");
+      t.add_cleanup(() => {
+      script.remove()
+      self.stringFromExecutedScript = undefined;
+    });
+    script.src = getURL(testData.contentType, singleHeader);
+    document.head.appendChild(script);
+    if (testData.executes) {
+      script.onload = t.step_func_done(() => {
+        assert_equals(self.stringFromExecutedScript, testData.encoding === "windows-1252" ? "€" : "€");
+      });
+      script.onerror = t.unreached_func();
+    } else {
+      script.onerror = t.step_func_done();
+      script.onload = t.unreached_func();
+    }
+  }, (singleHeader ? "combined" : "separate") + " " + testData.contentType.join(" "));
+}
+
+function getURL(input, singleHeader) {
+  // Edge does not support URLSearchParams
+  let url = "resources/content-type.py?"
+  if (singleHeader) {
+    url += "single_header&"
+  }
+  input.forEach(val => {
+    url += "value=" + encodeURIComponent(val) + "&";
+  });
+  url += "&content=" + encodeURIComponent("self.stringFromExecutedScript = \"€\"");
+  return url;
+}

mimesniff/mime-types/resources/mime-types.json

@@ -367,5 +367,17 @@
   {
     "input": "\u0100/\u0100",
     "output": null
+  },
+  {
+    "input": "text /html",
+    "output": null
+  },
+  {
+    "input": "text/ html",
+    "output": null
+  },
+  {
+    "input": "\"text/html\"",
+    "output": null
   }
 ]