Merge pull request #261 from taskcluster/render-new-schemas-refs

Bug 1463260 - Render new format of schemas and refs

layout/footer.pug

@@ -17,13 +17,6 @@ if !noAnchors
     }
     anchors.add();
     anchors.remove('.no-anchor');
-if docref
-  script.
-    // Make sure the URL's protocol is `https`.  Lots of schema and refernce
-    // URLs are hard-coded as http, but that will cause mixed-content warnings.
-    var fixUrlProtocol = function(url) {
-      return url.replace(/^http:/, 'https:');
-    }
 if sequence_diagrams
   script(src='/assets/raphael-min.js')
   script(src='/assets/underscore-min.js')

layout/schema.pug

@@ -189,10 +189,10 @@ mixin schemabox(schema)
       if schema.title
         h4.no-anchor
           | #{schema.title}
-          if schema.id
+          if schema.$id || schema.id
             | 
             small
-              a(href=schema.id, target='_blank', rel='noopener noreferrer')
+              a(href=schema.$id || schema.id, target='_blank', rel='noopener noreferrer')
                 | (source)
         | !{marked(schema.description || '')}
     table.table

lib/raw.js

@@ -13,6 +13,7 @@ var through = require('through2');
 var gutil = require('gulp-util');
 var PluginError = gutil.PluginError;
 var File = require('vinyl');
+const {buildSchemaId} = require('./utils');
 var fs = require('fs');
 var pug = require('pug');
 var marked = require('marked');
@@ -72,14 +73,30 @@ module.exports = function() {
       await Promise.all(_.flatMap(dat.entries, ref => {
         return _.map(['input', 'output', 'schema'], type => {
           if (ref[type]) {
-            let sname = path.basename(url.parse(ref[type]).pathname);
+            let sname = ref[type].startsWith('http://') ?
+              path.basename(url.parse(ref[type]).pathname) : // Old-style schema i.e. https://schemas.tc.net/...
+              ref[type].replace(/#$/, ''); // New-style relative schema i.e. /v1/something.json#
             // This JSON.parse stuff here and a few lines down lets us avoid going remote for remote refs
             // that are actually local. We do not allow remote (as in cross-project) refs in taskcluster.
             let schema = JSON.parse(fs.readFileSync(path.join('raw/reference', project, 'schemas', sname)));
+            if (schema.$id) {
+              schema.$id = buildSchemaId(schema.$id);
+            }
+            if (schema.id) {
+              schema.id = buildSchemaId(schema.id);
+            }
             return RefParser.dereference(schema, {
               resolve: {
-                any: {
+                taskcluster: {
                   order: 1,
+                  canRead: /^taskcluster:*/,
+                  read: (file, callback) => {
+                    let sname = url.parse(file.url).path.replace(/^\/schemas\/[^\/]*\//, '');
+                    callback(null, JSON.parse(fs.readFileSync(path.join('raw/reference', project, 'schemas', sname))));
+                  },
+                },
+                any: {
+                  order: 2,
                   canRead: /^http*/,
                   read: (file, callback) => {
                     let sname = path.basename(url.parse(file.url).pathname);

lib/render-schemas.js

@@ -2,6 +2,7 @@ const path = require('path');
 const fs = require('fs');
 const got = require('got');
 const pug = require('pug');
+const {buildSchemaId} = require('./utils');
 const {URL} = require('url');
 const {JSDOM} = require('jsdom');
 const through = require('through2');
@@ -29,11 +30,13 @@ module.exports = () => through.obj((file, enc, cb) => {
   const dom = JSDOM.fragment(file.contents);
   const schemas = dom.querySelectorAll('div[data-render-schema]');
   schemas.forEach(div => {
-    const schemaRef = div.dataset.renderSchema;
+    let schemaRef = div.dataset.renderSchema;
     if (schemaRef) {
       promises.push((async () => {
         let schema;
 
+        schemaRef = buildSchemaId(schemaRef);
+
         try {
           const url = new URL(schemaRef);
           schema = (await got(url)).body;
@@ -54,6 +57,7 @@ module.exports = () => through.obj((file, enc, cb) => {
 
         schema = await RefParser.dereference(JSON.parse(schema), {dereference: {circular: 'ignore'}});
         schema.id = schema.id || schemaRef;
+        schema.$id = schema.$id || schemaRef;
         div.innerHTML = template({schema, marked});
       })());
     }

lib/utils.js

@@ -0,0 +1,11 @@
+exports.buildSchemaId = schemaId => {
+  if (schemaId.startsWith('taskcluster:')) {
+    if (process.env.TASKCLUSTER_ROOT_URL && process.env.TASKCLUSTER_ROOT_URL !== 'https://taskclluster.net') {
+      return schemaId.replace(/^taskcluster:/, process.env.TASKCLUSTER_ROOT_URL);
+    } else {
+      schemaId = schemaId.replace(/^taskcluster:\/schemas\//, '');
+      return `https://schemas.taskcluster.net/${schemaId}`
+    }
+  }
+  return schemaId;
+};