From 50100f3a9cf164be2fa4a2a2e45f30675e7d3a63 Mon Sep 17 00:00:00 2001 From: Sam Ruby Date: Sun, 19 Aug 2018 14:03:21 -0400 Subject: [PATCH] tools: Include links to source code in documentation Parse source code using acorn; extracting exports. When producing documentation, match exports to headers. When a match is found, add a [src] link. This first commit handles simple exported classes and functions, and does so without requiring any changes to the source code or markdown. Subsequent commits will attempt to match more headers, and some of these changes are likely to require changes to the source code and/or markdown. PR-URL: https://github.com/nodejs/node/pull/22405 Reviewed-By: James M Snell Reviewed-By: Refael Ackermann Reviewed-By: Vse Mozhet Byt --- Makefile | 7 +- doc/api_assets/style.css | 5 + test/doctool/test-apilinks.js | 39 +++++++ test/doctool/test-doctool-html.js | 2 +- test/fixtures/apilinks/buffer.js | 12 +++ test/fixtures/apilinks/buffer.json | 4 + test/fixtures/apilinks/mod.js | 11 ++ test/fixtures/apilinks/mod.json | 3 + test/fixtures/apilinks/prototype.js | 13 +++ test/fixtures/apilinks/prototype.json | 5 + tools/doc/apilinks.js | 141 ++++++++++++++++++++++++++ tools/doc/generate.js | 7 +- tools/doc/html.js | 7 +- 13 files changed, 252 insertions(+), 4 deletions(-) create mode 100644 test/doctool/test-apilinks.js create mode 100644 test/fixtures/apilinks/buffer.js create mode 100644 test/fixtures/apilinks/buffer.json create mode 100644 test/fixtures/apilinks/mod.js create mode 100644 test/fixtures/apilinks/mod.json create mode 100644 test/fixtures/apilinks/prototype.js create mode 100644 test/fixtures/apilinks/prototype.json create mode 100644 tools/doc/apilinks.js diff --git a/Makefile b/Makefile index 2452244fc25967..8769b5b95c9f58 100644 --- a/Makefile +++ b/Makefile @@ -645,10 +645,15 @@ out/doc/api/assets/%: doc/api_assets/% out/doc/api/assets run-npm-ci = $(PWD)/$(NPM) ci gen-api = tools/doc/generate.js --node-version=$(FULLVERSION) \ + --apilinks=out/apilinks.json \ --analytics=$(DOCS_ANALYTICS) $< --output-directory=out/doc/api +gen-apilink = tools/doc/apilinks.js $(wildcard lib/*.js) > $@ + +out/apilinks.json: $(wildcard lib/*.js) tools/doc/apilinks.js + $(call available-node, $(gen-apilink)) out/doc/api/%.json out/doc/api/%.html: doc/api/%.md tools/doc/generate.js \ - tools/doc/html.js tools/doc/json.js + tools/doc/html.js tools/doc/json.js | out/apilinks.json $(call available-node, $(gen-api)) out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js diff --git a/doc/api_assets/style.css b/doc/api_assets/style.css index f59f3770048097..7d65b7405b41d1 100644 --- a/doc/api_assets/style.css +++ b/doc/api_assets/style.css @@ -283,6 +283,11 @@ h2, h3, h4, h5 { padding-right: 40px; } +.srclink { + float: right; + font-size: smaller; +} + h1 span, h2 span, h3 span, h4 span { position: absolute; display: block; diff --git a/test/doctool/test-apilinks.js b/test/doctool/test-apilinks.js new file mode 100644 index 00000000000000..e53c81a08a7203 --- /dev/null +++ b/test/doctool/test-apilinks.js @@ -0,0 +1,39 @@ +'use strict'; + +require('../common'); +const fixtures = require('../common/fixtures'); +const fs = require('fs'); +const assert = require('assert'); +const path = require('path'); +const { execFileSync } = require('child_process'); + +const script = path.join(__dirname, '..', '..', 'tools', 'doc', 'apilinks.js'); + +const apilinks = fixtures.path('apilinks'); +fs.readdirSync(apilinks).forEach((fixture) => { + if (!fixture.endsWith('.js')) return; + const file = path.join(apilinks, fixture); + + const expectedContent = fs.readFileSync(file + 'on', 'utf8'); + + const output = execFileSync( + process.execPath, + [script, file], + { encoding: 'utf-8' } + ); + + const expectedLinks = JSON.parse(expectedContent); + const actualLinks = JSON.parse(output); + + for (const [k, v] of Object.entries(expectedLinks)) { + assert.ok(k in actualLinks, `link not found: ${k}`); + assert.ok(actualLinks[k].endsWith('/' + v), + `link ${actualLinks[k]} expected to end with ${v}`); + delete actualLinks[k]; + } + + assert.strictEqual( + Object.keys(actualLinks).length, 0, + `unexpected links returned ${JSON.stringify(actualLinks)}` + ); +}); diff --git a/test/doctool/test-doctool-html.js b/test/doctool/test-doctool-html.js index 8c05ea6a0b3229..2fcb8315afd325 100644 --- a/test/doctool/test-doctool-html.js +++ b/test/doctool/test-doctool-html.js @@ -28,7 +28,7 @@ function toHTML({ input, filename, nodeVersion, analytics }, cb) { .use(html.firstHeader) .use(html.preprocessText) .use(html.preprocessElements, { filename }) - .use(html.buildToc, { filename }) + .use(html.buildToc, { filename, apilinks: {} }) .use(remark2rehype, { allowDangerousHTML: true }) .use(raw) .use(htmlStringify) diff --git a/test/fixtures/apilinks/buffer.js b/test/fixtures/apilinks/buffer.js new file mode 100644 index 00000000000000..8ee44123a5e8ca --- /dev/null +++ b/test/fixtures/apilinks/buffer.js @@ -0,0 +1,12 @@ +'use strict'; + +// Buffer instance methods are exported as 'buf'. + +function Buffer() { +} + +Buffer.prototype.instanceMethod = function() {} + +module.exports = { + Buffer +}; diff --git a/test/fixtures/apilinks/buffer.json b/test/fixtures/apilinks/buffer.json new file mode 100644 index 00000000000000..528eca6eb21548 --- /dev/null +++ b/test/fixtures/apilinks/buffer.json @@ -0,0 +1,4 @@ +{ + "buffer.Buffer": "buffer.js#L5", + "buf.instanceMethod": "buffer.js#L8" +} diff --git a/test/fixtures/apilinks/mod.js b/test/fixtures/apilinks/mod.js new file mode 100644 index 00000000000000..72606121f53fdb --- /dev/null +++ b/test/fixtures/apilinks/mod.js @@ -0,0 +1,11 @@ +'use strict'; + +// A module may export one or more methods. + +function foo() { +} + + +module.exports = { + foo +}; diff --git a/test/fixtures/apilinks/mod.json b/test/fixtures/apilinks/mod.json new file mode 100644 index 00000000000000..7d803e623672da --- /dev/null +++ b/test/fixtures/apilinks/mod.json @@ -0,0 +1,3 @@ +{ + "mod.foo": "mod.js#L5" +} diff --git a/test/fixtures/apilinks/prototype.js b/test/fixtures/apilinks/prototype.js new file mode 100644 index 00000000000000..40218e844e68ad --- /dev/null +++ b/test/fixtures/apilinks/prototype.js @@ -0,0 +1,13 @@ +'use strict'; + +// An exported class using classic prototype syntax. + +function Class() { +} + +Class.classMethod = function() {} +Class.prototype.instanceMethod = function() {} + +module.exports = { + Class +}; diff --git a/test/fixtures/apilinks/prototype.json b/test/fixtures/apilinks/prototype.json new file mode 100644 index 00000000000000..d61c32da62a64e --- /dev/null +++ b/test/fixtures/apilinks/prototype.json @@ -0,0 +1,5 @@ +{ + "prototype.Class": "prototype.js#L5", + "Class.classMethod": "prototype.js#L8", + "class.instanceMethod": "prototype.js#L9" +} diff --git a/tools/doc/apilinks.js b/tools/doc/apilinks.js new file mode 100644 index 00000000000000..98dd7827d67ac0 --- /dev/null +++ b/tools/doc/apilinks.js @@ -0,0 +1,141 @@ +'use strict'; + +// Scan API sources for definitions. +// +// Note the output is produced based on a world class parser, adherence to +// conventions, and a bit of guess work. Examples: +// +// * We scan for top level module.exports statements, and determine what +// is exported by looking at the source code only (i.e., we don't do +// an eval). If exports include `Foo`, it probably is a class, whereas +// if what is exported is `constants` it probably is prefixed by the +// basename of the source file (e.g., `zlib`), unless that source file is +// `buffer.js`, in which case the name is just `buf`. unless the constant +// is `kMaxLength`, in which case it is `buffer`. +// +// * We scan for top level definitions for those exports, handling +// most common cases (e.g., `X.prototype.foo =`, `X.foo =`, +// `function X(...) {...}`). Over time, we expect to handle more +// cases (example: ES2015 class definitions). + +const acorn = require('../../deps/acorn'); +const fs = require('fs'); +const path = require('path'); +const child_process = require('child_process'); + +// Run a command, capturing stdout, ignoring errors. +function execSync(command) { + try { + return child_process.execSync( + command, + { stdio: ['ignore', null, 'ignore'] } + ).toString().trim(); + } catch { + return ''; + } +} + +// Determine orign repo and tag (or hash) of the most recent commit. +const local_branch = execSync('git name-rev --name-only HEAD'); +const tracking_remote = execSync(`git config branch.${local_branch}.remote`); +const remote_url = execSync(`git config remote.${tracking_remote}.url`); +const repo = (remote_url.match(/(\w+\/\w+)\.git\r?\n?$/) || + ['', 'nodejs/node'])[1]; + +const hash = execSync('git log -1 --pretty=%H') || 'master'; +const tag = execSync(`git describe --contains ${hash}`).split('\n')[0] || hash; + +// Extract definitions from each file specified. +const definition = {}; +process.argv.slice(2).forEach((file) => { + const basename = path.basename(file, '.js'); + + // Parse source. + const source = fs.readFileSync(file, 'utf8'); + const ast = acorn.parse(source, { ecmaVersion: 10, locations: true }); + const program = ast.body; + + // Scan for exports. + const exported = { constructors: [], identifiers: [] }; + program.forEach((statement) => { + if (statement.type !== 'ExpressionStatement') return; + const expr = statement.expression; + if (expr.type !== 'AssignmentExpression') return; + + let lhs = expr.left; + if (expr.left.object.type === 'MemberExpression') lhs = lhs.object; + if (lhs.type !== 'MemberExpression') return; + if (lhs.object.name !== 'module') return; + if (lhs.property.name !== 'exports') return; + + let rhs = expr.right; + while (rhs.type === 'AssignmentExpression') rhs = rhs.right; + + if (rhs.type === 'NewExpression') { + exported.constructors.push(rhs.callee.name); + } else if (rhs.type === 'ObjectExpression') { + rhs.properties.forEach((property) => { + if (property.value.type === 'Identifier') { + exported.identifiers.push(property.value.name); + if (/^[A-Z]/.test(property.value.name[0])) { + exported.constructors.push(property.value.name); + } + } + }); + } else if (rhs.type === 'Identifier') { + exported.identifiers.push(rhs.name); + } + }); + + // Scan for definitions matching those exports; currently supports: + // + // ClassName.foo = ...; + // ClassName.prototype.foo = ...; + // function Identifier(...) {...}; + // + const link = `https://github.com/${repo}/blob/${tag}/` + + path.relative('.', file).replace(/\\/g, '/'); + + program.forEach((statement) => { + if (statement.type === 'ExpressionStatement') { + const expr = statement.expression; + if (expr.type !== 'AssignmentExpression') return; + if (expr.left.type !== 'MemberExpression') return; + + let object; + if (expr.left.object.type === 'MemberExpression') { + if (expr.left.object.property.name !== 'prototype') return; + object = expr.left.object.object; + } else if (expr.left.object.type === 'Identifier') { + object = expr.left.object; + } else { + return; + } + + if (!exported.constructors.includes(object.name)) return; + + let objectName = object.name; + if (expr.left.object.type === 'MemberExpression') { + objectName = objectName.toLowerCase(); + if (objectName === 'buffer') objectName = 'buf'; + } + + let name = expr.left.property.name; + if (expr.left.computed) { + name = `${objectName}[${name}]`; + } else { + name = `${objectName}.${name}`; + } + + definition[name] = `${link}#L${statement.loc.start.line}`; + } else if (statement.type === 'FunctionDeclaration') { + const name = statement.id.name; + if (!exported.identifiers.includes(name)) return; + if (basename.startsWith('_')) return; + definition[`${basename}.${name}`] = + `${link}#L${statement.loc.start.line}`; + } + }); +}); + +console.log(JSON.stringify(definition, null, 2)); diff --git a/tools/doc/generate.js b/tools/doc/generate.js index f2e3e8a1649985..28e09d003fa6db 100644 --- a/tools/doc/generate.js +++ b/tools/doc/generate.js @@ -40,6 +40,7 @@ let filename = null; let nodeVersion = null; let analytics = null; let outputDir = null; +let apilinks = {}; args.forEach(function(arg) { if (!arg.startsWith('--')) { @@ -50,6 +51,10 @@ args.forEach(function(arg) { analytics = arg.replace(/^--analytics=/, ''); } else if (arg.startsWith('--output-directory=')) { outputDir = arg.replace(/^--output-directory=/, ''); + } else if (arg.startsWith('--apilinks=')) { + apilinks = JSON.parse( + fs.readFileSync(arg.replace(/^--apilinks=/, ''), 'utf8') + ); } }); @@ -71,7 +76,7 @@ fs.readFile(filename, 'utf8', (er, input) => { .use(json.jsonAPI, { filename }) .use(html.firstHeader) .use(html.preprocessElements, { filename }) - .use(html.buildToc, { filename }) + .use(html.buildToc, { filename, apilinks }) .use(remark2rehype, { allowDangerousHTML: true }) .use(raw) .use(htmlStringify) diff --git a/tools/doc/html.js b/tools/doc/html.js index f1ac9e144e61e1..899605437ce7b3 100644 --- a/tools/doc/html.js +++ b/tools/doc/html.js @@ -329,7 +329,7 @@ function versionSort(a, b) { return +b.match(numberRe)[0] - +a.match(numberRe)[0]; } -function buildToc({ filename }) { +function buildToc({ filename, apilinks }) { return (tree, file) => { const startIncludeRefRE = /^\s*\s*$/; const endIncludeRefRE = /^\s*\s*$/; @@ -376,6 +376,11 @@ function buildToc({ filename }) { `id="${headingText}">#`; } + const api = headingText.replace(/^.*:\s+/, '').replace(/\(.*/, ''); + if (apilinks[api]) { + anchor = `[src]${anchor}`; + } + node.children.push({ type: 'html', value: anchor }); });