Permalink
Browse files

WIP. Start pulling in metadata from comments.

Also update the README to be more EXCITING!
  • Loading branch information...
1 parent 5589701 commit ccfa3518a7175ecd65285dbfd01a36b1cc51307e @csnover committed Apr 23, 2012
Showing with 262 additions and 80 deletions.
  1. +40 −5 README.md
  2. +48 −0 lib/Metadata.js
  3. +24 −21 lib/Value.js
  4. +50 −30 lib/esprimaParser.js
  5. +5 −1 lib/exporter/dojov1.js
  6. +61 −20 lib/processor/dojodoc.js
  7. +21 −0 tests/amd-with-dojoComments.js
  8. +13 −3 tests/dojoComments.js
View
45 README.md
@@ -4,15 +4,50 @@ js-doc-parse
A library for parsing JavaScript files and extracting inline documentation. Designed primarily for use with the Dojo
Toolkit, but extensible enough to work with any application or documentation format.
-New BSD License © 2011-2012 Colin Snover <http://zetafleet.com>
+New BSD License © 20112012 Colin Snover <http://zetafleet.com>.
Why is this library special?
----------------------------
-1. Works anywhere, not only on Rhino.
-2. Parses the actual JavaScript source code to extract API details, instead of just running some regular expressions
- over the source text to pluck out comment blocks. (= fewer boilerplate comments)
-3. Designed to be highly extensible, with planned support for two code commenting styles (dojodoc and jsdoc).
+1. Works anywhere, not only on Rhino. (n.b. by “anywhere” I mean “node.js”, but there’s only a handful of code that
+ needs to be abstracted for it to work on Rhino and in the browser, too, and I plan on doing just that.)
+2. It isn’t lazy! js-doc-parse completely parses the actual JavaScript source code to extract API details instead of
+ just running some crappy regular expressions over the source text to pluck out comment blocks.
+3. [Highly extensible](https://github.com/csnover/js-doc-parse/blob/esprima/config.js), with initially planned support
+ for two code commenting styles (dojodoc and jsdoc).
+
+Wait, _highly_ extensible? Tell me more!
+----------------------------------------
+
+Oh, alright! This documentation parser is designed to allow you, a JavaScript developer, to very easily extend four key
+areas of operation:
+
+1. Environments
+
+ Your code might run in a variety of environments and exploit natively available functionality of each environment.
+ If the documentation parser doesn’t know about all those delicious native objects, it can’t help you document
+ anything you borrow from them. Pluggable environments let you define all the built-ins of the environment your code
+ runs in so it doesn’t fall over when you try to `var slice = [].slice;`.
+
+2. Call handlers
+
+ Chances are good that whatever library you’re using has at least one function that retrieves, decorates, or iterates
+ over objects that you need to document. Adding custom call handlers makes it possible for the documentation parser
+ to understand and evaluate these special calls so that the correct properties exist on the correct objects and that
+ any special magic that these functions might represent can be annotated.
+
+3. Documentation processors
+
+ Some people don’t like the standard jsdoc syntax, and that’s cool. If you’ve extended jsdoc with some new tags, or
+ use some completely different format like dojodocs, Natural Docs, or docco – or, after an evening of heavy drinking,
+ created your own custom documentation format from scratch – writing a new documentation processor will enable you
+ to process any format your heart desires. It will not, however, cure your cirrhosis.
+
+4. Exporters
+
+ Once the code has been parsed and the documentation parsed, you’ll need to output the result to a format that you
+ (or some tool other than you) can actually use! Exporters are completely isolated from the documentation processing
+ workflow, so you can pick and choose one (or lots of) exporters for your final output.
Dependencies
------------
View
48 lib/Metadata.js
@@ -0,0 +1,48 @@
+define([ 'dojo/_base/lang' ], function (lang) {
+ function Metadata(/**Object*/ kwArgs) {
+ if (!(this instanceof Metadata)) {
+ throw new Error('Metadata is a constructor');
+ }
+
+ this.examples = [];
+ this.tags = [];
+
+ lang.mixin(this, kwArgs);
+ }
+
+ Metadata.prototype = {
+ constructor: Metadata,
+
+ /**
+ * A more detailed value type description.
+ * @type string
+ */
+ type: undefined,
+
+ /**
+ * A brief summary of the associated value.
+ * @type string
+ */
+ summary: undefined,
+
+ /**
+ * A detailed description of the associated value.
+ * @type string
+ */
+ description: undefined,
+
+ /**
+ * Code examples.
+ * @type Array.<string>
+ */
+ examples: [],
+
+ /**
+ * Tags.
+ * @type Array.<string>
+ */
+ tags: []
+ };
+
+ return Metadata;
+});
View
45 lib/Value.js
@@ -1,4 +1,9 @@
-define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console) {
+define([ 'dojo/_base/lang', './env', './console', './Metadata' ], function (lang, env, console, Metadata) {
+ // Sometimes the default hasOwnProperty property on a properties object is shadowed by an actual Value
+ // representing hasOwnProperty, so we have to use a fresh copy. The underscore on the name is to make
+ // jshint be quiet about it being a "bad name".
+ var _hasOwnProperty = Object.prototype.hasOwnProperty;
+
/**
* Represents a parsed data structure (object, etc.).
*/
@@ -12,7 +17,7 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
this.parameters = [];
this.returns = [];
this.throws = [];
- this.comments = [];
+ this.metadata = new Metadata();
this.file = env.file;
lang.mixin(this, kwArgs);
@@ -52,9 +57,15 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
* fully resolved yet.
*/
_type: Value.TYPE_UNDEFINED,
+
get type() {
return this._type;
},
+
+ /**
+ * Sets ES5-standard environment prototype and constructor properties on Values automatically based on their
+ * defined data type.
+ */
set type(value) {
this._type = value;
@@ -106,10 +117,12 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
properties: {},
/**
- * Metadata annotations for this data structure.
- * @type Object.<*>
+ * Metadata annotations for this data structure. This object should be used by all comment processors instead
+ * of modifying the Value object directly, since it may apply to multiple Values or variables that are
+ * overwritten with new Value objects.
+ * @type Metadata
*/
- metadata: {},
+ metadata: undefined,
/**
* If type is 'function' or 'constructor', the scope of the function containing information about variables
@@ -125,12 +138,6 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
parameters: [],
/**
- * Comments attached to this Value.
- * @type Array.<token>
- */
- comments: [],
-
- /**
* If type is 'function' or 'constructor', all return values from within the function.
* @type Array.<Value|Reference>
*/
@@ -161,18 +168,13 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
// whenever type gets set to one of these values.
if ((this.type === Value.TYPE_FUNCTION || this.type === Value.TYPE_INSTANCE) &&
(name[0] === 'constructor' || name[0] === 'prototype') &&
- !this.properties.hasOwnProperty(name[0])) {
+ !_hasOwnProperty.call(this.properties, name[0])) {
this.setProperty([ name[0] ], new Value({
type: name[0] === 'constructor' ? Value.TYPE_FUNCTION : Value.TYPE_OBJECT
}));
}
- // Sometimes the default hasOwnProperty property on a properties object is shadowed by an actual Value
- // representing hasOwnProperty, so we have to use a fresh copy. The underscore on the name is to make
- // jshint be quiet about it being a "bad name".
- var _hasOwnProperty = Object.prototype.hasOwnProperty;
-
for (var variable = this, i = 0, j = name.length; i < j; ++i) {
// hasOwnProperty checks are necessary to ensure built-in names like 'toString' are not picked up
// from the object's prototype
@@ -228,10 +230,12 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
obj = obj.properties[key];
}
- if (obj.properties[name[j]]) {
+ if (_hasOwnProperty.call(obj.properties, name[j])) {
console.info('Resetting property of object ' + this);
- // TODO: Should mixin, not overwrite, the property if it was once assumed to exist
+ // TODO: Should mixin, not overwrite, the property metadata if it was once assumed to exist,
+ // including combining arrays instead of overwriting them
+ value.metadata = lang.mixin(new Metadata(), obj.properties[name[j]].metadata, value.metadata);
}
obj.properties[name[j]] = value;
@@ -266,8 +270,7 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
}
var array = [],
- obj = this.properties,
- _hasOwnProperty = Object.prototype.hasOwnProperty;
+ obj = this.properties;
for (var k in obj) {
if (_hasOwnProperty.call(obj, k) && !isNaN(k)) {
View
80 lib/esprimaParser.js
@@ -40,16 +40,28 @@ define([
}
/**
- * Find comments to associate with a given node.
- * @param node The AST node.
- * @returns {Array} An array of comments.
+ * Attaches metadata to a given Value.
*/
- function getComments(/**Node*/ node, /**Node*/ contextNode) {
- return env.processors.reduce(function (comments, processor) {
- // Always want to concatenate an empty array, not some other value, if attachComment does not exist or
- // returns nothing
- return comments.concat((processor.attachComment && processor.attachComment(node, contextNode, env.parserState)) || []);
- }, []);
+ function attachMetadata(/**Value*/ value, /**Node*/ node, /**Value?*/ contextValue, /**Node?*/ contextNode) {
+ env.processors.forEach(function (processor) {
+ if (!processor.generateMetadata) {
+ return;
+ }
+
+ // TODO: Last processor always wins; this is OK for now, but maybe not later
+
+ value = {
+ raw: node,
+ evaluated: value
+ };
+
+ contextValue = contextValue || contextNode ? {
+ raw: contextNode,
+ evaluated: contextValue
+ } : undefined;
+
+ lang.mixin(value.evaluated.metadata, processor.generateMetadata(value, contextValue) || {});
+ });
}
/**
@@ -58,16 +70,17 @@ define([
* @returns {Value} A function Value.
*/
function createFunctionValue(/**Node*/ fn) {
- return new Value({
- type: Value.TYPE_FUNCTION,
- parameters: fn.params.map(function (identifier) {
- return new Parameter({
- name: identifier.name,
- comments: getComments(identifier, fn)
- });
- }),
- comments: getComments(fn.body, fn)
+ var functionValue = new Value({ type: Value.TYPE_FUNCTION });
+
+ functionValue.parameters = fn.params.map(function (identifier) {
+ var parameterValue = new Parameter({ name: identifier.name });
+ attachMetadata(parameterValue, identifier, functionValue, fn);
+ return parameterValue;
});
+
+ attachMetadata(functionValue, fn);
+
+ return functionValue;
}
/**
@@ -246,21 +259,23 @@ define([
}
if (expression.operator === '=') {
- // TODO: Avoid losing comments from LHS?
env.scope.setVariableValue(lhsIdentifier.map(function (id) { return id.name; }), rhsValue);
+
+ // TODO: Avoid losing metadata from lhsValue?
lhsValue = rhsValue;
- lhsValue.comments = lhsValue.comments.concat(getComments(expression.left), rhsValue.comments);
}
else {
lhsValue = read(expression.left);
- lhsValue.comments = lhsValue.comments.concat(getComments(expression.left));
if (!lhsValue) {
console.info('Cannot resolve origin value');
- return new Value({ type: Value.TYPE_UNDEFINED });
+ lhsValue = new Value({ type: Value.TYPE_UNDEFINED });
+ }
+ else {
+ lang.mixin(lhsValue, calculateBinary(lhsValue, expression.operator.replace(/\=$/, ''), rhsValue));
}
- lang.mixin(lhsValue, calculateBinary(lhsValue, expression.operator.replace(/\=$/, ''), rhsValue));
+ attachMetadata(lhsValue, expression.left);
}
return lhsValue;
@@ -276,8 +291,7 @@ define([
for (var i = 0, j = expression.elements.length, element, elementValue; i < j; ++i) {
element = expression.elements[i];
elementValue = element === undefined ? new Value({ type: Value.TYPE_UNDEFINED }) : read(element);
-
- elementValue.comments = elementValue.comments.concat(getComments(element, expression));
+ attachMetadata(elementValue, element, value, expression);
array.push(elementValue);
}
@@ -286,7 +300,7 @@ define([
// and a Value object is not a valid array length
value.properties = arrayToObject(array);
- value.comments = value.comments.concat(getComments(expression));
+ attachMetadata(value, expression);
return value;
},
@@ -496,6 +510,8 @@ define([
value.evaluate = function (/**Array.<Value>*/ args) {
var oldScope = env.setScope(value.scope);
+ args = args || [];
+
for (var i = 0, parameter; (parameter = value.parameters[i]); ++i) {
// TODO: Copy metadata from the parameter to the argument value
env.scope.setVariableValue(parameter.name, args[i] || parameter);
@@ -599,12 +615,13 @@ define([
for (var i = 0, property, propertyValue; (property = expression.properties[i]); ++i) {
// key might be a Literal or an Identifier
propertyValue = read(property.value);
+ propertyValue.evaluate && propertyValue.evaluate();
value.properties[property.key.value || property.key.name] = propertyValue;
- propertyValue.comments = propertyValue.comments.concat(getComments(property, expression));
+ attachMetadata(propertyValue, property, value, expression);
}
- value.comments = value.comments.concat(getComments(expression));
+ attachMetadata(value, expression);
return value;
},
@@ -623,7 +640,7 @@ define([
if (statement.argument) {
var value = read(statement.argument);
- value.comments = value.comments.concat(getComments(statement));
+ attachMetadata(value, statement);
env.functionScope.relatedFunction.returns.push(value);
}
},
@@ -731,7 +748,8 @@ define([
for (i = 0; (declaration = expression.declarations[i]); ++i) {
value = declaration.init ? read(declaration.init) : new Value({ type: Value.TYPE_UNDEFINED });
- value.comments = value.comments.concat(getComments(declaration, expression));
+ value.evaluate && value.evaluate();
+ attachMetadata(value, declaration, null, expression);
scope.setVariableValue(declaration.id.name, value);
}
},
@@ -798,6 +816,8 @@ define([
token.value.push(comment.value);
token.range[1] = comment.range[1];
} while (nextCommentIsRunOn());
+
+ token.value = token.value.join('\n');
}
return token;
View
6 lib/exporter/dojov1.js
@@ -5,6 +5,10 @@ define([ '../Module', '../Value', './util' ], function (Module, Value, util) {
* @param metadata The metadata to parse.
*/
function mixinMetadata(/**XmlNode*/ node, /**Object*/ metadata) {
+ if (metadata.type) {
+ node.attributes.type = metadata.type;
+ }
+
for (var metadataName in { summary: 1, description: 1 }) {
if (metadata.hasOwnProperty(metadataName)) {
node.createNode(metadataName).childNodes.push(metadata[metadataName]);
@@ -59,7 +63,7 @@ define([ '../Module', '../Value', './util' ], function (Module, Value, util) {
for (i = 0; (returnValue = property.returns[i]); ++i) {
returnsNode.createNode('return-type', {
- type: returnValue.type
+ type: returnValue.metadata.type || returnValue.type
});
}
}
View
81 lib/processor/dojodoc.js
@@ -1,4 +1,22 @@
-define([ '../env', './util' ], function (env, util) {
+define([ '../Value', '../env', './util' ], function (Value, env, util) {
+ function trim(str) {
+ return str.replace(/^[\s*]+|\s+$/g, '');
+ }
+
+ function processComment(comment, summaryId) {
+ var metadata = {};
+
+ if (summaryId) {
+ var type = new RegExp(summaryId + ':\\s+(.*?)\n', 'm').exec(comment);
+
+ if (type) {
+ metadata.type = trim(type[1]);
+ }
+ }
+
+ return metadata;
+ }
+
return {
/**
* Processes raw source code prior to being parsed.
@@ -10,34 +28,57 @@ define([ '../env', './util' ], function (env, util) {
},
/**
- * Attaches comments to the correct nearby nodes.
- * @param node An AST node.
- * @param node The nearest enclosing structure node (object, function, etc.).
+ * Returns metadata for a given Value.
+ * @param value Information on the Value being processed. Contains two properties:
+ * - raw: The raw AST node for the Value object.
+ * - evaluated: The Value object itself.
+ * @param contextValue Information on the nearest enclosing structure node (object, function, etc.).
+ * Contains two properties:
+ * - raw: The raw AST node for the context Value.
+ * - evaluated: The context Value itself. Note that not all enclosing structures may be associated with
+ * a Value (i.e. VariableDeclarations).
+ * @returns {Object} An object containing keys that will be mixed into the metadata of the main Value object.
*/
- attachComment: function (/**Node*/ node, /**Node*/ contextNode) {
+ generateMetadata: function (/**Object*/ value, /**Object?*/ context) {
var candidate;
- if (node.type === 'Identifier' && contextNode && contextNode.type === 'FunctionDeclaration') {
- candidate = util.getTokensInRange(node.range, true)[0];
- if (candidate.type === 'BlockComment') {
- return [ candidate.value ];
- }
+ // Function parameter
+ if (value.raw.type === 'Identifier' && context && context.raw.type === 'FunctionDeclaration') {
+ candidate = util.getTokensInRange(value.raw.range, true)[0];
+
+ return candidate.type === 'BlockComment' ? { type: trim(candidate.value) } : {};
+ }
+
+ // Function return statement
+ if (value.raw.type === 'ReturnStatement') {
+ candidate = /^[^\n]*\/\/(.*?)\n/.exec(util.getSourceForRange(value.raw.range));
+
+ return candidate ? { type: trim(candidate[1]) } : {};
}
- if (node.type === 'ReturnStatement') {
- candidate = /\/\/(.*)$/m.exec(util.getSourceForRange(node.range));
- return candidate ? [ candidate[1] ] : [];
+ // Function or object body
+ if (value.raw.type.indexOf('Function') > -1 || value.raw.type === 'ObjectExpression') {
+ // First token after the opening {
+ candidate = util.getTokensInRange(value.raw.type === 'ObjectExpression' ? value.raw.range : value.raw.body.range)[1];
+
+ return candidate.type === 'LineBlockComment' ? processComment(candidate.value) : {};
}
- return [];
- },
+ // Object property
+ if (value.raw.type === 'Property' && context && context.raw.type === 'ObjectExpression') {
+ candidate = util.getTokensInRange(value.raw.range, true)[0];
- /**
- * Performs additional processing on each fully defined value, such as copying data from attached comment
- * blocks onto the Value.
- */
- processValue: function (/**Value*/ value) {
+ if (candidate.type === 'LineBlockComment' &&
+ new RegExp('^\\s*' + value.raw.key.name + ':').test(candidate.value)) {
+
+ return processComment(candidate.value, value.raw.key.name);
+ }
+ else {
+ return {};
+ }
+ }
+ return {}; // strict mode
}
};
});
View
21 tests/amd-with-dojoComments.js
@@ -0,0 +1,21 @@
+define([], function () {
+ return {
+ // summary:
+ // This is an object.
+
+ // fooBar: SuperBoolean
+ // This is a property.
+ fooBar: true,
+
+ fn: function (/*FooType*/ foo, /**BarType*/ bar, baz) {
+ // summary:
+ // This is a function.
+ // returns:
+ // Some data, perhaps.
+
+ return [ foo, // AmazingArray
+ bar
+ ];
+ }
+ };
+});
View
16 tests/dojoComments.js
@@ -1,7 +1,17 @@
function fn(/*foo*/ foo, /**bar*/ bar, baz) {
- // function comment
- // many lines
- // one function
+ // summary:
+ // This is a function.
+ // returns:
+ // Some data, perhaps.
+
+ foo = {
+ // summary:
+ // This is an object.
+
+ // fooBar: SuperBoolean
+ // This is a property.
+ fooBar: true
+ };
return [ foo, // Array
bar

0 comments on commit ccfa351

Please sign in to comment.