WIP. Start pulling in metadata from comments.

Also update the README to be more EXCITING!

README.md

@@ -4,15 +4,50 @@ js-doc-parse
 A library for parsing JavaScript files and extracting inline documentation. Designed primarily for use with the Dojo
 Toolkit, but extensible enough to work with any application or documentation format.
 
-New BSD License © 2011-2012 Colin Snover <http://zetafleet.com>
+New BSD License © 2011–2012 Colin Snover <http://zetafleet.com>.
 
 Why is this library special?
 ----------------------------
 
-1. Works anywhere, not only on Rhino.
-2. Parses the actual JavaScript source code to extract API details, instead of just running some regular expressions
-   over the source text to pluck out comment blocks. (= fewer boilerplate comments)
-3. Designed to be highly extensible, with planned support for two code commenting styles (dojodoc and jsdoc).
+1. Works anywhere, not only on Rhino. (n.b. by “anywhere” I mean “node.js”, but there’s only a handful of code that
+   needs to be abstracted for it to work on Rhino and in the browser, too, and I plan on doing just that.)
+2. It isn’t lazy! js-doc-parse completely parses the actual JavaScript source code to extract API details instead of
+   just running some crappy regular expressions over the source text to pluck out comment blocks.
+3. [Highly extensible](https://github.com/csnover/js-doc-parse/blob/esprima/config.js), with initially planned support
+   for two code commenting styles (dojodoc and jsdoc).
+
+Wait, _highly_ extensible? Tell me more!
+----------------------------------------
+
+Oh, alright! This documentation parser is designed to allow you, a JavaScript developer, to very easily extend four key
+areas of operation:
+
+1. Environments
+
+   Your code might run in a variety of environments and exploit natively available functionality of each environment.
+   If the documentation parser doesn’t know about all those delicious native objects, it can’t help you document
+   anything you borrow from them. Pluggable environments let you define all the built-ins of the environment your code
+   runs in so it doesn’t fall over when you try to `var slice = [].slice;`.
+
+2. Call handlers
+
+   Chances are good that whatever library you’re using has at least one function that retrieves, decorates, or iterates
+   over objects that you need to document. Adding custom call handlers makes it possible for the documentation parser
+   to understand and evaluate these special calls so that the correct properties exist on the correct objects and that
+   any special magic that these functions might represent can be annotated.
+
+3. Documentation processors
+
+   Some people don’t like the standard jsdoc syntax, and that’s cool. If you’ve extended jsdoc with some new tags, or
+   use some completely different format like dojodocs, Natural Docs, or docco – or, after an evening of heavy drinking,
+   created your own custom documentation format from scratch – writing a new documentation processor will enable you
+   to process any format your heart desires. It will not, however, cure your cirrhosis.
+
+4. Exporters
+
+   Once the code has been parsed and the documentation parsed, you’ll need to output the result to a format that you
+   (or some tool other than you) can actually use! Exporters are completely isolated from the documentation processing
+   workflow, so you can pick and choose one (or lots of) exporters for your final output.
 
 Dependencies
 ------------

lib/Metadata.js

@@ -0,0 +1,48 @@
+define([ 'dojo/_base/lang' ], function (lang) {
+	function Metadata(/**Object*/ kwArgs) {
+		if (!(this instanceof Metadata)) {
+			throw new Error('Metadata is a constructor');
+		}
+
+		this.examples = [];
+		this.tags = [];
+
+		lang.mixin(this, kwArgs);
+	}
+
+	Metadata.prototype = {
+		constructor: Metadata,
+
+		/**
+		 * A more detailed value type description.
+		 * @type string
+		 */
+		type: undefined,
+
+		/**
+		 * A brief summary of the associated value.
+		 * @type string
+		 */
+		summary: undefined,
+
+		/**
+		 * A detailed description of the associated value.
+		 * @type string
+		 */
+		description: undefined,
+
+		/**
+		 * Code examples.
+		 * @type Array.<string>
+		 */
+		examples: [],
+
+		/**
+		 * Tags.
+		 * @type Array.<string>
+		 */
+		tags: []
+	};
+
+	return Metadata;
+});

lib/Value.js

@@ -1,4 +1,9 @@
-define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console) {
+define([ 'dojo/_base/lang', './env', './console', './Metadata' ], function (lang, env, console, Metadata) {
+	// Sometimes the default hasOwnProperty property on a properties object is shadowed by an actual Value
+	// representing hasOwnProperty, so we have to use a fresh copy. The underscore on the name is to make
+	// jshint be quiet about it being a "bad name".
+	var _hasOwnProperty = Object.prototype.hasOwnProperty;
+
 	/**
 	 * Represents a parsed data structure (object, etc.).
 	 */
@@ -12,7 +17,7 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
 		this.parameters = [];
 		this.returns = [];
 		this.throws = [];
-		this.comments = [];
+		this.metadata = new Metadata();
 		this.file = env.file;
 
 		lang.mixin(this, kwArgs);
@@ -52,9 +57,15 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
 		 * fully resolved yet.
 		 */
 		_type: Value.TYPE_UNDEFINED,
+
 		get type() {
 			return this._type;
 		},
+
+		/**
+		 * Sets ES5-standard environment prototype and constructor properties on Values automatically based on their
+		 * defined data type.
+		 */
 		set type(value) {
 			this._type = value;
 
@@ -106,10 +117,12 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
 		properties: {},
 
 		/**
-		 * Metadata annotations for this data structure.
-		 * @type Object.<*>
+		 * Metadata annotations for this data structure. This object should be used by all comment processors instead
+		 * of modifying the Value object directly, since it may apply to multiple Values or variables that are
+		 * overwritten with new Value objects.
+		 * @type Metadata
 		 */
-		metadata: {},
+		metadata: undefined,
 
 		/**
 		 * If type is 'function' or 'constructor', the scope of the function containing information about variables
@@ -124,12 +137,6 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
 		 */
 		parameters: [],
 
-		/**
-		 * Comments attached to this Value.
-		 * @type Array.<token>
-		 */
-		comments: [],
-
 		/**
 		 * If type is 'function' or 'constructor', all return values from within the function.
 		 * @type Array.<Value|Reference>
@@ -161,18 +168,13 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
 			// whenever type gets set to one of these values.
 			if ((this.type === Value.TYPE_FUNCTION || this.type === Value.TYPE_INSTANCE) &&
 				(name[0] === 'constructor' || name[0] === 'prototype') &&
-				!this.properties.hasOwnProperty(name[0])) {
+				!_hasOwnProperty.call(this.properties, name[0])) {
 
 				this.setProperty([ name[0] ], new Value({
 					type: name[0] === 'constructor' ? Value.TYPE_FUNCTION : Value.TYPE_OBJECT
 				}));
 			}
 
-			// Sometimes the default hasOwnProperty property on a properties object is shadowed by an actual Value
-			// representing hasOwnProperty, so we have to use a fresh copy. The underscore on the name is to make
-			// jshint be quiet about it being a "bad name".
-			var _hasOwnProperty = Object.prototype.hasOwnProperty;
-
 			for (var variable = this, i = 0, j = name.length; i < j; ++i) {
 				// hasOwnProperty checks are necessary to ensure built-in names like 'toString' are not picked up
 				// from the object's prototype
@@ -228,10 +230,12 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
 				obj = obj.properties[key];
 			}
 
-			if (obj.properties[name[j]]) {
+			if (_hasOwnProperty.call(obj.properties, name[j])) {
 				console.info('Resetting property of object ' + this);
 
-				// TODO: Should mixin, not overwrite, the property if it was once assumed to exist
+				// TODO: Should mixin, not overwrite, the property metadata if it was once assumed to exist,
+				// including combining arrays instead of overwriting them
+				value.metadata = lang.mixin(new Metadata(), obj.properties[name[j]].metadata, value.metadata);
 			}
 
 			obj.properties[name[j]] = value;
@@ -266,8 +270,7 @@ define([ 'dojo/_base/lang', './env', './console' ], function (lang, env, console
 			}
 
 			var array = [],
-				obj = this.properties,
-				_hasOwnProperty = Object.prototype.hasOwnProperty;
+				obj = this.properties;
 
 			for (var k in obj) {
 				if (_hasOwnProperty.call(obj, k) && !isNaN(k)) {

lib/esprimaParser.js

@@ -40,16 +40,28 @@ define([
 	}
 
 	/**
-	 * Find comments to associate with a given node.
-	 * @param node The AST node.
-	 * @returns {Array} An array of comments.
+	 * Attaches metadata to a given Value.
 	 */
-	function getComments(/**Node*/ node, /**Node*/ contextNode) {
-		return env.processors.reduce(function (comments, processor) {
-			// Always want to concatenate an empty array, not some other value, if attachComment does not exist or
-			// returns nothing
-			return comments.concat((processor.attachComment && processor.attachComment(node, contextNode, env.parserState)) || []);
-		}, []);
+	function attachMetadata(/**Value*/ value, /**Node*/ node, /**Value?*/ contextValue, /**Node?*/ contextNode) {
+		env.processors.forEach(function (processor) {
+			if (!processor.generateMetadata) {
+				return;
+			}
+
+			// TODO: Last processor always wins; this is OK for now, but maybe not later
+
+			value = {
+				raw: node,
+				evaluated: value
+			};
+
+			contextValue = contextValue || contextNode ? {
+				raw: contextNode,
+				evaluated: contextValue
+			} : undefined;
+
+			lang.mixin(value.evaluated.metadata, processor.generateMetadata(value, contextValue) || {});
+		});
 	}
 
 	/**
@@ -58,16 +70,17 @@ define([
 	 * @returns {Value} A function Value.
 	 */
 	function createFunctionValue(/**Node*/ fn) {
-		return new Value({
-			type: Value.TYPE_FUNCTION,
-			parameters: fn.params.map(function (identifier) {
-				return new Parameter({
-					name: identifier.name,
-					comments: getComments(identifier, fn)
-				});
-			}),
-			comments: getComments(fn.body, fn)
+		var functionValue = new Value({ type: Value.TYPE_FUNCTION });
+
+		functionValue.parameters = fn.params.map(function (identifier) {
+			var parameterValue = new Parameter({ name: identifier.name });
+			attachMetadata(parameterValue, identifier, functionValue, fn);
+			return parameterValue;
 		});
+
+		attachMetadata(functionValue, fn);
+
+		return functionValue;
 	}
 
 	/**
@@ -246,21 +259,23 @@ define([
 			}
 
 			if (expression.operator === '=') {
-				// TODO: Avoid losing comments from LHS?
 				env.scope.setVariableValue(lhsIdentifier.map(function (id) { return id.name; }), rhsValue);
+
+				// TODO: Avoid losing metadata from lhsValue?
 				lhsValue = rhsValue;
-				lhsValue.comments = lhsValue.comments.concat(getComments(expression.left), rhsValue.comments);
 			}
 			else {
 				lhsValue = read(expression.left);
-				lhsValue.comments = lhsValue.comments.concat(getComments(expression.left));
 
 				if (!lhsValue) {
 					console.info('Cannot resolve origin value');
-					return new Value({ type: Value.TYPE_UNDEFINED });
+					lhsValue = new Value({ type: Value.TYPE_UNDEFINED });
+				}
+				else {
+					lang.mixin(lhsValue, calculateBinary(lhsValue, expression.operator.replace(/\=$/, ''), rhsValue));
 				}
 
-				lang.mixin(lhsValue, calculateBinary(lhsValue, expression.operator.replace(/\=$/, ''), rhsValue));
+				attachMetadata(lhsValue, expression.left);
 			}
 
 			return lhsValue;
@@ -276,8 +291,7 @@ define([
 			for (var i = 0, j = expression.elements.length, element, elementValue; i < j; ++i) {
 				element = expression.elements[i];
 				elementValue = element === undefined ? new Value({ type: Value.TYPE_UNDEFINED }) : read(element);
-
-				elementValue.comments = elementValue.comments.concat(getComments(element, expression));
+				attachMetadata(elementValue, element, value, expression);
 
 				array.push(elementValue);
 			}
@@ -286,7 +300,7 @@ define([
 			// and a Value object is not a valid array length
 			value.properties = arrayToObject(array);
 
-			value.comments = value.comments.concat(getComments(expression));
+			attachMetadata(value, expression);
 
 			return value;
 		},
@@ -496,6 +510,8 @@ define([
 			value.evaluate = function (/**Array.<Value>*/ args) {
 				var oldScope = env.setScope(value.scope);
 
+				args = args || [];
+
 				for (var i = 0, parameter; (parameter = value.parameters[i]); ++i) {
 					// TODO: Copy metadata from the parameter to the argument value
 					env.scope.setVariableValue(parameter.name, args[i] || parameter);
@@ -599,12 +615,13 @@ define([
 			for (var i = 0, property, propertyValue; (property = expression.properties[i]); ++i) {
 				// key might be a Literal or an Identifier
 				propertyValue = read(property.value);
+				propertyValue.evaluate && propertyValue.evaluate();
 				value.properties[property.key.value || property.key.name] = propertyValue;
 
-				propertyValue.comments = propertyValue.comments.concat(getComments(property, expression));
+				attachMetadata(propertyValue, property, value, expression);
 			}
 
-			value.comments = value.comments.concat(getComments(expression));
+			attachMetadata(value, expression);
 
 			return value;
 		},
@@ -623,7 +640,7 @@ define([
 
 			if (statement.argument) {
 				var value = read(statement.argument);
-				value.comments = value.comments.concat(getComments(statement));
+				attachMetadata(value, statement);
 				env.functionScope.relatedFunction.returns.push(value);
 			}
 		},
@@ -731,7 +748,8 @@ define([
 
 			for (i = 0; (declaration = expression.declarations[i]); ++i) {
 				value = declaration.init ? read(declaration.init) : new Value({ type: Value.TYPE_UNDEFINED });
-				value.comments = value.comments.concat(getComments(declaration, expression));
+				value.evaluate && value.evaluate();
+				attachMetadata(value, declaration, null, expression);
 				scope.setVariableValue(declaration.id.name, value);
 			}
 		},
@@ -798,6 +816,8 @@ define([
 					token.value.push(comment.value);
 					token.range[1] = comment.range[1];
 				} while (nextCommentIsRunOn());
+
+				token.value = token.value.join('\n');
 			}
 
 			return token;