better doc parsing using marked.lexer

lib/tags.js

@@ -290,61 +290,122 @@ var tags = {
        *\/
    */
   doc: function() {
-    var description = this.tag.description.trimRight();
-    var resolvedPath = path.resolve(path.dirname(this.file), description);
+    var file = this.tag.description.trimRight();
+    var resolvedPath = path.resolve(path.dirname(this.file), file);
+    var lexer = new marked.Lexer();
+    var headingSet = false;
+    var description = '';
+    var result, parsedResult, isCode, isHeading, tag;
 
     try {
-      description = fs.readFileSync(resolvedPath, 'utf-8');
+      file = fs.readFileSync(resolvedPath, 'utf-8');
     } catch (e) {
       throw new ReferenceError('File not found \'' + resolvedPath + '\' (' +
         this.file + ':' + (this.tag.line+1) + ')');
     }
 
-    // code modified from mdconf
-    // @see https://github.com/tj/mdconf/blob/master/index.js
-    var toks = marked.lexer(description);
-    toks.forEach(function(tok) {
-      switch(tok.type) {
-        case 'heading':
+    // loop through the file by removing one line at a time and use the marked
+    // rules to look for heading and code blocks
+    //
+    // modified from the marked token parser
+    // @see https://github.com/chjj/marked/blob/master/lib/marked.js#L150
+    while (file.indexOf('\n') !== -1) {
+      isCode = false;
+      isHeading = false;
+
+      // code
+      if (result = lexer.rules.code.exec(file)) {
+        file = file.substring(result[0].length);
+        parsedResult = result[0].replace(/^ {4}/gm, '').replace(/\n+$/, '');
+        isCode = true;
+      }
 
-          // look at the @section tag and set its name if it's not defined
-          this.comment.tags.forEach(function(tag) {
-            if (tag.tag === 'section' && !tag.description) {
-              tag.description = tok.text;
-            }
-          }.bind(this));
-          break;
+      // fences (gfm)
+      else if (result = lexer.rules.fences.exec(file)) {
+        file = file.substring(result[0].length);
+        parsedResult = result[3];
+        isCode = true;
+      }
 
-        case 'code':
-          var tag;
+      // heading
+      else if (result = lexer.rules.heading.exec(file)) {
+        file = file.substring(result[0].length);
+        parsedResult = result[2];
+        isHeading = true;
+      }
 
-          if (tok.text.indexOf('@example') === 0) {
-            tag = 'example';
-          }
-          else if (tok.text.indexOf('@code') === 0) {
-            tag = 'code';
-          }
+      // lheading
+      else if (result = lexer.rules.lheading.exec(file)) {
+        file = file.substring(result[0].length);
+        parsedResult = result[1];
+        isHeading = true;
+      }
+
+      // go to next line
+      else if (file.indexOf('\n') !== -1) {
+        description += file.substring(0, file.indexOf('\n')+1);
+        file = file.substring(file.indexOf('\n')+1);
+      }
+
+      // handle code being able to create example or code tags
+      if (isCode) {
+        if (parsedResult.indexOf('@example') === 0) {
+          tag = 'example';
+        }
+        else if (parsedResult.indexOf('@code') === 0) {
+          tag = 'code';
+        }
 
-          // a code block without the @example or @code tag will just be added to
-          // the description wrapped in markdown code syntax
-          else {
-            this.comment.description += '\n\n```' + (tok.lang ? tok.lang : '') + '\n' + tok.text + '```';
-            break;
+        // any code block that is not @example or @code will be added to the
+        // description
+        else {
+          description += result[0];
+          continue;
+        }
+
+        this.comment.tags.push({
+          tag: tag,
+          type: (result[2] ? result[2] : ''),
+          description: parsedResult.replace('@' + tag, '').trimLeft()
+        });
+      }
+
+      // handle heading setting the section name
+      else if (isHeading) {
+
+        // if section name is already set then just add this heading to the
+        // description
+        if (headingSet) {
+          description += result[0];
+        }
+
+        else {
+          // look at the @section tag and set its name if it's not defined
+          for (var i = 0; (tag = this.comment.tags[i]); i++) {
+            if (tag.tag === 'section') {
+              if (!tag.description) {
+                tag.description = parsedResult;
+              }
+
+              // if section name is already set then just add this heading to the
+              // description
+              else {
+                description += result[0];
+              }
+
+              break;
+            }
           }
 
-          // generate tag
-          this.comment.tags.push({
-            tag: tag,
-            description: tok.text.replace('@' + tag, '').trimLeft()
-          });
-          break;
-
-        // everything else will be added as part of the description
-        default:
-          this.comment.description += '\n\n' + tok.text;
-          break;
+          headingSet = true;
+        }
       }
-    }.bind(this));
+    }
+
+    // add last line to description
+    description += file;
+
+    this.comment.description += description.trimRight();
   }
 };
 tags.code = tags.example;  // @code and @example generate the same structure

test/data/doc.md

@@ -4,11 +4,16 @@
 
 Description of the section.
 
+## Secondary Heading
+
+- list item 1
+- list item 2
+
 ```
 Code in the description
 ```
 
-```
+```html
 @example
 <div>foobar</div>
 ```

test/tags.spec.js

@@ -557,7 +557,7 @@ describe('tags', function() {
 
         parseComments(data, file, tags, {sections: sections, pages: pages});
 
-        expect(sections[0].description).to.equal('<p>Description of the section.</p>\n<pre><code>Code in the description\n</code></pre>');
+        expect(sections[0].description).to.equal('<p>Description of the section.</p>\n<h2 id="secondary-heading">Secondary Heading</h2>\n<ul>\n<li>list item 1</li>\n<li>list item 2</li>\n</ul>\n<pre><code>Code in the description\n</code></pre>');
 
         done();
       });