Expand tabs only when it's needed, as per CommonMark 0.21

lib/common/utils.js

@@ -151,6 +151,15 @@ function escapeRE (str) {
 
 ////////////////////////////////////////////////////////////////////////////////
 
+function isSpace(code) {
+  switch (code) {
+    case 0x09:
+    case 0x20:
+      return true;
+  }
+  return false;
+}
+
 // Zs (unicode class) || [\t\f\v\r\n]
 function isWhiteSpace(code) {
   if (code >= 0x2000 && code <= 0x200A) { return true; }
@@ -258,6 +267,7 @@ exports.fromCodePoint       = fromCodePoint;
 // exports.replaceEntities     = replaceEntities;
 exports.escapeHtml          = escapeHtml;
 exports.arrayReplaceAt      = arrayReplaceAt;
+exports.isSpace             = isSpace;
 exports.isWhiteSpace        = isWhiteSpace;
 exports.isMdAsciiPunct      = isMdAsciiPunct;
 exports.isPunctChar         = isPunctChar;

lib/parser_block.js

@@ -59,7 +59,7 @@ ParserBlock.prototype.tokenize = function (state, startLine, endLine) {
 
     // Termination condition for nested calls.
     // Nested calls currently used for blockquotes & lists
-    if (state.tShift[line] < state.blkIndent) { break; }
+    if (state.sCount[line] < state.blkIndent) { break; }
 
     // If nesting level exceeded - skip tail to the end. That's not ordinary
     // situation and we should not care about content.

lib/rules_block/blockquote.js

@@ -2,9 +2,11 @@
 
 'use strict';
 
+var isSpace = require('../common/utils').isSpace;
+
 
 module.exports = function blockquote(state, startLine, endLine, silent) {
-  var nextLine, lastLineEmpty, oldTShift, oldBMarks, oldIndent, oldParentType, lines,
+  var nextLine, lastLineEmpty, oldTShift, oldSCount, oldBMarks, oldIndent, oldParentType, lines, initial, offset, ch,
       terminatorRules, token,
       i, l, terminate,
       pos = state.bMarks[startLine] + state.tShift[startLine],
@@ -17,19 +19,39 @@ module.exports = function blockquote(state, startLine, endLine, silent) {
   // so no point trying to find the end of it in silent mode
   if (silent) { return true; }
 
-  // skip one optional space after '>'
+  // skip one optional space (but not tab, check cmark impl) after '>'
   if (state.src.charCodeAt(pos) === 0x20) { pos++; }
 
   oldIndent = state.blkIndent;
   state.blkIndent = 0;
 
+  // skip spaces after ">" and re-calculate offset
+  initial = offset = state.sCount[startLine] + pos - (state.bMarks[startLine] + state.tShift[startLine]);
+
   oldBMarks = [ state.bMarks[startLine] ];
   state.bMarks[startLine] = pos;
 
-  // check if we have an empty blockquote
-  pos = pos < max ? state.skipSpaces(pos) : pos;
+  while (pos < max) {
+    ch = state.src.charCodeAt(pos);
+
+    if (isSpace(ch)) {
+      if (ch === 0x09) {
+        offset += 4 - offset % 4;
+      } else {
+        offset++;
+      }
+    } else {
+      break;
+    }
+
+    pos++;
+  }
+
   lastLineEmpty = pos >= max;
 
+  oldSCount = [ state.sCount[startLine] ];
+  state.sCount[startLine] = offset - initial;
+
   oldTShift = [ state.tShift[startLine] ];
   state.tShift[startLine] = pos - state.bMarks[startLine];
 
@@ -54,7 +76,7 @@ module.exports = function blockquote(state, startLine, endLine, silent) {
   //      - - -
   //     ```
   for (nextLine = startLine + 1; nextLine < endLine; nextLine++) {
-    if (state.tShift[nextLine] < oldIndent) { break; }
+    if (state.sCount[nextLine] < oldIndent) { break; }
 
     pos = state.bMarks[nextLine] + state.tShift[nextLine];
     max = state.eMarks[nextLine];
@@ -67,15 +89,36 @@ module.exports = function blockquote(state, startLine, endLine, silent) {
     if (state.src.charCodeAt(pos++) === 0x3E/* > */) {
       // This line is inside the blockquote.
 
-      // skip one optional space after '>'
+      // skip one optional space (but not tab, check cmark impl) after '>'
       if (state.src.charCodeAt(pos) === 0x20) { pos++; }
 
+      // skip spaces after ">" and re-calculate offset
+      initial = offset = state.sCount[nextLine] + pos - (state.bMarks[nextLine] + state.tShift[nextLine]);
+
       oldBMarks.push(state.bMarks[nextLine]);
       state.bMarks[nextLine] = pos;
 
-      pos = pos < max ? state.skipSpaces(pos) : pos;
+      while (pos < max) {
+        ch = state.src.charCodeAt(pos);
+
+        if (isSpace(ch)) {
+          if (ch === 0x09) {
+            offset += 4 - offset % 4;
+          } else {
+            offset++;
+          }
+        } else {
+          break;
+        }
+
+        pos++;
+      }
+
       lastLineEmpty = pos >= max;
 
+      oldSCount.push(state.sCount[nextLine]);
+      state.sCount[nextLine] = offset - initial;
+
       oldTShift.push(state.tShift[nextLine]);
       state.tShift[nextLine] = pos - state.bMarks[nextLine];
       continue;
@@ -96,12 +139,11 @@ module.exports = function blockquote(state, startLine, endLine, silent) {
 
     oldBMarks.push(state.bMarks[nextLine]);
     oldTShift.push(state.tShift[nextLine]);
+    oldSCount.push(state.sCount[nextLine]);
 
-    // A negative number means that this is a paragraph continuation;
+    // A negative indentation means that this is a paragraph continuation
     //
-    // Any negative number will do the job here, but it's better for it
-    // to be large enough to make any bugs obvious.
-    state.tShift[nextLine] = -1;
+    state.sCount[nextLine] = -1;
   }
 
   oldParentType = state.parentType;
@@ -124,6 +166,7 @@ module.exports = function blockquote(state, startLine, endLine, silent) {
   for (i = 0; i < oldTShift.length; i++) {
     state.bMarks[i + startLine] = oldBMarks[i];
     state.tShift[i + startLine] = oldTShift[i];
+    state.sCount[i + startLine] = oldSCount[i];
   }
   state.blkIndent = oldIndent;
 

lib/rules_block/code.js

@@ -6,7 +6,7 @@
 module.exports = function code(state, startLine, endLine/*, silent*/) {
   var nextLine, last, token;
 
-  if (state.tShift[startLine] - state.blkIndent < 4) { return false; }
+  if (state.sCount[startLine] - state.blkIndent < 4) { return false; }
 
   last = nextLine = startLine + 1;
 
@@ -15,7 +15,7 @@ module.exports = function code(state, startLine, endLine/*, silent*/) {
       nextLine++;
       continue;
     }
-    if (state.tShift[nextLine] - state.blkIndent >= 4) {
+    if (state.sCount[nextLine] - state.blkIndent >= 4) {
       nextLine++;
       last = nextLine;
       continue;

lib/rules_block/fence.js

@@ -47,7 +47,7 @@ module.exports = function fence(state, startLine, endLine, silent) {
     pos = mem = state.bMarks[nextLine] + state.tShift[nextLine];
     max = state.eMarks[nextLine];
 
-    if (pos < max && state.tShift[nextLine] < state.blkIndent) {
+    if (pos < max && state.sCount[nextLine] < state.blkIndent) {
       // non-empty line with negative indent should stop the list:
       // - ```
       //  test
@@ -56,7 +56,7 @@ module.exports = function fence(state, startLine, endLine, silent) {
 
     if (state.src.charCodeAt(pos) !== marker) { continue; }
 
-    if (state.tShift[nextLine] - state.blkIndent >= 4) {
+    if (state.sCount[nextLine] - state.blkIndent >= 4) {
       // closing fence should be indented less than 4 spaces
       continue;
     }
@@ -77,7 +77,7 @@ module.exports = function fence(state, startLine, endLine, silent) {
   }
 
   // If a fence has heading spaces, they should be removed from its inner block
-  len = state.tShift[startLine];
+  len = state.sCount[startLine];
 
   state.line = nextLine + (haveEndMarker ? 1 : 0);
 

lib/rules_block/heading.js

@@ -2,6 +2,8 @@
 
 'use strict';
 
+var isSpace = require('../common/utils').isSpace;
+
 
 module.exports = function heading(state, startLine, endLine, silent) {
   var ch, level, tmp, token,
@@ -26,9 +28,9 @@ module.exports = function heading(state, startLine, endLine, silent) {
 
   // Let's cut tails like '    ###  ' from the end of string
 
-  max = state.skipCharsBack(max, 0x20, pos); // space
+  max = state.skipSpacesBack(max, pos);
   tmp = state.skipCharsBack(max, 0x23, pos); // #
-  if (tmp > pos && state.src.charCodeAt(tmp - 1) === 0x20/* space */) {
+  if (tmp > pos && isSpace(state.src.charCodeAt(tmp - 1))) {
     max = tmp;
   }
 

lib/rules_block/hr.js

@@ -2,6 +2,8 @@
 
 'use strict';
 
+var isSpace = require('../common/utils').isSpace;
+
 
 module.exports = function hr(state, startLine, endLine, silent) {
   var marker, cnt, ch, token,
@@ -17,12 +19,12 @@ module.exports = function hr(state, startLine, endLine, silent) {
     return false;
   }
 
-  // markers can be mixed with spaces, but there should be at least 3 one
+  // markers can be mixed with spaces, but there should be at least 3 of them
 
   cnt = 1;
   while (pos < max) {
     ch = state.src.charCodeAt(pos++);
-    if (ch !== marker && ch !== 0x20/* space */) { return false; }
+    if (ch !== marker && !isSpace(ch)) { return false; }
     if (ch === marker) { cnt++; }
   }
 

lib/rules_block/html_block.js

@@ -48,7 +48,7 @@ module.exports = function html_block(state, startLine, endLine, silent) {
   // Let's roll down till block end.
   if (!HTML_SEQUENCES[i][1].test(lineText)) {
     for (; nextLine < endLine; nextLine++) {
-      if (state.tShift[nextLine] < state.blkIndent) { break; }
+      if (state.sCount[nextLine] < state.blkIndent) { break; }
 
       pos = state.bMarks[nextLine] + state.tShift[nextLine];
       max = state.eMarks[nextLine];

lib/rules_block/lheading.js

@@ -8,11 +8,11 @@ module.exports = function lheading(state, startLine, endLine/*, silent*/) {
       next = startLine + 1;
 
   if (next >= endLine) { return false; }
-  if (state.tShift[next] < state.blkIndent) { return false; }
+  if (state.sCount[next] < state.blkIndent) { return false; }
 
   // Scan next line
 
-  if (state.tShift[next] - state.blkIndent > 3) { return false; }
+  if (state.sCount[next] - state.blkIndent > 3) { return false; }
 
   pos = state.bMarks[next] + state.tShift[next];
   max = state.eMarks[next];