replace custom showdown fork with markdown-it (#8451)

refs TryGhost/Admin#690, closes #1501, closes #2093, closes #4592, closes #4627, closes #4659, closes #5039, closes #5237, closes #5587, closes #5625, closes #5632, closes #5822, closes #5939, closes #6840, closes #7183, closes #7536

- replace custom showdown fork with markdown-it
- swaps showdown for markdown-it when rendering markdown
- match existing header ID behaviour
- allow headers without a space after the #s
- add duplicate header ID handling
- remove legacy markdown spec
- move markdown-it setup into markdown-converter util
- update mobiledoc specs to match markdown-it newline behaviour
- update data-generator HTML to match markdown-it newline behaviour
- fix Post "converts html to plaintext" test
- update rss spec to match markdown-it newline behaviour
- close almost all related showdown bugs

core/server/apps/default-cards/cards/markdown.js

@@ -1,14 +1,17 @@
 var SimpleDom   = require('simple-dom'),
     tokenizer   = require('simple-html-tokenizer').tokenize,
-    Showdown    = require('showdown-ghost'),
-    converter   = new Showdown.converter({extensions: ['ghostgfm', 'footnotes', 'highlight']}),
+    markdownConverter  = require('../../../utils/markdown-converter'),
     parser;
 
 module.exports = {
         name: 'card-markdown',
         type: 'dom',
         render(opts) {
             parser = new SimpleDom.HTMLParser(tokenizer, opts.env.dom, SimpleDom.voidMap);
-            return parser.parse('<div class="kg-card-markdown">' + converter.makeHtml(opts.payload.markdown || '') + '</div>');
+            return parser.parse(''
+                + '<div class="kg-card-markdown">'
+                + markdownConverter.render(opts.payload.markdown || '')
+                + '</div>'
+            );
         }
     };

core/server/apps/default-cards/tests/markdown_spec.js

@@ -15,6 +15,6 @@ describe('Markdown card', function () {
         };
 
         var serializer = new SimpleDom.HTMLSerializer([]);
-        serializer.serialize(card.render(opts)).should.match('<div class="kg-card-markdown"><h1 id="heading">HEADING</h1>\n\n<ul>\n<li>list</li>\n<li>items</li>\n</ul></div>');
+        serializer.serialize(card.render(opts)).should.match('<div class="kg-card-markdown"><h1 id="heading">HEADING</h1>\n<ul>\n<li>list</li>\n<li>items</li>\n</ul>\n</div>');
     });
 });

core/server/models/post.js

@@ -5,8 +5,7 @@ var _               = require('lodash'),
     Promise         = require('bluebird'),
     sequence        = require('../utils/sequence'),
     errors          = require('../errors'),
-    Showdown        = require('showdown-ghost'),
-    legacyConverter = new Showdown.converter({extensions: ['ghostgfm', 'footnotes', 'highlight']}),
+    legacyConverter = require('../utils/markdown-converter'),
     htmlToText      = require('html-to-text'),
     ghostBookshelf  = require('./base'),
     events          = require('../events'),
@@ -234,11 +233,11 @@ Post = ghostBookshelf.Model.extend({
         if (mobiledoc) {
             this.set('html', utils.mobiledocConverter.render(JSON.parse(mobiledoc)));
         } else {
-            // legacy showdown mode
-            this.set('html', legacyConverter.makeHtml(_.toString(this.get('markdown'))));
+            // legacy markdown mode
+            this.set('html', legacyConverter.render(_.toString(this.get('markdown'))));
         }
 
-        if (this.hasChanged('html')) {
+        if (this.hasChanged('html') || !this.get('plaintext')) {
             this.set('plaintext', htmlToText.fromString(this.get('html'), {
                 wordwrap: 80,
                 ignoreImage: true,

core/server/utils/index.js

@@ -112,7 +112,8 @@ utils = {
     tokens: require('./tokens'),
     sequence: require('./sequence'),
     ghostVersion: require('./ghost-version'),
-    mobiledocConverter: require('./mobiledoc-converter')
+    mobiledocConverter: require('./mobiledoc-converter'),
+    markdownConverter: require('./markdown-converter')
 };
 
 module.exports = utils;

core/server/utils/markdown-converter.js

@@ -0,0 +1,26 @@
+var MarkdownIt  = require('markdown-it'),
+    converter = new MarkdownIt({
+        html: true,
+        breaks: true,
+        linkify: true
+    })
+    .use(require('markdown-it-footnote'))
+    .use(require('markdown-it-lazy-headers'))
+    .use(require('markdown-it-mark'))
+    .use(require('markdown-it-named-headers'), {
+        // match legacy Showdown IDs otherwise default is github style dasherized
+        slugify: function (inputString, usedHeaders) {
+            var slug = inputString.replace(/[^\w]/g, '').toLowerCase();
+            if (usedHeaders[slug]) {
+                usedHeaders[slug] += 1;
+                slug += usedHeaders[slug];
+            }
+            return slug;
+        }
+    });
+
+module.exports = {
+    render: function (markdown) {
+        return converter.render(markdown);
+    }
+};

core/test/unit/rss_spec.js

@@ -167,7 +167,7 @@ describe('RSS', function () {
                 // item tags
                 xmlData.should.match(/<title><!\[CDATA\[Short and Sweet\]\]>/);
                 xmlData.should.match(/<description><!\[CDATA\[test stuff/);
-                xmlData.should.match(/<content:encoded><!\[CDATA\[<h2 id="testing">testing<\/h2>\n\n/);
+                xmlData.should.match(/<content:encoded><!\[CDATA\[<h2 id="testing">testing<\/h2>\n/);
                 xmlData.should.match(/<img src="http:\/\/placekitten.com\/500\/200"/);
                 xmlData.should.match(/<media:content url="http:\/\/placekitten.com\/500\/200" medium="image"\/>/);
                 xmlData.should.match(/<category><!\[CDATA\[public\]\]/);
@@ -197,7 +197,7 @@ describe('RSS', function () {
                 // special/optional tags
                 xmlData.should.match(/<title><!\[CDATA\[Short and Sweet\]\]>/);
                 xmlData.should.match(/<description><!\[CDATA\[test stuff/);
-                xmlData.should.match(/<content:encoded><!\[CDATA\[<h2 id="testing">testing<\/h2>\n\n/);
+                xmlData.should.match(/<content:encoded><!\[CDATA\[<h2 id="testing">testing<\/h2>\n/);
                 xmlData.should.match(/<img src="http:\/\/placekitten.com\/500\/200"/);
                 xmlData.should.match(/<media:content url="http:\/\/placekitten.com\/500\/200" medium="image"\/>/);