From 505a12a3bcfae4c794810e1c22ef02744ab6f62b Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 5 Nov 2018 10:54:20 -0600 Subject: [PATCH 01/26] extract markdownNodeAST function --- .../src/extend-node-type.js | 185 +++++++++--------- 1 file changed, 95 insertions(+), 90 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 07827d42b1458..b5cb2d0ea533c 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -121,97 +121,8 @@ module.exports = ( if (process.env.NODE_ENV !== `production` || !fileNodes) { fileNodes = getNodesByType(`File`) } - const ast = await new Promise((resolve, reject) => { - // Use Bluebird's Promise function "each" to run remark plugins serially. - Promise.each(pluginOptions.plugins, plugin => { - const requiredPlugin = require(plugin.resolve) - if (_.isFunction(requiredPlugin.mutateSource)) { - return requiredPlugin.mutateSource( - { - markdownNode, - files: fileNodes, - getNode, - reporter, - cache, - }, - plugin.pluginOptions - ) - } else { - return Promise.resolve() - } - }).then(() => { - const markdownAST = remark.parse(markdownNode.internal.content) - - if (pathPrefix) { - // Ensure relative links include `pathPrefix` - visit(markdownAST, [`link`, `definition`], node => { - if ( - node.url && - node.url.startsWith(`/`) && - !node.url.startsWith(`//`) - ) { - node.url = withPathPrefix(node.url, pathPrefix) - } - }) - } - // source => parse (can order parsing for dependencies) => typegen - // - // source plugins identify nodes, provide id, initial parse, know - // when nodes are created/removed/deleted - // get passed cached DataTree and return list of clean and dirty nodes. - // Also get passed `dirtyNodes` function which they can call with an array - // of node ids which will then get re-parsed and the inferred schema - // recreated (if inferring schema gets too expensive, can also - // cache the schema until a query fails at which point recreate the - // schema). - // - // parse plugins take data from source nodes and extend it, never mutate - // it. Freeze all nodes once done so typegen plugins can't change it - // this lets us save off the DataTree at that point as well as create - // indexes. - // - // typegen plugins identify further types of data that should be lazily - // computed due to their expense, or are hard to infer graphql type - // (markdown ast), or are need user input in order to derive e.g. - // markdown headers or date fields. - // - // wrap all resolve functions to (a) auto-memoize and (b) cache to disk any - // resolve function that takes longer than ~10ms (do research on this - // e.g. how long reading/writing to cache takes), and (c) track which - // queries are based on which source nodes. Also if connection of what - // which are always rerun if their underlying nodes change.. - // - // every node type in DataTree gets a schema type automatically. - // typegen plugins just modify the auto-generated types to add derived fields - // as well as computationally expensive fields. - if (process.env.NODE_ENV !== `production` || !fileNodes) { - fileNodes = getNodesByType(`File`) - } - // Use Bluebird's Promise function "each" to run remark plugins serially. - Promise.each(pluginOptions.plugins, plugin => { - const requiredPlugin = require(plugin.resolve) - if (_.isFunction(requiredPlugin)) { - return requiredPlugin( - { - markdownAST, - markdownNode, - getNode, - files: fileNodes, - pathPrefix, - reporter, - cache, - }, - plugin.pluginOptions - ) - } else { - return Promise.resolve() - } - }).then(() => { - resolve(markdownAST) - }) - }) - }) + const ast = await getMarkdownAST(markdownNode) // Save new AST to cache and return cache.set(cacheKey, ast) @@ -224,6 +135,100 @@ module.exports = ( } } + function getMarkdownAST(markdownNode) { + return new Promise((resolve, reject) => { + // Use Bluebird's Promise function "each" to run remark plugins serially. + Promise.each(pluginOptions.plugins, plugin => { + const requiredPlugin = require(plugin.resolve) + if (_.isFunction(requiredPlugin.mutateSource)) { + return requiredPlugin.mutateSource( + { + markdownNode, + files: fileNodes, + getNode, + reporter, + cache, + }, + plugin.pluginOptions + ) + } else { + return Promise.resolve() + } + }).then(() => { + const markdownAST = remark.parse(markdownNode.internal.content) + + if (pathPrefix) { + // Ensure relative links include `pathPrefix` + visit(markdownAST, [`link`, `definition`], node => { + if ( + node.url && + node.url.startsWith(`/`) && + !node.url.startsWith(`//`) + ) { + node.url = withPathPrefix(node.url, pathPrefix) + } + }) + } + + // source => parse (can order parsing for dependencies) => typegen + // + // source plugins identify nodes, provide id, initial parse, know + // when nodes are created/removed/deleted + // get passed cached DataTree and return list of clean and dirty nodes. + // Also get passed `dirtyNodes` function which they can call with an array + // of node ids which will then get re-parsed and the inferred schema + // recreated (if inferring schema gets too expensive, can also + // cache the schema until a query fails at which point recreate the + // schema). + // + // parse plugins take data from source nodes and extend it, never mutate + // it. Freeze all nodes once done so typegen plugins can't change it + // this lets us save off the DataTree at that point as well as create + // indexes. + // + // typegen plugins identify further types of data that should be lazily + // computed due to their expense, or are hard to infer graphql type + // (markdown ast), or are need user input in order to derive e.g. + // markdown headers or date fields. + // + // wrap all resolve functions to (a) auto-memoize and (b) cache to disk any + // resolve function that takes longer than ~10ms (do research on this + // e.g. how long reading/writing to cache takes), and (c) track which + // queries are based on which source nodes. Also if connection of what + // which are always rerun if their underlying nodes change.. + // + // every node type in DataTree gets a schema type automatically. + // typegen plugins just modify the auto-generated types to add derived fields + // as well as computationally expensive fields. + if (process.env.NODE_ENV !== `production` || !fileNodes) { + fileNodes = getNodesByType(`File`) + } + // Use Bluebird's Promise function "each" to run remark plugins serially. + Promise.each(pluginOptions.plugins, plugin => { + const requiredPlugin = require(plugin.resolve) + if (_.isFunction(requiredPlugin)) { + return requiredPlugin( + { + markdownAST, + markdownNode, + getNode, + files: fileNodes, + pathPrefix, + reporter, + cache, + }, + plugin.pluginOptions + ) + } else { + return Promise.resolve() + } + }).then(() => { + resolve(markdownAST) + }) + }) + }) + } + async function getHeadings(markdownNode) { const cachedHeadings = await cache.get(headingsCacheKey(markdownNode)) if (cachedHeadings) { From 099fcd0231c1975653fb997f148b0bb73f001770 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 5 Nov 2018 10:55:50 -0600 Subject: [PATCH 02/26] use async/await better --- .../src/extend-node-type.js | 138 +++++++++--------- 1 file changed, 68 insertions(+), 70 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index b5cb2d0ea533c..73cc8d8beae9d 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -136,9 +136,9 @@ module.exports = ( } function getMarkdownAST(markdownNode) { - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { // Use Bluebird's Promise function "each" to run remark plugins serially. - Promise.each(pluginOptions.plugins, plugin => { + await Promise.each(pluginOptions.plugins, plugin => { const requiredPlugin = require(plugin.resolve) if (_.isFunction(requiredPlugin.mutateSource)) { return requiredPlugin.mutateSource( @@ -154,78 +154,76 @@ module.exports = ( } else { return Promise.resolve() } - }).then(() => { - const markdownAST = remark.parse(markdownNode.internal.content) - - if (pathPrefix) { - // Ensure relative links include `pathPrefix` - visit(markdownAST, [`link`, `definition`], node => { - if ( - node.url && - node.url.startsWith(`/`) && - !node.url.startsWith(`//`) - ) { - node.url = withPathPrefix(node.url, pathPrefix) - } - }) - } + }) + const markdownAST = remark.parse(markdownNode.internal.content) - // source => parse (can order parsing for dependencies) => typegen - // - // source plugins identify nodes, provide id, initial parse, know - // when nodes are created/removed/deleted - // get passed cached DataTree and return list of clean and dirty nodes. - // Also get passed `dirtyNodes` function which they can call with an array - // of node ids which will then get re-parsed and the inferred schema - // recreated (if inferring schema gets too expensive, can also - // cache the schema until a query fails at which point recreate the - // schema). - // - // parse plugins take data from source nodes and extend it, never mutate - // it. Freeze all nodes once done so typegen plugins can't change it - // this lets us save off the DataTree at that point as well as create - // indexes. - // - // typegen plugins identify further types of data that should be lazily - // computed due to their expense, or are hard to infer graphql type - // (markdown ast), or are need user input in order to derive e.g. - // markdown headers or date fields. - // - // wrap all resolve functions to (a) auto-memoize and (b) cache to disk any - // resolve function that takes longer than ~10ms (do research on this - // e.g. how long reading/writing to cache takes), and (c) track which - // queries are based on which source nodes. Also if connection of what - // which are always rerun if their underlying nodes change.. - // - // every node type in DataTree gets a schema type automatically. - // typegen plugins just modify the auto-generated types to add derived fields - // as well as computationally expensive fields. - if (process.env.NODE_ENV !== `production` || !fileNodes) { - fileNodes = getNodesByType(`File`) - } - // Use Bluebird's Promise function "each" to run remark plugins serially. - Promise.each(pluginOptions.plugins, plugin => { - const requiredPlugin = require(plugin.resolve) - if (_.isFunction(requiredPlugin)) { - return requiredPlugin( - { - markdownAST, - markdownNode, - getNode, - files: fileNodes, - pathPrefix, - reporter, - cache, - }, - plugin.pluginOptions - ) - } else { - return Promise.resolve() + if (pathPrefix) { + // Ensure relative links include `pathPrefix` + visit(markdownAST, [`link`, `definition`], node => { + if ( + node.url && + node.url.startsWith(`/`) && + !node.url.startsWith(`//`) + ) { + node.url = withPathPrefix(node.url, pathPrefix) } - }).then(() => { - resolve(markdownAST) }) + } + + // source => parse (can order parsing for dependencies) => typegen + // + // source plugins identify nodes, provide id, initial parse, know + // when nodes are created/removed/deleted + // get passed cached DataTree and return list of clean and dirty nodes. + // Also get passed `dirtyNodes` function which they can call with an array + // of node ids which will then get re-parsed and the inferred schema + // recreated (if inferring schema gets too expensive, can also + // cache the schema until a query fails at which point recreate the + // schema). + // + // parse plugins take data from source nodes and extend it, never mutate + // it. Freeze all nodes once done so typegen plugins can't change it + // this lets us save off the DataTree at that point as well as create + // indexes. + // + // typegen plugins identify further types of data that should be lazily + // computed due to their expense, or are hard to infer graphql type + // (markdown ast), or are need user input in order to derive e.g. + // markdown headers or date fields. + // + // wrap all resolve functions to (a) auto-memoize and (b) cache to disk any + // resolve function that takes longer than ~10ms (do research on this + // e.g. how long reading/writing to cache takes), and (c) track which + // queries are based on which source nodes. Also if connection of what + // which are always rerun if their underlying nodes change.. + // + // every node type in DataTree gets a schema type automatically. + // typegen plugins just modify the auto-generated types to add derived fields + // as well as computationally expensive fields. + if (process.env.NODE_ENV !== `production` || !fileNodes) { + fileNodes = getNodesByType(`File`) + } + // Use Bluebird's Promise function "each" to run remark plugins serially. + await Promise.each(pluginOptions.plugins, plugin => { + const requiredPlugin = require(plugin.resolve) + if (_.isFunction(requiredPlugin)) { + return requiredPlugin( + { + markdownAST, + markdownNode, + getNode, + files: fileNodes, + pathPrefix, + reporter, + cache, + }, + plugin.pluginOptions + ) + } else { + return Promise.resolve() + } }) + resolve(markdownAST) }) } From dea318a07763535eb42f97ee2e81b8ce70d703f9 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 5 Nov 2018 11:08:36 -0600 Subject: [PATCH 03/26] delete unncessary promise to make code more readable --- .../src/extend-node-type.js | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 73cc8d8beae9d..13856492eb5cf 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -117,26 +117,26 @@ module.exports = ( // We are already generating AST, so let's wait for it return await ASTPromiseMap.get(cacheKey) } else { - const ASTGenerationPromise = new Promise(async resolve => { - if (process.env.NODE_ENV !== `production` || !fileNodes) { - fileNodes = getNodesByType(`File`) - } - - const ast = await getMarkdownAST(markdownNode) - - // Save new AST to cache and return - cache.set(cacheKey, ast) - // We can now release promise, as we cached result + const ASTGenerationPromise = getMarkdownAST(markdownNode) + ASTGenerationPromise.then(markdownAST => { + cache.set(cacheKey, markdownAST) + ASTPromiseMap.delete(cacheKey) + }).catch(err => { ASTPromiseMap.delete(cacheKey) - return resolve(ast) + throw err }) + // Save new AST to cache and return + // We can now release promise, as we cached result ASTPromiseMap.set(cacheKey, ASTGenerationPromise) - return await ASTGenerationPromise + return ASTGenerationPromise } } function getMarkdownAST(markdownNode) { return new Promise(async (resolve, reject) => { + if (process.env.NODE_ENV !== `production` || !fileNodes) { + fileNodes = getNodesByType(`File`) + } // Use Bluebird's Promise function "each" to run remark plugins serially. await Promise.each(pluginOptions.plugins, plugin => { const requiredPlugin = require(plugin.resolve) From 87a421dfc42d433911d601b5459a0e208982bca1 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 5 Nov 2018 11:19:41 -0600 Subject: [PATCH 04/26] extract remark target parameter --- .../gatsby-transformer-remark/src/extend-node-type.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 13856492eb5cf..5bdbef29ece60 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -117,7 +117,10 @@ module.exports = ( // We are already generating AST, so let's wait for it return await ASTPromiseMap.get(cacheKey) } else { - const ASTGenerationPromise = getMarkdownAST(markdownNode) + const ASTGenerationPromise = getMarkdownAST( + markdownNode, + markdownNode.internal.content + ) ASTGenerationPromise.then(markdownAST => { cache.set(cacheKey, markdownAST) ASTPromiseMap.delete(cacheKey) @@ -132,7 +135,7 @@ module.exports = ( } } - function getMarkdownAST(markdownNode) { + function getMarkdownAST(markdownNode, remarkTarget) { return new Promise(async (resolve, reject) => { if (process.env.NODE_ENV !== `production` || !fileNodes) { fileNodes = getNodesByType(`File`) @@ -155,7 +158,7 @@ module.exports = ( return Promise.resolve() } }) - const markdownAST = remark.parse(markdownNode.internal.content) + const markdownAST = remark.parse(remarkTarget) if (pathPrefix) { // Ensure relative links include `pathPrefix` From af59b305879d72c83ddb043aa4e466b211054da0 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 5 Nov 2018 11:36:16 -0600 Subject: [PATCH 05/26] update tests and return html excerpt --- .../src/__tests__/extend-node.js | 2 +- .../src/extend-node-type.js | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 53b3eeca6b783..2d43e7d1d0040 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -183,7 +183,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt).toMatch(`Where oh where is my little pony?`) + expect(node.excerpt).toMatch(`

Where oh where is my little pony?

`) }, { excerpt_separator: `` } ) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 5bdbef29ece60..02d11bc782853 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -380,9 +380,18 @@ module.exports = ( defaultValue: false, }, }, - resolve(markdownNode, { pruneLength, truncate }) { + resolve: async (markdownNode, { pruneLength, truncate }) => { if (markdownNode.excerpt) { - return Promise.resolve(markdownNode.excerpt) + const excerpt = await getMarkdownAST( + markdownNode, + markdownNode.excerpt + ) + const htmlAST = toHAST(excerpt, { allowDangerousHTML: true }) + const html = hastToHTML(htmlAST, { + allowDangerousHTML: true, + }) + + return html } return getAST(markdownNode).then(ast => { const excerptNodes = [] From 385f2abbcb73dc4cc6f37923f24c527bebebba49 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 5 Nov 2018 11:52:02 -0600 Subject: [PATCH 06/26] update snapshots --- .../src/__tests__/__snapshots__/extend-node.js.snap | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap index 4372b594b1f07..35d67762926c5 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap +++ b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap @@ -47,8 +47,7 @@ Object { exports[`Excerpt is generated correctly from schema correctly uses excerpt separator 1`] = ` Object { - "excerpt": "Where oh where is my little pony? -", + "excerpt": "

Where oh where is my little pony?

", "frontmatter": Object { "title": "my little pony", }, From 1613bbe17e9b2cd0b0896360da7bab5a963de4cc Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 5 Nov 2018 18:34:47 -0600 Subject: [PATCH 07/26] begin refactoring code to grab exerpts in the case where no separator is provided --- .../src/extend-node-type.js | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 02d11bc782853..0204a941d12dd 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -393,21 +393,25 @@ module.exports = ( return html } - return getAST(markdownNode).then(ast => { - const excerptNodes = [] - visit(ast, node => { - if (node.type === `text` || node.type === `inlineCode`) { - excerptNodes.push(node.value) - } + + const markdownAST = await getAST(markdownNode) + const excerptNodes = [] + const getExcerptString = () => + excerptNodes.map(node => node.value).join(` `) + visit(markdownAST, node => { + if (getExcerptString().length > pruneLength) { return - }) - if (!truncate) { - return prune(excerptNodes.join(` `), pruneLength, `…`) } - return _.truncate(excerptNodes.join(` `), { - length: pruneLength, - omission: `…`, - }) + if (node.type === `text` || node.type === `inlineCode`) { + excerptNodes.push(node) + } + }) + if (!truncate) { + return prune(getExcerptString(), pruneLength, `…`) + } + return _.truncate(getExcerptString(), { + length: pruneLength, + omission: `…`, }) }, }, From b431269ac589c107d7f29d6c535f616ecb7c45fd Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 5 Nov 2018 20:51:31 -0600 Subject: [PATCH 08/26] make headway on ast cloning --- .../src/__tests__/extend-node.js | 6 +- .../src/extend-node-type.js | 61 +++++++++++-------- .../src/hast-processing.js | 38 ++++++++++++ 3 files changed, 77 insertions(+), 28 deletions(-) create mode 100644 packages/gatsby-transformer-remark/src/hast-processing.js diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 2d43e7d1d0040..d6d1529ecce98 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -130,12 +130,12 @@ const bootstrapTest = ( describe(`Excerpt is generated correctly from schema`, () => { bootstrapTest( - `correctly loads an excerpt`, + `correctly loads a complicated excerpt`, `--- title: "my little pony" date: "2017-09-18T23:19:51.246Z" --- -Where oh where is my little pony?`, +Where oh [*where*](nick.com) **_is_** that pony?`, `excerpt frontmatter { title @@ -143,7 +143,7 @@ Where oh where is my little pony?`, `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt).toMatch(`Where oh where is my little pony?`) + expect(node.excerpt).toMatch(`Where oh where is that pony?`) } ) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 0204a941d12dd..d89d562273712 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -16,7 +16,7 @@ const toHAST = require(`mdast-util-to-hast`) const hastToHTML = require(`hast-util-to-html`) const mdastToToc = require(`mdast-util-toc`) const Promise = require(`bluebird`) -const prune = require(`underscore.string/prune`) +// const prune = require(`underscore.string/prune`) const unified = require(`unified`) const parse = require(`remark-parse`) const stringify = require(`remark-stringify`) @@ -24,6 +24,11 @@ const english = require(`retext-english`) const remark2retext = require(`remark-retext`) const stripPosition = require(`unist-util-remove-position`) const hastReparseRaw = require(`hast-util-raw`) +const { + preOrderTraversal, + getConcatenatedValue, + duplicateNode, +} = require(`./hast-processing`) let fileNodes let pluginsCacheStr = `` @@ -322,6 +327,13 @@ module.exports = ( } } + async function mdastToHTML(markdownAST) { + const htmlAST = toHAST(markdownAST, { allowDangerousHTML: true }) + return hastToHTML(htmlAST, { + allowDangerousHTML: true, + }) + } + const HeadingType = new GraphQLObjectType({ name: `MarkdownHeading`, fields: { @@ -382,37 +394,36 @@ module.exports = ( }, resolve: async (markdownNode, { pruneLength, truncate }) => { if (markdownNode.excerpt) { - const excerpt = await getMarkdownAST( + const excerptAST = await getMarkdownAST( markdownNode, markdownNode.excerpt ) - const htmlAST = toHAST(excerpt, { allowDangerousHTML: true }) - const html = hastToHTML(htmlAST, { - allowDangerousHTML: true, - }) - - return html + return mdastToHTML(excerptAST) } - const markdownAST = await getAST(markdownNode) - const excerptNodes = [] - const getExcerptString = () => - excerptNodes.map(node => node.value).join(` `) - visit(markdownAST, node => { - if (getExcerptString().length > pruneLength) { - return - } - if (node.type === `text` || node.type === `inlineCode`) { - excerptNodes.push(node) + const fullAST = await getHTMLAst(markdownNode) + let excerptAST + preOrderTraversal(fullAST, null, (node, parent) => { + console.log(`concating again`, node, parent) + // console.log(`concated value`, getConcatenatedValue(excerptAST)) + + // const totalExcerptSoFar = getConcatenatedValue(excerptAST) + // if (totalExcerptSoFar && totalExcerptSoFar.length > pruneLength) { + // return + // } + + // The first node has no parent. + // But we do need to retain a reference to tree + if (parent) { + parent.children.push(duplicateNode(node)) + } else { + excerptAST = node } }) - if (!truncate) { - return prune(getExcerptString(), pruneLength, `…`) - } - return _.truncate(getExcerptString(), { - length: pruneLength, - omission: `…`, - }) + console.log(`ended it`) + console.log(`final`, excerptAST) + + return hastToHTML(excerptAST) }, }, headings: { diff --git a/packages/gatsby-transformer-remark/src/hast-processing.js b/packages/gatsby-transformer-remark/src/hast-processing.js new file mode 100644 index 0000000000000..9f73482907d0b --- /dev/null +++ b/packages/gatsby-transformer-remark/src/hast-processing.js @@ -0,0 +1,38 @@ +function duplicateNode(node) { + return { + type: node.type, + children: [], + tagName: node.tagName, + value: node.value, + } +} + +function getConcatenatedValue(node) { + if (!node) { + return `` + } + if (node.type === `text`) { + return node.value + } else if (node.children && node.children.length) { + return node.children + .map(getConcatenatedValue) + .filter(value => value) + .join(``) + } + return `` +} + +function preOrderTraversal(node, parent, processingFunction) { + processingFunction(node, parent) + if (node.children) { + node.children.forEach(child => { + preOrderTraversal(child, node, processingFunction) + }) + } +} + +module.exports = { + duplicateNode, + getConcatenatedValue, + preOrderTraversal, +} From 7b40c31bdd131c114c1a3c8d829ccf64311f38a9 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 5 Nov 2018 20:58:49 -0600 Subject: [PATCH 09/26] abandon generalizing preorder traversal. tree duplication works --- .../src/extend-node-type.js | 30 +++++++++++-------- .../src/hast-processing.js | 10 ------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index d89d562273712..914634eefe8fa 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -24,11 +24,7 @@ const english = require(`retext-english`) const remark2retext = require(`remark-retext`) const stripPosition = require(`unist-util-remove-position`) const hastReparseRaw = require(`hast-util-raw`) -const { - preOrderTraversal, - getConcatenatedValue, - duplicateNode, -} = require(`./hast-processing`) +const { duplicateNode } = require(`./hast-processing`) let fileNodes let pluginsCacheStr = `` @@ -403,23 +399,31 @@ module.exports = ( const fullAST = await getHTMLAst(markdownNode) let excerptAST - preOrderTraversal(fullAST, null, (node, parent) => { - console.log(`concating again`, node, parent) - // console.log(`concated value`, getConcatenatedValue(excerptAST)) + let parent + function preOrderTraversal(node) { // const totalExcerptSoFar = getConcatenatedValue(excerptAST) // if (totalExcerptSoFar && totalExcerptSoFar.length > pruneLength) { // return // } - // The first node has no parent. - // But we do need to retain a reference to tree + const newNode = duplicateNode(node) if (parent) { - parent.children.push(duplicateNode(node)) + parent.children.push(newNode) } else { - excerptAST = node + excerptAST = newNode } - }) + + if (node.children) { + node.children.forEach(child => { + parent = newNode + preOrderTraversal(child) + }) + } + } + + preOrderTraversal(fullAST) + console.log(`ended it`) console.log(`final`, excerptAST) diff --git a/packages/gatsby-transformer-remark/src/hast-processing.js b/packages/gatsby-transformer-remark/src/hast-processing.js index 9f73482907d0b..5a04e831d2f49 100644 --- a/packages/gatsby-transformer-remark/src/hast-processing.js +++ b/packages/gatsby-transformer-remark/src/hast-processing.js @@ -22,17 +22,7 @@ function getConcatenatedValue(node) { return `` } -function preOrderTraversal(node, parent, processingFunction) { - processingFunction(node, parent) - if (node.children) { - node.children.forEach(child => { - preOrderTraversal(child, node, processingFunction) - }) - } -} - module.exports = { duplicateNode, getConcatenatedValue, - preOrderTraversal, } From 4a31459c3070575adf4a0e348f2d6651d652613b Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 5 Nov 2018 21:01:50 -0600 Subject: [PATCH 10/26] pruning mostly works --- .../gatsby-transformer-remark/src/extend-node-type.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 914634eefe8fa..0ea79944563de 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -24,7 +24,7 @@ const english = require(`retext-english`) const remark2retext = require(`remark-retext`) const stripPosition = require(`unist-util-remove-position`) const hastReparseRaw = require(`hast-util-raw`) -const { duplicateNode } = require(`./hast-processing`) +const { getConcatenatedValue, duplicateNode } = require(`./hast-processing`) let fileNodes let pluginsCacheStr = `` @@ -402,10 +402,10 @@ module.exports = ( let parent function preOrderTraversal(node) { - // const totalExcerptSoFar = getConcatenatedValue(excerptAST) - // if (totalExcerptSoFar && totalExcerptSoFar.length > pruneLength) { - // return - // } + const totalExcerptSoFar = getConcatenatedValue(excerptAST) + if (totalExcerptSoFar && totalExcerptSoFar.length > pruneLength) { + return + } const newNode = duplicateNode(node) if (parent) { From 440db7ac648aedc71c2e0718cae6dfd1a6a3ec32 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Thu, 8 Nov 2018 21:31:35 -0600 Subject: [PATCH 11/26] get the complicated test expectation correct --- .../gatsby-transformer-remark/src/__tests__/extend-node.js | 5 ++++- packages/gatsby-transformer-remark/src/extend-node-type.js | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index d6d1529ecce98..7c314ed40f0be 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -135,6 +135,7 @@ describe(`Excerpt is generated correctly from schema`, () => { title: "my little pony" date: "2017-09-18T23:19:51.246Z" --- + Where oh [*where*](nick.com) **_is_** that pony?`, `excerpt frontmatter { @@ -143,7 +144,9 @@ Where oh [*where*](nick.com) **_is_** that pony?`, `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt).toMatch(`Where oh where is that pony?`) + expect(node.excerpt).toMatch( + `

Where oh where is that pony?

` + ) } ) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 0ea79944563de..5c86a5c81a568 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -424,9 +424,6 @@ module.exports = ( preOrderTraversal(fullAST) - console.log(`ended it`) - console.log(`final`, excerptAST) - return hastToHTML(excerptAST) }, }, From c7b9cdd22cca52a3815bd1edce561c7da80af191 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Thu, 8 Nov 2018 21:31:45 -0600 Subject: [PATCH 12/26] rename complicated test --- packages/gatsby-transformer-remark/src/__tests__/extend-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 7c314ed40f0be..84a82898d419e 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -130,7 +130,7 @@ const bootstrapTest = ( describe(`Excerpt is generated correctly from schema`, () => { bootstrapTest( - `correctly loads a complicated excerpt`, + `correctly maps nested markdown to html`, `--- title: "my little pony" date: "2017-09-18T23:19:51.246Z" From cdc2324ea193d72dca37ce3f10b542c907b3655c Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Thu, 8 Nov 2018 21:49:16 -0600 Subject: [PATCH 13/26] refactor out tree cloning --- .../src/extend-node-type.js | 27 +++++-------------- .../src/hast-processing.js | 27 +++++++++++++++++++ 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 5c86a5c81a568..dc472cc084e94 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -24,7 +24,7 @@ const english = require(`retext-english`) const remark2retext = require(`remark-retext`) const stripPosition = require(`unist-util-remove-position`) const hastReparseRaw = require(`hast-util-raw`) -const { getConcatenatedValue, duplicateNode } = require(`./hast-processing`) +const { getConcatenatedValue, cloneTreeUntil } = require(`./hast-processing`) let fileNodes let pluginsCacheStr = `` @@ -398,31 +398,16 @@ module.exports = ( } const fullAST = await getHTMLAst(markdownNode) - let excerptAST - let parent + if (!fullAST.children.length) { + return `` + } - function preOrderTraversal(node) { + const excerptAST = cloneTreeUntil(fullAST, excerptAST => { const totalExcerptSoFar = getConcatenatedValue(excerptAST) if (totalExcerptSoFar && totalExcerptSoFar.length > pruneLength) { return } - - const newNode = duplicateNode(node) - if (parent) { - parent.children.push(newNode) - } else { - excerptAST = newNode - } - - if (node.children) { - node.children.forEach(child => { - parent = newNode - preOrderTraversal(child) - }) - } - } - - preOrderTraversal(fullAST) + }) return hastToHTML(excerptAST) }, diff --git a/packages/gatsby-transformer-remark/src/hast-processing.js b/packages/gatsby-transformer-remark/src/hast-processing.js index 5a04e831d2f49..d6fc9760d5559 100644 --- a/packages/gatsby-transformer-remark/src/hast-processing.js +++ b/packages/gatsby-transformer-remark/src/hast-processing.js @@ -22,7 +22,34 @@ function getConcatenatedValue(node) { return `` } +function cloneTreeUntil(root, endCondition) { + let clonedRoot + + function preOrderTraversal(node) { + if (endCondition(clonedRoot)) { + return + } + + const newNode = duplicateNode(node) + if (clonedRoot) { + clonedRoot.children.push(newNode) + } else { + clonedRoot = newNode + } + + if (node.children) { + node.children.forEach(child => { + clonedRoot = newNode + preOrderTraversal(child) + }) + } + } + preOrderTraversal(root) + return clonedRoot +} + module.exports = { duplicateNode, getConcatenatedValue, + cloneTreeUntil, } From 15932e7c930e0dce2fba3821914bf2c995aad246 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Thu, 8 Nov 2018 22:15:31 -0600 Subject: [PATCH 14/26] make progress towards pruning last text node --- .../src/extend-node-type.js | 22 +++++++++++++++++-- .../src/hast-processing.js | 16 ++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index dc472cc084e94..fd97af8668314 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -16,7 +16,7 @@ const toHAST = require(`mdast-util-to-hast`) const hastToHTML = require(`hast-util-to-html`) const mdastToToc = require(`mdast-util-toc`) const Promise = require(`bluebird`) -// const prune = require(`underscore.string/prune`) +const prune = require(`underscore.string/prune`) const unified = require(`unified`) const parse = require(`remark-parse`) const stringify = require(`remark-stringify`) @@ -24,7 +24,11 @@ const english = require(`retext-english`) const remark2retext = require(`remark-retext`) const stripPosition = require(`unist-util-remove-position`) const hastReparseRaw = require(`hast-util-raw`) -const { getConcatenatedValue, cloneTreeUntil } = require(`./hast-processing`) +const { + getConcatenatedValue, + cloneTreeUntil, + findLastTextNode, +} = require(`./hast-processing`) let fileNodes let pluginsCacheStr = `` @@ -408,6 +412,20 @@ module.exports = ( return } }) + const unprunedExcerpt = getConcatenatedValue(excerptAST) + if (!unprunedExcerpt) { + return `` + } + + if (pruneLength && unprunedExcerpt.length < pruneLength) { + return hastToHTML(excerptAST) + } + + const lastTextNode = findLastTextNode(excerptAST) + const amountToPruneLastNode = + pruneLength - (unprunedExcerpt.length - lastTextNode.value.length) + lastTextNode.value = prune(lastTextNode.value, amountToPruneLastNode) + console.log(`lastTextNode new value`, lastTextNode.value) return hastToHTML(excerptAST) }, diff --git a/packages/gatsby-transformer-remark/src/hast-processing.js b/packages/gatsby-transformer-remark/src/hast-processing.js index d6fc9760d5559..d66f170131e5d 100644 --- a/packages/gatsby-transformer-remark/src/hast-processing.js +++ b/packages/gatsby-transformer-remark/src/hast-processing.js @@ -48,8 +48,24 @@ function cloneTreeUntil(root, endCondition) { return clonedRoot } +function findLastTextNode(node, textNode) { + if (node.type === `text`) { + textNode = node + } + if (node.children) { + node.children.forEach(child => { + const laterTextNode = findLastTextNode(child) + if (laterTextNode !== textNode) { + textNode = laterTextNode + } + }) + } + return textNode +} + module.exports = { duplicateNode, getConcatenatedValue, cloneTreeUntil, + findLastTextNode, } From 235daa2fc312010398879f101b839a0f1aa33c59 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Fri, 9 Nov 2018 19:27:42 -0600 Subject: [PATCH 15/26] tests passing --- .../__tests__/__snapshots__/extend-node.js.snap | 10 +++++----- .../src/__tests__/extend-node.js | 12 +++++++++--- .../src/extend-node-type.js | 15 ++++++++++++--- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap index 35d67762926c5..c4a7341e5ad1c 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap +++ b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap @@ -9,9 +9,9 @@ Object { } `; -exports[`Excerpt is generated correctly from schema correctly loads an excerpt 1`] = ` +exports[`Excerpt is generated correctly from schema correctly maps nested markdown to html 1`] = ` Object { - "excerpt": "Where oh where is my little pony?", + "excerpt": "

Where oh where is that pony?

", "frontmatter": Object { "title": "my little pony", }, @@ -20,7 +20,7 @@ Object { exports[`Excerpt is generated correctly from schema correctly prunes length to default value 1`] = ` Object { - "excerpt": "Where oh where is my little pony? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla…", + "excerpt": "

Where oh where is my little pony? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla…

", "frontmatter": Object { "title": "my little pony", }, @@ -29,7 +29,7 @@ Object { exports[`Excerpt is generated correctly from schema correctly prunes length to provided parameter 1`] = ` Object { - "excerpt": "Where oh where is my little pony? Lorem ipsum…", + "excerpt": "

Where oh where is my little pony? Lorem ipsum…

", "frontmatter": Object { "title": "my little pony", }, @@ -38,7 +38,7 @@ Object { exports[`Excerpt is generated correctly from schema correctly prunes length to provided parameter with truncate 1`] = ` Object { - "excerpt": "Where oh where is my little pony? Lorem ipsum dol…", + "excerpt": "

Where oh where is my little pony? Lorem ipsum dol…

", "frontmatter": Object { "title": "my little pony", }, diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 84a82898d419e..9d6e9d1feceb5 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -209,7 +209,9 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt.length).toBe(139) + expect(node.excerpt).toMatch( + `

Where oh where is my little pony? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla…

` + ) } ) @@ -223,7 +225,9 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt.length).toBe(46) + expect(node.excerpt).toMatch( + `

Where oh where is my little pony? Lorem ipsum…

` + ) } ) @@ -237,7 +241,9 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt.length).toBe(50) + expect(node.excerpt).toMatch( + `

Where oh where is my little pony? Lorem ipsum dol…

` + ) } ) }) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index fd97af8668314..847451d382633 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -424,9 +424,18 @@ module.exports = ( const lastTextNode = findLastTextNode(excerptAST) const amountToPruneLastNode = pruneLength - (unprunedExcerpt.length - lastTextNode.value.length) - lastTextNode.value = prune(lastTextNode.value, amountToPruneLastNode) - console.log(`lastTextNode new value`, lastTextNode.value) - + if (!truncate) { + lastTextNode.value = prune( + lastTextNode.value, + amountToPruneLastNode, + `…` + ) + } else { + lastTextNode.value = _.truncate(lastTextNode.value, { + length: pruneLength, + omission: `…`, + }) + } return hastToHTML(excerptAST) }, }, From 830e6fc2c7785ed593dad7c828f0995d9444028d Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Fri, 9 Nov 2018 19:46:54 -0600 Subject: [PATCH 16/26] don't invoke plugin functions a second time if excerpt_separator used --- .../src/extend-node-type.js | 22 +++++++++---------- .../src/on-node-create.js | 4 ++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 847451d382633..7a5296275c958 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -122,10 +122,7 @@ module.exports = ( // We are already generating AST, so let's wait for it return await ASTPromiseMap.get(cacheKey) } else { - const ASTGenerationPromise = getMarkdownAST( - markdownNode, - markdownNode.internal.content - ) + const ASTGenerationPromise = getMarkdownAST(markdownNode) ASTGenerationPromise.then(markdownAST => { cache.set(cacheKey, markdownAST) ASTPromiseMap.delete(cacheKey) @@ -140,7 +137,7 @@ module.exports = ( } } - function getMarkdownAST(markdownNode, remarkTarget) { + function getMarkdownAST(markdownNode) { return new Promise(async (resolve, reject) => { if (process.env.NODE_ENV !== `production` || !fileNodes) { fileNodes = getNodesByType(`File`) @@ -163,7 +160,7 @@ module.exports = ( return Promise.resolve() } }) - const markdownAST = remark.parse(remarkTarget) + const markdownAST = remark.parse(markdownNode.internal.content) if (pathPrefix) { // Ensure relative links include `pathPrefix` @@ -394,9 +391,12 @@ module.exports = ( }, resolve: async (markdownNode, { pruneLength, truncate }) => { if (markdownNode.excerpt) { - const excerptAST = await getMarkdownAST( - markdownNode, - markdownNode.excerpt + const fullAST = await getAST(markdownNode) + const excerptAST = cloneTreeUntil( + fullAST, + node => + node.type === `html` && + node.value === markdownNode.excerpt_separator ) return mdastToHTML(excerptAST) } @@ -408,9 +408,7 @@ module.exports = ( const excerptAST = cloneTreeUntil(fullAST, excerptAST => { const totalExcerptSoFar = getConcatenatedValue(excerptAST) - if (totalExcerptSoFar && totalExcerptSoFar.length > pruneLength) { - return - } + return totalExcerptSoFar && totalExcerptSoFar.length > pruneLength }) const unprunedExcerpt = getConcatenatedValue(excerptAST) if (!unprunedExcerpt) { diff --git a/packages/gatsby-transformer-remark/src/on-node-create.js b/packages/gatsby-transformer-remark/src/on-node-create.js index 76b2931ca82ad..673dd7707d932 100644 --- a/packages/gatsby-transformer-remark/src/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/on-node-create.js @@ -17,7 +17,6 @@ module.exports = async function onCreateNode( } const content = await loadNodeContent(node) - try { let data = grayMatter(content, pluginOptions) // Convert date objects to string. Otherwise there's type mismatches @@ -48,7 +47,8 @@ module.exports = async function onCreateNode( _PARENT: node.id, } - markdownNode.excerpt = data.excerpt + markdownNode.excerpt_separator = + data.pluginOptions && data.pluginOptions.excerpt_separator markdownNode.rawMarkdownBody = data.content // Add path to the markdown file path From 76037bfb7866ea7f0664404b33ecd5ab8729267e Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Fri, 9 Nov 2018 22:50:18 -0600 Subject: [PATCH 17/26] code works --- .../__snapshots__/extend-node.js.snap | 9 ++++++++ .../src/__tests__/extend-node.js | 21 +++++++++++++++++++ .../src/on-node-create.js | 1 + 3 files changed, 31 insertions(+) diff --git a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap index c4a7341e5ad1c..0053edf1e6db7 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap +++ b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap @@ -45,6 +45,15 @@ Object { } `; +exports[`Excerpt is generated correctly from schema correctly prunes when there are many text nodes and the final text node needs to be shortend 1`] = ` +Object { + "excerpt": "

Where oh where is my little pony? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla…

", + "frontmatter": Object { + "title": "my little pony", + }, +} +`; + exports[`Excerpt is generated correctly from schema correctly uses excerpt separator 1`] = ` Object { "excerpt": "

Where oh where is my little pony?

", diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 9d6e9d1feceb5..3ac8be789cf71 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -246,6 +246,27 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid ) } ) + + bootstrapTest( + `correctly prunes when there are many text nodes and the final text node needs to be shortend`, + `--- +title: "my little pony" +date: "2017-09-18T23:19:51.246Z" +--- +Where oh *where* is my little pony? _Lorem_ ipsum *dolor sit amet*, consectetur adipiscing elit. Morbi _auctor sit amet velit id facilisis_. *Nulla viverra, eros at efficitur pulvinar*, lectus orci accumsan nisi, eu blandit elit nulla nec lectus. + `, + `excerpt(pruneLength: 140) + frontmatter { + title + } + `, + node => { + expect(node).toMatchSnapshot() + expect(node.excerpt).toMatch( + `

Where oh where is my little pony? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla…

` + ) + } + ) }) describe(`Wordcount and timeToRead are generated correctly from schema`, () => { diff --git a/packages/gatsby-transformer-remark/src/on-node-create.js b/packages/gatsby-transformer-remark/src/on-node-create.js index 673dd7707d932..02c9558138087 100644 --- a/packages/gatsby-transformer-remark/src/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/on-node-create.js @@ -17,6 +17,7 @@ module.exports = async function onCreateNode( } const content = await loadNodeContent(node) + try { let data = grayMatter(content, pluginOptions) // Convert date objects to string. Otherwise there's type mismatches From 97cfabb4d437541aacd165fbe72b5cbc1a1bd4ec Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 12 Nov 2018 19:51:24 -0600 Subject: [PATCH 18/26] migrate to excerpt and exerptHTML fields --- .../__snapshots__/extend-node.js.snap | 22 +++------- .../src/__tests__/extend-node.js | 44 +++---------------- .../src/extend-node-type.js | 34 ++++++++++++++ .../src/on-node-create.js | 3 +- 4 files changed, 49 insertions(+), 54 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap index 0053edf1e6db7..4372b594b1f07 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap +++ b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap @@ -9,9 +9,9 @@ Object { } `; -exports[`Excerpt is generated correctly from schema correctly maps nested markdown to html 1`] = ` +exports[`Excerpt is generated correctly from schema correctly loads an excerpt 1`] = ` Object { - "excerpt": "

Where oh where is that pony?

", + "excerpt": "Where oh where is my little pony?", "frontmatter": Object { "title": "my little pony", }, @@ -20,7 +20,7 @@ Object { exports[`Excerpt is generated correctly from schema correctly prunes length to default value 1`] = ` Object { - "excerpt": "

Where oh where is my little pony? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla…

", + "excerpt": "Where oh where is my little pony? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla…", "frontmatter": Object { "title": "my little pony", }, @@ -29,7 +29,7 @@ Object { exports[`Excerpt is generated correctly from schema correctly prunes length to provided parameter 1`] = ` Object { - "excerpt": "

Where oh where is my little pony? Lorem ipsum…

", + "excerpt": "Where oh where is my little pony? Lorem ipsum…", "frontmatter": Object { "title": "my little pony", }, @@ -38,16 +38,7 @@ Object { exports[`Excerpt is generated correctly from schema correctly prunes length to provided parameter with truncate 1`] = ` Object { - "excerpt": "

Where oh where is my little pony? Lorem ipsum dol…

", - "frontmatter": Object { - "title": "my little pony", - }, -} -`; - -exports[`Excerpt is generated correctly from schema correctly prunes when there are many text nodes and the final text node needs to be shortend 1`] = ` -Object { - "excerpt": "

Where oh where is my little pony? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla…

", + "excerpt": "Where oh where is my little pony? Lorem ipsum dol…", "frontmatter": Object { "title": "my little pony", }, @@ -56,7 +47,8 @@ Object { exports[`Excerpt is generated correctly from schema correctly uses excerpt separator 1`] = ` Object { - "excerpt": "

Where oh where is my little pony?

", + "excerpt": "Where oh where is my little pony? +", "frontmatter": Object { "title": "my little pony", }, diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 3ac8be789cf71..53b3eeca6b783 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -130,13 +130,12 @@ const bootstrapTest = ( describe(`Excerpt is generated correctly from schema`, () => { bootstrapTest( - `correctly maps nested markdown to html`, + `correctly loads an excerpt`, `--- title: "my little pony" date: "2017-09-18T23:19:51.246Z" --- - -Where oh [*where*](nick.com) **_is_** that pony?`, +Where oh where is my little pony?`, `excerpt frontmatter { title @@ -144,9 +143,7 @@ Where oh [*where*](nick.com) **_is_** that pony?`, `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt).toMatch( - `

Where oh where is that pony?

` - ) + expect(node.excerpt).toMatch(`Where oh where is my little pony?`) } ) @@ -186,7 +183,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt).toMatch(`

Where oh where is my little pony?

`) + expect(node.excerpt).toMatch(`Where oh where is my little pony?`) }, { excerpt_separator: `` } ) @@ -209,9 +206,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt).toMatch( - `

Where oh where is my little pony? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla…

` - ) + expect(node.excerpt.length).toBe(139) } ) @@ -225,9 +220,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt).toMatch( - `

Where oh where is my little pony? Lorem ipsum…

` - ) + expect(node.excerpt.length).toBe(46) } ) @@ -241,30 +234,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid `, node => { expect(node).toMatchSnapshot() - expect(node.excerpt).toMatch( - `

Where oh where is my little pony? Lorem ipsum dol…

` - ) - } - ) - - bootstrapTest( - `correctly prunes when there are many text nodes and the final text node needs to be shortend`, - `--- -title: "my little pony" -date: "2017-09-18T23:19:51.246Z" ---- -Where oh *where* is my little pony? _Lorem_ ipsum *dolor sit amet*, consectetur adipiscing elit. Morbi _auctor sit amet velit id facilisis_. *Nulla viverra, eros at efficitur pulvinar*, lectus orci accumsan nisi, eu blandit elit nulla nec lectus. - `, - `excerpt(pruneLength: 140) - frontmatter { - title - } - `, - node => { - expect(node).toMatchSnapshot() - expect(node.excerpt).toMatch( - `

Where oh where is my little pony? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla…

` - ) + expect(node.excerpt.length).toBe(50) } ) }) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 7a5296275c958..a7ea2be4cfcae 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -378,6 +378,40 @@ module.exports = ( }, }, excerpt: { + type: GraphQLString, + args: { + pruneLength: { + type: GraphQLInt, + defaultValue: 140, + }, + truncate: { + type: GraphQLBoolean, + defaultValue: false, + }, + }, + resolve(markdownNode, { pruneLength, truncate }) { + if (markdownNode.excerpt) { + return Promise.resolve(markdownNode.excerpt) + } + return getAST(markdownNode).then(ast => { + const excerptNodes = [] + visit(ast, node => { + if (node.type === `text` || node.type === `inlineCode`) { + excerptNodes.push(node.value) + } + return + }) + if (!truncate) { + return prune(excerptNodes.join(` `), pruneLength, `…`) + } + return _.truncate(excerptNodes.join(` `), { + length: pruneLength, + omission: `…`, + }) + }) + }, + }, + excerptHTML: { type: GraphQLString, args: { pruneLength: { diff --git a/packages/gatsby-transformer-remark/src/on-node-create.js b/packages/gatsby-transformer-remark/src/on-node-create.js index 02c9558138087..76b2931ca82ad 100644 --- a/packages/gatsby-transformer-remark/src/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/on-node-create.js @@ -48,8 +48,7 @@ module.exports = async function onCreateNode( _PARENT: node.id, } - markdownNode.excerpt_separator = - data.pluginOptions && data.pluginOptions.excerpt_separator + markdownNode.excerpt = data.excerpt markdownNode.rawMarkdownBody = data.content // Add path to the markdown file path From 9fc3aa2cce4503e8fe0a3efa896784a02c76f3c4 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Sun, 25 Nov 2018 15:23:01 -0600 Subject: [PATCH 19/26] print graphql errors properly --- .../src/__tests__/extend-node.js | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 53b3eeca6b783..9b1c42c38f8d5 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -105,6 +105,9 @@ const bootstrapTest = ( additionalParameters ).then(result => { try { + if (result.errors) { + done(result.errors) + } test(result.data.listNode[0]) done() } catch (err) { @@ -237,6 +240,27 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid expect(node.excerpt.length).toBe(50) } ) + + bootstrapTest( + `given an html format, it correctly maps nested markdown to html`, + `--- +title: "my little pony" +date: "2017-09-18T23:19:51.246Z" +--- + +Where oh [*where*](nick.com) **_is_** that pony?`, + `excerpt(format: 50) + frontmatter { + title + } + `, + node => { + expect(node).toMatchSnapshot() + expect(node.excerpt).toMatch( + `

Where oh where is that pony?

` + ) + } + ) }) describe(`Wordcount and timeToRead are generated correctly from schema`, () => { From df3dcf24bb15469c3299b9041e56505f0de9ea21 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Sun, 25 Nov 2018 16:17:45 -0600 Subject: [PATCH 20/26] excerpt printing in html --- .../src/__tests__/extend-node.js | 51 +++++++++++++++-- .../src/extend-node-type.js | 56 ++++++++++++++++++- 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 9b1c42c38f8d5..95b48e79b0829 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -105,9 +105,6 @@ const bootstrapTest = ( additionalParameters ).then(result => { try { - if (result.errors) { - done(result.errors) - } test(result.data.listNode[0]) done() } catch (err) { @@ -249,7 +246,7 @@ date: "2017-09-18T23:19:51.246Z" --- Where oh [*where*](nick.com) **_is_** that pony?`, - `excerpt(format: 50) + `excerpt(format: "html") frontmatter { title } @@ -261,6 +258,52 @@ Where oh [*where*](nick.com) **_is_** that pony?`, ) } ) + + bootstrapTest( + `given an html format, it prunes large excerpts`, + `--- +title: "my little pony" +date: "2017-09-18T23:19:51.246Z" +--- + +Where oh where is that pony? Is he in the stable or down by the stream?`, + `excerpt(format: "html", pruneLength: 50) + frontmatter { + title + } + `, + node => { + // expect(node).toMatchSnapshot() + expect(node.excerpt).toMatch( + `

Where oh where is that pony? Is he in the stable…

` + ) + } + ) + + bootstrapTest( + `given an html format, it respects the excerpt_separator`, + `--- +title: "my little pony" +date: "2017-09-18T23:19:51.246Z" +--- + +Where oh where is my little pony? + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla viverra, eros at efficitur pulvinar, lectus orci accumsan nisi, eu blandit elit nulla nec lectus. Integer porttitor imperdiet sapien. Quisque in orci sed nisi consequat aliquam. Aenean id mollis nisi. Sed auctor odio id erat facilisis venenatis. Quisque posuere faucibus libero vel fringilla. +`, + `excerpt(format: "html", pruneLength: 50) + frontmatter { + title + } + `, + node => { + // expect(node).toMatchSnapshot() + expect(node.excerpt).toMatch( + `

Where oh where is that pony? Is he in the stable…

` + ) + }, + { excerpt_separator: `` } + ) }) describe(`Wordcount and timeToRead are generated correctly from schema`, () => { diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index a7ea2be4cfcae..ef28f44c60b82 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -16,7 +16,6 @@ const toHAST = require(`mdast-util-to-hast`) const hastToHTML = require(`hast-util-to-html`) const mdastToToc = require(`mdast-util-toc`) const Promise = require(`bluebird`) -const prune = require(`underscore.string/prune`) const unified = require(`unified`) const parse = require(`remark-parse`) const stringify = require(`remark-stringify`) @@ -24,6 +23,8 @@ const english = require(`retext-english`) const remark2retext = require(`remark-retext`) const stripPosition = require(`unist-util-remove-position`) const hastReparseRaw = require(`hast-util-raw`) +const prune = require(`underscore.string/prune`) + const { getConcatenatedValue, cloneTreeUntil, @@ -388,8 +389,59 @@ module.exports = ( type: GraphQLBoolean, defaultValue: false, }, + format: { + type: GraphQLString, + defaultValue: `plain`, + }, }, - resolve(markdownNode, { pruneLength, truncate }) { + async resolve(markdownNode, { format, pruneLength, truncate }) { + if (format === `html`) { + if (markdownNode.excerpt) { + const fullAST = await getAST(markdownNode) + const excerptAST = cloneTreeUntil( + fullAST, + node => + node.type === `html` && + node.value === markdownNode.excerpt_separator + ) + return mdastToHTML(excerptAST) + } + + const fullAST = await getHTMLAst(markdownNode) + if (!fullAST.children.length) { + return `` + } + + const excerptAST = cloneTreeUntil(fullAST, excerptAST => { + const totalExcerptSoFar = getConcatenatedValue(excerptAST) + return totalExcerptSoFar && totalExcerptSoFar.length > pruneLength + }) + const unprunedExcerpt = getConcatenatedValue(excerptAST) + if (!unprunedExcerpt) { + return `` + } + + if (pruneLength && unprunedExcerpt.length < pruneLength) { + return hastToHTML(excerptAST) + } + + const lastTextNode = findLastTextNode(excerptAST) + const amountToPruneLastNode = + pruneLength - (unprunedExcerpt.length - lastTextNode.value.length) + if (!truncate) { + lastTextNode.value = prune( + lastTextNode.value, + amountToPruneLastNode, + `…` + ) + } else { + lastTextNode.value = _.truncate(lastTextNode.value, { + length: pruneLength, + omission: `…`, + }) + } + return hastToHTML(excerptAST) + } if (markdownNode.excerpt) { return Promise.resolve(markdownNode.excerpt) } From 2249f7c9ef0d088fec02c1e4aed30ad1f6312408 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Sun, 25 Nov 2018 16:20:23 -0600 Subject: [PATCH 21/26] more progress --- .../__snapshots__/extend-node.js.snap | 36 +++++++++++ .../src/__tests__/extend-node.js | 2 +- .../src/extend-node-type.js | 60 ------------------- 3 files changed, 37 insertions(+), 61 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap index 4372b594b1f07..c106e761e02f9 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap +++ b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap @@ -55,6 +55,42 @@ Object { } `; +exports[`Excerpt is generated correctly from schema given an html format, it correctly maps nested markdown to html 1`] = ` +Object { + "excerpt": "

Where oh where is that pony?

", + "frontmatter": Object { + "title": "my little pony", + }, +} +`; + +exports[`Excerpt is generated correctly from schema given an html format, it correctly maps nested markdown to html 2`] = ` +Object { + "excerpt": "

Where oh where is that pony?

", + "frontmatter": Object { + "title": "my little pony", + }, +} +`; + +exports[`Excerpt is generated correctly from schema given an html format, it prunes large excerpts 1`] = ` +Object { + "excerpt": "

Where oh where is that pony?

", + "frontmatter": Object { + "title": "my little pony", + }, +} +`; + +exports[`Excerpt is generated correctly from schema html format correctly transforms nested markdown into html 1`] = ` +Object { + "excerpt": "

Where oh where is that pony?

", + "frontmatter": Object { + "title": "my little pony", + }, +} +`; + exports[`Links are correctly prefixed correctly prefixes links 1`] = ` Object { "html": "

This is a link.

diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 95b48e79b0829..6d77c0d7b56ae 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -297,7 +297,7 @@ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet v } `, node => { - // expect(node).toMatchSnapshot() + expect(node).toMatchSnapshot() expect(node.excerpt).toMatch( `

Where oh where is that pony? Is he in the stable…

` ) diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index ef28f44c60b82..953fdc5bb2861 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -463,66 +463,6 @@ module.exports = ( }) }, }, - excerptHTML: { - type: GraphQLString, - args: { - pruneLength: { - type: GraphQLInt, - defaultValue: 140, - }, - truncate: { - type: GraphQLBoolean, - defaultValue: false, - }, - }, - resolve: async (markdownNode, { pruneLength, truncate }) => { - if (markdownNode.excerpt) { - const fullAST = await getAST(markdownNode) - const excerptAST = cloneTreeUntil( - fullAST, - node => - node.type === `html` && - node.value === markdownNode.excerpt_separator - ) - return mdastToHTML(excerptAST) - } - - const fullAST = await getHTMLAst(markdownNode) - if (!fullAST.children.length) { - return `` - } - - const excerptAST = cloneTreeUntil(fullAST, excerptAST => { - const totalExcerptSoFar = getConcatenatedValue(excerptAST) - return totalExcerptSoFar && totalExcerptSoFar.length > pruneLength - }) - const unprunedExcerpt = getConcatenatedValue(excerptAST) - if (!unprunedExcerpt) { - return `` - } - - if (pruneLength && unprunedExcerpt.length < pruneLength) { - return hastToHTML(excerptAST) - } - - const lastTextNode = findLastTextNode(excerptAST) - const amountToPruneLastNode = - pruneLength - (unprunedExcerpt.length - lastTextNode.value.length) - if (!truncate) { - lastTextNode.value = prune( - lastTextNode.value, - amountToPruneLastNode, - `…` - ) - } else { - lastTextNode.value = _.truncate(lastTextNode.value, { - length: pruneLength, - omission: `…`, - }) - } - return hastToHTML(excerptAST) - }, - }, headings: { type: new GraphQLList(HeadingType), args: { From 4e4c4989c63832d0f233b0de02c008d83e87b34e Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 26 Nov 2018 22:21:38 -0600 Subject: [PATCH 22/26] final changes to excerpt separator handling --- jest.config.js | 4 +-- .../__snapshots__/extend-node.js.snap | 22 ++-------------- .../src/__tests__/extend-node.js | 3 ++- .../src/extend-node-type.js | 25 ++++++------------- .../src/hast-processing.js | 5 ++-- 5 files changed, 17 insertions(+), 42 deletions(-) diff --git a/jest.config.js b/jest.config.js index 8a7938abadb5d..8c91872439dea 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,6 +1,6 @@ const path = require(`path`) const glob = require(`glob`) -const fs = require("fs") +const fs = require(`fs`) const pkgs = glob.sync(`./packages/*`).map(p => p.replace(/^\./, ``)) @@ -16,7 +16,7 @@ const coverageDirs = pkgs.map(p => path.join(p, `src/**/*.js`)) module.exports = { notify: true, - verbose: true, + verbose: false, roots: pkgs, modulePathIgnorePatterns: ignoreDirs, coveragePathIgnorePatterns: ignoreDirs, diff --git a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap index c106e761e02f9..648fc80a20a74 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap +++ b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/extend-node.js.snap @@ -64,27 +64,9 @@ Object { } `; -exports[`Excerpt is generated correctly from schema given an html format, it correctly maps nested markdown to html 2`] = ` +exports[`Excerpt is generated correctly from schema given an html format, it respects the excerpt_separator 1`] = ` Object { - "excerpt": "

Where oh where is that pony?

", - "frontmatter": Object { - "title": "my little pony", - }, -} -`; - -exports[`Excerpt is generated correctly from schema given an html format, it prunes large excerpts 1`] = ` -Object { - "excerpt": "

Where oh where is that pony?

", - "frontmatter": Object { - "title": "my little pony", - }, -} -`; - -exports[`Excerpt is generated correctly from schema html format correctly transforms nested markdown into html 1`] = ` -Object { - "excerpt": "

Where oh where is that pony?

", + "excerpt": "

Where oh where is that pony? Is he in the stable…

", "frontmatter": Object { "title": "my little pony", }, diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 6d77c0d7b56ae..f2d2cf9953bd0 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -287,7 +287,8 @@ title: "my little pony" date: "2017-09-18T23:19:51.246Z" --- -Where oh where is my little pony? +Where oh where is that pony? Is he in the stable or by the stream? + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla viverra, eros at efficitur pulvinar, lectus orci accumsan nisi, eu blandit elit nulla nec lectus. Integer porttitor imperdiet sapien. Quisque in orci sed nisi consequat aliquam. Aenean id mollis nisi. Sed auctor odio id erat facilisis venenatis. Quisque posuere faucibus libero vel fringilla. `, diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index 953fdc5bb2861..a17ac70924736 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -74,7 +74,6 @@ module.exports = ( if (type.name !== `MarkdownRemark`) { return {} } - pluginsCacheStr = pluginOptions.plugins.map(p => p.name).join(``) pathPrefixCacheStr = pathPrefix || `` @@ -325,13 +324,6 @@ module.exports = ( } } - async function mdastToHTML(markdownAST) { - const htmlAST = toHAST(markdownAST, { allowDangerousHTML: true }) - return hastToHTML(htmlAST, { - allowDangerousHTML: true, - }) - } - const HeadingType = new GraphQLObjectType({ name: `MarkdownHeading`, fields: { @@ -396,24 +388,23 @@ module.exports = ( }, async resolve(markdownNode, { format, pruneLength, truncate }) { if (format === `html`) { - if (markdownNode.excerpt) { - const fullAST = await getAST(markdownNode) + if (pluginOptions.excerpt_separator) { + const fullAST = await getHTMLAst(markdownNode) const excerptAST = cloneTreeUntil( fullAST, - node => - node.type === `html` && - node.value === markdownNode.excerpt_separator + ({ nextNode }) => + nextNode.type === `raw` && + nextNode.value === pluginOptions.excerpt_separator ) - return mdastToHTML(excerptAST) + return hastToHTML(excerptAST) } - const fullAST = await getHTMLAst(markdownNode) if (!fullAST.children.length) { return `` } - const excerptAST = cloneTreeUntil(fullAST, excerptAST => { - const totalExcerptSoFar = getConcatenatedValue(excerptAST) + const excerptAST = cloneTreeUntil(fullAST, ({ root }) => { + const totalExcerptSoFar = getConcatenatedValue(root) return totalExcerptSoFar && totalExcerptSoFar.length > pruneLength }) const unprunedExcerpt = getConcatenatedValue(excerptAST) diff --git a/packages/gatsby-transformer-remark/src/hast-processing.js b/packages/gatsby-transformer-remark/src/hast-processing.js index d66f170131e5d..46068438a9cff 100644 --- a/packages/gatsby-transformer-remark/src/hast-processing.js +++ b/packages/gatsby-transformer-remark/src/hast-processing.js @@ -24,9 +24,11 @@ function getConcatenatedValue(node) { function cloneTreeUntil(root, endCondition) { let clonedRoot + let endConditionMet = false function preOrderTraversal(node) { - if (endCondition(clonedRoot)) { + if (endConditionMet || endCondition({ root: clonedRoot, nextNode: node })) { + endConditionMet = true return } @@ -64,7 +66,6 @@ function findLastTextNode(node, textNode) { } module.exports = { - duplicateNode, getConcatenatedValue, cloneTreeUntil, findLastTextNode, From 64152c7c092261bfebc2d7e99d41dce2beef7a18 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Mon, 26 Nov 2018 22:38:56 -0600 Subject: [PATCH 23/26] use enum type for format arg --- .../src/__tests__/extend-node.js | 6 +++--- .../gatsby-transformer-remark/src/extend-node-type.js | 10 +++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index f2d2cf9953bd0..7fd5a833895c0 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -246,7 +246,7 @@ date: "2017-09-18T23:19:51.246Z" --- Where oh [*where*](nick.com) **_is_** that pony?`, - `excerpt(format: "html") + `excerpt(format: HTML) frontmatter { title } @@ -267,7 +267,7 @@ date: "2017-09-18T23:19:51.246Z" --- Where oh where is that pony? Is he in the stable or down by the stream?`, - `excerpt(format: "html", pruneLength: 50) + `excerpt(format: HTML, pruneLength: 50) frontmatter { title } @@ -292,7 +292,7 @@ Where oh where is that pony? Is he in the stable or by the stream? Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi auctor sit amet velit id facilisis. Nulla viverra, eros at efficitur pulvinar, lectus orci accumsan nisi, eu blandit elit nulla nec lectus. Integer porttitor imperdiet sapien. Quisque in orci sed nisi consequat aliquam. Aenean id mollis nisi. Sed auctor odio id erat facilisis venenatis. Quisque posuere faucibus libero vel fringilla. `, - `excerpt(format: "html", pruneLength: 50) + `excerpt(format: HTML, pruneLength: 50) frontmatter { title } diff --git a/packages/gatsby-transformer-remark/src/extend-node-type.js b/packages/gatsby-transformer-remark/src/extend-node-type.js index a17ac70924736..fd7ac66e8ecfd 100644 --- a/packages/gatsby-transformer-remark/src/extend-node-type.js +++ b/packages/gatsby-transformer-remark/src/extend-node-type.js @@ -354,6 +354,14 @@ module.exports = ( }, }) + const ExcerptFormats = new GraphQLEnumType({ + name: `ExcerptFormats`, + values: { + PLAIN: { value: `plain` }, + HTML: { value: `html` }, + }, + }) + return resolve({ html: { type: GraphQLString, @@ -382,7 +390,7 @@ module.exports = ( defaultValue: false, }, format: { - type: GraphQLString, + type: ExcerptFormats, defaultValue: `plain`, }, }, From 4db7c57e1c09e269ef4758bfef29cf05d51b9c7c Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Tue, 27 Nov 2018 08:07:37 -0600 Subject: [PATCH 24/26] undo accidental jest config changes --- jest.config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jest.config.js b/jest.config.js index 8c91872439dea..8a7938abadb5d 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,6 +1,6 @@ const path = require(`path`) const glob = require(`glob`) -const fs = require(`fs`) +const fs = require("fs") const pkgs = glob.sync(`./packages/*`).map(p => p.replace(/^\./, ``)) @@ -16,7 +16,7 @@ const coverageDirs = pkgs.map(p => path.join(p, `src/**/*.js`)) module.exports = { notify: true, - verbose: false, + verbose: true, roots: pkgs, modulePathIgnorePatterns: ignoreDirs, coveragePathIgnorePatterns: ignoreDirs, From e755e9b96ca22d31b572ac0fffce2778970ab2d1 Mon Sep 17 00:00:00 2001 From: Nick Drane Date: Tue, 27 Nov 2018 15:23:33 -0600 Subject: [PATCH 25/26] update README --- packages/gatsby-transformer-remark/README.md | 36 ++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-transformer-remark/README.md b/packages/gatsby-transformer-remark/README.md index 8be95e939140d..7eb6fe292d9ef 100644 --- a/packages/gatsby-transformer-remark/README.md +++ b/packages/gatsby-transformer-remark/README.md @@ -81,9 +81,11 @@ Using the following GraphQL query you'll be able to get the table of contents By default the tableOfContents is using the field `slug` to generate URLs. You can however provide another field using the pathToSlugField parameter. **Note** that providing a non existing field will cause the result to be null. -### Excerpt length +### Excerpts -By default, excerpts have a maximum length of 140 characters. You can change the default using the ```pruneLength``` argument. For example, if you need 500 characters, you can specify: +#### Length + +By default, excerpts have a maximum length of 140 characters. You can change the default using the `pruneLength` argument. For example, if you need 500 characters, you can specify: ```graphql { @@ -98,6 +100,36 @@ By default, excerpts have a maximum length of 140 characters. You can change the } ``` +#### Format + +By default, Gatsby will return excerpts as plaintext. This might be useful for populating [opengraph](https://en.wikipedia.org/wiki/Facebook_Platform#Open_Graph_protocol) html tags for SEO reasons. You can also explicitly specify a PLAIN format. + +```graphql +{ + allMarkdownRemark { + edges { + node { + excerpt(format: PLAIN) + } + } + } +} +``` + +It's also possible to ask Gatsby to return excerpts formatted as HTML. You might use this if you have a blog post whose excerpt contains markdown links, and you want these links to render as anchor tags. + +```graphql +{ + allMarkdownRemark { + edges { + node { + excerpt(format: HTML) + } + } + } +} +``` + ## Troubleshooting ### Excerpts for non-latin languages From dc120d9cc0ef2452c7c7e32d035fa8192cb4a425 Mon Sep 17 00:00:00 2001 From: Dustin Schau Date: Tue, 27 Nov 2018 15:40:54 -0600 Subject: [PATCH 26/26] Update README.md --- packages/gatsby-transformer-remark/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-transformer-remark/README.md b/packages/gatsby-transformer-remark/README.md index 7eb6fe292d9ef..2e96499042003 100644 --- a/packages/gatsby-transformer-remark/README.md +++ b/packages/gatsby-transformer-remark/README.md @@ -102,7 +102,7 @@ By default, excerpts have a maximum length of 140 characters. You can change the #### Format -By default, Gatsby will return excerpts as plaintext. This might be useful for populating [opengraph](https://en.wikipedia.org/wiki/Facebook_Platform#Open_Graph_protocol) html tags for SEO reasons. You can also explicitly specify a PLAIN format. +By default, Gatsby will return excerpts as plain text. This might be useful for populating [opengraph](https://en.wikipedia.org/wiki/Facebook_Platform#Open_Graph_protocol) HTML tags for SEO reasons. You can also explicitly specify a `PLAIN` format like so: ```graphql { @@ -116,7 +116,7 @@ By default, Gatsby will return excerpts as plaintext. This might be useful for p } ``` -It's also possible to ask Gatsby to return excerpts formatted as HTML. You might use this if you have a blog post whose excerpt contains markdown links, and you want these links to render as anchor tags. +It's also possible to ask Gatsby to return excerpts formatted as HTML. You might use this if you have a blog post whose an excerpt contains markdown content--e.g. header, link, etc.--and you want these links to render as HTML. ```graphql {