From eb1e3502571c6cef10112e4ad9e89d7d9b88ef49 Mon Sep 17 00:00:00 2001 From: RunDevelopment Date: Fri, 8 Jan 2021 15:58:41 +0100 Subject: [PATCH] Core: Fixed greedy rematching bug --- components/prism-core.js | 12 +++- components/prism-core.min.js | 2 +- docs/global.html | 8 +-- docs/prism-core.js.html | 12 +++- prism.js | 12 +++- tests/languages/javascript/issue2694.test | 70 +++++++++++++++++++++++ 6 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 tests/languages/javascript/issue2694.test diff --git a/components/prism-core.js b/components/prism-core.js index 29a7952d7b..acc60d6fa6 100644 --- a/components/prism-core.js +++ b/components/prism-core.js @@ -984,10 +984,18 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, rematch) { if (removeCount > 1) { // at least one Token object was removed, so we have to do some rematching // this can only happen if the current pattern is greedy - matchGrammar(text, tokenList, grammar, currentNode.prev, pos, { + + /** @type {RematchOptions} */ + var nestedRematch = { cause: token + ',' + j, reach: reach - }); + }; + matchGrammar(text, tokenList, grammar, currentNode.prev, pos, nestedRematch); + + // the reach might have been extended because of the rematching + if (rematch && nestedRematch.reach > rematch.reach) { + rematch.reach = nestedRematch.reach; + } } } } diff --git a/components/prism-core.min.js b/components/prism-core.min.js index 89173b8a76..38bfc4e4b1 100644 --- a/components/prism-core.min.js +++ b/components/prism-core.min.js @@ -1 +1 @@ -var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(u){var c=/\blang(?:uage)?-([\w-]+)\b/i,n=0,_={manual:u.Prism&&u.Prism.manual,disableWorkerMessageHandler:u.Prism&&u.Prism.disableWorkerMessageHandler,util:{encode:function e(n){return n instanceof M?new M(n.type,e(n.content),n.alias):Array.isArray(n)?n.map(e):n.replace(/&/g,"&").replace(/=l.reach);y+=m.value.length,m=m.next){var k=m.value;if(t.length>n.length)return;if(!(k instanceof M)){var b,x=1;if(h){if(!(b=W(p,y,n,f)))break;var w=b.index,A=b.index+b[0].length,P=y;for(P+=m.value.length;P<=w;)m=m.next,P+=m.value.length;if(P-=m.value.length,y=P,m.value instanceof M)continue;for(var S=m;S!==t.tail&&(Pl.reach&&(l.reach=N);var j=m.prev;O&&(j=z(t,j,O),y+=O.length),I(t,j,x);var C=new M(o,g?_.tokenize(E,g):E,d,E);m=z(t,j,C),L&&z(t,m,L),1"+a.content+""},!u.document)return u.addEventListener&&(_.disableWorkerMessageHandler||u.addEventListener("message",function(e){var n=JSON.parse(e.data),t=n.language,r=n.code,a=n.immediateClose;u.postMessage(_.highlight(r,_.languages[t],t)),a&&u.close()},!1)),_;var e=_.util.currentScript();function t(){_.manual||_.highlightAll()}if(e&&(_.filename=e.src,e.hasAttribute("data-manual")&&(_.manual=!0)),!_.manual){var r=document.readyState;"loading"===r||"interactive"===r&&e&&e.defer?document.addEventListener("DOMContentLoaded",t):window.requestAnimationFrame?window.requestAnimationFrame(t):window.setTimeout(t,16)}return _}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism); \ No newline at end of file +var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(u){var c=/\blang(?:uage)?-([\w-]+)\b/i,n=0,M={manual:u.Prism&&u.Prism.manual,disableWorkerMessageHandler:u.Prism&&u.Prism.disableWorkerMessageHandler,util:{encode:function e(n){return n instanceof W?new W(n.type,e(n.content),n.alias):Array.isArray(n)?n.map(e):n.replace(/&/g,"&").replace(/=l.reach);y+=m.value.length,m=m.next){var k=m.value;if(r.length>n.length)return;if(!(k instanceof W)){var b,x=1;if(h){if(!(b=z(p,y,n,f)))break;var w=b.index,A=b.index+b[0].length,P=y;for(P+=m.value.length;P<=w;)m=m.next,P+=m.value.length;if(P-=m.value.length,y=P,m.value instanceof W)continue;for(var S=m;S!==r.tail&&(Pl.reach&&(l.reach=N);var j=m.prev;O&&(j=I(r,j,O),y+=O.length),q(r,j,x);var C=new W(o,g?M.tokenize(E,g):E,d,E);if(m=I(r,j,C),L&&I(r,m,L),1l.reach&&(l.reach=_.reach)}}}}}}(e,a,n,a.head,0),function(e){var n=[],r=e.head.next;for(;r!==e.tail;)n.push(r.value),r=r.next;return n}(a)},hooks:{all:{},add:function(e,n){var r=M.hooks.all;r[e]=r[e]||[],r[e].push(n)},run:function(e,n){var r=M.hooks.all[e];if(r&&r.length)for(var t,a=0;t=r[a++];)t(n)}},Token:W};function W(e,n,r,t){this.type=e,this.content=n,this.alias=r,this.length=0|(t||"").length}function z(e,n,r,t){e.lastIndex=n;var a=e.exec(r);if(a&&t&&a[1]){var i=a[1].length;a.index+=i,a[0]=a[0].slice(i)}return a}function i(){var e={value:null,prev:null,next:null},n={value:null,prev:e,next:null};e.next=n,this.head=e,this.tail=n,this.length=0}function I(e,n,r){var t=n.next,a={value:r,prev:n,next:t};return n.next=a,t.prev=a,e.length++,a}function q(e,n,r){for(var t=n.next,a=0;a"+a.content+""},!u.document)return u.addEventListener&&(M.disableWorkerMessageHandler||u.addEventListener("message",function(e){var n=JSON.parse(e.data),r=n.language,t=n.code,a=n.immediateClose;u.postMessage(M.highlight(t,M.languages[r],r)),a&&u.close()},!1)),M;var e=M.util.currentScript();function r(){M.manual||M.highlightAll()}if(e&&(M.filename=e.src,e.hasAttribute("data-manual")&&(M.manual=!0)),!M.manual){var t=document.readyState;"loading"===t||"interactive"===t&&e&&e.defer?document.addEventListener("DOMContentLoaded",r):window.requestAnimationFrame?window.requestAnimationFrame(r):window.setTimeout(r,16)}return M}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism); \ No newline at end of file diff --git a/docs/global.html b/docs/global.html index 4838a92ced..c71bc32473 100644 --- a/docs/global.html +++ b/docs/global.html @@ -143,7 +143,7 @@

Grammar

Source:
@@ -274,7 +274,7 @@

GrammarToken

Source:
@@ -559,7 +559,7 @@

High
Source:
@@ -713,7 +713,7 @@

HookCallb
Source:
diff --git a/docs/prism-core.js.html b/docs/prism-core.js.html index ed5112a7fd..f17df7f3dc 100644 --- a/docs/prism-core.js.html +++ b/docs/prism-core.js.html @@ -1037,10 +1037,18 @@

prism-core.js

if (removeCount > 1) { // at least one Token object was removed, so we have to do some rematching // this can only happen if the current pattern is greedy - matchGrammar(text, tokenList, grammar, currentNode.prev, pos, { + + /** @type {RematchOptions} */ + var nestedRematch = { cause: token + ',' + j, reach: reach - }); + }; + matchGrammar(text, tokenList, grammar, currentNode.prev, pos, nestedRematch); + + // the reach might have been extended because of the rematching + if (rematch && nestedRematch.reach > rematch.reach) { + rematch.reach = nestedRematch.reach; + } } } } diff --git a/prism.js b/prism.js index 03688507c1..fdc707a4cd 100644 --- a/prism.js +++ b/prism.js @@ -989,10 +989,18 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, rematch) { if (removeCount > 1) { // at least one Token object was removed, so we have to do some rematching // this can only happen if the current pattern is greedy - matchGrammar(text, tokenList, grammar, currentNode.prev, pos, { + + /** @type {RematchOptions} */ + var nestedRematch = { cause: token + ',' + j, reach: reach - }); + }; + matchGrammar(text, tokenList, grammar, currentNode.prev, pos, nestedRematch); + + // the reach might have been extended because of the rematching + if (rematch && nestedRematch.reach > rematch.reach) { + rematch.reach = nestedRematch.reach; + } } } } diff --git a/tests/languages/javascript/issue2694.test b/tests/languages/javascript/issue2694.test new file mode 100644 index 0000000000..53c06c9a2a --- /dev/null +++ b/tests/languages/javascript/issue2694.test @@ -0,0 +1,70 @@ +replace(/'/, `'`) + +const var1 = `this is fine`; +const var2 = `this is fine`; + +// `load bearing comment` + +const var3 = `break starts here`; +const var4 = `break ends here`; + +---------------------------------------------------- + +[ + ["function", "replace"], + ["punctuation", "("], + ["regex", [ + ["regex-delimiter", "/"], + ["regex-source", "'"], + ["regex-delimiter", "/"] + ]], + ["punctuation", ","], + ["template-string", [ + ["template-punctuation", "`"], + ["string", "'"], + ["template-punctuation", "`"] + ]], + ["punctuation", ")"], + + ["keyword", "const"], + " var1 ", + ["operator", "="], + ["template-string", [ + ["template-punctuation", "`"], + ["string", "this is fine"], + ["template-punctuation", "`"] + ]], + ["punctuation", ";"], + + ["keyword", "const"], + " var2 ", + ["operator", "="], + ["template-string", [ + ["template-punctuation", "`"], + ["string", "this is fine"], + ["template-punctuation", "`"] + ]], + ["punctuation", ";"], + + ["comment", "// `load bearing comment`"], + + ["keyword", "const"], + " var3 ", + ["operator", "="], + ["template-string", [ + ["template-punctuation", "`"], + ["string", "break starts here"], + ["template-punctuation", "`"] + ]], + ["punctuation", ";"], + + ["keyword", "const"], + " var4 ", + ["operator", "="], + ["template-string", [ + ["template-punctuation", "`"], + ["string", "break ends here"], + ["template-punctuation", "`"] + ]], + ["punctuation", ";"] +] \ No newline at end of file