Support external language files in minified version of highlight.js (#…

…1888)

* protect API during minification (supports external language files)

* preprocess language api (apply core minification changes)

* add test, optimize API_REPLACES

src/highlight.js

@@ -40,6 +40,10 @@ https://highlightjs.org/
       languagePrefixRe = /\blang(?:uage)?-([\w-]+)\b/i,
       fixMarkupRe      = /((^(<[^>]+>|\t|)+|(?:\n)))/gm;
 
+  // The object will be assigned by the build tool. It used to synchronize API 
+  // of external language files with minified version of the highlight.js library.
+  var API_REPLACES;
+
   var spanEndTag = '</span>';
 
   // Global options used when within external APIs. This is modified when
@@ -224,6 +228,15 @@ https://highlightjs.org/
     return mode.cached_variants || (mode.endsWithParent && [inherit(mode)]) || [mode];
   }
 
+  function restoreLanguageApi(obj) {
+    if(API_REPLACES && !obj.langApiRestored) {
+      obj.langApiRestored = true;
+      for(var key in API_REPLACES)
+        obj[key] && (obj[API_REPLACES[key]] = obj[key]);
+      (obj.contains || []).concat(obj.variants || []).forEach(restoreLanguageApi);
+    }
+  }
+
   function compileLanguage(language) {
 
     function reStr(re) {
@@ -350,7 +363,7 @@ https://highlightjs.org/
         .filter(Boolean);
       mode.terminators = terminators.length ? langRe(joinRe(terminators, '|'), true) : {exec: function(/*s*/) {return null;}};
     }
-
+    
     compileMode(language);
   }
 
@@ -733,6 +746,7 @@ https://highlightjs.org/
 
   function registerLanguage(name, language) {
     var lang = languages[name] = language(hljs);
+    restoreLanguageApi(lang);
     if (lang.aliases) {
       lang.aliases.forEach(function(alias) {aliases[alias] = name;});
     }

test/index.js

@@ -20,3 +20,5 @@ require('./markup');
 // isn't actually used to test inside a browser but `jsdom` acts as a virtual
 // browser inside of node.js and runs together with all the other tests.
 require('./special');
+
+require("./tools");

test/tools.js

@@ -0,0 +1,40 @@
+'use strict';
+
+let utility = require('../tools/utility'),
+    bluebird = require('bluebird'),
+    path = require('path'),
+    readFile = bluebird.promisify(require('fs').readFile);
+
+describe("minification tools", () => {
+    it("should replace API calls with minified names", () => {
+        let content = "hljs.COMMENT(); hj.NUMBER_MODE == 0; a = hljs.endRe";
+        content.replace(utility.regex.replaces, utility.replaceClassNames).should.equal(
+            "hljs.C(); hj.NM == 0; a = hljs.eR"
+        );
+    });
+
+    it("should replace API calls with minified names and protect declarations", () => {
+        let content = "hj.NUMBER_MODE == 0; hljs.COMMENT = 1; a = hljs.endRe";
+        content.replace(utility.regex.replaces, utility.replaceClassNames).should.equal(
+            "hj.NM == 0; hljs.C = hljs.COMMENT = 1; a = hljs.eR"
+        );
+    });
+
+    it("should NOT protect non-public member declarations", () => {
+        let content = "hljs.endRe = 3;";
+        content.replace(utility.regex.replaces, utility.replaceClassNames).should.equal(
+            "hljs.eR = 3;"
+        );
+    });
+
+    it("should assign API_REPLACES to the REPLACES dictionary in the highlight.js code", (done) => {
+        readFile(path.join(__dirname, "../src/highlight.js"), 'utf-8').then(function(content) {
+            "abc".should.containEql("bc");
+            content.should.not.containEql("var API_REPLACES = " + JSON.stringify(utility.REPLACES));
+            content.replace(utility.regex.apiReplacesFrom, utility.regex.apiReplacesTo)
+                .should
+                .containEql("var API_REPLACES = " + JSON.stringify(utility.REPLACES));
+            done();
+        });
+    });
+});

tools/browser.js

@@ -130,7 +130,12 @@ module.exports = function(commander, dir) {
       task: ['replace', replace(regex.classname, '$1.className')]
     };
 
-    tasks.minify  = { requires: 'replace3', task: 'jsminify' };
+    tasks.replace4 = {
+      requires: 'replace3',
+      task: ['replace', replace(regex.apiReplacesFrom, regex.apiReplacesTo)]
+    };
+
+    tasks.minify  = { requires: 'replace4', task: 'jsminify' };
     requiresTask  = 'minify';
   }
 

tools/utility.js

@@ -60,18 +60,28 @@ const REPLACES = {
 };
 
 regex.replaces = new RegExp(
-  `\\b(${Object.keys(REPLACES).join('|')})\\b`, 'g');
+  `(?:([\\w\\d]+)\\.(${Object.keys(REPLACES).filter(r => r.toUpperCase() === r).join('|')})\\s*=(?!=)|\\b(${Object.keys(REPLACES).join('|')})\\b)`, 'g');
 
 regex.classname = /(block|parentNode)\.cN/g;
 
 regex.header = /^\s*(\/\*((.|\r?\n)*?)\*\/)?\s*/;
 
+regex.apiReplacesFrom = /\bvar\s*API_REPLACES\b/;
+regex.apiReplacesTo = `var API_REPLACES = ${JSON.stringify(REPLACES)}`;
+
 function replace(from, to) {
   return { regex: from, replace: to };
 }
 
-function replaceClassNames(match) {
-  return REPLACES[match];
+function replaceClassNames(match, gDeclObj, gDeclKey) {
+  if(gDeclObj)
+    return replaceAndSaveClassNames(gDeclObj, gDeclKey);
+  else
+    return REPLACES[match];
+}
+
+function replaceAndSaveClassNames(obj, key) {
+  return `${obj}.${REPLACES[key]} = ${obj}.${key} =`;
 }
 
 // All meta data, for each language definition, it store within the headers
@@ -165,5 +175,6 @@ module.exports = {
   regex: regex,
   replace: replace,
   replaceClassNames: replaceClassNames,
-  toQueue: toQueue
+  toQueue: toQueue,
+  REPLACES: REPLACES
 };