#8 Add checksum caching per domain for transpiled ts files

You do not normally change _all_ files constantly, so the performance boost should be pretty viable

src/TranspileWorker.js

@@ -1,4 +1,3 @@
-//import ParseTsModule from "./actions/ParseTsModule.js";
 
 const main = () => {
     self.importScripts(

src/WorkerManager.js

@@ -1,6 +1,8 @@
 import {oneSuccess} from "./utils.js";
 import {addPathToUrl} from "./UrlPathResolver.js";
+import blueimpMd5 from "./cdnEs6Wrappers/blueimpMd5.js";
 
+const whenMd5 = blueimpMd5.get();
 
 const EXPLICIT_EXTENSIONS = ['ts', 'js', 'tsx', 'jsx'];
 
@@ -12,7 +14,6 @@ const workers = [...Array(NUM_OF_WORKERS).keys()].map(i => {
 
     const workerUrl = addPathToUrl('./TranspileWorker.js', scriptUrl);
     // fuck you CORS
-    //const workerJsCode = fetch(workerUrl).then(rs => rs.text());
     const workerBlob = new Blob([
         'importScripts(' + JSON.stringify(workerUrl) + ')',
     ], {type: 'application/javascript'});
@@ -99,6 +100,45 @@ const withFreeWorker = (action) => new Promise((ok, err) => {
     checkFree();
 });
 
+const CACHE_PREFIX = 'ts-browser-cache:';
+
+const resetCache = () => {
+    console.log('ololo resetting cache');
+    for (const key of Object.keys(window.localStorage)) {
+        if (key.startsWith(CACHE_PREFIX)) {
+            console.log('removing key ' + key);
+            window.localStorage.removeItem(key);
+        }
+    }
+};
+
+const getFromCache = ({fullUrl, checksum}) => {
+    const absUrl = addPathToUrl(fullUrl, window.location.pathname);
+    const cacheKey = CACHE_PREFIX + absUrl;
+    const oldResultStr = window.localStorage.getItem(cacheKey);
+    let oldResult = null;
+    try {
+        oldResult = JSON.parse(oldResultStr || 'null');
+    } catch (exc) {
+        console.warn('Failed to parse cached ' + fullUrl, exc);
+    }
+
+    if (oldResult && oldResult.checksum === checksum) {
+        const {jsCode, ...rs} = oldResult.value;
+        return {...rs, whenJsCode: Promise.resolve(jsCode)};
+    } else {
+        return null;
+    }
+};
+
+const putToCache = ({fullUrl, checksum, jsCode, ...rs}) => {
+    const absUrl = addPathToUrl(fullUrl, window.location.pathname);
+    const cacheKey = CACHE_PREFIX + absUrl;
+    window.localStorage.setItem(cacheKey, JSON.stringify({
+        checksum, value: {...rs, jsCode},
+    }));
+};
+
 const WorkerManager = ({compilerOptions}) => {
     const fetchModuleSrc = (url) => {
         // typescript does not allow specifying extension in the import, but react
@@ -126,12 +166,26 @@ const WorkerManager = ({compilerOptions}) => {
     };
 
     const parseInWorker = async ({url, fullUrl, tsCode}) => {
-        const action = () => withFreeWorker(worker => worker.parseTsModule({
-            fullUrl, tsCode, compilerOptions,
-        }).then(({isJsSrc, staticDependencies, dynamicDependencies, whenJsCode}) => {
-            return {url, isJsSrc, staticDependencies, dynamicDependencies, whenJsCode};
-        }));
-        return action();
+        const md5 = await whenMd5;
+        const checksum = md5(tsCode);
+        const fromCache = getFromCache({fullUrl, checksum});
+        if (fromCache) {
+            return fromCache;
+        } else {
+            return withFreeWorker(worker => worker.parseTsModule({
+                fullUrl, tsCode, compilerOptions,
+            }).then(({isJsSrc, staticDependencies, dynamicDependencies, whenJsCode}) => {
+                return {url, isJsSrc, staticDependencies, dynamicDependencies, whenJsCode};
+            })).then(({whenJsCode, ...rs}) => {
+                if (fullUrl.endsWith('.ts') || fullUrl.endsWith('.tsx')) {
+                    // no caching for large raw js libs
+                    whenJsCode.then(jsCode => {
+                        putToCache({...rs, fullUrl, checksum, jsCode});
+                    });
+                }
+                return {...rs, whenJsCode};
+            });
+        }
     };
 
     return {
@@ -140,4 +194,6 @@ const WorkerManager = ({compilerOptions}) => {
     };
 };
 
+WorkerManager.resetCache = resetCache;
+
 export default WorkerManager;

src/cdnEs6Wrappers/blueimpMd5.js

@@ -0,0 +1,31 @@
+import {tryEvalLegacyJsModule} from "../sideEffectModules/sideEffectUtils.js";
+
+let whenLib = null;
+
+/** @return {Promise<ts>} */
+const get = () => {
+    if (!whenLib) {
+        if (window.md5) {
+            whenLib = Promise.resolve(window.md5);
+        } else {
+            // kind of lame that typescript does not provide it's own CDN
+            const url = 'https://cdnjs.cloudflare.com/ajax/libs/blueimp-md5/2.12.0/js/md5.js';
+            whenLib = fetch(url)
+                .then(rs => rs.text())
+                .then(jsCode => {
+                    jsCode += '\n//# sourceURL=' + url;
+                    const module = tryEvalLegacyJsModule(jsCode, false);
+                    if (module) {
+                        return module.default;
+                    } else {
+                        return Promise.reject(new Error('Failed to load ' + url));
+                    }
+                });
+        }
+    }
+    return whenLib;
+};
+
+export default {
+    get: get,
+};

src/cdnEs6Wrappers/typescriptServices.js

@@ -0,0 +1,32 @@
+import {tryEvalLegacyJsModule} from "../sideEffectModules/sideEffectUtils.js";
+
+let whenLib = null;
+
+/** @return {Promise<ts>} */
+const get = () => {
+    if (!whenLib) {
+        if (window.ts) {
+            whenLib = Promise.resolve(window.ts);
+        } else {
+            // kind of lame that typescript does not provide it's own CDN
+            const url = 'https://klesun-misc.github.io/TypeScript/lib/typescriptServices.js';
+            whenLib = fetch(url)
+                .then(rs => rs.text())
+                .then(jsCode => {
+                    jsCode += '\nwindow.ts = ts;';
+                    jsCode += '\n//# sourceURL=' + url;
+                    const module = tryEvalLegacyJsModule(jsCode, false);
+                    if (module) {
+                        return module.default;
+                    } else {
+                        return Promise.reject(new Error('Failed to load ' + url));
+                    }
+                });
+        }
+    }
+    return whenLib;
+};
+
+export default {
+    get: get,
+};

src/sideEffectModules/sideEffectUtils.js

@@ -0,0 +1,38 @@
+
+export const tryEvalLegacyJsModule = (jsCode, silent = true) => {
+    try {
+        // trying to support non es6 modules
+        const globalsBefore = new Set(Object.keys(window));
+        const self = {};
+        const evalResult = eval.apply(self, [jsCode]);
+        const newGlobals = Object.keys(window)
+            .filter(k => !globalsBefore.has(k));
+        const result = {};
+        for (const name of newGlobals) {
+            result[name] = window[name];
+        }
+        if (new Set(newGlobals.map(g => window[g])).size === 1) {
+            result['default'] = window[newGlobals[0]];
+        }
+        const name = jsCode.slice(-100).replace(/[\s\S]*\//, '');
+        console.debug('side-effects js lib loaded ' + name, {
+            newGlobals, evalResult, self,
+        });
+        if (newGlobals.length === 0) {
+            const msg = 'warning: imported lib ' + name + ' did not add any keys to window. ' +
+                'If it is imported in both html and js, you can only use it with side-effects ' +
+                'import like `import "someLib.js"; const someLib = window.someLib;`';
+            console.warn(msg);
+            return {warning: msg, self};
+        } else {
+            return result;
+        }
+    } catch (exc) {
+        if (silent) {
+            // Unexpected token 'import/export' - means it is a es6 module
+            return null;
+        } else {
+            throw exc;
+        }
+    }
+};

src/ts-browser.js

@@ -2,75 +2,19 @@
 import {b64EncodeUnicode} from "./utils.js";
 import {addPathToUrl} from "./UrlPathResolver.js";
 import WorkerManager from "./WorkerManager.js";
+import typescriptServices from "./cdnEs6Wrappers/typescriptServices.js";
+import {tryEvalLegacyJsModule} from "./sideEffectModules/sideEffectUtils.js";
 
 const CACHE_LOADED = 'ts-browser-loaded-modules';
 const IMPORT_DYNAMIC = 'ts-browser-import-dynamic';
 
+const whenTs = typescriptServices.get();
+
 /**
  * @module ts-browser - like ts-node, this tool allows you
  * to require typescript files and compiles then on the fly
  */
 
-const tryLoadSideEffectsJsModule = (jsCode) => {
-    try {
-        // trying to support non es6 modules
-        const globalsBefore = new Set(Object.keys(window));
-        const self = {};
-        const evalResult = eval.apply(self, [jsCode]);
-        const newGlobals = Object.keys(window)
-            .filter(k => !globalsBefore.has(k));
-        const result = {};
-        for (const name of newGlobals) {
-            result[name] = window[name];
-        }
-        if (new Set(newGlobals.map(g => window[g])).size === 1) {
-            result['default'] = window[newGlobals[0]];
-        }
-        const name = jsCode.slice(-100).replace(/[\s\S]*\//, '');
-        console.debug('side-effects js lib loaded ' + name, {
-            newGlobals, evalResult, self,
-        });
-        if (newGlobals.length === 0) {
-            const msg = 'warning: imported lib ' + name + ' did not add any keys to window. ' +
-                'If it is imported in both html and js, you can only use it with side-effects ' +
-                'import like `import "someLib.js"; const someLib = window.someLib;`';
-            console.warn(msg);
-            return {warning: msg};
-        } else {
-            return result;
-        }
-    } catch (exc) {
-        // Unexpected token 'import/export' - means it is a es6 module
-        return null;
-    }
-};
-
-let whenTypescriptServices = null;
-
-/** @return {Promise<ts>} */
-const getTs = () => {
-    if (!whenTypescriptServices) {
-        if (window.ts) {
-            whenTypescriptServices = Promise.resolve(window.ts);
-        } else {
-            // kind of lame that typescript does not provide it's own CDN
-            const url = 'https://klesun-misc.github.io/TypeScript/lib/typescriptServices.js';
-            whenTypescriptServices = fetch(url)
-                .then(rs => rs.text())
-                .then(jsCode => {
-                    jsCode += '\nwindow.ts = ts;';
-                    jsCode += '\n//# sourceURL=' + url;
-                    if (tryLoadSideEffectsJsModule(jsCode)) {
-                        return Promise.resolve(window.ts);
-                    } else {
-                        return Promise.reject(new Error('Failed to load typescriptServices.js'));
-                    }
-                });
-        }
-    }
-    return whenTypescriptServices;
-};
-
 const makeCircularRefProxy = (whenModule, newUrl) => {
     // from position of an app writer, it would be better to just not use circular
     // references, but since typescript supports them somewhat, so should I I guess
@@ -132,7 +76,7 @@ const loadModuleFromFiles = (baseUrl, cachedFiles) => {
             '//# sourceURL=' + baseUrl;
         const base64Code = b64EncodeUnicode(jsCode);
         if (fileData.isJsSrc) {
-            const loaded = tryLoadSideEffectsJsModule(jsCode);
+            const loaded = tryEvalLegacyJsModule(jsCode);
             if (loaded) {
                 // only side effect imports supported, as binding
                 // AMD/CJS modules with es6 has some problems
@@ -150,9 +94,9 @@ const LoadRootModule = async ({
     rootModuleUrl,
     compilerOptions = {},
 }) => {
-    const ts = await getTs();
+    const ts = await whenTs;
     compilerOptions.target = compilerOptions.target || ts.ScriptTarget.ES2018;
-    const workerManager = WorkerManager({ts, compilerOptions});
+    const workerManager = WorkerManager({compilerOptions});
 
     const cachedFiles = {};
     const urlToWhenFileData = {};
@@ -195,9 +139,15 @@ const LoadRootModule = async ({
     };
 
     const importDynamic = async (relUrl, baseUrl) => {
-        const url = addPathToUrl(relUrl, baseUrl);
-        await fetchDependencyFiles([url]);
-        return loadModuleFromFiles(url, cachedFiles);
+        try {
+            const url = addPathToUrl(relUrl, baseUrl);
+            await fetchDependencyFiles([url]);
+            return await loadModuleFromFiles(url, cachedFiles);
+        } catch (exc) {
+            console.warn('Resetting transpilation cache due to uncaught error');
+            WorkerManager.resetCache();
+            throw exc;
+        }
     };
 
     const main = async () => {