changed: application caching now done with Service Workers, closes #55

enketo · Mar 12, 2020 · 5cc2739 · 5cc2739
1 parent ee7a82d
commit 5cc2739
Show file tree

Hide file tree

Showing 28 changed files with 394 additions and 283 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,13 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+[Unreleased]
+----------------------
+**WARNING: IT IS HIGHLY RECOMMENDED TO DEPLOY THIS VERSION BEFORE CHROME 82 IS RELEASED SOME TIME IN APRIL 2020 AND PREFERABLY AFTER VERSION 1.86.1 HAS BEEN DEPLOYED. READ MORE [HERE](https://groups.google.com/forum/#!topic/enketo-users/1AewNMkAIiU).**
+
+##### Change
+- Switch offline application caching technology from ApplicationCache to Service Workers (major).
+
 [1.86.1] - 2020-03-12
 ----------------------
 **WARNING: IT IS HIGHLY RECOMMENDED TO DEPLOY THIS VERSION BEFORE MARCH 31ST, 2020. READ MORE [HERE](https://groups.google.com/forum/#!topic/enketo-users/1AewNMkAIiU).**

diff --git a/Gruntfile.js b/Gruntfile.js
@@ -1,5 +1,5 @@
 module.exports = grunt => {
-    const JS_INCLUDE = [ '**/*.js', '!**/node_modules/**', '!test/**/*.spec.js', '!public/js/build/*', '!test/client/config/karma.conf.js', '!docs/**', '!test-coverage/**' ];
+    const JS_INCLUDE = [ '**/*.js', '!**/offline-app-worker-partial.js', '!**/node_modules/**', '!test/**/*.spec.js', '!public/js/build/*', '!test/client/config/karma.conf.js', '!docs/**', '!test-coverage/**' ];
     const path = require( 'path' );
     const nodeSass = require( 'node-sass' );
     const bundles = require( './buildFiles' ).bundles;

diff --git a/app/controllers/manifest-controller.js → app/controllers/offline-controller.js b/app/controllers/manifest-controller.js → app/controllers/offline-controller.js
@@ -1,30 +1,36 @@
 /**
- * @module manifest-controller
+ * @module offline-resources-controller
  */
 
-const manifest = require( '../models/manifest-model' );
+const fs = require( 'fs' );
+const path = require( 'path' );
+const crypto = require( 'crypto' );
+const offlineResources = require( '../models/offline-resources-model' );
 const express = require( 'express' );
 const router = express.Router();
 const config = require( '../models/config-model' ).server;
-// var debug = require( 'debug' )( 'manifest-controller' );
+
+const partialOfflineAppWorkerScript = fs.readFileSync( path.join( process.cwd(), 'public/js/src/module/offline-app-worker-partial.js' ), 'utf8' );
+
+// This hash is not actually required but useful to see which offline-app-worker-partial.js is used during troubleshooting.
+const scriptHash = crypto.createHash( 'md5' ).update( partialOfflineAppWorkerScript ).digest( 'hex' );
+// var debug = require( 'debug' )( 'offline-controller' );
 
 module.exports = app => {
-    app.use( `${app.get( 'base path' )}/x/manifest.appcache*`, router );
-    // legacy:
-    app.use( `${app.get( 'base path' )}/_/manifest.appcache*`, router );
+    app.use( `${app.get( 'base path' )}/`, router );
 };
 router
-    .get( '*', ( req, res, next ) => {
+    .get( '/x/offline-app-worker.js', ( req, res, next ) => {
         if ( config[ 'offline enabled' ] === false ) {
             var error = new Error( 'Offline functionality has not been enabled for this application.' );
             error.status = 404;
             next( error );
         } else {
-            getManifest( req, res )
-                .then( manifestContent => {
+            getScriptContent( req, res )
+                .then( scriptContent => {
                     res
-                        .set( 'Content-Type', 'text/cache-manifest' )
-                        .send( manifestContent );
+                        .set( 'Content-Type', 'text/javascript' )
+                        .send( scriptContent );
                 } )
                 .catch( next );
         }
@@ -34,15 +40,25 @@ router
  * @param {module:api-controller~ExpressRequest} req
  * @param {module:api-controller~ExpressResponse} res
  */
-function getManifest( req, res ) {
+function getScriptContent( req, res ) {
     return Promise.all( [
             _getWebformHtml( req, res ),
             _getOfflineFallbackHtml( req, res )
         ] )
         .then( result => {
             // TODO: if we ever start supporting dialects, we need to change this
-            const lang = req.i18n.language.split( '-' )[ 0 ];
-            return manifest.get( result[ 0 ], result[ 1 ], lang );
+            // const lang = req.i18n.language.split( '-' )[ 0 ];
+            return offlineResources.get( result[ 0 ], result[ 1 ] );
+        } )
+        .then( dynamicContent => {
+            return `
+const version = '${dynamicContent.version}_${scriptHash}';
+const resources = [
+    '${dynamicContent.resources.join('\',\n    \'')}'
+];
+const fallback = '${dynamicContent.fallback}';
+
+${partialOfflineAppWorkerScript}`;
         } );
 }
 
@@ -53,7 +69,7 @@ function getManifest( req, res ) {
 function _getWebformHtml( req, res ) {
     return new Promise( ( resolve, reject ) => {
         res.render( 'surveys/webform', {
-            manifest: `${req.app.get( 'base path' )}/x/manifest.appcache`
+            offlinePath: config[ 'offline path' ]
         }, ( err, html ) => {
             if ( err ) {
                 reject( err );
@@ -70,7 +86,9 @@ function _getWebformHtml( req, res ) {
  */
 function _getOfflineFallbackHtml( req, res ) {
     return new Promise( ( resolve, reject ) => {
-        res.render( 'pages/offline', {}, ( err, html ) => {
+        res.render( 'pages/offline', {
+            offlinePath: config[ 'offline path' ]
+        }, ( err, html ) => {
             if ( err ) {
                 reject( err );
             } else {

diff --git a/app/controllers/pages-controller.js b/app/controllers/pages-controller.js
@@ -4,6 +4,7 @@
 
 const express = require( 'express' );
 const router = express.Router();
+const config = require( '../models/config-model' ).server;
 // var debug = require( 'debug' )( 'pages-controller' );
 
 module.exports = app => {
@@ -24,9 +25,10 @@ router
             title: 'Modern Browsers'
         } );
     } )
-    .get( '/offline', ( req, res ) => {
+    .get( '/x/offline', ( req, res ) => {
         res.render( 'pages/offline', {
-            title: 'Offline'
+            title: 'Offline',
+            offlinePath: config[ 'offline path' ]
         } );
     } )
     .get( '/thanks', ( req, res ) => {

diff --git a/app/controllers/survey-controller.js b/app/controllers/survey-controller.js
@@ -33,8 +33,7 @@ router.param( 'mod', ( req, rex, next, mod ) => {
 
 router
     //.get( '*', loggedInCheck )
-    .get( '/x/', offlineWebform )
-    .get( '/_/', offlineWebform )
+    .get( `${config[ 'offline path' ]}/`, offlineWebform )
     .get( '/:enketo_id', webform )
     .get( '/:mod/:enketo_id', webform )
     .get( '/preview/:enketo_id', preview )
@@ -74,8 +73,7 @@ function offlineWebform( req, res, next ) {
         error.status = 405;
         next( error );
     } else {
-        req.offline = true;
-        req.manifest = `${req.app.get( 'base path' )}/x/manifest.appcache`;
+        req.offlinePath = config[ 'offline path' ];
         webform( req, res, next );
     }
 }
@@ -87,8 +85,7 @@ function offlineWebform( req, res, next ) {
  */
 function webform( req, res, next ) {
     const options = {
-        offline: req.offline,
-        manifest: req.manifest,
+        offlinePath: req.offlinePath,
         iframe: req.iframe,
         print: req.query.print === 'true'
     };

diff --git a/app/models/config-model.js b/app/models/config-model.js
@@ -279,6 +279,7 @@ if ( config[ 'base path' ] && config[ 'base path' ].indexOf( '/' ) !== 0 ) {
 if ( config[ 'base path' ] && config[ 'base path' ].lastIndexOf( '/' ) === config[ 'base path' ].length - 1 ) {
     config[ 'base path' ] = config[ 'base path' ].substring( 0, config[ 'base path' ].length - 1 );
 }
+config[ 'offline path' ] = '/x';
 
 // ensure backwards compatibility of old external authentication configurations
 const authentication = config[ 'linked form and data server' ][ 'authentication' ];

diff --git a/app/models/manifest-model.js → app/models/offline-resources-model.js b/app/models/manifest-model.js → app/models/offline-resources-model.js
@@ -1,5 +1,5 @@
 /**
- * @module manifest-model
+ * @module offline-resources-model
  */
 
 const libxml = require( 'libxslt' ).libxmljs;
@@ -11,7 +11,7 @@ const client = require( 'redis' ).createClient( config.redis.cache.port, config.
     auth_pass: config.redis.cache.password
 } );
 const utils = require( '../lib/utils' );
-const debug = require( 'debug' )( 'manifest-model' );
+const debug = require( 'debug' )( 'offline-resources-model' );
 
 // in test environment, switch to different db
 if ( process.env.NODE_ENV === 'test' ) {
@@ -24,23 +24,23 @@ if ( process.env.NODE_ENV === 'test' ) {
  * @function
  * @param {string} html1
  * @param {string} html2
- * @param {string} lang
  * @return {Promise} Promise that resolves with manifest
  */
-function getManifest( html1, html2, lang ) {
-    const manifestKey = `ma:${lang}_manifest`;
-    const versionKey = `ma:${lang}_version`;
+
+function get( html1, html2 ) {
+    const resourcesKey = 'off:resources';
+    const versionKey = 'off:version';
 
     return new Promise( ( resolve, reject ) => {
-        // each language gets its own manifest
-        client.get( manifestKey, ( error, manifest ) => {
+        // There is only one list of resources for all forms and all languages
+        client.get( resourcesKey, ( error, obj ) => {
             if ( error ) {
                 reject( error );
-            } else if ( manifest && manifest !== 'null' ) {
-                debug( 'getting manifest from cache' );
-                resolve( manifest );
+            } else if ( obj && obj !== 'null' ) {
+                debug( 'getting offline resource list from cache' );
+                resolve( JSON.parse( obj ) );
             } else {
-                debug( 'building manifest from scratch' );
+                debug( 'building offline resource list from scratch' );
                 const doc1 = libxml.parseHtml( html1 );
                 const doc2 = libxml.parseHtml( html2 );
                 const themesSupported = config[ 'themes supported' ] || [];
@@ -53,8 +53,8 @@ function getManifest( html1, html2, lang ) {
                 // additional themes
                 resources = resources.concat( _getAdditionalThemes( resources, themesSupported ) );
 
-                // translations
-                resources = resources.concat( _getTranslations( lang ) );
+                // default language (TODO: configurable default?)
+                resources = resources.concat( [ `${config[ 'base path' ]}/x/locales/build/en/translation-combined.json` ] );
 
                 // any resources inside css files
                 resources = resources.concat( _getResourcesFromCss( resources ) );
@@ -63,17 +63,21 @@ function getManifest( html1, html2, lang ) {
                 resources = resources.concat( _getSrcAttributes( doc1 ) );
                 resources = resources.concat( _getSrcAttributes( doc2 ) );
 
-                // explicitly add the IE11 bundle, until we can drop IE11 support completely
-                resources = resources.concat( `/js/build/enketo-webform-ie11-bundle${process.env.NODE_ENV === 'production' || !process.env.NODE_ENV ? '.min' : ''}.js` );
-
-                // remove non-existing files, empties, duplicates and non-http urls
+                // remove empties, duplicates and non-http urls
                 resources = resources
                     .filter( _removeEmpties )
-                    .filter( _removeDuplicates )
-                    .filter( _removeNonHttpResources )
+                    .filter( _removeNonHttpResources );
+
+                // convert relative urls to absolute urls
+                resources = resources.map( _toAbsolute )
+                    // remove duplicates after converting URL to local URLs
+                    .filter( _removeDuplicates );
+
+                // remove non-existing files,
+                resources = resources
                     .filter( _removeNonExisting );
 
-                // calculate the hash to serve as the manifest version number
+                // calculate the hash to serve as the version number
                 const hash = _calculateHash( html1, html2, resources );
 
                 // add explicit entries in case user never lands on URL without querystring
@@ -82,37 +86,29 @@ function getManifest( html1, html2, lang ) {
                     `${config[ 'base path' ]}/x/`
                 ] );
 
+                const fallback = `${config[ 'base path' ]}/x/offline/`;
+
                 // determine version
                 _getVersionObj( versionKey )
                     .then( obj => {
                         let version = obj.version;
                         if ( obj.hash !== hash ) {
                             // create a new version
-                            const date = new Date().toISOString().replace( 'T', '|' );
-                            version = `${date.substring( 0, date.length - 8 )}|${lang}`;
+                            const date = new Date().toISOString().replace( 'T', '_' ).replace( ':', '-' );
+                            version = `${date.substring( 0, date.length - 8 )}`;
                             // update stored version, don't wait for result
                             _updateVersionObj( versionKey, hash, version );
                         }
-                        manifest = _getManifestString( version, resources );
-                        // cache manifest for an hour, don't wait for result
-                        client.set( manifestKey, manifest, 'EX', 1 * 60 * 60, () => {} );
-                        resolve( manifest );
+                        // cache for an hour, don't wait for result
+                        client.set( resourcesKey, JSON.stringify( { version, resources, fallback } ), 'EX', 1 * 60 * 60, () => {} );
+                        resolve( { version, resources, fallback } );
                     } );
             }
         } );
     } );
 
 }
 
-/**
- * @param {string} version
- * @param {Array} resources
- * @return {string} Manifest string
- */
-function _getManifestString( version, resources ) {
-    return `CACHE MANIFEST\n# version: ${version}\n\nCACHE:\n${resources.join( '\n' )}\n\nFALLBACK:\n/x ${config[ 'base path' ]}/offline\n/_ ${config[ 'base path' ]}/offline\n\nNETWORK:\n*\n`;
-}
-
 /**
  * @param {string} versionKey
  * @return {Promise<Error|object>}
@@ -182,23 +178,6 @@ function _getAdditionalThemes( resources, themes ) {
     return urls;
 }
 
-/**
- * @param {string} lang
- * @return {Array<string>} List of translations paths
- */
-function _getTranslations( lang ) {
-    const langs = [];
-
-    // fallback language
-    langs.push( `${config[ 'base path' ]}/locales/build/en/translation-combined.json` );
-
-    if ( lang && lang !== 'en' ) {
-        langs.push( `${config[ 'base path' ]}/locales/build/${lang}/translation-combined.json` );
-    }
-
-    return langs;
-}
-
 /**
  * @param {Array<string>} resources
  * @return {Array<string>} A list of urls
@@ -213,7 +192,12 @@ function _getResourcesFromCss( resources ) {
             const content = _getResourceContent( resource );
             let matches;
             while ( ( matches = urlReg.exec( content ) ) !== null ) {
-                urls.push( matches[ 1 ] );
+                let url = matches[ 1 ];
+                if ( url.startsWith( '../' ) ) {
+                    // change context one step down from public/css to public/
+                    url = url.substring( 3 );
+                }
+                urls.push( url );
             }
         }
     } );
@@ -255,12 +239,23 @@ function _removeNonExisting( resource ) {
  * @return {string} Local resource path
  */
 function _getLocalPath( resource ) {
-    const rel = ( resource.indexOf( `${config[ 'base path' ]}/locales/` ) === 0 ) ? '../../' : '../../public';
+    const rel = ( resource.indexOf( `${config[ 'base path' ]}${config['offline path']}/locales/` ) === 0 ) ? '../../' : '../../public';
     const resourceWithoutBase = resource.substring( config[ 'base path' ].length );
-    const localResourcePath = path.join( __dirname, rel, url.parse( resourceWithoutBase ).pathname );
+    const op = config[ 'offline path' ];
+    const resourceWithoutOfflinePath = op ? ( resourceWithoutBase.startsWith( op ) ? resourceWithoutBase.substring( op.length ) : resourceWithoutBase ) : resourceWithoutBase;
+    const localResourcePath = path.join( __dirname, rel, url.parse( resourceWithoutOfflinePath ).pathname );
+
     return localResourcePath;
 }
 
+/**
+ * Very crude convertor only from path/to/resource to /x/path/to/resource
+ * @param {*} resource 
+ */
+function _toAbsolute( resource ) {
+    return !resource.startsWith( '/' ) ? `${config['offline path']}/${resource}` : resource;
+}
+
 /**
  * @param {string} resource
  * @return {boolean} Whether a resource isn't empty
@@ -310,6 +305,4 @@ function _calculateHash( html1, html2, resources ) {
     return utils.md5( hash );
 }
 
-module.exports = {
-    get: getManifest
-};
+module.exports = { get };