(pouchdb#4652) - Migrate to ES6/Rollup, build one index.js

.gitignore

@@ -13,4 +13,6 @@ phantomjsdriver.log
 tests/pouchdb_server
 tests/performance-bundle.js
 dist/*
-pouchdb-webpack.js
+pouchdb-webpack.js
+lib
+lib_unit

.jshintrc

@@ -10,13 +10,12 @@
   "eqnull": true,
   "browser": true,
   "node": true,
-  "strict": true,
   "trailing": true,
-  "globalstrict": true,
   "white": true,
   "indent": 2,
   "forin": true,
   "futurehostile": true,
+  "esnext": true,
   "globals": {
     "PouchDB": true,
     "should": true,

.npmignore

@@ -11,3 +11,4 @@ phantomjsdriver.log
 bower_components
 ./docs/
 pouchdb-webpack.js
+lib_unit

.travis.yml

@@ -33,6 +33,8 @@ before_script:
     # Fail early so we dont run hours of saucelabs if we know there
     # is a lint failure
   - npm run jshint
+    # build before each test
+  - npm run build
 
 script: npm run $COMMAND
 

CONTRIBUTING.md

@@ -59,7 +59,9 @@ All dependancies installed? great, now building PouchDB itself is a breeze:
 
 You will now have various distributions of PouchDB in your `dist` folder, congratulations.
 
- * If you are on windows, you will need `node-gyp` to install levelup, visit https://github.com/TooTallNate/node-gyp#installation for installation instructions.
+Note that the source code is in `src/`, which is built by [Rollup](http://rollupjs.org/) as a
+Node module to `lib/`, which is then built by [Browserify](http://browserify.com/) as a browser-ready
+UMD module to `dist/`. All of this logic is in `bin/build.sh`.
 
 Testing PouchDB
 --------------------------------------

TESTING.md

@@ -25,6 +25,14 @@ or you can run:
 
 and open [http://127.0.0.1:8000/tests/integration/index.html](http://127.0.0.1:8000/tests/integration/index.html) in your browser of choice. The performance tests are located @ [http://localhost:8000/tests/performance/index.html](http://localhost:8000/tests/performance/index.html).
 
+### Unit tests
+
+    $ npm run build-as-modular-es5
+    $ npm run test-unit
+
+These are tests that confirm small parts of PouchDB functionality. In order to
+work correctly with ES6, they are first transpiled to `lib` as modular ES5 (`run run build-as-modular-es5`) using Babel, and then tested as CommonJS modules. See `build-as-modular-es5.sh` for details.
+
 ### Test Options
 
 #### Subset of tests:
@@ -35,8 +43,12 @@ or append `?grep=test.replication.js` if you opened the tests in a browser manua
 
 #### Test Coverage
 
+    $ npm run build-as-modular-es5
     $ COVERAGE=1 npm test
 
+Again, this uses `npm run build-as-modular-es5` in order to fully test the codebase
+as a non-bundle. See `build-as-modular-es5.sh` for details.
+
 #### Test alternative server
 
     $ COUCH_HOST=http://user:pass@myname.host.com npm run dev

bin/add-license.js

@@ -54,6 +54,7 @@ Object.keys(comments).forEach(function (name) {
     var contents = fs.readFileSync(filename);
     contents = comment + '\n' + contents;
 
-    fs.writeFileSync(filename, contents);
+    fs.writeFileSync(filename + '.tmp', contents);
+    fs.renameSync(filename + '.tmp', filename);
   });
 });

bin/build-as-modular-es5.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+#
+# This script builds `src` to `lib` in a modular style that preserves the
+# original file structure. This is useful for things like 1) unit tests, which
+# need to "reach inside" the bundle, and 2) coverage reports, which need to be
+# able to run all tests against the codebase, including the unit tests.
+#
+
+npm run build
+
+# In this case, Babel is roughly equivalent to what Rollup would do.
+# "add-module-exports" gives us a backwards-compatible CJS module structure
+# that doesn't oblige use to do require('foo').default for every module.
+
+./node_modules/.bin/babel \
+  --presets es2015 \
+  --plugins add-module-exports \
+  --out-dir lib src
+
+# Add a version number to both files (equivalent to build.sh)
+
+VERSION=$(node -e "console.log(require('./package.json').version)")
+./node_modules/.bin/replace --silent __VERSION__ $VERSION lib/*
+
+# Make the extras point back to the modular code (instead of being bundles).
+
+echo "module.exports = require('../deps/promise');" \
+    > lib/extras/promise.js
+
+echo "module.exports = require('../replicate/checkpointer');" \
+    > lib/extras/checkpointer.js
+
+echo "module.exports = require('../deps/ajax/prequest');" \
+    > lib/extras/ajax.js
+
+echo "module.exports = require('../replicate/generateReplicationId');" \
+    > lib/extras/generateReplicationId.js

bin/build.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+RIMRAF=./node_modules/.bin/rimraf
+MKDIRP=./node_modules/.bin/mkdirp
+BROWSERIFY=./node_modules/.bin/browserify
+ROLLUP=./node_modules/.bin/rollup
+REPLACE=./node_modules/.bin/replace
+DEREQUIRE=./node_modules/.bin/derequire
+UGLIFY=./node_modules/.bin/uglifyjs
+
+VERSION=$(node -e "console.log(require('./package.json').version)")
+
+$RIMRAF lib_staging
+$MKDIRP lib_staging
+
+EXTERNAL='argsarray,crypto,debug,double-ended-queue,es3ify,events,fruitdown,fs,inherits,inherits,js-extend,level-sublevel,level-sublevel/legacy,level-write-stream,levelup,lie,localstorage-down,memdown,path,pouchdb,pouchdb-collate,pouchdb-collections,request,scope-eval,spark-md5,through2,vuvuzela'
+
+# build for Node
+
+$ROLLUP --format=cjs --external $EXTERNAL \
+  src/index.js \
+  --output=lib_staging/index.js
+
+# build for Browserify/Webpack
+
+$RIMRAF src_browser
+cp -r src src_browser
+
+for file in `find src_browser | grep '\-browser.js'`; do
+  cp $file `echo $file | sed 's/-browser//g'`;
+done
+
+$ROLLUP --format=cjs --external $EXTERNAL \
+  src_browser/index.js \
+  --output=lib_staging/index-browser.js
+
+# add a version number to both files
+$REPLACE --silent __VERSION__ $VERSION lib_staging/*
+
+# rename to avoid concurrency issues in dev server
+$MKDIRP lib
+mv lib_staging/* lib/
+$RIMRAF lib_staging
+
+echo Built lib/index*
+
+# build for the browser (dist)
+
+$RIMRAF dist
+$MKDIRP dist
+
+$BROWSERIFY . -s PouchDB -p bundle-collapser/plugin \
+  | node ./bin/es3ify.js \
+  | $DEREQUIRE \
+  > dist/pouchdb.js
+
+$UGLIFY dist/pouchdb.js -m -c warnings=false > dist/pouchdb.min.js
+
+echo Built dist/*
+
+# build browser plugins
+
+$MKDIRP lib_staging/plugins
+
+for plugin in fruitdown localstorage memory; do
+  $ROLLUP --format=cjs --external $EXTERNAL \
+    src_browser/plugins/${plugin}/index.js \
+    > lib_staging/plugins/${plugin}.js
+done
+
+$MKDIRP lib/plugins
+mv lib_staging/plugins/* lib/plugins/
+$RIMRAF lib_staging
+
+echo Built lib/plugins/*
+
+# build extras API
+
+$MKDIRP lib_staging/extras
+
+$ROLLUP --format=cjs --external $EXTERNAL \
+    src_browser/deps/promise.js \
+    > lib_staging/extras/promise.js
+
+$ROLLUP --format=cjs --external $EXTERNAL \
+    src_browser/replicate/checkpointer.js \
+    > lib_staging/extras/checkpointer.js
+
+$ROLLUP --format=cjs --external $EXTERNAL \
+    src_browser/deps/ajax/prequest.js \
+    > lib_staging/extras/ajax.js
+
+$ROLLUP --format=cjs --external $EXTERNAL \
+    src_browser/replicate/generateReplicationId.js \
+    > lib_staging/extras/generateReplicationId.js
+
+$MKDIRP lib/extras
+mv lib_staging/extras/* lib/extras/
+$RIMRAF lib_staging
+
+echo Built lib/extras/*
+
+# build browser plugins to dist/
+
+for plugin in memory localstorage fruitdown; do
+  $BROWSERIFY ./extras/${plugin} \
+      -x pouchdb \
+      -p bundle-collapser/plugin \
+      | ./bin/es3ify.js \
+      | $DEREQUIRE > ./dist/pouchdb.${plugin}.js
+  $UGLIFY dist/pouchdb.${plugin}.js -m -c warnings=false \
+    > dist/pouchdb.${plugin}.min.js
+  echo Built dist/pouchdb.${plugin}.*
+done
+
+# cleanup
+$RIMRAF src_browser

bin/dev-server.js

@@ -4,25 +4,14 @@
 
 var fs = require('fs');
 var Promise = require('lie');
-var through = require('through2');
-var _derequire = require('derequire');
 var watchify = require('watchify');
+var watch = require('node-watch');
 var browserify = require('browserify');
 var cors_proxy = require('corsproxy');
 var http_proxy = require('pouchdb-http-proxy');
 var http_server = require('http-server');
-var mkdirp = require('mkdirp');
-
-function derequire() {
-  var out = new Buffer('');
-  return through(function (data, _, next) {
-    out = Buffer.concat([out, data]);
-    next();
-  }, function (next) {
-    this.push(_derequire(out.toString()));
-    next();
-  });
-}
+var spawn = require('child_process').spawn;
+
 var queryParams = {};
 
 if (process.env.ES5_SHIM || process.env.ES5_SHIMS) {
@@ -38,58 +27,61 @@ if (process.env.POUCHDB_SRC) {
   queryParams.src = process.env.POUCHDB_SRC;
 }
 
-var indexfile = "./lib/index.js";
-var outfile = "./dist/pouchdb.js";
 var perfRoot = './tests/performance/';
 var performanceBundle = './tests/performance-bundle.js';
 
-var w = watchify(browserify(indexfile, {
-  standalone: "PouchDB",
+var b = watchify(browserify({
+  entries: perfRoot,
   cache: {},
   packageCache: {},
   fullPaths: true,
   debug: true
-})).on('update', bundle);
-var b = watchify(browserify({
-    entries: perfRoot,
-    cache: {},
-    packageCache: {},
-    fullPaths: true,
-    debug: true
-  })).on('update', bundlePerfTests);
-
-function bundle(callback) {
-  mkdirp.sync('./dist');
-  w.bundle().pipe(derequire()).pipe(fs.createWriteStream(outfile))
-  .on('finish', function () {
-    console.log('Updated: ', outfile);
-    if (typeof callback === 'function') {
-      callback();
-    }
-  });
-}
+})).on('update', rebuildPerfTests);
 
-function bundlePerfTests(callback) {
-
+function rebuildPerfTests(callback) {
   b.bundle().pipe(fs.createWriteStream(performanceBundle))
-  .on('finish', function () {
-    console.log('Updated: ', performanceBundle);
-    if (typeof callback === 'function') {
-      callback();
-    }
-  });
+    .on('finish', function () {
+      console.log('Updated: ', performanceBundle);
+      if (typeof callback === 'function') {
+        callback();
+      }
+    });
+}
 
+var rebuildPromise = Promise.resolve();
+
+function rebuild(callback) {
+  // only run one build at a time
+  rebuildPromise = rebuildPromise.then(function () {
+    var child = spawn('npm', ['run', 'build']);
+    child.stdout.on('data', function (buf) {
+      console.log(String(buf).replace(/\s*$/, ''));
+    });
+    child.stderr.on('data', function (buf) {
+      console.log(String(buf).replace(/\s*$/, ''));
+    });
+    child.on('close', function () {
+      if (typeof callback === 'function') {
+        callback();
+      }
+    });
+  });
 }
 
+watch('./src', rebuild);
+
 var filesWritten = false;
-Promise.all([
-  new Promise(function (resolve) {
-    bundle(resolve);
-  }),
-  new Promise(function (resolve) {
-    bundlePerfTests(resolve);
-  })
-]).then(function () {
+
+new Promise(function (resolve) {
+  if (require.main === module) {
+    return rebuild(resolve);
+  }
+  resolve(); // don't bother rebuilding if we're in `npm run dev`
+}).then(function () {
+  return new Promise(function (resolve) {
+    rebuildPerfTests(resolve);
+  });
+}).then(function () {
   filesWritten = true;
   checkReady();
 });
@@ -135,7 +127,6 @@ function checkReady() {
   }
 }
 
-
 if (require.main === module) {
   startServers();
 } else {