Merge branch 'dev' into add_chunk_caching

.editorconfig

@@ -12,3 +12,6 @@ max_line_length = null
 
 [Makefile]
 indent_style = tab
+
+[*.gff3]
+indent_style = tab

.eslintrc

@@ -0,0 +1,18 @@
+{
+  "parser": "babel-eslint",
+  "extends": [
+  ],
+  "plugins": [
+  ],
+  "env": {
+      "browser": true,
+      "amd": true,
+      "es6": true
+  },
+  "globals": {
+  },
+  "rules": {
+    "no-underscore-dangle": 0,
+    "no-undef": "error"
+  }
+}

.gitattributes

@@ -1 +1,2 @@
 package-lock.json binary
+npm-shrinkwrap.json binary

.gitignore

@@ -36,3 +36,6 @@ themes/*/css
 JBrowse-*/
 JBrowse-*.zip
 dist/
+npm-debug.log
+yarn-error.log
+package-lock.json

.travis.yml

@@ -19,10 +19,18 @@ cache:
   - "$HOME/python/"
   - "$HOME/.nvm/"
   - "$HOME/geckodriver_$GECKODRIVER_VERSION"
+  - travis_phantomjs
+before_install:
 env:
     global:
         - GECKODRIVER_VERSION=0.19.1
+        - PHANTOMJS_VERSION=2.1.1
 install:
+- "export PATH=$PWD/travis_phantomjs/phantomjs-$PHANTOMJS_VERSION-linux-x86_64/bin:$PATH"
+- "if [ $(phantomjs --version) != $PHANTOMJS_VERSION ]; then rm -rf $PWD/travis_phantomjs; mkdir -p $PWD/travis_phantomjs; fi"
+- "if [ $(phantomjs --version) != $PHANTOMJS_VERSION ]; then wget https://github.com/Medium/phantomjs/releases/download/v$PHANTOMJS_VERSION/phantomjs-$PHANTOMJS_VERSION-linux-x86_64.tar.bz2 -O $PWD/travis_phantomjs/phantomjs-$PHANTOMJS_VERSION-linux-x86_64.tar.bz2; fi"
+- "if [ $(phantomjs --version) != $PHANTOMJS_VERSION ]; then tar -xvf $PWD/travis_phantomjs/phantomjs-$PHANTOMJS_VERSION-linux-x86_64.tar.bz2 -C $PWD/travis_phantomjs; fi"
+- "phantomjs --version"
 - |
   if [[ ! -f ~/geckodriver_$GECKODRIVER_VERSION/geckodriver ]]; then
     rm -rf ~/geckodriver_$GECKODRIVER_VERSION;
@@ -31,18 +39,18 @@ install:
   fi
 - export PATH=$PATH:$HOME/geckodriver_$GECKODRIVER_VERSION
 - cpanm --notest GD::Image Text::Markdown DateTime JSON
-- npm install -g jshint
 # only init the python virtualenv if it did not show up in our build cache
 - if [[ ! -f ~/python/bin/activate ]]; then virtualenv ~/python; fi
 - source ~/python/bin/activate
 - pip install nose selenium
 script:
-- jshint src/JBrowse/
 # these versions *should* already be set in version control, but set them again here just in case
 - if [[ "x$TRAVIS_TAG" != "x" ]]; then build/set_package_versions.pl $TRAVIS_TAG src/JBrowse/package.json package.json; fi
 - RELEASE_VERSION=`node -e 'require("fs").readFile("src/JBrowse/package.json", (e,d)=>console.log(JSON.parse(d).version))'`
 - BUILD_DIR=$PWD
+- set -e
 - ./setup.sh
+- yarn lint
 - prove -Isrc/perl5 -r -j3 tests/perl_tests;
 - utils/jb_run.js -p 9000 & sleep 2
 - node_modules/.bin/phantomjs tests/js_tests/run-jasmine.js http://localhost:9000/tests/js_tests/index.html
@@ -60,6 +68,9 @@ script:
     cd $BUILD_DIR;
     MOZ_HEADLESS=1 SELENIUM_BROWSER=firefox NOSEOPTS="-I jasmine" JBROWSE_URL=http://localhost:9000/JBrowse-$RELEASE_VERSION/index.html tests/selenium_tests/travis_wrapper.sh;
   fi
+- set +e
+after_failure:
+- cat setup.log
 deploy:
 # upload both minified and non-minified builds to GitHub releases
 - provider: releases

Makefile.PL

@@ -52,6 +52,7 @@ my %WriteMakefileArgs = (
   "PREREQ_PM" => {
     "Bio::Annotation::SimpleValue" => 0,
     "Bio::FeatureIO" => 0,
+    "Bio::GFF3::LowLevel" => "1.8",
     "Bio::GFF3::LowLevel::Parser" => "1.8",
     "Bio::Index::Fasta" => 0,
     "Bio::OntologyIO" => 0,

README.md

@@ -2,85 +2,81 @@
 
 # Installing JBrowse
 
-Users of JBrowse should get it from the main JBrowse site at http://jbrowse.org/install where official release are available.
+To install jbrowse, most users should visit http://jbrowse.org/install and download a zip file such as JBrowse-1.13.0.zip. See instructions at https://jbrowse.org/code/latest-release/docs/tutorial/ for a tutorial on setting up a sample instance.
 
-The `master` branch is always in line with the latest release however it is not minified JS. Features
-are merged into the `dev` branch following acceptance of an item submitted via pull request.
 
-# Install from github repo (development)
+# Install JBrowse from GitHub (for developers)
 
-Note: `jb_run.js` is a built-in [express](https://expressjs.com/) server that serves JBrowse.  However, any webserver like Apache or NGINX can be used.
-
-*If you are using a 3rd party webserver, you should clone JBrowse into your web root*
+To install from GitHub, you can simply clone the repo and run the setup.sh script
 
     git clone https://github.com/GMOD/jbrowse
     cd jbrowse
-    npm install
-    ./jb_setup.js   (optional -- sets up demo files such as Volvox)
-    ./jb_run.js     (optional -- begin serving JBrowse with built-in mini web server)
+    ./setup.sh
+
+At this point, if you are in the web root of your Apache or nginx folder, you can access it as http://localhost/jbrowse/?data=sample_data/json/volvox
+
+Alternatively, run `utils/jb_run.js -p 3000` and access http://localhost:3000/index.html?data=sample_data/json/volvox
+ to see the code running from a small express.js server.
 
-If you have installed the demo (with ./jb_setup.js), you can point your browser to
-http://localhost/jbrowse/index.html?data=sample_data/json/volvox
-and you should see the volvox example data.
+*Note: you should avoid using sudo tasks like ./setup.sh and instead use chown/chmod on folders to your own user as necessary.*
 
-`jb_run.js` will default to a non-privileged port (8080), this can be overridden with the `-p` option.
+*Also note: After editing a file, you must re-run the webpack build with `npm run build` or you can keep webpack running in "watch" mode by running  `npm run watch`.*
 
-Now you can simply edit files and your changes will be available in the browser (the build step is not required).
+*Also also note: git clone by default checks out the dev branch of jbrowse. The master branch contains the latest stable release*
 
 # Installing as an npm module
 
-This allows JBrowse to be easily integrated into other applications.  `jb_setup.js` and `jb_run.js` are copied into the application root and can be used to install the demo files and serve JBrowse, respectively.
+To install jbrowse from NPM directly, you can run.
 
     npm install GMOD/jbrowse
 
-# Generating Packaged Builds
+To setup a simple instance, you can use
 
-You can also optionally run build steps to create the minimized codebase. Extra perl dependencies Text::Markdown and DateTime are required to run the build step.
+    node_modules/.bin/jb_setup.js
+    node_modules/.bin/jb_run.js
+
+Then visit http://localhost:3000/?data=sample_data/json/volvox
 
-    make -f build/Makefile
 
-To build the Electron app (JBrowse desktop app), run the following
+# Contributing
 
-    npm install -g electron-packager
-    make -f build/Makefile release-electron-all
-
-To run the Electron app in debug mode run the following
-
-    npm install -g electron
-    electron browser/main.js
+Looking for places to contribute to the codebase?
+[Check out the "help wanted" label](https://github.com/GMOD/jbrowse/labels/help%20wanted).
 
 # Running the developer test suites
 
-## Server-side Perl
-
-Tests for the server-side Perl code.  You must have the JBrowse Perl
-module prerequisites installed for them to work.  Run with:
+The Travis-CI suite runs Perl, JavaScript, and Selenium automated tests. To run locally, you can use
 
     prove -Isrc/perl5 -lr tests
+    phantomjs tests/js_tests/run-jasmine.js http://localhost/jbrowse/tests/js_tests/index.html
+    pip install selenium nose
+    MOZ_HEADLESS=1 SELENIUM_BROWSER=firefox JBROWSE_URL='http://localhost/jbrowse/index.html' nosetests
+
+Supported browsers for SELENIUM_BROWSER are 'firefox', 'chrome', 'phantom', and 'travis_saucelabs'.  The Sauce Labs + Travis
+one will only work in a properly configured Travis CI build environment.
 
-## Client-side Unit Tests
+# Manual testing
 
-Point your browser at `http://my.dev.machine/jbrowse/tests/js_tests/index.html`
+<img style="display: block; margin: 1em auto" src="img/browserstack-logo-600x315.png" width="200" alt="Browserstack"/>
 
-You can also run them from phantomJS using
+JBrowse has a free open source account on [Browserstack](http://browserstack.com/) for manual testing.  Contact @rbuels for access.
 
-    phantomjs tests/js_tests/run-jasmine.js http://my.dev.machine/jbrowse/tests/js_tests/index.html
+# Generating Packaged Builds
 
-## Client-side Integration Tests
+You can also optionally run build steps to create the minimized codebase. Extra perl dependencies Text::Markdown and DateTime are required to run the build step.
 
-Integration tests for the client-side app.  You need to have Python
-eggs for `selenium` and `nose` installed.  Run the tests with:
+    make -f build/Makefile
 
-    MOZ_HEADLESS=1 SELENIUM_BROWSER=firefox JBROWSE_URL='http://localhost/jbrowse/index.html' nosetests
+To build the Electron app (JBrowse desktop app), run the following
 
-Supported browsers are 'firefox', 'chrome', 'phantom', and 'travis_saucelabs'.  The Sauce Labs + Travis
-one will only work in a properly configured Travis CI build environment.
+    npm install -g electron-packager
+    make -f build/Makefile release-electron-all
 
-# Manual testing
+To run the Electron app in debug mode run the following
 
-<img style="display: block; margin: 1em auto" src="img/browserstack-logo-600x315.png" width="200" alt="Browserstack"/>
+    npm install -g electron
+    electron browser/main.js
 
-JBrowse has a free open source account on [Browserstack](http://browserstack.com/) for manual testing.  Contact @rbuels for access.
 
 # Making a JBrowse release
 

bin/prepare-refseqs.pl

@@ -24,9 +24,11 @@ =head1 USAGE
    # OR:
        prepare-refseqs.pl --twobit <file>  [options]
    # OR:
-       prepare-refseqs.pl --conf <JBrowse config file>  [options]
+       prepare-refseqs.pl --conf <biodb config file>  [options]
    # OR:
        prepare-refseqs.pl --sizes <sizes file>  [options]
+   # OR:
+       prepare-refseqs.pl --gff-sizes <gff file>  [options]
 
 =head1 DESCRIPTION
 
@@ -39,8 +41,8 @@ =head1 DESCRIPTION
 use a JBrowse config file (pointing to a BioPerl database) or a FASTA
 file, together with a list of refseq names or a list of refseq IDs.
 If you use a GFF file, it should contain ##sequence-region lines as
-described in the GFF specs, and/or it should be GFF version 3 with an
-embedded FASTA section.
+described in the GFF specs (for --gff-sizes), and/or it should be GFF
+version 3 with an embedded FASTA section (for --gff).
 
 If you use a JBrowse config file or FASTA file, you can either provide
 a (comma-separated) list of refseq names, or (if the names aren't
@@ -52,21 +54,6 @@ =head1 OPTIONS
 
 =over 4
 
-=item --gff <file>
-
-Get reference sequences from a GFF3 file with embedded sequence
-information.
-
-=item --fasta <file>
-
-A FASTA file, optionally gzipped from which to load reference
-sequences.
-
-=item --indexed_fasta <file>
-
-A FASTA file (which should already have an accompaning '.fai' file)
-from which to load reference sequences.
-
 =item --noSort
 
 If using GFF or FASTA input, preserve the order of the reference

browserconfig.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+    <msapplication>
+        <tile>
+            <square150x150logo src="img/favicons/mstile-150x150.png"/>
+            <TileColor>#2d89ef</TileColor>
+        </tile>
+    </msapplication>
+</browserconfig>

build/Makefile

@@ -21,7 +21,7 @@ UNZIP=unzip -q
 SHASUM=shasum
 
 
-all: clean release-dev release-min
+all: clean release
 
 release: release-dev release-min release-notes.html
 	ls -lh *.zip

build/dojo-loader-config.js

@@ -1,7 +1,5 @@
-const path = require('path')
-const glob = require('glob')
 
-const {getPluginName} = require('./plugin-util')
+const {findPlugins} = require('./plugin-util')
 
 module.exports = function(env) {
 
@@ -47,19 +45,12 @@ module.exports = function(env) {
             }
         ]
         .concat(
-            glob.sync('plugins/*/')
-            .concat(
-                glob.sync('node_modules/**/*-jbrowse-plugin/')
-            )
-            .map( pluginDir => ({
-                name: getPluginName(pluginDir),
-                pluginDir,
-                location: pluginDir+'js'
-            }))
+            findPlugins('.')
         )
         ,
 
 		async: true
-    };
-	return dojoConfig;
+    }
+
+    return dojoConfig
 };

build/glob-loader.js

@@ -1,7 +1,7 @@
 const glob = require('glob')
 const path = require('path')
 
-const {getPluginName} = require('./plugin-util')
+const {findPlugins} = require('./plugin-util')
 
 const blacklist = {};
 `
@@ -24,18 +24,16 @@ const JBrowseModuleIds =
 log(`building ${JBrowseModuleIds.length} JBrowse modules`)
 
 const pluginModuleIds =
-    glob.sync('plugins/*/') // local plugins
-    .concat(glob.sync('node_modules/*-jbrowse-plugin/')) // unscoped modules
-    .concat(glob.sync('node_modules/@*/*-jbrowse-plugin/')) // scoped modules
-    .map( pluginDir => {
-        let pluginName = getPluginName(pluginDir)
-        return glob.sync(pluginDir+'/js/**/*.js')
-        .map( f => {
-            let mid = f.replace(pluginDir,pluginName+'/')
-                .replace('.js','')
-                .replace('/js/','/')
-            return mid
-        })
+    findPlugins('.')
+    .map( pluginConf => {
+        let pluginName = pluginConf.name
+        return glob.sync(pluginConf.pluginDir+'/js/**/*.js')
+            .map( f => {
+                let mid = f.replace(pluginConf.pluginDir,pluginName+'/')
+                    .replace('.js','')
+                    .replace('/js/','')
+                return mid
+            })
     })
     .reduce((a,b) => a.concat(b), [])
     .reverse()
@@ -45,6 +43,7 @@ const pluginModuleIds =
 pluginModuleIds.forEach( mid => log(`adding plugin module ${mid}`) )
 
 const mids = JBrowseModuleIds.concat(pluginModuleIds)
+log(`discovered a total of ${mids.length} modules`)
 
 // tiny Webpack Loader that replaces "//! webpackRequireGlob" expressions with a big bunch of requires
 module.exports = function(content,map,meta) {