Merge branch 'release-1.5.5'

refinery-platform · Apr 3, 2017 · abca285 · abca285
2 parents 5965130 + afe887d
commit abca285
Show file tree

Hide file tree

Showing 175 changed files with 4,683 additions and 2,380 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,24 +1,39 @@
 language: python
+cache: pip
+addons:
+  firefox: "49.0.1"
+  apt:
+    packages:
+      - xvfb
 python:
   - 2.7
-addons:
-  sauce_connect: true
-  # username and access_key provided as environment variables in travis config
 services:
   - postgresql
   - neo4j
 install:
   - pip install -r requirements.txt --quiet
+  # Install geckodriver required for selenium testing
+  - GECKODRIVER_VERSION=v0.15.0
+  - TAR=geckodriver-$GECKODRIVER_VERSION-linux64.tar
+  - wget https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/$TAR.gz
+  - gunzip $TAR.gz
+  - tar -xvf $TAR
+  - chmod a+x geckodriver
+  - export PATH=$PATH:`pwd`
 env:
-  - DJANGO_SETTINGS_MODULE=config.settings.dev REDUCE_TEST_OUTPUT=true PYTHONPATH=$PYTHONPATH:../refinery:../refinery/config
+  global:
+    # These env vars are available to every build
+    - REDUCE_TEST_OUTPUT=true
+    - PYTHONPATH=$PYTHONPATH:../refinery:../refinery/config
+    - DJANGO_SETTINGS_MODULE=config.settings.prod
+
 before_script:
   - createuser --createdb --no-superuser --no-createrole vagrant
   - createdb -O vagrant refinery
   - cd refinery
 # See http://www.stuartellis.eu/articles/erb/#running-erb-from-the-command-line
   - erb config/config.json.erb > config/config.json
   - python manage.py migrate --noinput
-  - python manage.py create_user 'guest' 'guest' 'guest@example.com' 'GuestFirst' 'GuestLast' 'GuestAffiliation' 'active'
   - npm install -g grunt-cli@0.1.13 bower@1.7.7 --progress false --quiet
   - cd ui
   - npm install --progress false --quiet
@@ -30,37 +45,20 @@ script:
   - flake8 --exclude=migrations,ui ..
 
   - echo 'travis_fold:start:grunt'
-  - pushd ui
-  - grunt
-  - popd
+  - pushd ui && grunt && popd
   - echo 'travis_fold:end:grunt'
 
-  - echo 'travis_fold:start:selenium'
-  # NOTE: Redirect runserver output since it's polluting logs
-  - python manage.py runserver > /dev/null 2>&1 &
-  - pushd ../selenium
-  # NOTE: Sauce Connect needs SAUCE_USERNAME and SAUCE_ACCESS_KEY
-  # (https://docs.travis-ci.com/user/sauce-connect/),
-  # while Pytest looks for SAUCELABS_USERNAME and SAUCELABS_API_KEY
-  # (http://pytest-selenium.readthedocs.io/en/latest/user_guide.html#sauce-labs).
-  # This has been recognized as a point of confusion, but there is no plan to change anything.
-  # https://github.com/pytest-dev/pytest-selenium/issues/53
-  - export SAUCELABS_USERNAME=$SAUCE_USERNAME SAUCELABS_API_KEY=$SAUCE_ACCESS_KEY
-  - printenv | sort | grep SAUCE | perl -pne 's/=(...).+/=$1.../' # Debug helper
-  - export BASE_URL=http://localhost:8000 CREDS_YML=guest_creds.yml UPLOAD=gff3.csv
-  - pytest --driver SauceLabs --capability browserName Firefox --capability tunnelIdentifier $TRAVIS_JOB_NUMBER -r fE
-  - popd
-  - echo 'travis_fold:end:selenium'
-
-  # If Django tests run first, the selenium test is truncated from the Travis log.
   - echo 'travis_fold:start:django-tests'
   - coverage run manage.py test
   - echo 'travis_fold:end:django-tests'
 
   - set +e # Currently, codecov does not always exit with 0, but that should not cause travis to fail.
+
 after_success:
+  - echo 'travis_fold:start:codecov'
   - codecov
   - npm run codecov
+  - echo 'travis_fold:end:codecov'
 notifications:
   slack:
     secure: nDs9Oj08nRizuD0edl6WcrSgaTPMyITQjZc4qPZpt+yOxUehWbrAmVhqYypfyvdj4qSi1E72rPTXftuBB1E1IZBgX4CCkrCkWGLgIxHaaValTd64oOX66eC3BbSehQxuJB7w1DWw54xBUkTy6+ufjAqiwhLpoEUeE296urAWYHU=
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,10 +1,53 @@
 # Change Log
 
-## [v1.5.4](https://github.com/refinery-platform/refinery-platform/tree/v1.5.4) (2017-02-16)
+## [v1.5.5](https://github.com/refinery-platform/refinery-platform/tree/v1.5.5) (2017-03-28)
+[Full Changelog](https://github.com/refinery-platform/refinery-platform/compare/v1.5.4...v1.5.5)
+
+**Implemented enhancements:**
+
+- Tabular Metadata File Upload Enhancement Con't [\#1638](https://github.com/refinery-platform/refinery-platform/issues/1638)
+- Meta Data Import - Fix "Optional Upload Data Files" Language [\#1637](https://github.com/refinery-platform/refinery-platform/issues/1637)
+- "?" help icon should be consistent across pages [\#1633](https://github.com/refinery-platform/refinery-platform/issues/1633)
+- create separate "Before You Get Started Section" [\#1632](https://github.com/refinery-platform/refinery-platform/issues/1632)
+- Remove "Make Public" field from data set upload page [\#1628](https://github.com/refinery-platform/refinery-platform/issues/1628)
+- remove base path field from file upload page [\#1627](https://github.com/refinery-platform/refinery-platform/issues/1627)
+- add screenshot annotations to tutorial [\#1625](https://github.com/refinery-platform/refinery-platform/issues/1625)
+- DataSet Enhancement - Add how-to intro [\#1616](https://github.com/refinery-platform/refinery-platform/issues/1616)
+- load data from non-Galaxy directories [\#367](https://github.com/refinery-platform/refinery-platform/issues/367)
+- change Galaxy settings to stop copying files into the system [\#366](https://github.com/refinery-platform/refinery-platform/issues/366)
+- Jkmarx metadata upload enhancements [\#1639](https://github.com/refinery-platform/refinery-platform/pull/1639) ([jkmarx](https://github.com/jkmarx))
+- Remove base path and make public. [\#1630](https://github.com/refinery-platform/refinery-platform/pull/1630) ([jkmarx](https://github.com/jkmarx))
+- Jkmarx/data upload enhancements [\#1626](https://github.com/refinery-platform/refinery-platform/pull/1626) ([jkmarx](https://github.com/jkmarx))
+
+**Fixed bugs:**
+
+- spacing in About page layout is broken [\#1640](https://github.com/refinery-platform/refinery-platform/issues/1640)
+- advanced section opens upon click anywhere on the row representing the collapsed section [\#1634](https://github.com/refinery-platform/refinery-platform/issues/1634)
+- Fix missing padding on about. [\#1641](https://github.com/refinery-platform/refinery-platform/pull/1641) ([jkmarx](https://github.com/jkmarx))
+- Fix styling. [\#1636](https://github.com/refinery-platform/refinery-platform/pull/1636) ([jkmarx](https://github.com/jkmarx))
+- Jkmarx/data set upload fixes [\#1635](https://github.com/refinery-platform/refinery-platform/pull/1635) ([jkmarx](https://github.com/jkmarx))
+
+**Closed issues:**
+
+- Attribute Filter UI Enhancement [\#1644](https://github.com/refinery-platform/refinery-platform/issues/1644)
+- Display will change depending on tool selected [\#1605](https://github.com/refinery-platform/refinery-platform/issues/1605)
+
+**Merged pull requests:**
+
+- Jkmarx/attribute filter updates [\#1647](https://github.com/refinery-platform/refinery-platform/pull/1647) ([jkmarx](https://github.com/jkmarx))
+- Jkmarx/tool launch module [\#1642](https://github.com/refinery-platform/refinery-platform/pull/1642) ([jkmarx](https://github.com/jkmarx))
+- Scottx611x/acceptance test bugfix [\#1631](https://github.com/refinery-platform/refinery-platform/pull/1631) ([scottx611x](https://github.com/scottx611x))
+- Add the ability to generate basic ToolDefinitons from properly annota… [\#1624](https://github.com/refinery-platform/refinery-platform/pull/1624) ([scottx611x](https://github.com/scottx611x))
+- Scottx611x/td api url routing [\#1622](https://github.com/refinery-platform/refinery-platform/pull/1622) ([scottx611x](https://github.com/scottx611x))
+- Scottx611x/td api [\#1620](https://github.com/refinery-platform/refinery-platform/pull/1620) ([scottx611x](https://github.com/scottx611x))
+- Add security group [\#1606](https://github.com/refinery-platform/refinery-platform/pull/1606) ([drj11](https://github.com/drj11))
+
+## [v1.5.4](https://github.com/refinery-platform/refinery-platform/tree/v1.5.4) (2017-03-02)
 [Full Changelog](https://github.com/refinery-platform/refinery-platform/compare/v1.5.3.1...v1.5.4)
 
 **Implemented enhancements:**
 
+- Data Upload Page Enhancement [\#1615](https://github.com/refinery-platform/refinery-platform/issues/1615)
 - Update Collaboration Page Styling [\#1597](https://github.com/refinery-platform/refinery-platform/issues/1597)
 - Data Set - Edit Capabilities [\#1591](https://github.com/refinery-platform/refinery-platform/issues/1591)
 - Data Set / share API Adjustment [\#1191](https://github.com/refinery-platform/refinery-platform/issues/1191)
@@ -14,6 +57,7 @@
 - implement user authentication via API keys or tokens [\#359](https://github.com/refinery-platform/refinery-platform/issues/359)
 - Bring unit tests for file server up to date [\#322](https://github.com/refinery-platform/refinery-platform/issues/322)
 - Start using Modernizr [\#267](https://github.com/refinery-platform/refinery-platform/issues/267)
+- Collapse advanced settings. [\#1617](https://github.com/refinery-platform/refinery-platform/pull/1617) ([jkmarx](https://github.com/jkmarx))
 - Update years in license  [\#1612](https://github.com/refinery-platform/refinery-platform/pull/1612) ([ngehlenborg](https://github.com/ngehlenborg))
 - Modify /sharing to return only owner and share\_list. [\#1611](https://github.com/refinery-platform/refinery-platform/pull/1611) ([jkmarx](https://github.com/jkmarx))
 - Jkmarx/colloration styling update [\#1598](https://github.com/refinery-platform/refinery-platform/pull/1598) ([jkmarx](https://github.com/jkmarx))
@@ -40,13 +84,15 @@
 
 **Merged pull requests:**
 
+- Utilize Transaction Management for case where Data already exists [\#1623](https://github.com/refinery-platform/refinery-platform/pull/1623) ([scottx611x](https://github.com/scottx611x))
 - Jkmarx/data set edit adjustments [\#1610](https://github.com/refinery-platform/refinery-platform/pull/1610) ([jkmarx](https://github.com/jkmarx))
 - Prevent console errors when in data set 2. [\#1607](https://github.com/refinery-platform/refinery-platform/pull/1607) ([jkmarx](https://github.com/jkmarx))
 - Fix relative imports in management commands [\#1601](https://github.com/refinery-platform/refinery-platform/pull/1601) ([hackdna](https://github.com/hackdna))
 - Jkmarx/mock uuid service [\#1596](https://github.com/refinery-platform/refinery-platform/pull/1596) ([jkmarx](https://github.com/jkmarx))
 - Improve AWS resource tags handling [\#1592](https://github.com/refinery-platform/refinery-platform/pull/1592) ([hackdna](https://github.com/hackdna))
 - Jkmarx/import module refactor [\#1590](https://github.com/refinery-platform/refinery-platform/pull/1590) ([jkmarx](https://github.com/jkmarx))
 - Jkmarx/jshint rule change [\#1584](https://github.com/refinery-platform/refinery-platform/pull/1584) ([jkmarx](https://github.com/jkmarx))
+- Scottx611x/acceptance testing [\#1534](https://github.com/refinery-platform/refinery-platform/pull/1534) ([scottx611x](https://github.com/scottx611x))
 
 ## [v1.5.3.1](https://github.com/refinery-platform/refinery-platform/tree/v1.5.3.1) (2017-01-23)
 [Full Changelog](https://github.com/refinery-platform/refinery-platform/compare/v1.5.3...v1.5.3.1)

diff --git a/deployment/Puppetfile b/deployment/Puppetfile
@@ -8,6 +8,7 @@ mod 'puppetlabs/lvm', '0.7.0'
 mod 'puppetlabs/postgresql', '4.8.0'
 mod 'puppetlabs/rabbitmq', '5.6.0'
 mod 'puppetlabs/stdlib', '4.13.1'
+mod 'puppet-archive', '1.2.0'
 mod 'saz/timezone', '3.2.0'
 mod 'stankevich/python', '1.9.1'
 mod 'thias/sysctl', '1.0.2'

diff --git a/deployment/manifests/aws.pp b/deployment/manifests/aws.pp
@@ -28,7 +28,9 @@
 
 # See code in refinery-modules/refinery/...
 include refinery
-include refinery::pg
-include refinery::neo4j
 include refinery::apache2
 include refinery::aws
+include refinery::neo4j
+include refinery::pg
+include refinery::solr
+
diff --git a/deployment/manifests/default.pp b/deployment/manifests/default.pp
@@ -33,6 +33,9 @@
 
 # See code in refinery-modules/refinery/...
 include refinery
-include refinery::pg
-include refinery::neo4j
 include refinery::apache2
+include refinery::neo4j
+include refinery::pg
+include refinery::selenium
+include refinery::solr
+
diff --git a/deployment/refinery-modules/refinery/manifests/init.pp b/deployment/refinery-modules/refinery/manifests/init.pp
@@ -131,87 +131,6 @@
   ensure => directory,
 }
 
-class solr {
-  $solr_version = "5.3.1"
-  $solr_archive = "solr-${solr_version}.tgz"
-  $solr_url = "http://archive.apache.org/dist/lucene/solr/${solr_version}/${solr_archive}"
-
-  package { 'java':
-    name => 'openjdk-7-jdk',
-  }
-  exec { "solr_download":
-    command => "wget ${solr_url} -O ${solr_archive}",
-    cwd     => "/usr/local/src",
-    creates => "/usr/local/src/${solr_archive}",
-    path    => "/usr/bin",
-    timeout => 600,  # downloading can take a long time
-  }
-  ->
-  exec { "solr_extract_installer":
-    command => "tar -xzf ${solr_archive} solr-${solr_version}/bin/install_solr_service.sh --strip-components=2",
-    cwd     => "/usr/local/src",
-    creates => "/usr/local/src/install_solr_service.sh",
-    path    => "/bin",
-  }
-  ->
-  file { "${django_root}/solr/core/conf/solrconfig.xml":
-    ensure  => file,
-    content => template("${django_root}/solr/core/conf/solrconfig.xml.erb"),
-  }
-  ->
-  file { "${django_root}/solr/data_set_manager/conf/solrconfig.xml":
-    ensure  => file,
-    content => template("${django_root}/solr/data_set_manager/conf/solrconfig.xml.erb"),
-  }
-  ->
-  exec { "solr_install":  # also starts the service
-    command => "sudo bash ./install_solr_service.sh ${solr_archive} -u ${app_user}",
-    cwd     => "/usr/local/src",
-    creates => "/opt/solr-${solr_version}",
-    path    => "/usr/bin:/bin",
-    require => [ File["/opt"], Package['java'] ],
-  }
-  ->
-  file_line { "solr_config_home":
-    path  => "/var/solr/solr.in.sh",
-    line  => "SOLR_HOME=${django_root}/solr",
-    match => "^SOLR_HOME",
-  }
-  ->
-  file_line { "solr_config_log":
-    path  => "/var/solr/log4j.properties",
-    line  => "solr.log=${django_root}/log",
-    match => "^solr.log",
-  }
-  ~>
-  service { 'solr':
-    ensure     => running,
-    hasrestart => true,
-  }
-}
-include solr
-
-class solrSynonymAnalyzer {
-  $version = "2.0.0"
-  $url = "https://github.com/refinery-platform/solr-synonyms-analyzer/releases/download/v${version}/hon-lucene-synonyms.jar"
-
-  # Need to remove the old file manually as wget throws a weird
-  # `HTTP request sent, awaiting response... 403 Forbidden` error when the file
-  # already exists.
-
-  exec { "solr-synonym-analyzer-download":
-    command => "rm -f ${solr_lib_dir}/hon-lucene-synonyms.jar && wget -P ${solr_lib_dir} ${url}",
-    creates => "${solr_lib_dir}/hon-lucene-synonyms.jar",
-    path    => "/usr/bin:/bin",
-    timeout => 120,  # downloading can take some time
-    notify => Service['solr'],
-    require => Exec['solr_install'],
-  }
-}
-include solrSynonymAnalyzer
-
-
-
 include '::rabbitmq'
 
 class ui {

diff --git a/deployment/refinery-modules/refinery/manifests/neo4j.pp b/deployment/refinery-modules/refinery/manifests/neo4j.pp
@@ -94,12 +94,16 @@
   include owl2neo4j
 
   class neo4jPrePopulatedDB {
-    package { 'unzip':
-      name        => 'unzip',
-      ensure      => latest,
-      require     => Class['apt::update'],
+    $neo4j_user = "neo4j"
+    $neo4j_group = "nogroup"
+    $dirname = "graph.db"
+    $filename = "${dirname}.zip"
+    $install_path = "/var/lib/neo4j/data/"
+
+    package{"unzip":
+      ensure   => 'installed',
     }
-    ->
+
     exec { "stop neo4j service":
       command     => "sudo service neo4j-service stop",
       user        => $app_user,
@@ -116,22 +120,24 @@
       path        => ['/usr/bin/'],
     }
     ->
-    exec { "fetch pre-generated db":
-      command     => "sudo wget -q http://data.cloud.refinery-platform.org.s3.amazonaws.com/data/stem-cell-commons/neo4j/2015/graph.db.zip && yes | sudo unzip graph.db.zip",
-      cwd         => "/var/lib/neo4j/data/",
-      user        => $app_user,
-      group       => $app_group,
-      path        => ['/usr/bin/'],
-      timeout     => 1800,
+    archive { "fetch pre-generated db":
+      path        => "/tmp/${filename}",
+      source      => "http://data.cloud.refinery-platform.org.s3.amazonaws.com/data/stem-cell-commons/neo4j/2015/${filename}",
+      extract       => true,
+      extract_path  => "${install_path}",
+      creates       => "${install_path}/${dirname}",
+      cleanup       => true,
+      user        => 'root',
+      group       => 'root',
       require     => [
         Package['unzip'],
         Class['neo4jFetch'],
       ],
     }
     ->
-    exec { "change db ownership":
-      command     => "sudo chown -R neo4j:nogroup /var/lib/neo4j/data/graph.db",
-      path        => ['/usr/bin/', '/bin/'],
+    exec { 'neo4j permissions':
+      command   => "sudo chown -R $neo4j_user:$neo4j_group $install_path/$dirname",
+      path        => ['/usr/bin/'],
     }
     ->
     exec { "start neo4j service":
@@ -142,48 +148,14 @@
       require     => Service["neo4j-service"],
     }
     ->
-    exec { "fetch neo4j fixture":
-      command     => "sudo wget -q https://raw.githubusercontent.com/refinery-platform/ontology-imports/master/django-fixure-stemcellcommons.json",
-      cwd         => $django_root,
-      creates     => "$django_root/django-fixure-stemcellcommons.json",
-      path        => ['/usr/bin/'],
-      user        => $app_user,
-      group       => $app_group,
-      timeout     => 1800,
-    }
-    ->
-    exec { "install neo4j fixture":
-      command     => "${virtualenv}/bin/python manage.py loaddata django-fixure-stemcellcommons.json",
-      environment => ["DJANGO_SETTINGS_MODULE=${django_settings_module}"],
-      cwd         => $django_root,
-      user        => $app_user,
-      group       => $app_group,
-      # Exit code 1 will occur the second time this command is run. This
-      # will turn into a DataMigration soon but I'm avoiding creating more
-      # Migrations in Django 1.6 and will address this once Django 1.7 is
-      # Merged
-      returns     => [0,1],
-      require     => [
-        Python::Requirements[$requirements],
-        Postgresql::Server::Db["refinery"]
-      ],
-    }
-    ->
     exec { "install neo4j annotations":
       command     => "${virtualenv}/bin/python manage.py import_annotations -c",
       environment => ["DJANGO_SETTINGS_MODULE=${django_settings_module}"],
       cwd         => $django_root,
       user        => $app_user,
       group       => $app_group,
-    }
-    ->
-    exec { "remove neo4j fixture":
-      command     => "rm -rf django-fixure-stemcellcommons.json",
-      cwd         => $django_root,
-      user        => $app_user,
-      group       => $app_group,
-      path        => ['/bin/'],
+      require     => Exec['migrate'],
     }
   }
   include neo4jPrePopulatedDB
-}
+}