Update to Python3.5 (#3474)

This closes #2619

.travis.yml

@@ -2,7 +2,7 @@ sudo: required
 
 language: python
 python:
-  - "2.7"
+  - "3.5"
 
 cache:
   directories:

deployment/bin/generate-superuser

@@ -7,7 +7,7 @@ import os
 from django.contrib.auth.hashers import make_password
 
 # The local package refinery/core/__init__.py
-import core     # Only used for its location
+import core  # Only used for its location
 
 core_path = os.path.dirname(os.path.abspath(core.__file__))
 
@@ -16,18 +16,19 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.dev")
 
 def main():
     with open(os.path.join(core_path, 'fixtures/superuser.json')) as j:
-	o = json.load(j)
+        o = json.load(j)
 
     admin_index = [index for (index, obj) in enumerate(o)
-      if obj['fields']['username'] == 'admin']
+                   if obj['fields']['username'] == 'admin']
     assert len(admin_index) == 1
     (admin_index,) = admin_index
 
     hashed_password = make_password(os.environ['ADMIN_PASSWORD'])
 
     o[admin_index]['fields']['password'] = hashed_password
 
-    print json.dumps(o, indent=2)
+    print(json.dumps(o, indent=2))
+
 
 if __name__ == '__main__':
     main()

deployment/puppet/refinery/manifests/apache2.pp

@@ -33,9 +33,9 @@
 
   class { 'apache::mod::wsgi':
     mod_path     => 'mod_wsgi.so',
-    package_name => 'libapache2-mod-wsgi',
+    package_name => 'libapache2-mod-wsgi-py3',
+    require      => Class['refinery::python'],
   }
-
   # recommended for use with AWS ELB to avoid HTTP 408 errors
   class { 'apache::mod::reqtimeout':
     timeouts => [
@@ -132,7 +132,8 @@
       'refinery' => {
         'user'        => $app_user,
         'group'       => $app_group,
-        'python-path' => "${django_root}:${virtualenv}/lib/python2.7/site-packages",
+        'python-home' => "${virtualenv}",
+        'python-path' => "${django_root}",
       }
     },
     wsgi_process_group      => 'refinery',

deployment/puppet/refinery/manifests/python.pp

@@ -8,6 +8,8 @@
 ) inherits refinery::params {
 
   class { '::python':
+    version    => 'python3.5',
+    ensure     => 'latest',
     dev        => 'present',
     gunicorn   => 'absent',
     pip        => 'latest',
@@ -16,7 +18,8 @@
 
   $base_dependencies = ['build-essential', 'libncurses5-dev']
   $crypto_dependencies = ['libffi-dev', 'libssl-dev']  # cryptography module
-  package { [$base_dependencies, $crypto_dependencies]: }
+  $pysam_dependecies = ['liblzma-dev', 'libbz2-dev', 'zlib1g-dev'] # pysam mod
+  package { [$base_dependencies, $crypto_dependencies, $pysam_dependecies]: }
 
   # for psycopg2 module
   package { 'libpq5':
@@ -41,6 +44,7 @@
   }
   ->
   python::virtualenv { $virtualenv:
+    version => '3.5',
     ensure  => present,
     owner   => $app_user,
     group   => $app_group,
@@ -49,9 +53,10 @@
         $base_dependencies,
         $crypto_dependencies,
         'libpq5',
-        'libpq-dev'
+        'libpq-dev',
+        $pysam_dependecies
       ],
-    ]
+    ],
   }
 
   python::requirements { "${project_root}/requirements.txt":

refinery/analysis_manager/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import models, migrations
 

refinery/analysis_manager/migrations/0002_analysisstatus_galaxy_import_progress.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import models, migrations
 

refinery/analysis_manager/migrations/0003_auto_20170726_1601.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import models, migrations
 

refinery/analysis_manager/migrations/0004_auto_20171211_1145.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import models, migrations
 

refinery/analysis_manager/migrations/0005_analysisstatus_refinery_import_task_group_id.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import migrations, models
 

refinery/analysis_manager/migrations/0006_analysisstatus_galaxy_import_task_group_id.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import migrations, models
 

refinery/analysis_manager/migrations/0007_analysisstatus_galaxy_export_task_group_id.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import migrations, models
 

refinery/analysis_manager/migrations/0008_analysisstatus_galaxy_workflow_task_group_id.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import migrations, models
 

refinery/analysis_manager/models.py

@@ -43,7 +43,7 @@ class AnalysisStatus(models.Model):
     class Meta:
         verbose_name_plural = 'analysis statuses'
 
-    def __unicode__(self):
+    def __str__(self):
         return self.analysis.name
 
     def set_galaxy_history_state(self, state):
@@ -110,7 +110,6 @@ def get_task_group_state(task_group_id):
     """return a list containing states of all tasks given a task set ID"""
     task_group_state = []
     percent_done = 0
-
     taskset = TaskSetResult.restore(task_group_id)
     if not taskset:
         logger.error("TaskSet with UUID '%s' doesn't exist", task_group_id)

refinery/analysis_manager/tasks.py

@@ -3,7 +3,7 @@
 
 @author: nils
 '''
-import urlparse
+from urllib.parse import urljoin
 
 from django.conf import settings
 
@@ -87,7 +87,10 @@ def _check_galaxy_history_state(analysis_uuid):
         run_analysis.retry(countdown=RETRY_INTERVAL)
     else:
         # workaround to avoid moving the progress bar backward
-        if analysis_status.galaxy_history_progress < percent_complete:
+        if not analysis_status.galaxy_history_progress:
+            analysis_status.galaxy_history_progress = percent_complete
+            analysis_status.save()
+        elif analysis_status.galaxy_history_progress < percent_complete:
             analysis_status.galaxy_history_progress = percent_complete
             analysis_status.save()
         if percent_complete < 100:
@@ -486,7 +489,7 @@ def _get_galaxy_download_task_ids(analysis):
     except galaxy.client.ConnectionError as exc:
         error_msg = \
             "Error downloading Galaxy history files for analysis '%s': %s"
-        logger.error(error_msg, analysis.name, exc.message)
+        logger.error(error_msg, analysis.name, exc)
         analysis.set_status(Analysis.FAILURE_STATUS, error_msg)
         analysis.galaxy_cleanup()
         return task_id_list
@@ -499,7 +502,7 @@ def _get_galaxy_download_task_ids(analysis):
             file_extension = results['file_ext']
             # size of file defined by galaxy
             file_size = results['file_size']
-            file_store_item = FileStoreItem(source=urlparse.urljoin(
+            file_store_item = FileStoreItem(source=urljoin(
                 galaxy_instance.base_url,
                 "datasets/{}/display?to_ext=txt".format(results['id'])
             ))

refinery/analysis_manager/test_models.py

@@ -17,7 +17,7 @@ def test_set_galaxy_history_state_with_invalid_state(self):
         with self.assertRaises(ValueError) as context:
             self.analysis_status.set_galaxy_history_state("NOT A VALID STATE")
             self.assertEqual(
-                context.exception.message,
+                str(context.exception),
                 "Invalid Galaxy history state given"
             )
 
@@ -32,7 +32,7 @@ def test_set_galaxy_import_state_with_invalid_state(self):
         with self.assertRaises(ValueError) as context:
             self.analysis_status.set_galaxy_import_state("NOT A VALID STATE")
             self.assertEqual(
-                context.exception.message,
+                str(context.exception),
                 "Invalid Galaxy history state given"
             )
 

refinery/analysis_manager/test_utils.py

@@ -122,7 +122,7 @@ def test_fetch_objects_required_for_analyses_bad_workflow(self):
                     "workflow_uuid": "COFFEE"
                 }
             )
-            self.assertIn("Couldn't fetch Workflow", context.exception.message)
+            self.assertIn("Couldn't fetch Workflow", str(context.exception))
 
     def test_fetch_objects_required_for_analyses_bad_study(self):
         with self.assertRaises(RuntimeError) as context:
@@ -133,7 +133,7 @@ def test_fetch_objects_required_for_analyses_bad_study(self):
                     "workflow_uuid": self.workflow.uuid
                 }
             )
-            self.assertIn("Couldn't fetch Study", context.exception.message)
+            self.assertIn("Couldn't fetch Study", str(context.exception))
 
     def test_fetch_objects_required_for_analyses_bad_user(self):
         with self.assertRaises(RuntimeError) as context:
@@ -144,4 +144,4 @@ def test_fetch_objects_required_for_analyses_bad_user(self):
                     "workflow_uuid": self.workflow.uuid
                 }
             )
-            self.assertIn("Couldn't fetch User", context.exception.message)
+            self.assertIn("Couldn't fetch User", str(context.exception))

refinery/analysis_manager/test_views.py

@@ -109,7 +109,7 @@ def test_analysis_returns_galaxy_file_import_state(
         response = analysis_status(request, self.analysis.uuid)
 
         self.assertEqual(
-            json.loads(response.content),
+            json.loads(response.content.decode()),
             {
                 "galaxyAnalysis": [],
                 "refineryImport": [],

refinery/analysis_manager/views.py

@@ -61,7 +61,7 @@ def analysis_cancel(request):
     # $http Angular service returns json format
     if request.method == 'POST':
         try:
-            uuid = json.loads(request.body)['uuid']
+            uuid = json.loads(request.body.decode())['uuid']
         except KeyError:
             return HttpResponseBadRequest()  # 400
         error_msg = "Cancellation failed for analysis '{}'".format(uuid)

refinery/annotation_server/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 from django.db import models, migrations
 

refinery/annotation_server/migrations/0002_annotation_server_data.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
+
 
 import json
 

refinery/annotation_server/models.py

@@ -16,7 +16,7 @@ class Taxon(models.Model):
     class Meta:
         unique_together = ("taxon_id", "name")
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s: %s" % (self.taxon_id, self.name)
 
 
@@ -31,7 +31,7 @@ class GenomeBuild(models.Model):
     available = models.BooleanField(default=True)
     default_build = models.BooleanField(default=False)
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s: %s" % (self.name, self.description)
 
 
@@ -128,7 +128,7 @@ class CytoBand (models.Model):
     name = models.CharField(max_length=255, db_index=True)
     gieStain = models.CharField(max_length=255)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.chrom + " - " + self.name
 
     class Meta:
@@ -168,7 +168,7 @@ class Gene (models.Model):
     cdsEndStat = models.CharField(max_length=10, db_index=True)
     exonFrames = models.CommaSeparatedIntegerField(max_length=3700)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name + " - " + self.chrom
 
     class Meta:
@@ -190,7 +190,7 @@ class GapRegionFile(models.Model):
     type = models.CharField(max_length=255)
     bridge = models.CharField(max_length=255)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.chrom + ":" + self.chromStart + "-" + self.chromEnd
 
     class Meta:
@@ -209,7 +209,7 @@ class WigDescription(models.Model):
     type = models.CharField(max_length=255)
     description = models.TextField()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name + "=" + self.description + ", " + self.type
 
 
@@ -231,7 +231,7 @@ class BedFile (models.Model):
     blockSizes = models.CommaSeparatedIntegerField(max_length=3700)
     blockStarts = models.CommaSeparatedIntegerField(max_length=3700)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name + " - " + self.chrom + ":" \
                + self.chromStart + "-" + self.chromEnd
 
@@ -255,7 +255,7 @@ class GffFile (models.Model):
     frame = models.CharField(max_length=100)
     attribute = models.TextField()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.feature + " - " + self.chrom + ":" + \
                self.start + "-" + self.end
 
@@ -271,7 +271,7 @@ class GtfFile (GffFile):
     gene_id = models.CharField(max_length=255, db_index=True)
     transcript_id = models.CharField(max_length=255, db_index=True)
 
-    def __unicode__(self):
+    def __str__(self):
         return self.gene_id + " - " + self.chrom + ":" + \
                self.start + "-" + self.end
 
@@ -291,7 +291,7 @@ class WigFile(models.Model):
     position = models.IntegerField(db_index=True)
     value = models.FloatField()
 
-    def __unicode__(self):
+    def __str__(self):
         return self.chrom + ":" + self.position + " = " + self.value
 
     class Meta:

refinery/config/settings/aws.py

@@ -6,11 +6,11 @@
 from .prod import *  # NOQA 405
 
 # Email
-EMAIL_BACKEND = 'django_smtp_ssl.SSLEmailBackend'
+EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
 EMAIL_HOST_USER = get_setting('EMAIL_HOST_USER')
 EMAIL_HOST_PASSWORD = get_setting('EMAIL_HOST_PASSWORD')
 EMAIL_USE_TLS = True
-EMAIL_PORT = 465
+EMAIL_PORT = 587
 
 # Storage
 INSTALLED_APPS += (