Merge pull request #4987 from mvdbeek/history_import_bug

[17.09][Bug] Encode file content with utf-8
galaxyproject · Nov 13, 2017 · a0b4a2d · a0b4a2d
2 parents 222ece4 + 0c9b420
commit a0b4a2d
Showing 1 changed file with 12 additions and 32 deletions.
diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py
@@ -4,7 +4,7 @@
 import os
 import shutil
 import tempfile
-from json import dumps, loads
+from json import dumps, load
 
 from sqlalchemy.orm import eagerload, eagerload_all
 from sqlalchemy.sql import expression
@@ -40,21 +40,6 @@ def file_in_dir(file_path, a_dir):
             abs_file_path = os.path.abspath(file_path)
             return os.path.split(abs_file_path)[0] == a_dir
 
-        def read_file_contents(file_path):
-            """ Read contents of a file. """
-            fp = open(file_path, 'rb')
-            buffsize = 1048576
-            file_contents = ''
-            try:
-                while True:
-                    file_contents += fp.read(buffsize)
-                    if not file_contents or len(file_contents) % buffsize != 0:
-                        break
-            except OverflowError:
-                pass
-            fp.close()
-            return file_contents
-
         def get_tag_str(tag, value):
             """ Builds a tag string for a tag, value pair. """
             if not value:
@@ -84,11 +69,10 @@ def get_tag_str(tag, value):
                 # Create history.
                 #
                 history_attr_file_name = os.path.join(archive_dir, 'history_attrs.txt')
-                history_attr_str = read_file_contents(history_attr_file_name)
-                history_attrs = loads(history_attr_str)
+                history_attrs = load(open(history_attr_file_name))
 
                 # Create history.
-                new_history = model.History(name='imported from archive: %s' % history_attrs['name'].encode('utf-8'),
+                new_history = model.History(name='imported from archive: %s' % history_attrs['name'],
                                             user=user)
                 new_history.importing = True
                 new_history.hid_counter = history_attrs['hid_counter']
@@ -110,12 +94,11 @@ def get_tag_str(tag, value):
                 # Create datasets.
                 #
                 datasets_attrs_file_name = os.path.join(archive_dir, 'datasets_attrs.txt')
-                datasets_attr_str = read_file_contents(datasets_attrs_file_name)
-                datasets_attrs = loads(datasets_attr_str)
+                datasets_attrs = load(open(datasets_attrs_file_name))
+                provenance_file_name = datasets_attrs_file_name + ".provenance"
 
-                if os.path.exists(datasets_attrs_file_name + ".provenance"):
-                    provenance_attr_str = read_file_contents(datasets_attrs_file_name + ".provenance")
-                    provenance_attrs = loads(provenance_attr_str)
+                if os.path.exists(provenance_file_name):
+                    provenance_attrs = load(open(provenance_file_name))
                     datasets_attrs += provenance_attrs
 
                 # Get counts of how often each dataset file is used; a file can
@@ -133,9 +116,9 @@ def get_tag_str(tag, value):
                     metadata = dataset_attrs['metadata']
 
                     # Create dataset and HDA.
-                    hda = model.HistoryDatasetAssociation(name=dataset_attrs['name'].encode('utf-8'),
+                    hda = model.HistoryDatasetAssociation(name=dataset_attrs['name'],
                                                           extension=dataset_attrs['extension'],
-                                                          info=dataset_attrs['info'].encode('utf-8'),
+                                                          info=dataset_attrs['info'],
                                                           blurb=dataset_attrs['blurb'],
                                                           peek=dataset_attrs['peek'],
                                                           designation=dataset_attrs['designation'],
@@ -210,10 +193,6 @@ def get_tag_str(tag, value):
                 # Create jobs.
                 #
 
-                # Read jobs attributes.
-                jobs_attr_file_name = os.path.join(archive_dir, 'jobs_attrs.txt')
-                jobs_attr_str = read_file_contents(jobs_attr_file_name)
-
                 # Decode jobs attributes.
                 def as_hda(obj_dct):
                     """ Hook to 'decode' an HDA; method uses history and HID to get the HDA represented by
@@ -222,7 +201,8 @@ def as_hda(obj_dct):
                         return self.sa_session.query(model.HistoryDatasetAssociation) \
                             .filter_by(history=new_history, hid=obj_dct['hid']).first()
                     return obj_dct
-                jobs_attrs = loads(jobs_attr_str, object_hook=as_hda)
+                jobs_attr_file_name = os.path.join(archive_dir, 'jobs_attrs.txt')
+                jobs_attrs = load(open(jobs_attr_file_name), object_hook=as_hda)
 
                 # Create each job.
                 for job_attrs in jobs_attrs:
@@ -244,7 +224,7 @@ def as_hda(obj_dct):
                     try:
                         imported_job.create_time = datetime.datetime.strptime(job_attrs["create_time"], "%Y-%m-%dT%H:%M:%S.%f")
                         imported_job.update_time = datetime.datetime.strptime(job_attrs["update_time"], "%Y-%m-%dT%H:%M:%S.%f")
-                    except:
+                    except Exception:
                         pass
                     self.sa_session.add(imported_job)
                     self.sa_session.flush()