Merge pull request #85 from galaxyproject/update_to_latest_galaxy

Upgrade to latest galaxy.

galaxy/jobs/metrics/__init__.py

@@ -16,7 +16,6 @@
 
 from galaxy import util
 from galaxy.util import plugin_config
-
 from ..metrics import formatting
 
 log = logging.getLogger(__name__)

galaxy/jobs/metrics/collectl/processes.py

@@ -8,7 +8,6 @@
 import tempfile
 
 from galaxy import util
-
 from ..collectl import stats
 
 if sys.version_info > (3,):

galaxy/jobs/metrics/instrumenters/collectl.py

@@ -4,7 +4,6 @@
 import shutil
 
 from galaxy import util
-
 from ..collectl import (
     cli,
     processes,

galaxy/jobs/metrics/instrumenters/core.py

@@ -8,6 +8,7 @@
 log = logging.getLogger(__name__)
 
 GALAXY_SLOTS_KEY = "galaxy_slots"
+GALAXY_MEMORY_MB_KEY = "galaxy_memory_mb"
 START_EPOCH_KEY = "start_epoch"
 END_EPOCH_KEY = "end_epoch"
 RUNTIME_SECONDS_KEY = "runtime_seconds"
@@ -19,6 +20,8 @@ def format(self, key, value):
         value = int(value)
         if key == GALAXY_SLOTS_KEY:
             return ("Cores Allocated", "%d" % value)
+        elif key == GALAXY_MEMORY_MB_KEY:
+            return ("Memory Allocated (MB)", "%d" % value)
         elif key == RUNTIME_SECONDS_KEY:
             return ("Job Runtime (Wall Clock)", formatting.seconds_to_str(value))
         else:
@@ -40,6 +43,7 @@ def __init__(self, **kwargs):
     def pre_execute_instrument(self, job_directory):
         commands = []
         commands.append(self.__record_galaxy_slots_command(job_directory))
+        commands.append(self.__record_galaxy_memory_mb_command(job_directory))
         commands.append(self.__record_seconds_since_epoch_to_file(job_directory, "start"))
         return commands
 
@@ -50,9 +54,11 @@ def post_execute_instrument(self, job_directory):
 
     def job_properties(self, job_id, job_directory):
         galaxy_slots_file = self.__galaxy_slots_file(job_directory)
+        galaxy_memory_mb_file = self.__galaxy_memory_mb_file(job_directory)
 
         properties = {}
         properties[GALAXY_SLOTS_KEY] = self.__read_integer(galaxy_slots_file)
+        properties[GALAXY_MEMORY_MB_KEY] = self.__read_integer(galaxy_memory_mb_file)
         start = self.__read_seconds_since_epoch(job_directory, "start")
         end = self.__read_seconds_since_epoch(job_directory, "end")
         if start is not None and end is not None:
@@ -65,6 +71,10 @@ def __record_galaxy_slots_command(self, job_directory):
         galaxy_slots_file = self.__galaxy_slots_file(job_directory)
         return '''echo "$GALAXY_SLOTS" > '%s' ''' % galaxy_slots_file
 
+    def __record_galaxy_memory_mb_command(self, job_directory):
+        galaxy_memory_mb_file = self.__galaxy_memory_mb_file(job_directory)
+        return '''echo "$GALAXY_MEMORY_MB" > '%s' ''' % galaxy_memory_mb_file
+
     def __record_seconds_since_epoch_to_file(self, job_directory, name):
         path = self._instrument_file_path(job_directory, "epoch_%s" % name)
         return 'date +"%s" > ' + path
@@ -76,6 +86,9 @@ def __read_seconds_since_epoch(self, job_directory, name):
     def __galaxy_slots_file(self, job_directory):
         return self._instrument_file_path(job_directory, "galaxy_slots")
 
+    def __galaxy_memory_mb_file(self, job_directory):
+        return self._instrument_file_path(job_directory, "galaxy_memory_mb")
+
     def __read_integer(self, path):
         value = None
         try:

galaxy/jobs/metrics/instrumenters/cpuinfo.py

@@ -3,7 +3,6 @@
 import re
 
 from galaxy import util
-
 from ..instrumenters import InstrumentPlugin
 from ...metrics import formatting
 

galaxy/jobs/metrics/instrumenters/meminfo.py

@@ -3,7 +3,6 @@
 import sys
 
 from galaxy import util
-
 from ..instrumenters import InstrumentPlugin
 from ...metrics import formatting
 

galaxy/objectstore/__init__.py

@@ -10,7 +10,6 @@
 import random
 import shutil
 import threading
-
 from xml.etree import ElementTree
 
 try:
@@ -22,11 +21,13 @@
 from galaxy.util import (
     directory_hash_id,
     force_symlink,
-    safe_makedirs,
-    safe_relpath,
     umask_fix_perms,
 )
 from galaxy.util.odict import odict
+from galaxy.util.path import (
+    safe_makedirs,
+    safe_relpath,
+)
 from galaxy.util.sleeper import Sleeper
 
 NO_SESSION_ERROR_MESSAGE = "Attempted to 'create' object store entity in configuration with no database session present."
@@ -722,6 +723,9 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None):
     elif store == 's3':
         from .s3 import S3ObjectStore
         return S3ObjectStore(config=config, config_xml=config_xml)
+    elif store == 'cloud':
+        from .cloud import Cloud
+        return Cloud(config=config, config_xml=config_xml)
     elif store == 'swift':
         from .s3 import SwiftObjectStore
         return SwiftObjectStore(config=config, config_xml=config_xml)

galaxy/objectstore/azure_blob.py

@@ -7,14 +7,8 @@
 import shutil
 import threading
 import time
-
 from datetime import datetime
 
-from galaxy.exceptions import ObjectInvalid, ObjectNotFound
-from galaxy.util import directory_hash_id, safe_relpath, umask_fix_perms
-from galaxy.util.sleeper import Sleeper
-from ..objectstore import convert_bytes, ObjectStore
-
 try:
     from azure.common import AzureHttpError
     from azure.storage import CloudStorageAccount
@@ -23,6 +17,21 @@
 except ImportError:
     BlockBlobService = None
 
+from galaxy.exceptions import (
+    ObjectInvalid,
+    ObjectNotFound
+)
+from galaxy.util import (
+    directory_hash_id,
+    umask_fix_perms
+)
+from galaxy.util.path import safe_relpath
+from galaxy.util.sleeper import Sleeper
+from ..objectstore import (
+    convert_bytes,
+    ObjectStore
+)
+
 NO_BLOBSERVICE_ERROR_MESSAGE = ("ObjectStore configured, but no azure.storage.blob dependency available."
                                 "Please install and properly configure azure.storage.blob or modify Object Store configuration.")