Merge branch '0.5.0-prerelease' into development

StanfordBioinformatics · Aug 15, 2017 · 6d33ef2 · 6d33ef2
2 parents 8e4c2de + 3923c3f
commit 6d33ef2
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 31 deletions.
diff --git a/bin/run_master.sh b/bin/run_master.sh
@@ -35,4 +35,4 @@ done
 $BIN_PATH/../loomengine/master/manage.py migrate
 $BIN_PATH/../loomengine/master/manage.py collectstatic --noinput
 
-gunicorn loomengine.master.master.wsgi --bind ${LOOM_MASTER_INTERNAL_IP}:${LOOM_MASTER_INTERNAL_PORT} --log-level ${LOOM_LOG_LEVEL} --capture-output -w ${LOOM_MASTER_GUNICORN_WORKERS_COUNT}
+gunicorn loomengine.master.master.wsgi --bind ${LOOM_MASTER_INTERNAL_IP}:${LOOM_MASTER_INTERNAL_PORT} --log-level ${LOOM_LOG_LEVEL} --capture-output -w ${LOOM_MASTER_GUNICORN_WORKERS_COUNT} --timeout 300
diff --git a/loomengine/utils/filemanager.py b/loomengine/utils/filemanager.py
@@ -11,12 +11,7 @@
 import tempfile
 import time
 import urlparse
-from google.auth.exceptions import TransportError
 import google.cloud.storage
-import google.cloud.streaming
-from google.cloud.exceptions import GoogleCloudError
-import requests
-from socket import error as SocketError
 
 from loomengine.utils import execute_with_retries
 from loomengine.utils import md5calc
@@ -209,12 +204,11 @@ def __init__(self, url, settings, retry=False):
 
         self.settings = settings
 
-        RETRYABLE_EXCEPTIONS = (GoogleCloudError, TransportError, SocketError)
         try:
             self.client = execute_with_retries(
                 lambda: google.cloud.storage.client.Client(
                     self.settings['GCE_PROJECT']),
-                RETRYABLE_EXCEPTIONS,
+                (Exception,),
                 logger,
                 'Get client')
         except ApplicationDefaultCredentialsError as e:
@@ -225,16 +219,14 @@ def __init__(self, url, settings, retry=False):
 
         try:
             if self.retry:
-                RETRYABLE_EXCEPTIONS = (
-                    GoogleCloudError, TransportError, SocketError)
                 self.bucket = execute_with_retries(
                     lambda: self.client.get_bucket(self.bucket_id),
-                    RETRYABLE_EXCEPTIONS,
+                    (Exception,),
                     logger,
                     'Get bucket')
                 self.blob = execute_with_retries(
                     lambda: self.bucket.get_blob(self.blob_id),
-                    RETRYABLE_EXCEPTIONS,
+                    (Exception,),
                     logger,
                     'Get blob')
             else:
@@ -354,23 +346,22 @@ def __init__(self, url, settings, retry=False):
         assert self.url.scheme == 'gs'
         self.bucket_id = self.url.hostname
         self.blob_id = self.url.path.lstrip('/')
-        RETRYABLE_EXCEPTIONS = (GoogleCloudError, TransportError, SocketError)
         self.client = execute_with_retries(
             lambda: google.cloud.storage.client.Client(
                 self.settings['GCE_PROJECT']),
-            RETRYABLE_EXCEPTIONS,
+            (Exception,),
             logger,
             'Get client')
         try:
             if self.retry:
                 self.bucket = execute_with_retries(
                     lambda: self.client.get_bucket(self.bucket_id),
-                    RETRYABLE_EXCEPTIONS,
+                    (Exception,),
                     logger,
                     'Get bucket')
                 self.blob = execute_with_retries(
                     lambda: self.bucket.get_blob(self.blob_id),
-                    RETRYABLE_EXCEPTIONS,
+                    (Exception,),
                     logger,
                     'Get blob')
             else:
@@ -461,15 +452,11 @@ def copy(self):
 class Local2GoogleStorageCopier(AbstractCopier):
 
     def copy(self):
-        RETRYABLE_EXCEPTIONS = [
-            e for e in
-            google.cloud.streaming.http_wrapper._RETRYABLE_EXCEPTIONS
-        ] + [GoogleCloudError, TransportError, SocketError]
         if self.source.retry or self.destination.retry:
             execute_with_retries(
                 lambda: self.destination.blob.upload_from_filename(
                     self.source.get_path()),
-                RETRYABLE_EXCEPTIONS,
+                (Exception,),
                 logger,
                 'File upload')
         else:
@@ -490,14 +477,10 @@ def copy(self):
                     (os.path.dirname(self.destination.get_path()),
                      e))
         if self.source.retry or self.destination.retry:
-            RETRYABLE_EXCEPTIONS = [
-                e for e in
-                google.cloud.streaming.http_wrapper._RETRYABLE_EXCEPTIONS
-            ] + [GoogleCloudError, TransportError, SocketError]
             execute_with_retries(
                 lambda: self.source.blob.download_to_filename(
                     self.destination.get_path()),
-                RETRYABLE_EXCEPTIONS,
+                (Exception,),
                 logger,
                 'File download')
         else:

diff --git a/requirements.txt b/requirements.txt
@@ -25,13 +25,13 @@ flower==0.9.1
 functools32==3.2.3-2
 future==0.16.0
 futures==3.0.5
-gax-google-logging-v2==0.8.3
-gax-google-pubsub-v1==0.8.3
+#gax-google-logging-v2==0.8.3
+#gax-google-pubsub-v1==0.8.3
 google-api-python-client==1.6.2
 google-apitools==0.5.8
-google-cloud==0.23.0
+google-cloud==0.26.1
 google-compute-engine==2.3.5
-google-gax==0.15.9
+#google-gax==0.15.9
 googleapis-common-protos==1.5.2
 greenlet==0.4.12
 grpc-google-logging-v2==0.8.1

diff --git a/setup.py b/setup.py
@@ -102,7 +102,7 @@
                         'flower==0.9.1',
                         'google-api-python-client==1.6.2',
                         'google-apitools>=0.5.8',
-                        'google-cloud==0.23.0',
+                        'google-cloud==0.26.1',
                         'gunicorn>=19.7.1',
                         'Jinja2==2.9.6',
                         'jsonfield==2.0.1',