From 4f59569fd9b6f54c641805ef950bab705cc5ca52 Mon Sep 17 00:00:00 2001 From: parthshahva Date: Thu, 7 Feb 2019 14:10:06 -0800 Subject: [PATCH] reduce threads running on upload and reraise error from Tenacy on upload methods --- hca/upload/api_client.py | 2 +- hca/upload/s3_agent.py | 4 ++-- hca/util/pool.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hca/upload/api_client.py b/hca/upload/api_client.py index c03e192c..c6cd5d42 100644 --- a/hca/upload/api_client.py +++ b/hca/upload/api_client.py @@ -33,7 +33,7 @@ def credentials(self, area_uuid): content=response.content)) return response.json() - @retry(wait=wait_fixed(2), stop=stop_after_attempt(3)) + @retry(reraise=True, wait=wait_fixed(2), stop=stop_after_attempt(3)) def file_upload_notification(self, area_uuid, filename): url = "{api_url_base}/area/{area_uuid}/{filename}".format(api_url_base=self.api_url_base, area_uuid=area_uuid, diff --git a/hca/upload/s3_agent.py b/hca/upload/s3_agent.py index 9a869f6c..1ce5a3de 100644 --- a/hca/upload/s3_agent.py +++ b/hca/upload/s3_agent.py @@ -65,7 +65,7 @@ def should_write_to_terminal(self): write_to_terminal = True return write_to_terminal - @retry(wait=wait_fixed(2), stop=stop_after_attempt(3)) + @retry(reraise=True, wait=wait_fixed(2), stop=stop_after_attempt(3)) def copy_s3_file(self, s3_path, target_bucket, target_key, content_type, report_progress=False): # Here we are using s3's managed copy to allow for s3 to s3 file upload # We override any original metadata or content types @@ -94,7 +94,7 @@ def copy_s3_file(self, s3_path, target_bucket, target_key, content_type, report_ upload_args['Callback'] = self.upload_progress_callback self.target_s3.meta.client.copy(**upload_args) - @retry(wait=wait_fixed(2), stop=stop_after_attempt(3)) + @retry(reraise=True, wait=wait_fixed(2), stop=stop_after_attempt(3)) def upload_local_file(self, local_path, target_bucket, target_key, content_type, report_progress=False): file_size = os.path.getsize(local_path) bucket = self.target_s3.Bucket(target_bucket) diff --git a/hca/util/pool.py b/hca/util/pool.py index 765c0e6c..761a65a3 100644 --- a/hca/util/pool.py +++ b/hca/util/pool.py @@ -10,7 +10,7 @@ """Based on https://askubuntu.com/questions/668538/cores-vs-threads-how-many-threads-should-i-run-on-this-machine and https://github.com/bloomreach/s4cmd/blob/master/s4cmd.py#L121.""" -DEFAULT_THREAD_COUNT = multiprocessing.cpu_count() * 4 +DEFAULT_THREAD_COUNT = multiprocessing.cpu_count() * 2 class Worker(Thread):