Skip to content

reduce threads running on upload and reraise error from tenacity on upload methods #228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
parthshahva merged 1 commit into from
Feb 12, 2019
Merged

reduce threads running on upload and reraise error from tenacity on upload methods #228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hca/upload/api_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def credentials(self, area_uuid):
content=response.content))
return response.json()

@retry(wait=wait_fixed(2), stop=stop_after_attempt(3))
@retry(reraise=True, wait=wait_fixed(2), stop=stop_after_attempt(3))
def file_upload_notification(self, area_uuid, filename):
url = "{api_url_base}/area/{area_uuid}/{filename}".format(api_url_base=self.api_url_base,
area_uuid=area_uuid,
Expand Down
4 changes: 2 additions & 2 deletions hca/upload/s3_agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def should_write_to_terminal(self):
write_to_terminal = True
return write_to_terminal

@retry(wait=wait_fixed(2), stop=stop_after_attempt(3))
@retry(reraise=True, wait=wait_fixed(2), stop=stop_after_attempt(3))
def copy_s3_file(self, s3_path, target_bucket, target_key, content_type, report_progress=False):
# Here we are using s3's managed copy to allow for s3 to s3 file upload
# We override any original metadata or content types
Expand Down Expand Up @@ -94,7 +94,7 @@ def copy_s3_file(self, s3_path, target_bucket, target_key, content_type, report_
upload_args['Callback'] = self.upload_progress_callback
self.target_s3.meta.client.copy(**upload_args)

@retry(wait=wait_fixed(2), stop=stop_after_attempt(3))
@retry(reraise=True, wait=wait_fixed(2), stop=stop_after_attempt(3))
def upload_local_file(self, local_path, target_bucket, target_key, content_type, report_progress=False):
file_size = os.path.getsize(local_path)
bucket = self.target_s3.Bucket(target_bucket)
Expand Down
2 changes: 1 addition & 1 deletion hca/util/pool.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

"""Based on https://askubuntu.com/questions/668538/cores-vs-threads-how-many-threads-should-i-run-on-this-machine
and https://github.com/bloomreach/s4cmd/blob/master/s4cmd.py#L121."""
DEFAULT_THREAD_COUNT = multiprocessing.cpu_count() * 4
DEFAULT_THREAD_COUNT = multiprocessing.cpu_count() * 2


class Worker(Thread):
Expand Down