From 6ba2b8293bcf505844f631c58ee26dbb7a1afff8 Mon Sep 17 00:00:00 2001 From: desimaniac <5501908+desimaniac@users.noreply.github.com> Date: Mon, 20 Jan 2020 22:49:38 -0600 Subject: [PATCH] Rclone: Added support for new `drive-stop-on-upload-limit` flag Ref: - https://github.com/rclone/rclone/issues/3857 - https://github.com/rclone/rclone/commit/58064bdd2b3a001111418656f301d041c3968330 --- README.md | 4 +++- cloudplow.py | 6 +++--- config.json.sample | 1 + utils/rclone.py | 6 ++++-- utils/uploader.py | 7 ++++--- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c9b0939..8e1767e 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,9 @@ Cloudplow has 3 main functions: "--stats": "60s", "--transfers": 8, "--verbose": 1, - "--skip-links": null + "--skip-links": null, + "--drive-stop-on-upload-limit": null, + "--user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36" }, "rclone_sleeps": { "Failed to copy: googleapi: Error 403: User rate limit exceeded": { diff --git a/cloudplow.py b/cloudplow.py index 61c38fc..e682c87 100755 --- a/cloudplow.py +++ b/cloudplow.py @@ -326,8 +326,8 @@ def do_upload(remote=None): else: for i in range(0, available_accounts_size): uploader.set_service_account(available_accounts[i]) - resp, resp_trigger = uploader.upload() - if resp: + resp, resp_trigger, return_code = uploader.upload() + if resp or return_code == 7: current_data = sa_delay[uploader_remote] current_data[available_accounts[i]] = time.time() + ((60 * 60) * resp) sa_delay[uploader_remote] = current_data @@ -374,7 +374,7 @@ def do_upload(remote=None): sa_delay[uploader_remote][available_accounts[i]] = None break else: - resp, resp_trigger = uploader.upload() + resp, resp_trigger, return_code = uploader.upload() if resp: if uploader_remote not in uploader_delay: # this uploader was not already in the delay dict, so lets put it there diff --git a/config.json.sample b/config.json.sample index 46311fa..70cf72f 100644 --- a/config.json.sample +++ b/config.json.sample @@ -53,6 +53,7 @@ "--transfers": 8, "--verbose": 1, "--skip-links": null, + "--drive-stop-on-upload-limit": null, "--user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36" }, "rclone_sleeps": { diff --git a/utils/rclone.py b/utils/rclone.py index 063e2e7..201359f 100644 --- a/utils/rclone.py +++ b/utils/rclone.py @@ -165,8 +165,10 @@ def upload(self, callback): # exec log.debug("Using: %s", cmd) - process.execute(cmd, callback) - return True + return_code = process.execute(cmd, callback) + if return_code == 7: + log.debug("Received 'Transfer Exceeded' response from Rclone.") + return True, return_code except Exception: log.exception("Exception occurred while uploading '%s' to remote: %s", self.config['upload_folder'], self.name) diff --git a/utils/uploader.py b/utils/uploader.py index 1654e9d..e02abc3 100644 --- a/utils/uploader.py +++ b/utils/uploader.py @@ -50,9 +50,10 @@ def upload(self): self.delayed_check = 0 self.delayed_trigger = None self.trigger_tracks = {} - rclone.upload(self.__logic) - log.info("Finished uploading to remote: %s", self.name) - return self.delayed_check, self.delayed_trigger + upload_status, return_code = rclone.upload(self.__logic) + if upload_status: + log.info("Finished uploading to remote: %s", self.name) + return self.delayed_check, self.delayed_trigger, return_code def remove_empty_dirs(self): path.remove_empty_dirs(self.rclone_config['upload_folder'], self.rclone_config['remove_empty_dir_depth'])