Merge pull request #162 from MichaelAquilina/support_minio

Add support for other services like Minio

s4/clients/s3.py

@@ -21,13 +21,16 @@
 S3Uri = collections.namedtuple("S3Uri", ["bucket", "key"])
 
 
-def get_s3_client(target, aws_access_key_id, aws_secret_access_key, region_name):
+def get_s3_client(
+    target, aws_access_key_id, aws_secret_access_key, endpoint_url, region_name
+):
     s3_uri = parse_s3_uri(target)
     s3_client = boto3.client(
         "s3",
         aws_access_key_id=aws_access_key_id,
         aws_secret_access_key=aws_secret_access_key,
         region_name=region_name,
+        endpoint_url=endpoint_url,
     )
     return S3SyncClient(s3_client, s3_uri.bucket, s3_uri.key)
 

s4/commands/__init__.py

@@ -21,6 +21,7 @@ def get_clients(self, entry):
         target_2 = entry["s3_uri"]
         aws_access_key_id = entry["aws_access_key_id"]
         aws_secret_access_key = entry["aws_secret_access_key"]
+        endpoint_url = entry.get("endpoint_url", None)
         region_name = entry["region_name"]
 
         # append trailing slashes to prevent incorrect prefix matching on s3
@@ -31,6 +32,10 @@ def get_clients(self, entry):
 
         client_1 = get_local_client(target_1)
         client_2 = get_s3_client(
-            target_2, aws_access_key_id, aws_secret_access_key, region_name
+            target_2,
+            aws_access_key_id,
+            aws_secret_access_key,
+            endpoint_url,
+            region_name,
         )
         return client_1, client_2

s4/commands/add_command.py

@@ -14,8 +14,18 @@ def run(self):
             self.logger.info("Choices are: %s", all_targets)
             return
 
+        self.logger.info("To add a new target, please enter the following\n")
+
         entry = {}
-        entry["local_folder"] = os.path.expanduser(utils.get_input("local folder: "))
+        local_folder = utils.get_input(
+            "local folder (leave blank for current folder): "
+        )
+        if not local_folder:
+            local_folder = os.getcwd()
+
+        entry["local_folder"] = os.path.expanduser(local_folder)
+
+        entry["endpoint_url"] = utils.get_input("endpoint url (leave blank for AWS): ")
         entry["s3_uri"] = utils.get_input("s3 uri: ")
         entry["region_name"] = utils.get_input("region name: ")
 

s4/commands/edit_command.py

@@ -21,12 +21,14 @@ def run(self):
         entry = self.config["targets"][self.args.target]
 
         local_folder = entry.get("local_folder", "")
+        endpoint_url = entry.get("endpoint_url")
         s3_uri = entry.get("s3_uri", "")
         aws_access_key_id = entry.get("aws_access_key_id")
         aws_secret_access_key = entry.get("aws_secret_access_key")
         region_name = entry.get("region_name")
 
         new_local_folder = utils.get_input("local folder [{}]: ".format(local_folder))
+        new_endpoint_url = utils.get_input("endpoint url [{}]: ".format(endpoint_url))
         new_s3_uri = utils.get_input("s3 uri [{}]: ".format(s3_uri))
         new_aws_access_key_id = utils.get_input(
             "AWS Access Key ID [{}]: ".format(aws_access_key_id)
@@ -45,6 +47,8 @@ def run(self):
             entry["aws_access_key_id"] = new_aws_access_key_id
         if new_aws_secret_access_key:
             entry["aws_secret_access_key"] = new_aws_secret_access_key
+        if new_endpoint_url:
+            entry["endpoint_url"] = new_endpoint_url
         if new_region_name:
             entry["region_name"] = new_region_name
 

s4/commands/sync_command.py

@@ -109,7 +109,7 @@ def run(self):
                         self.logger.exception(e)
                     else:
                         self.logger.error(
-                            "There was an error syncing '%s': %s", name, e
+                            "There was an error syncing '%s':\n%s", name, e
                         )
 
         except KeyboardInterrupt:

s4/sync.py

@@ -299,7 +299,7 @@ def run_resolutions(self, resolutions, dry_run=False):
                     success.append(key)
                 except Exception as e:
                     self.logger.error(
-                        "An error occurred while trying to update %s: %s", key, e
+                        "An error occurred while trying to update %s:\n%s", key, e
                     )
                     self.logger.debug(traceback.format_exc())
         except KeyboardInterrupt:

s4/utils.py

@@ -18,7 +18,9 @@ def get_input(*args, secret=False, **kwargs):
     if secret:
         return getpass.getpass(*args, **kwargs)
     else:
-        return input(*args, **kwargs)
+        value = input(*args, **kwargs)
+        # Normalise empty inputs to None
+        return value if value else None
 
 
 def get_config():
@@ -35,4 +37,4 @@ def set_config(config):
         os.makedirs(CONFIG_FOLDER_PATH)
 
     with open(CONFIG_FILE_PATH, "w") as fp:
-        json.dump(config, fp)
+        json.dump(config, fp, indent=5)

tests/commands/test_add_command.py

@@ -2,6 +2,7 @@
 
 import argparse
 import json
+import os
 
 import mock
 
@@ -16,6 +17,7 @@ def test_correct_behaviour(self, get_input, config_file):
         fake_stream = utils.FakeInputStream(
             [
                 "/home/user/Documents",
+                None,
                 "s3://mybucket/Documents",
                 "eu-west-2",
                 "aaaaaaaaaaaaaaaaaaaaaaaa",
@@ -36,6 +38,7 @@ def test_correct_behaviour(self, get_input, config_file):
             "targets": {
                 "Documents": {
                     "local_folder": "/home/user/Documents",
+                    "endpoint_url": None,
                     "s3_uri": "s3://mybucket/Documents",
                     "aws_access_key_id": "aaaaaaaaaaaaaaaaaaaaaaaa",
                     "aws_secret_access_key": "bbbbbbbbbbbbbbbbbbbbbbbb",
@@ -45,9 +48,32 @@ def test_correct_behaviour(self, get_input, config_file):
         }
         assert new_config == expected_config
 
+    def test_default_local_folder(self, get_input, config_file):
+        fake_stream = utils.FakeInputStream(
+            [
+                None,
+                None,
+                "s3://mybucket/Documents",
+                "eu-west-2",
+                "aaaaaaaaaaaaaaaaaaaaaaaa",
+                "bbbbbbbbbbbbbbbbbbbbbbbb",
+                "",
+            ]
+        )
+        get_input.side_effect = fake_stream
+        args = argparse.Namespace(copy_target_credentials=None)
+
+        command = AddCommand(args, {"targets": {}}, utils.create_logger())
+        command.run()
+
+        with open(config_file, "r") as fp:
+            config = json.load(fp)
+
+        assert config["targets"]["Documents"]["local_folder"] == os.getcwd()
+
     def test_copy_target_credentials(self, get_input, config_file):
         fake_stream = utils.FakeInputStream(
-            ["/home/user/Animals", "s3://mybucket/Zoo", "us-west-2", "Beasts"]
+            ["/home/user/Animals", None, "s3://mybucket/Zoo", "us-west-2", "Beasts"]
         )
         get_input.side_effect = fake_stream
         args = argparse.Namespace(copy_target_credentials="bar")
@@ -77,6 +103,7 @@ def test_copy_target_credentials(self, get_input, config_file):
                 },
                 "Beasts": {
                     "local_folder": "/home/user/Animals",
+                    "endpoint_url": None,
                     "s3_uri": "s3://mybucket/Zoo",
                     "aws_access_key_id": "so-much-bar",
                     "aws_secret_access_key": "bar-secretz",
@@ -88,7 +115,7 @@ def test_copy_target_credentials(self, get_input, config_file):
 
     def test_copy_target_credentials_bad_target(self, get_input, capsys):
         fake_stream = utils.FakeInputStream(
-            ["/home/user/Animals", "s3://mybucket/Zoo", "us-west-2", "Beasts"]
+            ["/home/user/Animals", "", "s3://mybucket/Zoo", "us-west-2", "Beasts"]
         )
         get_input.side_effect = fake_stream
         args = argparse.Namespace(copy_target_credentials="Foo")
@@ -104,6 +131,7 @@ def test_custom_target_name(self, get_input, config_file):
         fake_stream = utils.FakeInputStream(
             [
                 "/home/user/Music",
+                None,
                 "s3://mybucket/Musiccccc",
                 "us-west-1",
                 "1234567890",
@@ -124,6 +152,7 @@ def test_custom_target_name(self, get_input, config_file):
             "targets": {
                 "Tunes": {
                     "local_folder": "/home/user/Music",
+                    "endpoint_url": None,
                     "s3_uri": "s3://mybucket/Musiccccc",
                     "aws_access_key_id": "1234567890",
                     "aws_secret_access_key": "abcdefghij",

tests/commands/test_edit_command.py

@@ -31,14 +31,15 @@ def test_missing_target(self, get_input, capsys):
         assert err == ('"idontexist" is an unknown target\n' "Choices are: ['foo']\n")
 
     def test_no_changes(self, get_input, config_file):
-        fake_stream = utils.FakeInputStream(["", "", "", "", ""])
+        fake_stream = utils.FakeInputStream(["", "", "", "", "", ""])
         get_input.side_effect = fake_stream
 
         args = argparse.Namespace(target="foo")
         config = {
             "targets": {
                 "foo": {
                     "local_folder": "/home/mark/documents",
+                    "endpoint_url": "https://customurl.com",
                     "s3_uri": "s3://buckets/mybackup",
                     "aws_access_key_id": "23123123123313",
                     "aws_secret_access_key": "edwdadwadwadwdd",
@@ -56,6 +57,7 @@ def test_no_changes(self, get_input, config_file):
             "targets": {
                 "foo": {
                     "local_folder": "/home/mark/documents",
+                    "endpoint_url": "https://customurl.com",
                     "s3_uri": "s3://buckets/mybackup",
                     "aws_access_key_id": "23123123123313",
                     "aws_secret_access_key": "edwdadwadwadwdd",
@@ -69,6 +71,7 @@ def test_correct_output(self, get_input, config_file):
         fake_stream = utils.FakeInputStream(
             [
                 "/home/user/Documents",
+                "https://example.com",
                 "s3://buckets/mybackup222",
                 "9999999999",
                 "bbbbbbbbbbbbbbbbbbbbbbbb",
@@ -82,6 +85,7 @@ def test_correct_output(self, get_input, config_file):
             "targets": {
                 "foo": {
                     "local_folder": "/home/mark/documents",
+                    "endpoint_url": "",
                     "s3_uri": "s3://buckets/mybackup",
                     "aws_access_key_id": "23123123123313",
                     "aws_secret_access_key": "edwdadwadwadwdd",
@@ -99,6 +103,7 @@ def test_correct_output(self, get_input, config_file):
             "targets": {
                 "foo": {
                     "local_folder": "/home/user/Documents",
+                    "endpoint_url": "https://example.com",
                     "s3_uri": "s3://buckets/mybackup222",
                     "aws_access_key_id": "9999999999",
                     "aws_secret_access_key": "bbbbbbbbbbbbbbbbbbbbbbbb",

tests/commands/test_sync_command.py

@@ -130,8 +130,8 @@ def test_sync_error(self, SyncWorker, capsys):
         assert SyncWorker.call_count == 2
         assert out == ""
         assert err == (
-            "There was an error syncing 'bar': something bad happened\n"
-            "There was an error syncing 'foo': something bad happened\n"
+            "There was an error syncing 'bar':\nsomething bad happened\n"
+            "There was an error syncing 'foo':\nsomething bad happened\n"
         )
 
     def test_sync_error_debug(self, SyncWorker, capsys):