Merge pull request #75 from EGA-archive/feature/s3-clients-verification

S3 client verification
EGA-archive · May 24, 2019 · 5ccd152 · 5ccd152
2 parents 116002a + 9410fa5
commit 5ccd152
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 3 deletions.
diff --git a/deploy/bootstrap/run.sh b/deploy/bootstrap/run.sh
@@ -270,6 +270,8 @@ s3_access_key = ${S3_ACCESS_KEY}
 s3_secret_key = ${S3_SECRET_KEY}
 #region = lega
 cacertfile = /etc/ega/CA.cert
+certfile = /etc/ega/ssl.cert
+keyfile = /etc/ega/ssl.key
 EOF
 else
     # POSIX file system

diff --git a/lega/utils/storage.py b/lega/utils/storage.py
@@ -197,18 +197,28 @@ class S3Storage():
     def __init__(self, config_section, user):
         """Initialize S3 object Storage."""
         import boto3
+        import botocore
         self.endpoint = CONF.get_value(config_section, 's3_url')
         region = CONF.get_value(config_section, 's3_region')
         access_key = CONF.get_value(config_section, 's3_access_key')
         secret_key = CONF.get_value(config_section, 's3_secret_key')
         verify = CONF.get_value(config_section, 'cacertfile', default=None) or False
+        config_params = {
+            'connect_timeout': CONF.get_value(config_section, 'connect_timeout', conv=int, default=60),
+        }
+        certfile = CONF.get_value(config_section, 'certfile', default=None)
+        keyfile = CONF.get_value(config_section, 'keyfile', default=None)
+        if certfile and keyfile:
+            config_params['client_cert'] = (certfile, keyfile)
+        config = botocore.client.Config(**config_params)
         self.s3 = boto3.client('s3',
                                endpoint_url=self.endpoint,
                                region_name=region,
                                use_ssl=self.endpoint.startswith('https'),
                                verify=verify,
                                aws_access_key_id=access_key,
-                               aws_secret_access_key=secret_key)
+                               aws_secret_access_key=secret_key,
+                               config=config)
         # LOG.debug(f'S3 client: {self.s3!r}')
         try:
             LOG.debug('Creating "%s" bucket', user)

diff --git a/tests/unit/test_storage.py b/tests/unit/test_storage.py
@@ -6,6 +6,7 @@
 from io import UnsupportedOperation, BufferedReader
 from unittest import mock
 import boto3
+import botocore
 
 
 class TestFileStorage(unittest.TestCase):
@@ -116,24 +117,32 @@ def setUp(self):
         self.env.set('ARCHIVE_S3_REGION', 'lega')
         self.env.set('ARCHIVE_S3_ACCESS_KEY', 'test')
         self.env.set('ARCHIVE_S3_SECRET_KEY', 'test')
+        self.env.set('ARCHIVE_CACERTFILE', '/etc/ega/CA.cert')
+        self.env.set('ARCHIVE_CERTFILE', '/etc/ega/ssl.cert')
+        self.env.set('ARCHIVE_KEYFILE', '/etc/ega/ssl.key')
 
     def tearDown(self):
         """Remove setup variables."""
         self.env.unset('ARCHIVE_S3_URL')
         self.env.unset('ARCHIVE_S3_REGION')
         self.env.unset('ARCHIVE_S3_ACCESS_KEY')
         self.env.unset('ARCHIVE_S3_SECRET_KEY')
+        self.env.unset('ARCHIVE_CACERTFILE')
+        self.env.unset('ARCHIVE_CERTFILE')
+        self.env.unset('ARCHIVE_KEYFILE')
         self._dir.cleanup_all()
 
+    @mock.patch.object(botocore.client, 'Config')
     @mock.patch.object(boto3, 'client')
-    def test_upload(self, mock_boto):
+    def test_upload(self, mock_boto, mock_botocore):
         """Test copy to S3, should call boto3 client."""
         path = self._dir.write('test.file', 'data1'.encode('utf-8'))
         storage = S3Storage('archive', 'lega')
         storage.copy(path, 'lega')
+        mock_botocore.assert_called_with(connect_timeout=60, client_cert=('/etc/ega/ssl.cert', '/etc/ega/ssl.key'))
         mock_boto.assert_called_with('s3', aws_access_key_id='test', aws_secret_access_key='test',
                                      endpoint_url='https://localhost:5000', region_name='lega',
-                                     use_ssl=True, verify=False)
+                                     use_ssl=True, verify='/etc/ega/CA.cert', config=mock_botocore())
 
 
 class TestS3FileReader(unittest.TestCase):