implementation of aws s3 (unfinished)

docker-dev/requirements.txt

@@ -13,3 +13,4 @@ cryptography>=35.0.0
 pyjwt>=2.3.0
 gunicorn==20.1.0
 mongodb-migrations>=1.2.1
+boto3>=1.15.0

requirements.txt

@@ -13,3 +13,4 @@ cryptography>=35.0.0
 pyjwt>=2.3.0
 gunicorn==20.1.0
 mongodb-migrations>=1.2.1
+boto3>=1.15.0

src/settings.py

@@ -283,3 +283,9 @@
 PROPAGATE_TOKEN = environ.get("BAE_CB_PROPAGATE_TOKEN", PROPAGATE_TOKEN)
 if isinstance(PROPAGATE_TOKEN, str):
     PROPAGATE_TOKEN = PROPAGATE_TOKEN == "True"
+
+AWS_ACCESS_KEY_ID=environ.get("AWS_ACCESS_KEY_ID", '')
+AWS_SECRET_ACCESS_KEY=environ.get("AWS_ACCESS_KEY_ID", '')
+BUCKET_NAME = environ.get("AWS_ACCESS_KEY_ID", '')
+ACL_ENABLED=environ.get("AWS_ACCESS_KEY_ID", False)
+AWS_ENABLED= environ.get("AWS_ACCESS_KEY_ID", True)

src/wstore/asset_manager/asset_manager.py

@@ -35,17 +35,22 @@
 from wstore.store_commons.rollback import downgrade_asset, downgrade_asset_pa, rollback
 from wstore.store_commons.utils.name import is_valid_file
 from wstore.store_commons.utils.url import is_valid_url, url_fix
-
+import boto3
 logger = getLogger("wstore.default_logger")
 
 
 class AssetManager:
     def __init__(self):
-        pass
+        # AWS credentials
+        if settings.AWS_ENABLED:
+            self.s3 = boto3.client(
+                's3',
+                aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
+        )
 
-    def _save_resource_file(self, provider, file_):
+    def _save_resource_file(self, provider, file_, isPublic):
         logger.debug(f"Saving resource file {file_['name']}")
-
         # Load file contents
         if isinstance(file_, dict):
             file_name = file_["name"]
@@ -54,11 +59,67 @@ def _save_resource_file(self, provider, file_):
             file_name = file_.name
             file_.seek(0)
             content = file_.read()
-
         # Check file name
         if not is_valid_file(file_name):
             logger.debug(f"`{file_name}` is not a valid file name")
             raise ValueError("Invalid file name format: Unsupported character")
+        if isPublic is True and settings.AWS_ENABLED:
+            return self.__save_resource_aws(file_name, content)
+        else:
+            return self.__save_resource_local(provider, file_name, content)
+
+
+    def __save_resource_aws(self,  file_name, content):
+
+        # Create temp dir for assets if it does not exist
+        provider_dir = os.path.join(settings.MEDIA_ROOT, "assets", "temp")
+
+        if not os.path.isdir(provider_dir):
+            os.makedirs(provider_dir, exist_ok=True)
+        # Local file path for the asset file 
+        file_path = os.path.join(provider_dir, file_name)
+        logger.debug(f"File path for asset file: {file_path}")
+        # Key of s3
+        resource_path = os.path.join(settings.MEDIA_DIR, file_name)
+
+        if resource_path.startswith("/"):
+            resource_path = resource_path[1:]
+
+        # Check if the file already exists
+        if os.path.exists(file_path):
+            logger.error(f"The asset file `{file_name}` already exists")
+            raise ConflictError(f"The provided digital asset file ({file_name}) already exists")
+
+        logger.debug("Paths needed for saving temp resource file OK")
+
+        # Create file
+        with open(file_path, "wb") as f:
+            f.write(content)
+
+        self.rollback_logger["files"].append(file_path)
+
+        logger.debug("Asset temp file created")
+        # upload the file to S3
+        acl = {'ACL': 'public-read'} if settings.ACL_ENABLED else None
+        self.s3.upload_file(file_path, settings.BUCKET_NAME, resource_path, ExtraArgs=acl)
+
+        logger.debug(f'The file {file_name} is uploaded to {resource_path} in s3')
+
+        # delete file in local
+        try:
+            logger.debug(f"Deleting file {file_path}")
+            os.remove(file_path)
+            logger.debug(f"Deleting file {file_path}2")
+        except:
+            raise ConflictError(f"The provided digital asset file ({file_name}) cannot be removed")
+        site = settings.SITE
+        location = self.s3.get_bucket_location(Bucket=settings.BUCKET_NAME)['LocationConstraint']
+        logger.debug(location)
+        url = "https://s3-%s.amazonaws.com/%s/%s" % (location, settings.BUCKET_NAME, resource_path)
+        logger.debug(url)
+        return resource_path, url_fix(urljoin(site, url))
+
+    def __save_resource_local(self, provider, file_name, content):
 
         # Create provider dir for assets if it does not exist
         provider_dir = os.path.join(settings.MEDIA_ROOT, "assets", provider)
@@ -208,15 +269,15 @@ def _load_resource_info(self, provider, data, file_=None):
 
             elif isinstance(data["content"], dict):
                 resource_data["content_path"], download_link = self._save_resource_file(
-                    current_organization.name, data["content"]
+                    current_organization.name, data["content"], data["isPublic"]
                 )
 
             else:
                 logger.error("`content` field has an unsupported type, expected string or object")
                 raise TypeError("`content` field has an unsupported type, expected string or object")
 
         elif file_ is not None:
-            resource_data["content_path"], download_link = self._save_resource_file(current_organization.name, file_)
+            resource_data["content_path"], download_link = self._save_resource_file(current_organization.name, file_, data["isPublic"])
 
         else:
             logger.error("The digital asset has not been provided")

src/wstore/asset_manager/test/asset_manager_tests.py

@@ -165,11 +165,16 @@ def setUp(self):
         import wstore.store_commons.rollback
 
         wstore.store_commons.rollback.rollback = decorator_mock
-
         self._user = MagicMock()
         self._user.userprofile.current_organization.name = "test_user"
 
+        asset_manager.boto3.client = MagicMock()
+        asset_manager.boto3.client.return_value.get_bucket_location.return_value = {'LocationConstraint':'eu-north-1'}
         asset_manager.settings.SITE = "http://testdomain.com/"
+        asset_manager.settings.AWS_ACCESS_KEY_ID = "testKey/"
+        asset_manager.settings.AWS_SECRET_ACCESS_KEY = "testSecretKey/"
+        asset_manager.settings.BUCKET_NAME= "test"
+        asset_manager.settings.ACL_ENABLED= False
 
         asset_manager.Resource = MagicMock()
         self.res_mock = MagicMock()
@@ -181,6 +186,9 @@ def setUp(self):
         self._old_is_dir = asset_manager.os.path.isdir
         asset_manager.os.path.isdir = MagicMock()
 
+        self._old_remove = asset_manager.os.remove
+        asset_manager.os.remove = MagicMock()
+
         self._old_exists = asset_manager.os.path.exists
         asset_manager.os.path.exists = MagicMock()
         asset_manager.os.path.exists.return_value = False
@@ -192,7 +200,7 @@ def setUp(self):
     def tearDown(self):
         asset_manager.os.path.isdir = self._old_is_dir
         asset_manager.os.path.exists = self._old_exists
-
+        asset_manager.os.remove= self._old_remove
         import wstore.store_commons.rollback
 
         reload(wstore.store_commons.rollback)
@@ -218,12 +226,19 @@ def _file_conflict_err(self):
         asset_manager.os.path.exists.return_value = True
         self.res_mock.product_id = "1"
 
-    def _check_file_calls(self, file_name="example.wgt"):
-        asset_manager.os.path.isdir.assert_called_once_with("/home/test/media/assets/test_user")
-        asset_manager.os.path.exists.assert_called_once_with("/home/test/media/assets/test_user/{}".format(file_name))
-        self.open_mock.assert_called_once_with("/home/test/media/assets/test_user/{}".format(file_name), "wb")
+    def _check_file_calls(self, file_name="example.wgt", aws= False):
+        final_dir = "test_user" if not aws else "temp"
+        asset_manager.os.path.isdir.assert_called_once_with("/home/test/media/assets/{}".format(final_dir))
+        asset_manager.os.path.exists.assert_called_once_with("/home/test/media/assets/{}/{}".format(final_dir, file_name))
+        self.open_mock.assert_called_once_with("/home/test/media/assets/{}/{}".format(final_dir, file_name), "wb")
         self.open_mock().write.assert_called_once_with("Test data content".encode())
 
+    def _aws_mocks(self, side_effect, err):
+        if side_effect is not None:
+            asset_manager.boto3.client.return_value.upload_file.side_effect =  Exception({'Error': {'Code': side_effect, 'Message': err}}, 'upload_file')
+        else:
+            asset_manager.boto3.client.return_value.upload_file.return_value = None
+
     @parameterized.expand(
         [
             ("basic", UPLOAD_CONTENT),
@@ -236,7 +251,7 @@ def _check_file_calls(self, file_name="example.wgt"):
                 None,
                 "example file.wgt",
             ),
-            ("file", {"contentType": "application/x-widget"}, _use_file),
+            ("file", {"contentType": "application/x-widget", "isPublic":False}, _use_file),
             ("existing_override", UPLOAD_CONTENT, _file_conflict, True),
             (
                 "missing_type",
@@ -352,6 +367,46 @@ def test_upload_asset(
         else:
             self.assertTrue(isinstance(error, err_type))
             self.assertEquals(err_msg, str(error))
+
+    @parameterized.expand([("basic", UPLOAD_CONTENT_AWS, True), ("basic", UPLOAD_CONTENT_AWS, True, "AccessDenied", "Access Denied") ])
+    @override_settings(MEDIA_ROOT="/home/test/media")
+    def test_upload_asset_aws(
+        self,
+        name,
+        data,
+        awsEnabled,
+        side_effect=None,
+        err_msg=None,
+        file_name="example.wgt",
+    ):
+        asset_manager.settings.AWS_ENABLED= awsEnabled
+        am = asset_manager.AssetManager()
+        am.rollback_logger = {"files": [], "models": []}
+        self._aws_mocks(side_effect, err_msg)
+        error = None
+        try:
+            resource = am.upload_asset(self._user, data, file_=self._file)
+        except Exception as e:
+            error = e
+        # Check not error
+        if side_effect is None:
+            # Check calls
+            self.assertEquals("http://locationurl.com/", resource.get_url())
+            self.assertEqual("http://uri.com/", resource.get_uri())
+            self._check_file_calls(file_name, True)
+            # Check rollback logger
+            self.assertEquals(["/home/test/media/assets/temp/{}".format(file_name)],
+                am.rollback_logger["files"])
+            # Check file calls
+            if self._file is not None:
+                self._file.seek.assert_called_once_with(0)
+                asset_manager.os.makedirs.assert_called_once_with("/home/test/media/assets/temp", exist_ok=True)
+            if self._file is not None:
+                self._file.seek.assert_called_once_with(0)
+                asset_manager.os.makedirs.assert_called_once_with("/home/test/media/assets/temp", exist_ok=True)
+                # Check resource creation
+        else:
+            self.assertEquals("({'Error': {'Code': 'AccessDenied', 'Message': 'Access Denied'}}, 'upload_file')", str(error))
 
     def _mock_resource_type(self, form):
         asset_manager.ResourcePlugin = MagicMock()

src/wstore/asset_manager/test/resource_test_data.py

@@ -105,16 +105,25 @@
 UPLOAD_CONTENT = {
     "contentType": "application/x-widget",
     "content": {"name": "example.wgt", "data": "VGVzdCBkYXRhIGNvbnRlbnQ="},
+    "isPublic":False
+}
+
+UPLOAD_CONTENT_AWS = {
+    "contentType": "application/x-widget",
+    "content": {"name": "example.wgt", "data": "VGVzdCBkYXRhIGNvbnRlbnQ="},
+    "isPublic":True
 }
 
 UPLOAD_CONTENT_WHITESPACE = {
     "contentType": "application/x-widget",
     "content": {"name": "example file.wgt", "data": "VGVzdCBkYXRhIGNvbnRlbnQ="},
+    "isPublic":False
 }
 
 UPLOAD_INV_FILENAME = {
     "contentType": "application/x-widget",
     "content": {"name": "example/.wgt", "data": "VGVzdCBkYXRhIGNvbnRlbnQ="},
+    "isPublic":False
 }
 
 MISSING_TYPE = {"content": {"name": "exampleÑ.wgt", "data": "VGVzdCBkYXRhIGNvbnRlbnQ="}}