Skip to content
This repository has been archived by the owner on Aug 22, 2022. It is now read-only.

Configure PROFILE_IMAGE_BACKEND when using AWS s3 #370

Merged
merged 6 commits into from Dec 19, 2018
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 12 additions & 1 deletion instance/models/mixins/openedx_storage.py
Expand Up @@ -61,13 +61,24 @@ def _get_s3_settings(self):
# using shared s3 buckets
"EDXAPP_AWS_LOCATION": self.swift_container_name,
"EDXAPP_DEFAULT_FILE_STORAGE": 'storages.backends.s3boto.S3BotoStorage',

# Set up S3 image backend
"EDXAPP_PROFILE_IMAGE_BACKEND": {
"class": "storages.backends.s3boto.S3BotoStorage",
"options": {
"location": '{}/{}'.format(self.swift_container_name, 'profile-images'),
"headers": {
"Cache-Control": "max-age-{{ EDXAPP_PROFILE_IMAGE_MAX_AGE }}",
},
},
},
"EDXAPP_AWS_ACCESS_KEY_ID": self.s3_access_key,
"EDXAPP_AWS_SECRET_ACCESS_KEY": self.s3_secret_access_key,
"EDXAPP_AWS_STORAGE_BUCKET_NAME": self.s3_bucket_name,
"EDXAPP_AUTH_EXTRA": {
"AWS_STORAGE_BUCKET_NAME": self.s3_bucket_name,
},
"EDXAPP_AWS_S3_CUSTOM_DOMAIN": "{}.s3.amazonaws.com".format(self.s3_bucket_name),
"EDXAPP_AWS_S3_CUSTOM_DOMAIN": self.s3_custom_domain,
"EDXAPP_IMPORT_EXPORT_BUCKET": self.s3_bucket_name,
"EDXAPP_FILE_UPLOAD_BUCKET_NAME": self.s3_bucket_name,
"EDXAPP_FILE_UPLOAD_STORAGE_PREFIX": '{}/{}'.format(self.swift_container_name, 'submissions_attachments'),
Expand Down
7 changes: 7 additions & 0 deletions instance/models/mixins/storage.py
Expand Up @@ -226,6 +226,13 @@ def s3_hostname(self):
s3_hostname = settings.AWS_S3_CUSTOM_REGION_HOSTNAME.format(region=self.s3_region)
return s3_hostname

@property
def s3_custom_domain(self):
"""
The custom domain name built based on the bucket name.
"""
return "{}.s3.amazonaws.com".format(self.s3_bucket_name)

def create_iam_user(self):
"""
Create IAM user with access only to the s3 bucket set in s3_bucket_name
Expand Down
28 changes: 26 additions & 2 deletions instance/tests/models/test_openedx_storage_mixins.py
Expand Up @@ -21,15 +21,15 @@
"""

# Imports #####################################################################
from unittest.mock import patch, call
from unittest.mock import call, patch

import boto
import ddt
import yaml
from django.conf import settings
from django.test.utils import override_settings

from instance.models.mixins.storage import get_s3_cors_config, get_master_iam_connection, StorageContainer
from instance.models.mixins.storage import StorageContainer, get_master_iam_connection, get_s3_cors_config
from instance.tests.base import TestCase
from instance.tests.models.factories.openedx_appserver import make_test_appserver
from instance.tests.models.factories.openedx_instance import OpenEdXInstanceFactory
Expand Down Expand Up @@ -63,6 +63,13 @@ def check_s3_vars(self, yaml_vars_string):
self.assertEqual(parsed_vars['AWS_S3_LOGS_ACCESS_KEY_ID'], 'test-s3-access-key')
self.assertEqual(parsed_vars['AWS_S3_LOGS_SECRET_KEY'], 'test-s3-secret-access-key')

# Profile image backend
self.assertEqual(parsed_vars['EDXAPP_PROFILE_IMAGE_BACKEND']['class'], 'storages.backends.s3boto.S3BotoStorage')

opts = parsed_vars['EDXAPP_PROFILE_IMAGE_BACKEND']['options']
self.assertEqual(opts['headers'], {'Cache-Control': 'max-age-{{ EDXAPP_PROFILE_IMAGE_MAX_AGE }}'})
self.assertRegex(opts['location'], r'instance[\w]+_test_example_com/profile-images')

def test_ansible_s3_settings(self):
"""
Test that get_storage_settings() includes S3 vars, and that they get passed on to the
Expand All @@ -79,6 +86,21 @@ def test_ansible_s3_settings(self):
self.check_s3_vars(appserver.configuration_settings)


def get_s3_settings_profile_image(instance):
"""
Return expected s3 settings related to profile image backend
"""
s3_settings = (
'\n class: storages.backends.s3boto.S3BotoStorage'
'\n options:'
'\n headers:'
'\n Cache-Control: max-age-{{{{ EDXAPP_PROFILE_IMAGE_MAX_AGE }}}}'
'\n location: {instance.swift_container_name}/profile-images'
).format(instance=instance)

return s3_settings


def get_s3_settings(instance):
"""
Return expected s3 settings
Expand Down Expand Up @@ -120,6 +142,8 @@ def get_s3_settings(instance):
"AWS_S3_LOGS": 'true',
"AWS_S3_LOGS_ACCESS_KEY_ID": instance.s3_access_key,
"AWS_S3_LOGS_SECRET_KEY": instance.s3_secret_access_key,

"EDXAPP_PROFILE_IMAGE_BACKEND": get_s3_settings_profile_image(instance),
}

if instance.s3_region:
Expand Down