Update to S3Boto3Storage

The current file storage backend, S3BotoStorage, [cannot be used for
Python versions >=3.4.4](boto/boto#3433).
Therfore we need to update the plugin to use the S3Boto3Storage which
uses a different configuration logic. As a result, the following
configuration parameters are changed:
* `S3_ADDRESSING_STYLE` is added;
* `S3_SIGNATURE_VERSION` is added;
* `S3_REGION` is added;
* `S3_AUTO_CREATE_BUCKET` is removed;

Also we switch to presigned URLs which removes the need for
setting bucket ACL to be public.

Fixes: hastexo#8

README.md

@@ -5,12 +5,10 @@ This is a **experimental** plugin for
 [Tutor](https://docs.tutor.overhang.io) that allows Open edX to be
 configured to use a custom S3 host.
 
-**Do not use this plugin in production Tutor/Open edX environment.**
-
 Installation
 ------------
 
-    pip install tutor-s3
+    pip install tutor-contrib-s3
 
 Then, to enable this plugin, run::
 
@@ -24,45 +22,14 @@ Configuration
 * `S3_HOST` (default: `""`)
 * `S3_PORT` (default: `443`)
 * `S3_USE_SSL` (default: `"True"`)
-* `S3_AUTO_CREATE_BUCKET` (default: `"True"`)
 * `S3_STORAGE_BUCKET` (default: `"openedx"`)
 * `S3_FILE_UPLOAD_BUCKET` (default: `"{{ S3_STORAGE_BUCKET }}"`)
 * `S3_COURSE_IMPORT_EXPORT_BUCKET` (default:
     `"{{ S3_STORAGE_BUCKET }}"`)
 * `S3_PROFILE_IMAGE_BUCKET` (default: `"{{ S3_STORAGE_BUCKET }}"`)
 * `S3_PROFILE_IMAGE_CUSTOM_DOMAIN` (default: `""`)
 * `S3_PROFILE_IMAGE_MAX_AGE` (default: `31536000`)
-
+* `S3_ADDRESSING_STYLE` (default: `virtual`)
+* `S3_SIGNATURE_VERSION` (default: `s3v4`)
 These values can be modified with `tutor config save --set
 PARAM_NAME=VALUE` commands.
-
-Currently, you'll need to allow anonymous GET access to profile
-images. Assuming a properly configured s3cmd, first create the
-`S3_PROFILE_IMAGE_BUCKET` with public ACL set:
-
-    s3cmd mb s3://openedx --acl-public
-
-Next, set the policy for new objects. Create a `policy.json` file,
-substituting `openedx` for the configured value of
-`S3_PROFILE_IMAGE_BUCKET`, and `www.example.com` for the LMS domain::
-
-    {
-      "Version": "2012-10-17",
-      "Statement": [{
-        "Sid":"AddPerm",
-        "Effect":"Allow",
-        "Principal": "*",
-        "Action": ["s3:GetObject"],
-        "Resource": ["arn:aws:s3:::openedx/*"],
-        "Condition": {"StringEquals":{"aws:Referer":["www.example.com"]}
-      }]
-    }
-
-Next, run:
-
-    s3cmd setpolicy policy.json s3://openedx
-
-*This requirement will be removed before the first proper release of
-this plugin, and it will then use presigned URLs instead. If after
-that transition you continue to use the same S3 bucket, you will need
-to update your bucket and object ACLs.*

setup.py

@@ -8,12 +8,12 @@
     readme = f.read()
 
 setup(
-    name="tutor-s3",
+    name="tutor-contrib-s3",
     use_scm_version=True,
-    url="https://github.com/hastexo/tutor-s3/",
+    url="https://github.com/hastexo/tutor-contrib-s3/",
     project_urls={
-        "Code": "https://github.com/hastexo/tutor-s3",
-        "Issue tracker": "https://github.com/hastexo/tutor-s3/issues",
+        "Code": "https://github.com/hastexo/tutor-contrib-s3",
+        "Issue tracker": "https://github.com/hastexo/tutor-contrib-s3/issues",
     },
     license="AGPLv3",
     author='hastexo',

tutors3/__about__.py

@@ -5,4 +5,4 @@
 # Single-source package definition as suggested (among several
 # options) by:
 # https://packaging.python.org/guides/single-sourcing-package-version/
-__version__ = pkg_resources.get_distribution('tutor-s3').version
+__version__ = pkg_resources.get_distribution('tutor-contrib-s3').version

tutors3/patches/lms-env

@@ -1,12 +1,2 @@
 "FILE_UPLOAD_STORAGE_BUCKET_NAME": "{{ S3_FILE_UPLOAD_BUCKET }}",
-"COURSE_IMPORT_EXPORT_BUCKET": "{{ S3_COURSE_IMPORT_EXPORT_BUCKET }}",
-"PROFILE_IMAGE_BACKEND": {
-  "class": "storages.backends.s3boto.S3BotoStorage",
-  "options": {
-    "bucket": "{{ S3_PROFILE_IMAGE_BUCKET }}",
-    "custom_domain": "{{ S3_PROFILE_IMAGE_CUSTOM_DOMAIN }}",
-    "headers": {
-      "Cache-Control": "max-age-{{ S3_PROFILE_IMAGE_MAX_AGE }}"
-    }
-  }
-}
+"COURSE_IMPORT_EXPORT_BUCKET": "{{ S3_COURSE_IMPORT_EXPORT_BUCKET }}"

tutors3/patches/openedx-auth

@@ -1,2 +1,3 @@
 "AWS_STORAGE_BUCKET_NAME": "{{ S3_STORAGE_BUCKET }}",
-"AWS_S3_CUSTOM_DOMAIN": ""
+"AWS_S3_CUSTOM_DOMAIN": "",
+"AWS_QUERYSTRING_AUTH": true

tutors3/patches/openedx-cms-common-settings

@@ -0,0 +1,2 @@
+COURSE_IMPORT_EXPORT_STORAGE = DEFAULT_FILE_STORAGE
+USER_TASKS_ARTIFACT_STORAGE = DEFAULT_FILE_STORAGE

tutors3/patches/openedx-common-settings

@@ -1,18 +1,17 @@
-AWS_S3_HOST = "{{ S3_HOST }}"
-AWS_S3_PORT = {{ S3_PORT }}
+DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+VIDEO_IMAGE_SETTINGS["STORAGE_KWARGS"]["location"] = VIDEO_IMAGE_SETTINGS["STORAGE_KWARGS"]["location"].lstrip("/")
+VIDEO_TRANSCRIPTS_SETTINGS["STORAGE_KWARGS"]["location"] = VIDEO_TRANSCRIPTS_SETTINGS["STORAGE_KWARGS"]["location"].lstrip("/")
+GRADES_DOWNLOAD["STORAGE_KWARGS"]["location"] = GRADES_DOWNLOAD["STORAGE_KWARGS"]["location"].lstrip("/")
+
+ORA2_FILEUPLOAD_BACKEND = "s3"
+FILE_UPLOAD_STORAGE_BUCKET_NAME = "{{ S3_STORAGE_BUCKET }}"
+
+AWS_S3_SIGNATURE_VERSION = "{{ S3_SIGNATURE_VERSION }}"
+AWS_S3_ENDPOINT_URL = "{{ "https" if ENABLE_HTTPS else "http" }}://{{ S3_HOST }}:{{ S3_PORT }}"
 AWS_S3_USE_SSL = {{ "True" if S3_USE_SSL else "False" }}
 AWS_S3_SECURE_URLS = {{ "True" if S3_USE_SSL else "False" }}
-AWS_S3_CALLING_FORMAT = "boto.s3.connection.OrdinaryCallingFormat"
-AWS_AUTO_CREATE_BUCKET = {{ "True" if S3_AUTO_CREATE_BUCKET else "False" }}
+AWS_DEFAULT_ACL = None # inherit from the bucket
+AWS_S3_ADDRESSING_STYLE = "{{ S3_ADDRESSING_STYLE }}"
 
-# Configuring boto is required for ora2 because ora2 does not read
-# host/port/ssl settings from django. Hence this hack.
-# http://docs.pythonboto.org/en/latest/boto_config_tut.html
-import os
-os.environ["AWS_CREDENTIAL_FILE"] = "/tmp/boto.cfg"
-with open("/tmp/boto.cfg", "w") as f:
-    f.write("""[Boto]
-is_secure = {{ "True" if S3_USE_SSL else "False" }}
-[s3]
-host = {{ S3_HOST }}:{{ S3_PORT }}
-calling_format = boto.s3.connection.OrdinaryCallingFormat""")
+AWS_S3_REGION_NAME = "{{ S3_REGION }}"
+AWS_QUERYSTRING_EXPIRE = 7 * 24 * 60 * 60  # 1 week: this is necessary to generate valid download urls

tutors3/patches/openedx-lms-common-settings

@@ -1 +1,8 @@
-ORA2_FILEUPLOAD_BACKEND = "s3"
+PROFILE_IMAGE_BACKEND = {
+    "class": DEFAULT_FILE_STORAGE,
+    "options": {
+        "location": PROFILE_IMAGE_BACKEND["options"]["location"].lstrip("/"),
+        # the following non empty property is necessary in development
+        "base_url": "dummyprofileimagebaseurl",
+    },
+}

tutors3/plugin.py

@@ -11,14 +11,16 @@
         "VERSION": __version__,
         "HOST": "",
         "PORT": "443",
+        "REGION": "",
         "USE_SSL": "True",
-        "AUTO_CREATE_BUCKET": "True",
         "STORAGE_BUCKET": "openedx",
         "FILE_UPLOAD_BUCKET": "{{ S3_STORAGE_BUCKET }}",
         "COURSE_IMPORT_EXPORT_BUCKET": "{{ S3_STORAGE_BUCKET }}",
         "PROFILE_IMAGE_BUCKET": "{{ S3_STORAGE_BUCKET }}",
         "PROFILE_IMAGE_CUSTOM_DOMAIN": "",
         "PROFILE_IMAGE_MAX_AGE": "31536000",
+        "ADDRESSING_STYLE": "virtual",
+        "SIGNATURE_VERSION": "s3v4",
     },
 }