diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a0fdf90..427aa36 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,3 +1,4 @@ +--- name: build on: [push, pull_request] jobs: @@ -6,18 +7,35 @@ jobs: strategy: fail-fast: false matrix: - image-tag: - - 37 - - 38 - - 39 - - 310 + python-version: + - '3.7' + - '3.8' + - '3.9' + - '3.10' # container: thumbororg/thumbor-test:${{ matrix.image-tag }} steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Code Climate Before Build + if: matrix.python-version == '3.10' + env: + code_climate_token: ${{ secrets.CODE_CLIMATE_TOKEN }} + run: |- + curl -L -O https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 + chmod +x test-reporter-latest-linux-amd64 + ./test-reporter-latest-linux-amd64 before-build - name: APT Update run: sudo apt-get update -y - name: APT Install - run: sudo apt-get install -y python3-dev libcurl4-openssl-dev libgnutls28-dev python3-all-dev make zlib1g-dev gcc libssl-dev build-essential + run: sudo apt-get install -y python3-dev libcurl4-openssl-dev libgnutls28-dev + python3-all-dev make zlib1g-dev gcc libssl-dev build-essential + - name: Create Virtualenv + run: python3 -m venv ~/thumbor-aws + - name: Activate Virtualenv + run: source ~/thumbor-aws/bin/activate - name: Setup run: make setup - name: Run Unit Tests @@ -29,10 +47,15 @@ jobs: - name: Lint run: make flake pylint - name: Coveralls - if: matrix.image-tag == '310' + if: matrix.python-version == '3.10' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | pip install --upgrade coveralls coveralls --service=github - + - name: Code Climate After Build + if: matrix.python-version == '3.10' + env: + code_climate_token: ${{ secrets.CODE_CLIMATE_TOKEN }} + run: |- + ./test-reporter-latest-linux-amd64 after-build -t cobertura --exit-code 0 -r ${code_climate_token} diff --git a/.gitignore b/.gitignore index 09ae93e..87c8ee5 100644 --- a/.gitignore +++ b/.gitignore @@ -128,3 +128,6 @@ dmypy.json # Pyre type checker .pyre/ test-results/ +cobertura.xml +coverage +cc-test-reporter diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 98ce7c7..f02ca2d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.1.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -14,10 +14,14 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/flake8 - rev: '' # pick a git hash / tag to point to + rev: 4.0.1 # pick a git hash / tag to point to hooks: - id: flake8 - - repo: https://github.com/pycqa/pylint - rev: pylint-2.6.0 + - repo: local hooks: - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + args: [--rcfile=pylintrc] diff --git a/Makefile b/Makefile index 0a4526b..f7869bd 100644 --- a/Makefile +++ b/Makefile @@ -4,15 +4,23 @@ PYTHON = python OS := $(shell uname) setup: - @$(PYTHON) -m pip install -e .[tests] + @pip install -U pip + @pip install -U coverage + @pip install -e .[tests] -services: - @docker-compose up +services: docker-down + @docker-compose up --remove-orphans -ci: - @docker-compose up -d +ci: docker-down docker-up @./wait-for-it.sh localhost:4566 -- echo "Docker Compose is Up. Running tests..." - @pytest -sv --junit-xml=test-results/unit/results.xml --cov=thumbor_aws tests/ + @coverage run -m pytest tests && coverage xml && mv coverage.xml cobertura.xml + +docker-up: + @docker-compose up --remove-orphan -d + +docker-down: + @docker-compose stop + @docker-compose rm -f test: @$(MAKE) unit diff --git a/README.md b/README.md index 3c2736b..9373081 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,8 @@ This is a project to provide modern thumbor>7.0.0 AWS Extensions.

- - - + Coverage Status + @@ -37,6 +36,14 @@ This is a project to provide modern thumbor>7.0.0 AWS Extensions. ```bash pip install thumbor-aws ``` +## 🎯 Features + +- Asynchronous non-blocking AWS S3 support +- Conforms with thumbor 7 new storage and results storage specs +- Python 3 compliant +- S3 Storage - Retrieve and store source files, detector data and security keys; +- S3 Result Storage - Retrieve and store resulting images. These can be set to be public-read and served directly from S3. +- Compatibility mode for users of tc_aws: currently supported loader, storage and result storage. ## Usage @@ -45,22 +52,25 @@ pip install thumbor-aws Configure your `thumbor.conf` file to point to `thumbor_aws`: ``` -## The file storage thumbor should use to store original images. This must be the -## full name of a python module (python must be able to import it) -## Defaults to: 'thumbor.storages.file_storage' +## The loader thumbor should use to find source images. +## This must be the full name of a python module (python must be able to import it) +LOADER = "thumbor.loaders.http_loader" + +## The file storage thumbor should use to store original images. +## This must be the full name of a python module (python must be able to import it) STORAGE = 'thumbor_aws.storage' -## The result storage thumbor should use to store generated images. This must be -## the full name of a python module (python must be able to import it) -## Defaults to: None +## The result storage thumbor should use to store generated images. +## This must be the full name of a python module (python must be able to import it) RESULT_STORAGE = 'thumbor_aws.result_storage' ``` -As usual for thumbor, you don't need to use both at the same time. Feel free to use only what's needed. +You should use only the extensions required by your use case. +There's no dependency between them. ### Configuration -thumbor-aws allows you to configure each storage independently, so there are configuration keys for each. +thumbor-aws allows you to configure each extension independently: #### General @@ -73,6 +83,40 @@ Some S3 providers fail to return a valid location header when uploading a new ob AWS_DEFAULT_LOCATION = "https://{bucket_name}.s3.amazonaws.com" ``` +#### Loader + +thumbor-aws loader offer several configuration options: + +```python +################################## AWS Loader ################################## + +## Region where thumbor's objects are going to be loaded from. +## Defaults to: 'us-east-1' +#AWS_LOADER_REGION_NAME = 'us-east-1' + +## S3 Bucket where thumbor's objects are loaded from. +## Defaults to: 'thumbor' +#AWS_LOADER_BUCKET_NAME = 'thumbor' + +## Secret access key for S3 Loader. +## Defaults to: None +#AWS_LOADER_S3_SECRET_ACCESS_KEY = None + +## Access key ID for S3 Loader. +## Defaults to: None +#AWS_LOADER_S3_ACCESS_KEY_ID = None + +## Endpoint URL for S3 API. Very useful for testing. +## Defaults to: None +#AWS_LOADER_S3_ENDPOINT_URL = None + +## Loader prefix path. +## Defaults to: '/st' +#AWS_LOADER_ROOT_PATH = '/st' + +################################################################################ +``` + #### Storage Below you can see the result of running thumbor's config generation after importing thumbor-aws: @@ -82,37 +126,39 @@ Below you can see the result of running thumbor's config generation after import ## Region where thumbor's objects are going to be stored. ## Defaults to: 'us-east-1' -# AWS_STORAGE_REGION_NAME = 'us-east-1' +#AWS_STORAGE_REGION_NAME = 'us-east-1' ## S3 Bucket where thumbor's objects are going to be stored. ## Defaults to: 'thumbor' -# AWS_STORAGE_BUCKET_NAME = 'thumbor' +#AWS_STORAGE_BUCKET_NAME = 'thumbor' ## Secret access key for S3 to allow thumbor to store objects there. ## Defaults to: None -# AWS_STORAGE_S3_SECRET_ACCESS_KEY = None +#AWS_STORAGE_S3_SECRET_ACCESS_KEY = None ## Access key ID for S3 to allow thumbor to store objects there. ## Defaults to: None -# AWS_STORAGE_S3_ACCESS_KEY_ID = None +#AWS_STORAGE_S3_ACCESS_KEY_ID = None ## Endpoint URL for S3 API. Very useful for testing. ## Defaults to: None -# AWS_STORAGE_S3_ENDPOINT_URL = None - -## Endpoint URL for S3 API. Very useful for testing. -## Defaults to: None -# AWS_STORAGE_S3_ENDPOINT_URL = None +#AWS_STORAGE_S3_ENDPOINT_URL = None ## Storage prefix path. ## Defaults to: '/st' -# AWS_STORAGE_ROOT_PATH = '/st' +#AWS_STORAGE_ROOT_PATH = '/st' ## Storage ACL for files written in bucket ## Defaults to: 'public-read' -# AWS_STORAGE_S3_ACL = 'public-read' +#AWS_STORAGE_S3_ACL = 'public-read' + +## Default location to use if S3 does not return location header. Can use +## {bucket_name} var. +## Defaults to: 'https://{bucket_name}.s3.amazonaws.com' +#AWS_DEFAULT_LOCATION = 'https://{bucket_name}.s3.amazonaws.com' ################################################################################ + ``` #### Result Storage @@ -124,35 +170,103 @@ Below you can see the result of running thumbor's config generation after import ## Region where thumbor's objects are going to be stored. ## Defaults to: 'us-east-1' -# AWS_RESULT_STORAGE_REGION_NAME = 'us-east-1' +#AWS_RESULT_STORAGE_REGION_NAME = 'us-east-1' ## S3 Bucket where thumbor's objects are going to be stored. ## Defaults to: 'thumbor' -# AWS_RESULT_STORAGE_BUCKET_NAME = 'thumbor' +#AWS_RESULT_STORAGE_BUCKET_NAME = 'thumbor' ## Secret access key for S3 to allow thumbor to store objects there. ## Defaults to: None -# AWS_RESULT_STORAGE_S3_SECRET_ACCESS_KEY = None +#AWS_RESULT_STORAGE_S3_SECRET_ACCESS_KEY = None ## Access key ID for S3 to allow thumbor to store objects there. ## Defaults to: None -# AWS_RESULT_STORAGE_S3_ACCESS_KEY_ID = None +#AWS_RESULT_STORAGE_S3_ACCESS_KEY_ID = None ## Endpoint URL for S3 API. Very useful for testing. ## Defaults to: None -# AWS_RESULT_STORAGE_S3_ENDPOINT_URL = None +#AWS_RESULT_STORAGE_S3_ENDPOINT_URL = None ## Result Storage prefix path. ## Defaults to: '/rs' -# AWS_RESULT_STORAGE_ROOT_PATH = '/rs' +#AWS_RESULT_STORAGE_ROOT_PATH = '/rs' ## ACL to use for storing items in S3. ## Defaults to: None -# AWS_RESULT_STORAGE_S3_ACL = None +#AWS_RESULT_STORAGE_S3_ACL = None ################################################################################ ``` +### Configuring thumbor in compatibility mode with tc_aws + +If you are a [tc_aws](https://github.com/thumbor-community/aws) user, thumbor-aws has a compatibility mode where you can use the same configuration you are already used to: + +```python +############################# tc_aws Compatibility ############################# + +## Runs in compatibility mode using the configurations for tc_aws. +## Defaults to: False +THUMBOR_AWS_RUN_IN_COMPATIBILITY_MODE = True + +## AWS Region the bucket is located in. +## Defaults to: 'us-east-1' +TC_AWS_REGION = 'us-east-1' + +## Max retries for get image from S3 Bucket. Default is 0 +## Defaults to: 0 +TC_AWS_MAX_RETRY = 0 # This is not yet supported + +## S3 bucket for Loader. If given, source urls are interpreted as keys within +## this bucket. If not given, source urls are expected to containthe bucket +## name, such as 's3-bucket/keypath'. +## Defaults to: '' +TC_AWS_LOADER_BUCKET = 'my-bucket' + +## S3 path prefix for Loader bucket. If given, this is prefixed to all S3 keys. +## Defaults to: '' +TC_AWS_LOADER_ROOT_PATH = 'source-files' + +## S3 bucket for Storage +## Defaults to: '' +TC_AWS_STORAGE_BUCKET = 'my-bucket' + +## S3 path prefix for Storage bucket +## Defaults to: '' +TC_AWS_STORAGE_ROOT_PATH = 'source-files' + +## put data into S3 using the Server Side Encryption functionality to encrypt +## data at rest in S3 https://aws.amazon.com/about-aws/whats- +## new/2011/10/04/amazon-s3-announces-server-side-encryption-support/ +## Defaults to: False +TC_AWS_STORAGE_SSE = False # This is not yet supported + +## put data into S3 with Reduced Redundancy https://aws.amazon.com/about- +## aws/whats-new/2010/05/19/announcing-amazon-s3-reduced-redundancy-storage/ +## Defaults to: False +TC_AWS_STORAGE_RRS = False # This is not yet supported + +## S3 bucket for result Storage +## Defaults to: '' +TC_AWS_RESULT_STORAGE_BUCKET = 'my-bucket' + +## S3 path prefix for Result storage bucket +## Defaults to: '' +TC_AWS_RESULT_STORAGE_ROOT_PATH = 'result-storage' + +## Store result with metadata (for instance content-type) +## Defaults to: False +# This configuration won't matter as thumbor-aws stores metadata anyway +TC_AWS_STORE_METADATA = False + +################################################################################ +``` + +Please notice the addition of `THUMBOR_AWS_RUN_IN_COMPATIBILITY_MODE = True` to tell `thumbor_aws` you want compatibility with `tc_aws`. + +If you have any issues with this + #### Caveats 1. thumbor-aws does not create buckets for you. If they don't exist you are getting errors. @@ -175,14 +289,6 @@ thumbor-doctor If you still need help, please [raise an issue](https://github.com/thumbor/thumbor-aws/issues). -## 🎯 Features - -- Asynchronous non-blocking AWS S3 support -- Conforms with thumbor 7 new storage and results storage specs -- Python 3 compliant -- S3 Storage - Retrieve and store source files, detector data and security keys; -- S3 Result Storage - Retrieve and store resulting images. These can be set to be public-read and served directly from S3. - ## 👀 Thumbor [thumbor-aws](https://github.com/thumbor/thumbor-aws) stands on the shoulders of [thumbor](https://github.com/thumbor/thumbor)! If you are not familiar with [thumbor](https://github.com/thumbor/thumbor), please check the [docs](https://thumbor.readthedocs.io/en/latest/) or you can see a demo at http://thumborize.me/ diff --git a/fixtures.py b/fixtures.py index e72f5fd..41065a6 100644 --- a/fixtures.py +++ b/fixtures.py @@ -13,15 +13,17 @@ from os.path import abspath, dirname, isfile, join from aiobotocore.session import get_session - from thumbor.utils import logger ROOT_PATH = dirname(__file__) async def upload(): + """Uploads file to S3""" images_path = abspath(join(ROOT_PATH, "tests", "fixtures")) - all_images = [f for f in listdir(images_path) if isfile(join(images_path, f))] + all_images = [ + f for f in listdir(images_path) if isfile(join(images_path, f)) + ] session = get_session() async with session.create_client( @@ -53,13 +55,19 @@ async def upload(): ContentType="image/jpeg", ACL="public-read", ) - except Exception as e: - msg = f"Unable to upload image to {image_path}: {e} ({type(e)})" + except Exception as err: + msg = ( + "Unable to upload image to " + f"{image_path}: {err} ({type(err)})" + ) logger.error(msg) - raise RuntimeError(msg) + raise RuntimeError(msg) from err status_code = response["ResponseMetadata"]["HTTPStatusCode"] if status_code != 200: - msg = f"Unable to upload image to {image_path}: Status Code {status_code}" + msg = ( + "Unable to upload image to " + f"{image_path}: Status Code {status_code}" + ) logger.error(msg) raise RuntimeError(msg) diff --git a/pylintrc b/pylintrc index b245bc9..4183a3b 100644 --- a/pylintrc +++ b/pylintrc @@ -28,3 +28,7 @@ disable= fixme, R0801, deprecated-module, + +[MISCELLANEOUS] +# List of note tags to take in consideration, separated by a comma. +notes=FIXME diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..816862c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +[tool.black] +line-length = 79 +target-version = ['py37'] +include = '\.pyi?$' diff --git a/setup.cfg b/setup.cfg index 902ebb3..d137efc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,16 @@ [metadata] description-file = README.md +[isort] +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True +line_length=79 + [flake8] ignore = W801,E501,W605,W504,W606,W503,E203,E266 -max-line-length = 120 +max-line-length = 79 max-complexity = 20 select = B,C,E,F,W,T4 exclude = ./.tox/*,./build/*,./docs/conf.py diff --git a/setup.py b/setup.py index 3245c19..7b6684c 100644 --- a/setup.py +++ b/setup.py @@ -41,10 +41,12 @@ long_description=""" thumbor_aws provides extensions for thumbor using AWS services """, - keywords=("imaging face detection feature thumbnail imagemagick pil opencv"), + keywords=( + "imaging face detection feature thumbnail imagemagick pil opencv" + ), author="Bernardo Heynemann", author_email="heynemann@gmail.com", - url="https://github.com/thumbor/thumbor_aws", + url="https://github.com/thumbor/thumbor-aws", license="MIT", python_requires=">=3.6", classifiers=[ diff --git a/tests/__init__.py b/tests/__init__.py index 1c0aa23..77f8b78 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,11 +8,14 @@ # http://www.opensource.org/licenses/mit-license # Copyright (c) 2021 Bernardo Heynemann heynemann@gmail.com +import os + from thumbor.config import Config from thumbor.context import Context, ServerParameters from thumbor.importer import Importer from thumbor.testing import TestCase -from thumbor_aws.storage import Storage + +from thumbor_aws.s3_client import S3Client class BaseS3TestCase(TestCase): @@ -23,34 +26,97 @@ def bucket_name(self): """Name of the bucket to put test files in""" return self.context.config.AWS_STORAGE_BUCKET_NAME - def get_context(self): + @property + def region_name(self): + """Name of the bucket to put test files in""" + if self.context.config.THUMBOR_AWS_RUN_IN_COMPATIBILITY_MODE: + return self.context.config.TC_AWS_REGION + return self.context.config.AWS_STORAGE_REGION_NAME + + def get_config(self) -> Config: + # Required by Bot + os.environ["AWS_ACCESS_KEY_ID"] = "foobar" + os.environ["AWS_SECRET_ACCESS_KEY"] = "foobar" + cfg = Config(SECURITY_KEY="ACME-SEC") + cfg.LOADER = "thumbor_aws.loader" cfg.STORAGE = "thumbor_aws.storage" cfg.RESULT_STORAGE = "thumbor_aws.result_storage" # Storage Config cfg.AWS_STORAGE_REGION_NAME = "local" - cfg.AWS_STORAGE_BUCKET_NAME = "test-bucket" + cfg.AWS_STORAGE_BUCKET_NAME = "test-bucket-st" + cfg.AWS_STORAGE_ROOT_PATH = "/test-st" cfg.AWS_STORAGE_S3_ENDPOINT_URL = "https://localhost:4566" # Result Storage Config cfg.AWS_RESULT_STORAGE_REGION_NAME = "local" - cfg.AWS_RESULT_STORAGE_BUCKET_NAME = "test-bucket" + cfg.AWS_RESULT_STORAGE_BUCKET_NAME = "test-bucket-rs" cfg.AWS_RESULT_STORAGE_S3_ENDPOINT_URL = "https://localhost:4566" cfg.AWS_RESULT_STORAGE_ROOT_PATH = "/test-rs" cfg.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True + # Loader Config + cfg.AWS_LOADER_REGION_NAME = "local" + cfg.AWS_LOADER_BUCKET_NAME = "test-bucket-st" + cfg.AWS_LOADER_ROOT_PATH = "/test-st" + cfg.AWS_LOADER_S3_ENDPOINT_URL = "https://localhost:4566" + + return cfg + + def get_compatibility_config(self) -> Config: + # Required by Boto + os.environ["AWS_ACCESS_KEY_ID"] = "foobar" + os.environ["AWS_SECRET_ACCESS_KEY"] = "foobar" + + cfg = Config(SECURITY_KEY="ACME-SEC") + cfg.LOADER = "thumbor_aws.loader" + cfg.STORAGE = "thumbor_aws.storage" + cfg.RESULT_STORAGE = "thumbor_aws.result_storage" + + cfg.THUMBOR_AWS_RUN_IN_COMPATIBILITY_MODE = True + cfg.TC_AWS_REGION = "local" + cfg.TC_AWS_MAX_RETRY = 0 + cfg.TC_AWS_ENDPOINT = "https://localhost:4566" + + # Storage Config + cfg.TC_AWS_STORAGE_BUCKET = "test-bucket-compat" + cfg.TC_AWS_STORAGE_ROOT_PATH = "/test-compat-st" + + # Result Storage Config + cfg.TC_AWS_RESULT_STORAGE_BUCKET = "test-bucket-compat-rs" + cfg.TC_AWS_RESULT_STORAGE_ROOT_PATH = "/test-compat-rs" + cfg.STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True + + # Loader Config + cfg.TC_AWS_LOADER_BUCKET = "test-bucket-compat" + cfg.TC_AWS_LOADER_ROOT_PATH = "/test-compat-st" + + return cfg + + def get_context(self) -> Context: + cfg = self.get_config() importer = Importer(cfg) importer.import_modules() - server = ServerParameters(8889, "localhost", "thumbor.conf", None, "info", None) + server = ServerParameters( + 8889, "localhost", "thumbor.conf", None, "info", None + ) server.security_key = "ACME-SEC" return Context(server, cfg, importer) - async def ensure_bucket(self, cls=Storage): + async def ensure_bucket(self): """Ensures the test bucket is created""" - storage = cls(self.context) - location = {"LocationConstraint": self.context.config.AWS_STORAGE_REGION_NAME} - async with storage.get_client() as client: + s3client = S3Client(self.context) + if self.context.config.THUMBOR_AWS_RUN_IN_COMPATIBILITY_MODE is True: + s3client.configuration["region_name"] = self.config.TC_AWS_REGION + s3client.configuration[ + "endpoint_url" + ] = self.config.TC_AWS_ENDPOINT + + location = { + "LocationConstraint": self.region_name, + } + async with s3client.get_client() as client: try: await client.create_bucket( Bucket=self.bucket_name, @@ -58,4 +124,3 @@ async def ensure_bucket(self, cls=Storage): ) except client.exceptions.BucketAlreadyOwnedByYou: pass - return storage diff --git a/tests/test_loader.py b/tests/test_loader.py new file mode 100644 index 0000000..d2675e3 --- /dev/null +++ b/tests/test_loader.py @@ -0,0 +1,64 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# thumbor aws extensions +# https://github.com/thumbor/thumbor-aws + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2021 Bernardo Heynemann heynemann@gmail.com + +from uuid import uuid4 + +import pytest +from preggy import expect +from thumbor.config import Config +from tornado.testing import gen_test + +from tests import BaseS3TestCase +from thumbor_aws.loader import load +from thumbor_aws.storage import Storage + + +@pytest.mark.usefixtures("test_images") +class LoaderTestCase(BaseS3TestCase): + @property + def bucket_name(self): + """Name of the bucket to put test files in""" + return self.context.config.AWS_LOADER_BUCKET_NAME + + @gen_test + async def test_can_load_file_from_s3(self): + """ + Verifies that an image can be loaded from S3 + using Loader and that it's there + """ + await self.ensure_bucket() + storage = Storage(self.context) + filepath = f"/test/can_put_file_{uuid4()}" + expected = self.test_images["default"] + await storage.put(filepath, expected) + exists = await storage.exists(filepath) + expect(exists).to_be_true() + + result = await load(self.context, filepath) + + expect(result.successful).to_be_true() + expect(result.buffer).to_equal(expected) + expect(result.metadata["size"]).to_equal(len(expected)) + expect(result.metadata["updated_at"]).not_to_be_null() + + +@pytest.mark.usefixtures("test_images") +class LoaderCompatibilityModeTestCase(LoaderTestCase): + def get_config(self) -> Config: + return self.get_compatibility_config() + + @property + def _prefix(self): + return "/test-compat-st" + + @property + def bucket_name(self): + """Name of the bucket to put test files in""" + return "test-bucket-compat" diff --git a/tests/test_result_storage.py b/tests/test_result_storage.py index 1842bee..c8633dd 100644 --- a/tests/test_result_storage.py +++ b/tests/test_result_storage.py @@ -13,10 +13,11 @@ import pytest from mock import Mock from preggy import expect +from thumbor.config import Config from tornado.testing import gen_test from tests import BaseS3TestCase -from thumbor_aws.result_storage import Storage +from thumbor_aws.result_storage import Storage as ResultStorage @pytest.mark.usefixtures("test_images") @@ -25,33 +26,51 @@ class ResultStorageTestCase(BaseS3TestCase): def bucket_name(self): return self.context.config.AWS_RESULT_STORAGE_BUCKET_NAME + @property + def _prefix(self): + return "/test-rs" + + @property + def region_name(self): + """Name of the bucket to put test files in""" + if self.context.config.THUMBOR_AWS_RUN_IN_COMPATIBILITY_MODE: + return self.context.config.TC_AWS_REGION + return self.context.config.AWS_RESULT_STORAGE_REGION_NAME + @gen_test async def test_can_put_file_in_s3(self): - ( - """Verifies that submitting an image to S3 through""" - """Result Storage works and the image is there""" - ) - + """ + Verifies that submitting an image to S3 through + Result Storage works and the image is there + """ + await self.ensure_bucket() filepath = f"/test/can_put_file_{uuid4()}" self.context.request = Mock(url=filepath) - storage = await self.ensure_bucket(cls=Storage) + storage = ResultStorage(self.context) expected = self.test_images["default"] path = await storage.put(expected) expect(path).to_equal( - f"https://test-bucket.s3.localhost.localstack.cloud:4566/test-rs/default{filepath}", + f"https://{self.bucket_name}.s3.localhost.localstack.cloud:4566" + f"{self._prefix}/default{filepath}", + ) + status, data, _ = await storage.get_data( + f"{self._prefix}/default{filepath}" ) - status, data, _ = await storage.get_data(f"/test-rs/default{filepath}") expect(status).to_equal(200) expect(data).to_equal(expected) @gen_test async def test_can_get_result_in_s3(self): - """Verifies that an image can be retrieved from S3 through Result Storage""" + """ + Verifies that an image can be retrieved + from S3 through Result Storage + """ + await self.ensure_bucket() filepath = f"/test/can_put_file_{uuid4()}" self.context.request = Mock(url=filepath) - storage = await self.ensure_bucket(cls=Storage) + storage = ResultStorage(self.context) expected = self.test_images["default"] await storage.put(expected) @@ -65,10 +84,14 @@ async def test_can_get_result_in_s3(self): @gen_test async def test_can_get_result_in_s3_for_invalid_file(self): - """Verifies that if an image does not exist, Result Storage returns None""" + """ + Verifies that if an image does not exist, + Result Storage returns None + """ + await self.ensure_bucket() filepath = f"/test/can_put_file_{uuid4()}" self.context.request = Mock(url=filepath) - storage = await self.ensure_bucket(cls=Storage) + storage = ResultStorage(self.context) data = await storage.get() @@ -76,14 +99,32 @@ async def test_can_get_result_in_s3_for_invalid_file(self): @gen_test async def test_can_check_deprecated_last_updated_method(self): - """Verifies that Result Storage deprecated last_updated method works""" - + """ + Verifies that Result Storage deprecated + last_updated method works + """ + await self.ensure_bucket() filepath = f"/test/can_put_file_{uuid4()}" self.context.request = Mock(url=filepath) - storage = await self.ensure_bucket(cls=Storage) + storage = ResultStorage(self.context) expected = self.test_images["default"] await storage.put(expected) last_updated = await storage.last_updated() expect(last_updated).not_to_be_null() + + +@pytest.mark.usefixtures("test_images") +class ResultStorageCompatibilityTestCase(ResultStorageTestCase): + def get_config(self) -> Config: + return self.get_compatibility_config() + + @property + def _prefix(self): + return "/test-compat-rs" + + @property + def bucket_name(self): + """Name of the bucket to put test files in""" + return "test-bucket-compat-rs" diff --git a/tests/test_storage.py b/tests/test_storage.py index 1e855ce..cf206fb 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -13,41 +13,57 @@ import pytest from preggy import expect +from thumbor.config import Config from tornado.testing import gen_test from tests import BaseS3TestCase +from thumbor_aws.storage import Storage @pytest.mark.usefixtures("test_images") class StorageTestCase(BaseS3TestCase): + @property + def _prefix(self): + return "/test-st" + @gen_test async def test_can_put_file_in_s3(self): - """Verifies that an image can be placed in S3 using Storage and that it's there""" - - storage = await self.ensure_bucket() + """ + Verifies that an image can be placed in S3 + using Storage and that it's there + """ + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_put_file_{uuid4()}" expected = self.test_images["default"] path = await storage.put(filepath, expected) expect(path).to_equal( - f"https://test-bucket.s3.localhost.localstack.cloud:4566/st{filepath}", + f"https://{self.bucket_name}.s3.localhost.localstack.cloud:4566" + f"{self._prefix}{filepath}", + ) + status, data, _ = await storage.get_data( + storage.normalize_path(filepath) ) - status, data, _ = await storage.get_data(storage.normalize_path(filepath)) expect(status).to_equal(200) expect(data).to_equal(expected) @gen_test async def test_can_put_crypto_in_s3(self): - """Verifies that security information can be placed in S3 using Storage""" - - storage = await self.ensure_bucket() + """ + Verifies that security information can + be placed in S3 using Storage + """ + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_put_file_{uuid4()}" path = await storage.put_crypto(filepath) expect(path).to_equal( - f"https://test-bucket.s3.localhost.localstack.cloud:4566/st{filepath}.txt", + f"https://{self.bucket_name}.s3.localhost.localstack.cloud:4566" + f"{self._prefix}{filepath}.txt", ) status, data, _ = await storage.get_data( storage.normalize_path(filepath + ".txt") @@ -58,8 +74,8 @@ async def test_can_put_crypto_in_s3(self): @gen_test async def test_can_put_detector_data_in_s3(self): """Verifies that detector data can be placed in S3 using Storage""" - - storage = await self.ensure_bucket() + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_put_file_{uuid4()}" path = await storage.put_detector_data( @@ -70,7 +86,8 @@ async def test_can_put_detector_data_in_s3(self): ) expect(path).to_equal( - f"https://test-bucket.s3.localhost.localstack.cloud:4566/st{filepath}.detectors.txt", + f"https://{self.bucket_name}.s3.localhost.localstack.cloud:4566" + f"{self._prefix}{filepath}.detectors.txt", ) status, data, _ = await storage.get_data( storage.normalize_path(filepath + ".detectors.txt") @@ -81,8 +98,8 @@ async def test_can_put_detector_data_in_s3(self): @gen_test async def test_can_load_file_in_s3(self): """Verifies that an image can be loaded from S3 using Storage""" - - storage = await self.ensure_bucket() + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_load_file_{uuid4()}" expected = self.test_images["default"] await storage.put(filepath, expected) @@ -93,8 +110,12 @@ async def test_can_load_file_in_s3(self): @gen_test async def test_can_handle_expired_data(self): - """Verifies that an image won't be loaded from S3 using Storage if it is expired""" - storage = await self.ensure_bucket() + """ + Verifies that an image won't be loaded from S3 + using Storage if it is expired + """ + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_load_file_{uuid4()}" expected = self.test_images["default"] await storage.put(filepath, expected) @@ -109,10 +130,13 @@ async def test_can_handle_expired_data(self): @gen_test async def test_can_upload_with_valid_location(self): """Verifies that uploading with valid location returns location""" - storage = await self.ensure_bucket() + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_put_file_{uuid4()}" - with patch.object(storage.__class__, "get_location", return_value=None): + with patch.object( + storage.__class__, "get_location", return_value=None + ): response = await storage.upload( storage.normalize_path(filepath), b"ACME-SEC2", @@ -121,13 +145,18 @@ async def test_can_upload_with_valid_location(self): ) expect(response).to_equal( - f"https://my-site.com/{self.bucket_name}/st{filepath}" + "https://my-site.com/" + f"{self.bucket_name}{self._prefix}{filepath}" ) @gen_test async def test_can_get_crypto_from_s3(self): - """Verifies that security information can be loaded from S3 using Storage""" - storage = await self.ensure_bucket() + """ + Verifies that security information can be + loaded from S3 using Storage + """ + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_put_file_{uuid4()}" await storage.upload( @@ -144,7 +173,8 @@ async def test_can_get_crypto_from_s3(self): @gen_test async def test_can_get_detector_data_from_s3(self): """Verifies that detector data can be loaded from S3 using Storage""" - storage = await self.ensure_bucket() + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_put_file_{uuid4()}" await storage.upload( storage.normalize_path(filepath + ".detectors.txt"), @@ -164,8 +194,8 @@ async def test_can_get_detector_data_from_s3(self): @gen_test async def test_verify_file_does_not_exist(self): """Verifies that Storage can tell if a file does not exist in S3""" - - storage = await self.ensure_bucket() + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_put_file_{uuid4()}" exists = await storage.exists(filepath) @@ -175,8 +205,8 @@ async def test_verify_file_does_not_exist(self): @gen_test async def test_verify_file_does_exist(self): """Verifies that Storage can tell if a file exists in S3""" - - storage = await self.ensure_bucket() + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_put_file_{uuid4()}" await storage.put(filepath, self.test_images["default"]) @@ -184,12 +214,11 @@ async def test_verify_file_does_exist(self): expect(exists).to_be_true() - @pytest.mark.focus @gen_test async def test_can_remove(self): """Verifies that Storage can remove a file from S3""" - - storage = await self.ensure_bucket() + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_put_file_{uuid4()}" await storage.put(filepath, self.test_images["default"]) exists = await storage.exists(filepath) @@ -202,9 +231,12 @@ async def test_can_remove(self): @gen_test async def test_can_remove_invalid_file(self): - """Verifies that removing a file that does not exist does not cause Storage to fail""" - - storage = await self.ensure_bucket() + """ + Verifies that removing a file that does not exist + does not cause Storage to fail + """ + await self.ensure_bucket() + storage = Storage(self.context) filepath = f"/test/can_put_file_{uuid4()}" exists = await storage.exists(filepath) expect(exists).to_be_false() @@ -213,3 +245,18 @@ async def test_can_remove_invalid_file(self): exists = await storage.exists(filepath) expect(exists).to_be_false() + + +@pytest.mark.usefixtures("test_images") +class StorageCompatibilityModeTestCase(StorageTestCase): + def get_config(self) -> Config: + return self.get_compatibility_config() + + @property + def _prefix(self): + return "/test-compat-st" + + @property + def bucket_name(self): + """Name of the bucket to put test files in""" + return "test-bucket-compat" diff --git a/thumbor.conf b/thumbor.conf index ee9717c..c023fa4 100644 --- a/thumbor.conf +++ b/thumbor.conf @@ -2,15 +2,15 @@ ## Logging configuration as json ## Defaults to: None -#THUMBOR_LOG_CONFIG = None +# THUMBOR_LOG_CONFIG = None ## Log Format to be used by thumbor when writing log messages. ## Defaults to: '%(asctime)s %(name)s:%(levelname)s %(message)s' -#THUMBOR_LOG_FORMAT = '%(asctime)s %(name)s:%(levelname)s %(message)s' +# THUMBOR_LOG_FORMAT = '%(asctime)s %(name)s:%(levelname)s %(message)s' ## Date Format to be used by thumbor when writing log messages. ## Defaults to: '%Y-%m-%d %H:%M:%S' -#THUMBOR_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' +# THUMBOR_LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' ################################################################################ @@ -19,40 +19,40 @@ ## Max width in pixels for images read or generated by thumbor ## Defaults to: 0 -#MAX_WIDTH = 0 +# MAX_WIDTH = 0 ## Max height in pixels for images read or generated by thumbor ## Defaults to: 0 -#MAX_HEIGHT = 0 +# MAX_HEIGHT = 0 ## Max pixel count for images read by thumbor ## Defaults to: 75000000.0 -#MAX_PIXELS = 75000000.0 +# MAX_PIXELS = 75000000.0 ## Min width in pixels for images read or generated by thumbor ## Defaults to: 1 -#MIN_WIDTH = 1 +# MIN_WIDTH = 1 ## Min width in pixels for images read or generated by thumbor ## Defaults to: 1 -#MIN_HEIGHT = 1 +# MIN_HEIGHT = 1 ## Allowed domains for the http loader to download. These are regular ## expressions. ## Defaults to: [ -#] +# ] -#ALLOWED_SOURCES = [ -#] +# ALLOWED_SOURCES = [ +# ] ## Quality index used for generated JPEG images ## Defaults to: 80 -#QUALITY = 80 +# QUALITY = 80 ## Exports JPEG images with the `progressive` flag set. ## Defaults to: True -#PROGRESSIVE_JPEG = True +# PROGRESSIVE_JPEG = True ## Specify subsampling behavior for Pillow (see `subsampling` in ## http://pillow.readthedocs.org/en/latest/handbook/image-file- @@ -60,38 +60,38 @@ ## notation. Will ignore `quality`. Using `keep` will copy the original file's ## subsampling. ## Defaults to: None -#PILLOW_JPEG_SUBSAMPLING = None +# PILLOW_JPEG_SUBSAMPLING = None ## Specify quantization tables for Pillow (see `qtables` in ## http://pillow.readthedocs.org/en/latest/handbook/image-file- ## formats.html#jpeg). Will ignore `quality`. Using `keep` will copy the ## original file's qtables. ## Defaults to: None -#PILLOW_JPEG_QTABLES = None +# PILLOW_JPEG_QTABLES = None ## Specify resampling filter for Pillow resize method.One of LANCZOS, NEAREST, ## BILINEAR, BICUBIC, HAMMING (Pillow>=3.4.0). ## Defaults to: 'LANCZOS' -#PILLOW_RESAMPLING_FILTER = 'LANCZOS' +# PILLOW_RESAMPLING_FILTER = 'LANCZOS' ## Quality index used for generated WebP images. If not set (None) the same level ## of JPEG quality will be used. If 100 the `lossless` flag will be used. ## Defaults to: None -#WEBP_QUALITY = None +# WEBP_QUALITY = None ## Compression level for generated PNG images. ## Defaults to: 6 -#PNG_COMPRESSION_LEVEL = 6 +# PNG_COMPRESSION_LEVEL = 6 ## Indicates if final image should preserve indexed mode (P or 1) of original ## image ## Defaults to: True -#PILLOW_PRESERVE_INDEXED_MODE = True +# PILLOW_PRESERVE_INDEXED_MODE = True ## Specifies whether WebP format should be used automatically if the request ## accepts it (via Accept header) ## Defaults to: False -#AUTO_WEBP = False +# AUTO_WEBP = False ## Specifies whether a PNG image should be used automatically if the png image ## has no transparency (via alpha layer). WARNING: Depending on case, this is @@ -101,63 +101,63 @@ ## maybe will be bigger. You have to evaluate the majority of your use cases ## to take a decision about the usage of this conf. ## Defaults to: False -#AUTO_PNG_TO_JPG = False +# AUTO_PNG_TO_JPG = False ## Specify the ratio between 1in and 1px for SVG images. This is only used ## whenrasterizing SVG images having their size units in cm or inches. ## Defaults to: 150 -#SVG_DPI = 150 +# SVG_DPI = 150 ## Max AGE sent as a header for the image served by thumbor in seconds ## Defaults to: 86400 -#MAX_AGE = 86400 +# MAX_AGE = 86400 ## Indicates the Max AGE header in seconds for temporary images (images with ## failed smart detection) ## Defaults to: 0 -#MAX_AGE_TEMP_IMAGE = 0 +# MAX_AGE_TEMP_IMAGE = 0 ## Indicates whether thumbor should rotate images that have an Orientation EXIF ## header ## Defaults to: False -#RESPECT_ORIENTATION = False +# RESPECT_ORIENTATION = False ## Ignore errors during smart detections and return image as a temp image (not ## saved in result storage and with MAX_AGE_TEMP_IMAGE age) ## Defaults to: False -#IGNORE_SMART_ERRORS = False +# IGNORE_SMART_ERRORS = False ## Sends If-Modified-Since & Last-Modified headers; requires support from result ## storage ## Defaults to: False -#SEND_IF_MODIFIED_LAST_MODIFIED_HEADERS = False +# SEND_IF_MODIFIED_LAST_MODIFIED_HEADERS = False ## Preserves exif information in generated images. Increases image size in ## kbytes, use with caution. ## Defaults to: False -#PRESERVE_EXIF_INFO = False +# PRESERVE_EXIF_INFO = False ## Indicates whether thumbor should enable the EXPERIMENTAL support for animated ## gifs. ## Defaults to: True -#ALLOW_ANIMATED_GIFS = True +# ALLOW_ANIMATED_GIFS = True ## Indicates whether thumbor should use gifsicle engine. Please note that smart ## cropping and filters are not supported for gifs using gifsicle (but won't ## give an error). ## Defaults to: False -#USE_GIFSICLE_ENGINE = False +# USE_GIFSICLE_ENGINE = False ## Indicates whether thumbor should enable blacklist functionality to prevent ## processing certain images. ## Defaults to: False -#USE_BLACKLIST = False +# USE_BLACKLIST = False ## Size of the thread pool used for image transformations. The default value is 0 ## (don't use a threadpoool. Increase this if you are seeing your IOLoop ## getting blocked (often indicated by your upstream HTTP requests timing out) ## Defaults to: 0 -#ENGINE_THREADPOOL_SIZE = 0 +# ENGINE_THREADPOOL_SIZE = 0 ################################################################################ @@ -167,37 +167,37 @@ ## The metrics backend thumbor should use to measure internal actions. This must ## be the full name of a python module (python must be able to import it) ## Defaults to: 'thumbor.metrics.logger_metrics' -#METRICS = 'thumbor.metrics.logger_metrics' +# METRICS = 'thumbor.metrics.logger_metrics' ## The loader thumbor should use to load the original image. This must be the ## full name of a python module (python must be able to import it) ## Defaults to: 'thumbor.loaders.http_loader' -LOADER = 'thumbor.loaders.http_loader' +LOADER = "thumbor.loaders.http_loader" ## The file storage thumbor should use to store original images. This must be the ## full name of a python module (python must be able to import it) ## Defaults to: 'thumbor.storages.file_storage' -STORAGE = 'thumbor_aws.storage' +STORAGE = "thumbor_aws.storage" ## The result storage thumbor should use to store generated images. This must be ## the full name of a python module (python must be able to import it) ## Defaults to: None -RESULT_STORAGE = 'thumbor_aws.result_storage' +RESULT_STORAGE = "thumbor_aws.result_storage" ## The imaging engine thumbor should use to perform image operations. This must ## be the full name of a python module (python must be able to import it) ## Defaults to: 'thumbor.engines.pil' -#ENGINE = 'thumbor.engines.pil' +# ENGINE = 'thumbor.engines.pil' ## The gif engine thumbor should use to perform image operations. This must be ## the full name of a python module (python must be able to import it) ## Defaults to: 'thumbor.engines.gif' -#GIF_ENGINE = 'thumbor.engines.gif' +# GIF_ENGINE = 'thumbor.engines.gif' ## The url signer thumbor should use to verify url signatures.This must be the ## full name of a python module (python must be able to import it) ## Defaults to: 'libthumbor.url_signers.base64_hmac_sha1' -#URL_SIGNER = 'libthumbor.url_signers.base64_hmac_sha1' +# URL_SIGNER = 'libthumbor.url_signers.base64_hmac_sha1' ################################################################################ @@ -206,11 +206,11 @@ RESULT_STORAGE = 'thumbor_aws.result_storage' ## The security key thumbor uses to sign image URLs ## Defaults to: 'MY_SECURE_KEY' -#SECURITY_KEY = 'MY_SECURE_KEY' +# SECURITY_KEY = 'MY_SECURE_KEY' ## Indicates if the /unsafe URL should be available ## Defaults to: True -#ALLOW_UNSAFE_URL = True +# ALLOW_UNSAFE_URL = True ################################################################################ @@ -219,7 +219,7 @@ RESULT_STORAGE = 'thumbor_aws.result_storage' ## Enables automatically generated etags ## Defaults to: True -#ENABLE_ETAGS = True +# ENABLE_ETAGS = True ################################################################################ @@ -228,7 +228,7 @@ RESULT_STORAGE = 'thumbor_aws.result_storage' ## Set maximum id length for images when stored ## Defaults to: 32 -#MAX_ID_LENGTH = 32 +# MAX_ID_LENGTH = 32 ################################################################################ @@ -237,7 +237,7 @@ RESULT_STORAGE = 'thumbor_aws.result_storage' ## Set garbage collection interval in seconds ## Defaults to: None -#GC_INTERVAL = None +# GC_INTERVAL = None ################################################################################ @@ -246,7 +246,7 @@ RESULT_STORAGE = 'thumbor_aws.result_storage' ## Healthcheck route. ## Defaults to: '/healthcheck/?' -#HEALTHCHECK_ROUTE = '/healthcheck/?' +# HEALTHCHECK_ROUTE = '/healthcheck/?' ################################################################################ @@ -255,15 +255,15 @@ RESULT_STORAGE = 'thumbor_aws.result_storage' ## Host to send statsd instrumentation to ## Defaults to: None -#STATSD_HOST = None +# STATSD_HOST = None ## Port to send statsd instrumentation to ## Defaults to: 8125 -#STATSD_PORT = 8125 +# STATSD_PORT = 8125 ## Prefix for statsd ## Defaults to: None -#STATSD_PREFIX = None +# STATSD_PREFIX = None ################################################################################ @@ -272,7 +272,7 @@ RESULT_STORAGE = 'thumbor_aws.result_storage' ## The root path where the File Loader will try to find images ## Defaults to: '/home/heynemann' -#FILE_LOADER_ROOT_PATH = '/home/heynemann' +# FILE_LOADER_ROOT_PATH = '/home/heynemann' ################################################################################ @@ -282,82 +282,82 @@ RESULT_STORAGE = 'thumbor_aws.result_storage' ## The maximum number of seconds libcurl can take to connect to an image being ## loaded ## Defaults to: 5 -#HTTP_LOADER_CONNECT_TIMEOUT = 5 +# HTTP_LOADER_CONNECT_TIMEOUT = 5 ## The maximum number of seconds libcurl can take to download an image ## Defaults to: 20 -#HTTP_LOADER_REQUEST_TIMEOUT = 20 +# HTTP_LOADER_REQUEST_TIMEOUT = 20 ## Indicates whether libcurl should follow redirects when downloading an image ## Defaults to: True -#HTTP_LOADER_FOLLOW_REDIRECTS = True +# HTTP_LOADER_FOLLOW_REDIRECTS = True ## Indicates the number of redirects libcurl should follow when downloading an ## image ## Defaults to: 5 -#HTTP_LOADER_MAX_REDIRECTS = 5 +# HTTP_LOADER_MAX_REDIRECTS = 5 ## The maximum number of simultaneous HTTP connections the loader can make before ## queuing ## Defaults to: 10 -#HTTP_LOADER_MAX_CLIENTS = 10 +# HTTP_LOADER_MAX_CLIENTS = 10 ## Indicates whether thumbor should forward the user agent of the requesting user ## Defaults to: False -#HTTP_LOADER_FORWARD_USER_AGENT = False +# HTTP_LOADER_FORWARD_USER_AGENT = False ## Indicates whether thumbor should forward the headers of the request ## Defaults to: False -#HTTP_LOADER_FORWARD_ALL_HEADERS = False +# HTTP_LOADER_FORWARD_ALL_HEADERS = False ## Indicates which headers should be forwarded among all the headers of the ## request ## Defaults to: [ -#] +# ] -#HTTP_LOADER_FORWARD_HEADERS_WHITELIST = [ -#] +# HTTP_LOADER_FORWARD_HEADERS_WHITELIST = [ +# ] ## Default user agent for thumbor http loader requests ## Defaults to: 'Thumbor/7.0.0a5' -#HTTP_LOADER_DEFAULT_USER_AGENT = 'Thumbor/7.0.0a5' +# HTTP_LOADER_DEFAULT_USER_AGENT = 'Thumbor/7.0.0a5' ## The proxy host needed to load images through ## Defaults to: None -#HTTP_LOADER_PROXY_HOST = None +# HTTP_LOADER_PROXY_HOST = None ## The proxy port for the proxy host ## Defaults to: None -#HTTP_LOADER_PROXY_PORT = None +# HTTP_LOADER_PROXY_PORT = None ## The proxy username for the proxy host ## Defaults to: None -#HTTP_LOADER_PROXY_USERNAME = None +# HTTP_LOADER_PROXY_USERNAME = None ## The proxy password for the proxy host ## Defaults to: None -#HTTP_LOADER_PROXY_PASSWORD = None +# HTTP_LOADER_PROXY_PASSWORD = None ## The filename of CA certificates in PEM format ## Defaults to: None -#HTTP_LOADER_CA_CERTS = None +# HTTP_LOADER_CA_CERTS = None ## Validate the server’s certificate for HTTPS requests ## Defaults to: None -#HTTP_LOADER_VALIDATE_CERTS = None +# HTTP_LOADER_VALIDATE_CERTS = None ## The filename for client SSL key ## Defaults to: None -#HTTP_LOADER_CLIENT_KEY = None +# HTTP_LOADER_CLIENT_KEY = None ## The filename for client SSL certificate ## Defaults to: None -#HTTP_LOADER_CLIENT_CERT = None +# HTTP_LOADER_CLIENT_CERT = None ## If the CurlAsyncHTTPClient should be used ## Defaults to: False -#HTTP_LOADER_CURL_ASYNC_HTTP_CLIENT = False +# HTTP_LOADER_CURL_ASYNC_HTTP_CLIENT = False ################################################################################ @@ -369,19 +369,19 @@ RESULT_STORAGE = 'thumbor_aws.result_storage' ## timeout if the speed is below HTTP_LOADER_CURL_LOW_SPEED_LIMIT for that ## long ## Defaults to: 0 -#HTTP_LOADER_CURL_LOW_SPEED_TIME = 0 +# HTTP_LOADER_CURL_LOW_SPEED_TIME = 0 ## If HTTP_LOADER_CURL_LOW_SPEED_TIME and HTTP_LOADER_CURL_ASYNC_HTTP_CLIENT are ## set, then this is the limit in bytes per second as integer which should ## timeout if the speed is below that limit for ## HTTP_LOADER_CURL_LOW_SPEED_TIME seconds ## Defaults to: 0 -#HTTP_LOADER_CURL_LOW_SPEED_LIMIT = 0 +# HTTP_LOADER_CURL_LOW_SPEED_LIMIT = 0 ## Custom app class to override ThumborServiceApp. This config value is ## overridden by the -a command-line parameter. ## Defaults to: 'thumbor.app.ThumborServiceApp' -#APP_CLASS = 'thumbor.app.ThumborServiceApp' +# APP_CLASS = 'thumbor.app.ThumborServiceApp' ################################################################################ @@ -391,7 +391,7 @@ RESULT_STORAGE = 'thumbor_aws.result_storage' ## Expiration in seconds for the images in the File Storage. Defaults to one ## month ## Defaults to: 2592000 -#STORAGE_EXPIRATION_SECONDS = 2592000 +# STORAGE_EXPIRATION_SECONDS = 2592000 ## Indicates whether thumbor should store the signing key for each image in the ## file storage. This allows the key to be changed and old images to still be @@ -401,7 +401,7 @@ STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True ## The root path where the File Storage will try to find images ## Defaults to: '/tmp/thumbor/storage' -#FILE_STORAGE_ROOT_PATH = '/tmp/thumbor/storage' +# FILE_STORAGE_ROOT_PATH = '/tmp/thumbor/storage' ################################################################################ @@ -411,31 +411,31 @@ STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True ## Max size in bytes for images uploaded to thumbor ## Aliases: MAX_SIZE ## Defaults to: 0 -#UPLOAD_MAX_SIZE = 0 +# UPLOAD_MAX_SIZE = 0 ## Indicates whether thumbor should enable File uploads ## Aliases: ENABLE_ORIGINAL_PHOTO_UPLOAD ## Defaults to: False -#UPLOAD_ENABLED = False +# UPLOAD_ENABLED = False ## The type of storage to store uploaded images with ## Aliases: ORIGINAL_PHOTO_STORAGE ## Defaults to: 'thumbor.storages.file_storage' -#UPLOAD_PHOTO_STORAGE = 'thumbor.storages.file_storage' +# UPLOAD_PHOTO_STORAGE = 'thumbor.storages.file_storage' ## Indicates whether image deletion should be allowed ## Aliases: ALLOW_ORIGINAL_PHOTO_DELETION ## Defaults to: False -#UPLOAD_DELETE_ALLOWED = False +# UPLOAD_DELETE_ALLOWED = False ## Indicates whether image overwrite should be allowed ## Aliases: ALLOW_ORIGINAL_PHOTO_PUTTING ## Defaults to: False -#UPLOAD_PUT_ALLOWED = False +# UPLOAD_PUT_ALLOWED = False ## Default filename for image uploaded ## Defaults to: 'image' -#UPLOAD_DEFAULT_FILENAME = 'image' +# UPLOAD_DEFAULT_FILENAME = 'image' ################################################################################ @@ -445,17 +445,17 @@ STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True ## Mixed Storage file storage. This must be the full name of a python module ## (python must be able to import it) ## Defaults to: 'thumbor.storages.no_storage' -#MIXED_STORAGE_FILE_STORAGE = 'thumbor.storages.no_storage' +# MIXED_STORAGE_FILE_STORAGE = 'thumbor.storages.no_storage' ## Mixed Storage signing key storage. This must be the full name of a python ## module (python must be able to import it) ## Defaults to: 'thumbor.storages.no_storage' -#MIXED_STORAGE_CRYPTO_STORAGE = 'thumbor.storages.no_storage' +# MIXED_STORAGE_CRYPTO_STORAGE = 'thumbor.storages.no_storage' ## Mixed Storage detector information storage. This must be the full name of a ## python module (python must be able to import it) ## Defaults to: 'thumbor.storages.no_storage' -#MIXED_STORAGE_DETECTOR_STORAGE = 'thumbor.storages.no_storage' +# MIXED_STORAGE_DETECTOR_STORAGE = 'thumbor.storages.no_storage' ################################################################################ @@ -465,7 +465,7 @@ STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True ## The callback function name that should be used by the META route for JSONP ## access ## Defaults to: None -#META_CALLBACK_NAME = None +# META_CALLBACK_NAME = None ################################################################################ @@ -476,23 +476,23 @@ STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True ## of them must be full names of python modules (python must be able to import ## it) ## Defaults to: [ -#] +# ] -#DETECTORS = [ -#] +# DETECTORS = [ +# ] ## The cascade file that opencv will use to detect faces. ## Defaults to: 'haarcascade_frontalface_alt.xml' -#FACE_DETECTOR_CASCADE_FILE = 'haarcascade_frontalface_alt.xml' +# FACE_DETECTOR_CASCADE_FILE = 'haarcascade_frontalface_alt.xml' ## The cascade file that opencv will use to detect glasses. ## Defaults to: 'haarcascade_eye_tree_eyeglasses.xml' -#GLASSES_DETECTOR_CASCADE_FILE = 'haarcascade_eye_tree_eyeglasses.xml' +# GLASSES_DETECTOR_CASCADE_FILE = 'haarcascade_eye_tree_eyeglasses.xml' ## The cascade file that opencv will use to detect profile faces. ## Defaults to: 'haarcascade_profileface.xml' -#PROFILE_DETECTOR_CASCADE_FILE = 'haarcascade_profileface.xml' +# PROFILE_DETECTOR_CASCADE_FILE = 'haarcascade_profileface.xml' ################################################################################ @@ -501,24 +501,24 @@ STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True ## List of optimizers that thumbor will use to optimize images ## Defaults to: [ -#] +# ] -#OPTIMIZERS = [ -#] +# OPTIMIZERS = [ +# ] ## Path for the jpegtran binary ## Defaults to: '/usr/bin/jpegtran' -#JPEGTRAN_PATH = '/usr/bin/jpegtran' +# JPEGTRAN_PATH = '/usr/bin/jpegtran' ## Path for the progressive scans file to use with jpegtran optimizer. Implies ## progressive jpeg output ## Defaults to: '' -#JPEGTRAN_SCANS_FILE = '' +# JPEGTRAN_SCANS_FILE = '' ## Path for the ffmpeg binary used to generate gifv(h.264) ## Defaults to: '/usr/local/bin/ffmpeg' -#FFMPEG_PATH = '/usr/local/bin/ffmpeg' +# FFMPEG_PATH = '/usr/local/bin/ffmpeg' ################################################################################ @@ -559,9 +559,9 @@ STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True # 'thumbor.filters.upscale', # 'thumbor.filters.proportion', # 'thumbor.filters.stretch', -#] +# ] -#FILTERS = [ +# FILTERS = [ # 'thumbor.filters.brightness', # 'thumbor.filters.colorize', # 'thumbor.filters.contrast', @@ -592,7 +592,7 @@ STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True # 'thumbor.filters.upscale', # 'thumbor.filters.proportion', # 'thumbor.filters.stretch', -#] +# ] ################################################################################ @@ -606,7 +606,7 @@ STORES_CRYPTO_KEY_FOR_EACH_IMAGE = True ## Path where the Result storage will store generated images ## Defaults to: '/tmp/thumbor/result_storage' -#RESULT_STORAGE_FILE_STORAGE_ROOT_PATH = '/tmp/thumbor/result_storage' +# RESULT_STORAGE_FILE_STORAGE_ROOT_PATH = '/tmp/thumbor/result_storage' ## Indicates whether unsafe requests should also be stored in the Result Storage ## Defaults to: False @@ -619,19 +619,19 @@ RESULT_STORAGE_STORES_UNSAFE = True ## Server host for the queued redis detector ## Defaults to: 'localhost' -#REDIS_QUEUE_SERVER_HOST = 'localhost' +# REDIS_QUEUE_SERVER_HOST = 'localhost' ## Server port for the queued redis detector ## Defaults to: 6379 -#REDIS_QUEUE_SERVER_PORT = 6379 +# REDIS_QUEUE_SERVER_PORT = 6379 ## Server database index for the queued redis detector ## Defaults to: 0 -#REDIS_QUEUE_SERVER_DB = 0 +# REDIS_QUEUE_SERVER_DB = 0 ## Server password for the queued redis detector ## Defaults to: None -#REDIS_QUEUE_SERVER_PASSWORD = None +# REDIS_QUEUE_SERVER_PASSWORD = None ################################################################################ @@ -640,15 +640,15 @@ RESULT_STORAGE_STORES_UNSAFE = True ## AWS key id ## Defaults to: None -#SQS_QUEUE_KEY_ID = None +# SQS_QUEUE_KEY_ID = None ## AWS key secret ## Defaults to: None -#SQS_QUEUE_KEY_SECRET = None +# SQS_QUEUE_KEY_SECRET = None ## AWS SQS region ## Defaults to: 'us-east-1' -#SQS_QUEUE_REGION = 'us-east-1' +# SQS_QUEUE_REGION = 'us-east-1' ################################################################################ @@ -658,20 +658,20 @@ RESULT_STORAGE_STORES_UNSAFE = True ## This configuration indicates whether thumbor should use a custom error ## handler. ## Defaults to: False -#USE_CUSTOM_ERROR_HANDLING = False +# USE_CUSTOM_ERROR_HANDLING = False ## Error reporting module. Needs to contain a class called ErrorHandler with a ## handle_error(context, handler, exception) method. ## Defaults to: 'thumbor.error_handlers.sentry' -#ERROR_HANDLER_MODULE = 'thumbor.error_handlers.sentry' +# ERROR_HANDLER_MODULE = 'thumbor.error_handlers.sentry' ## File of error log as json ## Defaults to: None -#ERROR_FILE_LOGGER = None +# ERROR_FILE_LOGGER = None ## File of error log name is parametrized with context attribute ## Defaults to: False -#ERROR_FILE_NAME_USE_CONTEXT = False +# ERROR_FILE_NAME_USE_CONTEXT = False ################################################################################ @@ -681,11 +681,11 @@ RESULT_STORAGE_STORES_UNSAFE = True ## Sentry thumbor project dsn. i.e.: http://5a63d58ae7b94f1dab3dee740b301d6a:73ee ## a45d3e8649239a973087e8f21f98@localhost:9000/2 ## Defaults to: '' -#SENTRY_DSN_URL = '' +# SENTRY_DSN_URL = '' ## Sentry environment i.e.: staging ## Defaults to: None -#SENTRY_ENVIRONMENT = None +# SENTRY_ENVIRONMENT = None ################################################################################ @@ -695,12 +695,12 @@ RESULT_STORAGE_STORES_UNSAFE = True ## The amount of time to wait before shutting down the server, i.e. stop ## accepting requests. ## Defaults to: 0 -#MAX_WAIT_SECONDS_BEFORE_SERVER_SHUTDOWN = 0 +# MAX_WAIT_SECONDS_BEFORE_SERVER_SHUTDOWN = 0 ## The amount of time to waut before shutting down all io, after the server has ## been stopped ## Defaults to: 0 -#MAX_WAIT_SECONDS_BEFORE_IO_SHUTDOWN = 0 +# MAX_WAIT_SECONDS_BEFORE_IO_SHUTDOWN = 0 ################################################################################ @@ -712,13 +712,13 @@ RESULT_STORAGE_STORES_UNSAFE = True # 'thumbor.handler_lists.healthcheck', # 'thumbor.handler_lists.upload', # 'thumbor.handler_lists.blacklist', -#] +# ] -#HANDLER_LISTS = [ +# HANDLER_LISTS = [ # 'thumbor.handler_lists.healthcheck', # 'thumbor.handler_lists.upload', # 'thumbor.handler_lists.blacklist', -#] +# ] ################################################################################ @@ -730,19 +730,19 @@ RESULT_STORAGE_STORES_UNSAFE = True ## compatibility loader. Please only use this if you can't use up-to-date ## loaders. ## Defaults to: None -#COMPATIBILITY_LEGACY_LOADER = None +# COMPATIBILITY_LEGACY_LOADER = None ## Storage that will be used with the compatibility layer, instead of the ## compatibility storage. Please only use this if you can't use up-to-date ## storages. ## Defaults to: None -#COMPATIBILITY_LEGACY_STORAGE = None +# COMPATIBILITY_LEGACY_STORAGE = None ## Result Storage that will be used with the compatibility layer, instead of the ## compatibility result storage. Please only use this if you can't use up-to- ## date result storages. ## Defaults to: None -#COMPATIBILITY_LEGACY_RESULT_STORAGE = None +# COMPATIBILITY_LEGACY_RESULT_STORAGE = None ################################################################################ @@ -756,11 +756,11 @@ AWS_DEFAULT_LOCATION = "https://{bucket_name}.s3.amazonaws.com" ## Region where thumbor's objects are going to be stored. ## Defaults to: 'us-east-1' -AWS_STORAGE_REGION_NAME = 'local' +AWS_STORAGE_REGION_NAME = "local" ## S3 Bucket where thumbor's objects are going to be stored. ## Defaults to: 'thumbor' -AWS_STORAGE_BUCKET_NAME = 'mystorage' +AWS_STORAGE_BUCKET_NAME = "mystorage" ## Secret access key for S3 to allow thumbor to store objects there. ## Defaults to: None @@ -772,15 +772,15 @@ AWS_STORAGE_S3_ACCESS_KEY_ID = None ## Endpoint URL for S3 API. Very useful for testing. ## Defaults to: None -AWS_STORAGE_S3_ENDPOINT_URL = 'https://localhost:4566' +AWS_STORAGE_S3_ENDPOINT_URL = "https://localhost:4566" ## Storage prefix path. ## Defaults to: '/st' -AWS_STORAGE_ROOT_PATH = '/st' +AWS_STORAGE_ROOT_PATH = "/st" ## Storage ACL for files written in bucket ## Defaults to: 'public-read' -AWS_STORAGE_S3_ACL = 'public-read' +AWS_STORAGE_S3_ACL = "public-read" ################################################################################ @@ -789,11 +789,11 @@ AWS_STORAGE_S3_ACL = 'public-read' ## Region where thumbor's objects are going to be stored. ## Defaults to: 'us-east-1' -AWS_RESULT_STORAGE_REGION_NAME = 'local' +AWS_RESULT_STORAGE_REGION_NAME = "local" ## S3 Bucket where thumbor's objects are going to be stored. ## Defaults to: 'thumbor' -AWS_RESULT_STORAGE_BUCKET_NAME = 'mystorage' +AWS_RESULT_STORAGE_BUCKET_NAME = "mystorage" ## Secret access key for S3 to allow thumbor to store objects there. ## Defaults to: None @@ -805,12 +805,12 @@ AWS_RESULT_STORAGE_S3_ACCESS_KEY_ID = None ## Endpoint URL for S3 API. Very useful for testing. ## Defaults to: None -AWS_RESULT_STORAGE_S3_ENDPOINT_URL = 'https://localhost:4566' +AWS_RESULT_STORAGE_S3_ENDPOINT_URL = "https://localhost:4566" ## Result Storage prefix path. ## Defaults to: '/rs' -AWS_RESULT_STORAGE_ROOT_PATH = '/rs' +AWS_RESULT_STORAGE_ROOT_PATH = "/rs" -AWS_RESULT_STORAGE_S3_ACL = 'public-read' +AWS_RESULT_STORAGE_S3_ACL = "public-read" ################################################################################ diff --git a/thumbor_aws/config.py b/thumbor_aws/config.py new file mode 100644 index 0000000..9788bfb --- /dev/null +++ b/thumbor_aws/config.py @@ -0,0 +1,133 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# thumbor aws extensions +# https://github.com/thumbor/thumbor-aws + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2021 Bernardo Heynemann heynemann@gmail.com + + +from thumbor.config import Config, config + +Config.define( + "AWS_DEFAULT_LOCATION", + "https://{bucket_name}.s3.amazonaws.com", + ( + "Default location to use if S3 does not return location header." + " Can use {bucket_name} var." + ), + "AWS Storage", +) + +# TC_AWS Compatibility settings +Config.define( + "THUMBOR_AWS_RUN_IN_COMPATIBILITY_MODE", + False, + "Runs in compatibility mode using the configurations for tc_aws.", + "tc_aws Compatibility", +) + +Config.define( + "TC_AWS_REGION", + "us-east-1", + "AWS Region the bucket is located in.", + "tc_aws Compatibility", +) + +# TODO: Support retries +Config.define( + "TC_AWS_MAX_RETRY", + 0, + "Max retries for get image from S3 Bucket. Default is 0", + "tc_aws Compatibility", +) + +# TC_AWS Loader Settings + +Config.define( + "TC_AWS_LOADER_BUCKET", + "", + "S3 bucket for Loader. If given, source urls are interpreted as keys " + "within this bucket. If not given, source urls are expected to contain" + "the bucket name, such as 's3-bucket/keypath'.", + "tc_aws Compatibility", +) + +Config.define( + "TC_AWS_LOADER_ROOT_PATH", + "", + "S3 path prefix for Loader bucket. " + "If given, this is prefixed to all S3 keys.", + "tc_aws Compatibility", +) + +# TC_AWS Storage Settings + +Config.define( + "TC_AWS_STORAGE_BUCKET", + "", + "S3 bucket for Storage", + "tc_aws Compatibility", +) + +Config.define( + "TC_AWS_STORAGE_ROOT_PATH", + "", + "S3 path prefix for Storage bucket", + "tc_aws Compatibility", +) + +# TODO: Support SSE +Config.define( + "TC_AWS_STORAGE_SSE", + False, + "put data into S3 using the Server Side Encryption functionality to " + "encrypt data at rest in S3 " + "https://aws.amazon.com/about-aws/whats-new" + "/2011/10/04/amazon-s3-announces-server-side-encryption-support/", + "tc_aws Compatibility", +) + +# TODO: Support RRS +Config.define( + "TC_AWS_STORAGE_RRS", + False, + "put data into S3 with Reduced Redundancy " + "https://aws.amazon.com/about-aws/whats-new" + "/2010/05/19/announcing-amazon-s3-reduced-redundancy-storage/", + "tc_aws Compatibility", +) + +# Result Storage + +Config.define( + "TC_AWS_RESULT_STORAGE_BUCKET", + "", + "S3 bucket for result Storage", + "tc_aws Compatibility", +) + +Config.define( + "TC_AWS_RESULT_STORAGE_ROOT_PATH", + "", + "S3 path prefix for Result storage bucket", + "tc_aws Compatibility", +) + +# Thumbor AWS already does this +Config.define( + "TC_AWS_STORE_METADATA", + False, + "Store result with metadata (for instance content-type)", + "tc_aws Compatibility", +) + + +def __generate_config(): + config.generate_config() + + +if __name__ == "__main__": + __generate_config() diff --git a/thumbor_aws/loader.py b/thumbor_aws/loader.py new file mode 100644 index 0000000..d67b1eb --- /dev/null +++ b/thumbor_aws/loader.py @@ -0,0 +1,106 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# thumbor imaging service +# https://github.com/thumbor/thumbor/wiki + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2011 globo.com thumbor@googlegroups.com + + +from thumbor.loaders import LoaderResult + +from thumbor_aws.config import Config +from thumbor_aws.s3_client import S3Client + +Config.define( + "AWS_LOADER_REGION_NAME", + "us-east-1", + "Region where thumbor's objects are going to be loaded from.", + "AWS Loader", +) + +Config.define( + "AWS_LOADER_BUCKET_NAME", + "thumbor", + "S3 Bucket where thumbor's objects are loaded from.", + "AWS Loader", +) + +Config.define( + "AWS_LOADER_S3_SECRET_ACCESS_KEY", + None, + "Secret access key for S3 Loader.", + "AWS Loader", +) + +Config.define( + "AWS_LOADER_S3_ACCESS_KEY_ID", + None, + "Access key ID for S3 Loader.", + "AWS Loader", +) + +Config.define( + "AWS_LOADER_S3_ENDPOINT_URL", + None, + "Endpoint URL for S3 API. Very useful for testing.", + "AWS Loader", +) + +Config.define( + "AWS_LOADER_ROOT_PATH", + "/st", + "Loader prefix path.", + "AWS Loader", +) + + +async def load(context, path): + """Loader to get source files from S3""" + + client = S3Client(context) + client.configuration = { + "region_name": context.config.AWS_LOADER_REGION_NAME, + "secret_access_key": context.config.AWS_LOADER_S3_SECRET_ACCESS_KEY, + "access_key_id": context.config.AWS_LOADER_S3_ACCESS_KEY_ID, + "endpoint_url": context.config.AWS_LOADER_S3_ENDPOINT_URL, + "bucket_name": context.config.AWS_LOADER_BUCKET_NAME, + "root_path": context.config.AWS_LOADER_ROOT_PATH, + } + if client.compatibility_mode is True: + client.configuration["region_name"] = context.config.TC_AWS_REGION + client.configuration["endpoint_url"] = context.config.TC_AWS_ENDPOINT + client.configuration[ + "bucket_name" + ] = context.config.TC_AWS_LOADER_BUCKET + client.configuration[ + "root_path" + ] = context.config.TC_AWS_LOADER_ROOT_PATH + + norm_path = normalize_url(client.configuration["root_path"], path) + result = LoaderResult() + + status_code, body, last_modified = await client.get_data(norm_path) + + if status_code != 200: + result.error = LoaderResult.ERROR_NOT_FOUND + result.extra = body + result.successful = False + return result + + result.successful = True + result.buffer = body + + result.metadata.update( + size=len(body), + updated_at=last_modified, + ) + + return result + + +def normalize_url(prefix: str, path: str) -> str: + """Function to normalize URLs before reaching into S3""" + return f"{prefix.rstrip('/')}/{path.lstrip('/')}" diff --git a/thumbor_aws/result_storage.py b/thumbor_aws/result_storage.py index 6d7f046..1c7cf75 100644 --- a/thumbor_aws/result_storage.py +++ b/thumbor_aws/result_storage.py @@ -17,38 +17,122 @@ from thumbor.result_storages import BaseStorage, ResultStorageResult from thumbor.utils import logger +from thumbor_aws.config import Config from thumbor_aws.s3_client import S3Client +Config.define( + "AWS_RESULT_STORAGE_REGION_NAME", + "us-east-1", + "Region where thumbor's objects are going to be stored.", + "AWS Result Storage", +) + +Config.define( + "AWS_RESULT_STORAGE_BUCKET_NAME", + "thumbor", + "S3 Bucket where thumbor's objects are going to be stored.", + "AWS Result Storage", +) + +Config.define( + "AWS_RESULT_STORAGE_S3_SECRET_ACCESS_KEY", + None, + "Secret access key for S3 to allow thumbor to store objects there.", + "AWS Result Storage", +) + +Config.define( + "AWS_RESULT_STORAGE_S3_ACCESS_KEY_ID", + None, + "Access key ID for S3 to allow thumbor to store objects there.", + "AWS Result Storage", +) + +Config.define( + "AWS_RESULT_STORAGE_S3_ENDPOINT_URL", + None, + "Endpoint URL for S3 API. Very useful for testing.", + "AWS Result Storage", +) + +Config.define( + "AWS_RESULT_STORAGE_ROOT_PATH", + "/rs", + "Result Storage prefix path.", + "AWS Result Storage", +) + +Config.define( + "AWS_RESULT_STORAGE_S3_ACL", + None, + "ACL to use for storing items in S3.", + "AWS Result Storage", +) + class Storage(BaseStorage, S3Client): + def __init__(self, context): + BaseStorage.__init__(self, context) + S3Client.__init__(self, context) + if self.compatibility_mode: + self.configuration["region_name"] = self.config.TC_AWS_REGION + self.configuration["endpoint_url"] = self.config.TC_AWS_ENDPOINT + self.configuration[ + "bucket_name" + ] = self.config.TC_AWS_RESULT_STORAGE_BUCKET + self.configuration[ + "root_path" + ] = self.config.TC_AWS_RESULT_STORAGE_ROOT_PATH + @property def region_name(self) -> str: - return self.context.config.AWS_RESULT_STORAGE_REGION_NAME + return self.configuration.get( + "region_name", self.context.config.AWS_RESULT_STORAGE_REGION_NAME + ) @property def secret_access_key(self) -> str: - return self.context.config.AWS_RESULT_STORAGE_S3_SECRET_ACCESS_KEY + return self.configuration.get( + "secret_access_key", + self.context.config.AWS_RESULT_STORAGE_S3_SECRET_ACCESS_KEY, + ) @property def access_key_id(self) -> str: - return self.context.config.AWS_RESULT_STORAGE_S3_ACCESS_KEY_ID + return self.configuration.get( + "access_key_id", + self.context.config.AWS_RESULT_STORAGE_S3_ACCESS_KEY_ID, + ) @property def endpoint_url(self) -> str: - return self.context.config.AWS_RESULT_STORAGE_S3_ENDPOINT_URL + return self.configuration.get( + "endpoint_url", + self.context.config.AWS_RESULT_STORAGE_S3_ENDPOINT_URL, + ) @property def bucket_name(self) -> str: - return self.context.config.AWS_RESULT_STORAGE_BUCKET_NAME + return self.configuration.get( + "bucket_name", + self.context.config.AWS_RESULT_STORAGE_BUCKET_NAME, + ) @property def file_acl(self) -> str: - return self.context.config.AWS_RESULT_STORAGE_S3_ACL + return self.configuration.get( + "file_acl", + self.context.config.AWS_RESULT_STORAGE_S3_ACL, + ) @property def root_path(self) -> str: """Defines the path prefix for all result storage images in S3""" - return self.context.config.AWS_RESULT_STORAGE_ROOT_PATH.rstrip("/") + + return self.configuration.get( + "root_path", + self.context.config.AWS_RESULT_STORAGE_ROOT_PATH, + ) async def put(self, image_bytes: bytes) -> str: file_abspath = self.normalize_path(self.context.request.url) @@ -60,19 +144,30 @@ async def put(self, image_bytes: bytes) -> str: content_type, self.context.config.AWS_DEFAULT_LOCATION, ) - logger.info("[RESULT_STORAGE] Image uploaded successfully to %s", file_abspath) + logger.info( + "[RESULT_STORAGE] Image uploaded successfully to %s", file_abspath + ) return response @property def is_auto_webp(self) -> bool: - """Identifies the current request if it's being auto converted to webp""" - return self.context.config.AUTO_WEBP and self.context.request.accepts_webp + """ + Identifies the current request if it's + being auto converted to webp + """ + return ( + self.context.config.AUTO_WEBP and self.context.request.accepts_webp + ) def normalize_path(self, path: str) -> str: """Returns the path used for result storage""" prefix = "auto_webp" if self.is_auto_webp else "default" fs_path = unquote(path).lstrip("/") - return f"{self.root_path}/{prefix}/{fs_path}" + return ( + f"{self.root_path.rstrip('/')}/" + f"{prefix.lstrip('/')}/" + f"{fs_path.lstrip('/')}" + ) async def get(self) -> ResultStorageResult: path = self.context.request.url @@ -82,7 +177,9 @@ async def get(self) -> ResultStorageResult: exists = await self.object_exists(file_abspath) if not exists: - logger.debug("[RESULT_STORAGE] image not found at %s", file_abspath) + logger.debug( + "[RESULT_STORAGE] image not found at %s", file_abspath + ) return None status, body, last_modified = await self.get_data(file_abspath) @@ -94,7 +191,8 @@ async def get(self) -> ResultStorageResult: return None logger.info( - "[RESULT_STORAGE] Image retrieved successfully at %s.", file_abspath + "[RESULT_STORAGE] Image retrieved successfully at %s.", + file_abspath, ) return ResultStorageResult( diff --git a/thumbor_aws/s3_client.py b/thumbor_aws/s3_client.py index 01b639f..2156d47 100644 --- a/thumbor_aws/s3_client.py +++ b/thumbor_aws/s3_client.py @@ -13,152 +13,71 @@ from aiobotocore.client import AioBaseClient from aiobotocore.session import AioSession, get_session -from thumbor.config import Config, config +from thumbor.config import Config from thumbor.context import Context from thumbor.utils import logger -Config.define( - "AWS_DEFAULT_LOCATION", - "https://{bucket_name}.s3.amazonaws.com", - ( - "Default location to use if S3 does not return location header." - " Can use {bucket_name} var." - ), - "AWS Storage", -) - -Config.define( - "AWS_STORAGE_REGION_NAME", - "us-east-1", - "Region where thumbor's objects are going to be stored.", - "AWS Storage", -) - -Config.define( - "AWS_STORAGE_BUCKET_NAME", - "thumbor", - "S3 Bucket where thumbor's objects are going to be stored.", - "AWS Storage", -) - -Config.define( - "AWS_STORAGE_S3_SECRET_ACCESS_KEY", - None, - "Secret access key for S3 to allow thumbor to store objects there.", - "AWS Storage", -) - -Config.define( - "AWS_STORAGE_S3_ACCESS_KEY_ID", - None, - "Access key ID for S3 to allow thumbor to store objects there.", - "AWS Storage", -) - -Config.define( - "AWS_STORAGE_S3_ENDPOINT_URL", - None, - "Endpoint URL for S3 API. Very useful for testing.", - "AWS Storage", -) - -Config.define( - "AWS_STORAGE_ROOT_PATH", - "/st", - "Storage prefix path.", - "AWS Storage", -) - -Config.define( - "AWS_STORAGE_S3_ACL", - "public-read", - "Storage ACL for files written in bucket", - "AWS Storage", -) - -Config.define( - "AWS_RESULT_STORAGE_REGION_NAME", - "us-east-1", - "Region where thumbor's objects are going to be stored.", - "AWS Result Storage", -) - -Config.define( - "AWS_RESULT_STORAGE_BUCKET_NAME", - "thumbor", - "S3 Bucket where thumbor's objects are going to be stored.", - "AWS Result Storage", -) - -Config.define( - "AWS_RESULT_STORAGE_S3_SECRET_ACCESS_KEY", - None, - "Secret access key for S3 to allow thumbor to store objects there.", - "AWS Result Storage", -) - -Config.define( - "AWS_RESULT_STORAGE_S3_ACCESS_KEY_ID", - None, - "Access key ID for S3 to allow thumbor to store objects there.", - "AWS Result Storage", -) - -Config.define( - "AWS_RESULT_STORAGE_S3_ENDPOINT_URL", - None, - "Endpoint URL for S3 API. Very useful for testing.", - "AWS Result Storage", -) - -Config.define( - "AWS_RESULT_STORAGE_ROOT_PATH", - "/rs", - "Result Storage prefix path.", - "AWS Result Storage", -) - -Config.define( - "AWS_RESULT_STORAGE_S3_ACL", - None, - "ACL to use for storing items in S3.", - "AWS Result Storage", -) - class S3Client: __session: AioSession = None context: Context = None + configuration: dict[str, object] = None + + def __init__(self, context): + self.context = context + self.configuration = {} + + @property + def config(self) -> Config: + """Thumbor config from context""" + return self.context.config + + @property + def compatibility_mode(self) -> bool: + """Should thumbor-aws run in compatibility mode?""" + return self.context.config.THUMBOR_AWS_RUN_IN_COMPATIBILITY_MODE @property def region_name(self) -> str: """Region to save the file to""" - return self.context.config.AWS_STORAGE_REGION_NAME + return self.configuration.get( + "region_name", self.config.AWS_STORAGE_REGION_NAME + ) @property def secret_access_key(self) -> str: """Secret access key to connect to AWS with""" - return self.context.config.AWS_STORAGE_S3_SECRET_ACCESS_KEY + return self.configuration.get( + "secret_access_key", self.config.AWS_STORAGE_S3_SECRET_ACCESS_KEY + ) @property def access_key_id(self) -> str: """Access key ID to connect to AWS with""" - return self.context.config.AWS_STORAGE_S3_ACCESS_KEY_ID + return self.configuration.get( + "access_key_id", self.config.AWS_STORAGE_S3_ACCESS_KEY_ID + ) @property def endpoint_url(self) -> str: """AWS Endpoint URL. Very useful for testing""" - return self.context.config.AWS_STORAGE_S3_ENDPOINT_URL + return self.configuration.get( + "endpoint_url", self.config.AWS_STORAGE_S3_ENDPOINT_URL + ) @property def bucket_name(self) -> str: """Bucket to save the file to""" - return self.context.config.AWS_STORAGE_BUCKET_NAME + return self.configuration.get( + "bucket_name", self.config.AWS_STORAGE_BUCKET_NAME + ) @property def file_acl(self) -> str: """ACL to save the files with""" - return self.context.config.AWS_STORAGE_S3_ACL + return self.configuration.get( + "file_acl", self.config.AWS_STORAGE_S3_ACL + ) @property def session(self) -> AioSession: @@ -216,7 +135,9 @@ async def upload( "Location Headers was not found in response" ) logger.warning(msg) - location = default_location.format(bucket_name=self.bucket_name) + location = default_location.format( + bucket_name=self.bucket_name + ) return f"{location.rstrip('/')}/{path.lstrip('/')}" @@ -227,7 +148,9 @@ async def get_data( async with self.get_client() as client: try: - response = await client.get_object(Bucket=self.bucket_name, Key=path) + response = await client.get_object( + Bucket=self.bucket_name, Key=path + ) except client.exceptions.NoSuchKey: return 404, b"", None @@ -250,7 +173,9 @@ async def object_exists(self, filepath: str): async with self.get_client() as client: try: - await client.get_object_acl(Bucket=self.bucket_name, Key=filepath) + await client.get_object_acl( + Bucket=self.bucket_name, Key=filepath + ) return True except client.exceptions.NoSuchKey: return False @@ -259,7 +184,9 @@ async def get_object_acl(self, filepath: str): """Gets an object's metadata""" async with self.get_client() as client: - return await client.get_object_acl(Bucket=self.bucket_name, Key=filepath) + return await client.get_object_acl( + Bucket=self.bucket_name, Key=filepath + ) def get_status_code(self, response: Mapping[str, Any]) -> int: """Gets the status code from an AWS response object""" @@ -291,18 +218,10 @@ def _is_expired( """Identifies whether an AWS S3 object is expired""" if expiration is None: - expiration = self.context.config.STORAGE_EXPIRATION_SECONDS + expiration = self.config.STORAGE_EXPIRATION_SECONDS if expiration is None: return False timediff = datetime.datetime.now(datetime.timezone.utc) - last_modified return timediff.total_seconds() > expiration - - -def __generate_config(): - config.generate_config() - - -if __name__ == "__main__": - __generate_config() diff --git a/thumbor_aws/storage.py b/thumbor_aws/storage.py index eed3f96..9bb2e5a 100644 --- a/thumbor_aws/storage.py +++ b/thumbor_aws/storage.py @@ -16,14 +16,80 @@ from thumbor.engines import BaseEngine from thumbor.utils import logger +from thumbor_aws.config import Config from thumbor_aws.s3_client import S3Client +Config.define( + "AWS_STORAGE_REGION_NAME", + "us-east-1", + "Region where thumbor's objects are going to be stored.", + "AWS Storage", +) + +Config.define( + "AWS_STORAGE_BUCKET_NAME", + "thumbor", + "S3 Bucket where thumbor's objects are going to be stored.", + "AWS Storage", +) + +Config.define( + "AWS_STORAGE_S3_SECRET_ACCESS_KEY", + None, + "Secret access key for S3 to allow thumbor to store objects there.", + "AWS Storage", +) + +Config.define( + "AWS_STORAGE_S3_ACCESS_KEY_ID", + None, + "Access key ID for S3 to allow thumbor to store objects there.", + "AWS Storage", +) + +Config.define( + "AWS_STORAGE_S3_ENDPOINT_URL", + None, + "Endpoint URL for S3 API. Very useful for testing.", + "AWS Storage", +) + +Config.define( + "AWS_STORAGE_ROOT_PATH", + "/st", + "Storage prefix path.", + "AWS Storage", +) + +Config.define( + "AWS_STORAGE_S3_ACL", + "public-read", + "Storage ACL for files written in bucket", + "AWS Storage", +) + class Storage(storages.BaseStorage, S3Client): + def __init__(self, context): + S3Client.__init__(self, context) + storages.BaseStorage.__init__(self, context) + if self.compatibility_mode: + self.configuration["region_name"] = self.config.TC_AWS_REGION + self.configuration["endpoint_url"] = self.config.TC_AWS_ENDPOINT + self.configuration[ + "bucket_name" + ] = self.config.TC_AWS_STORAGE_BUCKET + self.configuration[ + "root_path" + ] = self.config.TC_AWS_STORAGE_ROOT_PATH + @property def root_path(self) -> str: """Defines the path prefix for all storage images in S3""" - return self.context.config.AWS_STORAGE_ROOT_PATH.rstrip("/") + return self.configuration.get( + "root_path", + self.config.AWS_STORAGE_ROOT_PATH, + ) async def put(self, path: str, file_bytes: bytes) -> str: content_type = BaseEngine.get_mimetype(file_bytes) @@ -100,7 +166,6 @@ async def get_detector_data(self, path: str) -> Any: async def exists(self, path: str) -> bool: normalized_path = self.normalize_path(path) - print(normalized_path) return await self.object_exists(normalized_path) async def remove(self, path: str): @@ -123,4 +188,4 @@ async def remove(self, path: str): def normalize_path(self, path: str) -> str: """Returns the path used for storage""" path = unquote(path).lstrip("/") - return f"{self.root_path}/{path}" + return f"{self.root_path.rstrip('/')}/{path.lstrip('/')}"