Merge pull request #1 from oarepo/build

Initial release

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "3rdparty/s3-client"]
 	path = 3rdparty/s3-client
-	url = git@github.com:CESNET/s3-client
+	url = https://github.com/CESNET/s3-client

.travis.yml

@@ -22,20 +22,14 @@ cache:
   - pip
 
 env:
-  global:
-    - BOTO_CONFIG=/dev/null
   matrix:
-    - REQUIREMENTS=lowest
     - REQUIREMENTS=release DEPLOY=true
     - REQUIREMENTS=devel
 
 python:
-  - "3.6"
-  - "3.7"
   - "3.8"
 
 before_install:
-  - "nvm install 6; nvm use 6"
   - "travis_retry pip install --upgrade pip setuptools py"
   - "travis_retry pip install twine wheel coveralls requirements-builder"
   - "requirements-builder -e all --level=min setup.py > .travis-lowest-requirements.txt"
@@ -44,6 +38,7 @@ before_install:
 
 install:
   - "travis_retry pip install -r .travis-${REQUIREMENTS}-requirements.txt"
+  - "travis_retry pip install 3rdparty/s3-client"
   - "travis_retry pip install -e .[all]"
   - "pip freeze"
 
@@ -55,12 +50,15 @@ after_success:
 
 deploy:
   provider: pypi
-  user: oarepo
+  user: "__token__"
   password:
-    secure: "enp0kMohUFWXHsDRvFjNGUxDRPg0LZGR99IHnQE+KP5rt3HRBfRRQWbcqBxBI/61019PPEPbMxIZwNMIYBrEVPHKYTwTT6RNcGH8nMjosxU7zzHweUrugTJjQFnincuU6PEDjkj9mN76OGPnmFAaDWCBd3aIZ4vt1nN5klJfSCsU0tqnKLRy2M/MegMs2j27gFUaU+eINul8wDePBACyLwftli2y3jaA8t+McNON33J4w3/HG504AzVvYh7M7rfR13tpMih5BhdHsC9ww0pNMqgDImgWq1JEdKqBu6OODEjGu+9t5u5pgnbMJco13Pi1Vhm3/1WBieItKQhQTPhu3r1Em4nknKymGJBHUiaz7218DAPah8gVGX0I5q2OmDNP7HVw9kmLv7a2wFj2jN6rnB0BRRwkLapu0J9ASCzfn2JgNElazTe46ykEoPUUpW8t1GXTN+sGC6+ldNop42JBdtCmrmC6cwVXFIP3jRIf+PYfFTSkz96shE6iOnt4+NCcxnahQ2uIoYF17v24M0b2+TWMIOvcAAieUm9jenLeh8juV9U+mcleHxzNor71p+8czn1/ppJEZdorzcvDVE9uDe6noNmYDlrjyhQ3ci0fmR2jQylUvZ3vI0hPgLL7+KDL45nhoPCbs8dFk8UDbj34j/YV4YYTbaIf55F1tYIZAik"
+    secure: "AZDDkNIYFnHTINqmPPuRDC+X7Hwi92iMBi2DvqsAc2GJ375th+bC70cjJbhysnBwCgMvJGMYtuAxtSrPWLEMIC3kRd7y0X9ZGvRlGlG0M4YhegTfGl9pshwgQL/Jbqz6Ct8O5wAcXloqnUqfCUpWqVtoDiPUnfgW+Nv3rZfpbUJK4/i5S8GVSsk9TCxJECgGvg09RicsgR0pN+sBLLrEYgC1WnNnDB8396oNaGD92BCTLlrwYd9H8TJWe19VsTeVxY9keAkmqXdhv/5y2sypWmXSl1NDvkZUTmCanC+vTnPGsrziAq9OyFr068q3FT5ETdTk6nhWtSElwo0kUWwIAvQ6SbqgI2QOntGvBIf5nbHTsAhPxUU3vr6N7sJmLtKoeLnrQqWFypCoLOsIVCnu6aPNFd/FU+LQdAIFolsQ/RjMZrMn71/K2hKMDVHjiyhxySgXZWOwIc3AplrX1I3a5+PgNwvXueNtXUdDywmgP6DrYSzqEpNbZ9Jj9a172LaMb2uqru1Lc62PCix9omZxM4Zf3eThQUFLftHrtb5mLERtFrt6LSM+hhdNGeNmATasd6RiHiLJZOTmXikNWs9WI79/TmK5bFl3u/HzDsE5XDnPyHcSe9ePEcttrEzORebodOqoqr6MtWevDPYDTmd4aiNdLYlr5ud9V0y+wG+Jh+0="
   distributions: "sdist bdist_wheel"
   on:
     tags: true
-    python: "3.6"
+    branch: "master"
+    python: "3.8"
     repo: oarepo/oarepo-s3
     condition: $DEPLOY = true
+  distributions: "sdist bdist_wheel"
+  skip_existing: true

MANIFEST.in

@@ -21,3 +21,13 @@ recursive-include docs Makefile
 recursive-include examples *.py
 recursive-include examples *.sh
 recursive-include tests *.py
+recursive-include 3rdparty *.py
+recursive-include 3rdparty *.md
+recursive-exclude 3rdparty .gitkeep
+recursive-exclude 3rdparty *.yaml
+recursive-include 3rdparty *.toml
+recursive-include 3rdparty *.txt
+include .gitmodules
+include docker-compose.yml
+include docker-services.yml
+exclude .envrc

oarepo_s3/api.py

@@ -93,9 +93,11 @@ def __init__(self, key, expires, size,
 
 
 @use_kwargs(multipart_init_args)
-def multipart_uploader(record, key, files, pid, request, endpoint, resolver, size=None, part_size=None, **kwargs):
+def multipart_uploader(record, key, files, pid, request, endpoint,
+                       resolver, size=None, part_size=None, **kwargs):
     """Multipart upload handler."""
-    from oarepo_s3.views import MultipartUploadCompleteResource, MultipartUploadAbortResource
+    from oarepo_s3.views import MultipartUploadAbortResource, \
+        MultipartUploadCompleteResource
 
     expiration = current_app.config['S3_MULTIPART_UPLOAD_EXPIRATION']
     complete = resolver(MultipartUploadCompleteResource.view_name, key=key)

oarepo_s3/proxies.py

@@ -43,6 +43,4 @@
 from flask import current_app
 from werkzeug.local import LocalProxy
 
-from oarepo_s3.ext import OARepoS3State
-
-current_s3: OARepoS3State = LocalProxy(lambda: current_app.extensions['oarepo-s3'])
+current_s3 = LocalProxy(lambda: current_app.extensions['oarepo-s3'])

oarepo_s3/s3.py

@@ -51,10 +51,13 @@ def __init__(self, access_key, secret_key, client_kwargs=dict,
         self.endpoint_url = client_kwargs.get('endpoint_url', None)
         self.client_kwargs = client_kwargs
         self.config_kwargs = config_kwargs
-        self.client = S3MultipartClient(self.endpoint_url, access_key, secret_key)
+        self.client = S3MultipartClient(
+            self.endpoint_url, access_key, secret_key)
 
     def init_multipart_upload(self, bucket, object_name, object_size):
-        """Creates a multipart upload to AWS S3 API and returns session configuration with pre-signed urls."""
+        """Creates a multipart upload to AWS S3 API and returns
+           session configuration with pre-signed urls.
+        """
         return self.client.signed_s3_multipart_upload(bucket=bucket,
                                                       object_name=object_name,
                                                       size=object_size,
@@ -64,9 +67,12 @@ def init_multipart_upload(self, bucket, object_name, object_size):
 
     def complete_multipart_upload(self, bucket, object_name, parts, upload_id):
         """Completes a multipart upload to AWS S3."""
-        self.client.finish_multipart_upload(bucket, object_name, parts, upload_id)
-        return self.client.finish_file_metadata(bucket, object_name, object_name)
+        self.client.finish_multipart_upload(
+            bucket, object_name, parts, upload_id)
+        return self.client.finish_file_metadata(
+            bucket, object_name, object_name)
 
     def abort_multipart_upload(self, bucket, object_name, upload_id):
         """Cancels an in-progress multipart upload to AWS S3."""
-        return self.client.abort_multipart_upload(bucket, object_name, upload_id)
+        return self.client.abort_multipart_upload(
+            bucket, object_name, upload_id)

oarepo_s3/storage.py

@@ -10,9 +10,9 @@
 from functools import wraps
 
 from invenio_files_rest.storage import pyfs_storage_factory
-from invenio_s3 import S3FSFileStorage
 from s3fs.core import split_path
 
+from invenio_s3 import S3FSFileStorage
 from oarepo_s3.api import MultipartUpload
 from oarepo_s3.proxies import current_s3
 
@@ -58,7 +58,8 @@ def multipart_save(self, bucket, mu: MultipartUpload):
         urls on the multipartUpload to be used by the client to
         directly communicate with the S3 multipart APIs.
         """
-        mu.session = current_s3.client.init_multipart_upload(bucket, mu.key, mu.size)
+        mu.session = current_s3.client.init_multipart_upload(
+            bucket, mu.key, mu.size)
         mu.session['bucket'] = bucket
         return self.fileurl, mu.size, None
 

oarepo_s3/views.py

@@ -77,7 +77,8 @@ def post(self, pid, record, key, parts):
             parts,
             mup['upload_id'])
 
-        file_rec['multipart_upload']['status'] = MultipartUploadStatus.COMPLETED
+        file_rec['multipart_upload']['status'] = \
+            MultipartUploadStatus.COMPLETED
         files.flush()
         record.commit()
 
@@ -112,13 +113,16 @@ def post(self, pid, record, key):
 
 
 def multipart_actions(code, files, rest_endpoint, extra, is_draft):
-    # decide if view should be created on this resource and return blueprint mapping
+    # decide if view should be created on this
+    # resource and return blueprint mapping
     # rest path -> view
     return {
-        'files/<key>/complete-multipart': MultipartUploadCompleteResource.as_view(
-            MultipartUploadCompleteResource.view_name.format(endpoint=code)
-        ),
-        'files/<key>/abort-multipart': MultipartUploadAbortResource.as_view(
-            MultipartUploadAbortResource.view_name.format(endpoint=code)
-        )
+        'files/<key>/complete-multipart':
+            MultipartUploadCompleteResource.as_view(
+                MultipartUploadCompleteResource.view_name.format(endpoint=code)
+            ),
+        'files/<key>/abort-multipart':
+            MultipartUploadAbortResource.as_view(
+                MultipartUploadAbortResource.view_name.format(endpoint=code)
+            )
     }

pytest.ini

@@ -7,5 +7,5 @@
 
 [pytest]
 pep8ignore = docs/conf.py ALL
-;addopts = --pep8 --cov=oarepo_s3 --cov-report=term-missing
+addopts = --pep8 --cov=oarepo_s3 --cov-report=term-missing
 testpaths = tests oarepo_s3

run-tests.sh

@@ -7,7 +7,6 @@
 # under the terms of the MIT License; see LICENSE file for more details.
 
 pydocstyle invenio_s3 tests docs && \
-isort -rc -c -df && \
+isort -c -df oarepo_s3 && \
 check-manifest --ignore ".travis-*" && \
-sphinx-build -qnNW docs docs/_build/html && \
 python setup.py test

setup.cfg

@@ -8,11 +8,6 @@
 [aliases]
 test = pytest
 
-[build_sphinx]
-source-dir = docs/
-build-dir = docs/_build
-all_files = 1
-
 [bdist_wheel]
 universal = 1
 

tests/conftest.py

@@ -16,11 +16,12 @@
 
 import boto3
 import pytest
-from flask import current_app, Flask, url_for
+from flask import Flask, current_app, url_for
 from flask.testing import FlaskClient
 from invenio_app.factory import create_api
 from invenio_base.signals import app_loaded
-from invenio_db import InvenioDB, db as _db
+from invenio_db import InvenioDB
+from invenio_db import db as _db
 from invenio_files_rest import InvenioFilesREST
 from invenio_files_rest.models import Bucket, Location
 from invenio_indexer import InvenioIndexer
@@ -34,56 +35,66 @@
 from invenio_records_rest.utils import PIDConverter, allow_all
 from invenio_records_rest.views import create_blueprint_from_app
 from invenio_rest import InvenioREST
-from invenio_s3 import InvenioS3
 from invenio_search import InvenioSearch
 from invenio_search.cli import destroy, init
-from marshmallow import Schema, fields, INCLUDE
+from marshmallow import INCLUDE, Schema, fields
 from mock import patch
 from moto import mock_s3
 from oarepo_records_draft.ext import RecordsDraft
-from oarepo_validate import SchemaKeepingRecordMixin, MarshmallowValidatedRecordMixin
+from oarepo_validate import MarshmallowValidatedRecordMixin, \
+    SchemaKeepingRecordMixin
 from s3_client_lib.utils import get_file_chunk_size
-from sqlalchemy_utils import database_exists, create_database
+from sqlalchemy_utils import create_database, database_exists
 
+from invenio_s3 import InvenioS3
 from oarepo_s3 import S3FileStorage
 from oarepo_s3.ext import OARepoS3
 from oarepo_s3.s3 import S3Client
 from tests.utils import draft_entrypoints
 
-SAMPLE_ALLOWED_SCHEMAS = ['http://localhost:5000/schemas/records/record-v1.0.0.json']
-SAMPLE_PREFERRED_SCHEMA = 'http://localhost:5000/schemas/records/record-v1.0.0.json'
+SAMPLE_ALLOWED_SCHEMAS = [
+    'http://localhost:5000/schemas/records/record-v1.0.0.json']
+SAMPLE_PREFERRED_SCHEMA = \
+    'http://localhost:5000/schemas/records/record-v1.0.0.json'
 
 
 class TestSchemaV1(Schema):
     """Testing record schema."""
+
     title = fields.String()
 
     class Meta:
+        """Test schema Meta class."""
+
         unknown = INCLUDE
 
 
 class TestRecord(SchemaKeepingRecordMixin,
                  MarshmallowValidatedRecordMixin,
                  Record):
     """Fake test record."""
+
     ALLOWED_SCHEMAS = SAMPLE_ALLOWED_SCHEMAS
     PREFERRED_SCHEMA = SAMPLE_PREFERRED_SCHEMA
     MARSHMALLOW_SCHEMA = TestSchemaV1
 
 
 class TestIndexer(RecordIndexer):
     """Fake record indexer."""
+
     def index(self, record, arguments=None, **kwargs):
         """Fake index implementation."""
         return {}
 
+
 class MockedS3Client(S3Client):
     """Fake S3 client."""
 
     def init_multipart_upload(self, bucket, object_name, object_size):
         """Fake init multipart upload implementation."""
         max_parts, chunk_size = get_file_chunk_size(object_size)
-        return {"parts_url": [f'http://localhost/test/{i}' for i in range(1, max_parts + 1)],
+        parts = [f'http://localhost/test/{i}' for i in range(1, max_parts + 1)]
+        return {"parts_url": parts,
                 "chunk_size": chunk_size,
                 "checksum_update": "",
                 "upload_id": str(uuid.uuid4()),
@@ -102,7 +113,10 @@ def abort_multipart_upload(self, bucket, object_name, upload_id):
 
 
 class JsonClient(FlaskClient):
+    """Test REST JSON client."""
+
     def open(self, *args, **kwargs):
+        """Opens a new connection."""
         kwargs.setdefault('content_type', 'application/json')
         kwargs.setdefault('Accept', 'application/json')
         return super().open(*args, **kwargs)
@@ -177,7 +191,7 @@ def db(app):
     """Create database for the tests."""
     with app.app_context():
         if not database_exists(str(_db.engine.url)) and \
-            app.config['SQLALCHEMY_DATABASE_URI'] != 'sqlite://':
+           app.config['SQLALCHEMY_DATABASE_URI'] != 'sqlite://':
             create_database(_db.engine.url)
         _db.create_all()
 
@@ -225,7 +239,8 @@ def app_config(app_config):
             'indexer_class': TestIndexer,
             'list_route': '/records/',
             'item_route': '/records/<pid(recid, '
-                          'record_class="invenio_records_files.api.Record"):pid_value>',
+                          'record_class="invenio_records_files.api.Record"'
+                          '):pid_value>',
         },
         'drecid': {
             'create_permission_factory_imp': allow_all,
@@ -253,7 +268,9 @@ def app_config(app_config):
 
 @pytest.fixture(scope='module')
 def draft_config(app_config):
-    app_config['RECORDS_DRAFT_ENDPOINTS'] = app_config['RECORDS_REST_ENDPOINTS']
+    """Draft endpoints configuration."""
+    app_config['RECORDS_DRAFT_ENDPOINTS'] = \
+        app_config['RECORDS_REST_ENDPOINTS']
     app_config['RECORDS_DRAFT_ENDPOINTS'].update(
         dict(
             recid=dict(
@@ -269,19 +286,13 @@ def draft_config(app_config):
 
 @pytest.fixture()
 def prepare_es(app, db):
-    runner = app.test_cli_runner()
-    result = runner.invoke(destroy, ['--yes-i-know', '--force'])
-    if result.exit_code:
-        print(result.output, file=sys.stderr)
-    assert result.exit_code == 0
-    result = runner.invoke(init)
-    if result.exit_code:
-        print(result.output, file=sys.stderr)
-    assert result.exit_code == 0
+    """Prepare ES indices."""
+    return
 
 
 @pytest.fixture()
 def draft_app(app):
+    """Drafts app fixture."""
     return app
 
 
@@ -335,6 +346,7 @@ def __init__(self, **kwargs):
 
 @pytest.fixture
 def draft_records_url(app):
+    """Draft endpoint url."""
     return url_for('oarepo_records_rest.draft_records_list')
 
 
@@ -376,7 +388,8 @@ def objects():
     objs = []
     for key, content in [
         ('LICENSE', b'license file'),
-        ('README.rst', b'readme file')]:
+        ('README.rst', b'readme file')
+    ]:
         objs.append(
             (key, BytesIO(content), len(content))
         )
@@ -412,7 +425,7 @@ def record(app, db, s3_location):
 
 @pytest.fixture()
 def draft_record(app, db, prepare_es, s3_location):
-    # let's create a record
+    """Testing draft-enabled record."""
     draft_uuid = uuid.uuid4()
     data = {
         'title': 'blah',