Merge pull request #5 from rohanpm/s3-test

Introduce prototype for upload API, based on S3

docs/api-guide.rst

@@ -0,0 +1,69 @@
+.. _api-guide:
+
+API User Guide
+==============
+
+This document provides an overview of the APIs available in exodus-gw
+along with general usage information.
+
+For a full reference on the available endpoints, see:
+:ref:`api-reference`.
+
+.. warning::
+
+    exodus-gw is in early stages of development. All APIs are subject
+    to backwards-incompatible changes without warning.
+
+
+Authentication
+--------------
+
+Authentication is currently unimplemented in exodus-gw; any clients with
+access to the service can perform all operations.
+
+
+Uploading blobs
+---------------
+
+.. seealso:: `Upload API reference <api.html#tag/upload>`_
+
+.. automodule:: exodus_gw.s3.api
+
+Using boto3 with the upload API
+...............................
+
+As the upload API is partially compatible with S3, it is possible to use
+the AWS SDK to perform uploads. This is the recommended method of using the API.
+
+Use ``endpoint_url`` when creating a boto resource or client to point at exodus-gw.
+Region and credentials will be ignored.
+
+Note that, as the upload API provides only a subset of the S3 API, many boto methods
+will not work. Uploading files is supported.
+
+Usage:
+
+.. code-block:: python
+
+    import boto3
+    from botocore.config import Config
+
+    # Prepare S3 resource pointing at exodus-gw
+    s3 = boto3.resource('s3',
+                        endpoint_url='https://exodus-gw.example.com/upload',
+                        # If SSL needs to be configured:
+                        verify='/path/to/bundle.pem',
+                        config=Config(client_cert=('client.crt', 'client.key')))
+
+    # Basic APIs such as upload_file now work as usual
+    bucket = s3.Bucket('exodus-cdn-dev')
+    bucket.upload_file('/tmp/hello.txt',
+                       'aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f')
+
+
+
+Publishing content
+------------------
+
+exodus-gw is expected to provide an API for making uploaded content
+externally accessible on the exodus CDN. This API is not yet implemented.

docs/api.rst

@@ -0,0 +1,10 @@
+.. _api-reference:
+
+API Reference
+=============
+
+This page is a placeholder, expected to be overwritten with openapi/api.html
+during doc build.
+
+If you are seeing this page in published docs, something has gone wrong,
+please report it!

docs/conf.py

@@ -94,7 +94,6 @@
     "github_banner": True,
     "description": "Publishing microservice for Red Hat's Content Delivery Network",
     "extra_nav_links": {
-        "API": "api.html",
         "Source": "https://github.com/release-engineering/exodus-gw",
         "Index": "genindex.html",
     },
@@ -109,6 +108,9 @@
 
 # This dir contains the generated openapi spec and the static redoc-based
 # api.html for viewing. Copy it alongside the other docs.
+# Note: to enable sphinx linking to api.html, we also have an api.rst
+# and it's expected to be overwritten here. We depend on the implementation
+# detail that html_extra_path copying happens after .rst rendering.
 html_extra_path = ["openapi"]
 
 # Custom sidebar templates, must be a dictionary that maps document names

docs/index.rst

@@ -1,8 +1,22 @@
 exodus-gw
 =========
 
-Publishing microservice for Red Hat's Content Delivery Network
+Publishing microservice for Red Hat's Content Delivery Network.
 
 .. toctree::
    :maxdepth: 2
    :caption: Contents:
+
+   api-guide
+   api
+
+Overview
+--------
+
+exodus-gw (or "exodus gateway") is a microservice for uploading and publishing content
+onto Red Hat's CDN. It exposes an API for use by publishing tools.
+
+- For general usage information on the exodus-gw API, see :ref:`api-guide`.
+- For a technical reference on the exodus-gw API, see :ref:`api-reference`.
+- For more general information on the CDN architecture and how exodus-gw fits in,
+  see `Architecture <https://release-engineering.github.io/exodus-lambda/arch.html>`_.

examples/s3-upload

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# Upload a list of files from the command-line to s3, possibly
+# using a custom endpoint.
+#
+# This command is used as a baseline for testing s3 uploads
+# via exodus-gw or via s3 directly. It can be used to compare
+# performance and compatibility.
+#
+# Usage:
+#
+#  # Using default S3
+#  examples/s3-upload file1 [file2 [...]]
+#
+#  # Testing same thing via exodus-gw
+#  uvicorn exodus_gw.gateway:app &
+#  examples/s3-upload --endpoint-url http://localhost:8000/upload file1 [file2 [...]]
+#
+# It is recommended to test using a mixture of files both smaller
+# and larger than 10MB to cover both multipart and single part
+# uploads.
+
+import boto3
+import sys
+import hashlib
+import argparse
+
+
+def get_object_key(filename):
+    with open(filename, "rb") as f:
+        hasher = hashlib.sha256()
+        while True:
+            chunk = f.read(1024 * 1024 * 10)
+            if not chunk:
+                break
+            hasher.update(chunk)
+        return hasher.hexdigest()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--endpoint-url", default=None)
+    parser.add_argument("--bucket", default="exodus-cdn-dev")
+    parser.add_argument("files", nargs="+")
+
+    args = parser.parse_args()
+    s3 = boto3.resource("s3", endpoint_url=args.endpoint_url)
+    bucket = s3.Bucket(args.bucket)
+
+    print(
+        "Using endpoint:",
+        "[default]" if not args.endpoint_url else args.endpoint_url,
+    )
+
+    for filename in args.files:
+        bucket.upload_file(filename, get_object_key(filename))
+        print("Uploaded:", filename)
+
+
+if __name__ == "__main__":
+    main()

exodus_gw/__init__.py

@@ -1,5 +1,5 @@
 # Any modules which add endpoints onto the app must be imported here
-from . import gateway
+from . import gateway, s3
 
 # Per ASGI conventions, main entry point is accessible as "exodus_gw:application"
 from .app import app as application

exodus_gw/s3/__init__.py

@@ -0,0 +1 @@
+from . import api