Restore /lambdas

.circleci/config.yml

@@ -5,6 +5,9 @@ workflows:
   test:
     jobs:
       - test-3.6
+      - test-lambdas-indexer
+      - test-lambdas-preview
+      - test-lambdas-shared
   build_and_deploy:
     jobs:
       - build:
@@ -38,6 +41,60 @@ jobs:
             cd api/python && pytest --cov=./
             codecov
 
+  test-lambdas-indexer:
+    docker:
+      - image: circleci/python:3.6-jessie
+    steps:
+      - checkout
+      - run:
+          name: Setup
+          command: |
+            virtualenv venv
+            . venv/bin/activate
+            pip install lambdas/shared/
+            pip install -r lambdas/es/indexer/requirements.txt
+            pip install -r lambdas/es/indexer/test-requirements.txt
+      - run:
+          name: Pytest
+          command: |
+            . venv/bin/activate
+            cd lambdas/es/indexer && pytest --cov=./
+
+  test-lambdas-preview:
+    docker:
+      - image: circleci/python:3.6-jessie
+    steps:
+      - checkout
+      - run:
+          name: Setup
+          command: |
+            virtualenv venv
+            . venv/bin/activate
+            pip install lambdas/shared/
+            pip install -r lambdas/preview/requirements.txt
+            pip install -r lambdas/preview/test-requirements.txt
+      - run:
+          name: Pytest
+          command: |
+            . venv/bin/activate
+            cd lambdas/preview && pytest --cov=./
+
+  test-lambdas-shared:
+    docker:
+      - image: circleci/python:3.6-jessie
+    steps:
+      - checkout
+      - run:
+          name: Setup
+          command: |
+            virtualenv venv
+            . venv/bin/activate
+            pip install lambdas/shared/[tests]
+      - run:
+          name: Pytest
+          command: |
+            . venv/bin/activate
+            cd lambdas/shared && pytest --cov=./
   build:
     docker:
       - image: circleci/python:3.6-jessie

lambdas/.gitignore

@@ -0,0 +1 @@
+__pycache__

lambdas/bucket-setup/cfnresponse.py

@@ -0,0 +1,43 @@
+# Copied from https://github.com/jorgebastida/cfn-response
+# Added .encode('utf-8')
+
+import json
+
+try:
+    from urllib2 import HTTPError, build_opener, HTTPHandler, Request
+except ImportError:
+    from urllib.error import HTTPError
+    from urllib.request import build_opener, HTTPHandler, Request
+
+
+SUCCESS = "SUCCESS"
+FAILED = "FAILED"
+
+
+def send(event, context, response_status, reason=None, response_data=None, physical_resource_id=None):
+    response_data = response_data or {}
+    response_body = json.dumps(
+        {
+            'Status': response_status,
+            'Reason': reason or "See the details in CloudWatch Log Stream: " + context.log_stream_name,
+            'PhysicalResourceId': physical_resource_id or context.log_stream_name,
+            'StackId': event['StackId'],
+            'RequestId': event['RequestId'],
+            'LogicalResourceId': event['LogicalResourceId'],
+            'Data': response_data
+        }
+    ).encode('utf-8')
+
+    opener = build_opener(HTTPHandler)
+    request = Request(event['ResponseURL'], data=response_body)
+    request.add_header('Content-Type', '')
+    request.add_header('Content-Length', len(response_body))
+    request.get_method = lambda: 'PUT'
+    try:
+        response = opener.open(request)
+        print("Status code: {}".format(response.getcode()))
+        print("Status message: {}".format(response.msg))
+        return True
+    except HTTPError as exc:
+        print("Failed executing HTTP request: {}".format(exc.code))
+        return False

lambdas/bucket-setup/index.py

@@ -0,0 +1,76 @@
+"""
+Modify bucket properties so that bucket works well with Quilt;
+intended for QuiltBucket (CloudFormation parameter)
+"""
+import boto3
+import botocore
+import cfnresponse
+
+S3_CLIENT = boto3.client('s3')
+
+def handler(event, context):
+    """entry point"""
+    if event['RequestType'] == 'Delete':
+        cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
+        return
+
+    try:
+        props = event['ResourceProperties']
+        bucket = props['Bucket']
+        catalog_host = props['QuiltWebHost']
+        enable_versioning(bucket)
+        set_cors(bucket, catalog_host)
+        cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
+        return
+    except Exception as ex:
+        print(ex)
+        cfnresponse.send(event, context, cfnresponse.FAILED, {})
+        raise
+
+def enable_versioning(bucket):
+    """switch on object versionsing"""
+    S3_CLIENT.put_bucket_versioning(
+        Bucket=bucket,
+        VersioningConfiguration={
+            'Status': 'Enabled'
+        }
+    )
+
+def set_cors(bucket, catalog_host):
+    """set CORS so catalog works properly"""
+    new_cors_rule = {
+        'AllowedHeaders': ['*'],
+        'AllowedMethods': [
+            'GET',
+            'HEAD',
+            'PUT',
+            'POST'
+        ],
+        'AllowedOrigins': [
+            'https://' + catalog_host
+        ],
+        'ExposeHeaders': [
+            'Content-Length',
+            'Content-Range',
+            'x-amz-meta-helium'
+        ],
+        'MaxAgeSeconds': 3000
+    }
+
+    try:
+        existing_cors_rules = S3_CLIENT.get_bucket_cors(Bucket=bucket)['CORSRules']
+    # if there's no CORS set at all, we'll get an error
+    except botocore.exceptions.ClientError as problem:
+        if 'NoSuchCORSConfiguration' in str(problem):
+            existing_cors_rules = []
+        else:
+            raise
+
+    if new_cors_rule not in existing_cors_rules:
+        existing_cors_rules.append(new_cors_rule)
+        S3_CLIENT.put_bucket_cors(
+            Bucket=bucket,
+            CORSConfiguration={
+                'CORSRules': existing_cors_rules
+            }
+        )

lambdas/bucket-setup/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+    name='lambda_function',
+    version='0.0.1',
+    py_modules=['index', 'cfnresponse'],
+)