Merge 4240f13 into 6f8bc4e

.gitignore

@@ -9,3 +9,4 @@ dist/
 dist-aws/
 tests/db/
 .pytest_cache
+vendor/

.travis.yml

@@ -1,4 +1,4 @@
-sudo: false
+sudo: true #required to use docker
 notifications:
   email: false
 language: python
@@ -8,22 +8,20 @@ env:
   - TOX_ENV=py36
   - TOX_ENV=coveralls
   - TOX_ENV=flake8
+services:
+  - docker
 install:
   - pip install tox
 script:
   - tox -e $TOX_ENV
 jobs:
   include:
-    - stage: s3 deploy
+    - stage: publish image
       install:
         - pip install pipenv awscli
       script: skip
-      before_deploy: bash build.sh
+      before_deploy: make dist
       deploy:
         skip_cleanup: true
-        provider: s3
-        access_key_id: $AWS_ACCESS_KEY_ID
-        secret_access_key: $AWS_SECRET_ACCESS_KEY
-        bucket: "carbon-deploy"
-        local_dir: "dist-aws"
-      after_deploy: bash publish.sh
+        provider: script
+        script: make publish

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.6
+
+ENV LD_LIBRARY_PATH /opt/lib/
+
+RUN apt-get update && apt-get install -y unzip libaio1
+RUN pip install --upgrade pip
+RUN pip install pipenv
+COPY vendor/instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip /
+RUN unzip -j instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip -d /opt/lib/
+
+COPY Pipfile* /
+RUN pipenv install --system --deploy
+COPY dist/carbon*.whl /
+RUN pip install carbon*.whl
+
+ENTRYPOINT ["carbon"]
+CMD ["--help"]

Makefile

@@ -1,29 +1,51 @@
-.PHONY: all clean install release test tests update
+.PHONY: install deps wheel container dist clean test tests update
 SHELL=/bin/bash
-RELEASE_TYPE=patch
+S3_BUCKET=carbon-deploy
+LIBAIO_SO=libaio.so.1.0.1
+ORACLE_ZIP=instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip
+ECR_REGISTRY=672626379771.dkr.ecr.us-east-1.amazonaws.com
 
-all: test
+install:
+	pipenv install
+
+vendor/$(ORACLE_ZIP):
+	aws s3 cp s3://$(S3_BUCKET)/$(ORACLE_ZIP) vendor/$(ORACLE_ZIP)
+
+deps: vendor/$(ORACLE_ZIP)
+
+wheel:
+	pipenv run python setup.py bdist_wheel
+
+container:
+	docker build -t $(ECR_REGISTRY)/carbon:latest \
+		-t $(ECR_REGISTRY)/carbon:`git describe --always` \
+		-t carbon:latest .
+
+dist: deps wheel container
+	@tput setaf 2
+	@tput bold
+	@echo "Finished building docker image. Try running:"
+	@echo "  $$ docker run --rm carbon:latest"
+	@tput sgr0
 
 clean:
 	find . -name "*.pyc" -print0 | xargs -0 rm -f
 	find . -name '__pycache__' -print0 | xargs -0 rm -rf
-	rm -rf .coverage .tox *.egg-info .eggs build/ build-aws/ dist/ dist-aws/
+	rm -rf .coverage .tox *.egg-info .eggs build/ dist/
 
-install:
-	pipenv install
+distclean: clean
+	rm -rf vendor/
 
 test:
 	tox
 
 tests: test
 
-release:
-	pipenv run bumpversion $(RELEASE_TYPE)
-	@tput setaf 2
-	@echo Built release for `git describe --tag`. Make sure to run:
-	@echo "  $$ git push origin `git rev-parse --abbrev-ref HEAD` tag `git describe --tag`"
-	@tput sgr0
-
 update:
+	pipenv clean
 	pipenv update --dev
-	pipenv lock -r > requirements.txt
+
+publish:
+	$(aws ecr get-login --no-include-email --region us-east-1)
+	docker push $(ECR_REGISTRY)/carbon:latest
+	docker push $(ECR_REGISTRY)/carbon:`git describe --always`

Pipfile

@@ -3,7 +3,6 @@ url = "https://pypi.python.org/simple"
 verify_ssl = true
 
 [dev-packages]
-bumpversion = "*"
 coveralls = "*"
 pytest = "*"
 pytest-cov = "*"

README.rst

@@ -1,7 +1,7 @@
 carbon
 ======
 
-Carbon is a tool for generating a feed of people that can be loaded into Symplectic Elements. It is run as a Lambda by a scheduled CloudWatch event.
+Carbon is a tool for generating a feed of people that can be loaded into Symplectic Elements. It is designed to be run as a container.
 
 Developing
 ----------
@@ -25,27 +25,25 @@ If you do need to connect to the data warehouse, you will also need to install t
 
 On Linux, you will also need to make sure you have libaio installed. You can probably just use your system's package manager to install this easily. The package may be called ``libaio1``.
 
-The Lambda Package
-------------------
+Building
+--------
 
-The Lambda package is built with the ``build.sh`` script.
+Running ``make dist`` creates a new container that is tagged as ``carbon:latest``. It will also add tags for the ECR registry with both the ``latest`` and the git short hash. The first build will take some time, but subsequent builds should be fast.
 
-**IMPORTANT**: Both the ``cx_Oracle`` and ``lxml`` packages use platform dependent wheels, so you if you are planning on using the Lambda package you *must* do this step on a Linux x86_64 architecture.
+We are restricted from distributing the Oracle client library, so a copy is kept in a private S3 bucket for use when building the container image. If you are updating this, make sure you are using a Linux x86_64 version.
 
-We are restricted from distributing the Oracle client library, so a copy is kept in a private S3 bucket for use when building the Lambda. The Lambda execution context does not provide libaio, so a copy of this is also kept in the bucket. If you are updating either of these, make sure you are using a Linux x86_64 version. To be on the safe side, use https://github.com/lambci/docker-lambda to get the libaio library as that will ensure you are getting one that's been linked against an appropriate version of glibc.
-
-The build process downloads each of these files from S3 so you should have the AWS CLI installed and configured to authenticate using an account with appropriate access.
+The build process downloads this file from S3 so you should have the AWS CLI installed and configured to authenticate using an account with appropriate access.
 
 Deploying
 ---------
 
-Deployment is currently being handled by Travis. When a PR is merged onto the master branch Travis will build a new Lambda package, push it to S3 and update the Lambda function to point to the new package.
+Deployment is currently being handled by Travis. When a PR is merged onto the master branch Travis will build a new container image, tag it both with ``latest`` and with the git short hash, and then push both tags to the ECR registry.
 
-If you need to deploy a new package outside of Travis then do the following, *noting the restrictions on running build.sh described above*::
+If you need to deploy a new image outside of Travis then do the following::
 
     $ cd carbon
-    $ ./build.sh
-    $ ./publish.sh --upload
+    $ make clean
+    $ make dist && make publish
 
 Configuration
 ^^^^^^^^^^^^^

carbon/__init__.py

@@ -5,5 +5,3 @@
 
     A tool for generating people.
 """
-
-__version__ = '1.2.0'

setup.py

@@ -3,33 +3,30 @@
 Carbon, a people loader.
 """
 
-import io
-import re
 from setuptools import find_packages, setup
+import subprocess
 
-
-with io.open('LICENSE') as f:
+with open('LICENSE') as f:
     license = f.read()
 
-with open('carbon/__init__.py', 'r') as fp:
-    version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fp.read(),
-                        re.MULTILINE).group(1)
+try:
+    output = subprocess.run(['git', 'describe', '--always'],
+                            stdout=subprocess.PIPE, encoding='utf-8')
+    version = output.stdout.strip()
+except subprocess.CalledProcessError as e:
+    version = 'unknown'
 
 setup(
     name='carbon',
-    version=version,
+    version="1.0.0-" + version,
     description='Load people into Elements',
     long_description=__doc__,
     url='https://github.com/MITLibraries/carbon',
     license=license,
     author='Mike Graves',
     author_email='mgraves@mit.edu',
     packages=find_packages(exclude=['tests']),
-    install_requires=[
-        'click',
-        'lxml',
-        'SQLAlchemy',
-    ],
+    install_requires=[],
     entry_points={
         'console_scripts': [
             'carbon = carbon.cli:main',
@@ -40,10 +37,7 @@
         'Intended Audience :: Developers',
         'Environment :: Console',
         'License :: OSI Approved :: Apache Software License',
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
-        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.6',
     ]
 )

tests/test_cli.py

@@ -21,14 +21,14 @@ def runner():
 def test_people_returns_people(runner, xml_data):
     res = runner.invoke(main, ['--db', 'sqlite://', 'people'])
     assert res.exit_code == 0
-    assert res.output_bytes == \
+    assert res.stdout_bytes == \
         ET.tostring(xml_data, encoding="UTF-8", xml_declaration=True)
 
 
 def test_articles_returns_articles(runner, articles_data):
     res = runner.invoke(main, ['--db', 'sqlite://', 'articles'])
     assert res.exit_code == 0
-    assert res.output_bytes == \
+    assert res.stdout_bytes == \
         ET.tostring(articles_data, encoding='UTF-8', xml_declaration=True)
 
 

tox.ini

@@ -3,6 +3,7 @@ envlist = py36,coverage,flake8
 skipsdist = True
 
 [testenv]
+basepython = python3.6
 passenv = HOME
 deps =
   pipenv