From d15a84776491b47c22bd97dd3260e7fbd1da7a54 Mon Sep 17 00:00:00 2001 From: Mike Graves Date: Wed, 10 Oct 2018 13:44:00 -0400 Subject: [PATCH 1/4] Create docker build tooling Since we are switching to Fargate, this changes the build process to create a container. The app code itself has not changed as it already supports this mode of operation. This commit adds the necessary tooling to create the container. The Makefile has been updated to handle the whole build process. Running `make dist` should result in built container. Versioning of the app has been switched from semver to the commit hash. The travis deploy step will need to be updated once we are set up with ECR. --- .travis.yml | 24 +++--- Dockerfile | 20 +++++ Makefile | 44 ++++++---- Pipfile | 1 - Pipfile.lock | 202 ++++++++++++++++++++++----------------------- carbon/__init__.py | 2 - setup.py | 25 +++--- tests/test_cli.py | 4 +- tox.ini | 1 + 9 files changed, 174 insertions(+), 149 deletions(-) create mode 100644 Dockerfile diff --git a/.travis.yml b/.travis.yml index 9639866..b4efaa1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: false +sudo: true #required to use docker notifications: email: false language: python @@ -8,22 +8,24 @@ env: - TOX_ENV=py36 - TOX_ENV=coveralls - TOX_ENV=flake8 +services: + - docker install: - pip install tox script: - tox -e $TOX_ENV jobs: include: - - stage: s3 deploy + - stage: ECR deploy install: - pip install pipenv awscli script: skip - before_deploy: bash build.sh - deploy: - skip_cleanup: true - provider: s3 - access_key_id: $AWS_ACCESS_KEY_ID - secret_access_key: $AWS_SECRET_ACCESS_KEY - bucket: "carbon-deploy" - local_dir: "dist-aws" - after_deploy: bash publish.sh + before_deploy: make dist + #deploy: + # skip_cleanup: true + # provider: s3 + # access_key_id: $AWS_ACCESS_KEY_ID + # secret_access_key: $AWS_SECRET_ACCESS_KEY + # bucket: "carbon-deploy" + # local_dir: "dist-aws" + #after_deploy: bash publish.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6e4169a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.6 + +ENV LD_LIBRARY_PATH /opt/lib/ + +RUN apt-get update && apt-get install -y unzip +RUN pip install --upgrade pip +RUN pip install pipenv +COPY dist/libaio.so.1.0.1 /opt/lib/ +RUN ln -s /opt/lib/libaio.so.1.0.1 libaio.so.1 && \ + ln -s /opt/lib/libaio.so.1 liaio.so +COPY dist/instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip / +RUN unzip -j instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip -d /opt/lib/ + +COPY Pipfile* / +RUN pipenv install --system --deploy +COPY dist/carbon*.whl / +RUN pip install carbon*.whl + +ENTRYPOINT ["carbon"] +CMD ["--help"] diff --git a/Makefile b/Makefile index e1fc0be..24238f9 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,43 @@ -.PHONY: all clean install release test tests update +.PHONY: install deps wheel container dist clean test tests update SHELL=/bin/bash -RELEASE_TYPE=patch +S3_BUCKET=carbon-deploy +LIBAIO_SO=libaio.so.1.0.1 +ORACLE_ZIP=instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip -all: test +install: + pipenv install + +dist/$(LIBAIO_SO): + aws s3 cp s3://$(S3_BUCKET)/$(LIBAIO_SO) dist/$(LIBAIO_SO) + +dist/$(ORACLE_ZIP): + aws s3 cp s3://$(S3_BUCKET)/$(ORACLE_ZIP) dist/$(ORACLE_ZIP) + +deps: dist/$(LIBAIO_SO) dist/$(ORACLE_ZIP) + +wheel: + pipenv run python setup.py bdist_wheel + +container: + docker build -t carbon:`git describe --always` -t carbon:latest . + +dist: deps wheel container + @tput setaf 2 + @tput bold + @echo "Finished building docker image. Try running:" + @echo " $$ docker run --rm carbon:`git describe --always`" + @tput sgr0 clean: find . -name "*.pyc" -print0 | xargs -0 rm -f find . -name '__pycache__' -print0 | xargs -0 rm -rf - rm -rf .coverage .tox *.egg-info .eggs build/ build-aws/ dist/ dist-aws/ - -install: - pipenv install + rm -rf .coverage .tox *.egg-info .eggs build/ dist/ test: tox tests: test -release: - pipenv run bumpversion $(RELEASE_TYPE) - @tput setaf 2 - @echo Built release for `git describe --tag`. Make sure to run: - @echo " $$ git push origin `git rev-parse --abbrev-ref HEAD` tag `git describe --tag`" - @tput sgr0 - update: + pipenv clean pipenv update --dev - pipenv lock -r > requirements.txt diff --git a/Pipfile b/Pipfile index 64dce16..43c8b4d 100644 --- a/Pipfile +++ b/Pipfile @@ -3,7 +3,6 @@ url = "https://pypi.python.org/simple" verify_ssl = true [dev-packages] -bumpversion = "*" coveralls = "*" pytest = "*" pytest-cov = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 84a8c0f..a86bee1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9cb0c8c59b46204ba9c4851537e3e8fecbe8c7313981270a3a29ee320c2e0f63" + "sha256": "105531717c77e769b2953c64c43b3af967caabcb313e8299eb70fd820619f57b" }, "pipfile-spec": 6, "requires": {}, @@ -15,70 +15,70 @@ "default": { "click": { "hashes": [ - "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", - "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" ], - "version": "==6.7" + "version": "==7.0" }, "cx-oracle": { "hashes": [ - "sha256:075dba1d4ed45dea9d5d7ed27d301baeb0656f1668790c7190473d8f0ae6b2d8", - "sha256:1d15e1137c6cd0739da679a58254880a0cb5502cb7347b8adc9a131f81d0c21c", - "sha256:23270b341f7b92126be46e2d9fe39ff0d620ccba6bfd567e2326fab13214b466", - "sha256:3519bf3263c9892aaadc844735aca02d3773ed9b92f97e069cd1726882a7d1b6", - "sha256:53612d52f8ae290bba01eda24db1f013c03028d7c08aa8c068a7336f8f4c30b3", - "sha256:671f323eca25d957b52ce93438f7cdd274b601281ddc8d3bd4558b72bb3cbdf9", - "sha256:70b2bd1a04d142999c5f83171160147f9d52f6e77924f40f3f13d0d5e35813d0", - "sha256:817be3a0ac11e5afedf73709c1bf0e6ddbf5e491a2bfaa2202f4c503c3b08462", - "sha256:b76f05402499592861a45dfef431eab888096c87ce9040ed8536a011c12f699c", - "sha256:d3d98b83a1f4d6a1ea2bb020d3d3d2db2285e65d8b38f0ff24bc5ed5deb1d4a0", - "sha256:d5e0ad3fbd1178cdaa5f0450cebc04bc1b14e5bf8db5c8e6e22547b80d65f08c", - "sha256:d964183361e044f35d110373705752244ecaa7f7974f5ef179316a1e612b038c", - "sha256:e0ae4c0a401fe508d76883d6123b5c8641d626182f94f9fddd8668390292dd35", - "sha256:fc4903e03b6b3aee10f7d26649939683b9e1f702cea74cfcc1644a29c4780fe4" + "sha256:0095ceb8aa0e0db6cfba7cb43a930e71c573242a74e75bc5df938a761a072767", + "sha256:01194cd86c2660e49d41f502209d2d509223fe3902daaa621b5deb1853c65347", + "sha256:0b8dd8c694b133ed0601eb9d187c01dc2568f44eb083436c4ea8185bf834b17e", + "sha256:1157e26b1d3c5b5f1564e43b29293bd2b77e0718670fe5f47b64f0db95342f68", + "sha256:3528b8a80f1212cdb4767afc908afde6c79fdf539d6572fb0bd78163389e1b5e", + "sha256:45cf78c4df9873aaa2084250d9e3fc9772f1d67f374c2c9b236fd88c0ecc2e90", + "sha256:6e1e97203b42853eb67cba12b51dc4ebb19727c1a611f88abf12d761b9c60d07", + "sha256:6ee61bfba245afbc1254c65f33b6b2f3b0433a0b27926ebafb1800c3ab5c2c5a", + "sha256:75ee5edccf385d8e8b1443058909fbf3477bb1db12ab7f367aafba8d993fc828", + "sha256:81748e3cdda8d6fc4ae0c81ee358cacb247f32fbe115ef9e217fe97b969dfbdc", + "sha256:85e8adad50bb94097973b8786b10641b4f132e3f597155679effd3ba3bafe13a", + "sha256:97b9868047eaa8f7ebac0d61344dc1f040eea8bae68f9c76db90edcff1596661", + "sha256:ea9abc49a2ef10501f8669e7518c55876466c0f65bcf3dd895ee6b812508eb13", + "sha256:f7bc65bbe043da66491c3d9ac35503e4d054e3082656672ff637774f8efeb81d" ], - "version": "==6.4.1" + "version": "==7.0.0" }, "lxml": { "hashes": [ - "sha256:0cf1eca0652c4409e0655e04b840d6d85b7eb18718f5fba3862acad5500e3480", - "sha256:10624ef1b468252309f269b13af4f837e3a82be366b5f3e49b0e83f1ad66205f", - "sha256:1259e374da3a575615fe402e0966c5894bae3d2e229c2239ba4ebf2bb020c4b6", - "sha256:26bb748af1ead0097eb8272b8a06f15a0015b8f312eef772a95f223a16e7de56", - "sha256:27d0b13bcfcf2f6a5664e64fc3d684c76db1cdba5a5761795d154063559e0b59", - "sha256:2b013fdabcbc21bc2770437099b921ec290235752b5baaac7a601f75094a378d", - "sha256:2e469ea2c0b722b9b393187649e7d126c537a68512fc92a676fe86e57050c2a9", - "sha256:37f7c2cdf513a0ea239c1609681880fb2f0073de0d2996e0ae9a7f0ef15d8b95", - "sha256:68c6afc7a4411db2df28307e2493c945cb3d887e8f431b81811c1ea6ba087b8b", - "sha256:73fe3452fc02c0b418914f842f897bdad0f1184368d8d9c315294ff7b94946f2", - "sha256:7584d83d7315f641510e5f97f4d636ea225fd76e3f8aee965b2e8c93a8169b4d", - "sha256:76e3ec6b26b1198dd5e6e20539d8360dcd3b224cd80cadba9307b790fda79161", - "sha256:8288a889cbaa446e5fa168837456e63098b91953c89e5857968a5091b337cdca", - "sha256:ad9e1fee284dec97b74cd88e925eca1575145598c974243cfb5e859f406adc32", - "sha256:b360c3769cf0fd7d82577e40e37d4caf693f67744d0d61d11d66b5c31eaccf7a", - "sha256:c4aaf320284a2713428163bae0e7df0db3b489237ab4830179210a12d56d3068", - "sha256:c530274e43b0f376cd94e8e0a3e6ea28de1739ec4326689bdbf626e172d2e614", - "sha256:ca4e79294fd0f3f9e0e5a4c309df84b5f2abc62349bfaf2aaf8965e5108ef8e2", - "sha256:ce2dc5a104e885abbd48d0cc92ae74afa1d685ee65d6e3497067207d6a26e177", - "sha256:d295cac30d3e13e82473081ea7df2a11352b5625cb54187fcd5a8be5d9ebf315", - "sha256:d498338b39c4757ba88bdc705b3a0647d18554856cd2d394ac3bb919ac890c9d", - "sha256:d537f8e613074805e17039e345edaa822534f66f07d315c89cff9824aa996d65", - "sha256:df8ba3f52ef59a553b0e94593ea526c34faa4f531c1ab7f5ca7f392bc770c9e3", - "sha256:e2553800d2d461a2dc329682d0a9068f238ec11d763e5454c61c4df7a0346ed2", - "sha256:e2afbe403090f5893e254958d02875e0732975e73c4c0cdd33c1f009a61963ca", - "sha256:e740efa625883f3c4de20c7e1411228d7ce2ab47b9e874a703f6681ec0558a30", - "sha256:ec7864b62da0f5ae44973351247f2250a25b9b544fc6aff8bd6a75da1156cc70", - "sha256:f26ddab491b10279b7e8e3fdcbaaaba3ab282fbaecfa48a19874dfc4d53b9d4f", - "sha256:f6a16681f30918521066ddcc4ba79c1e033c9837dd94f78f5a9f6110e7572185", - "sha256:f968623ac9b81a6253d4bbbe3f4d1e6be5f33707f397b566935783511bfa281a" + "sha256:02bc220d61f46e9b9d5a53c361ef95e9f5e1d27171cd461dddb17677ae2289a5", + "sha256:22f253b542a342755f6cfc047fe4d3a296515cf9b542bc6e261af45a80b8caf6", + "sha256:2f31145c7ff665b330919bfa44aacd3a0211a76ca7e7b441039d2a0b0451e415", + "sha256:36720698c29e7a9626a0dc802ef8885f8f0239bfd1689628ecd459a061f2807f", + "sha256:438a1b0203545521f6616132bfe0f4bca86f8a401364008b30e2b26ec408ce85", + "sha256:4815892904c336bbaf73dafd54f45f69f4021c22b5bad7332176bbf4fb830568", + "sha256:5be031b0f15ad63910d8e5038b489d95a79929513b3634ad4babf77100602588", + "sha256:5c93ae37c3c588e829b037fdfbd64a6e40c901d3f93f7beed6d724c44829a3ad", + "sha256:60842230678674cdac4a1cf0f707ef12d75b9a4fc4a565add4f710b5fcf185d5", + "sha256:62939a8bb6758d1bf923aa1c13f0bcfa9bf5b2fc0f5fa917a6e25db5fe0cfa4e", + "sha256:75830c06a62fe7b8fe3bbb5f269f0b308f19f3949ac81cfd40062f47c1455faf", + "sha256:81992565b74332c7c1aff6a913a3e906771aa81c9d0c68c68113cffcae45bc53", + "sha256:8c892fb0ee52c594d9a7751c7d7356056a9682674b92cc1c4dc968ff0f30c52f", + "sha256:9d862e3cf4fc1f2837dedce9c42269c8c76d027e49820a548ac89fdcee1e361f", + "sha256:a623965c086a6e91bb703d4da62dabe59fe88888e82c4117d544e11fd74835d6", + "sha256:a7783ab7f6a508b0510490cef9f857b763d796ba7476d9703f89722928d1e113", + "sha256:aab09fbe8abfa3b9ce62aaf45aca2d28726b1b9ee44871dbe644050a2fff4940", + "sha256:abf181934ac3ef193832fb973fd7f6149b5c531903c2ec0f1220941d73eee601", + "sha256:ae07fa0c115733fce1e9da96a3ac3fa24801742ca17e917e0c79d63a01eeb843", + "sha256:b9c78242219f674ab645ec571c9a95d70f381319a23911941cd2358a8e0521cf", + "sha256:bccb267678b870d9782c3b44d0cefe3ba0e329f9af8c946d32bf3778e7a4f271", + "sha256:c4df4d27f4c93b2cef74579f00b1d3a31a929c7d8023f870c4b476f03a274db4", + "sha256:caf0e50b546bb60dfa99bb18dfa6748458a83131ecdceaf5c071d74907e7e78a", + "sha256:d3266bd3ac59ac4edcd5fa75165dee80b94a3e5c91049df5f7c057ccf097551c", + "sha256:db0d213987bcd4e6d41710fb4532b22315b0d8fb439ff901782234456556aed1", + "sha256:dbbd5cf7690a40a9f0a9325ab480d0fccf46d16b378eefc08e195d84299bfae1", + "sha256:e16e07a0ec3a75b5ee61f2b1003c35696738f937dc8148fbda9fe2147ccb6e61", + "sha256:e175a006725c7faadbe69e791877d09936c0ef2cf49d01b60a6c1efcb0e8be6f", + "sha256:edd9c13a97f6550f9da2236126bb51c092b3b1ce6187f2bd966533ad794bbb5e", + "sha256:fa39ea60d527fbdd94215b5e5552f1c6a912624521093f1384a491a8ad89ad8b" ], - "version": "==4.2.4" + "version": "==4.2.5" }, "sqlalchemy": { "hashes": [ - "sha256:72325e67fb85f6e9ad304c603d83626d1df684fdf0c7ab1f0352e71feeab69d8" + "sha256:c5951d9ef1d5404ed04bae5a16b60a0779087378928f997a294d1229c6ca4d3e" ], - "version": "==1.2.10" + "version": "==1.2.12" } }, "develop": { @@ -91,31 +91,25 @@ }, "atomicwrites": { "hashes": [ - "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585", - "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6" + "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", + "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" ], - "version": "==1.1.5" + "markers": "python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.2.*'", + "version": "==1.2.1" }, "attrs": { "hashes": [ - "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265", - "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b" + "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", + "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" ], - "version": "==18.1.0" - }, - "bumpversion": { - "hashes": [ - "sha256:6744c873dd7aafc24453d8b6a1a0d6d109faf63cd0cd19cb78fd46e74932c77e", - "sha256:6753d9ff3552013e2130f7bc03c1007e24473b4835952679653fb132367bdd57" - ], - "version": "==0.5.3" + "version": "==18.2.0" }, "certifi": { "hashes": [ - "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7", - "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0" + "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", + "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" ], - "version": "==2018.4.16" + "version": "==2018.8.24" }, "cffi": { "hashes": [ @@ -199,34 +193,35 @@ }, "coveralls": { "hashes": [ - "sha256:32569a43c9dbc13fa8199247580a4ab182ef439f51f65bb7f8316d377a1340e8", - "sha256:664794748d2e5673e347ec476159a9d87f43e0d2d44950e98ed0e27b98da8346" + "sha256:ab638e88d38916a6cedbf80a9cd8992d5fa55c77ab755e262e00b36792b7cd6d", + "sha256:b2388747e2529fa4c669fb1e3e2756e4e07b6ee56c7d9fce05f35ccccc913aa0" ], - "version": "==1.3.0" + "version": "==1.5.1" }, "cryptography": { "hashes": [ - "sha256:21af753934f2f6d1a10fe8f4c0a64315af209ef6adeaee63ca349797d747d687", - "sha256:27bb401a20a838d6d0ea380f08c6ead3ccd8c9d8a0232dc9adcc0e4994576a66", - "sha256:29720c4253263cff9aea64585adbbe85013ba647f6e98367efff9db2d7193ded", - "sha256:2a35b7570d8f247889784010aac8b384fd2e4a47b33e15c4a60b45a7c1944120", - "sha256:42c531a6a354407f42ee07fda5c2c0dc822cf6d52744949c182f2b295fbd4183", - "sha256:5eb86f03f9c4f0ac2336ac5431271072ddf7ecc76b338e26366732cfac58aa19", - "sha256:67f7f57eae8dede577f3f7775957f5bec93edd6bdb6ce597bb5b28e1bdf3d4fb", - "sha256:6ec84edcbc966ae460560a51a90046503ff0b5b66157a9efc61515c68059f6c8", - "sha256:7ba834564daef87557e7fcd35c3c3183a4147b0b3a57314e53317360b9b201b3", - "sha256:7d7f084cbe1fdb82be5a0545062b59b1ad3637bc5a48612ac2eb428ff31b31ea", - "sha256:82409f5150e529d699e5c33fa8fd85e965104db03bc564f5f4b6a9199e591f7c", - "sha256:87d092a7c2a44e5f7414ab02fb4145723ebba411425e1a99773531dd4c0e9b8d", - "sha256:8c56ef989342e42b9fcaba7c74b446f0cc9bed546dd00034fa7ad66fc00307ef", - "sha256:9449f5d4d7c516a6118fa9210c4a00f34384cb1d2028672100ee0c6cce49d7f6", - "sha256:bc2301170986ad82d9349a91eb8884e0e191209c45f5541b16aa7c0cfb135978", - "sha256:c132bab45d4bd0fff1d3fe294d92b0a6eb8404e93337b3127bdec9f21de117e6", - "sha256:c3d945b7b577f07a477700f618f46cbc287af3a9222cd73035c6ef527ef2c363", - "sha256:cee18beb4c807b5c0b178f4fa2fae03cef9d51821a358c6890f8b23465b7e5d2", - "sha256:d01dfc5c2b3495184f683574e03c70022674ca9a7be88589c5aba130d835ea90" + "sha256:02602e1672b62e803e08617ec286041cc453e8d43f093a5f4162095506bc0beb", + "sha256:10b48e848e1edb93c1d3b797c83c72b4c387ab0eb4330aaa26da8049a6cbede0", + "sha256:17db09db9d7c5de130023657be42689d1a5f60502a14f6f745f6f65a6b8195c0", + "sha256:227da3a896df1106b1a69b1e319dce218fa04395e8cc78be7e31ca94c21254bc", + "sha256:2cbaa03ac677db6c821dac3f4cdfd1461a32d0615847eedbb0df54bb7802e1f7", + "sha256:31db8febfc768e4b4bd826750a70c79c99ea423f4697d1dab764eb9f9f849519", + "sha256:4a510d268e55e2e067715d728e4ca6cd26a8e9f1f3d174faf88e6f2cb6b6c395", + "sha256:6a88d9004310a198c474d8a822ee96a6dd6c01efe66facdf17cb692512ae5bc0", + "sha256:76936ec70a9b72eb8c58314c38c55a0336a2b36de0c7ee8fb874a4547cadbd39", + "sha256:7e3b4aecc4040928efa8a7cdaf074e868af32c58ffc9bb77e7bf2c1a16783286", + "sha256:8168bcb08403ef144ff1fb880d416f49e2728101d02aaadfe9645883222c0aa5", + "sha256:8229ceb79a1792823d87779959184a1bf95768e9248c93ae9f97c7a2f60376a1", + "sha256:8a19e9f2fe69f6a44a5c156968d9fc8df56d09798d0c6a34ccc373bb186cee86", + "sha256:8d10113ca826a4c29d5b85b2c4e045ffa8bad74fb525ee0eceb1d38d4c70dfd6", + "sha256:be495b8ec5a939a7605274b6e59fbc35e76f5ad814ae010eb679529671c9e119", + "sha256:dc2d3f3b1548f4d11786616cf0f4415e25b0fbecb8a1d2cd8c07568f13fdde38", + "sha256:e4aecdd9d5a3d06c337894c9a6e2961898d3f64fe54ca920a72234a3de0f9cb3", + "sha256:e79ab4485b99eacb2166f3212218dd858258f374855e1568f728462b0e6ee0d9", + "sha256:f995d3667301e1754c57b04e0bae6f0fa9d710697a9f8d6712e8cca02550910f" ], - "version": "==2.3" + "markers": "python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.2.*'", + "version": "==2.3.1" }, "docopt": { "hashes": [ @@ -259,17 +254,18 @@ }, "py": { "hashes": [ - "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7", - "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e" + "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1", + "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6" ], - "markers": "python_version != '3.0.*' and python_version != '3.3.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version >= '2.7'", - "version": "==1.5.4" + "markers": "python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.2.*'", + "version": "==1.6.0" }, "pycparser": { "hashes": [ - "sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226" + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" ], - "version": "==2.18" + "markers": "python_version != '3.1.*' and python_version != '3.0.*' and python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.2.*'", + "version": "==2.19" }, "pyftpdlib": { "hashes": [ @@ -286,17 +282,17 @@ }, "pytest": { "hashes": [ - "sha256:86a8dbf407e437351cef4dba46736e9c5a6e3c3ac71b2e942209748e76ff2086", - "sha256:e74466e97ac14582a8188ff4c53e6cc3810315f342f6096899332ae864c1d432" + "sha256:7e258ee50338f4e46957f9e09a0f10fb1c2d05493fa901d113a8dafd0790de4e", + "sha256:9332147e9af2dcf46cd7ceb14d5acadb6564744ddff1fe8c17f0ce60ece7d9a2" ], - "version": "==3.7.1" + "version": "==3.8.2" }, "pytest-cov": { "hashes": [ - "sha256:03aa752cf11db41d281ea1d807d954c4eda35cfa1b21d6971966cc041bbf6e2d", - "sha256:890fe5565400902b0c78b5357004aab1c814115894f4f21370e2433256a3eeec" + "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", + "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" ], - "version": "==2.5.1" + "version": "==2.6.0" }, "pyyaml": { "hashes": [ diff --git a/carbon/__init__.py b/carbon/__init__.py index 805e827..7fa032c 100644 --- a/carbon/__init__.py +++ b/carbon/__init__.py @@ -5,5 +5,3 @@ A tool for generating people. """ - -__version__ = '1.2.0' diff --git a/setup.py b/setup.py index 220d00d..69a6e66 100644 --- a/setup.py +++ b/setup.py @@ -3,17 +3,19 @@ Carbon, a people loader. """ -import io -import re from setuptools import find_packages, setup +import subprocess -with io.open('LICENSE') as f: +with open('LICENSE') as f: license = f.read() -with open('carbon/__init__.py', 'r') as fp: - version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fp.read(), - re.MULTILINE).group(1) +try: + output = subprocess.run(['git', 'describe', '--always'], + stdout=subprocess.PIPE, encoding='utf-8') + version = output.stdout.strip() +except subprocess.CalledProcessError as e: + version = 'unknown' setup( name='carbon', @@ -25,11 +27,7 @@ author='Mike Graves', author_email='mgraves@mit.edu', packages=find_packages(exclude=['tests']), - install_requires=[ - 'click', - 'lxml', - 'SQLAlchemy', - ], + install_requires=[], entry_points={ 'console_scripts': [ 'carbon = carbon.cli:main', @@ -40,10 +38,7 @@ 'Intended Audience :: Developers', 'Environment :: Console', 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.6', ] ) diff --git a/tests/test_cli.py b/tests/test_cli.py index a77aef1..3457b3a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -21,14 +21,14 @@ def runner(): def test_people_returns_people(runner, xml_data): res = runner.invoke(main, ['--db', 'sqlite://', 'people']) assert res.exit_code == 0 - assert res.output_bytes == \ + assert res.stdout_bytes == \ ET.tostring(xml_data, encoding="UTF-8", xml_declaration=True) def test_articles_returns_articles(runner, articles_data): res = runner.invoke(main, ['--db', 'sqlite://', 'articles']) assert res.exit_code == 0 - assert res.output_bytes == \ + assert res.stdout_bytes == \ ET.tostring(articles_data, encoding='UTF-8', xml_declaration=True) diff --git a/tox.ini b/tox.ini index 3c89f77..e988eeb 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,7 @@ envlist = py36,coverage,flake8 skipsdist = True [testenv] +basepython = python3.6 passenv = HOME deps = pipenv From fb6f25acfad88080e2357d5fdfb25c141663c04a Mon Sep 17 00:00:00 2001 From: Mike Graves Date: Wed, 10 Oct 2018 15:21:53 -0400 Subject: [PATCH 2/4] Fix versioning problem This fixes an issue where pip won't install a wheel whose version number starts with a letter. I've also moved the required S3 libs into a vendor directory to save having to download them every time you run `make clean`. --- .gitignore | 1 + Dockerfile | 4 ++-- Makefile | 13 ++++++++----- setup.py | 3 +-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 1171f93..54ac84a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ dist/ dist-aws/ tests/db/ .pytest_cache +vendor/ diff --git a/Dockerfile b/Dockerfile index 6e4169a..b9baece 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,10 @@ ENV LD_LIBRARY_PATH /opt/lib/ RUN apt-get update && apt-get install -y unzip RUN pip install --upgrade pip RUN pip install pipenv -COPY dist/libaio.so.1.0.1 /opt/lib/ +COPY vendor/libaio.so.1.0.1 /opt/lib/ RUN ln -s /opt/lib/libaio.so.1.0.1 libaio.so.1 && \ ln -s /opt/lib/libaio.so.1 liaio.so -COPY dist/instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip / +COPY vendor/instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip / RUN unzip -j instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip -d /opt/lib/ COPY Pipfile* / diff --git a/Makefile b/Makefile index 24238f9..694f91b 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,13 @@ ORACLE_ZIP=instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip install: pipenv install -dist/$(LIBAIO_SO): - aws s3 cp s3://$(S3_BUCKET)/$(LIBAIO_SO) dist/$(LIBAIO_SO) +vendor/$(LIBAIO_SO): + aws s3 cp s3://$(S3_BUCKET)/$(LIBAIO_SO) vendor/$(LIBAIO_SO) -dist/$(ORACLE_ZIP): - aws s3 cp s3://$(S3_BUCKET)/$(ORACLE_ZIP) dist/$(ORACLE_ZIP) +vendor/$(ORACLE_ZIP): + aws s3 cp s3://$(S3_BUCKET)/$(ORACLE_ZIP) vendor/$(ORACLE_ZIP) -deps: dist/$(LIBAIO_SO) dist/$(ORACLE_ZIP) +deps: vendor/$(LIBAIO_SO) vendor/$(ORACLE_ZIP) wheel: pipenv run python setup.py bdist_wheel @@ -33,6 +33,9 @@ clean: find . -name '__pycache__' -print0 | xargs -0 rm -rf rm -rf .coverage .tox *.egg-info .eggs build/ dist/ +distclean: clean + rm -rf vendor/ + test: tox diff --git a/setup.py b/setup.py index 69a6e66..a24db5e 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,6 @@ from setuptools import find_packages, setup import subprocess - with open('LICENSE') as f: license = f.read() @@ -19,7 +18,7 @@ setup( name='carbon', - version=version, + version="1.0.0-" + version, description='Load people into Elements', long_description=__doc__, url='https://github.com/MITLibraries/carbon', From b46cce0aa252196d2a7e1fb4694bf653a7c05337 Mon Sep 17 00:00:00 2001 From: Mike Graves Date: Wed, 10 Oct 2018 16:19:07 -0400 Subject: [PATCH 3/4] Remove libaio from S3 Since we're deploying into a container we control, we can just install libaio with the system package manager. --- Dockerfile | 5 +---- Makefile | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index b9baece..fcce9b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,12 +2,9 @@ FROM python:3.6 ENV LD_LIBRARY_PATH /opt/lib/ -RUN apt-get update && apt-get install -y unzip +RUN apt-get update && apt-get install -y unzip libaio1 RUN pip install --upgrade pip RUN pip install pipenv -COPY vendor/libaio.so.1.0.1 /opt/lib/ -RUN ln -s /opt/lib/libaio.so.1.0.1 libaio.so.1 && \ - ln -s /opt/lib/libaio.so.1 liaio.so COPY vendor/instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip / RUN unzip -j instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip -d /opt/lib/ diff --git a/Makefile b/Makefile index 694f91b..da219ad 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,10 @@ ORACLE_ZIP=instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip install: pipenv install -vendor/$(LIBAIO_SO): - aws s3 cp s3://$(S3_BUCKET)/$(LIBAIO_SO) vendor/$(LIBAIO_SO) - vendor/$(ORACLE_ZIP): aws s3 cp s3://$(S3_BUCKET)/$(ORACLE_ZIP) vendor/$(ORACLE_ZIP) -deps: vendor/$(LIBAIO_SO) vendor/$(ORACLE_ZIP) +deps: vendor/$(ORACLE_ZIP) wheel: pipenv run python setup.py bdist_wheel From 4240f13d528e8cabcb4d7236318c36aec0f918b0 Mon Sep 17 00:00:00 2001 From: Mike Graves Date: Thu, 11 Oct 2018 14:41:28 -0400 Subject: [PATCH 4/4] Update travis deploy The deploy stage on travis will now build the image and push it to ECR. A new make target has been added to handle this. --- .travis.yml | 14 +++++--------- Makefile | 12 ++++++++++-- README.rst | 22 ++++++++++------------ 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index b4efaa1..198823d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,16 +16,12 @@ script: - tox -e $TOX_ENV jobs: include: - - stage: ECR deploy + - stage: publish image install: - pip install pipenv awscli script: skip before_deploy: make dist - #deploy: - # skip_cleanup: true - # provider: s3 - # access_key_id: $AWS_ACCESS_KEY_ID - # secret_access_key: $AWS_SECRET_ACCESS_KEY - # bucket: "carbon-deploy" - # local_dir: "dist-aws" - #after_deploy: bash publish.sh + deploy: + skip_cleanup: true + provider: script + script: make publish diff --git a/Makefile b/Makefile index da219ad..a2173c3 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ SHELL=/bin/bash S3_BUCKET=carbon-deploy LIBAIO_SO=libaio.so.1.0.1 ORACLE_ZIP=instantclient-basiclite-linux.x64-18.3.0.0.0dbru.zip +ECR_REGISTRY=672626379771.dkr.ecr.us-east-1.amazonaws.com install: pipenv install @@ -16,13 +17,15 @@ wheel: pipenv run python setup.py bdist_wheel container: - docker build -t carbon:`git describe --always` -t carbon:latest . + docker build -t $(ECR_REGISTRY)/carbon:latest \ + -t $(ECR_REGISTRY)/carbon:`git describe --always` \ + -t carbon:latest . dist: deps wheel container @tput setaf 2 @tput bold @echo "Finished building docker image. Try running:" - @echo " $$ docker run --rm carbon:`git describe --always`" + @echo " $$ docker run --rm carbon:latest" @tput sgr0 clean: @@ -41,3 +44,8 @@ tests: test update: pipenv clean pipenv update --dev + +publish: + $(aws ecr get-login --no-include-email --region us-east-1) + docker push $(ECR_REGISTRY)/carbon:latest + docker push $(ECR_REGISTRY)/carbon:`git describe --always` diff --git a/README.rst b/README.rst index 41d8afe..cc11e0e 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ carbon ====== -Carbon is a tool for generating a feed of people that can be loaded into Symplectic Elements. It is run as a Lambda by a scheduled CloudWatch event. +Carbon is a tool for generating a feed of people that can be loaded into Symplectic Elements. It is designed to be run as a container. Developing ---------- @@ -25,27 +25,25 @@ If you do need to connect to the data warehouse, you will also need to install t On Linux, you will also need to make sure you have libaio installed. You can probably just use your system's package manager to install this easily. The package may be called ``libaio1``. -The Lambda Package ------------------- +Building +-------- -The Lambda package is built with the ``build.sh`` script. +Running ``make dist`` creates a new container that is tagged as ``carbon:latest``. It will also add tags for the ECR registry with both the ``latest`` and the git short hash. The first build will take some time, but subsequent builds should be fast. -**IMPORTANT**: Both the ``cx_Oracle`` and ``lxml`` packages use platform dependent wheels, so you if you are planning on using the Lambda package you *must* do this step on a Linux x86_64 architecture. +We are restricted from distributing the Oracle client library, so a copy is kept in a private S3 bucket for use when building the container image. If you are updating this, make sure you are using a Linux x86_64 version. -We are restricted from distributing the Oracle client library, so a copy is kept in a private S3 bucket for use when building the Lambda. The Lambda execution context does not provide libaio, so a copy of this is also kept in the bucket. If you are updating either of these, make sure you are using a Linux x86_64 version. To be on the safe side, use https://github.com/lambci/docker-lambda to get the libaio library as that will ensure you are getting one that's been linked against an appropriate version of glibc. - -The build process downloads each of these files from S3 so you should have the AWS CLI installed and configured to authenticate using an account with appropriate access. +The build process downloads this file from S3 so you should have the AWS CLI installed and configured to authenticate using an account with appropriate access. Deploying --------- -Deployment is currently being handled by Travis. When a PR is merged onto the master branch Travis will build a new Lambda package, push it to S3 and update the Lambda function to point to the new package. +Deployment is currently being handled by Travis. When a PR is merged onto the master branch Travis will build a new container image, tag it both with ``latest`` and with the git short hash, and then push both tags to the ECR registry. -If you need to deploy a new package outside of Travis then do the following, *noting the restrictions on running build.sh described above*:: +If you need to deploy a new image outside of Travis then do the following:: $ cd carbon - $ ./build.sh - $ ./publish.sh --upload + $ make clean + $ make dist && make publish Configuration ^^^^^^^^^^^^^