From d0f518ca62b0ddb87d0f0754fe1394051629f1fa Mon Sep 17 00:00:00 2001 From: Mike Graves Date: Mon, 21 Oct 2019 15:50:06 -0400 Subject: [PATCH] Add custom plugin for running containerized tasks This adds a custom Airflow plugin that includes an operator and sensor that can be used together to run completely isolated workflows. There's an example workflow included in the root of this repo to show how it works. --- Dockerfile | 4 + Pipfile | 4 +- Pipfile.lock | 420 ++++++++++++++++++++++------------------ README.rst | 23 ++- example_ecs.py | 30 +++ mit/__init__.py | 10 + mit/operators.py | 34 ++++ mit/sensors.py | 32 +++ tests/test_operators.py | 27 +++ tox.ini | 6 +- workflows/log_config.py | 224 +++++++++++++++++++++ 11 files changed, 616 insertions(+), 198 deletions(-) create mode 100644 example_ecs.py create mode 100644 mit/__init__.py create mode 100644 mit/operators.py create mode 100644 mit/sensors.py create mode 100644 tests/test_operators.py create mode 100644 workflows/log_config.py diff --git a/Dockerfile b/Dockerfile index 668f18f..02125df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,12 @@ COPY Pipfile* / RUN pipenv install --system --ignore-pipfile --deploy COPY entrypoint.sh / RUN mkdir -p /airflow/dags +RUN mkdir -p /airflow/plugins COPY workflows/* /airflow/dags/ +COPY mit /airflow/plugins/mit +RUN chown -R airflow:airflow /airflow USER airflow +ENV AIRFLOW_HOME /airflow ENTRYPOINT ["/entrypoint.sh"] CMD ["--help"] diff --git a/Pipfile b/Pipfile index c8d2d7c..9b7e48d 100644 --- a/Pipfile +++ b/Pipfile @@ -11,12 +11,12 @@ coveralls = "*" [packages] apache-airflow = {extras = ["celery", "crypto", "password", "postgres"],version = "~=1.10.0"} -psycopg2 = "*" click = "*" -boto3 = "*" redis = "*" jmespath = "*" colorama = "*" +psycopg2-binary = "*" +boto3 = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index c89022b..32a6bb4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8a9e30889c46bc8b2d2b510c0461e7d8a5f0917688544b9b69914b134e2b8c3c" + "sha256": "bdd619b18d2b13b57ff15352fcda442d97668ac151f4cccd248e61504fb5c726" }, "pipfile-spec": 6, "requires": { @@ -53,19 +53,12 @@ ], "version": "==3.0.0" }, - "asn1crypto": { - "hashes": [ - "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292", - "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f" - ], - "version": "==1.0.1" - }, "attrs": { "hashes": [ - "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", - "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.2.0" + "version": "==19.3.0" }, "babel": { "hashes": [ @@ -80,6 +73,7 @@ "sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42", "sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294", "sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161", + "sha256:6305557019906466fc42dbc53b46da004e72fd7a551c044a827e572c82191752", "sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31", "sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5", "sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c", @@ -90,6 +84,7 @@ "sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09", "sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105", "sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133", + "sha256:ce4e4f0deb51d38b1611a27f330426154f2980e66582dc5f438aad38b5f24fc1", "sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7", "sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc" ], @@ -104,18 +99,18 @@ }, "boto3": { "hashes": [ - "sha256:9b18eeb4f0943af80fc40cd480931e4900f7fb3850dfafe68903148fdd832e6b", - "sha256:a985ed608640f8f7be49f83f298172876b695cc2218e335334a9da3bf6b0a968" + "sha256:839285fbd6f3ab16170af449ae9e33d0eccf97ca22de17d9ff68b8da2310ea06", + "sha256:d93f1774c4bc66e02acdda2067291acb9e228a035435753cb75f83ad2904cbe3" ], "index": "pypi", - "version": "==1.9.245" + "version": "==1.9.253" }, "botocore": { "hashes": [ - "sha256:16a09307cef306312d4c3ea18ed3902ae1e084c905bda091db2689e9852753ef", - "sha256:b21b694a6bccbe12b64d3e452d081016b67172c92d0b1be2904f60cd4dd5598d" + "sha256:3baf129118575602ada9926f5166d82d02273c250d0feb313fc270944b27c48b", + "sha256:dc080aed4f9b220a9e916ca29ca97a9d37e8e1d296fe89cbaeef929bf0c8066b" ], - "version": "==1.12.245" + "version": "==1.12.253" }, "cached-property": { "hashes": [ @@ -140,36 +135,40 @@ }, "cffi": { "hashes": [ - "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", - "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", - "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", - "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", - "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", - "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", - "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", - "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", - "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", - "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", - "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", - "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", - "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", - "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", - "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", - "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", - "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", - "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", - "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", - "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", - "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", - "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", - "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", - "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", - "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", - "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", - "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", - "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" - ], - "version": "==1.12.3" + "sha256:08f99e8b38d5134d504aa7e486af8e4fde66a2f388bbecc270cdd1e00fa09ff8", + "sha256:1112d2fc92a867a6103bce6740a549e74b1d320cf28875609f6e93857eee4f2d", + "sha256:1b9ab50c74e075bd2ae489853c5f7f592160b379df53b7f72befcbe145475a36", + "sha256:24eff2997436b6156c2f30bed215c782b1d8fd8c6a704206053c79af95962e45", + "sha256:2eff642fbc9877a6449026ad66bf37c73bf4232505fb557168ba5c502f95999b", + "sha256:362e896cea1249ed5c2a81cf6477fabd9e1a5088aa7ea08358a4c6b0998294d2", + "sha256:40eddb3589f382cb950f2dcf1c39c9b8d7bd5af20665ce273815b0d24635008b", + "sha256:5ed40760976f6b8613d4a0db5e423673ca162d4ed6c9ed92d1f4e58a47ee01b5", + "sha256:632c6112c1e914c486f06cfe3f0cc507f44aa1e00ebf732cedb5719e6aa0466a", + "sha256:64d84f0145e181f4e6cc942088603c8db3ae23485c37eeda71cb3900b5e67cb4", + "sha256:6cb4edcf87d0e7f5bdc7e5c1a0756fbb37081b2181293c5fdf203347df1cd2a2", + "sha256:6f19c9df4785305669335b934c852133faed913c0faa63056248168966f7a7d5", + "sha256:719537b4c5cd5218f0f47826dd705fb7a21d83824920088c4214794457113f3f", + "sha256:7b0e337a70e58f1a36fb483fd63880c9e74f1db5c532b4082bceac83df1523fa", + "sha256:853376efeeb8a4ae49a737d5d30f5db8cdf01d9319695719c4af126488df5a6a", + "sha256:85bbf77ffd12985d76a69d2feb449e35ecdcb4fc54a5f087d2bd54158ae5bb0c", + "sha256:8978115c6f0b0ce5880bc21c967c65058be8a15f1b81aa5fdbdcbea0e03952d1", + "sha256:8f7eec920bc83692231d7306b3e311586c2e340db2dc734c43c37fbf9c981d24", + "sha256:8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226", + "sha256:92068ebc494b5f9826b822cec6569f1f47b9a446a3fef477e1d11d7fac9ea895", + "sha256:b57e1c8bcdd7340e9c9d09613b5e7fdd0c600be142f04e2cc1cc8cb7c0b43529", + "sha256:ba956c9b44646bc1852db715b4a252e52a8f5a4009b57f1dac48ba3203a7bde1", + "sha256:ca42034c11eb447497ea0e7b855d87ccc2aebc1e253c22e7d276b8599c112a27", + "sha256:dc9b2003e9a62bbe0c84a04c61b0329e86fccd85134a78d7aca373bbbf788165", + "sha256:dd308802beb4b2961af8f037becbdf01a1e85009fdfc14088614c1b3c383fae5", + "sha256:e77cd105b19b8cd721d101687fcf665fd1553eb7b57556a1ef0d453b6fc42faa", + "sha256:f56dff1bd81022f1c980754ec721fb8da56192b026f17f0f99b965da5ab4fbd2", + "sha256:fa4cc13c03ea1d0d37ce8528e0ecc988d2365e8ac64d8d86cafab4038cb4ce89", + "sha256:fa8cf1cb974a9f5911d2a0303f6adc40625c05578d8e7ff5d313e1e27850bd59", + "sha256:fb003019f06d5fc0aa4738492ad8df1fa343b8a37cbcf634018ad78575d185df", + "sha256:fd409b7778167c3bcc836484a8f49c0e0b93d3e745d975749f83aa5d18a5822f", + "sha256:fe5d65a3ee38122003245a82303d11ac05ff36531a8f5ce4bc7d4bbc012797e1" + ], + "version": "==1.13.0" }, "chardet": { "hashes": [ @@ -218,24 +217,29 @@ }, "cryptography": { "hashes": [ - "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c", - "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643", - "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216", - "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799", - "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a", - "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9", - "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc", - "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8", - "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53", - "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1", - "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609", - "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292", - "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e", - "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6", - "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed", - "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d" - ], - "version": "==2.7" + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" }, "defusedxml": { "hashes": [ @@ -411,13 +415,6 @@ "index": "pypi", "version": "==0.9.4" }, - "js-regex": { - "hashes": [ - "sha256:90bbc2d2b76c2d9b07058e59e089fb175c7eccb8f82f55bea5bdb3515c03ed8d", - "sha256:aee930ccbca4a254e4547c64e5882c71834ba5210ec397b50627c5662b1bf18d" - ], - "version": "==1.0.0" - }, "json-merge-patch": { "hashes": [ "sha256:09898b6d427c08754e2a97c709cf2dfd7e28bd10c5683a538914975eab778d39" @@ -426,10 +423,10 @@ }, "jsonschema": { "hashes": [ - "sha256:4f4ddc3d51f33a5363c042dc62c85010e9e3b8353bcf108afff394dde70854b3", - "sha256:edcdc3030424f3ebbcd95a9ea8310320ccbe655a3a693296569258bfa0707b37" + "sha256:2fa0684276b6333ff3c0b1b27081f4b2305f0a36cf702a23db50edb141893c3f", + "sha256:94c0a13b4a0616458b42529091624e66700a17f847453e52279e35509a5b7631" ], - "version": "==3.1.0" + "version": "==3.1.1" }, "kombu": { "hashes": [ @@ -544,24 +541,29 @@ }, "numpy": { "hashes": [ - "sha256:05dbfe72684cc14b92568de1bc1f41e5f62b00f714afc9adee42f6311738091f", - "sha256:0d82cb7271a577529d07bbb05cb58675f2deb09772175fab96dc8de025d8ac05", - "sha256:10132aa1fef99adc85a905d82e8497a580f83739837d7cbd234649f2e9b9dc58", - "sha256:12322df2e21f033a60c80319c25011194cd2a21294cc66fee0908aeae2c27832", - "sha256:16f19b3aa775dddc9814e02a46b8e6ae6a54ed8cf143962b4e53f0471dbd7b16", - "sha256:3d0b0989dd2d066db006158de7220802899a1e5c8cf622abe2d0bd158fd01c2c", - "sha256:438a3f0e7b681642898fd7993d38e2bf140a2d1eafaf3e89bb626db7f50db355", - "sha256:5fd214f482ab53f2cea57414c5fb3e58895b17df6e6f5bca5be6a0bb6aea23bb", - "sha256:73615d3edc84dd7c4aeb212fa3748fb83217e00d201875a47327f55363cef2df", - "sha256:7bd355ad7496f4ce1d235e9814ec81ee3d28308d591c067ce92e49f745ba2c2f", - "sha256:7d077f2976b8f3de08a0dcf5d72083f4af5411e8fddacd662aae27baa2601196", - "sha256:a4092682778dc48093e8bda8d26ee8360153e2047826f95a3f5eae09f0ae3abf", - "sha256:b458de8624c9f6034af492372eb2fee41a8e605f03f4732f43fc099e227858b2", - "sha256:e70fc8ff03a961f13363c2c95ef8285e0cf6a720f8271836f852cc0fa64e97c8", - "sha256:ee8e9d7cad5fe6dde50ede0d2e978d81eafeaa6233fb0b8719f60214cf226578", - "sha256:f4a4f6aba148858a5a5d546a99280f71f5ee6ec8182a7d195af1a914195b21a2" - ], - "version": "==1.17.2" + "sha256:0b0dd8f47fb177d00fa6ef2d58783c4f41ad3126b139c91dd2f7c4b3fdf5e9a5", + "sha256:25ffe71f96878e1da7e014467e19e7db90ae7d4e12affbc73101bcf61785214e", + "sha256:26efd7f7d755e6ca966a5c0ac5a930a87dbbaab1c51716ac26a38f42ecc9bc4b", + "sha256:28b1180c758abf34a5c3fea76fcee66a87def1656724c42bb14a6f9717a5bdf7", + "sha256:2e418f0a59473dac424f888dd57e85f77502a593b207809211c76e5396ae4f5c", + "sha256:30c84e3a62cfcb9e3066f25226e131451312a044f1fe2040e69ce792cb7de418", + "sha256:4650d94bb9c947151737ee022b934b7d9a845a7c76e476f3e460f09a0c8c6f39", + "sha256:4dd830a11e8724c9c9379feed1d1be43113f8bcce55f47ea7186d3946769ce26", + "sha256:4f2a2b279efde194877aff1f76cf61c68e840db242a5c7169f1ff0fd59a2b1e2", + "sha256:62d22566b3e3428dfc9ec972014c38ed9a4db4f8969c78f5414012ccd80a149e", + "sha256:669795516d62f38845c7033679c648903200980d68935baaa17ac5c7ae03ae0c", + "sha256:75fcd60d682db3e1f8fbe2b8b0c6761937ad56d01c1dc73edf4ef2748d5b6bc4", + "sha256:9395b0a41e8b7e9a284e3be7060db9d14ad80273841c952c83a5afc241d2bd98", + "sha256:9e37c35fc4e9410093b04a77d11a34c64bf658565e30df7cbe882056088a91c1", + "sha256:a0678793096205a4d784bd99f32803ba8100f639cf3b932dc63b21621390ea7e", + "sha256:b46554ad4dafb2927f88de5a1d207398c5385edbb5c84d30b3ef187c4a3894d8", + "sha256:c867eeccd934920a800f65c6068acdd6b87e80d45cd8c8beefff783b23cdc462", + "sha256:dd0667f5be56fb1b570154c2c0516a528e02d50da121bbbb2cbb0b6f87f59bc2", + "sha256:de2b1c20494bdf47f0160bd88ed05f5e48ae5dc336b8de7cfade71abcc95c0b9", + "sha256:f1df7b2b7740dd777571c732f98adb5aad5450aee32772f1b39249c8a50386f6", + "sha256:ffca69e29079f7880c5392bf675eb8b4146479d976ae1924d01cd92b04cccbcc" + ], + "version": "==1.17.3" }, "ordereddict": { "hashes": [ @@ -571,23 +573,27 @@ }, "pandas": { "hashes": [ - "sha256:18d91a9199d1dfaa01ad645f7540370ba630bdcef09daaf9edf45b4b1bca0232", - "sha256:3f26e5da310a0c0b83ea50da1fd397de2640b02b424aa69be7e0784228f656c9", - "sha256:4182e32f4456d2c64619e97c58571fa5ca0993d1e8c2d9ca44916185e1726e15", - "sha256:426e590e2eb0e60f765271d668a30cf38b582eaae5ec9b31229c8c3c10c5bc21", - "sha256:5eb934a8f0dc358f0e0cdf314072286bbac74e4c124b64371395e94644d5d919", - "sha256:717928808043d3ea55b9bcde636d4a52d2236c246f6df464163a66ff59980ad8", - "sha256:8145f97c5ed71827a6ec98ceaef35afed1377e2d19c4078f324d209ff253ecb5", - "sha256:8744c84c914dcc59cbbb2943b32b7664df1039d99e834e1034a3372acb89ea4d", - "sha256:c1ac1d9590d0c9314ebf01591bd40d4c03d710bfc84a3889e5263c97d7891dee", - "sha256:cb2e197b7b0687becb026b84d3c242482f20cbb29a9981e43604eb67576da9f6", - "sha256:d4001b71ad2c9b84ff18b182cea22b7b6cbf624216da3ea06fb7af28d1f93165", - "sha256:d8930772adccb2882989ab1493fa74bd87d47c8ac7417f5dd3dd834ba8c24dc9", - "sha256:dfbb0173ee2399bc4ed3caf2d236e5c0092f948aafd0a15fbe4a0e77ee61a958", - "sha256:eebfbba048f4fa8ac711b22c78516e16ff8117d05a580e7eeef6b0c2be554c18", - "sha256:f1b21bc5cf3dbea53d33615d1ead892dfdae9d7052fa8898083bec88be20dcd2" - ], - "version": "==0.25.1" + "sha256:0f484f43658a72e7d586a74978259657839b5bd31b903e963bb1b1491ab51775", + "sha256:0ffc6f9e20e77f3a7dc8baaad9c7fd25b858b084d3a2d8ce877bc3ea804e0636", + "sha256:23e0eac646419c3057f15eb96ab821964848607bf1d4ea5a82f26565986ec5e9", + "sha256:27c0603b15b5c6fa24885253bbe49a0c289381e7759385c59308ba4f0b166cf1", + "sha256:397fe360643fffc5b26b41efdf608647e3334a618d185a07976cd2dc51c90bce", + "sha256:3dbb3aa41c01504255bff2bd56944bdb915f6c9ce4bac7e2868efbace0b2a639", + "sha256:4e07c63247c59d61c6ebdbbb50196143cec6c5044403510c4e1a9d31854a83d6", + "sha256:4fa6d9235c6d2fecbeca82c3d326abd255866cafbfd37f66a0e826544e619760", + "sha256:56cb88b3876363d410a9d7724f43641ff164e2c9902d3266a648213e2efd5e6d", + "sha256:7ce1be1614455f83710b9a5dc1fc602a755bdddbe4dda1d41515062923a37bbf", + "sha256:ae1c96ffdeec376895e533107e0b0f9da16225a2184fbb24a5abc866769db75e", + "sha256:b6f27c9231be8a23de846f2302373991467dd8e397a4804d2614e8c5aa8d5a90", + "sha256:c6056067f894f9355bedcd168dd740aa849908d41c0a74756f6e38f203e941b3", + "sha256:ca91a19d1f0a280874a24dca44aadce42da7f3a7edb7e9ab7c7baad8febee2be", + "sha256:cbe4985f5c82a173f8cff6b7fe92d551addf99fb4ea9ff4eb4b1fe606bb098ec", + "sha256:e3e9893bfe80a8b8e6d56d36ebb7afe1df77db7b4068a6e2ef3636a91f6f1caa", + "sha256:e7b218e8711910dac3fed0d19376cd1ef0e386be5175965d332fd0c65d02a43b", + "sha256:ec48d18b8b63a5dbb838e8ea7892ee1034299e03f852bd9b6dffe870310414dd", + "sha256:f4ab6280277e3208a59bfa9f2e51240304d09e69ffb65abfb4a21d678b495f74" + ], + "version": "==0.25.2" }, "pendulum": { "hashes": [ @@ -655,9 +661,44 @@ "sha256:f36b333e9f86a2fba960c72b90c34be6ca71819e300f7b1fc3d2b0f0b2c546cd", "sha256:f4526d078aedd5187d0508aa5f9a01eae6a48a470ed678406da94b4cd6524b7e" ], - "index": "pypi", "version": "==2.7.7" }, + "psycopg2-binary": { + "hashes": [ + "sha256:040234f8a4a8dfd692662a8308d78f63f31a97e1c42d2480e5e6810c48966a29", + "sha256:086f7e89ec85a6704db51f68f0dcae432eff9300809723a6e8782c41c2f48e03", + "sha256:18ca813fdb17bc1db73fe61b196b05dd1ca2165b884dd5ec5568877cabf9b039", + "sha256:19dc39616850342a2a6db70559af55b22955f86667b5f652f40c0e99253d9881", + "sha256:2166e770cb98f02ed5ee2b0b569d40db26788e0bf2ec3ae1a0d864ea6f1d8309", + "sha256:3a2522b1d9178575acee4adf8fd9f979f9c0449b00b4164bb63c3475ea6528ed", + "sha256:3aa773580f85a28ffdf6f862e59cb5a3cc7ef6885121f2de3fca8d6ada4dbf3b", + "sha256:3b5deaa3ee7180585a296af33e14c9b18c218d148e735c7accf78130765a47e3", + "sha256:407af6d7e46593415f216c7f56ba087a9a42bd6dc2ecb86028760aa45b802bd7", + "sha256:4c3c09fb674401f630626310bcaf6cd6285daf0d5e4c26d6e55ca26a2734e39b", + "sha256:4c6717962247445b4f9e21c962ea61d2e884fc17df5ddf5e35863b016f8a1f03", + "sha256:50446fae5681fc99f87e505d4e77c9407e683ab60c555ec302f9ac9bffa61103", + "sha256:5057669b6a66aa9ca118a2a860159f0ee3acf837eda937bdd2a64f3431361a2d", + "sha256:5dd90c5438b4f935c9d01fcbad3620253da89d19c1f5fca9158646407ed7df35", + "sha256:659c815b5b8e2a55193ede2795c1e2349b8011497310bb936da7d4745652823b", + "sha256:69b13fdf12878b10dc6003acc8d0abf3ad93e79813fd5f3812497c1c9fb9be49", + "sha256:7a1cb80e35e1ccea3e11a48afe65d38744a0e0bde88795cc56a4d05b6e4f9d70", + "sha256:7e6e3c52e6732c219c07bd97fff6c088f8df4dae3b79752ee3a817e6f32e177e", + "sha256:7f42a8490c4fe854325504ce7a6e4796b207960dabb2cbafe3c3959cb00d1d7e", + "sha256:84156313f258eafff716b2961644a4483a9be44a5d43551d554844d15d4d224e", + "sha256:8578d6b8192e4c805e85f187bc530d0f52ba86c39172e61cd51f68fddd648103", + "sha256:890167d5091279a27e2505ff0e1fb273f8c48c41d35c5b92adbf4af80e6b2ed6", + "sha256:9aadff9032e967865f9778485571e93908d27dab21d0fdfdec0ca779bb6f8ad9", + "sha256:9f24f383a298a0c0f9b3113b982e21751a8ecde6615494a3f1470eb4a9d70e9e", + "sha256:a73021b44813b5c84eda4a3af5826dd72356a900bac9bd9dd1f0f81ee1c22c2f", + "sha256:afd96845e12638d2c44d213d4810a08f4dc4a563f9a98204b7428e567014b1cd", + "sha256:b73ddf033d8cd4cc9dfed6324b1ad2a89ba52c410ef6877998422fcb9c23e3a8", + "sha256:dbc5cd56fff1a6152ca59445178652756f4e509f672e49ccdf3d79c1043113a4", + "sha256:eac8a3499754790187bb00574ab980df13e754777d346f85e0ff6df929bcd964", + "sha256:eaed1c65f461a959284649e37b5051224f4db6ebdc84e40b5e65f2986f101a08" + ], + "index": "pypi", + "version": "==2.8.4" + }, "pycparser": { "hashes": [ "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" @@ -748,11 +789,11 @@ }, "redis": { "hashes": [ - "sha256:98a22fb750c9b9bb46e75e945dc3f61d0ab30d06117cbb21ff9cd1d315fedd3b", - "sha256:c504251769031b0dd7dd5cf786050a6050197c6de0d37778c80c08cb04ae8275" + "sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62", + "sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2" ], "index": "pypi", - "version": "==3.3.8" + "version": "==3.3.11" }, "requests": { "hashes": [ @@ -784,9 +825,9 @@ }, "sqlalchemy": { "hashes": [ - "sha256:272a835758908412e75e87f75dd0179a51422715c125ce42109632910526b1fd" + "sha256:0f0768b5db594517e1f5e1572c73d14cf295140756431270d89496dc13d5e46c" ], - "version": "==1.3.9" + "version": "==1.3.10" }, "tabulate": { "hashes": [ @@ -889,13 +930,6 @@ } }, "develop": { - "asn1crypto": { - "hashes": [ - "sha256:0b199f211ae690df3db4fd6c1c4ff976497fb1da689193e368eedbadc53d9292", - "sha256:bca90060bd995c3f62c4433168eab407e44bdbdb567b3f3a396a676c1a4c4a3f" - ], - "version": "==1.0.1" - }, "atomicwrites": { "hashes": [ "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", @@ -905,16 +939,16 @@ }, "attrs": { "hashes": [ - "sha256:ec20e7a4825331c1b5ebf261d111e16fa9612c1f7a5e1f884f12bd53a664dfd2", - "sha256:f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==19.2.0" + "version": "==19.3.0" }, "aws-sam-translator": { "hashes": [ - "sha256:6563aa3b534e7ad672d580ecd3dfa92021e81b4e5983604c0df7ee0a07b3ed99" + "sha256:11c62c00f37b57c39a55d7a29d93f4704a88549c29a6448ebc953147173fbe85" ], - "version": "==1.15.0" + "version": "==1.15.1" }, "aws-xray-sdk": { "hashes": [ @@ -932,18 +966,18 @@ }, "boto3": { "hashes": [ - "sha256:9b18eeb4f0943af80fc40cd480931e4900f7fb3850dfafe68903148fdd832e6b", - "sha256:a985ed608640f8f7be49f83f298172876b695cc2218e335334a9da3bf6b0a968" + "sha256:839285fbd6f3ab16170af449ae9e33d0eccf97ca22de17d9ff68b8da2310ea06", + "sha256:d93f1774c4bc66e02acdda2067291acb9e228a035435753cb75f83ad2904cbe3" ], "index": "pypi", - "version": "==1.9.245" + "version": "==1.9.253" }, "botocore": { "hashes": [ - "sha256:16a09307cef306312d4c3ea18ed3902ae1e084c905bda091db2689e9852753ef", - "sha256:b21b694a6bccbe12b64d3e452d081016b67172c92d0b1be2904f60cd4dd5598d" + "sha256:3baf129118575602ada9926f5166d82d02273c250d0feb313fc270944b27c48b", + "sha256:dc080aed4f9b220a9e916ca29ca97a9d37e8e1d296fe89cbaeef929bf0c8066b" ], - "version": "==1.12.245" + "version": "==1.12.253" }, "certifi": { "hashes": [ @@ -954,36 +988,40 @@ }, "cffi": { "hashes": [ - "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", - "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", - "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", - "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", - "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", - "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", - "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", - "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", - "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", - "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", - "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", - "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", - "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", - "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", - "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", - "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", - "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", - "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", - "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", - "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", - "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", - "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", - "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", - "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", - "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", - "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", - "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", - "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" - ], - "version": "==1.12.3" + "sha256:08f99e8b38d5134d504aa7e486af8e4fde66a2f388bbecc270cdd1e00fa09ff8", + "sha256:1112d2fc92a867a6103bce6740a549e74b1d320cf28875609f6e93857eee4f2d", + "sha256:1b9ab50c74e075bd2ae489853c5f7f592160b379df53b7f72befcbe145475a36", + "sha256:24eff2997436b6156c2f30bed215c782b1d8fd8c6a704206053c79af95962e45", + "sha256:2eff642fbc9877a6449026ad66bf37c73bf4232505fb557168ba5c502f95999b", + "sha256:362e896cea1249ed5c2a81cf6477fabd9e1a5088aa7ea08358a4c6b0998294d2", + "sha256:40eddb3589f382cb950f2dcf1c39c9b8d7bd5af20665ce273815b0d24635008b", + "sha256:5ed40760976f6b8613d4a0db5e423673ca162d4ed6c9ed92d1f4e58a47ee01b5", + "sha256:632c6112c1e914c486f06cfe3f0cc507f44aa1e00ebf732cedb5719e6aa0466a", + "sha256:64d84f0145e181f4e6cc942088603c8db3ae23485c37eeda71cb3900b5e67cb4", + "sha256:6cb4edcf87d0e7f5bdc7e5c1a0756fbb37081b2181293c5fdf203347df1cd2a2", + "sha256:6f19c9df4785305669335b934c852133faed913c0faa63056248168966f7a7d5", + "sha256:719537b4c5cd5218f0f47826dd705fb7a21d83824920088c4214794457113f3f", + "sha256:7b0e337a70e58f1a36fb483fd63880c9e74f1db5c532b4082bceac83df1523fa", + "sha256:853376efeeb8a4ae49a737d5d30f5db8cdf01d9319695719c4af126488df5a6a", + "sha256:85bbf77ffd12985d76a69d2feb449e35ecdcb4fc54a5f087d2bd54158ae5bb0c", + "sha256:8978115c6f0b0ce5880bc21c967c65058be8a15f1b81aa5fdbdcbea0e03952d1", + "sha256:8f7eec920bc83692231d7306b3e311586c2e340db2dc734c43c37fbf9c981d24", + "sha256:8fe230f612c18af1df6f348d02d682fe2c28ca0a6c3856c99599cdacae7cf226", + "sha256:92068ebc494b5f9826b822cec6569f1f47b9a446a3fef477e1d11d7fac9ea895", + "sha256:b57e1c8bcdd7340e9c9d09613b5e7fdd0c600be142f04e2cc1cc8cb7c0b43529", + "sha256:ba956c9b44646bc1852db715b4a252e52a8f5a4009b57f1dac48ba3203a7bde1", + "sha256:ca42034c11eb447497ea0e7b855d87ccc2aebc1e253c22e7d276b8599c112a27", + "sha256:dc9b2003e9a62bbe0c84a04c61b0329e86fccd85134a78d7aca373bbbf788165", + "sha256:dd308802beb4b2961af8f037becbdf01a1e85009fdfc14088614c1b3c383fae5", + "sha256:e77cd105b19b8cd721d101687fcf665fd1553eb7b57556a1ef0d453b6fc42faa", + "sha256:f56dff1bd81022f1c980754ec721fb8da56192b026f17f0f99b965da5ab4fbd2", + "sha256:fa4cc13c03ea1d0d37ce8528e0ecc988d2365e8ac64d8d86cafab4038cb4ce89", + "sha256:fa8cf1cb974a9f5911d2a0303f6adc40625c05578d8e7ff5d313e1e27850bd59", + "sha256:fb003019f06d5fc0aa4738492ad8df1fa343b8a37cbcf634018ad78575d185df", + "sha256:fd409b7778167c3bcc836484a8f49c0e0b93d3e745d975749f83aa5d18a5822f", + "sha256:fe5d65a3ee38122003245a82303d11ac05ff36531a8f5ce4bc7d4bbc012797e1" + ], + "version": "==1.13.0" }, "cfn-lint": { "hashes": [ @@ -1046,24 +1084,29 @@ }, "cryptography": { "hashes": [ - "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c", - "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643", - "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216", - "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799", - "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a", - "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9", - "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc", - "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8", - "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53", - "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1", - "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609", - "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292", - "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e", - "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6", - "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed", - "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d" - ], - "version": "==2.7" + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" }, "datetime": { "hashes": [ @@ -1135,13 +1178,6 @@ "index": "pypi", "version": "==0.9.4" }, - "js-regex": { - "hashes": [ - "sha256:90bbc2d2b76c2d9b07058e59e089fb175c7eccb8f82f55bea5bdb3515c03ed8d", - "sha256:aee930ccbca4a254e4547c64e5882c71834ba5210ec397b50627c5662b1bf18d" - ], - "version": "==1.0.0" - }, "jsondiff": { "hashes": [ "sha256:7e18138aecaa4a8f3b7ac7525b8466234e6378dd6cae702b982c9ed851d2ae21" @@ -1171,10 +1207,10 @@ }, "jsonschema": { "hashes": [ - "sha256:4f4ddc3d51f33a5363c042dc62c85010e9e3b8353bcf108afff394dde70854b3", - "sha256:edcdc3030424f3ebbcd95a9ea8310320ccbe655a3a693296569258bfa0707b37" + "sha256:2fa0684276b6333ff3c0b1b27081f4b2305f0a36cf702a23db50edb141893c3f", + "sha256:94c0a13b4a0616458b42529091624e66700a17f847453e52279e35509a5b7631" ], - "version": "==3.1.0" + "version": "==3.1.1" }, "markupsafe": { "hashes": [ diff --git a/README.rst b/README.rst index 88f2b5d..4b85057 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,6 @@ Workflow This is the MIT Libraries' `Airflow `_ implementation. **Please read these instructions carefully, because this is very much a shared codebase.** .. contents:: Table of Contents -.. section-numbering:: Intro ----- @@ -40,6 +39,28 @@ Workflow Tips and Things to Remember - It's better to have several small tasks than a single large task for your workflow. The Airflow worker nodes will need to be restarted from time to time for different reasons. When a node is restarted, it's given about 2 minutes for any tasks currently running to finish at which point it is forceably killed. - Airflow schedules tasks to be run, but there are no hard guarantees about when a scheduled task will actually be run as it depends on there being workers available to run it. +Running Containerized Workflows +------------------------------- + +We are experimenting with a somewhat different execution model for workflows. With this model, instead of your task being done in the context of Airflow it's done in its own container. There are several really good reasons to consider doing things this way: + +1. Your task does not share the same dependencies as all the other tasks or Airflow itself. It is running in an isolated container that has nothing to do with Airflow. +2. Your task can be written in whatever language you want. You bring your own container and can put whatever you want on it. +3. If your long-running task is busy doing work when the cluster goes down for a redeploy it will keep going and your workflow will pick up where it left off when the cluster restarts. + +If you are interested in using this, read on. There are two custom operators you will be using to build your workflow. The workflow (DAG) will still be the same Airflow workflow that you are used to, it's just that you will only use these two operators. + +The first is the ``airflow.operators.mit.ECSOperator``. Note that this is different from the one included in Airflow. This is the operator that will be used to start your containerized task. It will start the task and then immediately exit. The next step in your workflow will be an ``airflow.sensors.mit.ECSTaskSensor``. As soon as the previous ``ECSOperator`` has exited this sensor will periodically monitor the container. If the container failed to start for some reason, or exited with an error, the sensor step will fail. Otherwise, the sensor step will succeed and your workflow can continue. You can use as many of these as your want in a workflow. You'll just chain them together so that it will look something like this:: + + ECSOperator --> ECSTaskSensor --> ECSOperator --> ECSTaskSensor ... + +The ``ECSOperator`` needs a few arguments that have to be provided at runtime from the environment. The ``cluster`` can be retrieved from the ``ECS_CLUSTER_NAME`` envvar. The ``network_configuration`` can be retrieved from the ``ECS_NETWORK_CONFIG``, but note that this is a base64 encoded JSON string (sorry, this is the only way to get this from Terraform into the environment). You can use:: + + json.loads(base64.b64decode(os.getenv('ECS_NETWORK_CONFIG'))) + +to retrieve the network configuration. The ``task_definition`` will need to be manually configured after the task has been created in Terraform. There is an example workflow (``example_ecs.py``) in the root of this repo. + + Developing Locally ------------------ diff --git a/example_ecs.py b/example_ecs.py new file mode 100644 index 0000000..108c2e3 --- /dev/null +++ b/example_ecs.py @@ -0,0 +1,30 @@ +import base64 +from datetime import datetime +import json +import os + +from airflow import DAG +from airflow.operators.mit import ECSOperator +from airflow.sensors.mit import ECSTaskSensor + + +network_config = json.loads(base64.b64decode(os.getenv('ECS_NETWORK_CONFIG'))) +cluster = os.getenv('ECS_CLUSTER') + +dag = DAG('example', + description='An example for running containerized workflows.', + start_date=datetime.now()) + +task1 = ECSOperator(task_id='ex_step_1', + dag=dag, + cluster=cluster, + task_definition='example_task_definition', + overrides={}, + network_configuration=network_config) + +task2 = ECSTaskSensor(task_id='ex_step_2', + dag=dag, + cluster=cluster, + ecs_task_id='ex_step_1') + +task1 >> task2 diff --git a/mit/__init__.py b/mit/__init__.py new file mode 100644 index 0000000..694a923 --- /dev/null +++ b/mit/__init__.py @@ -0,0 +1,10 @@ +from airflow.plugins_manager import AirflowPlugin + +from mit.operators import ECSOperator +from mit.sensors import ECSTaskSensor + + +class MitPlugin(AirflowPlugin): + name = 'mit' + operators = [ECSOperator] + sensors = [ECSTaskSensor] diff --git a/mit/operators.py b/mit/operators.py new file mode 100644 index 0000000..3db428f --- /dev/null +++ b/mit/operators.py @@ -0,0 +1,34 @@ +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults +import boto3 + + +class ECSOperator(BaseOperator): + @apply_defaults + def __init__(self, + cluster, + task_definition, + overrides, + network_configuration, + launch_type='FARGATE', + **kwargs): + super().__init__(**kwargs) + self.cluster = cluster + self.task_definition = task_definition + self.overrides = overrides + self.launch_type = launch_type + self.network_configuration = network_configuration + + def execute(self, context): + ecs = boto3.client('ecs') + res = ecs.run_task( + cluster=self.cluster, + taskDefinition=self.task_definition, + overrides=self.overrides, + count=1, + launchType=self.launch_type, + startedBy='Airflow', + networkConfiguration=self.network_configuration) + if len(res['failures']) > 0: + raise Exception(res) + return res['tasks'][0]['taskArn'] diff --git a/mit/sensors.py b/mit/sensors.py new file mode 100644 index 0000000..93a336a --- /dev/null +++ b/mit/sensors.py @@ -0,0 +1,32 @@ +from airflow.sensors.base_sensor_operator import BaseSensorOperator +from airflow.utils.decorators import apply_defaults +import boto3 + + +class ECSTaskSensor(BaseSensorOperator): + @apply_defaults + def __init__(self, + cluster, + ecs_task_id, + mode='reschedule', + *args, + **kwargs): + super().__init__(*args, mode=mode, **kwargs) + self.cluster = cluster + self.ecs_task_id = ecs_task_id + + def poke(self, context): + ecs = boto3.client('ecs') + arn = context['task_instance'].xcom_pull(task_ids=self.ecs_task_id) + res = ecs.describe_tasks(cluster=self.cluster, tasks=[arn]) + if len(res['failures']) > 0: + raise Exception(res) + + task = res['tasks'][0] + if task['lastStatus'] != 'STOPPED': + return False + cntr = task['containers'][0] + if cntr['exitCode'] > 0: + raise Exception( + f'Container exited with exit code {cntr["exitCode"]}') + return True diff --git a/tests/test_operators.py b/tests/test_operators.py new file mode 100644 index 0000000..cfc2939 --- /dev/null +++ b/tests/test_operators.py @@ -0,0 +1,27 @@ +import re + +import boto3 +import pytest + +from mit.operators import ECSOperator + + +arn_search = re.compile(r'arn:aws:ecs:us-east-1:012345678910:task/[0-9a-z-]+') + + +@pytest.fixture +def task(cluster): + task_name = 'workflow-task' + ecs = boto3.client('ecs') + ecs.register_task_definition(family=task_name, containerDefinitions=[]) + return task_name + + +def test_ecs_operator_runs_task(cluster, task): + op = ECSOperator(cluster=cluster.name, + task_definition=task, + overrides={}, + network_configuration={}, + task_id='test') + arn = op.execute(context={}) + assert arn_search.search(arn) diff --git a/tox.ini b/tox.ini index 5e5986a..36a3d2c 100644 --- a/tox.ini +++ b/tox.ini @@ -11,13 +11,13 @@ deps = flake8: flake8 setenv = PIPENV_VERBOSITY=-1 - coverage: PYTEST_COV=--cov manager + coverage: PYTEST_COV=--cov manager --cov mit commands = pipenv install --dev --ignore-pipfile --deploy pipenv run pytest tests {env:PYTEST_COV:} {posargs:--tb=short} [testenv:flake8] -commands = pipenv run flake8 manager tests +commands = pipenv run flake8 manager tests mit [testenv:safety] commands = pipenv check @@ -27,7 +27,7 @@ passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH COVERALLS_REPO_TOKEN commands = pipenv install --dev - pytest tests --cov=manager + pytest tests --cov=manager --cov=mit coveralls [flake8] diff --git a/workflows/log_config.py b/workflows/log_config.py new file mode 100644 index 0000000..295a2a2 --- /dev/null +++ b/workflows/log_config.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from typing import Dict, Any + +import six + +from airflow import configuration as conf +from airflow.utils.file import mkdirs + +# TODO: Logging format and level should be configured +# in this file instead of from airflow.cfg. Currently +# there are other log format and level configurations in +# settings.py and cli.py. Please see AIRFLOW-1455. +LOG_LEVEL = conf.get('core', 'LOGGING_LEVEL').upper() + + +# Flask appbuilder's info level log is very verbose, +# so it's set to 'WARN' by default. +FAB_LOG_LEVEL = conf.get('core', 'FAB_LOGGING_LEVEL').upper() + +LOG_FORMAT = conf.get('core', 'LOG_FORMAT') + +COLORED_LOG_FORMAT = conf.get('core', 'COLORED_LOG_FORMAT') + +COLORED_LOG = conf.getboolean('core', 'COLORED_CONSOLE_LOG') + +COLORED_FORMATTER_CLASS = conf.get('core', 'COLORED_FORMATTER_CLASS') + +BASE_LOG_FOLDER = conf.get('core', 'BASE_LOG_FOLDER') + +PROCESSOR_LOG_FOLDER = conf.get('scheduler', 'CHILD_PROCESS_LOG_DIRECTORY') + +DAG_PROCESSOR_MANAGER_LOG_LOCATION = \ + conf.get('core', 'DAG_PROCESSOR_MANAGER_LOG_LOCATION') + +FILENAME_TEMPLATE = conf.get('core', 'LOG_FILENAME_TEMPLATE') + +PROCESSOR_FILENAME_TEMPLATE = conf.get('core', 'LOG_PROCESSOR_FILENAME_TEMPLATE') + +# Storage bucket url for remote logging +# s3 buckets should start with "s3://" +# gcs buckets should start with "gs://" +# wasb buckets should start with "wasb" +# just to help Airflow select correct handler +REMOTE_BASE_LOG_FOLDER = conf.get('core', 'REMOTE_BASE_LOG_FOLDER') + +ELASTICSEARCH_HOST = conf.get('elasticsearch', 'HOST') + +ELASTICSEARCH_LOG_ID_TEMPLATE = conf.get('elasticsearch', 'LOG_ID_TEMPLATE') + +ELASTICSEARCH_END_OF_LOG_MARK = conf.get('elasticsearch', 'END_OF_LOG_MARK') + +ELASTICSEARCH_WRITE_STDOUT = conf.get('elasticsearch', 'WRITE_STDOUT') + +ELASTICSEARCH_JSON_FORMAT = conf.get('elasticsearch', 'JSON_FORMAT') + +ELASTICSEARCH_JSON_FIELDS = conf.get('elasticsearch', 'JSON_FIELDS') + +FORMATTER_CLASS_KEY = '()' if six.PY2 else 'class' + +DEFAULT_LOGGING_CONFIG = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'airflow': { + 'format': LOG_FORMAT + }, + 'airflow_coloured': { + 'format': COLORED_LOG_FORMAT if COLORED_LOG else LOG_FORMAT, + FORMATTER_CLASS_KEY: COLORED_FORMATTER_CLASS if COLORED_LOG else 'logging.Formatter' + }, + }, + 'handlers': { + 'console': { + 'class': 'airflow.utils.log.logging_mixin.RedirectStdHandler', + 'formatter': 'airflow_coloured', + 'stream': 'sys.stdout' + }, + 'task': { + 'class': 'airflow.utils.log.file_task_handler.FileTaskHandler', + 'formatter': 'airflow', + 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER), + 'filename_template': FILENAME_TEMPLATE, + }, + 'processor': { + 'class': 'airflow.utils.log.file_processor_handler.FileProcessorHandler', + 'formatter': 'airflow', + 'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER), + 'filename_template': PROCESSOR_FILENAME_TEMPLATE, + } + }, + 'loggers': { + 'airflow.processor': { + 'handlers': ['processor'], + 'level': LOG_LEVEL, + 'propagate': False, + }, + 'airflow.task': { + 'handlers': ['task'], + 'level': LOG_LEVEL, + 'propagate': False, + }, + 'flask_appbuilder': { + 'handler': ['console'], + 'level': FAB_LOG_LEVEL, + 'propagate': True, + } + }, + 'root': { + 'handlers': ['console'], + 'level': LOG_LEVEL, + } +} # type: Dict[str, Any] + +DEFAULT_DAG_PARSING_LOGGING_CONFIG = { + 'handlers': { + 'processor_manager': { + 'class': 'logging.handlers.RotatingFileHandler', + 'formatter': 'airflow', + 'filename': DAG_PROCESSOR_MANAGER_LOG_LOCATION, + 'mode': 'a', + 'maxBytes': 104857600, # 100MB + 'backupCount': 5 + } + }, + 'loggers': { + 'airflow.processor_manager': { + 'handlers': ['processor_manager'], + 'level': LOG_LEVEL, + 'propagate': False, + } + } +} + +REMOTE_HANDLERS = { + 's3': { + 'task': { + 'class': 'airflow.utils.log.s3_task_handler.S3TaskHandler', + 'formatter': 'airflow', + 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER), + 's3_log_folder': REMOTE_BASE_LOG_FOLDER, + 'filename_template': FILENAME_TEMPLATE, + }, + }, + 'gcs': { + 'task': { + 'class': 'airflow.utils.log.gcs_task_handler.GCSTaskHandler', + 'formatter': 'airflow', + 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER), + 'gcs_log_folder': REMOTE_BASE_LOG_FOLDER, + 'filename_template': FILENAME_TEMPLATE, + }, + }, + 'wasb': { + 'task': { + 'class': 'airflow.utils.log.wasb_task_handler.WasbTaskHandler', + 'formatter': 'airflow', + 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER), + 'wasb_log_folder': REMOTE_BASE_LOG_FOLDER, + 'wasb_container': 'airflow-logs', + 'filename_template': FILENAME_TEMPLATE, + 'delete_local_copy': False, + }, + }, + 'elasticsearch': { + 'task': { + 'class': 'airflow.utils.log.es_task_handler.ElasticsearchTaskHandler', + 'formatter': 'airflow', + 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER), + 'log_id_template': ELASTICSEARCH_LOG_ID_TEMPLATE, + 'filename_template': FILENAME_TEMPLATE, + 'end_of_log_mark': ELASTICSEARCH_END_OF_LOG_MARK, + 'host': ELASTICSEARCH_HOST, + 'write_stdout': ELASTICSEARCH_WRITE_STDOUT, + 'json_format': ELASTICSEARCH_JSON_FORMAT, + 'json_fields': ELASTICSEARCH_JSON_FIELDS + }, + }, +} + +REMOTE_LOGGING = conf.getboolean('core', 'remote_logging') + +# Only update the handlers and loggers when CONFIG_PROCESSOR_MANAGER_LOGGER is set. +# This is to avoid exceptions when initializing RotatingFileHandler multiple times +# in multiple processes. +if os.environ.get('CONFIG_PROCESSOR_MANAGER_LOGGER') == 'True': + DEFAULT_LOGGING_CONFIG['handlers'] \ + .update(DEFAULT_DAG_PARSING_LOGGING_CONFIG['handlers']) + DEFAULT_LOGGING_CONFIG['loggers'] \ + .update(DEFAULT_DAG_PARSING_LOGGING_CONFIG['loggers']) + + # Manually create log directory for processor_manager handler as RotatingFileHandler + # will only create file but not the directory. + processor_manager_handler_config = DEFAULT_DAG_PARSING_LOGGING_CONFIG['handlers'][ + 'processor_manager'] + directory = os.path.dirname(processor_manager_handler_config['filename']) + mkdirs(directory, 0o755) + +if REMOTE_LOGGING and REMOTE_BASE_LOG_FOLDER.startswith('s3://'): + DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['s3']) +elif REMOTE_LOGGING and REMOTE_BASE_LOG_FOLDER.startswith('gs://'): + DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['gcs']) +elif REMOTE_LOGGING and REMOTE_BASE_LOG_FOLDER.startswith('wasb'): + DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['wasb']) +elif REMOTE_LOGGING and ELASTICSEARCH_HOST: + DEFAULT_LOGGING_CONFIG['handlers'].update(REMOTE_HANDLERS['elasticsearch'])