From 3627f0160b4db536c0817c196dd51f78215bae5e Mon Sep 17 00:00:00 2001
From: xiaoxshe <109769013+xiaoxshe@users.noreply.github.com>
Date: Thu, 25 Aug 2022 16:04:36 -0700
Subject: [PATCH 1/8] support Spark 3.2 and EMR 6.7
---
Makefile | 2 +-
Pipfile | 41 +
Pipfile.lock | 1052 +++++++++++++++++
new_images.yml | 4 +-
scripts/fetch-ec2-instance-type-info.sh | 2 +-
setup.py | 48 +
.../container-bootstrap-config/bootstrap.sh | 1 +
.../3.2/py3/docker/py39/Dockerfile.cpu | 128 ++
.../3.2/py3/hadoop-config/core-site.xml | 26 +
.../3.2/py3/hadoop-config/hdfs-site.xml | 67 ++
.../3.2/py3/hadoop-config/spark-defaults.conf | 10 +
.../3.2/py3/hadoop-config/spark-env.sh | 3 +
.../3.2/py3/hadoop-config/yarn-site.xml | 34 +
.../3.2/py3/nginx-config/default.conf | 17 +
.../3.2/py3/nginx-config/nginx.conf | 66 ++
spark/processing/3.2/py3/yum/emr-apps.repo | 17 +
16 files changed, 1514 insertions(+), 4 deletions(-)
create mode 100644 Pipfile
create mode 100644 Pipfile.lock
create mode 100644 setup.py
create mode 100644 spark/processing/3.2/py3/container-bootstrap-config/bootstrap.sh
create mode 100644 spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
create mode 100644 spark/processing/3.2/py3/hadoop-config/core-site.xml
create mode 100644 spark/processing/3.2/py3/hadoop-config/hdfs-site.xml
create mode 100644 spark/processing/3.2/py3/hadoop-config/spark-defaults.conf
create mode 100644 spark/processing/3.2/py3/hadoop-config/spark-env.sh
create mode 100644 spark/processing/3.2/py3/hadoop-config/yarn-site.xml
create mode 100644 spark/processing/3.2/py3/nginx-config/default.conf
create mode 100644 spark/processing/3.2/py3/nginx-config/nginx.conf
create mode 100644 spark/processing/3.2/py3/yum/emr-apps.repo
diff --git a/Makefile b/Makefile
index e3a2488..8b2be39 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ SHELL := /bin/sh
# Set variables if testing locally
ifeq ($(IS_RELEASE_BUILD),)
- SPARK_VERSION := 3.1
+ SPARK_VERSION := 3.2
PROCESSOR := cpu
FRAMEWORK_VERSION := py39
SM_VERSION := 1.0
diff --git a/Pipfile b/Pipfile
new file mode 100644
index 0000000..8c11d47
--- /dev/null
+++ b/Pipfile
@@ -0,0 +1,41 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+tenacity = "==8.0.1"
+psutil = "==5.9.0"
+click = "==8.1.2"
+watchdog = "==0.10.3"
+waitress = "==2.1.2"
+types-waitress = "==2.0.6"
+requests = "==2.27.1"
+types-requests = "==2.27.16"
+rsa = "==4.3"
+pyasn1 = "==0.4.8"
+boto3 = "==1.21.33"
+safety = "==1.10.3"
+black = "==22.3.0"
+mypy = "==0.942"
+flake8 = "==4.0.1"
+flake8-docstrings = "==1.5.0"
+pytest = "==7.1.1"
+pytest-cov = "==2.10.0"
+pytest-xdist = "==2.5.0"
+docker = "==5.0.3"
+docker-compose = "==1.29.2"
+cryptography = "==36.0.2"
+typing-extensions = "==4.1.1"
+sagemaker = "==2.83.0"
+smspark = {editable = true, path = "."}
+importlib-metadata = "==4.11.3"
+pytest-parallel = "==0.1.1"
+pytest-rerunfailures = "10.0"
+numpy = "==1.22.2"
+protobuf = "==3.20.1"
+
+[requires]
+python_version = "3.9"
diff --git a/Pipfile.lock b/Pipfile.lock
new file mode 100644
index 0000000..e50f52b
--- /dev/null
+++ b/Pipfile.lock
@@ -0,0 +1,1052 @@
+{
+ "_meta": {
+ "hash": {
+ "sha256": "5d91621d369abeef58c8db372b5c8461708b798aa0ecf86a2caf5849d7b11d76"
+ },
+ "pipfile-spec": 6,
+ "requires": {
+ "python_version": "3.9"
+ },
+ "sources": [
+ {
+ "name": "pypi",
+ "url": "https://pypi.org/simple",
+ "verify_ssl": true
+ }
+ ]
+ },
+ "default": {
+ "attrs": {
+ "hashes": [
+ "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
+ "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==20.3.0"
+ },
+ "bcrypt": {
+ "hashes": [
+ "sha256:0b0f0c7141622a31e9734b7f649451147c04ebb5122327ac0bd23744df84be90",
+ "sha256:1c3334446fac200499e8bc04a530ce3cf0b3d7151e0e4ac5c0dddd3d95e97843",
+ "sha256:2d0dd19aad87e4ab882ef1d12df505f4c52b28b69666ce83c528f42c07379227",
+ "sha256:594780b364fb45f2634c46ec8d3e61c1c0f1811c4f2da60e8eb15594ecbf93ed",
+ "sha256:7c7dd6c1f05bf89e65261d97ac3a6520f34c2acb369afb57e3ea4449be6ff8fd",
+ "sha256:845b1daf4df2dd94d2fdbc9454953ca9dd0e12970a0bfc9f3dcc6faea3fa96e4",
+ "sha256:8780e69f9deec9d60f947b169507d2c9816e4f11548f1f7ebee2af38b9b22ae4",
+ "sha256:bf413f2a9b0a2950fc750998899013f2e718d20fa4a58b85ca50b6df5ed1bbf9",
+ "sha256:bfb67f6a6c72dfb0a02f3df51550aa1862708e55128b22543e2b42c74f3620d7",
+ "sha256:c59c170fc9225faad04dde1ba61d85b413946e8ce2e5f5f5ff30dfd67283f319",
+ "sha256:dc6ec3dc19b1c193b2f7cf279d3e32e7caf447532fbcb7af0906fe4398900c33",
+ "sha256:ede0f506554571c8eda80db22b83c139303ec6b595b8f60c4c8157bdd0bdee36"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==4.0.0"
+ },
+ "black": {
+ "hashes": [
+ "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b",
+ "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176",
+ "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09",
+ "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a",
+ "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015",
+ "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79",
+ "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb",
+ "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20",
+ "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464",
+ "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968",
+ "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82",
+ "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21",
+ "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0",
+ "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265",
+ "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b",
+ "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a",
+ "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72",
+ "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce",
+ "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0",
+ "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a",
+ "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163",
+ "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad",
+ "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"
+ ],
+ "index": "pypi",
+ "version": "==22.3.0"
+ },
+ "boto3": {
+ "hashes": [
+ "sha256:9d8ddfefe0c4a993423e2c40831034c78fcb7b3425bf3610cf0087301dd9098b",
+ "sha256:c06b9b29f80da8cf6d9fac8f41d74a74d0f5347927acf11b15428b295fcbdd31"
+ ],
+ "index": "pypi",
+ "version": "==1.21.33"
+ },
+ "botocore": {
+ "hashes": [
+ "sha256:663d8f02b98641846eb959c54c840cc33264d5f2dee5b8fc09ee8adbef0f8dcf",
+ "sha256:89a203bba3c8f2299287e48a9e112e2dbe478cf67eaac26716f0e7f176446146"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==1.24.46"
+ },
+ "certifi": {
+ "hashes": [
+ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d",
+ "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==2022.6.15"
+ },
+ "cffi": {
+ "hashes": [
+ "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5",
+ "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef",
+ "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104",
+ "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426",
+ "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405",
+ "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375",
+ "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a",
+ "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e",
+ "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc",
+ "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf",
+ "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185",
+ "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497",
+ "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3",
+ "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35",
+ "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c",
+ "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83",
+ "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21",
+ "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca",
+ "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984",
+ "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac",
+ "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd",
+ "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee",
+ "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a",
+ "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2",
+ "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192",
+ "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7",
+ "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585",
+ "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f",
+ "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e",
+ "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27",
+ "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b",
+ "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e",
+ "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e",
+ "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d",
+ "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c",
+ "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415",
+ "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82",
+ "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02",
+ "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314",
+ "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325",
+ "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c",
+ "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3",
+ "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914",
+ "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045",
+ "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d",
+ "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9",
+ "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5",
+ "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2",
+ "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c",
+ "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3",
+ "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2",
+ "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8",
+ "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d",
+ "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d",
+ "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9",
+ "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162",
+ "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76",
+ "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4",
+ "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e",
+ "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9",
+ "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6",
+ "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b",
+ "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01",
+ "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"
+ ],
+ "version": "==1.15.1"
+ },
+ "charset-normalizer": {
+ "hashes": [
+ "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597",
+ "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"
+ ],
+ "markers": "python_version >= '3'",
+ "version": "==2.0.12"
+ },
+ "click": {
+ "hashes": [
+ "sha256:24e1a4a9ec5bf6299411369b208c1df2188d9eb8d916302fe6bf03faed227f1e",
+ "sha256:479707fe14d9ec9a0757618b7a100a0ae4c4e236fac5b7f80ca68028141a1a72"
+ ],
+ "index": "pypi",
+ "version": "==8.1.2"
+ },
+ "coverage": {
+ "hashes": [
+ "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2",
+ "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820",
+ "sha256:1238b08f3576201ebf41f7c20bf59baa0d05da941b123c6656e42cdb668e9827",
+ "sha256:14a32ec68d721c3d714d9b105c7acf8e0f8a4f4734c811eda75ff3718570b5e3",
+ "sha256:15e38d853ee224e92ccc9a851457fb1e1f12d7a5df5ae44544ce7863691c7a0d",
+ "sha256:354df19fefd03b9a13132fa6643527ef7905712109d9c1c1903f2133d3a4e145",
+ "sha256:35ef1f8d8a7a275aa7410d2f2c60fa6443f4a64fae9be671ec0696a68525b875",
+ "sha256:4179502f210ebed3ccfe2f78bf8e2d59e50b297b598b100d6c6e3341053066a2",
+ "sha256:42c499c14efd858b98c4e03595bf914089b98400d30789511577aa44607a1b74",
+ "sha256:4b7101938584d67e6f45f0015b60e24a95bf8dea19836b1709a80342e01b472f",
+ "sha256:564cd0f5b5470094df06fab676c6d77547abfdcb09b6c29c8a97c41ad03b103c",
+ "sha256:5f444627b3664b80d078c05fe6a850dd711beeb90d26731f11d492dcbadb6973",
+ "sha256:6113e4df2fa73b80f77663445be6d567913fb3b82a86ceb64e44ae0e4b695de1",
+ "sha256:61b993f3998ee384935ee423c3d40894e93277f12482f6e777642a0141f55782",
+ "sha256:66e6df3ac4659a435677d8cd40e8eb1ac7219345d27c41145991ee9bf4b806a0",
+ "sha256:67f9346aeebea54e845d29b487eb38ec95f2ecf3558a3cffb26ee3f0dcc3e760",
+ "sha256:6913dddee2deff8ab2512639c5168c3e80b3ebb0f818fed22048ee46f735351a",
+ "sha256:6a864733b22d3081749450466ac80698fe39c91cb6849b2ef8752fd7482011f3",
+ "sha256:7026f5afe0d1a933685d8f2169d7c2d2e624f6255fb584ca99ccca8c0e966fd7",
+ "sha256:783bc7c4ee524039ca13b6d9b4186a67f8e63d91342c713e88c1865a38d0892a",
+ "sha256:7a98d6bf6d4ca5c07a600c7b4e0c5350cd483c85c736c522b786be90ea5bac4f",
+ "sha256:8d032bfc562a52318ae05047a6eb801ff31ccee172dc0d2504614e911d8fa83e",
+ "sha256:98c0b9e9b572893cdb0a00e66cf961a238f8d870d4e1dc8e679eb8bdc2eb1b86",
+ "sha256:9c7b9b498eb0c0d48b4c2abc0e10c2d78912203f972e0e63e3c9dc21f15abdaa",
+ "sha256:9cc4f107009bca5a81caef2fca843dbec4215c05e917a59dec0c8db5cff1d2aa",
+ "sha256:9d6e1f3185cbfd3d91ac77ea065d85d5215d3dfa45b191d14ddfcd952fa53796",
+ "sha256:a095aa0a996ea08b10580908e88fbaf81ecf798e923bbe64fb98d1807db3d68a",
+ "sha256:a3b2752de32c455f2521a51bd3ffb53c5b3ae92736afde67ce83477f5c1dd928",
+ "sha256:ab066f5ab67059d1f1000b5e1aa8bbd75b6ed1fc0014559aea41a9eb66fc2ce0",
+ "sha256:c1328d0c2f194ffda30a45f11058c02410e679456276bfa0bbe0b0ee87225fac",
+ "sha256:c35cca192ba700979d20ac43024a82b9b32a60da2f983bec6c0f5b84aead635c",
+ "sha256:cbbb0e4cd8ddcd5ef47641cfac97d8473ab6b132dd9a46bacb18872828031685",
+ "sha256:cdbb0d89923c80dbd435b9cf8bba0ff55585a3cdb28cbec65f376c041472c60d",
+ "sha256:cf2afe83a53f77aec067033199797832617890e15bed42f4a1a93ea24794ae3e",
+ "sha256:d5dd4b8e9cd0deb60e6fcc7b0647cbc1da6c33b9e786f9c79721fd303994832f",
+ "sha256:dfa0b97eb904255e2ab24166071b27408f1f69c8fbda58e9c0972804851e0558",
+ "sha256:e16c45b726acb780e1e6f88b286d3c10b3914ab03438f32117c4aa52d7f30d58",
+ "sha256:e1fabd473566fce2cf18ea41171d92814e4ef1495e04471786cbc943b89a3781",
+ "sha256:e3d3c4cc38b2882f9a15bafd30aec079582b819bec1b8afdbde8f7797008108a",
+ "sha256:e431e305a1f3126477abe9a184624a85308da8edf8486a863601d58419d26ffa",
+ "sha256:e7b4da9bafad21ea45a714d3ea6f3e1679099e420c8741c74905b92ee9bfa7cc",
+ "sha256:ee2b2fb6eb4ace35805f434e0f6409444e1466a47f620d1d5763a22600f0f892",
+ "sha256:ee6ae6bbcac0786807295e9687169fba80cb0617852b2fa118a99667e8e6815d",
+ "sha256:ef6f44409ab02e202b31a05dd6666797f9de2aa2b4b3534e9d450e42dea5e817",
+ "sha256:f67cf9f406cf0d2f08a3515ce2db5b82625a7257f88aad87904674def6ddaec1",
+ "sha256:f855b39e4f75abd0dfbcf74a82e84ae3fc260d523fcb3532786bcbbcb158322c",
+ "sha256:fc600f6ec19b273da1d85817eda339fb46ce9eef3e89f220055d8696e0a06908",
+ "sha256:fcbe3d9a53e013f8ab88734d7e517eb2cd06b7e689bedf22c0eb68db5e4a0a19",
+ "sha256:fde17bc42e0716c94bf19d92e4c9f5a00c5feb401f5bc01101fdf2a8b7cacf60",
+ "sha256:ff934ced84054b9018665ca3967fc48e1ac99e811f6cc99ea65978e1d384454b"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==6.4.4"
+ },
+ "cryptography": {
+ "hashes": [
+ "sha256:0a3bf09bb0b7a2c93ce7b98cb107e9170a90c51a0162a20af1c61c765b90e60b",
+ "sha256:1f64a62b3b75e4005df19d3b5235abd43fa6358d5516cfc43d87aeba8d08dd51",
+ "sha256:32db5cc49c73f39aac27574522cecd0a4bb7384e71198bc65a0d23f901e89bb7",
+ "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d",
+ "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6",
+ "sha256:53e0285b49fd0ab6e604f4c5d9c5ddd98de77018542e88366923f152dbeb3c29",
+ "sha256:70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9",
+ "sha256:7b2d54e787a884ffc6e187262823b6feb06c338084bbe80d45166a1cb1c6c5bf",
+ "sha256:7be666cc4599b415f320839e36367b273db8501127b38316f3b9f22f17a0b815",
+ "sha256:8241cac0aae90b82d6b5c443b853723bcc66963970c67e56e71a2609dc4b5eaf",
+ "sha256:82740818f2f240a5da8dfb8943b360e4f24022b093207160c77cadade47d7c85",
+ "sha256:8897b7b7ec077c819187a123174b645eb680c13df68354ed99f9b40a50898f77",
+ "sha256:c2c5250ff0d36fd58550252f54915776940e4e866f38f3a7866d92b32a654b86",
+ "sha256:ca9f686517ec2c4a4ce930207f75c00bf03d94e5063cbc00a1dc42531511b7eb",
+ "sha256:d2b3d199647468d410994dbeb8cec5816fb74feb9368aedf300af709ef507e3e",
+ "sha256:da73d095f8590ad437cd5e9faf6628a218aa7c387e1fdf67b888b47ba56a17f0",
+ "sha256:e167b6b710c7f7bc54e67ef593f8731e1f45aa35f8a8a7b72d6e42ec76afd4b3",
+ "sha256:ea634401ca02367c1567f012317502ef3437522e2fc44a3ea1844de028fa4b84",
+ "sha256:ec6597aa85ce03f3e507566b8bcdf9da2227ec86c4266bd5e6ab4d9e0cc8dab2",
+ "sha256:f64b232348ee82f13aac22856515ce0195837f6968aeaa94a3d0353ea2ec06a6"
+ ],
+ "index": "pypi",
+ "version": "==36.0.2"
+ },
+ "dill": {
+ "hashes": [
+ "sha256:33501d03270bbe410c72639b350e941882a8b0fd55357580fbc873fba0c59302",
+ "sha256:d75e41f3eff1eee599d738e76ba8f4ad98ea229db8b085318aa2b3333a208c86"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
+ "version": "==0.3.5.1"
+ },
+ "distro": {
+ "hashes": [
+ "sha256:151aeccf60c216402932b52e40ee477a939f8d58898927378a02abbe852c1c39",
+ "sha256:d596311d707e692c2160c37807f83e3820c5d539d5a83e87cfb6babd8ba3a06b"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==1.7.0"
+ },
+ "docker": {
+ "hashes": [
+ "sha256:7a79bb439e3df59d0a72621775d600bc8bc8b422d285824cb37103eab91d1ce0",
+ "sha256:d916a26b62970e7c2f554110ed6af04c7ccff8e9f81ad17d0d40c75637e227fb"
+ ],
+ "index": "pypi",
+ "version": "==5.0.3"
+ },
+ "docker-compose": {
+ "hashes": [
+ "sha256:4c8cd9d21d237412793d18bd33110049ee9af8dab3fe2c213bbd0733959b09b7",
+ "sha256:8d5589373b35c8d3b1c8c1182c6e4a4ff14bffa3dd0b605fcd08f73c94cef809"
+ ],
+ "index": "pypi",
+ "version": "==1.29.2"
+ },
+ "dockerpty": {
+ "hashes": [
+ "sha256:69a9d69d573a0daa31bcd1c0774eeed5c15c295fe719c61aca550ed1393156ce"
+ ],
+ "version": "==0.4.1"
+ },
+ "docopt": {
+ "hashes": [
+ "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
+ ],
+ "version": "==0.6.2"
+ },
+ "dparse": {
+ "hashes": [
+ "sha256:b1514fb08895d85b18d4eba3b1b7025ff9e6ea07286282021e19def872129975",
+ "sha256:c348994a1f41c85f664d8f5a47442647bc4e22c5af5b1b26ef29aff0fa5dddcd"
+ ],
+ "markers": "python_version >= '3.5'",
+ "version": "==0.5.2"
+ },
+ "execnet": {
+ "hashes": [
+ "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5",
+ "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+ "version": "==1.9.0"
+ },
+ "flake8": {
+ "hashes": [
+ "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d",
+ "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"
+ ],
+ "index": "pypi",
+ "version": "==4.0.1"
+ },
+ "flake8-docstrings": {
+ "hashes": [
+ "sha256:3d5a31c7ec6b7367ea6506a87ec293b94a0a46c0bce2bb4975b7f1d09b6f3717",
+ "sha256:a256ba91bc52307bef1de59e2a009c3cf61c3d0952dbe035d6ff7208940c2edc"
+ ],
+ "index": "pypi",
+ "version": "==1.5.0"
+ },
+ "google-pasta": {
+ "hashes": [
+ "sha256:4612951da876b1a10fe3960d7226f0c7682cf901e16ac06e473b267a5afa8954",
+ "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed",
+ "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e"
+ ],
+ "version": "==0.2.0"
+ },
+ "idna": {
+ "hashes": [
+ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
+ "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
+ ],
+ "markers": "python_version >= '3'",
+ "version": "==3.3"
+ },
+ "importlib-metadata": {
+ "hashes": [
+ "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6",
+ "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"
+ ],
+ "index": "pypi",
+ "version": "==4.11.3"
+ },
+ "iniconfig": {
+ "hashes": [
+ "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+ "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+ ],
+ "version": "==1.1.1"
+ },
+ "jmespath": {
+ "hashes": [
+ "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
+ "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==1.0.1"
+ },
+ "jsonschema": {
+ "hashes": [
+ "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163",
+ "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"
+ ],
+ "version": "==3.2.0"
+ },
+ "mccabe": {
+ "hashes": [
+ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+ "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+ ],
+ "version": "==0.6.1"
+ },
+ "multiprocess": {
+ "hashes": [
+ "sha256:00ef48461d43d1e30f8f4b2e1b287ecaaffec325a37053beb5503e0d69e5a3cd",
+ "sha256:01c1137d2f18d0cd262d0fdb7294b1fe9fc3e8dc8b126e506085434ae8eb3677",
+ "sha256:0f4faf4811019efdb2f91db09240f893ee40cbfcb06978f3b8ed8c248e73babe",
+ "sha256:17cb4229aa43e6973679d67c66a454cbf8b6b0d038425cba3220ea5a06d61b58",
+ "sha256:2e096dd618a84d15aa369a9cf6695815e5539f853dc8fa4f4b9153b11b1d0b32",
+ "sha256:34e9703bd5b9fee5455c93a74e44dbabe55481c214d03be1e65f037be9d0c520",
+ "sha256:3ec1c8015e19182bfa01b5887a9c25805c48df3c71863f48fe83803147cde5d6",
+ "sha256:48315eefe02c35dd7560da3fa8af66d9f4a61b9dc8f7c40801c5f972ab4604b1",
+ "sha256:5436d1cd9f901f7ddc4f20b6fd0b462c87dcc00d941cc13eeb2401fc5bd00e42",
+ "sha256:5974bdad390ba466cc130288d2ef1048fdafedd01cf4641fc024f6088af70bfe",
+ "sha256:5a6dca5f29f0224c855d0d5cad963476175cfc8de112d3eebe85914cb735f130",
+ "sha256:62e556a0c31ec7176e28aa331663ac26c276ee3536b5e9bb5e850681e7a00f11",
+ "sha256:6cdde49defcb933062df382ebc9b5299beebcd157a98b3a65291c1c94a2edc41",
+ "sha256:7be9e320a41d2d0d0eddacfe693cfb07b4cb9c0d3d10007f4304255c15215778",
+ "sha256:7e6a689da3490412caa7b3e27c3385d8aaa49135f3a353ace94ca47e4c926d37",
+ "sha256:92003c247436f8699b7692e95346a238446710f078500eb364bc23bb0503dd4f",
+ "sha256:99bb68dd0d5b3d30fe104721bee26e4637667112d5951b51feb81479fd560876",
+ "sha256:af0a48440aa8f793d8bb100f20102c12f192de5a608638819a998f2cc59e1fcd",
+ "sha256:b7415f61bddfffdade73396904551be8124a4a363322aa9c72d42e349c5fca39",
+ "sha256:b9a3be43ecee6776a9e7223af96914a0164f306affcf4624b213885172236b77",
+ "sha256:c4a97216e8319039c69a266252cc68a392b96f9e67e3ed02ad88be9e6f2d2969"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
+ "version": "==0.70.13"
+ },
+ "mypy": {
+ "hashes": [
+ "sha256:0e2dd88410937423fba18e57147dd07cd8381291b93d5b1984626f173a26543e",
+ "sha256:10daab80bc40f84e3f087d896cdb53dc811a9f04eae4b3f95779c26edee89d16",
+ "sha256:17e44649fec92e9f82102b48a3bf7b4a5510ad0cd22fa21a104826b5db4903e2",
+ "sha256:1a0459c333f00e6a11cbf6b468b870c2b99a906cb72d6eadf3d1d95d38c9352c",
+ "sha256:246e1aa127d5b78488a4a0594bd95f6d6fb9d63cf08a66dafbff8595d8891f67",
+ "sha256:2b184db8c618c43c3a31b32ff00cd28195d39e9c24e7c3b401f3db7f6e5767f5",
+ "sha256:2bc249409a7168d37c658e062e1ab5173300984a2dada2589638568ddc1db02b",
+ "sha256:3841b5433ff936bff2f4dc8d54cf2cdbfea5d8e88cedfac45c161368e5770ba6",
+ "sha256:4c3e497588afccfa4334a9986b56f703e75793133c4be3a02d06a3df16b67a58",
+ "sha256:5bf44840fb43ac4074636fd47ee476d73f0039f4f54e86d7265077dc199be24d",
+ "sha256:64235137edc16bee6f095aba73be5334677d6f6bdb7fa03cfab90164fa294a17",
+ "sha256:6776e5fa22381cc761df53e7496a805801c1a751b27b99a9ff2f0ca848c7eca0",
+ "sha256:6ce34a118d1a898f47def970a2042b8af6bdcc01546454726c7dd2171aa6dfca",
+ "sha256:6f6ad963172152e112b87cc7ec103ba0f2db2f1cd8997237827c052a3903eaa6",
+ "sha256:6f7106cbf9cc2f403693bf50ed7c9fa5bb3dfa9007b240db3c910929abe2a322",
+ "sha256:7742d2c4e46bb5017b51c810283a6a389296cda03df805a4f7869a6f41246534",
+ "sha256:9521c1265ccaaa1791d2c13582f06facf815f426cd8b07c3a485f486a8ffc1f3",
+ "sha256:a1b383fe99678d7402754fe90448d4037f9512ce70c21f8aee3b8bf48ffc51db",
+ "sha256:b840cfe89c4ab6386c40300689cd8645fc8d2d5f20101c7f8bd23d15fca14904",
+ "sha256:d8d3ba77e56b84cd47a8ee45b62c84b6d80d32383928fe2548c9a124ea0a725c",
+ "sha256:dcd955f36e0180258a96f880348fbca54ce092b40fbb4b37372ae3b25a0b0a46",
+ "sha256:e865fec858d75b78b4d63266c9aff770ecb6a39dfb6d6b56c47f7f8aba6baba8",
+ "sha256:edf7237137a1a9330046dbb14796963d734dd740a98d5e144a3eb1d267f5f9ee"
+ ],
+ "index": "pypi",
+ "version": "==0.942"
+ },
+ "mypy-extensions": {
+ "hashes": [
+ "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+ "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+ ],
+ "version": "==0.4.3"
+ },
+ "numpy": {
+ "hashes": [
+ "sha256:03ae5850619abb34a879d5f2d4bb4dcd025d6d8fb72f5e461dae84edccfe129f",
+ "sha256:076aee5a3763d41da6bef9565fdf3cb987606f567cd8b104aded2b38b7b47abf",
+ "sha256:0b536b6840e84c1c6a410f3a5aa727821e6108f3454d81a5cd5900999ef04f89",
+ "sha256:15efb7b93806d438e3bc590ca8ef2f953b0ce4f86f337ef4559d31ec6cf9d7dd",
+ "sha256:168259b1b184aa83a514f307352c25c56af111c269ffc109d9704e81f72e764b",
+ "sha256:2638389562bda1635b564490d76713695ff497242a83d9b684d27bb4a6cc9d7a",
+ "sha256:3556c5550de40027d3121ebbb170f61bbe19eb639c7ad0c7b482cd9b560cd23b",
+ "sha256:4a176959b6e7e00b5a0d6f549a479f869829bfd8150282c590deee6d099bbb6e",
+ "sha256:515a8b6edbb904594685da6e176ac9fbea8f73a5ebae947281de6613e27f1956",
+ "sha256:55535c7c2f61e2b2fc817c5cbe1af7cb907c7f011e46ae0a52caa4be1f19afe2",
+ "sha256:59153979d60f5bfe9e4c00e401e24dfe0469ef8da6d68247439d3278f30a180f",
+ "sha256:60cb8e5933193a3cc2912ee29ca331e9c15b2da034f76159b7abc520b3d1233a",
+ "sha256:6767ad399e9327bfdbaa40871be4254d1995f4a3ca3806127f10cec778bd9896",
+ "sha256:76a4f9bce0278becc2da7da3b8ef854bed41a991f4226911a24a9711baad672c",
+ "sha256:8cf33634b60c9cef346663a222d9841d3bbbc0a2f00221d6bcfd0d993d5543f6",
+ "sha256:94dd11d9f13ea1be17bac39c1942f527cbf7065f94953cf62dfe805653da2f8f",
+ "sha256:aafa46b5a39a27aca566198d3312fb3bde95ce9677085efd02c86f7ef6be4ec7",
+ "sha256:badca914580eb46385e7f7e4e426fea6de0a37b9e06bec252e481ae7ec287082",
+ "sha256:d76a26c5118c4d96e264acc9e3242d72e1a2b92e739807b3b69d8d47684b6677"
+ ],
+ "index": "pypi",
+ "version": "==1.22.2"
+ },
+ "packaging": {
+ "hashes": [
+ "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
+ "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==21.3"
+ },
+ "pandas": {
+ "hashes": [
+ "sha256:07238a58d7cbc8a004855ade7b75bbd22c0db4b0ffccc721556bab8a095515f6",
+ "sha256:0daf876dba6c622154b2e6741f29e87161f844e64f84801554f879d27ba63c0d",
+ "sha256:16ad23db55efcc93fa878f7837267973b61ea85d244fc5ff0ccbcfa5638706c5",
+ "sha256:1d9382f72a4f0e93909feece6fef5500e838ce1c355a581b3d8f259839f2ea76",
+ "sha256:24ea75f47bbd5574675dae21d51779a4948715416413b30614c1e8b480909f81",
+ "sha256:2893e923472a5e090c2d5e8db83e8f907364ec048572084c7d10ef93546be6d1",
+ "sha256:2ff7788468e75917574f080cd4681b27e1a7bf36461fe968b49a87b5a54d007c",
+ "sha256:41fc406e374590a3d492325b889a2686b31e7a7780bec83db2512988550dadbf",
+ "sha256:48350592665ea3cbcd07efc8c12ff12d89be09cd47231c7925e3b8afada9d50d",
+ "sha256:605d572126eb4ab2eadf5c59d5d69f0608df2bf7bcad5c5880a47a20a0699e3e",
+ "sha256:6dfbf16b1ea4f4d0ee11084d9c026340514d1d30270eaa82a9f1297b6c8ecbf0",
+ "sha256:6f803320c9da732cc79210d7e8cc5c8019aad512589c910c66529eb1b1818230",
+ "sha256:721a3dd2f06ef942f83a819c0f3f6a648b2830b191a72bbe9451bcd49c3bd42e",
+ "sha256:755679c49460bd0d2f837ab99f0a26948e68fa0718b7e42afbabd074d945bf84",
+ "sha256:78b00429161ccb0da252229bcda8010b445c4bf924e721265bec5a6e96a92e92",
+ "sha256:958a0588149190c22cdebbc0797e01972950c927a11a900fe6c2296f207b1d6f",
+ "sha256:a3924692160e3d847e18702bb048dc38e0e13411d2b503fecb1adf0fcf950ba4",
+ "sha256:d51674ed8e2551ef7773820ef5dab9322be0828629f2cbf8d1fc31a0c4fed640",
+ "sha256:d5ebc990bd34f4ac3c73a2724c2dcc9ee7bf1ce6cf08e87bb25c6ad33507e318",
+ "sha256:d6c0106415ff1a10c326c49bc5dd9ea8b9897a6ca0c8688eb9c30ddec49535ef",
+ "sha256:e48fbb64165cda451c06a0f9e4c7a16b534fcabd32546d531b3c240ce2844112"
+ ],
+ "markers": "python_version >= '3.8'",
+ "version": "==1.4.3"
+ },
+ "paramiko": {
+ "hashes": [
+ "sha256:003e6bee7c034c21fbb051bf83dc0a9ee4106204dd3c53054c71452cc4ec3938",
+ "sha256:655f25dc8baf763277b933dfcea101d636581df8d6b9774d1fb653426b72c270"
+ ],
+ "version": "==2.11.0"
+ },
+ "pathos": {
+ "hashes": [
+ "sha256:1c44373d8692897d5d15a8aa3b3a442ddc0814c5e848f4ff0ded5491f34b1dac",
+ "sha256:6a6ddb514ce2719f63fb88d5ec4f4490e436b636b54f1102d952c9f7c52f18e2",
+ "sha256:a8dbddcd3d9af32ada7c6dc088d845588c513a29a0ba19ab9f64c5cd83692934"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
+ "version": "==0.2.9"
+ },
+ "pathspec": {
+ "hashes": [
+ "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a",
+ "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"
+ ],
+ "version": "==0.9.0"
+ },
+ "pathtools": {
+ "hashes": [
+ "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0"
+ ],
+ "version": "==0.1.2"
+ },
+ "platformdirs": {
+ "hashes": [
+ "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788",
+ "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==2.5.2"
+ },
+ "pluggy": {
+ "hashes": [
+ "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
+ "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==1.0.0"
+ },
+ "pox": {
+ "hashes": [
+ "sha256:541b5c845aacb806c1364d4142003efb809d654c9ca8db82e650ee86c81e680b",
+ "sha256:cbb0c0acd650c0ffb620999da611e93aae5105c46a084c4ceaf2f704ed708c1e"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
+ "version": "==0.3.1"
+ },
+ "ppft": {
+ "hashes": [
+ "sha256:07166097d7dd45af7b98859654390d579d11dadf20780f6baca4bded3f55a580",
+ "sha256:47e0dab87a516c0b9992cd5b0c908348e4c7d964304d106b227fad28ae03219e"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
+ "version": "==1.7.6.5"
+ },
+ "protobuf": {
+ "hashes": [
+ "sha256:06059eb6953ff01e56a25cd02cca1a9649a75a7e65397b5b9b4e929ed71d10cf",
+ "sha256:097c5d8a9808302fb0da7e20edf0b8d4703274d140fd25c5edabddcde43e081f",
+ "sha256:284f86a6207c897542d7e956eb243a36bb8f9564c1742b253462386e96c6b78f",
+ "sha256:32ca378605b41fd180dfe4e14d3226386d8d1b002ab31c969c366549e66a2bb7",
+ "sha256:3cc797c9d15d7689ed507b165cd05913acb992d78b379f6014e013f9ecb20996",
+ "sha256:62f1b5c4cd6c5402b4e2d63804ba49a327e0c386c99b1675c8a0fefda23b2067",
+ "sha256:69ccfdf3657ba59569c64295b7d51325f91af586f8d5793b734260dfe2e94e2c",
+ "sha256:6f50601512a3d23625d8a85b1638d914a0970f17920ff39cec63aaef80a93fb7",
+ "sha256:7403941f6d0992d40161aa8bb23e12575637008a5a02283a930addc0508982f9",
+ "sha256:755f3aee41354ae395e104d62119cb223339a8f3276a0cd009ffabfcdd46bb0c",
+ "sha256:77053d28427a29987ca9caf7b72ccafee011257561259faba8dd308fda9a8739",
+ "sha256:7e371f10abe57cee5021797126c93479f59fccc9693dafd6bd5633ab67808a91",
+ "sha256:9016d01c91e8e625141d24ec1b20fed584703e527d28512aa8c8707f105a683c",
+ "sha256:9be73ad47579abc26c12024239d3540e6b765182a91dbc88e23658ab71767153",
+ "sha256:adc31566d027f45efe3f44eeb5b1f329da43891634d61c75a5944e9be6dd42c9",
+ "sha256:adfc6cf69c7f8c50fd24c793964eef18f0ac321315439d94945820612849c388",
+ "sha256:af0ebadc74e281a517141daad9d0f2c5d93ab78e9d455113719a45a49da9db4e",
+ "sha256:cb29edb9eab15742d791e1025dd7b6a8f6fcb53802ad2f6e3adcb102051063ab",
+ "sha256:cd68be2559e2a3b84f517fb029ee611546f7812b1fdd0aa2ecc9bc6ec0e4fdde",
+ "sha256:cdee09140e1cd184ba9324ec1df410e7147242b94b5f8b0c64fc89e38a8ba531",
+ "sha256:db977c4ca738dd9ce508557d4fce0f5aebd105e158c725beec86feb1f6bc20d8",
+ "sha256:dd5789b2948ca702c17027c84c2accb552fc30f4622a98ab5c51fcfe8c50d3e7",
+ "sha256:e250a42f15bf9d5b09fe1b293bdba2801cd520a9f5ea2d7fb7536d4441811d20",
+ "sha256:ff8d8fa42675249bb456f5db06c00de6c2f4c27a065955917b28c4f15978b9c3"
+ ],
+ "index": "pypi",
+ "version": "==3.20.1"
+ },
+ "protobuf3-to-dict": {
+ "hashes": [
+ "sha256:1e42c25b5afb5868e3a9b1962811077e492c17557f9c66f0fe40d821375d2b5a"
+ ],
+ "version": "==0.1.5"
+ },
+ "psutil": {
+ "hashes": [
+ "sha256:072664401ae6e7c1bfb878c65d7282d4b4391f1bc9a56d5e03b5a490403271b5",
+ "sha256:1070a9b287846a21a5d572d6dddd369517510b68710fca56b0e9e02fd24bed9a",
+ "sha256:1d7b433519b9a38192dfda962dd8f44446668c009833e1429a52424624f408b4",
+ "sha256:3151a58f0fbd8942ba94f7c31c7e6b310d2989f4da74fcbf28b934374e9bf841",
+ "sha256:32acf55cb9a8cbfb29167cd005951df81b567099295291bcfd1027365b36591d",
+ "sha256:3611e87eea393f779a35b192b46a164b1d01167c9d323dda9b1e527ea69d697d",
+ "sha256:3d00a664e31921009a84367266b35ba0aac04a2a6cad09c550a89041034d19a0",
+ "sha256:4e2fb92e3aeae3ec3b7b66c528981fd327fb93fd906a77215200404444ec1845",
+ "sha256:539e429da49c5d27d5a58e3563886057f8fc3868a5547b4f1876d9c0f007bccf",
+ "sha256:55ce319452e3d139e25d6c3f85a1acf12d1607ddedea5e35fb47a552c051161b",
+ "sha256:58c7d923dc209225600aec73aa2c4ae8ea33b1ab31bc11ef8a5933b027476f07",
+ "sha256:7336292a13a80eb93c21f36bde4328aa748a04b68c13d01dfddd67fc13fd0618",
+ "sha256:742c34fff804f34f62659279ed5c5b723bb0195e9d7bd9907591de9f8f6558e2",
+ "sha256:7641300de73e4909e5d148e90cc3142fb890079e1525a840cf0dfd39195239fd",
+ "sha256:76cebf84aac1d6da5b63df11fe0d377b46b7b500d892284068bacccf12f20666",
+ "sha256:7779be4025c540d1d65a2de3f30caeacc49ae7a2152108adeaf42c7534a115ce",
+ "sha256:7d190ee2eaef7831163f254dc58f6d2e2a22e27382b936aab51c835fc080c3d3",
+ "sha256:8293942e4ce0c5689821f65ce6522ce4786d02af57f13c0195b40e1edb1db61d",
+ "sha256:869842dbd66bb80c3217158e629d6fceaecc3a3166d3d1faee515b05dd26ca25",
+ "sha256:90a58b9fcae2dbfe4ba852b57bd4a1dded6b990a33d6428c7614b7d48eccb492",
+ "sha256:9b51917c1af3fa35a3f2dabd7ba96a2a4f19df3dec911da73875e1edaf22a40b",
+ "sha256:b2237f35c4bbae932ee98902a08050a27821f8f6dfa880a47195e5993af4702d",
+ "sha256:c3400cae15bdb449d518545cbd5b649117de54e3596ded84aacabfbb3297ead2",
+ "sha256:c51f1af02334e4b516ec221ee26b8fdf105032418ca5a5ab9737e8c87dafe203",
+ "sha256:cb8d10461c1ceee0c25a64f2dd54872b70b89c26419e147a05a10b753ad36ec2",
+ "sha256:d62a2796e08dd024b8179bd441cb714e0f81226c352c802fca0fd3f89eeacd94",
+ "sha256:df2c8bd48fb83a8408c8390b143c6a6fa10cb1a674ca664954de193fdcab36a9",
+ "sha256:e5c783d0b1ad6ca8a5d3e7b680468c9c926b804be83a3a8e95141b05c39c9f64",
+ "sha256:e9805fed4f2a81de98ae5fe38b75a74c6e6ad2df8a5c479594c7629a1fe35f56",
+ "sha256:ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3",
+ "sha256:ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c",
+ "sha256:ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3"
+ ],
+ "index": "pypi",
+ "version": "==5.9.0"
+ },
+ "py": {
+ "hashes": [
+ "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719",
+ "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+ "version": "==1.11.0"
+ },
+ "pyasn1": {
+ "hashes": [
+ "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359",
+ "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576",
+ "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf",
+ "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7",
+ "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d",
+ "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00",
+ "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8",
+ "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86",
+ "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12",
+ "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776",
+ "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba",
+ "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2",
+ "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"
+ ],
+ "index": "pypi",
+ "version": "==0.4.8"
+ },
+ "pycodestyle": {
+ "hashes": [
+ "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20",
+ "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+ "version": "==2.8.0"
+ },
+ "pycparser": {
+ "hashes": [
+ "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",
+ "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"
+ ],
+ "version": "==2.21"
+ },
+ "pydocstyle": {
+ "hashes": [
+ "sha256:1d41b7c459ba0ee6c345f2eb9ae827cab14a7533a88c5c6f7e94923f72df92dc",
+ "sha256:6987826d6775056839940041beef5c08cc7e3d71d63149b48e36727f70144dc4"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==6.1.1"
+ },
+ "pyflakes": {
+ "hashes": [
+ "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c",
+ "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==2.4.0"
+ },
+ "pynacl": {
+ "hashes": [
+ "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858",
+ "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d",
+ "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93",
+ "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1",
+ "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92",
+ "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff",
+ "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba",
+ "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394",
+ "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b",
+ "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==1.5.0"
+ },
+ "pyparsing": {
+ "hashes": [
+ "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
+ "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"
+ ],
+ "markers": "python_full_version >= '3.6.8'",
+ "version": "==3.0.9"
+ },
+ "pyrsistent": {
+ "hashes": [
+ "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c",
+ "sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc",
+ "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e",
+ "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26",
+ "sha256:636ce2dc235046ccd3d8c56a7ad54e99d5c1cd0ef07d9ae847306c91d11b5fec",
+ "sha256:6455fc599df93d1f60e1c5c4fe471499f08d190d57eca040c0ea182301321286",
+ "sha256:6bc66318fb7ee012071b2792024564973ecc80e9522842eb4e17743604b5e045",
+ "sha256:7bfe2388663fd18bd8ce7db2c91c7400bf3e1a9e8bd7d63bf7e77d39051b85ec",
+ "sha256:7ec335fc998faa4febe75cc5268a9eac0478b3f681602c1f27befaf2a1abe1d8",
+ "sha256:914474c9f1d93080338ace89cb2acee74f4f666fb0424896fcfb8d86058bf17c",
+ "sha256:b568f35ad53a7b07ed9b1b2bae09eb15cdd671a5ba5d2c66caee40dbf91c68ca",
+ "sha256:cdfd2c361b8a8e5d9499b9082b501c452ade8bbf42aef97ea04854f4a3f43b22",
+ "sha256:d1b96547410f76078eaf66d282ddca2e4baae8964364abb4f4dcdde855cd123a",
+ "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96",
+ "sha256:d7a096646eab884bf8bed965bad63ea327e0d0c38989fc83c5ea7b8a87037bfc",
+ "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1",
+ "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07",
+ "sha256:e4f3149fd5eb9b285d6bfb54d2e5173f6a116fe19172686797c056672689daf6",
+ "sha256:e92a52c166426efbe0d1ec1332ee9119b6d32fc1f0bbfd55d5c1088070e7fc1b",
+ "sha256:f87cc2863ef33c709e237d4b5f4502a62a00fab450c9e020892e8e2ede5847f5",
+ "sha256:fd8da6d0124efa2f67d86fa70c851022f87c98e205f0594e1fae044e7119a5a6"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==0.18.1"
+ },
+ "pytest": {
+ "hashes": [
+ "sha256:841132caef6b1ad17a9afde46dc4f6cfa59a05f9555aae5151f73bdf2820ca63",
+ "sha256:92f723789a8fdd7180b6b06483874feca4c48a5c76968e03bb3e7f806a1869ea"
+ ],
+ "index": "pypi",
+ "version": "==7.1.1"
+ },
+ "pytest-cov": {
+ "hashes": [
+ "sha256:1a629dc9f48e53512fcbfda6b07de490c374b0c83c55ff7a1720b3fccff0ac87",
+ "sha256:6e6d18092dce6fad667cd7020deed816f858ad3b49d5b5e2b1cc1c97a4dba65c"
+ ],
+ "index": "pypi",
+ "version": "==2.10.0"
+ },
+ "pytest-forked": {
+ "hashes": [
+ "sha256:8b67587c8f98cbbadfdd804539ed5455b6ed03802203485dd2f53c1422d7440e",
+ "sha256:bbbb6717efc886b9d64537b41fb1497cfaf3c9601276be8da2cccfea5a3c8ad8"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==1.4.0"
+ },
+ "pytest-parallel": {
+ "hashes": [
+ "sha256:9aac3fc199a168c0a8559b60249d9eb254de7af58c12cee0310b54d4affdbfab",
+ "sha256:9e3703015b0eda52be9e07d2ba3498f09340a56d5c79a39b50f22fc5c38212fe"
+ ],
+ "index": "pypi",
+ "version": "==0.1.1"
+ },
+ "pytest-rerunfailures": {
+ "hashes": [
+ "sha256:9e1e1bad51e07642c5bbab809fc1d4ec8eebcb7de86f90f1a26e6ef9de446697",
+ "sha256:d31d8e828dfd39363ad99cd390187bf506c7a433a89f15c3126c7d16ab723fe2"
+ ],
+ "index": "pypi",
+ "version": "==10.2"
+ },
+ "pytest-xdist": {
+ "hashes": [
+ "sha256:4580deca3ff04ddb2ac53eba39d76cb5dd5edeac050cb6fbc768b0dd712b4edf",
+ "sha256:6fe5c74fec98906deb8f2d2b616b5c782022744978e7bd4695d39c8f42d0ce65"
+ ],
+ "index": "pypi",
+ "version": "==2.5.0"
+ },
+ "python-dateutil": {
+ "hashes": [
+ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+ "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+ "version": "==2.8.2"
+ },
+ "python-dotenv": {
+ "hashes": [
+ "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f",
+ "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938"
+ ],
+ "markers": "python_version >= '3.5'",
+ "version": "==0.20.0"
+ },
+ "pytz": {
+ "hashes": [
+ "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197",
+ "sha256:cea221417204f2d1a2aa03ddae3e867921971d0d76f14d87abb4414415bbdcf5"
+ ],
+ "version": "==2022.2.1"
+ },
+ "pyyaml": {
+ "hashes": [
+ "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf",
+ "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696",
+ "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393",
+ "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77",
+ "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922",
+ "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5",
+ "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8",
+ "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10",
+ "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc",
+ "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018",
+ "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e",
+ "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253",
+ "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347",
+ "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183",
+ "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541",
+ "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb",
+ "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185",
+ "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc",
+ "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db",
+ "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa",
+ "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46",
+ "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122",
+ "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b",
+ "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63",
+ "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df",
+ "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc",
+ "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247",
+ "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6",
+ "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+ "version": "==5.4.1"
+ },
+ "requests": {
+ "hashes": [
+ "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61",
+ "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"
+ ],
+ "index": "pypi",
+ "version": "==2.27.1"
+ },
+ "rsa": {
+ "hashes": [
+ "sha256:7b863ff461f751373b4203dc09cfd07d92564575e8fafa45cc24fcde039153a0",
+ "sha256:d111bb8b92767d68c153cb248da8465c28ccdec449a266b76ac7ada0b48d6bbe"
+ ],
+ "index": "pypi",
+ "version": "==4.3"
+ },
+ "s3transfer": {
+ "hashes": [
+ "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971",
+ "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==0.5.2"
+ },
+ "safety": {
+ "hashes": [
+ "sha256:30e394d02a20ac49b7f65292d19d38fa927a8f9582cdfd3ad1adbbc66c641ad5",
+ "sha256:5f802ad5df5614f9622d8d71fedec2757099705c2356f862847c58c6dfe13e84"
+ ],
+ "index": "pypi",
+ "version": "==1.10.3"
+ },
+ "sagemaker": {
+ "hashes": [
+ "sha256:6e4c7f483e06532e196f30962bc4a3b3e1934b589fc771aeb7ae3e58623ae937"
+ ],
+ "index": "pypi",
+ "version": "==2.83.0"
+ },
+ "setuptools": {
+ "hashes": [
+ "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82",
+ "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==65.3.0"
+ },
+ "six": {
+ "hashes": [
+ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+ "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+ "version": "==1.16.0"
+ },
+ "smdebug-rulesconfig": {
+ "hashes": [
+ "sha256:104da3e6931ecf879dfc687ca4bbb3bee5ea2bc27f4478e9dbb3ee3655f1ae61",
+ "sha256:7a19e6eb2e6bcfefbc07e4a86ef7a88f32495001a038bf28c7d8e77ab793fcd6"
+ ],
+ "markers": "python_version >= '2.7'",
+ "version": "==1.0.1"
+ },
+ "smspark": {
+ "editable": true,
+ "path": "."
+ },
+ "snowballstemmer": {
+ "hashes": [
+ "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1",
+ "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"
+ ],
+ "version": "==2.2.0"
+ },
+ "tblib": {
+ "hashes": [
+ "sha256:059bd77306ea7b419d4f76016aef6d7027cc8a0785579b5aad198803435f882c",
+ "sha256:289fa7359e580950e7d9743eab36b0691f0310fce64dee7d9c31065b8f723e23"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+ "version": "==1.7.0"
+ },
+ "tenacity": {
+ "hashes": [
+ "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f",
+ "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a"
+ ],
+ "index": "pypi",
+ "version": "==8.0.1"
+ },
+ "texttable": {
+ "hashes": [
+ "sha256:42ee7b9e15f7b225747c3fa08f43c5d6c83bc899f80ff9bae9319334824076e9",
+ "sha256:dd2b0eaebb2a9e167d1cefedab4700e5dcbdb076114eed30b58b97ed6b37d6f2"
+ ],
+ "version": "==1.6.4"
+ },
+ "toml": {
+ "hashes": [
+ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
+ "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
+ ],
+ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
+ "version": "==0.10.2"
+ },
+ "tomli": {
+ "hashes": [
+ "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+ "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
+ ],
+ "markers": "python_version < '3.11'",
+ "version": "==2.0.1"
+ },
+ "types-requests": {
+ "hashes": [
+ "sha256:2437a5f4d16c0c8bd7539a8126d492b7aeb41e6cda670d76b286c7f83a658d42",
+ "sha256:c8010c18b291a7efb60b1452dbe12530bc25693dd657e70c62803fcdc4bffe9b"
+ ],
+ "index": "pypi",
+ "version": "==2.27.16"
+ },
+ "types-urllib3": {
+ "hashes": [
+ "sha256:333e675b188a1c1fd980b4b352f9e40572413a4c1ac689c23cd546e96310070a",
+ "sha256:b78e819f0e350221d0689a5666162e467ba3910737bafda14b5c2c85e9bb1e56"
+ ],
+ "version": "==1.26.23"
+ },
+ "types-waitress": {
+ "hashes": [
+ "sha256:d7843d13487effb0e0774ec294f42ca63ed9f74a9296b47e4e290ddb21a05292",
+ "sha256:fdd57199a5a7b5b3e65973feb137964bd750cdb1af4f7cc7c9d6053342f86ff2"
+ ],
+ "index": "pypi",
+ "version": "==2.0.6"
+ },
+ "typing-extensions": {
+ "hashes": [
+ "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42",
+ "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"
+ ],
+ "index": "pypi",
+ "version": "==4.1.1"
+ },
+ "urllib3": {
+ "hashes": [
+ "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e",
+ "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
+ "version": "==1.26.12"
+ },
+ "waitress": {
+ "hashes": [
+ "sha256:7500c9625927c8ec60f54377d590f67b30c8e70ef4b8894214ac6e4cad233d2a",
+ "sha256:780a4082c5fbc0fde6a2fcfe5e26e6efc1e8f425730863c04085769781f51eba"
+ ],
+ "index": "pypi",
+ "version": "==2.1.2"
+ },
+ "watchdog": {
+ "hashes": [
+ "sha256:4214e1379d128b0588021880ccaf40317ee156d4603ac388b9adcf29165e0c04"
+ ],
+ "index": "pypi",
+ "version": "==0.10.3"
+ },
+ "websocket-client": {
+ "hashes": [
+ "sha256:2e50d26ca593f70aba7b13a489435ef88b8fc3b5c5643c1ce8808ff9b40f0b32",
+ "sha256:d376bd60eace9d437ab6d7ee16f4ab4e821c9dae591e1b783c58ebd8aaf80c5c"
+ ],
+ "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==0.59.0"
+ },
+ "zipp": {
+ "hashes": [
+ "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2",
+ "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==3.8.1"
+ }
+ },
+ "develop": {}
+}
diff --git a/new_images.yml b/new_images.yml
index 2da32fd..ed93865 100644
--- a/new_images.yml
+++ b/new_images.yml
@@ -1,7 +1,7 @@
---
new_images:
- - spark: "3.1.1"
+ - spark: "3.2"
use-case: "processing"
processors: ["cpu"]
python: ["py39"]
- sm_version: "1.3"
+ sm_version: "1.0"
diff --git a/scripts/fetch-ec2-instance-type-info.sh b/scripts/fetch-ec2-instance-type-info.sh
index 056af07..2989d1c 100755
--- a/scripts/fetch-ec2-instance-type-info.sh
+++ b/scripts/fetch-ec2-instance-type-info.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..5802b8b
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,48 @@
+# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import glob
+import os
+
+from setuptools import find_packages, setup
+
+with open("VERSION", "r") as version_file:
+ version = version_file.read()
+
+setup(
+ name="smspark",
+ description="Library that enables running Spark Processing jobs on Amazon SageMaker",
+ version=version,
+ python_requires=">3.7.0",
+ packages=find_packages("src"),
+ package_dir={"": "src"},
+ py_modules=[os.path.splitext(os.path.basename(path))[0] for path in glob.glob("src/smspark/*.py")],
+ author="Amazon Web Services",
+ url="https://github.com/aws/smspark/",
+ license="Apache License 2.0",
+ keywords="ML Amazon AWS AI SageMaker Processing Spark",
+ classifiers=[
+ "Development Status :: 3 - Alpha",
+ "Intended Audience :: Developers",
+ "Natural Language :: English",
+ "License :: OSI Approved :: Apache Software License",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3.9",
+ ],
+ setup_requires=["setuptools", "wheel"],
+ entry_points={
+ "console_scripts": [
+ "smspark-submit=smspark.cli:submit_main",
+ "smspark-history-server=smspark.history_server_cli:run_history_server",
+ ]
+ },
+)
diff --git a/spark/processing/3.2/py3/container-bootstrap-config/bootstrap.sh b/spark/processing/3.2/py3/container-bootstrap-config/bootstrap.sh
new file mode 100644
index 0000000..e4e23bb
--- /dev/null
+++ b/spark/processing/3.2/py3/container-bootstrap-config/bootstrap.sh
@@ -0,0 +1 @@
+echo "Not implemented"
\ No newline at end of file
diff --git a/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu b/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
new file mode 100644
index 0000000..db9d795
--- /dev/null
+++ b/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
@@ -0,0 +1,128 @@
+FROM 137112412989.dkr.ecr.us-west-2.amazonaws.com/amazonlinux:2
+ARG REGION
+ENV AWS_REGION ${REGION}
+
+RUN yum clean all \
+ && yum update -y \
+ && yum install -y awscli bigtop-utils curl gcc gzip unzip zip gunzip tar wget liblapack* libblas* libopencv* libopenblas*
+
+# Install python 3.9
+ARG PYTHON_BASE_VERSION=3.9
+ARG PYTHON_WITH_BASE_VERSION=python${PYTHON_BASE_VERSION}
+ARG PIP_WITH_BASE_VERSION=pip${PYTHON_BASE_VERSION}
+ARG PYTHON_VERSION=${PYTHON_BASE_VERSION}.12
+RUN yum -y groupinstall 'Development Tools' \
+ && yum -y install openssl-devel bzip2-devel libffi-devel sqlite-devel xz-devel \
+ && wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
+ && tar xzf Python-${PYTHON_VERSION}.tgz \
+ && cd Python-*/ \
+ && ./configure --enable-optimizations \
+ && make altinstall \
+ && echo -e 'alias python3=python3.9\nalias pip3=pip3.9' >> ~/.bashrc \
+ && ln -s $(which ${PYTHON_WITH_BASE_VERSION}) /usr/local/bin/python3 \
+ && ln -s $(which ${PIP_WITH_BASE_VERSION}) /usr/local/bin/pip3 \
+ && cd .. \
+ && rm Python-${PYTHON_VERSION}.tgz \
+ && rm -rf Python-${PYTHON_VERSION}
+
+# install nginx amazonlinux:2.0.20200304.0 does not have nginx, so need to install epel-release first
+RUN wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+RUN yum install -y epel-release-latest-7.noarch.rpm
+RUN yum install -y nginx
+
+RUN rm -rf /var/cache/yum
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+# http://blog.stuart.axelbrooke.com/python-3-on-spark-return-of-the-pythonhashseed
+ENV PYTHONHASHSEED 0
+ENV PYTHONIOENCODING UTF-8
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+
+# Install EMR Spark/Hadoop
+ENV HADOOP_HOME /usr/lib/hadoop
+ENV HADOOP_CONF_DIR /usr/lib/hadoop/etc/hadoop
+ENV SPARK_HOME /usr/lib/spark
+
+COPY yum/emr-apps.repo /etc/yum.repos.d/emr-apps.repo
+
+# Install hadoop / spark dependencies from EMR's yum repository for Spark optimizations.
+# replace placeholder with region in repository URL
+RUN sed -i "s/REGION/${AWS_REGION}/g" /etc/yum.repos.d/emr-apps.repo
+RUN adduser -N hadoop
+
+# These packages are a subset of what EMR installs in a cluster with the
+# "hadoop", "spark", and "hive" applications.
+# They include EMR-optimized libraries and extras.
+RUN yum install -y aws-hm-client \
+ aws-java-sdk \
+ aws-sagemaker-spark-sdk \
+ emr-goodies \
+ emr-ruby \
+ emr-scripts \
+ emr-s3-select \
+ emrfs \
+ hadoop \
+ hadoop-client \
+ hadoop-hdfs \
+ hadoop-hdfs-datanode \
+ hadoop-hdfs-namenode \
+ hadoop-httpfs \
+ hadoop-kms \
+ hadoop-lzo \
+ hadoop-yarn \
+ hadoop-yarn-nodemanager \
+ hadoop-yarn-proxyserver \
+ hadoop-yarn-resourcemanager \
+ hadoop-yarn-timelineserver \
+ hive \
+ hive-hcatalog \
+ hive-hcatalog-server \
+ hive-jdbc \
+ hive-server2 \
+ s3-dist-cp \
+ spark-core \
+ spark-datanucleus \
+ spark-external \
+ spark-history-server \
+ spark-python
+
+
+# Point Spark at proper python binary
+ENV PYSPARK_PYTHON=/usr/local/bin/python3.9
+
+# Setup Spark/Yarn/HDFS user as root
+ENV PATH="/usr/bin:/opt/program:${PATH}"
+ENV YARN_RESOURCEMANAGER_USER="root"
+ENV YARN_NODEMANAGER_USER="root"
+ENV HDFS_NAMENODE_USER="root"
+ENV HDFS_DATANODE_USER="root"
+ENV HDFS_SECONDARYNAMENODE_USER="root"
+
+
+
+# Set up bootstrapping program and Spark configuration
+COPY hadoop-config /opt/hadoop-config
+COPY nginx-config /opt/nginx-config
+COPY aws-config /opt/aws-config
+COPY Pipfile Pipfile.lock setup.py *.whl /opt/program/
+ENV PIPENV_PIPFILE=/opt/program/Pipfile
+# Use --system flag, so it will install all packages into the system python,
+# and not into the virtualenv. Since docker containers do not need to have virtualenvs
+# pipenv > 2022.4.8 fails to build smspark
+RUN /usr/local/bin/python3.9 -m pip install pipenv==2022.4.8 \
+ && pipenv install --system \
+ && /usr/local/bin/python3.9 -m pip install /opt/program/*.whl
+
+# Setup container bootstrapper
+COPY container-bootstrap-config /opt/container-bootstrap-config
+RUN chmod +x /opt/container-bootstrap-config/bootstrap.sh \
+ && /opt/container-bootstrap-config/bootstrap.sh
+
+# With this config, spark history server will not run as daemon, otherwise there
+# will be no server running and container will terminate immediately
+ENV SPARK_NO_DAEMONIZE TRUE
+
+WORKDIR $SPARK_HOME
+
+ENTRYPOINT ["smspark-submit"]
diff --git a/spark/processing/3.2/py3/hadoop-config/core-site.xml b/spark/processing/3.2/py3/hadoop-config/core-site.xml
new file mode 100644
index 0000000..52db7b2
--- /dev/null
+++ b/spark/processing/3.2/py3/hadoop-config/core-site.xml
@@ -0,0 +1,26 @@
+
+
+
+
+
+
+ fs.defaultFS
+ hdfs://nn_uri/
+ NameNode URI
+
+
+ fs.s3a.aws.credentials.provider
+ com.amazonaws.auth.DefaultAWSCredentialsProviderChain
+ AWS S3 credential provider
+
+
+ fs.s3.impl
+ org.apache.hadoop.fs.s3a.S3AFileSystem
+ s3a filesystem implementation
+
+
+ fs.AbstractFileSystem.s3a.imp
+ org.apache.hadoop.fs.s3a.S3A
+ s3a filesystem implementation
+
+
diff --git a/spark/processing/3.2/py3/hadoop-config/hdfs-site.xml b/spark/processing/3.2/py3/hadoop-config/hdfs-site.xml
new file mode 100644
index 0000000..37e0a5b
--- /dev/null
+++ b/spark/processing/3.2/py3/hadoop-config/hdfs-site.xml
@@ -0,0 +1,67 @@
+
+
+
+
+
+
+ dfs.datanode.data.dir
+ file:///opt/amazon/hadoop/hdfs/datanode
+ Comma separated list of paths on the local filesystem of a DataNode where it should store its\
+ blocks.
+
+
+
+ dfs.namenode.name.dir
+ file:///opt/amazon/hadoop/hdfs/namenode
+ Path on the local filesystem where the NameNode stores the namespace and transaction logs per\
+ sistently.
+
+
+
+
+ dfs.client.block.write.replace-datanode-on-failure.enable
+ true
+
+ If there is a datanode/network failure in the write pipeline,
+ DFSClient will try to remove the failed datanode from the pipeline
+ and then continue writing with the remaining datanodes. As a result,
+ the number of datanodes in the pipeline is decreased. The feature is
+ to add new datanodes to the pipeline.
+
+ This is a site-wide property to enable/disable the feature.
+
+ When the cluster size is extremely small, e.g. 3 nodes or less, cluster
+ administrators may want to set the policy to NEVER in the default
+ configuration file or disable this feature. Otherwise, users may
+ experience an unusually high rate of pipeline failures since it is
+ impossible to find new datanodes for replacement.
+
+ See also dfs.client.block.write.replace-datanode-on-failure.policy
+
+
+
+
+ dfs.client.block.write.replace-datanode-on-failure.policy
+ ALWAYS
+
+ This property is used only if the value of
+ dfs.client.block.write.replace-datanode-on-failure.enable is true.
+
+ ALWAYS: always add a new datanode when an existing datanode is
+ removed.
+
+ NEVER: never add a new datanode.
+
+ DEFAULT:
+ Let r be the replication number.
+ Let n be the number of existing datanodes.
+ Add a new datanode only if r is greater than or equal to 3 and either
+ (1) floor(r/2) is greater than or equal to n; or
+ (2) r is greater than n and the block is hflushed/appended.
+
+
+
diff --git a/spark/processing/3.2/py3/hadoop-config/spark-defaults.conf b/spark/processing/3.2/py3/hadoop-config/spark-defaults.conf
new file mode 100644
index 0000000..734086a
--- /dev/null
+++ b/spark/processing/3.2/py3/hadoop-config/spark-defaults.conf
@@ -0,0 +1,10 @@
+spark.driver.extraClassPath /usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop/hadoop-aws.jar:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/goodies/lib/emr-spark-goodies.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*:/usr/share/aws/hmclient/lib/aws-glue-datacatalog-spark-client.jar:/usr/share/java/Hive-JSON-Serde/hive-openx-serde.jar:/usr/share/aws/sagemaker-spark-sdk/lib/sagemaker-spark-sdk.jar:/usr/share/aws/emr/s3select/lib/emr-s3-select-spark-connector.jar
+spark.driver.extraLibraryPath /usr/lib/hadoop/lib/native:/usr/lib/hadoop-lzo/lib/native
+spark.executor.extraClassPath /usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop/hadoop-aws.jar:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/goodies/lib/emr-spark-goodies.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*:/usr/share/aws/hmclient/lib/aws-glue-datacatalog-spark-client.jar:/usr/share/java/Hive-JSON-Serde/hive-openx-serde.jar:/usr/share/aws/sagemaker-spark-sdk/lib/sagemaker-spark-sdk.jar:/usr/share/aws/emr/s3select/lib/emr-s3-select-spark-connector.jar
+spark.executor.extraLibraryPath /usr/lib/hadoop/lib/native:/usr/lib/hadoop-lzo/lib/native
+spark.driver.host=sd_host
+spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version=2
+
+# Fix for "Uncaught exception: org.apache.spark.rpc.RpcTimeoutException: Cannot
+# receive any reply from 10.0.109.30:35219 in 120 seconds.""
+spark.rpc.askTimeout=300s
diff --git a/spark/processing/3.2/py3/hadoop-config/spark-env.sh b/spark/processing/3.2/py3/hadoop-config/spark-env.sh
new file mode 100644
index 0000000..1b58aa1
--- /dev/null
+++ b/spark/processing/3.2/py3/hadoop-config/spark-env.sh
@@ -0,0 +1,3 @@
+#EMPTY FILE AVOID OVERRIDDING ENV VARS
+# Specifically, without copying the empty file, SPARK_HISTORY_OPTS will be overriden,
+# spark.history.ui.port defaults to 18082, and spark.eventLog.dir defaults to local fs
diff --git a/spark/processing/3.2/py3/hadoop-config/yarn-site.xml b/spark/processing/3.2/py3/hadoop-config/yarn-site.xml
new file mode 100644
index 0000000..1c92988
--- /dev/null
+++ b/spark/processing/3.2/py3/hadoop-config/yarn-site.xml
@@ -0,0 +1,34 @@
+
+
+
+
+ yarn.resourcemanager.hostname
+ rm_hostname
+ The hostname of the RM.
+
+
+ yarn.nodemanager.hostname
+ nm_hostname
+ The hostname of the NM.
+
+
+ yarn.nodemanager.webapp.address
+ nm_webapp_address
+
+
+ yarn.nodemanager.vmem-pmem-ratio
+ 5
+ Ratio between virtual memory to physical memory.
+
+
+ yarn.resourcemanager.am.max-attempts
+ 1
+ The maximum number of application attempts.
+
+
+ yarn.nodemanager.env-whitelist
+ JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_HOME,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_REGION
+ Environment variable whitelist
+
+
+
diff --git a/spark/processing/3.2/py3/nginx-config/default.conf b/spark/processing/3.2/py3/nginx-config/default.conf
new file mode 100644
index 0000000..a8a50a5
--- /dev/null
+++ b/spark/processing/3.2/py3/nginx-config/default.conf
@@ -0,0 +1,17 @@
+server {
+ listen 15050;
+ server_name localhost;
+ client_header_buffer_size 128k;
+ large_client_header_buffers 4 128k;
+
+ location ~ ^/history/(.*)/(.*)/jobs/$ {
+ proxy_pass http://localhost:18080/history/$1/jobs/;
+ proxy_redirect http://localhost:18080/history/$1/jobs/ $domain_name/proxy/15050/history/$1/jobs/;
+ expires off;
+ }
+
+ location / {
+ proxy_pass http://localhost:18080;
+ expires off;
+ }
+}
\ No newline at end of file
diff --git a/spark/processing/3.2/py3/nginx-config/nginx.conf b/spark/processing/3.2/py3/nginx-config/nginx.conf
new file mode 100644
index 0000000..1e3a51c
--- /dev/null
+++ b/spark/processing/3.2/py3/nginx-config/nginx.conf
@@ -0,0 +1,66 @@
+# For more information on configuration, see:
+# * Official English Documentation: http://nginx.org/en/docs/
+# * Official Russian Documentation: http://nginx.org/ru/docs/
+
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log;
+pid /run/nginx.pid;
+
+# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
+include /usr/share/nginx/modules/*.conf;
+
+events {
+ worker_connections 1024;
+}
+
+http {
+ log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+ '$status $body_bytes_sent "$http_referer" '
+ '"$http_user_agent" "$http_x_forwarded_for"';
+
+ access_log /var/log/nginx/access.log main;
+
+ sendfile on;
+ tcp_nopush on;
+ tcp_nodelay on;
+ keepalive_timeout 65;
+ types_hash_max_size 2048;
+
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+
+ # Load modular configuration files from the /etc/nginx/conf.d directory.
+ # See http://nginx.org/en/docs/ngx_core_module.html#include
+ # for more information.
+ include /etc/nginx/conf.d/*.conf;
+
+ server {
+ listen 80 default_server;
+ listen [::]:80 default_server;
+ server_name _;
+ root /usr/share/nginx/html;
+
+ # Load configuration files for the default server block.
+ include /etc/nginx/default.d/*.conf;
+
+ location /proxy/15050 {
+ proxy_pass http://localhost:15050/;
+ }
+
+ location ~ ^/proxy/15050/(.*) {
+ proxy_pass http://localhost:15050/$1;
+ }
+
+ location / {
+ }
+
+ error_page 404 /404.html;
+ location = /40x.html {
+ }
+
+ error_page 500 502 503 504 /50x.html;
+ location = /50x.html {
+ }
+ }
+}
\ No newline at end of file
diff --git a/spark/processing/3.2/py3/yum/emr-apps.repo b/spark/processing/3.2/py3/yum/emr-apps.repo
new file mode 100644
index 0000000..66e03f8
--- /dev/null
+++ b/spark/processing/3.2/py3/yum/emr-apps.repo
@@ -0,0 +1,17 @@
+[emr-apps]
+name = EMR Application Repository
+gpgkey = https://s3-REGION.amazonaws.com/repo.REGION.emr.amazonaws.com/apps-repository/emr-6.7.0/947ef300-59e5-4093-bc24-944efa7271df-v2/repoPublicKey.txt
+enabled = 1
+baseurl = https://s3-REGION.amazonaws.com/repo.REGION.emr.amazonaws.com/apps-repository/emr-6.7.0/947ef300-59e5-4093-bc24-944efa7271df-v2
+priority = 5
+gpgcheck = 0
+
+[emr-puppet]
+mirrorlist: http://amazonlinux.$awsregion.$awsdomain/$releasever/extras/emr-puppet/latest/$basearch/mirror.list
+enabled: 1
+gpgcheck: 1
+name: Amazon Extras repo for emr-puppet
+gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-amazon-linux-2
+priority: 10
+skip_if_unavailable: 1
+report_instanceid: yes
From e0b042ff344b259aadf55815096d580367818c27 Mon Sep 17 00:00:00 2001
From: xiaoxshe <109769013+xiaoxshe@users.noreply.github.com>
Date: Fri, 26 Aug 2022 10:35:50 -0700
Subject: [PATCH 2/8] fix previous commit format error by running black
---
DEVELOPMENT.md | 30 ++++++++++++++++++++++++++++--
src/smspark/bootstrapper.py | 12 +++++++-----
src/smspark/constants.py | 3 +--
3 files changed, 36 insertions(+), 9 deletions(-)
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
index 92b3904..3634cd9 100644
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@@ -126,14 +126,27 @@ docker push $AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/$SPARK_REPOSITORY:$V
make test-sagemaker
```
+6. please run following command before you raise CR:
+
+```
+make test-unit
+make install-container-library
+```
+
+
## Push the code
1. You need to create PR request in order to merge the code. How to create PR request lists here:https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request
2. You need to get Github access of AWS organization. please following here:https://w.amazon.com/?Open_Source/GitHub
3. Get access to permission specific to a team, example is here:https://github.com/orgs/aws/teams/sagemakerwrite/members
4. Ask a person to review the code and merge it in.This repo needs at least one code reviewer.
5. The code needs to be signed before pushing. More detail about signing is here:https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits.
-Remember in your local, you need to set up: git config --global user.signingkey [key id] and also upload public key into your github account.
-6. The email you specify when you created public key must match github email in github settings.
+
+```
+$ git commit -S -m "your commit message"
+```
+
+6.Remember in your local, you need to set up: git config --global user.signingkey [key id] and also upload public key into your github account.
+The email you specify when you created public key must match github email in github settings.
### FAQ
@@ -168,3 +181,16 @@ make: *** [install-container-library] Error 255
```
* you need to update smsparkbuild/py39/Pipfile corresponding package version.
+
+6. Code build may fail because of the format,
+for example
+```
+2 files would be reformatted, 13 files would be left unchanged.
+```
+
+you can fix it by running
+
+```
+black src/smspark/bootstrapper.py
+```
+see https://www.freecodecamp.org/news/auto-format-your-python-code-with-black/ for detail.
\ No newline at end of file
diff --git a/src/smspark/bootstrapper.py b/src/smspark/bootstrapper.py
index bf13cce..9c3cdcf 100644
--- a/src/smspark/bootstrapper.py
+++ b/src/smspark/bootstrapper.py
@@ -239,7 +239,7 @@ def start_spark_standalone_primary(self) -> None:
subprocess.Popen(cmd_start_primary, shell=True)
def deserialize_user_configuration(
- self, configuration_dict_or_list: Union[Dict[str, Any], List[Dict[str, Any]]]
+ self, configuration_dict_or_list: Union[Dict[str, Any], List[Dict[str, Any]]]
) -> Union[Sequence[Configuration], Configuration]:
if isinstance(configuration_dict_or_list, dict):
return self.deserialize_user_configuration_dict(configuration_dict_or_list)
@@ -247,7 +247,7 @@ def deserialize_user_configuration(
return self._deserialize_user_configuration_to_sequence(configuration_dict_or_list)
def _deserialize_user_configuration_to_sequence(
- self, configuration_list: List[Dict[str, Any]]
+ self, configuration_list: List[Dict[str, Any]]
) -> Sequence[Configuration]:
return [self.deserialize_user_configuration_dict(conf) for conf in configuration_list]
@@ -373,7 +373,7 @@ def set_yarn_spark_resource_config(self) -> None:
logging.info("Configuration at {} is: \n{}".format(spark_config.path, spark_config_string))
def get_yarn_spark_resource_config(
- self, instance_count: int, instance_mem_mb: int, instance_cores: int
+ self, instance_count: int, instance_mem_mb: int, instance_cores: int
) -> Tuple[Configuration, Configuration]:
aws_region = os.getenv("AWS_REGION")
executor_cores = instance_cores
@@ -383,8 +383,10 @@ def get_yarn_spark_resource_config(
driver_mem_mb = int(instance_mem_mb * constants.DRIVER_MEM_INSTANCE_MEM_RATIO)
driver_mem_overhead_mb = int(driver_mem_mb * constants.DRIVER_MEM_OVERHEAD_RATIO)
- executor_mem_mb = int(((instance_mem_mb * constants.EXECUTOR_MEM_INSTANCE_MEM_RATIO)
- / executor_count_per_instance) * (1 - constants.EXECUTOR_MEM_OVERHEAD_RATIO))
+ executor_mem_mb = int(
+ ((instance_mem_mb * constants.EXECUTOR_MEM_INSTANCE_MEM_RATIO) / executor_count_per_instance)
+ * (1 - constants.EXECUTOR_MEM_OVERHEAD_RATIO)
+ )
executor_mem_overhead_mb = int(executor_mem_mb * constants.EXECUTOR_MEM_OVERHEAD_RATIO)
driver_gc_config = (
diff --git a/src/smspark/constants.py b/src/smspark/constants.py
index 3896481..62cae5a 100644
--- a/src/smspark/constants.py
+++ b/src/smspark/constants.py
@@ -1,5 +1,4 @@
-
DRIVER_MEM_INSTANCE_MEM_RATIO = 0.9
DRIVER_MEM_OVERHEAD_RATIO = 0.1
EXECUTOR_MEM_INSTANCE_MEM_RATIO = 0.95
-EXECUTOR_MEM_OVERHEAD_RATIO = 0.1
\ No newline at end of file
+EXECUTOR_MEM_OVERHEAD_RATIO = 0.1
From b6c50bf378c66bbb147e35344bc1153e0fc00063 Mon Sep 17 00:00:00 2001
From: xiaoxshe <109769013+xiaoxshe@users.noreply.github.com>
Date: Fri, 26 Aug 2022 10:59:25 -0700
Subject: [PATCH 3/8] fix missing docstring in public module
---
DEVELOPMENT.md | 6 +++++-
src/smspark/constants.py | 1 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
index 3634cd9..e72f86b 100644
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@@ -193,4 +193,8 @@ you can fix it by running
```
black src/smspark/bootstrapper.py
```
-see https://www.freecodecamp.org/news/auto-format-your-python-code-with-black/ for detail.
\ No newline at end of file
+see https://www.freecodecamp.org/news/auto-format-your-python-code-with-black/ for detail.
+
+7. Remember to define module at start of python file. Missing docstring error.
+
+see more detail here https://stackoverflow.com/questions/46192576/how-can-i-fix-flake8-d100-missing-docstring-error-in-atom-editor
diff --git a/src/smspark/constants.py b/src/smspark/constants.py
index 62cae5a..d2de860 100644
--- a/src/smspark/constants.py
+++ b/src/smspark/constants.py
@@ -1,3 +1,4 @@
+"""Configuration to define driver and executor memory."""
DRIVER_MEM_INSTANCE_MEM_RATIO = 0.9
DRIVER_MEM_OVERHEAD_RATIO = 0.1
EXECUTOR_MEM_INSTANCE_MEM_RATIO = 0.95
From 55ba750eabc1ddc0d25447b0cef1b91f22d86cd9 Mon Sep 17 00:00:00 2001
From: xiaoxshe <109769013+xiaoxshe@users.noreply.github.com>
Date: Fri, 26 Aug 2022 14:09:09 -0700
Subject: [PATCH 4/8] fix epel release error
---
spark/processing/3.2/py3/docker/py39/Dockerfile.cpu | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu b/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
index db9d795..e386e56 100644
--- a/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
+++ b/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
@@ -26,8 +26,8 @@ RUN yum -y groupinstall 'Development Tools' \
&& rm -rf Python-${PYTHON_VERSION}
# install nginx amazonlinux:2.0.20200304.0 does not have nginx, so need to install epel-release first
-RUN wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-RUN yum install -y epel-release-latest-7.noarch.rpm
+RUN wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+RUN yum install -y epel-release-latest-9.noarch.rpm
RUN yum install -y nginx
RUN rm -rf /var/cache/yum
From 556bd8b29e003917a24eaaab7c09283d6b79a815 Mon Sep 17 00:00:00 2001
From: xiaoxshe <109769013+xiaoxshe@users.noreply.github.com>
Date: Fri, 26 Aug 2022 14:27:34 -0700
Subject: [PATCH 5/8] fix epel release error
---
spark/processing/3.2/py3/docker/py39/Dockerfile.cpu | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu b/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
index e386e56..19d6e1c 100644
--- a/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
+++ b/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
@@ -26,8 +26,7 @@ RUN yum -y groupinstall 'Development Tools' \
&& rm -rf Python-${PYTHON_VERSION}
# install nginx amazonlinux:2.0.20200304.0 does not have nginx, so need to install epel-release first
-RUN wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
-RUN yum install -y epel-release-latest-9.noarch.rpm
+RUN yum install epel-release
RUN yum install -y nginx
RUN rm -rf /var/cache/yum
From 2a538b422e93f1a90698e4e1c7521961fbc29373 Mon Sep 17 00:00:00 2001
From: xiaoxshe <109769013+xiaoxshe@users.noreply.github.com>
Date: Fri, 26 Aug 2022 15:48:00 -0700
Subject: [PATCH 6/8] revert back epel change
---
spark/processing/3.2/py3/docker/py39/Dockerfile.cpu | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu b/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
index 19d6e1c..d6bd8b8 100644
--- a/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
+++ b/spark/processing/3.2/py3/docker/py39/Dockerfile.cpu
@@ -26,7 +26,8 @@ RUN yum -y groupinstall 'Development Tools' \
&& rm -rf Python-${PYTHON_VERSION}
# install nginx amazonlinux:2.0.20200304.0 does not have nginx, so need to install epel-release first
-RUN yum install epel-release
+RUN wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
RUN yum install -y nginx
RUN rm -rf /var/cache/yum
From 6b41ebdf213c2233c3a802aeaddcc243fe53d3c0 Mon Sep 17 00:00:00 2001
From: xiaoxshe <109769013+xiaoxshe@users.noreply.github.com>
Date: Tue, 30 Aug 2022 08:37:58 -0700
Subject: [PATCH 7/8] change for CR feedback
---
scripts/fetch-ec2-instance-type-info.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/fetch-ec2-instance-type-info.sh b/scripts/fetch-ec2-instance-type-info.sh
index 2989d1c..056af07 100755
--- a/scripts/fetch-ec2-instance-type-info.sh
+++ b/scripts/fetch-ec2-instance-type-info.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
From 3d1df4e7b3553828d2fe2a37a3905440a76ea195 Mon Sep 17 00:00:00 2001
From: xiaoxshe <109769013+xiaoxshe@users.noreply.github.com>
Date: Tue, 30 Aug 2022 08:41:10 -0700
Subject: [PATCH 8/8] delete setup.py
---
setup.py | 48 ------------------------------------------------
1 file changed, 48 deletions(-)
delete mode 100644 setup.py
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 5802b8b..0000000
--- a/setup.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-# http://aws.amazon.com/apache2.0/
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-import glob
-import os
-
-from setuptools import find_packages, setup
-
-with open("VERSION", "r") as version_file:
- version = version_file.read()
-
-setup(
- name="smspark",
- description="Library that enables running Spark Processing jobs on Amazon SageMaker",
- version=version,
- python_requires=">3.7.0",
- packages=find_packages("src"),
- package_dir={"": "src"},
- py_modules=[os.path.splitext(os.path.basename(path))[0] for path in glob.glob("src/smspark/*.py")],
- author="Amazon Web Services",
- url="https://github.com/aws/smspark/",
- license="Apache License 2.0",
- keywords="ML Amazon AWS AI SageMaker Processing Spark",
- classifiers=[
- "Development Status :: 3 - Alpha",
- "Intended Audience :: Developers",
- "Natural Language :: English",
- "License :: OSI Approved :: Apache Software License",
- "Programming Language :: Python",
- "Programming Language :: Python :: 3.9",
- ],
- setup_requires=["setuptools", "wheel"],
- entry_points={
- "console_scripts": [
- "smspark-submit=smspark.cli:submit_main",
- "smspark-history-server=smspark.history_server_cli:run_history_server",
- ]
- },
-)