Update ci and pycodestyle

.python-version

@@ -1,3 +1,2 @@
-2.7.17
-3.6.8
-3.8.1
+3.6.15
+3.8.16

awscurl/awscurl.py

@@ -198,7 +198,7 @@ def task_1_create_a_canonical_request(
     # Step 4: Create the canonical headers and signed headers. Header names
     # and value must be trimmed and lowercase, and sorted in ASCII order.
     # Note that there is a trailing \n.
-    canonical_headers = ('host:' + fullhost + '\n' +
+    canonical_headers = ('host:' + fullhost + '\n' + 
                          'x-amz-date:' + amzdate + '\n')
 
     if security_token:

ci-in-docker.sh

@@ -5,10 +5,11 @@ set -o pipefail
 set -o nounset
 set -o xtrace
 
-images=("ubuntu" "centos")
+docker build -t awscurl-ci-ubuntu -f ./ci/ci-ubuntu/Dockerfile .
+docker run   -t awscurl-ci-ubuntu bash -c "source /root/venv/bin/activate && cd dd && tox" 
 
-for image in "${images[@]}"
-do
-#  echo $image
-  docker run -it -v $(pwd):/root "${image}" bash -c "cd && ./install.sh && ./ci.sh"
-done
+docker build -t awscurl-ci-alpine -f ./ci/ci-alpine/Dockerfile .
+docker run   -t awscurl-ci-alpine bash -c "source /root/venv/bin/activate && cd dd && tox" 
+
+docker build -t awscurl-ci-centos -f ./ci/ci-centos/Dockerfile .
+docker run   -t awscurl-ci-centos bash -c "source /root/venv/bin/activate && cd dd && tox" 

ci/ci-alpine/Dockerfile

@@ -0,0 +1,36 @@
+FROM alpine
+
+# RUN echo 'tzdata tzdata/Areas select Europe' | debconf-set-selections && \
+#     echo 'tzdata tzdata/Zones/Europe select Paris' | debconf-set-selections && \
+#     apk add --no-cache tzdata && \
+#     cp /usr/share/zoneinfo/Europe/Paris /etc/localtime && \
+#     echo "Europe/Paris" > /etc/timezone && \
+#     apk del tzdata
+
+RUN apk update && \
+    apk add sudo curl git build-base autoconf automake libtool \
+    openssl-dev readline-dev zlib-dev sqlite-dev ncurses-dev \
+    xz-dev tk-dev libffi-dev bzip2-dev bash gcc make musl-dev \
+    libffi-dev openssl-dev zlib-dev readline-dev sqlite-dev 
+
+RUN curl https://pyenv.run | bash
+
+ENV PATH="/root/.pyenv/bin:$PATH"
+
+RUN eval "$(pyenv init -)" && \
+    eval "$(pyenv virtualenv-init -)"
+
+COPY .python-version .python-version
+RUN cat .python-version | xargs -n 1 /root/.pyenv/bin/pyenv install
+
+RUN /root/.pyenv/versions/3.8*/bin/python -m venv /root/venv
+
+RUN bash -c "source /root/venv/bin/activate && pip install tox"
+
+COPY awscurl dd/awscurl
+COPY tox.ini dd/tox.ini
+COPY requirements.txt dd/requirements.txt
+COPY requirements-test.txt dd/requirements-test.txt
+COPY tests dd/tests
+
+RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"

ci/ci-centos/Dockerfile

@@ -0,0 +1,28 @@
+FROM tgagor/centos
+
+RUN yum update -y
+RUN yum group install -y "Development Tools"
+RUN yum install -y libffi-devel readline-devel zlib-devel bzip2-devel sqlite-devel openssl-devel git
+
+
+RUN curl https://pyenv.run | bash
+
+ENV PATH="/root/.pyenv/bin:$PATH"
+
+RUN eval "$(pyenv init -)" && \
+    eval "$(pyenv virtualenv-init -)"
+
+COPY .python-version .python-version
+RUN cat .python-version | xargs -n 1 /root/.pyenv/bin/pyenv install
+
+RUN /root/.pyenv/versions/3.8*/bin/python -m venv /root/venv
+
+RUN bash -c "source /root/venv/bin/activate && pip install tox"
+
+COPY awscurl dd/awscurl
+COPY tox.ini dd/tox.ini
+COPY requirements.txt dd/requirements.txt
+COPY requirements-test.txt dd/requirements-test.txt
+COPY tests dd/tests
+
+RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"

ci/ci-ubuntu/Dockerfile

@@ -0,0 +1,38 @@
+FROM ubuntu
+
+RUN apt update
+RUN apt install -y sudo
+RUN echo 'tzdata tzdata/Areas select Europe' | debconf-set-selections
+RUN echo 'tzdata tzdata/Zones/Europe select Paris' | debconf-set-selections
+RUN DEBIAN_FRONTEND="noninteractive" apt install -y tzdata
+
+RUN apt install -y curl git \
+    build-essential \
+    autoconf \
+    automake \
+    libtool \
+    libffi-dev libreadline-dev libz-dev libsqlite-dev libssl-dev \
+    libreadline-dev libsqlite3-dev wget curl libncurses5-dev libncursesw5-dev \
+    xz-utils tk-dev libffi-dev libbz2-dev liblzma-dev git
+
+RUN curl https://pyenv.run | bash
+
+ENV PATH="/root/.pyenv/bin:$PATH"
+
+RUN eval "$(pyenv init -)" && \
+    eval "$(pyenv virtualenv-init -)"
+
+COPY .python-version .python-version
+RUN cat .python-version | xargs -n 1 /root/.pyenv/bin/pyenv install
+
+RUN /root/.pyenv/versions/3.8*/bin/python -m venv /root/venv
+
+RUN bash -c "source /root/venv/bin/activate && pip install tox"
+
+COPY awscurl dd/awscurl
+COPY tox.ini dd/tox.ini
+COPY requirements.txt dd/requirements.txt
+COPY requirements-test.txt dd/requirements-test.txt
+COPY tests dd/tests
+
+RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate"

install.sh

@@ -11,6 +11,9 @@ OS_RELEASE=$(. /etc/os-release; echo "${NAME}")
 if [ "${OS_RELEASE}" = "Ubuntu" ]; then
     apt update
     apt install -y sudo
+    echo 'tzdata tzdata/Areas select Europe' | debconf-set-selections
+    echo 'tzdata tzdata/Zones/Europe select Paris' | debconf-set-selections
+    DEBIAN_FRONTEND="noninteractive" apt install -y tzdata
 
     apt install -y curl git \
     build-essential \
@@ -19,7 +22,7 @@ if [ "${OS_RELEASE}" = "Ubuntu" ]; then
     libtool \
     libffi-dev libreadline-dev libz-dev libsqlite-dev libssl-dev \
     libreadline-dev libsqlite3-dev wget curl libncurses5-dev libncursesw5-dev \
-    xz-utils tk-dev libffi-dev liblzma-dev python-openssl git
+    xz-utils tk-dev libffi-dev libbz2-dev liblzma-dev git
 elif [ "${OS_RELEASE}" = "CentOS Linux" ]; then
     yum update -y
     yum group install -y "Development Tools"

tests/integration_test.py

@@ -19,10 +19,8 @@
 sys.path.append(extra_path)
 print(f'sys.path2={sys.path}')
 
-import pytest
-from mock import patch
+from awscurl.awscurl import make_request, inner_main  # nopep8: E402
 
-from awscurl.awscurl import make_request, inner_main
 
 __author__ = 'iokulist'
 

tests/load_aws_config_test.py

@@ -16,15 +16,15 @@ def test(self):
                                                            "./tests/data/credentials",
                                                            "default")
 
-        self.assertEquals([access_key, secret_access, token], ['access_key_id', 'secret_access_key', None])
+        self.assertEqual([access_key, secret_access, token], ['access_key_id', 'secret_access_key', None])
 
         access_key, secret_access, token = load_aws_config(None,
                                                            None,
                                                            "ttt",
                                                            "./tests/data/credentials",
                                                            "default")
 
-        self.assertEquals([access_key, secret_access, token], ['access_key_id', 'secret_access_key', 'ttt'])
+        self.assertEqual([access_key, secret_access, token], ['access_key_id', 'secret_access_key', 'ttt'])
 
         # TODO: remove this test as I think it's not valid to loads secret_key if session_key was already provided
         # access_key, secret_access, token = load_aws_config('aaa',

tests/stages_test.py

@@ -11,6 +11,7 @@
     task_3_calculate_the_signature,
     task_4_build_auth_headers_for_the_request)
 
+
 class TestStages(TestCase):
     """
     Suite to test all stages.

tests/unit_test.py

@@ -128,6 +128,7 @@ def test_make_request(self, *args, **kvargs):
 
         pass
 
+
 class TestMakeRequestVerifySSLPass(TestCase):
     maxDiff = None
 
@@ -160,33 +161,34 @@ def test_make_request(self, *args, **kvargs):
 
 
 class TestMakeRequestWithBinaryData(TestCase):
-  maxDiff = None
-
-  @patch('requests.get', new_callable=my_mock_get)
-  @patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request)
-  @patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
-  def test_make_request(self, *args, **kvargs):
-    headers = {}
-    params = {'method': 'GET',
-              'service': 'ec2',
-              'region': 'region',
-              'uri': 'https://user:pass@host:123/path/?a=b&c=d',
-              'headers': headers,
-              'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v',
-              'access_key': '',
-              'secret_key': '',
-              'security_token': '',
-              'data_binary': True}
-    make_request(**params)
-
-    expected = {'x-amz-date': '19700101T000000Z',
-                'Authorization': 'AWS4-HMAC-SHA256 Credential=/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-date, Signature=6ebcf316c9bb50bb7b2bbabf128dddde3babbf16badfd31ddc40838e7592d5df',
-                'x-amz-content-sha256': '3f514228bd64bbff67daaa80e482aee0e0b0c51891d3a64e4abfa145f4364b99',
-                'x-amz-security-token': ''}
-
-    self.assertEqual(expected, headers)
+    maxDiff = None
+
+    @patch('requests.get', new_callable=my_mock_get)
+    @patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request)
+    @patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow)
+    def test_make_request(self, *args, **kvargs):
+        headers = {}
+        params = {'method': 'GET',
+                  'service': 'ec2',
+                  'region': 'region',
+                  'uri': 'https://user:pass@host:123/path/?a=b&c=d',
+                  'headers': headers,
+                  'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v',
+                  'access_key': '',
+                  'secret_key': '',
+                  'security_token': '',
+                  'data_binary': True}
+        make_request(**params)
+
+        expected = {'x-amz-date': '19700101T000000Z',
+                    'Authorization': 'AWS4-HMAC-SHA256 Credential=/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-date, Signature=6ebcf316c9bb50bb7b2bbabf128dddde3babbf16badfd31ddc40838e7592d5df',
+                    'x-amz-content-sha256': '3f514228bd64bbff67daaa80e482aee0e0b0c51891d3a64e4abfa145f4364b99',
+                    'x-amz-security-token': ''}
+
+        self.assertEqual(expected, headers)
+
+        pass
 
-    pass
 
 class TestMakeRequestWithToken(TestCase):
     maxDiff = None
@@ -217,6 +219,7 @@ def test_make_request(self, *args, **kvargs):
 
         pass
 
+
 class TestMakeRequestWithTokenAndBinaryData(TestCase):
     maxDiff = None
 
@@ -246,6 +249,7 @@ def test_make_request(self, *args, **kvargs):
 
         pass
 
+
 class TestHostFromHeaderUsedInCanonicalHeader(TestCase):
     maxDiff = None
 
@@ -276,54 +280,56 @@ def test_make_request(self, *args, **kvargs):
 
         pass
 
+
 class TestRequestResponse(TestCase):
-  maxDiff = None
-
-  @patch('awscurl.awscurl.__send_request')
-  def test_make_request(self, mocked_resp):
-    resp = Response()
-    resp.status_code=200
-    resp._content = b'{"file_name": "test.yml", "env": "staging", "hash": "\xe5\xad\x97"}'
-    resp.encoding = 'UTF-8'
-    mocked_resp.return_value = resp
-
-    headers = {}
-    params = {'method': 'GET',
-              'service': 'ec2',
-              'region': 'region',
-              'uri': 'https://user:pass@host:123/path/?a=b&c=d',
-              'headers': headers,
-              'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v',
-              'access_key': '',
-              'secret_key': '',
-              'security_token': '',
-              'data_binary': True}
-    r = make_request(**params)
-
-    expected = u'\u5b57'
-
-    ### assert that the unicode character is in the response.text output
-    self.assertTrue(expected in r.text)
-
-    ### assert that the unicode character is _not_ in the response.text.encode('utf-8')
-    ### which has been converted to 8-bit string with unicode characters escaped
-    ### in py2 this raises an exception on the assertion (`expected in x` below)
-    ### in py3 we can compare the two directly, and the assertion should be false
-    if sys.version_info[0] == 2:
-        with self.assertRaises(UnicodeDecodeError):
-            x = str(r.text.encode('utf-8'))
-            expected in x
-    else:
-        self.assertFalse(expected in str(r.text.encode('utf-8')))
+    maxDiff = None
+
+    @patch('awscurl.awscurl.__send_request')
+    def test_make_request(self, mocked_resp):
+        resp = Response()
+        resp.status_code=200
+        resp._content = b'{"file_name": "test.yml", "env": "staging", "hash": "\xe5\xad\x97"}'
+        resp.encoding = 'UTF-8'
+        mocked_resp.return_value = resp
+
+        headers = {}
+        params = {'method': 'GET',
+                  'service': 'ec2',
+                  'region': 'region',
+                  'uri': 'https://user:pass@host:123/path/?a=b&c=d',
+                  'headers': headers,
+                  'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v',
+                  'access_key': '',
+                  'secret_key': '',
+                  'security_token': '',
+                  'data_binary': True}
+        r = make_request(**params)
+
+        expected = u'\u5b57'
+
+        ### assert that the unicode character is in the response.text output
+        self.assertTrue(expected in r.text)
+
+        ### assert that the unicode character is _not_ in the response.text.encode('utf-8')
+        ### which has been converted to 8-bit string with unicode characters escaped
+        ### in py2 this raises an exception on the assertion (`expected in x` below)
+        ### in py3 we can compare the two directly, and the assertion should be false
+        if sys.version_info[0] == 2:
+            with self.assertRaises(UnicodeDecodeError):
+                x = str(r.text.encode('utf-8'))
+                expected in x
+        else:
+            self.assertFalse(expected in str(r.text.encode('utf-8')))
+
+        pass
 
-    pass
 
 class TestAwsUrlEncode(TestCase):
-  def test_aws_url_encode(self):
-    self.assertEqual(aws_url_encode(""), "")
-    self.assertEqual(aws_url_encode("AZaz09-_.~"), "AZaz09-_.~")
-    self.assertEqual(aws_url_encode(" /:@[`{"), "%20%2F%3A%40%5B%60%7B")
-    self.assertEqual(aws_url_encode("a=,=b"), "a==%2C==b")
-    self.assertEqual(aws_url_encode("\u0394-\u30a1"), "%CE%94-%E3%82%A1")
-
-  pass
+    def test_aws_url_encode(self):
+        self.assertEqual(aws_url_encode(""), "")
+        self.assertEqual(aws_url_encode("AZaz09-_.~"), "AZaz09-_.~")
+        self.assertEqual(aws_url_encode(" /:@[`{"), "%20%2F%3A%40%5B%60%7B")
+        self.assertEqual(aws_url_encode("a=,=b"), "a==%2C==b")
+        self.assertEqual(aws_url_encode("\u0394-\u30a1"), "%CE%94-%E3%82%A1")
+
+    pass

tests/url_parsing_test.py

@@ -47,4 +47,3 @@ def test(self, *args, **kvargs):
                           'query': '',
                           'schema': 'http',
                           'user': None})
-