Added region localization, Integrate repository with CircleCI to enab…

…le package build visualization, Unit tests bug fixes for python3.5

.circleci/config.yml

@@ -0,0 +1,61 @@
+version: 2.1
+executors:
+  python:
+    parameters:
+      image:
+        type: string
+    docker:
+      - image: << parameters.image >>
+commands:
+  runtest:
+    steps:
+      - checkout
+      - restore_cache:
+          key: efs-utils-{{ .Branch }}-{{ checksum "requirements.txt" }}
+      - run:
+          name: Install Python dependencies in a virtual env
+          command: |
+            python -m pip install --user virtualenv
+            python -m virtualenv ~/.env/efs-utils
+            source ~/.env/efs-utils/bin/activate
+            pip install -r requirements.txt
+      - save_cache:
+          key: efs-utils-{{ .Branch }}-{{ checksum "requirements.txt" }}
+          paths:
+            - ~/.env/efs-utils
+      - run:
+          name: Run all tests
+          command: |
+            source ~/.env/efs-utils/bin/activate
+            make test
+jobs:
+  test:
+    parameters:
+      image:
+        type: string
+    executor:
+      name: python
+      image: << parameters.image >>
+    steps:
+      - runtest
+workflows:
+  workflow:
+    jobs:
+      - test:
+          name: python38
+          image: circleci/python:3.8.1
+      - test:
+          name: python37
+          image: circleci/python:3.7.3
+      - test:
+          name: python36
+          image: circleci/python:3.6.9
+      - test:
+          name: python35
+          image: circleci/python:3.5.9
+      - test:
+          name: python34
+          image: circleci/python:3.4.9
+      - test:
+          name: python27
+          image: circleci/python:2.7.13

build-deb.sh

@@ -11,7 +11,7 @@ set -ex
 
 BASE_DIR=$(pwd)
 BUILD_ROOT=${BASE_DIR}/build/debbuild
-VERSION=1.18
+VERSION=1.19
 
 echo 'Cleaning deb build workspace'
 rm -rf ${BUILD_ROOT}

config.ini

@@ -7,4 +7,4 @@
 #
 
 [global]
-version=1.18
+version=1.19

dist/amazon-efs-utils.control

@@ -1,6 +1,6 @@
 Package: amazon-efs-utils
 Architecture: all
-Version: 1.18
+Version: 1.19
 Section: utils
 Depends: python|python2, nfs-common, stunnel4 (>= 4.56)
 Priority: optional

dist/amazon-efs-utils.spec

@@ -20,7 +20,7 @@
 %endif
 
 Name      : amazon-efs-utils
-Version   : 1.18
+Version   : 1.19
 Release   : 1%{?dist}
 Summary   : This package provides utilities for simplifying the use of EFS file systems
 

dist/efs-utils.conf

@@ -29,6 +29,18 @@ stunnel_check_cert_validity = false
 port_range_lower_bound = 20049
 port_range_upper_bound = 20449
 
+[mount.cn-north-1]
+dns_name_suffix = amazonaws.com.cn
+
+[mount.cn-northwest-1]
+dns_name_suffix = amazonaws.com.cn
+
+[mount.us-iso-east-1]
+dns_name_suffix = c2s.ic.gov
+
+[mount.us-isob-east-1]
+dns_name_suffix = sc2s.sgov.gov
+
 [mount-watchdog]
 enabled = true
 poll_interval_sec = 1

src/mount_efs/__init__.py

@@ -68,7 +68,7 @@
     from urllib.error import URLError
     from urllib.request import urlopen
 
-VERSION = '1.18'
+VERSION = '1.19'
 SERVICE = 'elasticfilesystem'
 
 CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'

src/watchdog/__init__.py

@@ -46,7 +46,7 @@
     from urllib.error import URLError
     from urllib.request import urlopen
 
-VERSION = '1.18'
+VERSION = '1.19'
 SERVICE = 'elasticfilesystem'
 
 CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'

test/__init__.py

@@ -0,0 +1,7 @@
+#
+# Copyright 2017-2018 Amazon.com, Inc. and its affiliates. All Rights Reserved.
+#
+# Licensed under the MIT License. See the LICENSE accompanying this file
+# for the specific language governing permissions and limitations under
+# the License.
+#

test/mount_efs_test/test_check_network_status.py

@@ -12,6 +12,8 @@
 
 from mock import MagicMock
 
+from .. import utils
+
 FS_ID = 'fs-deadbeef'
 
 
@@ -25,15 +27,15 @@ def test_non_systemd(mocker):
 
     mount_efs.check_network_status(FS_ID, 'init')
 
-    call_mock.assert_not_called()
+    utils.assert_not_called(call_mock)
 
 
 def test_systemd_network_up(mocker):
     call_mock = _mock_subprocess_call(mocker)
 
     mount_efs.check_network_status(FS_ID, 'systemd')
 
-    call_mock.assert_called_once()
+    utils.assert_called_once(call_mock)
 
 
 def test_systemd_network_down(mocker):
@@ -42,5 +44,5 @@ def test_systemd_network_down(mocker):
     with pytest.raises(SystemExit) as ex:
         mount_efs.check_network_status(FS_ID, 'systemd')
 
-    call_mock.assert_called_once()
+    utils.assert_called_once(call_mock)
     assert 0 == ex.value.code

test/mount_efs_test/test_get_dns_name.py

@@ -13,10 +13,17 @@
 
 from mock import MagicMock
 
+from .. import utils
+
 FS_ID = 'fs-deadbeef'
 DEFAULT_REGION = 'us-east-1'
-SPECIAL_DNS_NAME_SUFFIX = 'amazonaws.com.cn'
-SPECIAL_REGION = 'cn-north-1'
+SPECIAL_REGION_DNS_DICT = {
+    "cn-north-1": "amazonaws.com.cn",
+    "cn-northwest-1": "amazonaws.com.cn",
+    "us-iso-east-1": "c2s.ic.gov",
+    "us-isob-east-1": "sc2s.sgov.gov"
+}
+SPECIAL_REGIONS = ["cn-north-1", "cn-northwest-1", "us-iso-east-1", "us-isob-east-1"]
 
 
 @pytest.fixture(autouse=True)
@@ -64,7 +71,7 @@ def test_get_dns_name_region_hardcoded(mocker):
 
     dns_name = mount_efs.get_dns_name(config, FS_ID)
 
-    get_region_mock.assert_not_called()
+    utils.assert_not_called(get_region_mock)
 
     assert '%s.efs.%s.amazonaws.com' % (FS_ID, DEFAULT_REGION) == dns_name
 
@@ -76,7 +83,7 @@ def test_get_dns_name_region_and_suffix_hardcoded(mocker):
 
     dns_name = mount_efs.get_dns_name(config, FS_ID)
 
-    get_region_mock.assert_not_called()
+    utils.assert_not_called(get_region_mock)
 
     assert '%s.elastic-file-system.us-west-2.amazonaws.com' % FS_ID == dns_name
 
@@ -123,25 +130,32 @@ def test_get_dns_name_unresolvable(mocker, capsys):
 
 
 def test_get_dns_name_special_region(mocker):
-    mocker.patch('mount_efs.get_region', return_value=SPECIAL_REGION)
+    for special_region in SPECIAL_REGIONS:
+        mocker.patch('mount_efs.get_region', return_value=special_region)
 
-    config_section = 'mount.%s' % SPECIAL_REGION
-    config = _get_mock_config(dns_name_suffix=SPECIAL_DNS_NAME_SUFFIX, config_section=config_section)
-    config.has_section.return_value = True
+        config_section = 'mount.%s' % special_region
+        special_dns_name_suffix = SPECIAL_REGION_DNS_DICT[special_region]
 
-    dns_name = mount_efs.get_dns_name(config, FS_ID)
+        config = _get_mock_config(dns_name_suffix=special_dns_name_suffix, config_section=config_section)
+        config.has_section.return_value = True
+
+        dns_name = mount_efs.get_dns_name(config, FS_ID)
 
-    assert '%s.efs.%s.%s' % (FS_ID, SPECIAL_REGION, SPECIAL_DNS_NAME_SUFFIX) == dns_name
+        assert '%s.efs.%s.%s' % (FS_ID, special_region, special_dns_name_suffix) == dns_name
 
 
 def test_get_dns_name_region_in_suffix(mocker):
     get_region_mock = mocker.patch('mount_efs.get_region')
 
-    dns_name_suffix = '%s.%s' % (SPECIAL_REGION, SPECIAL_DNS_NAME_SUFFIX)
-    config = _get_mock_config('{fs_id}.efs.{dns_name_suffix}', dns_name_suffix=dns_name_suffix)
+    for special_region in SPECIAL_REGIONS:
+        special_dns_name_suffix = SPECIAL_REGION_DNS_DICT[special_region]
+        dns_name_suffix = '%s.%s' % (special_region, special_dns_name_suffix)
 
-    dns_name = mount_efs.get_dns_name(config, FS_ID)
+        config = _get_mock_config('{fs_id}.efs.{dns_name_suffix}', dns_name_suffix=dns_name_suffix)
+
+        dns_name = mount_efs.get_dns_name(config, FS_ID)
+
+        utils.assert_not_called(get_region_mock)
 
-    get_region_mock.assert_not_called()
+        assert '%s.efs.%s.%s' % (FS_ID, special_region, special_dns_name_suffix) == dns_name
 
-    assert '%s.efs.%s.%s' % (FS_ID, SPECIAL_REGION, SPECIAL_DNS_NAME_SUFFIX) == dns_name

test/mount_efs_test/test_main.py

@@ -14,6 +14,9 @@
 
 from mock import patch
 
+from .. import utils
+
+
 AP_ID = 'fsap-0123456789abcdef0'
 BAD_AP_ID_INCORRECT_START = 'bad-fsap-0123456789abc'
 BAD_AP_ID_TOO_SHORT = 'fsap-0123456789abcdef'
@@ -61,15 +64,15 @@ def _test_main(mocker, tls=False, root=True, ap_id=None, iam=False, awsprofile=N
 
     mount_efs.main()
 
-    bootstrap_logging_mock.assert_called_once()
-    get_dns_mock.assert_called_once()
-    parse_arguments_mock.assert_called_once()
-    mount_mock.assert_called_once()
+    utils.assert_called_once(bootstrap_logging_mock)
+    utils.assert_called_once(get_dns_mock)
+    utils.assert_called_once(parse_arguments_mock)
+    utils.assert_called_once(mount_mock)
 
     if tls:
-        bootstrap_tls_mock.assert_called_once()
+        utils.assert_called_once(bootstrap_tls_mock)
     else:
-        bootstrap_tls_mock.assert_not_called()
+        utils.assert_not_called(bootstrap_tls_mock)
 
 
 def _test_main_assert_error(mocker, capsys, expected_err, **kwargs):

test/mount_efs_test/test_match_device.py

@@ -11,6 +11,9 @@
 
 import mount_efs
 
+from .. import utils
+
+
 CORRECT_DEVICE_DESCRIPTORS_FS_ID = [
     ('fs-deadbeef', ('fs-deadbeef', '/')),
     ('fs-deadbeef:/', ('fs-deadbeef', '/')),
@@ -36,8 +39,8 @@ def test_match_device_correct_descriptors_cname_dns_primary(mocker):
                                          return_value=('fs-deadbeef.efs.us-east-1.amazonaws.com', [], None))
     for device, (fs_id, path) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS:
         assert (fs_id, path) == mount_efs.match_device(None, device)
-    get_dns_name_mock.assert_called()
-    gethostbyname_ex_mock.assert_called()
+    utils.assert_called(get_dns_name_mock)
+    utils.assert_called(gethostbyname_ex_mock)
 
 
 def test_match_device_correct_descriptors_cname_dns_secondary(mocker):
@@ -46,8 +49,8 @@ def test_match_device_correct_descriptors_cname_dns_secondary(mocker):
                                          return_value=(None, ['fs-deadbeef.efs.us-east-1.amazonaws.com'], None))
     for device, (fs_id, path) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS:
         assert (fs_id, path) == mount_efs.match_device(None, device)
-    get_dns_name_mock.assert_called()
-    gethostbyname_ex_mock.assert_called()
+    utils.assert_called(get_dns_name_mock)
+    utils.assert_called(gethostbyname_ex_mock)
 
 
 def test_match_device_correct_descriptors_cname_dns_tertiary(mocker):
@@ -56,8 +59,8 @@ def test_match_device_correct_descriptors_cname_dns_tertiary(mocker):
                                          return_value=(None, [None, 'fs-deadbeef.efs.us-east-1.amazonaws.com'], None))
     for device, (fs_id, path) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS:
         assert (fs_id, path) == mount_efs.match_device(None, device)
-    get_dns_name_mock.assert_called()
-    gethostbyname_ex_mock.assert_called()
+    utils.assert_called(get_dns_name_mock)
+    utils.assert_called(gethostbyname_ex_mock)
 
 
 def test_match_device_correct_descriptors_cname_dns_amongst_invalid(mocker):
@@ -70,8 +73,8 @@ def test_match_device_correct_descriptors_cname_dns_amongst_invalid(mocker):
     )
     for device, (fs_id, path) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS:
         assert (fs_id, path) == mount_efs.match_device(None, device)
-    get_dns_name_mock.assert_called()
-    gethostbyname_ex_mock.assert_called()
+    utils.assert_called(get_dns_name_mock)
+    utils.assert_called(gethostbyname_ex_mock)
 
 
 def test_match_device_unresolvable_domain(mocker, capsys):
@@ -93,7 +96,7 @@ def test_match_device_no_hostnames(mocker, capsys):
     assert 0 != ex.value.code
     out, err = capsys.readouterr()
     assert 'did not resolve to an EFS mount target' in err
-    gethostbyname_ex_mock.assert_called()
+    utils.assert_called(gethostbyname_ex_mock)
 
 
 def test_match_device_no_hostnames2(mocker, capsys):
@@ -105,7 +108,7 @@ def test_match_device_no_hostnames2(mocker, capsys):
     assert 0 != ex.value.code
     out, err = capsys.readouterr()
     assert 'did not resolve to an EFS mount target' in err
-    gethostbyname_ex_mock.assert_called()
+    utils.assert_called(gethostbyname_ex_mock)
 
 
 def test_match_device_resolve_to_invalid_efs_dns_name(mocker, capsys):
@@ -117,7 +120,7 @@ def test_match_device_resolve_to_invalid_efs_dns_name(mocker, capsys):
     assert 0 != ex.value.code
     out, err = capsys.readouterr()
     assert 'did not resolve to a valid DNS name' in err
-    gethostbyname_ex_mock.assert_called()
+    utils.assert_called(gethostbyname_ex_mock)
 
 
 def test_match_device_resolve_to_unexpected_efs_dns_name(mocker, capsys):
@@ -130,5 +133,5 @@ def test_match_device_resolve_to_unexpected_efs_dns_name(mocker, capsys):
     assert 0 != ex.value.code
     out, err = capsys.readouterr()
     assert 'did not resolve to a valid DNS name' in err
-    get_dns_name_mock.assert_called()
-    gethostbyname_ex_mock.assert_called()
+    utils.assert_called(get_dns_name_mock)
+    utils.assert_called(gethostbyname_ex_mock)