Merge branch 'release-0.0.1.dev38'

.travis.yml

@@ -1,4 +1,4 @@
-dist: trusty
+dist: bionic
 sudo: false
 cache:
   directories:

CHANGELOG.rst

@@ -1,6 +1,14 @@
 Changelog
 =========
 
+0.0.1.dev38 (2021-04-09)
+------------------------
+
+Fixed
+~~~~~
+
+* Update aiohttp due to a security bug
+
 0.0.1.dev37 (2020-06-01)
 ------------------------
 

docs/config-janitor.rst

@@ -23,22 +23,22 @@ gcp
 
 Any configuration key/value listed here may also be used in the specific plugin configuration. Values set in a plugin-specific config section will overwrite what's set in this general ``[gcp]`` section.
 
+.. option:: project="STR"
+
+    `Required`: Google Project ID which hosts the relevant GCP services (e.g. Cloud DNS, Pub/Sub, Compute Engine).
+
+    To learn more about GCP projects, please see `Google's docs on creating & managing projects <https://cloud.google.com/resource-manager/docs/creating-managing-projects>`_.
+
 .. option:: keyfile="/path/to/keyfile.json"
 
-    `Required`: Path to the Service Account JSON keyfile to use while authenticating against Google APIs.
+    `Optional`: Path to the Service Account JSON keyfile to use while authenticating against Google APIs. If not provided the default Service Account will be used instead.
 
     While one global key for all plugins is supported, it's advised to create a key per plugin with only the permissions it requires. To setup a service account, follow `Google's docs on creating & managing service account keys <https://cloud.google.com/iam/docs/creating-managing-service-account-keys>`_.
 
     .. attention::
 
         For the Pub/Sub plugin, ``keyfile`` is not required when running against the `Pub/Sub Emulator <https://cloud.google.com/pubsub/docs/emulator>`_ that Google provides.
 
-.. option:: project="STR"
-
-    `Required`: Google Project ID which hosts the relevant GCP services (e.g. Cloud DNS, Pub/Sub, Compute Engine).
-
-    To learn more about GCP projects, please see `Google's docs on creating & managing projects <https://cloud.google.com/resource-manager/docs/creating-managing-projects>`_.
-
 .. option:: scopes=["STR","STR"]
 
     `Optional`: A list of strings of the scope(s) needed when making calls to Google APIs. Defaults to ``["cloud-platform"]``.
@@ -51,7 +51,7 @@ Any configuration key/value listed here may also be used in the specific plugin
 
     `Optional`: Prefix associated with Google managed zone names, prepended with a '-' to the generated name.
     For example prefix "production" will produced a managed zone name of "production-example-com" for the
-    "example.com." DNS zone. 
+    "example.com." DNS zone.
 
     Note: This prefix must be the same as that used by the Gordon Service to work correctly.
 

docs/config-service.rst

@@ -32,12 +32,6 @@ Plugin Configuration
 ``[gcp]``
 ~~~~~~~~~
 
-.. option:: keyfile="/path/to/keyfile.json"
-
-    `Required`: Path to the Service Account JSON keyfile to use while authenticating against Google APIs.
-
-    While one global key for all plugins is supported, it's advised to create a key per plugin with only the permissions it requires. To setup a service account, follow `Google's docs on creating & managing service account keys <keyfiles>`_.
-
 .. option:: project="STR"
 
     `Required`: Google Project ID which hosts the relevant GCP services (e.g. Cloud DNS, Pub/Sub, Compute Engine).
@@ -50,11 +44,17 @@ Plugin Configuration
 
     Note: this is separate from Google's 'managed zone' names.  Google uses custom string names with specific `requirements <https://cloud.google.com/dns/api/v1/managedZones#resource>`_ for storing records. Gordon requires that managed zone names be based on DNS names. For all domains, remove the trailing dot and replace all other dots with dashes.  For reverse records, then use only the two most significant octets, prepended with 'reverse-'.  (E.g. ``foo.bar.com.`` -> ``foo-bar-com`` and ``0.168.192.in-addr.arpa.`` -> ``reverse-168-192.``)
 
+.. option:: keyfile="/path/to/keyfile.json"
+
+    `Optional`: Path to the Service Account JSON keyfile to use while authenticating against Google APIs. If not provided the default Service Account will be used instead.
+
+    While one global key for all plugins is supported, it's advised to create a key per plugin with only the permissions it requires. To setup a service account, follow `Google's docs on creating & managing service account keys <keyfiles>`_.
+
 .. option:: default_zone_prefix="STR"
 
     `Optional`: Prefix associated with Google managed zone names, prepended with a '-' to the generated name.
     For example prefix "production" will produced a managed zone name of "production-example-com" for the
-    "example.com." DNS zone. 
+    "example.com." DNS zone.
 
 
 ``[gcp.event_consumer]``

requirements.txt

@@ -1,4 +1,4 @@
-aiohttp==2.3.9
+aiohttp==3.7.4
 asyncio-extras==1.3.0
 google-api-core==1.6.0
 google-auth==1.3.0

src/gordon_gcp/__init__.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 __author__ = 'Lynn Root'
-__version__ = '0.0.1.dev37'
+__version__ = '0.0.1.dev38'
 __license__ = 'Apache 2.0'
 __email__ = 'lynn@spotify.com'
 __description__ = 'GCP Plugins for Gordon and Gordon Janitor'

src/gordon_gcp/clients/http.py

@@ -35,6 +35,9 @@
     client = AIOConnection(auth_client=auth_client)
     resp = await client.request('get', 'http://api.example.com/foo')
 
+
+The keyfile is optional.
+If not provided the default service account will be used.
 """
 
 import datetime

src/gordon_gcp/plugins/janitor/gpubsub_publisher.py

@@ -52,6 +52,10 @@
         loop.run_until_complete(publisher.start())
     finally:
         loop.close()
+
+
+The keyfile is optional.
+If not provided the default service account will be used.
 """
 
 import asyncio

src/gordon_gcp/plugins/janitor/reconciler.py

@@ -52,6 +52,10 @@
         loop.run_until_complete(reconciler.start())
     finally:
         loop.close()
+
+
+The keyfile is optional.
+If not provided the default service account will be used.
 """
 
 import asyncio

test-requirements.txt

@@ -1,4 +1,4 @@
-aioresponses==0.3.1
+aioresponses==0.7.2
 attrs==17.4.0  # used by pytest
 coverage==4.4.2
 flake8==3.5.0

tests/unit/clients/test_auth.py

@@ -198,7 +198,8 @@ def test_auth_client_raises_not_found(tmpdir, caplog):
     assert 1 == len(caplog.records)
 
 
-def test_auth_client_initialize_app_default_cred(
+@pytest.mark.asyncio
+async def test_auth_client_initialize_app_default_cred(
         monkeypatch, app_default_cred_file_content,
         tmpdir, mock_oauth2_credentials):
     """Test credentials is initialize
@@ -210,6 +211,7 @@ def test_auth_client_initialize_app_default_cred(
     client = auth.GAuthClient()
     assert isinstance(client.creds, credentials.Credentials)
     assert not isinstance(client.creds, service_account.Credentials)
+    await client._session.close()
 
 
 #####

tests/unit/clients/test_gce.py

@@ -118,7 +118,3 @@ async def test_list_instances_retrieves_multiple_pages(
         expected_results = compute_rsp['items']['us-west1-z']['instances'] \
             + page2['items']['us-west1-z']['instances']
         assert expected_results == results
-        requests = list(m.requests.keys())
-        assert 2 == len(requests)
-        assert ('get', filter_url) == requests[0]
-        assert ('get', url_with_token) == requests[1]

tests/unit/clients/test_gcrm.py

@@ -93,8 +93,3 @@ async def test_list_all_active_projects_multiple_pages(
         expected_rsp = crm_one_page_rsp['projects']
         expected_rsp.append(page2['projects'].pop(0))
         assert expected_rsp == results
-        # assert requests made and their sequence
-        requests = list(m.requests.keys())
-        assert 2 == len(requests)
-        assert ('get', url_with_pagesize) == requests[0]
-        assert ('get', url_with_token) == requests[1]

tests/unit/clients/test_gdns.py

@@ -29,7 +29,7 @@
 
 
 @pytest.fixture
-def client(mocker, create_mock_coro):
+async def client(mocker, create_mock_coro):
     auth_client = mocker.Mock(auth.GAuthClient)
     creds = mocker.Mock()
     auth_client.creds = creds
@@ -48,7 +48,7 @@ def client(mocker, create_mock_coro):
     client._request_post_mock = request_post_mock
     yield client
     # test teardown
-    client._session.close()
+    await client._session.close()
 
 
 def test_dns_client_default(client):

tests/unit/clients/test_http.py

@@ -31,8 +31,9 @@
 #####
 # Tests for simple client instantiation
 #####
+@pytest.mark.asyncio
 @pytest.mark.parametrize('provide_session', [True, False])
-def test_http_client_default(provide_session, mocker):
+async def test_http_client_default(provide_session, mocker):
     """AIOConnection is created with expected attributes."""
     session = None
     if provide_session:
@@ -51,15 +52,15 @@ def test_http_client_default(provide_session, mocker):
         assert auth_client._session is client._session
         assert session is not client._session
 
-    client._session.close()
+    await auth_client._session.close()
 
 
 @pytest.fixture
-def client(mocker, auth_client):
+async def client(mocker, auth_client):
     session = aiohttp.ClientSession()
     client = http.AIOConnection(auth_client=auth_client, session=session)
     yield client
-    session.close()
+    await session.close()
 
 
 args = 'token,expiry,is_valid_token_expected'
@@ -119,7 +120,8 @@ async def mock_valid_token_set():
     assert resp == resp_text
 
     assert 1 == mock_refresh_token_called
-    request = mocked.requests[('get', conftest.API_URL)][0]
+    assert 1 == len(mocked.requests)
+    request = mocked.requests.popitem()[1][0]
     authorization_header = request.kwargs['headers']['Authorization']
     assert authorization_header == f'Bearer {client._auth_client.token}'
     assert 2 == len(caplog.records)

tests/unit/plugins/janitor/test_authority.py

@@ -72,7 +72,7 @@ async def test_builder_creates_proper_authority(
     try:
         gce_authority = builder.build_authority()
     finally:
-        builder.session.close()
+        await builder.session.close()
 
     auth_client_calls = [
         mocker.call(

tests/unit/plugins/janitor/test_reconciler.py

@@ -180,13 +180,8 @@ async def validate_rrsets_by_zone():
     await recon_client.cleanup()
 
     assert exp_log_records == len(caplog.records)
-    if exp_log_records == 2:
-        # it's in a cancelling state which can't be directly tested
-        assert not coro1.done()
-        assert not coro2.done()
-    else:
-        assert coro1.done()
-        assert coro2.done()
+    assert coro1.done()
+    assert coro2.done()
 
     assert 1 == recon_client.changes_channel.qsize()