Bug fixes for GCP modules

ansible · Aug 19, 2019 · 96aad20 · 96aad20
1 parent 555b841
commit 96aad20
Show file tree

Hide file tree

Showing 14 changed files with 117 additions and 120 deletions.
diff --git a/lib/ansible/module_utils/gcp_utils.py b/lib/ansible/module_utils/gcp_utils.py
@@ -78,53 +78,118 @@ def __init__(self, module, product):
         self._validate()
 
     def get(self, url, body=None, **kwargs):
-        kwargs.update({'json': body, 'headers': self._headers()})
-        try:
-            return self.session().get(url, **kwargs)
-        except getattr(requests.exceptions, 'RequestException') as inst:
-            self.module.fail_json(msg=inst.message)
+        """
+        This method should be avoided in favor of full_get
+        """
+        kwargs.update({'json': body})
+        return self.full_get(url, **kwargs)
 
     def post(self, url, body=None, headers=None, **kwargs):
-        if headers:
-            headers = self._merge_dictionaries(headers, self._headers())
-        else:
-            headers = self._headers()
+        """
+        This method should be avoided in favor of full_post
+        """
+        kwargs.update({'json': body, 'headers': headers})
+        return self.full_post(url, **kwargs)
+
+    def post_contents(self, url, file_contents=None, headers=None, **kwargs):
+        """
+        This method should be avoided in favor of full_post
+        """
+        kwargs.update({'data': file_contents, 'headers': headers})
+        return self.full_post(url, **kwargs)
+
+    def delete(self, url, body=None):
+        """
+        This method should be avoided in favor of full_delete
+        """
+        kwargs = {'json': body}
+        return self.full_delete(url, **kwargs)
+
+    def put(self, url, body=None):
+        """
+        This method should be avoided in favor of full_put
+        """
+        kwargs = {'json': body}
+        return self.full_put(url, **kwargs)
+
+    def patch(self, url, body=None, **kwargs):
+        """
+        This method should be avoided in favor of full_patch
+        """
+        kwargs.update({'json': body})
+        return self.full_patch(url, **kwargs)
+
+    def list(self, url, callback, params=None, array_name='items',
+             pageToken='nextPageToken', **kwargs):
+        """
+        This should be used for calling the GCP list APIs. It will return
+        an array of items
+
+        This takes a callback to a `return_if_object(module, response)`
+        function that will decode the response + return a dictionary. Some
+        modules handle the decode + error processing differently, so we should
+        defer to the module to handle this.
+        """
+        resp = callback(self.module, self.full_get(url, params, **kwargs))
+        items = resp.get(array_name) if resp.get(array_name) else []
+        while resp.get(pageToken):
+            if params:
+                params['pageToken'] = resp.get(pageToken)
+            else:
+                params = {'pageToken': resp[pageToken]}
 
+            resp = callback(self.module, self.full_get(url, params, **kwargs))
+            if resp.get(array_name):
+                items = items + resp.get(array_name)
+        return items
+
+    # The following methods fully mimic the requests API and should be used.
+    def full_get(self, url, params=None, **kwargs):
+        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
         try:
-            return self.session().post(url, json=body, headers=headers)
+            return self.session().get(url, params=params, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
+            # Only log the message to avoid logging any sensitive info.
             self.module.fail_json(msg=inst.message)
 
-    def post_contents(self, url, file_contents=None, headers=None, **kwargs):
-        if headers:
-            headers = self._merge_dictionaries(headers, self._headers())
-        else:
-            headers = self._headers()
+    def full_post(self, url, data=None, json=None, **kwargs):
+        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
 
         try:
-            return self.session().post(url, data=file_contents, headers=headers)
+            return self.session().post(url, data=data, json=json, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
             self.module.fail_json(msg=inst.message)
 
-    def delete(self, url, body=None):
+    def full_put(self, url, data=None, **kwargs):
+        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
+
         try:
-            return self.session().delete(url, json=body, headers=self._headers())
+            return self.session().put(url, data=data, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
             self.module.fail_json(msg=inst.message)
 
-    def put(self, url, body=None):
+    def full_patch(self, url, data=None, **kwargs):
+        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
+
         try:
-            return self.session().put(url, json=body, headers=self._headers())
+            return self.session().patch(url, data=data, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
             self.module.fail_json(msg=inst.message)
 
-    def patch(self, url, body=None, **kwargs):
-        kwargs.update({'json': body, 'headers': self._headers()})
+    def full_delete(self, url, **kwargs):
+        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
+
         try:
-            return self.session().patch(url, **kwargs)
+            return self.session().delete(url, **kwargs)
         except getattr(requests.exceptions, 'RequestException') as inst:
             self.module.fail_json(msg=inst.message)
 
+    def _set_headers(self, headers):
+        if headers:
+            return self._merge_dictionaries(headers, self._headers())
+        else:
+            return self._headers()
+
     def session(self):
         return AuthorizedSession(
             self._credentials())

diff --git a/lib/ansible/modules/cloud/google/gcp_compute_target_https_proxy_info.py b/lib/ansible/modules/cloud/google/gcp_compute_target_https_proxy_info.py
@@ -143,12 +143,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
     module.exit_json(**return_value)
 
 
@@ -158,8 +153,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
 
 
 def query_options(filters):

diff --git a/lib/ansible/modules/cloud/google/gcp_compute_target_pool.py b/lib/ansible/modules/cloud/google/gcp_compute_target_pool.py
@@ -419,7 +419,11 @@ def wait_for_operation(module, response):
         return {}
     status = navigate_hash(op_result, ['status'])
     wait_done = wait_for_completion(status, op_result, module)
-    return decode_response(fetch_resource(module, navigate_hash(wait_done, ['targetLink']), 'compute#targetPool'), module)
+    response = fetch_resource(module, navigate_hash(wait_done, ['targetLink']), 'compute#targetPool')
+    if response:
+        return decode_response(response, module)
+    else:
+        return {}
 
 
 def wait_for_completion(status, op_result, module):

diff --git a/lib/ansible/modules/cloud/google/gcp_compute_target_pool_info.py b/lib/ansible/modules/cloud/google/gcp_compute_target_pool_info.py
@@ -179,12 +179,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
     module.exit_json(**return_value)
 
 
@@ -194,8 +189,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
 
 
 def query_options(filters):

diff --git a/lib/ansible/modules/cloud/google/gcp_compute_target_ssl_proxy_info.py b/lib/ansible/modules/cloud/google/gcp_compute_target_ssl_proxy_info.py
@@ -139,12 +139,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
     module.exit_json(**return_value)
 
 
@@ -154,8 +149,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
 
 
 def query_options(filters):

diff --git a/lib/ansible/modules/cloud/google/gcp_compute_target_tcp_proxy_info.py b/lib/ansible/modules/cloud/google/gcp_compute_target_tcp_proxy_info.py
@@ -125,12 +125,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
     module.exit_json(**return_value)
 
 
@@ -140,8 +135,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
 
 
 def query_options(filters):

diff --git a/lib/ansible/modules/cloud/google/gcp_compute_target_vpn_gateway_info.py b/lib/ansible/modules/cloud/google/gcp_compute_target_vpn_gateway_info.py
@@ -141,12 +141,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
     module.exit_json(**return_value)
 
 
@@ -156,8 +151,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
 
 
 def query_options(filters):

diff --git a/lib/ansible/modules/cloud/google/gcp_compute_url_map_info.py b/lib/ansible/modules/cloud/google/gcp_compute_url_map_info.py
@@ -221,12 +221,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
     module.exit_json(**return_value)
 
 
@@ -236,8 +231,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
 
 
 def query_options(filters):

diff --git a/lib/ansible/modules/cloud/google/gcp_compute_vpn_tunnel_info.py b/lib/ansible/modules/cloud/google/gcp_compute_vpn_tunnel_info.py
@@ -168,12 +168,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/compute']
 
-    items = fetch_list(module, collection(module), query_options(module.params['filters']))
-    if items.get('items'):
-        items = items.get('items')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module), query_options(module.params['filters']))}
     module.exit_json(**return_value)
 
 
@@ -183,8 +178,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'compute')
-    response = auth.get(link, params={'filter': query})
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='items', params={'filter': query})
 
 
 def query_options(filters):

diff --git a/lib/ansible/modules/cloud/google/gcp_container_cluster_info.py b/lib/ansible/modules/cloud/google/gcp_container_cluster_info.py
@@ -586,12 +586,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform']
 
-    items = fetch_list(module, collection(module))
-    if items.get('clusters'):
-        items = items.get('clusters')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module))}
     module.exit_json(**return_value)
 
 
@@ -601,8 +596,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'container')
-    response = auth.get(link)
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='clusters')
 
 
 def return_if_object(module, response):

diff --git a/lib/ansible/modules/cloud/google/gcp_container_node_pool_info.py b/lib/ansible/modules/cloud/google/gcp_container_node_pool_info.py
@@ -371,12 +371,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/cloud-platform']
 
-    items = fetch_list(module, collection(module))
-    if items.get('nodePools'):
-        items = items.get('nodePools')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module))}
     module.exit_json(**return_value)
 
 
@@ -387,8 +382,7 @@ def collection(module):
 
 def fetch_list(module, link):
     auth = GcpSession(module, 'container')
-    response = auth.get(link)
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='nodePools')
 
 
 def return_if_object(module, response):

diff --git a/lib/ansible/modules/cloud/google/gcp_dns_managed_zone_info.py b/lib/ansible/modules/cloud/google/gcp_dns_managed_zone_info.py
@@ -214,12 +214,7 @@ def main():
     if not module.params['scopes']:
         module.params['scopes'] = ['https://www.googleapis.com/auth/ndev.clouddns.readwrite']
 
-    items = fetch_list(module, collection(module), module.params['dns_name'])
-    if items.get('managedZones'):
-        items = items.get('managedZones')
-    else:
-        items = []
-    return_value = {'resources': items}
+    return_value = {'resources': fetch_list(module, collection(module), module.params['dns_name'])}
     module.exit_json(**return_value)
 
 
@@ -229,8 +224,7 @@ def collection(module):
 
 def fetch_list(module, link, query):
     auth = GcpSession(module, 'dns')
-    response = auth.get(link, params={'dnsName': query})
-    return return_if_object(module, response)
+    return auth.list(link, return_if_object, array_name='managedZones', params={'dnsName': query})
 
 
 def return_if_object(module, response):