gcp - region psuedo resource from static data, and makefile data update target

Makefile

@@ -129,6 +129,14 @@ pkg-publish-wheel:
 	twine upload -r $(PKG_REPO) dist/*
 	for pkg in $(PKG_SET); do cd $$pkg && twine upload -r $(PKG_REPO) dist/* && cd ../..; done
 
+data-update:
+# aws data sets
+	python tools/dev/cfntypedb.py -f tests/data/cfn-types.json
+	python tools/dev/arnref.py -f tests/data/arn-types.json
+	python tools/dev/iamdb.py -f tests/data/iam-actions.json
+# gcp data sets
+	python tools/dev/gcpiamdb.py -f tools/c7n_gcp/tests/data/iam-permissions.json
+	python tools/dev/gcpregion.py -f tools/c7n_gcp/c7n_gcp/regions.json
 
 ###
 # Static analyzers

tools/c7n_gcp/c7n_gcp/actions/notify.py

@@ -100,7 +100,8 @@ def publish_message(self, message, client):
 
     @classmethod
     def register_resource(cls, registry, resource_class):
-        resource_class.action_registry.register('notify', Notify)
+        if resource_class.action_registry:
+            resource_class.action_registry.register('notify', Notify)
 
 
 gcp_resources.subscribe(Notify.register_resource)

tools/c7n_gcp/c7n_gcp/filters/metrics.py

@@ -212,7 +212,8 @@ def process_resource(self, resource):
 
     @classmethod
     def register_resources(klass, registry, resource_class):
-        resource_class.filter_registry.register('metrics', klass)
+        if resource_class.filter_registry:
+            resource_class.filter_registry.register('metrics', klass)
 
 
 gcp_resources.subscribe(GCPMetricsFilter.register_resources)

tools/c7n_gcp/c7n_gcp/filters/sccfindings.py

@@ -79,7 +79,8 @@ def process_resource(self, resource):
 
     @classmethod
     def register_resources(klass, registry, resource_class):
-        resource_class.filter_registry.register('scc-findings', klass)
+        if resource_class.filter_registry:
+            resource_class.filter_registry.register('scc-findings', klass)
 
 
 gcp_resources.subscribe(SecurityComandCenterFindingsFilter.register_resources)

tools/c7n_gcp/c7n_gcp/regions.json

@@ -0,0 +1,41 @@
+[
+  "asia-east1",
+  "asia-east2",
+  "asia-northeast1",
+  "asia-northeast2",
+  "asia-northeast3",
+  "asia-south1",
+  "asia-south2",
+  "asia-southeast1",
+  "asia-southeast2",
+  "australia-southeast1",
+  "australia-southeast2",
+  "europe-central2",
+  "europe-north1",
+  "europe-southwest1",
+  "europe-west1",
+  "europe-west12",
+  "europe-west2",
+  "europe-west3",
+  "europe-west4",
+  "europe-west6",
+  "europe-west8",
+  "europe-west9",
+  "me-central1",
+  "me-west1",
+  "northamerica-northeast1",
+  "northamerica-northeast2",
+  "southamerica-east1",
+  "southamerica-west1",
+  "us-central1",
+  "us-central2",
+  "us-east1",
+  "us-east4",
+  "us-east5",
+  "us-east7",
+  "us-south1",
+  "us-west1",
+  "us-west2",
+  "us-west3",
+  "us-west4"
+]

tools/c7n_gcp/c7n_gcp/resources/resource_map.py

@@ -77,6 +77,7 @@
     "gcp.pubsub-snapshot": "c7n_gcp.resources.pubsub.PubSubSnapshot",
     "gcp.pubsub-subscription": "c7n_gcp.resources.pubsub.PubSubSubscription",
     "gcp.pubsub-topic": "c7n_gcp.resources.pubsub.PubSubTopic",
+    "gcp.region": "c7n_gcp.region.Region",
     "gcp.route": "c7n_gcp.resources.network.Route",
     "gcp.router": "c7n_gcp.resources.network.Router",
     "gcp.service": "c7n_gcp.resources.service.Service",

tools/dev/arnref.py

@@ -59,7 +59,7 @@ def header_matches(string, table):
 
 @click.command()
 @click.option("--docs-dir", type=click.Path())
-@click.option("--output", type=click.File("wb"), default="-")
+@click.option("-f", "--output", type=click.File("wb"), default="-")
 def main(docs_dir, output):
     docs_dir = Path(docs_dir)
 

tools/dev/cfntypedb.py

@@ -1,13 +1,17 @@
 # Copyright The Cloud Custodian Authors.
 # SPDX-License-Identifier: Apache-2.0
-import boto3
 import json
+
+import boto3
+import click
 import jmespath
 
 from botocore.paginate import Paginator
 
 
-def main():
+@click.command()
+@click.option('-f', '--output', default='-', type=click.File('w'))
+def main(output):
 
     client = boto3.client('cloudformation')
 
@@ -31,7 +35,8 @@ def main():
     for m in missing:
         if m not in type_names:
             type_names.append(m)
-    print(json.dumps(sorted(type_names), indent=2))
+
+    output.write(json.dumps(sorted(type_names), indent=2))
 
 
 if __name__ == '__main__':

tools/dev/gcpiamdb.py

@@ -7,7 +7,8 @@
 
 
 @click.command()
-def main():
+@click.option('-f', '--output', default='-', type=click.File('w'))
+def main(output):
     """GCP IAM DataSet
     """
     response = requests.get(
@@ -19,8 +20,7 @@ def main():
             continue
         perms.append(row.td.code.text)
 
-    with open('gcp-iam-perms.json', 'w') as fh:
-        json.dump({'permissions': perms}, fp=fh, indent=2)
+    json.dump({'permissions': perms}, fp=output, indent=2)
 
 
 if __name__ == '__main__':

tools/dev/gcpregion.py

@@ -0,0 +1,17 @@
+import click
+import requests
+import jmespath
+import json
+
+
+@click.command()
+@click.option('-f', '--output', type=click.File('w'), default='-')
+def main(output):
+    data = requests.get('https://www.gstatic.com/ipranges/cloud.json').json()
+    regions = sorted(list(set(jmespath.search('prefixes[].scope', data))))
+    regions.remove('global')
+    output.write(json.dumps([{'id': r} for r in regions], indent=2))
+
+
+if __name__ == '__main__':
+    main()

tools/dev/iamdb.py

@@ -1,6 +1,7 @@
 # Copyright The Cloud Custodian Authors.
 # SPDX-License-Identifier: Apache-2.0
 
+import click
 import requests
 import json
 
@@ -9,7 +10,9 @@
 URL = "https://awspolicygen.s3.amazonaws.com/js/policies.js"
 
 
-def main():
+@click.command()
+@click.option('-f', '--output', default='-', type=click.File('w'))
+def main(output):
     raw_data = requests.get(URL).text
     data = json.loads(raw_data[raw_data.find('=') + 1:])
 
@@ -21,8 +24,7 @@ def main():
     for k in sorted(perms):
         sorted_perms[k] = sorted(set(perms[k]))
 
-    with open('iam-permissions.json', 'w') as fh:
-        json.dump(sorted_perms, fp=fh, indent=2)
+    json.dump(sorted_perms, fp=output, indent=2)
 
 
 if __name__ == '__main__':