Merge pull request #261 from CloudWanderer-io/bugfix/remove-non-strin…

…g-id-parts Bugfix/remove non string id parts
CloudWanderer-io · Jan 1, 2022 · def2bc9 · def2bc9
2 parents 1c467d5 + 2da78c8
commit def2bc9
Show file tree

Hide file tree

Showing 27 changed files with 511 additions and 433 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -20,3 +20,11 @@ repos:
     hooks:
       - id: mypy
         exclude: ^doc_source/|^tests/
+  - repo: https://github.com/pre-commit/pre-commit-hooks.git
+    rev: v4.1.0
+    hooks:
+      - id: pretty-format-json
+        args:
+          - --autofix
+          - --no-sort-keys
+          - --indent=4
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,11 @@
+# 0.29.1
+
+- `PartialURN` now throws an error if you pass a non-string as a resource_id_part (Relates to the lamber layers section of #260)
+- `ServiceResource.get_urn` now converts integer resource_id_parts to strings (Relates to the lamber layers section of #260)
+- Ensured lambda layer versions load full metadata with `get_layer_version` call.
+- Added enum for resource map `ResourceIndependenceType` normalising on `baseResource` rather than `resource` as was used in some places.
+- `CloudWandererAWSInterface.get_resource` now throws an error if you try to get a dependent resource as this will cause `parent_urn` not to be populated.
+
 # 0.29.0
 
 - Fix bug where wafv2 `RegionalWebAcl` resources would not correctly load their data (Fixes [#250](https://github.com/CloudWanderer-io/CloudWanderer/issues/250))

diff --git a/cloudwanderer/aws_interface/interface.py b/cloudwanderer/aws_interface/interface.py
@@ -11,7 +11,7 @@
 from ..base import CloudInterface, ServiceResourceTypeFilter
 from ..cloud_wanderer_resource import CloudWandererResource
 from ..exceptions import UnsupportedResourceTypeError
-from ..models import ActionSet, ServiceResourceType, TemplateActionSet
+from ..models import ActionSet, ResourceIndependenceType, ServiceResourceType, TemplateActionSet
 from ..urn import URN
 from .aws_services import AWS_SERVICES
 from .models import AWSResourceTypeFilter, ResourceMap
@@ -94,6 +94,10 @@ def get_resource(
                     service_name=cast(AWS_SERVICES, urn.service), region_name=service.service_map.global_service_region
                 )
             resource = service.resource(resource_type=urn.resource_type, identifiers=urn.resource_id_parts)
+            if resource.resource_map.type == ResourceIndependenceType.DEPENDENT_RESOURCE:
+                raise UnsupportedResourceTypeError(
+                    f"Resource type {urn.resource_type} is a dependent resource, please enumerate its parent."
+                )
             if not hasattr(resource, "load"):
                 raise UnsupportedResourceTypeError(f"Resource type {urn.resource_type} doesn't support get_resource()")
             logger.info("Loading resource data.")
@@ -227,7 +231,11 @@ def _get_dependent_resources(
                         dependent_resource,
                     )
                     continue
-                logger.debug("Found %s", dependent_resource)
+                logger.debug(
+                    "Found %s, it %s",
+                    dependent_resource,
+                    ["does not require loading", "requires loading"][dependent_resource.resource_map.requires_load],
+                )
                 if dependent_resource.resource_map.requires_load or (
                     not dependent_resource.meta.data and hasattr(dependent_resource, "load")
                 ):

diff --git a/cloudwanderer/aws_interface/models.py b/cloudwanderer/aws_interface/models.py
@@ -13,6 +13,7 @@
     RelationshipDirection,
     RelationshipRegionSource,
     ResourceIdUniquenessScope,
+    ResourceIndependenceType,
 )
 from ..utils import camel_to_snake, snake_to_pascal
 from .utils import _get_urn_components_from_string
@@ -114,8 +115,8 @@ class ResourceMap(NamedTuple):
 
     #: The PascalCase name of the resource (e.g. ``Instance``)
     name: str
-    #: The snake_case type of the resource (e.g. ``instance``)
-    type: Optional[str]
+    #: The ResourceIndependenceType of the resource (baseResource, secondaryAttribute, dependentResource)
+    type: ResourceIndependenceType
     #: The scope in which this resource's ID is unique.
     id_uniqueness_scope: ResourceIdUniquenessScope
     #: An optional definition for how to perform a secondary query to discover the region in
@@ -147,7 +148,7 @@ def factory(
     ) -> "ResourceMap":
         return cls(
             name=name,
-            type=definition.get("type"),
+            type=ResourceIndependenceType[camel_to_snake(definition.get("type", "baseResource"))],
             region_request=ResourceRegionRequest.factory(definition.get("regionRequest")),
             regional_resource=definition.get("regionalResource", True),
             default_aws_resource_type_filter=AWSResourceTypeFilter(

diff --git a/cloudwanderer/aws_interface/resource_definitions/autoscaling/2011-01-01/resources-cw-1.json b/cloudwanderer/aws_interface/resource_definitions/autoscaling/2011-01-01/resources-cw-1.json
@@ -2,7 +2,7 @@
   "service": {},
   "resources": {
     "AutoScalingGroup": {
-      "type": "resource",
+      "type": "baseResource",
       "relationships": [
         {
           "basePath": "@",
@@ -47,7 +47,7 @@
       ]
     },
     "LaunchConfiguration": {
-      "type": "resource",
+      "type": "baseResource",
       "relationships": [
         {
           "basePath": "@",

diff --git a/...wanderer/aws_interface/resource_definitions/cloudformation/2010-05-15/resources-cw-1.json b/...wanderer/aws_interface/resource_definitions/cloudformation/2010-05-15/resources-cw-1.json
@@ -1,8 +1,8 @@
 {
-    "service": {},
-    "resources": {
-        "Stack": {
-            "type": "resource"
-        }
+  "service": {},
+  "resources": {
+    "Stack": {
+      "type": "baseResource"
     }
+  }
 }
diff --git a/cloudwanderer/aws_interface/resource_definitions/ec2/2016-11-15/resources-cw-1.json b/cloudwanderer/aws_interface/resource_definitions/ec2/2016-11-15/resources-cw-1.json
@@ -5,7 +5,7 @@
   },
   "resources": {
     "NatGateway": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -85,7 +85,7 @@
       ]
     },
     "RouteTable": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -120,7 +120,7 @@
       ]
     },
     "NetworkInterface": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -168,7 +168,7 @@
       ]
     },
     "NetworkAcl": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -190,7 +190,7 @@
       ]
     },
     "InternetGateway": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -212,7 +212,7 @@
       ]
     },
     "Volume": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -247,14 +247,14 @@
       ]
     },
     "DhcpOptions": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
       }
     },
     "Vpc": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -276,7 +276,7 @@
       ]
     },
     "Subnet": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -307,7 +307,7 @@
       ]
     },
     "Image": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -334,7 +334,7 @@
       }
     },
     "Snapshot": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -346,7 +346,7 @@
       }
     },
     "Instance": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
@@ -421,14 +421,14 @@
       ]
     },
     "SecurityGroup": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false
       }
     },
     "ElasticIp": {
-      "type": "resource",
+      "type": "baseResource",
       "idUniquenessScope": {
         "requiresAccountId": false,
         "requiresRegion": false

diff --git a/cloudwanderer/aws_interface/resource_definitions/elb/2012-06-01/resources-cw-1.json b/cloudwanderer/aws_interface/resource_definitions/elb/2012-06-01/resources-cw-1.json
@@ -1,62 +1,62 @@
 {
-    "service": {},
-    "resources": {
-        "LoadBalancer": {
-            "type": "resource",
-            "relationships": [
-                {
-                    "basePath": "@",
-                    "idParts": [
-                        {
-                            "path": "VPCId"
-                        }
-                    ],
-                    "service": "ec2",
-                    "resourceType": "vpc",
-                    "regionSource": "sameAsResource",
-                    "accountIdSource": "unknown",
-                    "direction": "inbound"
-                },
-                {
-                    "basePath": "Instances[]",
-                    "idParts": [
-                        {
-                            "path": "InstanceId"
-                        }
-                    ],
-                    "service": "ec2",
-                    "resourceType": "instance",
-                    "regionSource": "sameAsResource",
-                    "accountIdSource": "unknown",
-                    "direction": "outbound"
-                },
-                {
-                    "basePath": "SecurityGroups[]",
-                    "idParts": [
-                        {
-                            "path": "@"
-                        }
-                    ],
-                    "service": "ec2",
-                    "resourceType": "security_group",
-                    "regionSource": "sameAsResource",
-                    "accountIdSource": "unknown",
-                    "direction": "outbound"
-                },
-                {
-                    "basePath": "Subnets[]",
-                    "idParts": [
-                        {
-                            "path": "@"
-                        }
-                    ],
-                    "service": "ec2",
-                    "resourceType": "subnet",
-                    "regionSource": "sameAsResource",
-                    "accountIdSource": "unknown",
-                    "direction": "outbound"
-                }
-            ]
+  "service": {},
+  "resources": {
+    "LoadBalancer": {
+      "type": "baseResource",
+      "relationships": [
+        {
+          "basePath": "@",
+          "idParts": [
+            {
+              "path": "VPCId"
+            }
+          ],
+          "service": "ec2",
+          "resourceType": "vpc",
+          "regionSource": "sameAsResource",
+          "accountIdSource": "unknown",
+          "direction": "inbound"
+        },
+        {
+          "basePath": "Instances[]",
+          "idParts": [
+            {
+              "path": "InstanceId"
+            }
+          ],
+          "service": "ec2",
+          "resourceType": "instance",
+          "regionSource": "sameAsResource",
+          "accountIdSource": "unknown",
+          "direction": "outbound"
+        },
+        {
+          "basePath": "SecurityGroups[]",
+          "idParts": [
+            {
+              "path": "@"
+            }
+          ],
+          "service": "ec2",
+          "resourceType": "security_group",
+          "regionSource": "sameAsResource",
+          "accountIdSource": "unknown",
+          "direction": "outbound"
+        },
+        {
+          "basePath": "Subnets[]",
+          "idParts": [
+            {
+              "path": "@"
+            }
+          ],
+          "service": "ec2",
+          "resourceType": "subnet",
+          "regionSource": "sameAsResource",
+          "accountIdSource": "unknown",
+          "direction": "outbound"
         }
+      ]
     }
+  }
 }
diff --git a/cloudwanderer/aws_interface/resource_definitions/elbv2/2015-12-01/resources-cw-1.json b/cloudwanderer/aws_interface/resource_definitions/elbv2/2015-12-01/resources-cw-1.json
@@ -11,7 +11,7 @@
       ]
     },
     "TargetGroup": {
-      "type": "resource",
+      "type": "baseResource",
       "relationships": [
         {
           "basePath": "@",
@@ -99,7 +99,7 @@
       ]
     },
     "LoadBalancer": {
-      "type": "resource",
+      "type": "baseResource",
       "relationships": [
         {
           "basePath": "@",