Merge pull request #671 from prancer-io/development

Implemented feature to crawl resources of all cloud accounts

setup.py

@@ -18,7 +18,7 @@
 setup(
     name='prancer-basic',
     # also update the version in processor.__init__.py file
-    version='3.0.4',
+    version='3.0.5',
     description='Prancer Basic, http://prancer.io/',
     long_description=LONG_DESCRIPTION,
     license = "BSD",

src/processor/__init__.py

@@ -1,3 +1,3 @@
 # Prancer Basic
 
-__version__ = '3.0.4'
+__version__ = '3.0.5'

src/processor/connector/populate_json.py

@@ -5,6 +5,7 @@
 from processor.connector.snapshot_custom import get_custom_data, git_clone_dir
 from processor.logging.log_handler import getlogger
 from subprocess import Popen, PIPE
+import copy
 import tempfile
 import re
 import os
@@ -68,6 +69,7 @@ def validate_master_snapshot_data(master_snapshot_json, document_json, file_loca
         logger.error("Invalid json %s: 'Snapshots' field is not type list in snapshot configuration file." % file_location)
         return False
 
+    snapshot_list = []
     for snapshot in snapshots:
         if "type" not in snapshot:
             logger.error("Invalid json %s: 'type' field is not exists in snapshot configuration file." % file_location)
@@ -86,6 +88,7 @@ def validate_master_snapshot_data(master_snapshot_json, document_json, file_loca
                     for key, value in connector_user.items():
                         if key != "id":
                             snapshot[key] = value
+                    snapshot_list.append(copy.deepcopy(snapshot))
 
             if not found_connector_user:
                 logger.error("Invalid json %s: `testUser` in snapshot configuration file is not mactch with the 'connectorUser' in remote snapshot file." % file_location)
@@ -130,7 +133,7 @@ def validate_master_snapshot_data(master_snapshot_json, document_json, file_loca
         document_json.pop("connector")
         document_json.pop("remoteFile")
         document_json.pop("connectorUsers")
-        document_json["snapshots"] = master_snapshot_json["snapshots"]
+        document_json["snapshots"] = snapshot_list
 
     return validate    
 

src/processor/connector/snapshot_aws.py

@@ -1202,6 +1202,7 @@ def populate_aws_snapshot(snapshot, container=None):
                                         'masterSnapshotId': node['masterSnapshotId'],
                                         'collection': data['collection'],
                                         'arn' : data['arn'],
+                                        'account_id': account_id,
                                         'status' : 'active'
                                     }
                                     if node.get("boto_type"):

src/processor/connector/snapshot_azure.py

@@ -227,7 +227,7 @@ def get_node(token, sub_name, sub_id, node, user, snapshot_source, all_data_reco
             exmatch = re.search(r'/subscriptions.*/resourceGroups/.*?/', node['path'], re.I)
             if exmatch:
                 export_template_url = 'https://management.azure.com%sexportTemplate?api-version=2021-04-01' % (exmatch.group(0).lower())
-                status, data = export_template(export_template_url, hdrs, node['path'])
+                status, data = export_template(export_template_url, hdrs, node['path'], retry_count=0)
 
         db_record['path'] = node['path']
 
@@ -298,15 +298,7 @@ def check_include_path_validation(path, include_paths, include_regex_list):
     return include_path or include_regex
 
 
-def populate_azure_snapshot(snapshot, container=None, snapshot_type='azure'):
-    """ Populates the resources from azure."""
-    dbname = config_value('MONGODB', 'dbname')
-    snapshot_source = get_field_value(snapshot, 'source')
-    snapshot_user = get_field_value(snapshot, 'testUser')
-    snapshot_nodes = get_field_value(snapshot, 'nodes')
-    snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes)
-    client_id, client_secret, sub_name, sub_id, tenant_id = \
-        get_web_client_data(snapshot_type, snapshot_source, snapshot_user, container)
+def populate_client_secret(client_id, client_secret, snapshot_user):
     if not client_id:
         # logger.info("No client_id in the snapshot to access azure resource!...")
         raise Exception("No client id in the snapshot to access azure resource!...")
@@ -328,6 +320,20 @@ def populate_azure_snapshot(snapshot, container=None, snapshot_type='azure'):
 
     if not client_secret:
         raise Exception("No `client_secret` key in the connector file to access azure resource!...")
+
+    return client_secret
+
+def populate_azure_snapshot(snapshot, container=None, snapshot_type='azure'):
+    """ Populates the resources from azure."""
+    dbname = config_value('MONGODB', 'dbname')
+    snapshot_source = get_field_value(snapshot, 'source')
+    snapshot_user = get_field_value(snapshot, 'testUser')
+    snapshot_nodes = get_field_value(snapshot, 'nodes')
+    snapshot_data, valid_snapshotids = validate_snapshot_nodes(snapshot_nodes)
+    client_id, client_secret, sub_name, sub_id, tenant_id = \
+        get_web_client_data(snapshot_type, snapshot_source, snapshot_user, container)
+
+    client_secret = populate_client_secret(client_id, client_secret, snapshot_user)
 
     logger.info('\t\tSubscription: %s', sub_id)
     logger.info('\t\tTenant: %s', tenant_id)
@@ -447,7 +453,8 @@ def populate_azure_snapshot(snapshot, container=None, snapshot_type='azure'):
                                         'snapshotId': data['snapshotId'],
                                         'path': data['path'],
                                         'validate': validate,
-                                        'status': 'active'
+                                        'status': 'active',
+                                        'subscriptionId': sub_id
                                     })
 
         for data in all_data_records:

src/processor/connector/snapshot_google.py

@@ -590,6 +590,7 @@ def set_snapshot_data(node, items, snapshot_data, project_id=None, credentials=N
                         "collection": node['collection'],
                         "path": path,
                         "status" : "active",
+                        'project_id': project_id,
                         "validate" : node['validate'] if 'validate' in node else True
                     })
     return snapshot_data

src/processor/crawler/master_snapshot.py

@@ -45,6 +45,7 @@
 from processor.connector.populate_json import pull_json_data
 from processor.helper.file.file_utils import exists_file,remove_file
 from processor.template_processor.base.base_template_processor import set_processed_templates
+from processor.crawler.utils import check_container_for_all_accounts
 
 doc_id = None
 logger = getlogger()
@@ -60,6 +61,9 @@
 REMOVE_SNAPSHOTGEN_FIELDS = [
     "exclude",
     "source",
+    "subscriptionId",
+    "account_id",
+    "project_id"
 ]
 
 def generate_snapshot(snapshot_json_data, snapshot_file_data):
@@ -84,8 +88,17 @@ def generate_snapshot(snapshot_json_data, snapshot_file_data):
                                     # if structure and structure == 'aws':
                                     #     newnode = {}
                                     # else:
+                                    snapshot_type = snapshot.get("type")
                                     if "source" in sid_data and sid_data["source"] != snapshot.get("source"):
                                         continue
+
+                                    if snapshot_type == "azure" and sid_data.get("subscriptionId") != snapshot.get("subscriptionId"):
+                                        continue
+                                    elif snapshot_type == "aws" and sid_data.get("account_id") != snapshot.get("accountId"):
+                                        continue
+                                    elif snapshot_type == "google" and sid_data.get("project_id") != snapshot.get("project-id"):
+                                        continue
+
                                     newnode = copy.deepcopy(node)
                                     newnode.update(sid_data)
 
@@ -163,7 +176,7 @@ def generate_mastersnapshots_from_json(mastersnapshot_json_data, snapshot_json_d
     return snapshot_data
 
 
-def generate_snapshots_from_mastersnapshot_file(mastersnapshot_file):
+def generate_snapshots_from_mastersnapshot_file(mastersnapshot_file, container, file_name):
     """
     Each snapshot file from the filesystem is loaded as a json datastructue
      and generate all the nodes in this json datastructure.
@@ -174,6 +187,8 @@ def generate_snapshots_from_mastersnapshot_file(mastersnapshot_file):
     if not mastersnapshot_json_data:
         logger.error("masterSnapshot file %s looks to be empty, next!...", mastersnapshot_file)
         return {}, {}
+
+    mastersnapshot_json_data = check_container_for_all_accounts(container, mastersnapshot_json_data, file_name)
 
     if "connector" in mastersnapshot_json_data and "remoteFile" in mastersnapshot_json_data and mastersnapshot_json_data["connector"] and mastersnapshot_json_data["remoteFile"]:
         _, pull_response = pull_json_data(mastersnapshot_json_data)
@@ -242,7 +257,7 @@ def generate_container_mastersnapshots_filesystem(container, mastersnapshotfile=
         if parts[-1] in snapshots or parts[-1] == mastersnapshotfile_name_json:
             if parts[-1] not in populated:
                 # Take the snapshot and crawl for the  resource types.
-                snapshot_file_data, snapshot_json_data = generate_snapshots_from_mastersnapshot_file(snapshot_file)
+                snapshot_file_data, snapshot_json_data = generate_snapshots_from_mastersnapshot_file(snapshot_file, container, parts[-1])
                 file_name = '%s.json' % snapshot_file if snapshot_file and not snapshot_file.endswith('.json') else snapshot_file
                 # snapshot_json_data = json_from_file(file_name)
                 generate_snapshot(snapshot_json_data, snapshot_file_data)
@@ -309,6 +324,7 @@ def generate_container_mastersnapshots_database(container, mastersnapshotfile=No
             for doc in docs:
                 if doc['json']:
                     snapshot = doc['name']
+                    doc['json'] = check_container_for_all_accounts(container, doc['json'], snapshot)
                     if "connector" in doc['json'] and "remoteFile" in doc['json'] and doc['json']["connector"] and doc['json']["remoteFile"]:
                         _, pull_response = pull_json_data(doc['json'])
                         if not pull_response: