Update python script and add terraform import script (SumoLogic#35)

* Write terraform resource reference for collector_ids and role_ids

* Only get the role names once and before beginning to write to the config file

* Fix how roles are written to file

* Add roles files output from script to gitignore

* Add script that imports resources into terraform

* Add docstrings to functions

* Add new line

* Fix return type for function

.gitignore

@@ -2,5 +2,5 @@
 .vscode
 collectors-resources.tf
 sources-resources.tf
-roles-resources.tf
 users-resources.tf
+*-role.tf

tools/get-existing-sumologic-resources/create-tf-config-file.py

@@ -6,7 +6,7 @@
 import requests as req
 from requests.auth import HTTPBasicAuth
 from sumologic import SumoLogic
-from typing import TextIO
+from typing import List, TextIO, Tuple
 
 
 """
@@ -41,6 +41,33 @@ def sumo_login() -> tuple:
         print('Credentials for Sumo not set in environment.')
         sys.exit(1)
 
+def get_role_names(versioned_endpoint: str, users: List[dict], credentials: tuple) -> dict:
+    """
+    Makes request to Sumo Logic api to get role names using the role ids in users
+    :param versioned_endpoint: Versioned endpoint to use for api calls
+    :param users: list of users returned from the Sumo Logic api
+    :param credentials: tuple with the Sumo Logic access ID and key
+    :return: dictionary mapping role ids to their corresponding role name
+    """
+    role_ids_to_names = {}
+    role_ids = []
+
+    # get all the role ids
+    for user in users:
+        role_ids += user["roleIds"]
+
+    # remove duplicates
+    role_ids = set(role_ids)
+
+    for role_id in role_ids:
+        role = req.get(f'{versioned_endpoint}/roles/{role_id}',
+                        auth=HTTPBasicAuth(credentials[0], credentials[1]))
+        if role.status_code != 200:
+            print (f'Could not get name for role id {role_id}')
+            return None
+        role_ids_to_names[role_id] = replace_invalid_chars(role.json()['name'])
+    return role_ids_to_names
+
 
 def get_sources(versioned_endpoint: str, source_type: str, resource_mapping: dict, credentials: tuple) -> dict:
     """
@@ -57,7 +84,7 @@ def get_sources(versioned_endpoint: str, source_type: str, resource_mapping: dic
     access_id = credentials[0]
     access_key = credentials[1]
 
-    # defined in config file but don't want to pass around entire resource_mappings dictionary
+    # defined in json file but don't want to pass around entire resource_mappings dictionary
     collectors_url = '/collectors/'
     collectors = req.get(f'{versioned_endpoint}{collectors_url}',
                             params={'filter': 'hosted'},
@@ -85,13 +112,13 @@ def get_sources(versioned_endpoint: str, source_type: str, resource_mapping: dic
     return {'sources': result}
 
 
-def get_sumo_resources(resource_type: str, resource_mapping: dict, credentials: tuple) -> dict:
+def get_sumo_resources(resource_type: str, resource_mapping: dict, credentials: tuple) -> Tuple[str, dict]:
     """
     Makes request for resources from Sumo Logic API
     :param resource_type: Sumo Logic resource type
     :param resource_mapping: dictionary containing relevant Terraform and Sumo Logic information
     :param credentials: tuple with the Sumo Logic access ID and key
-    :return: dictionary of Sumo Logic resources data
+    :return: tuple of versioned_endpoint string and dictionary of Sumo Logic roles data
     """
     valid_resource_types = ["collectors", "sources", "roles", "users"]
 
@@ -113,11 +140,11 @@ def get_sumo_resources(resource_type: str, resource_mapping: dict, credentials:
         if response.status_code != 200:
             print ("Status code not equal to 200. Check that your Sumo Logic credentials are set")
             return None
-        return response.json()
+        return versioned_endpoint, response.json()
 
     # There is no url field, so need to get sources
     else:
-        return get_sources(versioned_endpoint, resource_mapping['source_type'], resource_mapping, credentials)
+        return versioned_endpoint, get_sources(versioned_endpoint, resource_mapping['source_type'], resource_mapping, credentials)
 
 
 def replace_invalid_chars(resource_name: str) -> str:
@@ -134,7 +161,7 @@ def replace_invalid_chars(resource_name: str) -> str:
 
 def get_second_import_arg(resource_type: str, resource: dict) -> str:
     """
-    Constructs the second argument for terraform import based on the format expected for the given resource_type.
+    Constructs the second argument for terraform import based on the format expected for the given resource type
     :param resource_type: Sumo Logic resource type
     :param resource: a dictionary containing key, value pairs returned from the Sumo Logic api
     :return: The second argument to terraform import for the given resource
@@ -147,7 +174,7 @@ def get_second_import_arg(resource_type: str, resource: dict) -> str:
 
 def get_valid_resource_name(resource_type: str, resource: dict) -> str:
     """
-    Creates a unique terraform resource name to use in the config file.
+    Creates a unique terraform resource name to use in the config file
     :param resource_type: Sumo Logic resource type
     :param resource: a dictionary containing key, value pairs returned from the Sumo Logic api
     :return: A valid Terraform resource name
@@ -162,16 +189,65 @@ def get_valid_resource_name(resource_type: str, resource: dict) -> str:
             valid_resource_name = f'{resource["collector_name"]}_{valid_resource_name}'
         return valid_resource_name
 
+def write_tf_resource_ref_to_file(versioned_endpoint: str, resource_type: str, key: str, resource: dict, fp: TextIO, role_names: dict):
+    """
+    Writes resource to file using a reference to another terraform resource
+    :param versioned_endpoint: Versioned endpoint to use for api calls
+    :param resource_type: Sumo Logic resource type
+    :param key: the terraform argument to write to the file
+    :param resource: a dictionary containing key, value pairs returned from the Sumo Logic api
+    :param fp: File pointer for the file where resource will be written
+    :param role_names: a dictionary containing role names for each role id, equal to None if not writing users config
+    :return: None
+    """
+    val = ''
+    if resource_type == "sources":
+        val = f'sumologic_collector.{resource["collector_name"]}.id'
+        fp.write(f"""    {key} = {val}\n""")
 
-def write_resource_to_file(resource_type: str, resource: dict, resource_name: str, resource_mapping: dict, fp: TextIO):
+    # need to make an api call to Sumo Logic api to get role name
+    if resource_type == "users":
+        role_ids = resource["roleIds"]
+        role_ids.reverse()
+        for i, role_id in enumerate(role_ids):
+            role_name = role_names[role_id]
+            if i == 0:
+                fp.write(f"""    {key} = [sumologic_role.{role_name}.id""")
+            else:
+                fp.write(f""", sumologic_role.{role_name}.id""")
+        fp.write(f"""]\n""")
+
+def write_roles_to_files(versioned_endpoint: str, roles: List[dict], resource_mapping: dict, role_names: dict):
+    """
+    Writes each of the roles to separate files
+    :param versioned_endpoint: Versioned endpoint to use for api calls
+    :param roles: list of roles returned from the Sumo Logic api
+    :param resource_mapping: dictionary containing relevant Terraform and Sumo Logic information
+    :param role_names: a dictionary containing role names for each role id, equal to None if not writing users config
+    :return: None
+    """
+    for role in roles:
+        valid_resource_name = get_valid_resource_name("roles", role)
+        second_tf_import_arg = get_second_import_arg("roles", role)
+
+        # needed for Terraform import
+        print (f'{valid_resource_name}')
+        print (f'{second_tf_import_arg}')
+
+        with open(f'{valid_resource_name}-role.tf', 'w') as f:
+            write_resource_to_file(versioned_endpoint, "roles", role, valid_resource_name, resource_mapping, f, role_names)
+
+def write_resource_to_file(versioned_endpoint: str, resource_type: str, resource: dict, resource_name: str, resource_mapping: dict, fp: TextIO, role_names: dict):
     """
-    Writes resource to the given file.
+    Writes resource to the given file
+    :param versioned_endpoint: Versioned endpoint to use for api calls
     :param resource_type: Sumo Logic resource type
     :param resource: a dictionary containing key, value pairs returned from the Sumo Logic api
     :param resource_mapping: dictionary containing relevant Terraform and Sumo Logic information
     :param fp: File pointer for the file where resource will be written
     :return: None
     """
+
     fp.write(f'resource "{resource_mapping["tf_name"]}" "{resource_name}" {{\n')
     for arg in resource_mapping['tf_supported']:
         key, val = '', ''
@@ -180,18 +256,16 @@ def write_resource_to_file(resource_type: str, resource: dict, resource_name: st
         # convert from resource_type naming to Terraform naming
         if arg in resource_mapping['api_to_tf']:
             key = resource_mapping['api_to_tf'][arg]
-        # We need the collector id to be a reference to the collector resource
-        if resource_type == "sources" and arg == "collector_id":
-            val = f'sumologic_collector.{resource["collector_name"]}.id'
-            tf.write(f"""    {key} = {val}\n""")
-            continue
         if key:
             # write bool before checking val, otherwise values of false won't be written
             if isinstance(val, bool):
                     fp.write(f"""    {key} = {str(val).lower()}\n""")
             else:
                 if val:
-                    if isinstance(val, list):
+                    if (resource_type == "sources" and key == "collector_id") or (resource_type == "users" and key == "role_ids"):
+                        write_tf_resource_ref_to_file(versioned_endpoint, resource_type, key, resource, fp, role_names)
+                        continue
+                    elif isinstance(val, list):
                         if resource_type == "users":
                             val.reverse()
                         fp.write(f"""    {key} = {val}\n""".replace("'", '"'))
@@ -207,19 +281,29 @@ def write_resource_to_file(resource_type: str, resource: dict, resource_name: st
 
 def generate_tf_config(resource_type: str, resource_mapping: dict, credentials: tuple):
     """
-    Generates a Terraform config file for the given resource_type and prints arguments needed for terraform import.
+    Generates a Terraform config file for the given resource type and prints arguments needed for terraform import
     :param resource_type: Sumo Logic resource type
     :param resource_mapping: dictionary containing relevant Terraform and Sumo Logic information
     :param credentials: tuple with the Sumo Logic access ID and key
     :return: None
     """
-    data = get_sumo_resources(resource_type, resource_mapping, credentials)
+    versioned_endpoint, data = get_sumo_resources(resource_type, resource_mapping, credentials)
     if not data:
         print ("No data was returned from the Sumo Logic api")
         return None
 
     resources = data[resource_mapping['data_key']]
 
+    # need to get role names to reference in users config
+    role_names = None
+    if resource_type == "users":
+        role_names = get_role_names(versioned_endpoint, resources, credentials)
+
+    # roles should be written to separate files
+    if resource_type == "roles":
+        write_roles_to_files(versioned_endpoint, resources, resource_mapping, role_names)
+        return
+
     with open(f'{resource_type}-resources.tf', 'w') as tf:
         for resource in resources:
             valid_resource_name = get_valid_resource_name(resource_type, resource)
@@ -229,7 +313,7 @@ def generate_tf_config(resource_type: str, resource_mapping: dict, credentials:
             print (f'{valid_resource_name}')
             print (f'{second_tf_import_arg}')
 
-            write_resource_to_file(resource_type, resource, valid_resource_name, resource_mapping, tf)
+            write_resource_to_file(versioned_endpoint, resource_type, resource, valid_resource_name, resource_mapping, tf, role_names)
 
 if __name__ == "__main__":
     if len(sys.argv[1:]) != 1:

tools/get-existing-sumologic-resources/tf-import.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Can only import one resource type for each run of this script
+
+if [ -z "$1" ] || [ -z "$2" ] ; then
+    echo "Usage: $0 <file with output from create-tf-config-file.py> <resource type>";
+    echo "Resource type options: sumologic_collector, sumologic_http_source, sumologic_role, sumologic_user";
+    exit 1;
+fi
+
+filename=$1 # a file containing the output of the create-tf-config-file.py script
+resource_type=$2 # name of resource type in terraform, e.g. sumologic_collector
+
+# delete the first line in the file because it's the Sumo Logic api url
+sed '1d' $filename > tmpfile; mv tmpfile $filename
+
+terraform init --backend=false
+
+n=1
+while read line; do
+
+# even numbered lines contain the second argument needed for terraform import
+if [ $(($n%2)) == 0 ]; then
+    second_arg=$line
+    terraform import "$resource_type.$resource_name" $second_arg
+# odd numbered lines contain the name of the resource in terraform
+else
+    resource_name=$line
+fi
+n=$((n+1))
+done < $filename