Merge af92e24 into a07a839

sandbox_scripts/QualiEnvironmentUtils/ConfigFileManager.py

@@ -1,5 +1,6 @@
 # coding=utf-8
 from sandbox_scripts.QualiEnvironmentUtils.Sandbox import *
+from time import gmtime, strftime
 import traceback
 
 
@@ -26,30 +27,60 @@ def create_concrete_config_from_template(self, template_config_data, config_set_
         try:
             concrete_config_data = template_config_data
             # Replace {ConfigPool.PARAM} with PARAM's value from the pool
-            it = re.finditer(r"\{ConfigPool\.[^}]*\}", concrete_config_data, flags=re.IGNORECASE)
+            it = re.finditer(r"\{ConfigPool\:[^}]*\}", concrete_config_data, flags=re.IGNORECASE)
             for match in it:
                 param = match.group()
                 concrete_config_data = concrete_config_data.replace(param, config_set_pool_data[param.lower()])
 
+            # Replace {QUALI-NOTATION} WITH A NOTE
+            it = re.finditer(r"\{QUALI NOTATION\}", concrete_config_data,flags=re.IGNORECASE)
+            for match in it:
+                param = match.group()
+                quali_note = "Built from template: " + strftime("%Y-%b-%d %H:%M:%S", gmtime())
+                concrete_config_data = concrete_config_data.replace(param, quali_note)
+
+            # Replace {Device.Self.Name} with the resource's name
+            it = re.finditer(r"\{Device:Self:Name\}", concrete_config_data,flags=re.IGNORECASE)
+            for match in it:
+                param = match.group()
+                concrete_config_data = concrete_config_data.replace(param, resource.name)
+
             # Replace {Device.Self.Address} with the resource's management ip
-            it = re.finditer(r"\{Device.Self.Address\}", concrete_config_data,flags=re.IGNORECASE)
+            it = re.finditer(r"\{Device:Self:Address\}", concrete_config_data,flags=re.IGNORECASE)
             for match in it:
                 param = match.group()
                 concrete_config_data = concrete_config_data.replace(param, resource.address)
 
             # Replace {Device.Self.ATTRIBUTE_NAME} with the resource's attribute value
-            it = re.finditer(r"\{Device.Self\.[^}]*\}", concrete_config_data,flags=re.IGNORECASE)
+            # Need to decode password attributes: Password, Enable Password, and SNMP Read Community
+            it = re.finditer(r"\{Device:Self\:[^}]*\}", concrete_config_data,flags=re.IGNORECASE)
             for match in it:
                 param = match.group()
-                idx = param.rfind('.')+1
+                idx = param.rfind(':')+1
                 att_name = param[idx:len(param)-1]
                 param_val = resource.get_attribute(att_name)
                 #param_val = resource.get_attribute(param)
                 concrete_config_data = concrete_config_data.replace(param, param_val)
 
-            # TODO - Implement replacemant of params from types: {Device.NAME.Attribute_name}
+            # Replacemant of params from types: {Device:ALIAS:Attribute_name}
+            root_resources = self.sandbox.get_root_networking_resources()
+            it = re.finditer(r"\{Device:[^}]*\}", concrete_config_data, flags=re.IGNORECASE)
+            for match in it:
+                param = match.group()
+                junk, sb_alias, alias_attribname = param.split(":")
+                alias_attribname = alias_attribname.replace("}","")
+                concrete_name = ''
+                for resource in root_resources:
+                    if resource.alias == sb_alias:
+                        concrete_name = resource.name
+                        param_val = resource.get_attribute(alias_attribname)
+                        concrete_config_data = concrete_config_data.replace(param, param_val)
+                        break
+                if concrete_name <= ' ':
+                    raise ('did not find concrete device with alias ' + sb_alias + '; likely missing from blueprint.')
 
             return concrete_config_data
         except:
+            print str(Exception.message)
             self.sandbox.report_error("Failed to create a concrete config file from the template\'s data. "
                                       "Unexpected error: " + traceback.format_exc())

sandbox_scripts/QualiEnvironmentUtils/ConfigPoolManager.py

@@ -59,13 +59,13 @@ def push_data_from_pool_to_sandbox(self):
     def pool_data_to_dict(self):
         pool_data_dict = dict()
         for attribute in self.pool_resource.attributes:
-            pool_data_dict[str('{ConfigPool.' + attribute.Name + '}').lower()] = attribute.Value
+            pool_data_dict[str('{ConfigPool:' + attribute.Name + '}').lower()] = attribute.Value
         for resource_from_pool in self.pool_resource.details.ChildResources:
             split_name = resource_from_pool.Name.split('/')
             name_of_resource_from_pool = split_name[len(split_name)-1]
             #resource_attributes_dict = dict()
             for attribute in resource_from_pool.ResourceAttributes:
-                resource_dict_key = str('{ConfigPool.' + name_of_resource_from_pool + '.' + attribute.Name + '}').lower()
+                resource_dict_key = str('{ConfigPool:' + name_of_resource_from_pool + ':' + attribute.Name + '}').lower()
                 #resource_attributes_dict[resource_dict_key] = attribute.Value
                 pool_data_dict[resource_dict_key] = attribute.Value
         return pool_data_dict

sandbox_scripts/QualiEnvironmentUtils/RepositoryClients/GitLabClient.py

@@ -1,49 +1,64 @@
 from RepositoryClient import *
 from sandbox_scripts.QualiEnvironmentUtils.Sandbox import *
 import base64
+import pip
+
 try:
     import gitlab
     imported_gitlab = True
-except Exception as e:
-    imported_gitlab = False
+except:
+    try:
+        pip.main(["install","pyapi-gitlab"])
+        import gitlab
+    except:
+        imported_gitlab = False
+
 
 class GitLabClient(RepositoryClient):
+    # ----------------------------------
+    # ----------------------------------
     def __init__(self, sandbox,repository_resource ):
         super(GitLabClient,self).__init__(sandbox, repository_resource)
         if imported_gitlab == False:
-            self.sandbox.report_error('gitlab lib was not installed', write_to_output_window=True)
+            self.sandbox.report_error('gitlab lib was not found', write_to_output_window=True)
         self.url = repository_resource.get_attribute("GitLab URL")
         self.token = repository_resource.get_attribute("GitLab Token")
         self.project_name = repository_resource.get_attribute("GitLab Project Name")
+        self.repository_path = repository_resource.get_attribute("Repository Path")
 
+    # ----------------------------------
+    # ----------------------------------
     def download(self, source, destination):
-        #TODO - Implement this function
-        raise Exception("need to implement")
-
 
-'''
-    def RetrieveFileFromGit (GitURL, GitToken, GitProjectName, TemplateName):
-        Git = gitlab.Gitlab(GitURL, GitToken)
-        Gitprojid = 0
-        Gitprojects = ''
+        gl = gitlab.Gitlab(self.url, self.token)
+        projid = 0
+        projects = ''
         try:
-            for project in Git.getall(Git.getprojects):
+            for project in gl.getall(gl.getprojects):
                 #print project['name']
-                if project['name'] == GitProjectName:
-                    Gitprojid = project['id']
+                if project['name'] == self.project_name:
+                    projid = project['id']
                     break
                 else:
-                    Gitprojects += str(project['id']) + '-' + project['name'] + '\n'
-        except:
-            return 3, "Could not access repository at %s" % GitURL
+                    projects += str(project['id']) + '-' + project['name'] + '\n'
+        except Exception as ex:
+            raise QualiError("GitLabClient", "ERROR: Could not access repository at %s" % self.url + " : " + str(ex.message))
 
-        if Gitprojid == 0:
-            return 2, "Failed to locate project by name among \n" + Gitprojects
+        if projid == 0:
+            raise QualiError("GitLabClient","ERROR: Failed to locate project by name among \n" + projects)
+
+        try:
+            source = self.repository_path + source
+            filebase64 = gl.getfile(projid, source, 'master')
+            filetext = base64.b64decode(filebase64['content']).decode()
+        except Exception as ex:
+            raise QualiError("GitLabClient", "ERROR: Failed to retrieve file, which may be expected for " +
+                             source + " : " + str(ex.message))
 
         try:
-            tmplt64 = Git.getfile(Gitprojid, TemplateName, 'master')
-            CfgTemplate = base64.b64decode(tmplt64['content']).decode()
-            return 0, CfgTemplate
-        except:
-            return 1, "Failed to retrieve file."
-'''
+            with open(destination,'w') as dest:
+                    dest.write(filetext)
+            print "Downloaded: " + source
+            return 0, "Successfully retrieved file from repository and saved to destination"
+        except Exception as ex:
+            raise QualiError("GitLabClient","ERROR: Retrieved file from repository - failed to save to destination : " + str(ex.message))

sandbox_scripts/QualiEnvironmentUtils/RepositoryClients/RepositoryClient.py

@@ -9,7 +9,6 @@ def __init__(self, sandbox, repository_resource):
         self.sandbox = sandbox
         self.repository_resource = repository_resource
 
-
     @abstractmethod
     def download(self, source, destination):
        raise NotImplementedError('subclasses must override download()!')

sandbox_scripts/QualiEnvironmentUtils/Resource.py

@@ -5,7 +5,6 @@
 from cloudshell.api.cloudshell_api import *
 from cloudshell.api.common_cloudshell_api import *
 from QualiUtils import *
-
 import datetime
 import json
 
@@ -57,7 +56,11 @@ def get_attribute(self, attribute_name):
         attribute_name = attribute_name.lower()
         for attribute in self.attributes:
             if attribute.Name.lower() == attribute_name:
-                return attribute.Value
+                if attribute.Type == 'Password':
+                    decrypted = self.api_session.DecryptPassword(attribute.Value)
+                    return decrypted.Value
+                else:
+                    return attribute.Value
         raise QualiError(self.name, "Attribute: " + attribute_name + " not found")
 
     # -----------------------------------------
@@ -96,9 +99,9 @@ def health_check(self,reservation_id):
         if self.has_command('health_check'):
             try:
                 # Return a detailed description in case of a failure
-                out = self.execute_command(reservation_id, 'health_check', printOutput=False)#.Output()
+                out = self.execute_command(reservation_id, 'health_check', printOutput=False) #.Output()
                 if out.Output.find(' passed') == -1:
-                    err = "Health check did not pass for device " + self.name + ". " + out
+                    err = "Health check did not pass for device " + self.name + ". "  +out.Output
                     return err
 
             except QualiError as qe:
@@ -110,17 +113,38 @@ def health_check(self,reservation_id):
     # -----------------------------------------
     def load_network_config(self, reservation_id, config_path, config_type, restore_method='Override'):
         """
-        Load config from a configuration file on the device
+        Load config from a configuration file onto the device
         :param str reservation_id:  Reservation id.
         :param config_path:  The path to the config file
         :param config_type:  StartUp or Running
         :param restore_method:  Optional. Restore method. Can be Append or Override.
         """
         # Run executeCommand with the restore command and its params (ConfigPath,RestoreMethod)
         try:
-            command_inputs = [InputNameValue('path', str(config_path)),
-                              InputNameValue('restore_method', str(restore_method)),
-                              InputNameValue('configuration_type', str(config_type))]
+            the_path = "undef"
+            the_cfgtype = "undef"
+            the_restoremeth = "undef"
+            for command in self.commands:
+                if command.Name == 'restore':
+                    for parm in command.Parameters:
+                        if parm.Name in ["path", "src_Path"]:
+                            the_path = parm.Name
+                        if parm.Name in ["configuration_type","config_type"]:
+                            the_cfgtype = parm.Name
+                        if parm.Name in ["restore_method"]:
+                            the_restoremeth = parm.Name
+
+            if the_path == "undef" or the_cfgtype == "undef" or the_restoremeth == "undef":
+                raise QualiError(self.name, "Failed to find viable restore command for " + self.name \
+                      + " : " + the_path + ", " + the_cfgtype + ", " + the_restoremeth)
+
+        except QualiError as qerror:
+            raise QualiError(self.name, "Failed building restore command input parm names." + qerror.message)
+
+        try:
+            command_inputs = [InputNameValue(the_path, str(config_path)),
+                              InputNameValue(the_cfgtype, str(config_type)),
+                              InputNameValue(the_restoremeth, str(restore_method))]
 
             if self.attribute_exist('VRF Management Name'):
                 vrf_name = self.get_attribute('VRF Management Name')
@@ -131,38 +155,11 @@ def load_network_config(self, reservation_id, config_path, config_type, restore_
                                  commandInputs=command_inputs,
                                  printOutput=True)
 
-        except:
-            try:
-                command_inputs = [InputNameValue('src_Path', str(config_path)),
-                                    InputNameValue('restore_method', str(restore_method)),
-                                    InputNameValue('config_type', str(config_type))]
-
-                if self.attribute_exist('VRF Management Name'):
-                    vrf_name = self.get_attribute('VRF Management Name')
-                    if vrf_name !='':
-                        command_inputs.append(InputNameValue('vrf_management_name', str(vrf_name)))
-
-                self.execute_command(reservation_id, 'Restore',
-                                     commandInputs=command_inputs,
-                                     printOutput=True)
-            except:
-                try:
-                    command_inputs = [InputNameValue('path', str(config_path)),
-                                        InputNameValue('restore_method', str(restore_method)),
-                                        InputNameValue('config_type', str(config_type))]
-
-                    if self.attribute_exist('VRF Management Name'):
-                        vrf_name = self.get_attribute('VRF Management Name')
-                        if vrf_name !='':
-                            command_inputs.append(InputNameValue('vrf', str(vrf_name)))
+        except QualiError as qerror:
+            raise QualiError(self.name, "Failed to load configuration: " + qerror.message)
 
-                    self.execute_command(reservation_id, 'restore',
-                                         commandInputs=command_inputs,
-                                         printOutput=True)
-                except QualiError as qerror:
-                    raise QualiError(self.name, "Failed to load configuration: " + qerror.message)
-                except:
-                    raise QualiError(self.name, "Failed to load configuration. Unexpected error:" + str(sys.exc_info()[0]))
+        except:
+            raise "Failed to load configuration. Unexpected error:" + str(sys.exc_info()[0])
 
     # -----------------------------------------
     # -----------------------------------------
@@ -351,7 +348,10 @@ def execute_connected_command(self, reservation_id, commandName,tag,commandInput
         if self.connected_commands.__sizeof__() > 0:
             # Run executeCommand with the restore command and its params (ConfigPath,RestoreMethod)
             try:
-               return self.api_session.ExecuteResourceConnectedCommand(reservation_id,self.name,commandName=commandName,commandTag=tag,parameterValues = commandInputs)
+               return self.api_session.ExecuteResourceConnectedCommand(reservation_id,self.name,
+                                                                       commandName=commandName,
+                                                                       commandTag=tag,
+                                                                       parameterValues = commandInputs)
 
             except CloudShellAPIError as error:
                 raise QualiError(self.name, error.message)
@@ -450,5 +450,5 @@ def get_version(self, reservation_id):
             return  version
         except QualiError as qerror:
             raise QualiError(self.name, "Failed to get the version: " + qerror.message)
-        except:
-            raise QualiError(self.name, "Failed to get the version. Unexpected error:" + str(sys.exc_info()[0]))
+        except Exception as ex:
+            raise QualiError(self.name, "Failed to get the version. Unexpected error:" + str(ex.message))

sandbox_scripts/QualiEnvironmentUtils/StorageClients/FTPClient.py

@@ -70,15 +70,17 @@ def upload(self, destination, source):
             file_idx=destination.rfind('/')
             destination_dir = destination[:(file_idx-len(destination))]
             destination_dir = self._remove_header(destination_dir)
+            destination_file = destination[file_idx+1:]
             #destination_dir = destination_dir.replace('//','/')
             self.ftp.cwd(destination_dir)
-            destination_file =destination[file_idx+1:]
             myfile = open(source, 'r')
             self.ftp.storlines('STOR ' + destination_file, myfile)
             myfile.close()
+            self.sandbox.report_info("Successfully uploaded " + source + " to " + destination_file)
         except Exception as e:
-            self.sandbox.report_error("Failed to upload file " + myfile + " to " + destination_file +
+            self.sandbox.report_error("Failed to upload file " + source + " to " + destination_file +
                                       " on  the FTP server. Error is: " + str(e), raise_error=True)
+            print "Failed to upload file " + source + " to " + destination_file + " on  the FTP server. Error is: " + str(e)
 
 
     # ----------------------------------
@@ -96,8 +98,9 @@ def dir_exist(self, dir_name):
     # ----------------------------------
     # ----------------------------------
     def _remove_header(self, path):
-        #path = path.replace('ftp://' + self.username + ':' + self.password + '@' + self.address +"/",'')
-        path = path.replace('ftp://' + self.username + ':' + self.password + '@' + self.address ,'')
+        # Jim: need the line with '/' added to replace - it was set to the other line!
+        path = path.replace('ftp://' + self.username + ':' + self.password + '@' + self.address +"/",'')
+        #path = path.replace('ftp://' + self.username + ':' + self.password + '@' + self.address,'')
         path = path.replace(' ', '_')
         return path
 

sandbox_scripts/QualiEnvironmentUtils/StorageClients/TFTPClient.py

@@ -3,12 +3,17 @@
 from StorageClient import *
 from sandbox_scripts.QualiEnvironmentUtils.Sandbox import *
 import tempfile
+import pip
 
 try:
     import tftpy
     imported_tftpy = True
-except Exception as e:
-    imported_tftpy = False
+except:
+    try:
+        pip.main(["install","tftpy"])
+        import tftpy
+    except:
+        imported_tftpy = False
 
 
 class TFTPClient(StorageClient):
@@ -17,7 +22,7 @@ class TFTPClient(StorageClient):
     def __init__(self, sandbox, storage_resource):
         super(TFTPClient,self).__init__(sandbox, storage_resource)
         if imported_tftpy == False:
-            self.sandbox.report_error('TFTPY lib was not installed', write_to_output_window=True)
+            self.sandbox.report_error('tftpy lib was not installed', write_to_output_window=True)
 
         self.username = storage_resource.get_attribute("Storage username")
         self.password = storage_resource.get_attribute("Storage password")