From c8a8eac826647e3ecea517ff8ef9de82c4902030 Mon Sep 17 00:00:00 2001 From: Rajendra Dendukuri Date: Mon, 26 Sep 2022 07:25:40 -0700 Subject: [PATCH] [ZTP] Improvements to allow in-band zero touch provisioning - Download all plugins before processing each configuration sections. This ensures that configuration section plugins are available at all times. If the plugin of a configuration section needs to be downloaded at a later time when ZTP is in-progress and the configuration section is actually processed, set the field "pre-ztp-plugin-download" : false in the corresponding configuration section data in ztp.json. - Additional unit test cases added --- .../python3/dist-packages/ztp/Downloader.py | 6 +- .../lib/python3/dist-packages/ztp/ZTPLib.py | 2 +- .../python3/dist-packages/ztp/ZTPSections.py | 2 +- .../lib/python3/dist-packages/ztp/defaults.py | 1 + src/usr/lib/ztp/dhcp/inband-ztp-ip | 6 +- src/usr/lib/ztp/ztp-engine.py | 92 +++++++++++++++---- tests/test_ZTPJson.py | 18 ++++ tests/test_ztp_engine.py | 41 +++++++++ 8 files changed, 143 insertions(+), 25 deletions(-) diff --git a/src/usr/lib/python3/dist-packages/ztp/Downloader.py b/src/usr/lib/python3/dist-packages/ztp/Downloader.py index 7f3348d..08ef3d1 100644 --- a/src/usr/lib/python3/dist-packages/ztp/Downloader.py +++ b/src/usr/lib/python3/dist-packages/ztp/Downloader.py @@ -247,6 +247,10 @@ def getUrl(self, url=None, dst_file=None, incl_http_headers=None, is_secure=True else: break - os.chmod(dst_file, stat.S_IRWXU) + try: + os.chmod(dst_file, stat.S_IRWXU) + except FileNotFoundError: + return (20, None) + # Use curl result return (0, dst_file) diff --git a/src/usr/lib/python3/dist-packages/ztp/ZTPLib.py b/src/usr/lib/python3/dist-packages/ztp/ZTPLib.py index 5882ee2..f2354ce 100644 --- a/src/usr/lib/python3/dist-packages/ztp/ZTPLib.py +++ b/src/usr/lib/python3/dist-packages/ztp/ZTPLib.py @@ -95,7 +95,7 @@ def runCommand(cmd, capture_stdout=True, use_shell=False): else: shcmd = cmd if capture_stdout is True: - proc = subprocess.Popen(shcmd, shell=use_shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, close_fds=True) + proc = subprocess.Popen(shcmd, shell=use_shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) pid = proc.pid runcmd_pids.append(pid) output_stdout, output_stderr = proc.communicate() diff --git a/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py b/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py index 0da76d3..a5fee48 100644 --- a/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py +++ b/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py @@ -91,7 +91,7 @@ def __buildDefaults(self, section): @param section (dict) Configuration Section input data read from JSON file. ''' - default_objs = ['ignore-result', 'reboot-on-success', 'reboot-on-failure', 'halt-on-failure'] + default_objs = ['ignore-result', 'reboot-on-success', 'reboot-on-failure', 'halt-on-failure', 'pre-ztp-plugin-download'] # Loop through objects and update them with default values for key in default_objs: _val = getField(section, key, bool, getCfg(key)) diff --git a/src/usr/lib/python3/dist-packages/ztp/defaults.py b/src/usr/lib/python3/dist-packages/ztp/defaults.py index 34464d3..710375e 100644 --- a/src/usr/lib/python3/dist-packages/ztp/defaults.py +++ b/src/usr/lib/python3/dist-packages/ztp/defaults.py @@ -48,6 +48,7 @@ "log-file" : "/var/log/ztp.log", \ "log-level" : "INFO", \ "monitor-startup-config" : True, \ + "pre-ztp-plugin-download" : True, \ "restart-ztp-interval": 300, \ "reboot-on-success" : False, \ "reboot-on-failure" : False, \ diff --git a/src/usr/lib/ztp/dhcp/inband-ztp-ip b/src/usr/lib/ztp/dhcp/inband-ztp-ip index f86ac48..2eef315 100755 --- a/src/usr/lib/ztp/dhcp/inband-ztp-ip +++ b/src/usr/lib/ztp/dhcp/inband-ztp-ip @@ -38,7 +38,7 @@ case $reason in if inband_interface_check ${interface} ; then if [ -n "$old_ip_address" ] && [ -n "$old_subnet_mask" ]; then prefix=$(IPprefix_by_netmask "${old_subnet_mask}") - config interface ip remove ${interface} ${old_ip_address}${prefix} + /usr/local/bin/config interface ip remove ${interface} ${old_ip_address}${prefix} fi fi ;; @@ -47,11 +47,11 @@ case $reason in if [ -n "$new_ip_address" ] && [ -n "$new_subnet_mask" ]; then if [ -n "$old_ip_address" ] && [ -n "$old_subnet_mask" ]; then prefix=$(IPprefix_by_netmask "${old_subnet_mask}") - config interface ip remove ${interface} ${old_ip_address}${prefix} + /usr/local/bin/config interface ip remove ${interface} ${old_ip_address}${prefix} fi prefix=$(IPprefix_by_netmask "${new_subnet_mask}") if [ "${prefix}" != "/0" ]; then - config interface ip add ${interface} ${new_ip_address}${prefix} + /usr/local/bin/config interface ip add ${interface} ${new_ip_address}${prefix} fi fi fi diff --git a/src/usr/lib/ztp/ztp-engine.py b/src/usr/lib/ztp/ztp-engine.py index 1d6ad89..4bdfa9c 100755 --- a/src/usr/lib/ztp/ztp-engine.py +++ b/src/usr/lib/ztp/ztp-engine.py @@ -432,13 +432,55 @@ def __evalZTPResult(self): # Check reboot on result flags and take action self.__rebootAction(self.objztpJson.ztpDict, delayed_reboot=True) + def __downloadPlugins(self): + '''! + Check and download plugins used by configuration sections + @return False - If failed to download one more plugins of a required configuration section + True - Successfully downloaded plugins of all configuration sections + ''' + + # Obtain a copy of the list of configuration sections + section_names = list(self.objztpJson.section_names) + abort = False + logger.debug('Verifying and downloading plugins user by configuration sections: %s' % ', '.join(section_names)) + for sec in sorted(section_names): + section = self.objztpJson.ztpDict.get(sec) + t = getTimestamp() + try: + # Retrieve individual section's progress + sec_status = section.get('status') + download_check = section.get('pre-ztp-plugin-download') + if sec_status == 'BOOT' and download_check: + logger.info('Verifying and downloading plugin used by the configuration section %s.' % (sec)) + updateActivity('Verifying and downloading plugin used by the configuration section %s' % sec) + # Get the appropriate plugin to be used for this configuration section + plugin = self.objztpJson.plugin(sec) + if plugin is None: + # Mark section status as failed + section['error'] = 'Unable to find or download requested plugin' + section['start-timestamp'] = t + self.objztpJson.updateStatus(section, 'FAILED') + if not section.get('ignore-result'): + abort = True + + except: + logger.debug('Exception [%s] encountered while verifying plugin for configuration section %s.' % (str(e), sec)) + logger.info('Exception encountered while verifying plugin for configuration section %s. Marking it as FAILED.' % sec) + section['error'] = 'Exception [%s] encountered while verifying the plugin' % (str(e)) + section['start-timestamp'] = t + self.objztpJson.updateStatus(section, 'FAILED') + if not section.get('ignore-result'): + abort = True + return abort + def __processConfigSections(self): '''! Process and execute individual configuration sections defined in ZTP JSON. Plugin for each configuration section is resolved and executed. Configuration section data is provided as command line argument to the plugin. Each and every section is processed before this function returns. - + @return False - If error encountered processing configuration sections and request restarting ZTP + True - If processing of configuration sections has been completed ''' # Obtain a copy of the list of configuration sections @@ -447,6 +489,9 @@ def __processConfigSections(self): # set temporary flags abort = False sort = True + if self.__downloadPlugins(): + logger.info('Halting ZTP as download of one or more plugins FAILED.') + return False logger.debug('Processing configuration sections: %s' % ', '.join(section_names)) # Loop through each sections till all of them are processed @@ -537,6 +582,7 @@ def __processConfigSections(self): # Check reboot on result flags self.__rebootAction(section) + return True def __processZTPJson(self): '''! @@ -598,32 +644,40 @@ def __processZTPJson(self): self.__loadZTPProfile("resume") # Process available configuration sections in ZTP JSON - self.__processConfigSections() - - # Determine ZTP result - self.__evalZTPResult() - - # Check restart ZTP condition - # ZTP result is failed and restart-ztp-on-failure is set or - _restart_ztp_on_failure = (self.objztpJson['status'] == 'FAILED' and \ - self.objztpJson['restart-ztp-on-failure'] == True) - - # ZTP completed and no startup-config is found, restart-ztp-no-config and config-fallback is not set - _restart_ztp_missing_config = ( (self.objztpJson['status'] == 'SUCCESS' or self.objztpJson['status'] == 'FAILED') and \ - self.objztpJson['restart-ztp-no-config'] == True and \ - self.objztpJson['config-fallback'] == False and - os.path.isfile(getCfg('config-db-json')) is False ) + _processing_completed = self.__processConfigSections() + # In test mode always mark processing as completed + if self.test_mode: + _processing_completed = True + + _restart_ztp_missing_config = False + _restart_ztp_on_failure = False + if _processing_completed: + # Determine ZTP result + self.__evalZTPResult() + + # Check restart ZTP condition + # ZTP result is failed and restart-ztp-on-failure is set or + _restart_ztp_on_failure = (self.objztpJson['status'] == 'FAILED' and \ + self.objztpJson['restart-ztp-on-failure'] == True) + + # ZTP completed and no startup-config is found, restart-ztp-no-config and config-fallback is not set + _restart_ztp_missing_config = ( (self.objztpJson['status'] == 'SUCCESS' or self.objztpJson['status'] == 'FAILED') and \ + self.objztpJson['restart-ztp-no-config'] == True and \ + self.objztpJson['config-fallback'] == False and + os.path.isfile(getCfg('config-db-json')) is False ) # Mark ZTP for restart - if _restart_ztp_missing_config or _restart_ztp_on_failure: + if not _processing_completed or _restart_ztp_missing_config or _restart_ztp_on_failure: os.remove(getCfg('ztp-json')) if os.path.isfile(getCfg('ztp-json-shadow')): - os.remove(getCfg('ztp-json-shadow')) + os.remove(getCfg('ztp-json-shadow')) self.objztpJson = None # Remove startup-config file to obtain a new one through ZTP if getCfg('monitor-startup-config') is True and os.path.isfile(getCfg('config-db-json')): os.remove(getCfg('config-db-json')) - if _restart_ztp_missing_config: + if not _processing_completed: + return ("restart", "Restarting ZTP due to error processing configuration sections") + elif _restart_ztp_missing_config: return ("restart", "ZTP completed but startup configuration '%s' not found" % (getCfg('config-db-json'))) elif _restart_ztp_on_failure: return ("restart", "ZTP completed with FAILED status") diff --git a/tests/test_ZTPJson.py b/tests/test_ZTPJson.py index b008de8..c38e215 100644 --- a/tests/test_ZTPJson.py +++ b/tests/test_ZTPJson.py @@ -191,6 +191,7 @@ def test_ztp_dynamic_url_invalid_arg_type(self, tmpdir): "source": "/tmp/test_firmware_%s.json" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -199,6 +200,7 @@ def test_ztp_dynamic_url_invalid_arg_type(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, @@ -305,6 +307,7 @@ def test_ztp_non_existent_plugin_section_name(self, tmpdir): "source": "http://localhost:2000/ztp/scripts/post_install.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -313,6 +316,7 @@ def test_ztp_non_existent_plugin_section_name(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, @@ -459,6 +463,7 @@ def test_ztp_url_reusing_plugin(self, tmpdir): "source": "file:///tmp/test_firmware.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -467,6 +472,7 @@ def test_ztp_url_reusing_plugin(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, @@ -526,6 +532,7 @@ def test_ztp_url_reusing_plugin_2(self, tmpdir): "destination": "/tmp/firmware_check.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -533,6 +540,7 @@ def test_ztp_url_reusing_plugin_2(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -682,6 +690,7 @@ def test_ztp_url_could_not_interpreted(self, tmpdir): "source": "http://localhost:2000/ztp/scripts/post_install.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -689,6 +698,7 @@ def test_ztp_url_could_not_interpreted(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": None, @@ -765,6 +775,7 @@ def test_ztp_return_another_invalid_url_section(self, tmpdir): "source": "http://localhost:2000/ztp/scripts/post_install.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -772,6 +783,7 @@ def test_ztp_return_another_invalid_url_section(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": None, @@ -813,6 +825,7 @@ def test_ztp_dynamic_url_download(self, tmpdir): } } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -820,6 +833,7 @@ def test_ztp_dynamic_url_download(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -874,6 +888,7 @@ def test_ztp_dynamic_url_download_2(self, tmpdir): } } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -881,6 +896,7 @@ def test_ztp_dynamic_url_download_2(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -927,6 +943,7 @@ def test_ztp_graphservice_do_not_exist(self, tmpdir): "halt-on-failure": false, "ignore-result": false, "plugin": "graphservice", + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -935,6 +952,7 @@ def test_ztp_graphservice_do_not_exist(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, diff --git a/tests/test_ztp_engine.py b/tests/test_ztp_engine.py index 7d73d9c..091c028 100644 --- a/tests/test_ztp_engine.py +++ b/tests/test_ztp_engine.py @@ -448,6 +448,47 @@ def test_ztp_json_ignore_fail(self): self.cfgSet('monitor-startup-config', True) self.cfgSet('restart-ztp-no-config', True) + def test_ztp_json_invalid_plugins(self): + '''! + Simple ZTP test with 3-sections, invalid plugin in the second section, ZTP Failed + ''' + content = """{ + "ztp": { + "0001-test-plugin": { + "message" : "0001-test-plugin", + "message-file" : "/etc/ztp.results" + }, + "0002-invalid-plugin": { + "pre-ztp-plugin-download" : false + }, + "0003-test-plugin": { + "pre-ztp-plugin-download" : false, + "message" : "0003-test-plugin", + "message-file" : "/etc/ztp.results" + } + } +}""" + expected_result = """0001-test-plugin +0003-test-plugin +""" + self.__init_ztp_data() + self.cfgSet('monitor-startup-config', False) + self.cfgSet('restart-ztp-no-config', False) + self.__write_file("/tmp/ztp_input.json", content) + self.__write_file(self.cfgGet("opt67-url"), "file:///tmp/ztp_input.json") + runCommand(COVERAGE + ZTP_ENGINE_CMD) + runCommand(COVERAGE + ZTP_CMD + ' status -v') + os.remove("/tmp/ztp_input.json") + objJson, jsonDict = JsonReader(self.cfgGet('ztp-json'), indent=4) + assert(jsonDict.get('ztp').get('status') == 'FAILED') + assert(jsonDict.get('ztp').get('0001-test-plugin').get('status') == 'SUCCESS') + assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('status') == 'FAILED') + assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('error') == 'Unable to find or download requested plugin') + result = self.__read_file("/etc/ztp.results") + assert(result == expected_result) + self.cfgSet('monitor-startup-config', True) + self.cfgSet('restart-ztp-no-config', True) + def test_ztp_json_ignore_ztp_success(self): '''! Simple ZTP test with 3-sections, Failure in 3 sections but ignore-result set in 2sections, ignore-result set in ztp, ZTP Success