diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 222e2c323..f4902f1eb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -56,6 +56,9 @@ parameters: - name: sensormond root_dir: sonic-sensormond python3: true + - name: stormond + root_dir: sonic-stormond + python3: true - name: artifactBranch type: string default: 'refs/heads/master' diff --git a/sonic-stormond/pytest.ini b/sonic-stormond/pytest.ini new file mode 100644 index 000000000..d90ee9ed9 --- /dev/null +++ b/sonic-stormond/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --cov=scripts --cov-report html --cov-report term --cov-report xml --junitxml=test-results.xml -vv diff --git a/sonic-stormond/scripts/stormond b/sonic-stormond/scripts/stormond new file mode 100755 index 000000000..dd1834156 --- /dev/null +++ b/sonic-stormond/scripts/stormond @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 + +""" + stormond + Storage device Monitoring daemon for SONiC +""" + +import os +import signal +import sys +import threading +import subprocess +import shutil +import json +import time + +from sonic_py_common import daemon_base, device_info, syslogger +from swsscommon import swsscommon +from sonic_platform_base.sonic_storage.storage_devices import StorageDevices, BLKDEV_BASE_PATH + +# +# Constants ==================================================================== +# + +SYSLOG_IDENTIFIER = "stormond" + +STORAGE_DEVICE_TABLE = "STORAGE_INFO" +FSSTATS_SYNC_TIME_KEY = "FSSTATS_SYNC" + +# This directory binds to /host/pmon/stormond/ on the host +FSIO_RW_JSON_FILE = "/usr/share/stormond/fsio-rw-stats.json" + +STORMOND_PERIODIC_STATEDB_SYNC_SECS = 3600 #one hour +STORMOND_SYNC_TO_DISK_SECS = 86400 #one day + +STORAGEUTIL_LOAD_ERROR = 127 + +exit_code = 0 + +# +# Daemon ======================================================================= +# + + +class DaemonStorage(daemon_base.DaemonBase): + + def __init__(self, log_identifier): + + self.log = syslogger.SysLogger(SYSLOG_IDENTIFIER) + super(DaemonStorage, self).__init__(log_identifier) + + self.timeout = STORMOND_PERIODIC_STATEDB_SYNC_SECS + self.fsstats_sync_interval = STORMOND_SYNC_TO_DISK_SECS + self.stop_event = threading.Event() + self.state_db = None + self.config_db = None + self.device_table = None + self.storage = StorageDevices() + + # These booleans are for FSIO RW information reconciliation + self.fsio_json_file_loaded = False + self.statedb_storage_info_loaded = False + + self.use_fsio_json_baseline = False + self.use_statedb_baseline = False + + # These dicts are to load info from disk/database into memory, respectively + self.fsio_rw_json = {disk:{} for disk in self.storage.devices} + self.fsio_rw_statedb = {disk:{} for disk in self.storage.devices} + + # This time is set at init and then subsequently after each FSIO JSON file sync + self.fsio_sync_time = time.time() + + # These are the various static and dynamic fields that are posted to state_db + self.static_fields = ["device_model", "serial"] + self.dynamic_fields = ["firmware", \ + "health", \ + "temperature", \ + "latest_fsio_reads", \ + "latest_fsio_writes", \ + "total_fsio_reads", \ + "total_fsio_writes", \ + "disk_io_reads", \ + "disk_io_writes", \ + "reserved_blocks"] + + # These are the fields that we are interested in saving to disk to protect against + # reboots or crashes + self.statedb_json_sync_fields = self.dynamic_fields[3:7] + + # Connect to STATE_DB and create Storage device table + self.state_db = daemon_base.db_connect("STATE_DB") + self.device_table = swsscommon.Table(self.state_db, STORAGE_DEVICE_TABLE) + + # Load the FSIO RW values from state_db and JSON file and reconcile latest information + self._load_fsio_rw_statedb() + self._load_fsio_rw_json() + self._determine_sot() + + def get_configdb_intervals(self): + self.config_db = daemon_base.db_connect("CONFIG_DB") + config_info = dict(self.config_db.hgetall('STORMOND_CONFIG|INTERVALS')) + self.timeout = int(config_info.get('daemon_polling_interval', STORMOND_PERIODIC_STATEDB_SYNC_SECS)) + self.fsstats_sync_interval = int(config_info.get('fsstats_sync_interval', STORMOND_SYNC_TO_DISK_SECS)) + + self.log_info("Polling Interval set to {} seconds".format(self.timeout)) + self.log_info("FSIO JSON file Interval set to {} seconds".format(self.fsstats_sync_interval)) + + + # Get the total and latest FSIO reads and writes from JSON file + def _load_fsio_rw_json(self): + try: + if not os.path.exists(FSIO_RW_JSON_FILE): + self.log_info("{} not present.".format(FSIO_RW_JSON_FILE)) + return + + # Load JSON file + with open(FSIO_RW_JSON_FILE, 'r') as f: + self.fsio_rw_json = json.load(f) + + # Verify that none of the values in the JSON file are None + for storage_device in self.storage.devices: + for field in self.statedb_json_sync_fields: + + if self.fsio_rw_json[storage_device][field] == None: + self.log_warning("{}:{} value = None in JSON file".format(storage_device, field)) + return + + self.fsio_json_file_loaded = True + + except Exception as e: + self.log_error("JSON file could not be loaded: {}".format(str(e))) + + return + + + # Sync the total and latest procfs reads and writes from STATE_DB to JSON file on disk + def sync_fsio_rw_json(self): + + self.log_info("Syncing total and latest procfs reads and writes from STATE_DB to JSON file") + + json_file_dict = {disk:{} for disk in self.storage.devices} + try: + for device in self.storage.devices: + for field in self.statedb_json_sync_fields: + json_file_dict[device][field] = self.state_db.hget('STORAGE_INFO|{}'.format(device), field) + + self.fsio_sync_time = time.time() + json_file_dict["successful_sync_time"] = str(self.fsio_sync_time) + + with open(FSIO_RW_JSON_FILE, 'w+') as f: + json.dump(json_file_dict, f) + + return True + + except Exception as ex: + self.log_error("Unable to sync state_db to disk: {}".format(str(ex))) + return False + + + # Update the successful sync time to STATE_DB + def write_sync_time_statedb(self): + self.state_db.hset("{}|{}".format(STORAGE_DEVICE_TABLE,FSSTATS_SYNC_TIME_KEY), "successful_sync_time", str(self.fsio_sync_time)) + + # Run a sanity check on the state_db. If successful, get total, latest + # FSIO reads and writes for each storage device from STATE_DB + def _load_fsio_rw_statedb(self): + + # Sanity Check: + + # If the number of STORAGE_INFO|* keys does not equal the + # number of storage disks on the device + FSSTATS_SYNC field, + # there has been a corruption to the database. In this case we + # pivot to the JSON file being the Source of Truth. + try: + if (len(self.state_db.keys("STORAGE_INFO|*")) != (len(self.storage.devices) + 1)): + return + + # For each storage device on the switch, + for storage_device in self.storage.devices: + + # Get the total and latest procfs reads and writes from STATE_DB + for field in self.statedb_json_sync_fields: + value = self.state_db.hget('STORAGE_INFO|{}'.format(storage_device), field) + self.fsio_rw_statedb[storage_device][field] = "0" if value is None else value + + if value is None: + self.log_warning("{}:{} value = None in StateDB".format(storage_device, field)) + return + + self.statedb_storage_info_loaded = True + except Exception as e: + self.log_error("Reading STATE_DB failed with: {}".format(str(e))) + + + def _determine_sot(self): + + # This daemon considers the storage information values held in the STATE_DB to be its + # Source of Truth. + + # If stormond is coming back up after a daemon crash, storage information would be saved in the + # STATE_DB. In that scenario, we use the STATE_DB information as the SoT to reconcile the FSIO + # reads and writes values. + if self.statedb_storage_info_loaded == True: + self.use_fsio_json_baseline = False + self.use_statedb_baseline = True + + # If the state_db information did not load successfully but the JSON file did, + # we consider the JSON file to be the SoT. + + elif self.statedb_storage_info_loaded == False and self.fsio_json_file_loaded == True: + self.use_fsio_json_baseline = True + self.use_statedb_baseline = False + + # If neither the STATE_DB nor the JSON file information was loaded, we consider + # that akin to an INIT state. + + + def _reconcile_fsio_rw_values(self, fsio_dict, device): + + # If stormond is coming up for the first time, neither STATE_DB info nor JSON file would be present. + # In that case, neither resource would have any prior information stored. The baseline is 0 for every field. + if self.use_statedb_baseline == False and self.use_fsio_json_baseline == False: + fsio_dict["total_fsio_reads"] = fsio_dict["latest_fsio_reads"] + fsio_dict["total_fsio_writes"] = fsio_dict["latest_fsio_writes"] + + # If the daemon is re-init-ing after a planned reboot or powercycle, there would be no storage info + # in the STATE_DB. Therefore, we would need to parse the total and hitherto latest procfs reads + # and writes from the FSIO JSON file and use those reads/writes values as a baseline. + elif self.use_statedb_baseline == False and self.use_fsio_json_baseline == True: + fsio_dict["total_fsio_reads"] = str(int(self.fsio_rw_json[device]["total_fsio_reads"]) + int(fsio_dict["latest_fsio_reads"])) + fsio_dict["total_fsio_writes"] = str(int(self.fsio_rw_json[device]["total_fsio_writes"]) + int(fsio_dict["latest_fsio_writes"])) + + # The only scenario where there would be storage info present in the STATE_DB is when the daemon is + # coming back up after a crash. + + # In this scenario, we use the STATE_DB values as the SoT. We use the 'latest_fsio_reads/writes' + # values from STATE_DB, which is the values from the last invocation of get_fs_io_reads/writes + # on the storage disk that was posted to STATE_DB, and the values obtained from the most recent + # invocation of get_fs_io_reads/writes (prior to this function being called) to determine the + # additional procfs reads and writes that have happened on the FS while the daemon was down. + + # We then add these additional values to the previous values of total_fsio_reads/writes to + # determine the new total procfs reads/writes. + + elif self.use_statedb_baseline == True: + additional_procfs_reads = int(fsio_dict["latest_fsio_reads"]) - int(self.fsio_rw_statedb[device]["latest_fsio_reads"]) + additional_procfs_writes = int(fsio_dict["latest_fsio_writes"]) - int(self.fsio_rw_statedb[device]["latest_fsio_writes"]) + + fsio_dict["total_fsio_reads"] = str(int(self.fsio_rw_statedb[device]["total_fsio_reads"]) + additional_procfs_reads) + fsio_dict["total_fsio_writes"] = str(int(self.fsio_rw_statedb[device]["total_fsio_writes"]) + additional_procfs_writes) + + return fsio_dict["total_fsio_reads"], fsio_dict["total_fsio_writes"] + + + + # Update the Storage device info to State DB + def update_storage_info_status_db(self, disk_device, kvp_dict): + + fvp = swsscommon.FieldValuePairs([(field, str(value)) for field, value in kvp_dict.items()]) + self.device_table.set(disk_device, fvp) + + + # Get Static attributes and update the State DB, once + def get_static_fields_update_state_db(self): + + # Get relevant information about each Storage Device on the switch + for storage_device, storage_object in self.storage.devices.items(): + try: + # Unlikely scenario + if storage_object is None: + self.log_info("{} does not have an instantiated object. Static Information cannot be gathered.".format(storage_device)) + continue + + static_kvp_dict = {} + + static_kvp_dict["device_model"] = storage_object.get_model() + static_kvp_dict["serial"] = storage_object.get_serial() + + self.log_info("Storage Device: {}, Device Model: {}, Serial: {}".format(storage_device, static_kvp_dict["device_model"], static_kvp_dict["serial"])) + + # update Storage Device Status to DB + self.update_storage_info_status_db(storage_device, static_kvp_dict) + + except Exception as ex: + self.log_error("get_static_fields_update_state_db() failed with: {}".format(str(ex))) + + # Get Dynamic attributes and update the State DB + def get_dynamic_fields_update_state_db(self): + + # Get relevant information about each storage disk on the device + for storage_device, storage_object in self.storage.devices.items(): + try: + if storage_object is None: + self.log_info("Storage device '{}' does not have an instantiated object. Dynamic Information cannot be gathered.".format(storage_device)) + continue + + # Fetch the latest dynamic info + blkdevice = os.path.join(BLKDEV_BASE_PATH, storage_device) + storage_object.fetch_parse_info(blkdevice) + + dynamic_kvp_dict = {} + + dynamic_kvp_dict["firmware"] = storage_object.get_firmware() + dynamic_kvp_dict["health"] = storage_object.get_health() + dynamic_kvp_dict["temperature"] = storage_object.get_temperature() + dynamic_kvp_dict["latest_fsio_reads"] = storage_object.get_fs_io_reads() + dynamic_kvp_dict["latest_fsio_writes"] = storage_object.get_fs_io_writes() + dynamic_kvp_dict["disk_io_reads"] = storage_object.get_disk_io_reads() + dynamic_kvp_dict["disk_io_writes"] = storage_object.get_disk_io_writes() + dynamic_kvp_dict["reserved_blocks"] = storage_object.get_reserved_blocks() + + dynamic_kvp_dict["total_fsio_reads"], dynamic_kvp_dict["total_fsio_writes"] = self._reconcile_fsio_rw_values(dynamic_kvp_dict, storage_device) + + self.log_info("Storage Device: {}, Firmware: {}, health: {}%, Temp: {}C, FS IO Reads: {}, FS IO Writes: {}".format(\ + storage_device, dynamic_kvp_dict["firmware"], dynamic_kvp_dict["health"], dynamic_kvp_dict["temperature"], dynamic_kvp_dict["total_fsio_reads"],dynamic_kvp_dict["total_fsio_writes"])) + self.log_info("Latest FSIO Reads: {}, Latest FSIO Writes: {}".format(dynamic_kvp_dict["latest_fsio_reads"], dynamic_kvp_dict["latest_fsio_writes"])) + self.log_info("Disk IO Reads: {}, Disk IO Writes: {}, Reserved Blocks: {}".format(dynamic_kvp_dict["disk_io_reads"], dynamic_kvp_dict["disk_io_writes"], \ + dynamic_kvp_dict["reserved_blocks"])) + + # Update storage device statistics to STATE_DB + self.update_storage_info_status_db(storage_device, dynamic_kvp_dict) + + except Exception as ex: + self.log_info("get_dynamic_fields_update_state_db() failed with: {}".format(str(ex))) + + + # Override signal handler from DaemonBase + def signal_handler(self, sig, frame): + FATAL_SIGNALS = [signal.SIGINT, signal.SIGTERM] + NONFATAL_SIGNALS = [signal.SIGHUP] + + global exit_code + + if sig in FATAL_SIGNALS: + self.log_info("Caught signal '{}'".format(signal.Signals(sig).name)) + + if self.sync_fsio_rw_json(): + self.write_sync_time_statedb() + else: + self.log_warning("Unable to sync latest and total procfs RW to disk") + + self.log_info("Exiting with {}".format(signal.Signals(sig).name)) + + # Make sure we exit with a non-zero code so that supervisor will try to restart us + exit_code = 128 + sig + self.stop_event.set() + elif sig in NONFATAL_SIGNALS: + self.log_info("Caught signal '{}' - ignoring...".format(signal.Signals(sig).name)) + else: + self.log_warning("Caught unhandled signal '{}' - ignoring...".format(signal.Signals(sig).name)) + + # Main daemon logic + def run(self): + + # Connect to CONFIG_DB and get polling and sync intervals -- + # this is to be able to dynamically configure the polling and sync times. + self.get_configdb_intervals() + + # Repeatedly read and update Dynamic Fields to the StateDB + self.get_dynamic_fields_update_state_db() + + if self.stop_event.wait(self.timeout): + # We received a fatal signal + return False + + # Check if time elapsed since init is > fsstats_sync_interval OR + # If sync interval has elapsed or if difference in elapsed_time and sync interval is less than polling interval + + # If so, sync the appropriate fields to FSIO JSON file + + elapsed_time = time.time() - self.fsio_sync_time + if (elapsed_time > self.fsstats_sync_interval) or ((self.fsstats_sync_interval - elapsed_time) < self.timeout): + if self.sync_fsio_rw_json(): + self.write_sync_time_statedb() + else: + self.log_warning("Unable to sync latest and total procfs RW to disk") + + return True +# +# Main ========================================================================= +# + + +def main(): + stormon = DaemonStorage(SYSLOG_IDENTIFIER) + + stormon.log_info("Starting Storage Monitoring Daemon") + + # Read and update Static Fields to the StateDB once + stormon.get_static_fields_update_state_db() + + while stormon.run(): + pass + + stormon.log_info("Shutting down Storage Monitoring Daemon") + + return exit_code + +if __name__ == '__main__': + sys.exit(main()) diff --git a/sonic-stormond/setup.cfg b/sonic-stormond/setup.cfg new file mode 100644 index 000000000..b7e478982 --- /dev/null +++ b/sonic-stormond/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test=pytest diff --git a/sonic-stormond/setup.py b/sonic-stormond/setup.py new file mode 100644 index 000000000..0e2013829 --- /dev/null +++ b/sonic-stormond/setup.py @@ -0,0 +1,42 @@ +from setuptools import setup + +setup( + name='sonic-stormond', + version='1.0', + description='Storage Device Monitoring Daemon for SONiC', + license='Apache 2.0', + author='SONiC Team', + author_email='linuxnetdev@microsoft.com', + url='https://github.com/sonic-net/sonic-platform-daemons', + maintainer='Ashwin Srinivasan', + maintainer_email='assrinivasan@microsoft.com', + scripts=[ + 'scripts/stormond', + ], + setup_requires=[ + 'pytest-runner', + 'wheel' + ], + install_requires=[ + 'enum34', + 'sonic-py-common', + ], + tests_require=[ + 'mock>=2.0.0', + 'pytest', + 'pytest-cov', + ], + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: No Input/Output (Daemon)', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Operating System :: POSIX :: Linux', + 'Topic :: System :: Hardware', + ], + keywords='sonic SONiC ssd Ssd SSD ssdmond storage stormond storagemond', + test_suite='setup.get_test_suite' +) diff --git a/sonic-stormond/tests/__init__.py b/sonic-stormond/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/sonic-stormond/tests/mock_swsscommon.py b/sonic-stormond/tests/mock_swsscommon.py new file mode 100644 index 000000000..3023099a6 --- /dev/null +++ b/sonic-stormond/tests/mock_swsscommon.py @@ -0,0 +1,63 @@ +''' + Mock implementation of swsscommon package for unit testing +''' + +from swsssdk import ConfigDBConnector, SonicDBConfig, SonicV2Connector + +STATE_DB = '' + + +class Table: + def __init__(self, db, table_name): + self.table_name = table_name + self.mock_dict = {} + self.mock_keys = ['sda'] + + def _del(self, key): + del self.mock_dict[key] + pass + + def set(self, key, fvs): + self.mock_dict[key] = fvs.fv_dict + pass + + def get(self, key): + if key in self.mock_dict: + return self.mock_dict[key] + return None + + def get_size(self): + return (len(self.mock_dict)) + + def getKeys(self): + return self.mock_keys + + def hgetall(self): + return self.mock_dict + + +class FieldValuePairs: + fv_dict = {} + + def __init__(self, tuple_list): + if isinstance(tuple_list, list) and isinstance(tuple_list[0], tuple): + self.fv_dict = dict(tuple_list) + + def __setitem__(self, key, kv_tuple): + self.fv_dict[kv_tuple[0]] = kv_tuple[1] + + def __getitem__(self, key): + return self.fv_dict[key] + + def __eq__(self, other): + if not isinstance(other, FieldValuePairs): + # don't attempt to compare against unrelated types + return NotImplemented + + return self.fv_dict == other.fv_dict + + def __repr__(self): + return repr(self.fv_dict) + + def __str__(self): + return repr(self.fv_dict) diff --git a/sonic-stormond/tests/mocked_libs/sonic_platform/__init__.py b/sonic-stormond/tests/mocked_libs/sonic_platform/__init__.py new file mode 100644 index 000000000..47d228696 --- /dev/null +++ b/sonic-stormond/tests/mocked_libs/sonic_platform/__init__.py @@ -0,0 +1,6 @@ +""" + Mock implementation of sonic_platform package for unit testing +""" + +from . import pcie + diff --git a/sonic-stormond/tests/mocked_libs/sonic_platform/pcie.py b/sonic-stormond/tests/mocked_libs/sonic_platform/pcie.py new file mode 100644 index 000000000..df68a999e --- /dev/null +++ b/sonic-stormond/tests/mocked_libs/sonic_platform/pcie.py @@ -0,0 +1,13 @@ +""" + Mock implementation of sonic_platform package for unit testing +""" + +from sonic_platform_base.pcie_base import PcieBase + + +class Pcie(PcieBase): + def __init__(self): + self.platform_pcieutil = "/tmp/Pcie" + + def __str__(self): + return self.platform_pcieutil diff --git a/sonic-stormond/tests/mocked_libs/sonic_platform_base/__init__.py b/sonic-stormond/tests/mocked_libs/sonic_platform_base/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/sonic-stormond/tests/mocked_libs/sonic_platform_base/__init__.py @@ -0,0 +1 @@ + diff --git a/sonic-stormond/tests/mocked_libs/sonic_platform_base/sonic_storage/__init__.py b/sonic-stormond/tests/mocked_libs/sonic_platform_base/sonic_storage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/sonic-stormond/tests/mocked_libs/sonic_platform_base/sonic_storage/storage_base.py b/sonic-stormond/tests/mocked_libs/sonic_platform_base/sonic_storage/storage_base.py new file mode 100644 index 000000000..4ad4e8cf7 --- /dev/null +++ b/sonic-stormond/tests/mocked_libs/sonic_platform_base/sonic_storage/storage_base.py @@ -0,0 +1,102 @@ +# +# storage_base.py +# +# Base class for implementing common SSD health features +# + + +class StorageBase(object): + """ + Base class for interfacing with a SSD + """ + def __init__(self, diskdev): + """ + Constructor + + Args: + diskdev: Linux device name to get parameters for + """ + pass + + def get_health(self): + """ + Retrieves current disk health in percentages + + Returns: + A float number of current ssd health + e.g. 83.5 + """ + raise NotImplementedError + + def get_temperature(self): + """ + Retrieves current disk temperature in Celsius + + Returns: + A float number of current temperature in Celsius + e.g. 40.1 + """ + raise NotImplementedError + + def get_model(self): + """ + Retrieves model for the given disk device + + Returns: + A string holding disk model as provided by the manufacturer + """ + raise NotImplementedError + + def get_firmware(self): + """ + Retrieves firmware version for the given disk device + + Returns: + A string holding disk firmware version as provided by the manufacturer + """ + raise NotImplementedError + + def get_serial(self): + """ + Retrieves serial number for the given disk device + + Returns: + A string holding disk serial number as provided by the manufacturer + """ + raise NotImplementedError + + def get_vendor_output(self): + """ + Retrieves vendor specific data for the given disk device + + Returns: + A string holding some vendor specific disk information + """ + raise NotImplementedError + + def get_disk_io_reads(self): + """ + Retrieves the total number of Input/Output (I/O) reads done on a storage disk + + Returns: + An integer value of the total number of I/O reads + """ + raise NotImplementedError + + def get_disk_io_writes(self): + """ + Retrieves the total number of Input/Output (I/O) writes done on a storage disk + + Returns: + An integer value of the total number of I/O writes + """ + raise NotImplementedError + + def get_reserved_blocks(self): + """ + Retrieves the total number of reserved blocks in an storage disk + + Returns: + An integer value of the total number of reserved blocks + """ + raise NotImplementedError diff --git a/sonic-stormond/tests/mocked_libs/sonic_platform_base/sonic_storage/storage_devices.py b/sonic-stormond/tests/mocked_libs/sonic_platform_base/sonic_storage/storage_devices.py new file mode 100644 index 000000000..1ecffe13e --- /dev/null +++ b/sonic-stormond/tests/mocked_libs/sonic_platform_base/sonic_storage/storage_devices.py @@ -0,0 +1,12 @@ + +BLKDEV_BASE_PATH = '' + +class StorageDevices: + def __init__(self): + self.devices = {'sda' : None} + + def _get_storage_devices(self): + pass + + def _storage_device_object_factory(self, key): + pass \ No newline at end of file diff --git a/sonic-stormond/tests/mocked_libs/swsscommon/__init__.py b/sonic-stormond/tests/mocked_libs/swsscommon/__init__.py new file mode 100644 index 000000000..012af621e --- /dev/null +++ b/sonic-stormond/tests/mocked_libs/swsscommon/__init__.py @@ -0,0 +1,5 @@ +''' + Mock implementation of swsscommon package for unit testing +''' + +from . import swsscommon diff --git a/sonic-stormond/tests/mocked_libs/swsscommon/swsscommon.py b/sonic-stormond/tests/mocked_libs/swsscommon/swsscommon.py new file mode 100644 index 000000000..ddb3cd686 --- /dev/null +++ b/sonic-stormond/tests/mocked_libs/swsscommon/swsscommon.py @@ -0,0 +1,66 @@ +''' + Mock implementation of swsscommon package for unit testing +''' + +STATE_DB = '' + + +class Table: + def __init__(self, db, table_name): + self.table_name = table_name + self.mock_dict = {} + + def _del(self, key): + del self.mock_dict[key] + pass + + def set(self, key, fvs): + self.mock_dict[key] = fvs.fv_dict + pass + + def get(self, key): + if key in self.mock_dict: + return self.mock_dict[key] + return None + + def get_size(self): + return (len(self.mock_dict)) + + def getKeys(self): + return list(self.mock_dict.keys()) + + +class FieldValuePairs: + fv_dict = {} + + def __init__(self, tuple_list): + if isinstance(tuple_list, list) and isinstance(tuple_list[0], tuple): + self.fv_dict = dict(tuple_list) + + def __setitem__(self, key, kv_tuple): + self.fv_dict[kv_tuple[0]] = kv_tuple[1] + + def __getitem__(self, key): + return self.fv_dict[key] + + def __eq__(self, other): + if not isinstance(other, FieldValuePairs): + # don't attempt to compare against unrelated types + return NotImplemented + + return self.fv_dict == other.fv_dict + + def __repr__(self): + return repr(self.fv_dict) + + def __str__(self): + return repr(self.fv_dict) + +class ConfigDBConnector: + pass + +class SonicDBConfig: + pass + +class SonicV2Connector: + pass diff --git a/sonic-stormond/tests/test_DaemonStorage.py b/sonic-stormond/tests/test_DaemonStorage.py new file mode 100644 index 000000000..dafb05fd9 --- /dev/null +++ b/sonic-stormond/tests/test_DaemonStorage.py @@ -0,0 +1,308 @@ +import datetime +import os +import sys +from imp import load_source + +# TODO: Clean this up once we no longer need to support Python 2 +if sys.version_info.major == 3: + from unittest.mock import patch, MagicMock, mock_open +else: + from mock import patch, MagicMock, mock_open + +# Add mocked_libs path so that the file under test can load mocked modules from there +tests_path = os.path.dirname(os.path.abspath(__file__)) +mocked_libs_path = os.path.join(tests_path, "mocked_libs") +sys.path.insert(0, mocked_libs_path) + +from .mocked_libs.swsscommon import swsscommon +from sonic_py_common import daemon_base + +# Add path to the file under test so that we can load it +modules_path = os.path.dirname(tests_path) +scripts_path = os.path.join(modules_path, "scripts") +sys.path.insert(0, modules_path) +load_source('stormond', os.path.join(scripts_path, 'stormond')) + +import stormond +import pytest + + +log_identifier = 'storage_daemon_test' + + +#daemon_base.db_connect = MagicMock() + +config_intvls = ''' +daemon_polling_interval, +60, +fsstats_sync_interval, +300 +''' + +fsio_dict = {"total_fsio_reads": "", "total_fsio_writes": "", "latest_fsio_reads": "1000", "latest_fsio_writes": "2000"} +fsio_json_dict = { 'sda' : {"total_fsio_reads": "10500", "total_fsio_writes": "21000", "latest_fsio_reads": "1000", "latest_fsio_writes": "2000"}} +bad_fsio_json_dict = { 'sda' : {"total_fsio_reads": None, "total_fsio_writes": "21000", "latest_fsio_reads": "1000", "latest_fsio_writes": "2000"}} +fsio_statedb_dict = { 'sda' : {"total_fsio_reads": "10500", "total_fsio_writes": "21000", "latest_fsio_reads": "200", "latest_fsio_writes": "400"}} + +dynamic_dict = {'firmware': 'ILLBBK', 'health': '40', 'temperature': '5000', 'latest_fsio_reads': '150', 'latest_fsio_writes': '270', 'disk_io_reads': '1000', 'disk_io_writes': '2000', 'reserved_blocks': '3'} + +class TestDaemonStorage(object): + """ + Test cases to cover functionality in DaemonStorage class + """ + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_default_configdb_intervals_no_config(self): + + stormon_daemon = stormond.DaemonStorage(log_identifier) + + assert (stormon_daemon.timeout) == 3600 + assert (stormon_daemon.fsstats_sync_interval) == 86400 + + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_storage_devices(self): + + def new_mock_factory(self, key): + return MagicMock() + + with patch('sonic_platform_base.sonic_storage.storage_devices.StorageDevices._storage_device_object_factory', new=new_mock_factory): + + stormon_daemon = stormond.DaemonStorage(log_identifier) + + assert(list(stormon_daemon.storage.devices.keys()) == ['sda']) + + @patch('os.path.exists', MagicMock(return_value=True)) + @patch('json.load', MagicMock(return_value=bad_fsio_json_dict)) + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_load_fsio_rw_json_false(self): + + with patch('builtins.open', new_callable=mock_open, read_data='{}') as mock_fd: + stormon_daemon = stormond.DaemonStorage(log_identifier) + + assert stormon_daemon.fsio_json_file_loaded == False + + @patch('os.path.exists', MagicMock(return_value=True)) + @patch('json.load', MagicMock(return_value=fsio_json_dict)) + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_load_fsio_rw_json_true(self): + + with patch('builtins.open', new_callable=mock_open, read_data='{}') as mock_fd: + stormon_daemon = stormond.DaemonStorage(log_identifier) + + assert stormon_daemon.fsio_json_file_loaded == True + + + @patch('os.path.exists', MagicMock(return_value=True)) + @patch('json.load', MagicMock(side_effect=Exception)) + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_load_fsio_rw_json_exception(self): + + with patch('builtins.open', new_callable=mock_open, read_data='{}') as mock_fd: + stormon_daemon = stormond.DaemonStorage(log_identifier) + + assert stormon_daemon.fsio_json_file_loaded == False + + @patch('sonic_py_common.daemon_base.db_connect') + def testget_configdb_intervals(self, mock_daemon_base): + + mock_daemon_base = MagicMock() + + stormon_daemon = stormond.DaemonStorage(log_identifier) + stormon_daemon.get_configdb_intervals() + + assert mock_daemon_base.call_count == 0 + + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + @patch('json.dump', MagicMock()) + def test_sync_fsio_rw_json_exception(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + + with patch('builtins.open', new_callable=mock_open, read_data='{}') as mock_fd: + stormon_daemon.sync_fsio_rw_json() + + assert stormon_daemon.state_db.call_count == 0 + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + @patch('json.dump', MagicMock()) + @patch('time.time', MagicMock(return_value=1000)) + def test_sync_fsio_rw_json_happy(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + + with patch('builtins.open', new_callable=mock_open, read_data='{}') as mock_fd: + stormon_daemon.sync_fsio_rw_json() + + assert stormon_daemon.state_db.call_count == 0 + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_reconcile_fsio_rw_values_init(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + stormon_daemon.use_statedb_baseline = False + stormon_daemon.use_fsio_json_baseline = False + + (reads, writes) = stormon_daemon._reconcile_fsio_rw_values(fsio_dict, MagicMock()) + + assert reads == '1000' + assert writes == '2000' + + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_reconcile_fsio_rw_values_reboot(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + + stormon_daemon.use_statedb_baseline = False + stormon_daemon.use_fsio_json_baseline = True + stormon_daemon.fsio_rw_json = fsio_json_dict + + (reads, writes) = stormon_daemon._reconcile_fsio_rw_values(fsio_dict, 'sda') + + assert reads == '11500' + assert writes == '23000' + + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_reconcile_fsio_rw_values_daemon_crash(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + + stormon_daemon.use_statedb_baseline = True + stormon_daemon.use_fsio_json_baseline = True + stormon_daemon.fsio_rw_statedb = fsio_statedb_dict + + (reads, writes) = stormon_daemon._reconcile_fsio_rw_values(fsio_dict, 'sda') + + assert reads == '11300' + assert writes == '22600' + + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_update_storage_info_status_db(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + + stormon_daemon.update_storage_info_status_db('sda', fsio_json_dict['sda']) + + assert stormon_daemon.device_table.getKeys() == ['sda'] + + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_get_static_fields(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + + mock_storage_device_object = MagicMock() + mock_storage_device_object.get_model.return_value = "Skynet" + mock_storage_device_object.get_serial.return_value = "T1000" + + stormon_daemon.storage.devices = {'sda' : mock_storage_device_object} + stormon_daemon.get_static_fields_update_state_db() + + assert stormon_daemon.device_table.getKeys() == ['sda'] + assert stormon_daemon.device_table.get('sda') == {'device_model': 'Skynet', 'serial': 'T1000'} + + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_get_dynamic_fields(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + + mock_storage_device_object = MagicMock() + mock_storage_device_object.get_firmware.return_value = "ILLBBK" + mock_storage_device_object.get_health.return_value = "40" + mock_storage_device_object.get_temperature.return_value = "5000" + mock_storage_device_object.get_fs_io_reads.return_value = "150" + mock_storage_device_object.get_fs_io_writes.return_value = "270" + mock_storage_device_object.get_disk_io_reads.return_value = "1000" + mock_storage_device_object.get_disk_io_writes.return_value = "2000" + mock_storage_device_object.get_reserved_blocks.return_value = "3" + + stormon_daemon.storage.devices = {'sda' : mock_storage_device_object} + stormon_daemon.get_dynamic_fields_update_state_db() + + assert stormon_daemon.device_table.getKeys() == ['sda'] + for field, value in dynamic_dict.items(): + assert stormon_daemon.device_table.get('sda')[field] == value + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + @patch('json.dump', MagicMock()) + @patch('time.time', MagicMock(return_value=1000)) + def test_write_sync_time_statedb(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + stormon_daemon.sync_fsio_rw_json = MagicMock(return_value=True) + + stormon_daemon.write_sync_time_statedb() + assert stormon_daemon.state_db.call_count == 0 + + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_signal_handler(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + stormon_daemon.sync_fsio_rw_json = MagicMock() + + stormon_daemon.stop_event.set = MagicMock() + stormon_daemon.log_info = MagicMock() + stormon_daemon.log_warning = MagicMock() + + # Test SIGHUP + stormon_daemon.signal_handler(stormond.signal.SIGHUP, None) + assert stormon_daemon.log_info.call_count == 1 + stormon_daemon.log_info.assert_called_with("Caught signal 'SIGHUP' - ignoring...") + assert stormon_daemon.log_warning.call_count == 0 + assert stormon_daemon.stop_event.set.call_count == 0 + assert stormond.exit_code == 0 + + # Reset + stormon_daemon.log_info.reset_mock() + stormon_daemon.log_warning.reset_mock() + stormon_daemon.stop_event.set.reset_mock() + + # Test SIGINT + test_signal = stormond.signal.SIGINT + stormon_daemon.signal_handler(test_signal, None) + assert stormon_daemon.log_info.call_count == 2 + stormon_daemon.log_info.assert_called_with("Exiting with SIGINT") + assert stormon_daemon.log_warning.call_count == 0 + assert stormon_daemon.stop_event.set.call_count == 1 + assert stormond.exit_code == (128 + test_signal) + + # Reset + stormon_daemon.log_info.reset_mock() + stormon_daemon.log_warning.reset_mock() + stormon_daemon.stop_event.set.reset_mock() + + # Test SIGTERM + test_signal = stormond.signal.SIGTERM + stormon_daemon.signal_handler(test_signal, None) + assert stormon_daemon.log_info.call_count == 2 + stormon_daemon.log_info.assert_called_with("Exiting with SIGTERM") + assert stormon_daemon.log_warning.call_count == 0 + assert stormon_daemon.stop_event.set.call_count == 1 + assert stormond.exit_code == (128 + test_signal) + + # Reset + stormon_daemon.log_info.reset_mock() + stormon_daemon.log_warning.reset_mock() + stormon_daemon.stop_event.set.reset_mock() + stormond.exit_code = 0 + + # Test an unhandled signal + stormon_daemon.signal_handler(stormond.signal.SIGUSR1, None) + assert stormon_daemon.log_warning.call_count == 1 + stormon_daemon.log_warning.assert_called_with("Caught unhandled signal 'SIGUSR1' - ignoring...") + assert stormon_daemon.log_info.call_count == 0 + assert stormon_daemon.stop_event.set.call_count == 0 + assert stormond.exit_code == 0 + + + @patch('sonic_py_common.daemon_base.db_connect', MagicMock()) + def test_run(self): + stormon_daemon = stormond.DaemonStorage(log_identifier) + stormon_daemon.get_dynamic_fields_update_state_db = MagicMock() + + def mock_intervals(): + stormon_daemon.timeout = 10 + stormon_daemon.fsstats_sync_interval = 30 + + with patch.object(stormon_daemon, 'get_configdb_intervals', new=mock_intervals): + stormon_daemon.run() + + assert stormon_daemon.get_dynamic_fields_update_state_db.call_count == 1 + +