feat(enodebd): Firmware upgrade for Sercomm

lte/gateway/configs/enodebd.yml

@@ -38,6 +38,50 @@ sas:
   sas_location: "indoor"
   sas_height_type: "AMSL"
 
+# TODO: @xbend TR069 Firmware Upgrade config (TR069 download flow)
+# This config section emulates a per eNB firmware upgrade mechanism
+# Eventually we would want this to be passed from Orchestrator as part
+# of mconfig
+# Currently only supported/used by FreedomFi One eNB
+
+# Main config section for TR069 Download request for FW Upgrade
+# `firmwares` is a dictionary of available firmwares with download details
+#   key is the firmware version signature. Value is an object with:
+#     "url": full url to the firmware file
+#     "username": [optional] auth credentials
+#     "password": [optional] auth credentials
+# `enbs` is a dictionary with serial-level eNB specific firmware upgrade configuration.
+#   key is the eNB serial number.
+#   value is a string matching firmware version listed in the `firmwares` section.
+#   Takes priority over model-level definition.
+#   Currently only works if serial number is part of eNB's data model.
+# `models` is a dictionary with model-level eNB firmware upgrade configuration.
+#   key is the device model name as stated in enodebd device_utils.py.
+#   value is a string matching firmware version listed in the `firmwares` section.
+#   If specified, applies to all eNBs for that device model that do not
+#   have individual firmware upgrade config in `enbs` section
+# Example:
+# firmware_upgrade_download:
+#   firmwares:
+#     "some_version":
+#       url: "http://some_url/some_fw_file.ffw"
+#       username: "user"
+#       password: "password"
+#     "some_other_version":
+#       url: "http://some_other_url/some_other_fw_file.ffw"
+#     "some_yet_unused_version":
+#       url: "some_url"
+#   enbs:
+#     "some_serial": "some_version"
+#     "some_other_serial": "some_other_version"
+#   models:
+#     "some_model": "some_version"
+
+firmware_upgrade_download:
+  firmwares: {}
+  enbs: {}
+  models: {}
+
 # primSync determines the primary source for synchronization.
 # Used by FreedomFi One eNB.
 # "GNSS" - When using SAS client in eNB, GPS is used.

lte/gateway/python/magma/enodebd/devices/freedomfi_one.py

@@ -53,14 +53,17 @@
     AcsMsgAndTransition,
     AcsReadMsgResult,
     AddObjectsState,
+    CheckFirmwareUpgradeDownloadState,
     DeleteObjectsState,
     EnbSendRebootState,
     EndSessionState,
     EnodebAcsState,
     ErrorState,
+    FirmwareUpgradeDownloadState,
     GetParametersState,
     NotifyDPState,
     SetParameterValuesState,
+    WaitForFirmwareUpgradeDownloadResponse,
     WaitGetParametersState,
     WaitInformMRebootState,
     WaitInformState,
@@ -488,11 +491,28 @@ def _init_state_map(self) -> None:
             # Inform comes in -> Respond with InformResponse
             'wait_inform': WaitInformState(self, when_done='get_rpc_methods'),
             # If first inform after boot -> GetRpc request comes in, if not
-            # empty request comes in => Transition to get_transient_params
+            # empty request comes in => Transition
             'get_rpc_methods': FreedomFiOneGetInitState(
+                self,
+                when_done='check_fw_upgrade_download',
+            ),
+
+            # Download flow
+            'check_fw_upgrade_download': CheckFirmwareUpgradeDownloadState(
+                self,
+                when_download='fw_upgrade_download',
+                when_skip='get_transient_params',
+            ),
+            'fw_upgrade_download': FirmwareUpgradeDownloadState(
+                self,
+                when_done='wait_fw_upgrade_download_response',
+            ),
+            'wait_fw_upgrade_download_response': WaitForFirmwareUpgradeDownloadResponse(
                 self,
                 when_done='get_transient_params',
+                when_skip='get_transient_params',
             ),
+            # Download flow ends
 
             # Read transient readonly params.
             'get_transient_params': FreedomFiOneSendGetTransientParametersState(
@@ -1370,7 +1390,7 @@ def get_msg(self, message: Any) -> AcsMsgAndTransition:
             AcsMsgAndTransition
         """
         request = models.DummyInput()
-        if self.acs.desired_cfg.get_parameter(SASParameters.SAS_METHOD):
+        if self.acs.desired_cfg and self.acs.desired_cfg.get_parameter(SASParameters.SAS_METHOD):
             return AcsMsgAndTransition(request, self.notify_dp)
         return AcsMsgAndTransition(request, None)
 
@@ -1382,7 +1402,7 @@ def state_description(self) -> str:
             str
         """
         description = 'Completed provisioning eNB. Awaiting new Inform'
-        if self.acs.desired_cfg.get_parameter(SASParameters.SAS_METHOD):
+        if self.acs.desired_cfg and self.acs.desired_cfg.get_parameter(SASParameters.SAS_METHOD):
             description = 'Completed initial provisioning of the eNB. Awaiting update from DProxy'
         return description
 
@@ -1397,10 +1417,14 @@ def enter(self):
         Enter the state
         """
         request = CBSDRequest(
-            serial_number=self.acs.device_cfg.get_parameter(ParameterName.SERIAL_NUMBER),
+            serial_number=self.acs.device_cfg.get_parameter(
+                ParameterName.SERIAL_NUMBER,
+            ),
         )
         state = get_cbsd_state(request)
-        ff_one_update_desired_config_from_cbsd_state(state, self.acs.desired_cfg)
+        ff_one_update_desired_config_from_cbsd_state(
+            state, self.acs.desired_cfg,
+        )
 
 
 def ff_one_update_desired_config_from_cbsd_state(state: CBSDStateResult, config: EnodebConfiguration) -> None:
@@ -1416,8 +1440,13 @@ def ff_one_update_desired_config_from_cbsd_state(state: CBSDStateResult, config:
     if not state.radio_enabled:
         return
 
-    earfcn = calc_earfcn(state.channel.low_frequency_hz, state.channel.high_frequency_hz)
-    bandwidth_mhz = calc_bandwidth_mhz(state.channel.low_frequency_hz, state.channel.high_frequency_hz)
+    earfcn = calc_earfcn(
+        state.channel.low_frequency_hz,
+        state.channel.high_frequency_hz,
+    )
+    bandwidth_mhz = calc_bandwidth_mhz(
+        state.channel.low_frequency_hz, state.channel.high_frequency_hz,
+    )
     bandwidth_rbs = calc_bandwidth_rbs(bandwidth_mhz)
     tx_power = _calc_tx_power(state.channel.max_eirp_dbm_mhz, bandwidth_mhz)
 

lte/gateway/python/magma/enodebd/state_machines/acs_state_utils.py

@@ -342,3 +342,103 @@ def get_optional_param_to_check(
         except KeyError:
             return param
     return None
+
+
+def should_transition_to_firmware_upgrade_download(acs):
+    device_sw_version = ""
+    device_serial_number = ""
+    if acs.device_cfg.has_parameter(ParameterName.SW_VERSION):
+        device_sw_version = acs.device_cfg.get_parameter(
+            ParameterName.SW_VERSION,
+        )
+    if acs.device_cfg.has_parameter(ParameterName.SERIAL_NUMBER):
+        device_serial_number = acs.device_cfg.get_parameter(
+            ParameterName.SERIAL_NUMBER,
+        )
+    if acs.is_fw_upgrade_in_progress():
+        logger.debug(
+            'Skipping FW Download for eNB [%s], firmware upgrade in progress.',
+            device_serial_number,
+        )
+        return False
+    fw_upgrade_config = get_firmware_upgrade_download_config(acs)
+    if not fw_upgrade_config:
+        logger.debug(
+            'Skipping FW Download for eNB [%s], missing firmware upgrade config in enodebd.yml.',
+            device_serial_number,
+        )
+        return False
+    target_software_version = fw_upgrade_config.get('version', '')
+    if device_sw_version == target_software_version:
+        logger.debug(
+            'Skipping FW Download for eNB [%s], eNB Software Version [%s] up to date with firmware upgrade config.',
+            device_serial_number,
+            target_software_version,
+        )
+        acs.stop_fw_upgrade_timeout()
+        return False
+    logger.info(
+        'Initiate FW Download for eNB [%s], eNB SW Version [%s], target SW Version [%s]',
+        device_serial_number, device_sw_version, target_software_version,
+    )
+    return True
+
+
+def get_firmware_upgrade_download_config(acs):
+    device_serial_number = ''
+    if acs.device_cfg.has_parameter(ParameterName.SERIAL_NUMBER):
+        device_serial_number = acs.device_cfg.get_parameter(
+            ParameterName.SERIAL_NUMBER,
+        )
+    fw_upgrade_config = _get_firmware_upgrade_download_config_for_serial(
+        acs, device_serial_number,
+    )
+    if fw_upgrade_config:
+        logger.info(f'Found {fw_upgrade_config=} for {device_serial_number=}')
+        return fw_upgrade_config
+    device_model = acs.device_name
+    fw_upgrade_config = _get_firmware_upgrade_download_config_for_model(
+        acs, device_model,
+    )
+    if fw_upgrade_config:
+        logger.info(f'Found {fw_upgrade_config=} for {device_model=}')
+    return fw_upgrade_config
+
+
+def _get_firmware_upgrade_download_config_for_serial(acs, serial: str):
+    enbs = acs.service_config.get(
+        'firmware_upgrade_download', {},
+    ).get('enbs', {})
+    fw_version = enbs.get(serial, '')
+    return _get_firmware_upgrade_download_config(acs, fw_version)
+
+
+def _get_firmware_upgrade_download_config_for_model(acs, model: str):
+    ouis = acs.service_config.get(
+        'firmware_upgrade_download', {},
+    ).get('models', {})
+    fw_version = ouis.get(model, '')
+    return _get_firmware_upgrade_download_config(acs, fw_version)
+
+
+def _get_firmware_upgrade_download_config(acs, fw_version: str):
+    if not fw_version:
+        return {}
+    firmware_cfg = acs.service_config.get('firmware_upgrade_download', {}).get(
+        'firmwares', {},
+    ).get(fw_version, {})
+    if not firmware_cfg:
+        logger.debug(
+            f'Could not find Firmware Upgrade download version {fw_version} in config.',
+        )
+        return {}
+
+    if not firmware_cfg.get('url', ''):
+        logger.debug(
+            f'Firmware Upgrade download config for {fw_version} does not have a valid url.',
+        )
+        return {}
+
+    # add a 'version' key for easy extraction later
+    firmware_cfg['version'] = fw_version
+    return firmware_cfg

lte/gateway/python/magma/enodebd/state_machines/enb_acs_impl.py

@@ -54,6 +54,16 @@ class BasicEnodebAcsStateMachine(EnodebAcsStateMachine):
     # Check the MME connection status every 15 seconds
     MME_CHECK_TIMER = 15
 
+    # When a FW upgrade download is initiated, we should wait for eNB to apply
+    # the FW image. Currently for Sercomm (FreedomFi One), the eNB accepts Download,
+    # eventually reports TransferComplete, but will reboot in a random time after that.
+    # When DownloadResponse is a success, the state should start the fw upgrade timer.
+    # When FW Upgrade check indentifies that eNB Software Version matches desired
+    # SW version, the timer should be stopped.
+    # Otherwise, keep the timer (even during state reset) and attempt new fw upgrade download
+    # flow once the timer finishes
+    FW_UPGRADE_TIMEOUT = 5 * 60
+
     def __init__(
             self,
             service: MagmaService,
@@ -64,6 +74,7 @@ def __init__(
         self.timeout_handler = None
         self.mme_timeout_handler = None
         self.mme_timer = None
+        self.fw_upgrade_timeout_handler = None
         self._start_state_machine(service)
 
     def get_state(self) -> str:
@@ -98,7 +109,10 @@ def handle_tr069_message(
             return self._get_tr069_msg(message)
 
     def transition(self, next_state: str) -> Any:
-        logger.debug('State transition to <%s>', next_state)
+        logger.debug(
+            'State transition from <%s> to <%s>',
+            self.state.__class__.__name__, next_state,
+        )
         self.state.exit()
         self.state = self.state_map[next_state]
         self.state.enter()
@@ -118,6 +132,48 @@ def stop_state_machine(self) -> None:
         self._data_model = None
 
         self.mme_timer = None
+        self.fw_upgrade_timeout_handler = None
+
+    def start_fw_upgrade_timeout(self) -> None:
+        """
+        Start a firmware upgrade timeout timer.
+
+        When initialing a firmware upgrade download, the eNB can take
+        an unknown amount of time for the download to finish. This process
+        is indicated by a TransferComplete TR069 message, but the enB
+        can still operate and can apply the firmware at any time and reboot.
+
+        Since we do not want to re-issue a download request (eNB hasn't updated,
+        its SW version is still 'old' and the firmware version check will still detect
+        and older FW version still present on the eNB) we want to hold with the download
+        flow for some time - which is what this timer is for.
+        """
+        if self.fw_upgrade_timeout_handler is not None:
+            return
+
+        logger.debug(
+            'ACS starting fw upgrade timeout for %d seconds',
+            self.FW_UPGRADE_TIMEOUT,
+        )
+        self.fw_upgrade_timeout_handler = self.event_loop.call_later(
+            self.FW_UPGRADE_TIMEOUT,
+            self.stop_fw_upgrade_timeout,
+        )
+
+    def stop_fw_upgrade_timeout(self) -> None:
+        """
+        Stop firmware upgrade timeout timer.
+
+        Invoking this method will re-enable firmware software version checking
+        in the Download states.
+        """
+        if self.fw_upgrade_timeout_handler is not None:
+            logger.debug('ACS stopping fw upgrade timeout.')
+            self.fw_upgrade_timeout_handler.cancel()
+            self.fw_upgrade_timeout_handler = None
+
+    def is_fw_upgrade_in_progress(self) -> bool:
+        return self.fw_upgrade_timeout_handler != None
 
     def _start_state_machine(
             self,
@@ -247,7 +303,9 @@ def _check_mme_connection(self) -> None:
                     'within %s seconds - rebooting!',
                     self.MME_DISCONNECT_ENODEB_REBOOT_TIMER,
                 )
-                metrics.STAT_ENODEB_REBOOTS.labels(cause='MME disconnect').inc()
+                metrics.STAT_ENODEB_REBOOTS.labels(
+                    cause='MME disconnect',
+                ).inc()
                 metrics.STAT_ENODEB_REBOOT_TIMER_ACTIVE.set(0)
                 self.mme_timer = None
                 self.reboot_asap()