diff --git a/etc/os-net-config/samples/dcb_config_sample.yaml b/etc/os-net-config/samples/dcb_config_sample.yaml new file mode 100644 index 0000000..cdd2bdc --- /dev/null +++ b/etc/os-net-config/samples/dcb_config_sample.yaml @@ -0,0 +1,17 @@ +# For reconfiguring the DCB, the below template could be used. +# use ``os-net-config-dcb -c `` to perform the +# reconfiguration. + +dcb_config: + - + type: dcb_config + name: ens1f0np0 + dscp2prio: + # Add the dscp configs. + # It requires priority and protocol + - priority: 5 + protocol: 45 + - priority: 5 + protocol: 46 + - priority: 6 + protocol: 47 diff --git a/etc/os-net-config/samples/dcb_sample.yaml b/etc/os-net-config/samples/dcb_sample.yaml new file mode 100644 index 0000000..5ec431c --- /dev/null +++ b/etc/os-net-config/samples/dcb_sample.yaml @@ -0,0 +1,24 @@ +network_config: + - + type: sriov_pf + name: ens1f0np0 + numvfs: 4 + use_dhcp: false + dcb: + dscp2prio: + # Add the dscp configs. + # It requires priority and protocol + - priority: 5 + protocol: 45 + - priority: 5 + protocol: 46 + - priority: 6 + protocol: 47 + - + type: sriov_pf + name: ens1f1np1 + numvfs: 4 + use_dhcp: false + dcb: + # Remove the dscp configurations + dscp2prio: [] diff --git a/os_net_config/cli.py b/os_net_config/cli.py index 8543822..1e8da82 100644 --- a/os_net_config/cli.py +++ b/os_net_config/cli.py @@ -22,6 +22,7 @@ import yaml from os_net_config import common +from os_net_config import dcb_config from os_net_config import impl_eni from os_net_config import impl_ifcfg from os_net_config import impl_iproute @@ -279,6 +280,11 @@ def main(argv=sys.argv, main_logger=None): else: main_logger.warning('\n'.join(validation_errors)) + # Reset the DCB Config during rerun. + # This is required to apply the new values and clear the old ones + if utils.is_dcb_config_required(): + common.reset_dcb_map() + # Look for the presence of SriovPF types in the first parse of the json # if SriovPFs exists then PF devices needs to be configured so that the VF # devices are created. @@ -347,6 +353,13 @@ def main(argv=sys.argv, main_logger=None): files_changed = provider.apply(cleanup=opts.cleanup, activate=not opts.no_activate) + + if utils.is_dcb_config_required(): + # Apply the DCB Config + utils.configure_dcb_config_service() + dcb_apply = dcb_config.DcbApplyConfig() + dcb_apply.apply() + if opts.noop: if configure_sriov: files_changed.update(pf_files_changed) diff --git a/os_net_config/common.py b/os_net_config/common.py index d41fbb5..7d1ed18 100644 --- a/os_net_config/common.py +++ b/os_net_config/common.py @@ -56,6 +56,18 @@ # promisc: "on"/"off" SRIOV_CONFIG_FILE = '/var/lib/os-net-config/sriov_config.yaml' +# File to contain the list of DCB configurations +# Format of the file shall be +# - name: +# dscp2prio: +# - protocol: 44 +# selector: 5 +# priority: 6 +# - protocol: 42 +# selector: 5 +# priority: 3 + +DCB_CONFIG_FILE = '/var/lib/os-net-config/dcb_config.yaml' _SYS_BUS_PCI_DEV = '/sys/bus/pci/devices' SYS_CLASS_NET = '/sys/class/net' @@ -146,6 +158,74 @@ def get_file_data(filename): return '' +def write_yaml_config(filepath, data): + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, 'w') as f: + yaml.safe_dump(data, f, default_flow_style=False) + + +def update_dcb_map(ifname, pci_addr, driver, noop, dscp2prio=None): + if not noop: + dcb_map = get_dcb_config_map() + for item in dcb_map: + if item['pci_addr'] == pci_addr: + item['name'] = ifname + item['driver'] = driver + item['dscp2prio'] = dscp2prio + break + else: + new_item = {} + new_item['pci_addr'] = pci_addr + new_item['driver'] = driver + new_item['name'] = ifname + new_item['dscp2prio'] = dscp2prio + dcb_map.append(new_item) + + write_yaml_config(DCB_CONFIG_FILE, dcb_map) + + +def write_dcb_map(dcb_map): + write_yaml_config(DCB_CONFIG_FILE, dcb_map) + + +def get_dcb_config_map(): + contents = get_file_data(DCB_CONFIG_FILE) + dcb_config_map = yaml.safe_load(contents) if contents else [] + return dcb_config_map + + +def get_empty_dcb_map(): + contents = get_file_data(DCB_CONFIG_FILE) + dcb_config_map = yaml.safe_load(contents) if contents else [] + for entry in dcb_config_map: + entry['dscp2prio'] = [] + return dcb_config_map + + +def add_dcb_entry(dcb_config_map, data): + for entry in dcb_config_map: + if entry['pci_addr'] == data.pci_addr: + entry['dscp2prio'] = data.dscp2prio + entry['name'] = data.name + entry['driver'] = data.driver + break + else: + new_entry = {} + new_entry['name'] = data.name + new_entry['pci_addr'] = data.pci_addr + new_entry['driver'] = data.driver + new_entry['dscp2prio'] = data.dscp2prio + + dcb_config_map.append(new_entry) + return dcb_config_map + + +def reset_dcb_map(): + dcb_map = get_empty_dcb_map() + if dcb_map != []: + write_dcb_map(dcb_map) + + def get_sriov_map(pf_name=None): contents = get_file_data(SRIOV_CONFIG_FILE) sriov_map = yaml.safe_load(contents) if contents else [] diff --git a/os_net_config/dcb_config.py b/os_net_config/dcb_config.py new file mode 100644 index 0000000..8d9e339 --- /dev/null +++ b/os_net_config/dcb_config.py @@ -0,0 +1,531 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os +import sys +import yaml + +from collections import defaultdict +from os_net_config import common +from os_net_config import dcb_netlink +from os_net_config import objects +from os_net_config import validator +from oslo_concurrency import processutils +from pyroute2 import netlink +from pyroute2.netlink.nlsocket import NetlinkSocket +from pyroute2.netlink.rtnl import RTM_GETDCB +from pyroute2.netlink.rtnl import RTM_SETDCB + +# Maximum retries for getting the reply for +# netlink msg with correct sequence number +DCB_MAX_RETRIES = 3 + +# Bitmask indicating the mode - OS Controlled vs FW Controlled +DCB_CAP_DCBX_HOST = 0x1 + +IEEE_8021QAZ_TSA_STRICT = 0 +IEEE_8021QAZ_TSA_ETS = 2 +IEEE_8021QAZ_TSA_VENDOR = 255 + +logger = common.configure_logger() + + +class DCBErrorException(ValueError): + pass + + +class DcbApp: + def __init__(self, selector, priority, protocol): + self.selector = selector + self.priority = priority + self.protocol = protocol + + def is_equal(self, dcbapp2): + if (self.selector == dcbapp2.selector and + self.priority == dcbapp2.priority and + self.protocol == dcbapp2.protocol): + return True + return False + + def dump(self): + log = (f'DcbApp {{Priority: {self.priority} ' + f'Protocol: {self.protocol} Selector: {self.selector}}}') + return log + + +class DcbAppTable: + def __init__(self): + self.apps = {} + + def dump(self, selector): + s = ["", "", "", "", "", "", "", ""] + + for i in range(len(self.apps)): + if self.apps[i].selector == selector: + s[self.apps[i].priority] += '%02d ' % self.apps[i].protocol + + msg = "" + for i in range(8): + pad = "\tprio:%d dscp:" % i + while (len(s[i]) > 24): + msg += pad + s[i][:24] + "\n" + s[i] = s[i][24:] + if s[i] != "": + msg += pad + s[i] + + return msg + + def set_values(self, dcb_cfg): + for i in range(len(self.apps)): + dcb_cfg.set_ieee_app(self.apps[i].selector, + self.apps[i].priority, + self.apps[i].protocol) + + def count_app_selector(self, selector): + count = 0 + for i in range(len(self.apps)): + if self.apps[i].selector == selector: + count = count + 1 + return count + + def del_app_entry(self, dcb_cfg, + selector=dcb_netlink.IEEE_8021QAZ_APP_SEL_DSCP): + for i in range(len(self.apps)): + if self.apps[i].selector == selector: + dcb_cfg.del_ieee_app(self.apps[i].selector, + self.apps[i].priority, + self.apps[i].protocol) + + def set_default_dscp(self, dcb_cfg, selector, max_protocol): + for i in range(max_protocol): + dcb_cfg.set_ieee_app(selector, i >> 3, i) # firmware default + return + + +class DcbMessage(dcb_netlink.dcbmsg): + def __init__(self, nlmsg=None): + super(dcb_netlink.dcbmsg, self).__init__(nlmsg) + self['family'] = 0 + + def set_header(self, cmd, msg_type, seq): + self['cmd'] = cmd + self['header']['sequence_number'] = seq + self['header']['pid'] = os.getpid() + self['header']['type'] = msg_type + self['header']['flags'] = netlink.NLM_F_REQUEST + + def set_attr(self, attr): + self['attrs'] = attr + + +class DcbConfig(): + def __init__(self, iface_name): + self.iface_name = iface_name + self._seq = 0 + self.nlsock = NetlinkSocket(family=netlink.NETLINK_ROUTE) + self.nlsock.bind() + + def seq(self): + self._seq += 1 + return self._seq + + def check_error(self, msg, seq): + if msg['header']['sequence_number'] == seq: + if msg['header']['type'] == netlink.NLMSG_ERROR: + return netlink.NLMSG_ERROR + return msg['header']['type'] + else: + return -1 + + def send_and_receive(self, cmd, msg_type, attr): + msg = DcbMessage() + seq = self.seq() + msg.set_header(cmd=cmd, msg_type=msg_type, + seq=seq) + iface_attr = ['DCB_ATTR_IFNAME', self.iface_name] + msg.set_attr(attr=[iface_attr] + attr) + msg.encode() + try: + logger.debug(f'Sending message {msg_type_to_name(msg_type)} ' + f'cmd {cmd_to_name(cmd)} attr {attr}') + self.nlsock.sendto(msg.data, (0, 0)) + except Exception: + e_msg = (f'Failed to send {msg_type_to_name(msg_type)}' + f' to {self.iface_name}') + raise DCBErrorException(e_msg) + + try: + retry = 0 + while retry < DCB_MAX_RETRIES: + rd_data = self.nlsock.recv(netlink.NLMSG_MAX_LEN) + r_msg = DcbMessage(rd_data) + r_msg.decode() + logger.debug(f'Received message {r_msg}') + err = self.check_error(r_msg, seq) + if err == netlink.NLMSG_ERROR: + e_msg = f'NLMSG_ERROR for command {cmd_to_name(cmd)}' + raise DCBErrorException(e_msg) + if err < 0: + retry += 1 + else: + break + except Exception: + e_msg = f'Failed to get the reply for {cmd}' + raise DCBErrorException(e_msg) + return r_msg + + def get_dcbx(self): + r_msg = self.send_and_receive(cmd=dcb_netlink.DCB_CMD_GDCBX, + msg_type=RTM_GETDCB, + attr=[]) + dcbx = r_msg.get_encoded('DCB_ATTR_DCBX') + logger.debug(f"DCBX mode for {self.iface_name} is {dcbx}") + return dcbx + + def set_dcbx(self, mode): + dcbx_data = ['DCB_ATTR_DCBX', mode] + logger.debug(f'Setting DCBX mode for {self.iface_name}\ + mode:{dcbx_data}') + r_msg = self.send_and_receive(cmd=dcb_netlink.DCB_CMD_SDCBX, + msg_type=RTM_SETDCB, + attr=[dcbx_data]) + dcbx = r_msg.get_encoded('DCB_ATTR_DCBX') + logger.debug(f"Got DCBX mode for {self.iface_name} mode:{dcbx}") + return dcbx + + def get_ieee_ets(self): + r_msg = self.send_and_receive(cmd=dcb_netlink.DCB_CMD_IEEE_GET, + msg_type=RTM_GETDCB, + attr=[]) + iface_name = r_msg.get_encoded('DCB_ATTR_IFNAME') + ieee_ets = r_msg.get_nested('DCB_ATTR_IEEE', + 'DCB_ATTR_IEEE_ETS') + if ieee_ets: + tc_tx_bw = ieee_ets['tc_tx_bw'] + tc_tsa = ieee_ets['tc_tsa'] + prio_tc = ieee_ets['prio_tc'] + + else: + return None, None, None + + logger.debug(f'Received for interface {iface_name}\n' + f'tc_tx_bw: {tc_tx_bw} tc_tsa: {tc_tsa}' + f'prio_tc: {prio_tc}') + + return prio_tc, tc_tsa, tc_tx_bw + + def get_ieee_app_table(self): + r_msg = self.send_and_receive(cmd=dcb_netlink.DCB_CMD_IEEE_GET, + msg_type=RTM_GETDCB, + attr=[]) + dcb_app_list = [] + ieee_app_table = r_msg.get_nested('DCB_ATTR_IEEE', + 'DCB_ATTR_IEEE_APP_TABLE') + if ieee_app_table: + dcb_app_list = self.get_nested_attr(ieee_app_table, + 'DCB_ATTR_IEEE_APP') + + appTable = DcbAppTable() + for i in range(len(dcb_app_list)): + selector = dcb_app_list[i]['selector'] + priority = dcb_app_list[i]['priority'] + protocol = dcb_app_list[i]['protocol'] + appTable.apps[i] = DcbApp(selector, priority, protocol) + + return appTable + + def add_nested_attr(self, attr, attr_data): + return [attr, {'attrs': [attr_data]}] + + def get_nested_attr(self, attr_data, attr): + nested_attr_data = attr_data['attrs'] + desired_attr_list = [] + for entry in nested_attr_data: + if attr in entry: + desired_attr_list.append(entry[1]) + return desired_attr_list + + def set_ieee_app(self, selector, priority, protocol): + dcb_app = {'selector': selector, + 'priority': priority, + 'protocol': protocol} + logger.debug(f'Adding ieee app {dcb_app}') + ieee_app = ['DCB_ATTR_IEEE_APP', dcb_app] + ieee_app_table = self.add_nested_attr('DCB_ATTR_IEEE_APP_TABLE', + ieee_app) + ieee = self.add_nested_attr('DCB_ATTR_IEEE', ieee_app_table) + + self.send_and_receive(cmd=dcb_netlink.DCB_CMD_IEEE_SET, + msg_type=RTM_SETDCB, + attr=[ieee]) + + def del_ieee_app(self, selector, priority, protocol): + dcb_app = {'selector': selector, + 'priority': priority, + 'protocol': protocol} + logger.debug(f'Deleting ieee app {dcb_app}') + ieee_app = ['DCB_ATTR_IEEE_APP', dcb_app] + ieee_app_table = self.add_nested_attr('DCB_ATTR_IEEE_APP_TABLE', + ieee_app) + ieee = self.add_nested_attr('DCB_ATTR_IEEE', ieee_app_table) + self.send_and_receive(cmd=dcb_netlink.DCB_CMD_IEEE_DEL, + msg_type=RTM_SETDCB, + attr=[ieee]) + + +class DcbApplyConfig(): + def __init__(self): + self.dcb_user_config = common.get_dcb_config_map() + + def show(self): + mode = {0: 'FW Controlled', 1: 'OS Controlled'} + + for cfg in self.dcb_user_config: + iface_name = cfg['name'] + dcb_config = DcbConfig(iface_name) + dscp_map = None + + dcbx_mode = dcb_config.get_dcbx() & DCB_CAP_DCBX_HOST + app_table = dcb_config.get_ieee_app_table() + count = app_table.count_app_selector( + dcb_netlink.IEEE_8021QAZ_APP_SEL_DSCP) + if count == 0: + trust = "pcp" + else: + trust = "dscp" + dscp_map = app_table.dump( + dcb_netlink.IEEE_8021QAZ_APP_SEL_DSCP) + + prio_tc, tsa, tc_bw = dcb_config.get_ieee_ets() + + logger.info(f'-----------------------------') + logger.info(f'Interface: {iface_name}') + logger.info(f'DCBX Mode : {mode[dcbx_mode]}') + logger.info(f'Trust mode: {trust}') + if dscp_map: + logger.info(f'dscp2prio mapping: {dscp_map}') + + if prio_tc is None: + logger.info('Failed to get IEEE ETS') + return + tc2up = defaultdict(list) + for up in range(len(prio_tc)): + tc = prio_tc[up] + tc2up[int(tc)].append(up) + + for tc in sorted(tc2up): + msg = "" + try: + msg = "tc: %d , tsa: " % (tc) + except Exception: + pass + try: + if (tsa[tc] == IEEE_8021QAZ_TSA_ETS): + msg += "ets, bw: %s%%" % (tc_bw[tc]) + elif (tsa[tc] == IEEE_8021QAZ_TSA_STRICT): + msg += "strict" + elif (tsa[tc] == IEEE_8021QAZ_TSA_VENDOR): + msg += "vendor" + else: + msg += "unknown" + except Exception: + pass + + msg += f', priority: ' + try: + for up in tc2up[tc]: + msg += f' {up} ' + except Exception: + pass + if msg: + logger.info(f'{msg}') + + def apply(self): + + for cfg in self.dcb_user_config: + dcb_config = DcbConfig(cfg['name']) + + dcbx_mode = dcb_config.get_dcbx() & DCB_CAP_DCBX_HOST + # In case of mellanox nic, set the mstconfig and do fwreset + # If the DCBX mode is already set to FW (0), ignore + # performing mstconfig and mstfwreset. + if 'mlx' in cfg['driver'] and dcbx_mode != 0: + mstconfig(cfg['name'], cfg['pci_addr']) + mstfwreset(cfg['name'], cfg['pci_addr']) + + # Set the mode to Firmware + dcb_config.set_dcbx(mode=0) + curr_apptable = dcb_config.get_ieee_app_table() + add_app_table = DcbAppTable() + user_dscp2prio = cfg['dscp2prio'] + i = 0 + for index in range(len(user_dscp2prio)): + selector = user_dscp2prio[index]['selector'] + priority = user_dscp2prio[index]['priority'] + protocol = user_dscp2prio[index]['protocol'] + dcb_app = DcbApp(selector, priority, protocol) + for key in curr_apptable.apps.keys(): + if dcb_app.is_equal(curr_apptable.apps[key]): + logger.debug(f'Not adding {dcb_app.dump()}') + curr_apptable.apps.pop(key) + break + else: + logger.debug(f'Adding {dcb_app.dump()}') + add_app_table.apps[i] = dcb_app + i += 1 + + curr_apptable.del_app_entry(dcb_config, + dcb_netlink.IEEE_8021QAZ_APP_SEL_DSCP) + add_app_table.set_values(dcb_config) + + +def mstconfig(name, pci_addr): + """Allow FW controlled mode for mellanox devices. + + :name Interface name where firmware configurations needs change + :pci_addr pci address of the interface + """ + + logger.info(f"Running mstconfig for {name}") + try: + processutils.execute('mstconfig', '-y', '-d', pci_addr, 'set', + 'LLDP_NB_DCBX_P1=TRUE', 'LLDP_NB_TX_MODE_P1=2', + 'LLDP_NB_RX_MODE_P1=2', 'LLDP_NB_DCBX_P2=TRUE', + 'LLDP_NB_TX_MODE_P2=2', 'LLDP_NB_RX_MODE_P2=2') + except processutils.ProcessExecutionError: + logger.error(f"mstconfig failed for {name}") + raise + + +def mstfwreset(name, pci_addr): + """mstfwreset is an utility to reset the PCI device and load new FW""" + logger.info(f"Running mstfwreset for {name}") + try: + processutils.execute('mstfwreset', '--device', pci_addr, + '--level', '3', '-y', 'r') + except processutils.ProcessExecutionError: + logger.error(f"mstfwreset failed for {name}") + raise + + +def cmd_to_name(cmd): + cmds_map = {dcb_netlink.DCB_CMD_IEEE_SET: 'DCB_CMD_IEEE_SET', + dcb_netlink.DCB_CMD_IEEE_GET: 'DCB_CMD_IEEE_GET', + dcb_netlink.DCB_CMD_GDCBX: 'DCB_CMD_GDCBX', + dcb_netlink.DCB_CMD_SDCBX: 'DCB_CMD_SDCBX', + dcb_netlink.DCB_CMD_IEEE_DEL: 'DCB_CMD_IEEE_DEL'} + return cmds_map[cmd] + + +def msg_type_to_name(msg_type): + msg_type_map = {RTM_SETDCB: 'RTM_SETDCB', + RTM_GETDCB: 'RTM_GETDCB'} + return msg_type_map[msg_type] + + +def parse_opts(argv): + parser = argparse.ArgumentParser( + description='Configure the DSCP settings for the interfaces using' + ' a YAML config file format.') + + parser.add_argument( + '-d', '--debug', + dest="debug", + action='store_true', + help="Print debugging output.", + required=False) + + parser.add_argument( + '-v', '--verbose', + dest="verbose", + action='store_true', + help="Print verbose output.", + required=False) + + parser.add_argument( + '-s', '--show', + dest="show", + action='store_true', + help="Print the DCB configurations.", + required=False) + + parser.add_argument('-c', '--config-file', metavar='CONFIG_FILE', + help="""path to the configuration file.""", + required=False) + + opts = parser.parse_args(argv[1:]) + + return opts + + +def parse_config(user_config_file): + # Read config file containing network configs to apply + if os.path.exists(user_config_file): + try: + with open(user_config_file) as cf: + iface_array = yaml.safe_load(cf.read()).get("dcb_config") + logger.debug(f"dcb_config: {iface_array}") + except IOError: + logger.error(f"Error reading file: {user_config_file}") + return 1 + else: + logger.error(f"No config file exists at: {user_config_file}") + return 1 + + # Validate the configurations for schematic errors + validation_errors = validator.validate_config(iface_array) + if validation_errors: + logger.error('\n'.join(validation_errors)) + return 1 + + # Get the DCB Map and clear all the dscp2prio map for all + # previously configured interfaces. Add the dscp2prio entries + # from the new configuration and write the contents to + # DCB Config File + dcb_map = common.get_empty_dcb_map() + for iface_json in iface_array: + obj = objects.object_from_json(iface_json) + if isinstance(obj, objects.Dcb): + common.add_dcb_entry(dcb_map, obj) + else: + e_msg = 'Only dcb objects are handled' + raise DCBErrorException(e_msg) + common.write_dcb_map(dcb_map) + + +def main(argv=sys.argv): + opts = parse_opts(argv) + common.logger_level(logger, opts.verbose, opts.debug) + + if opts.config_file: + # Validate and parse the user configurations. + parse_config(opts.config_file) + + dcb_apply = DcbApplyConfig() + if opts.show: + # Enable verbose logs to display the output + common.logger_level(logger, True, opts.debug) + dcb_apply.show() + else: + # Apply the new DCB configuration + dcb_apply.apply() + common.logger_level(logger, True, opts.debug) + dcb_apply.show() + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/os_net_config/dcb_netlink.py b/os_net_config/dcb_netlink.py new file mode 100644 index 0000000..b7f8a5b --- /dev/null +++ b/os_net_config/dcb_netlink.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pyroute2.netlink import nla +from pyroute2.netlink import nlmsg + +# DCB Commands +DCB_CMD_IEEE_SET = 20 +DCB_CMD_IEEE_GET = 21 +DCB_CMD_GDCBX = 22 +DCB_CMD_SDCBX = 23 +DCB_CMD_IEEE_DEL = 27 + +# DSCP Selector +IEEE_8021QAZ_APP_SEL_DSCP = 5 + + +class dcbmsg(nlmsg): + + pack = 'struct' + + """C Structure + struct dcbmsg { + __u8 dcb_family; + __u8 cmd; + __u16 dcb_pad; + }; + """ + fields = ( + ('family', 'B'), + ('cmd', 'B'), + ('pad', 'H'), + ) + nla_map = ( + (1, 'DCB_ATTR_IFNAME', 'asciiz'), + (13, 'DCB_ATTR_IEEE', 'ieee_attrs'), + (14, 'DCB_ATTR_DCBX', 'uint8'), + ) + + class ieee_attrs(nla): + pack = 'struct' + nla_map = ( + (1, 'DCB_ATTR_IEEE_ETS', 'ieee_ets'), + (2, 'DCB_ATTR_IEEE_PFC', 'ieee_pfc'), + (3, 'DCB_ATTR_IEEE_APP_TABLE', 'ieee_app_table'), + ) + + """This structure contains the IEEE 802.1Qaz ETS managed object + + :willing: willing bit in ETS configuration TLV + :ets_cap: indicates supported capacity of ets feature + :cbs: credit based shaper ets algorithm supported + :tc_tx_bw: tc tx bandwidth indexed by traffic class + :tc_rx_bw: tc rx bandwidth indexed by traffic class + :tc_tsa: TSA Assignment table, indexed by traffic class + :prio_tc: priority assignment table mapping 8021Qp to tc + :tc_reco_bw: recommended tc bw indexed by tc for TLV + :tc_reco_tsa: recommended tc bw indexed by tc for TLV + :reco_prio_tc: recommended tc tx bw indexed by tc for TLV + + Recommended values are used to set fields in the ETS + recommendation TLV with hardware offloaded LLDP. + + ---- + TSA Assignment 8 bit identifiers + 0 strict priority + 1 credit-based shaper + 2 enhanced transmission selection + 3-254 reserved + 255 vendor specific + """ + class ieee_ets(nla): + pack = 'struct' + fields = ( + ('willing', 'B'), + ('ets_cap', 'B'), + ('cbs', 'B'), + ('tc_tx_bw', 'BBBBBBBB'), + ('tc_rx_bw', 'BBBBBBBB'), + ('tc_tsa', 'BBBBBBBB'), + ('prio_tc', 'BBBBBBBB'), + ('tc_reco_bw', 'BBBBBBBB'), + ('tc_reco_tsa', 'BBBBBBBB'), + ('reco_prio_tc', 'BBBBBBBB'), + ) + + class ieee_app_table(nla): + pack = 'struct' + nla_map = ( + (0, 'DCB_ATTR_IEEE_APP_UNSPEC', 'none'), + (1, 'DCB_ATTR_IEEE_APP', 'dcb_app'), + ) + + """This structure contains the IEEE 802.1Qaz APP managed object. This + object is also used for the CEE std as well. + + :selector: protocol identifier type + :protocol: protocol of type indicated + :priority: 3-bit unsigned integer indicating priority for IEEE + 8-bit 802.1p user priority bitmap for CEE + """ + class dcb_app(nla): + pack = 'struct' + fields = ( + ('selector', 'B'), + ('priority', 'B'), + ('protocol', 'H'), + ) + + """This structure contains the IEEE 802.1Qaz PFC managed object + + :pfc_cap: Indicates the number of traffic classes on the local device + that may simultaneously have PFC enabled. + :pfc_en: bitmap indicating pfc enabled traffic classes + :mbc: enable macsec bypass capability + :delay: the allowance made for a round-trip propagation delay of the + link in bits. + :requests: count of the sent pfc frames + :indications: count of the received pfc frames + """ + class ieee_pfc(nla): + pack = 'struct' + fields = ( + ('pfc_cap', 'B'), + ('pfc_en', 'B'), + ('mbc', 'B'), + ('delay', 'H'), + ('requests', 'QQQQQQQQ'), + ('indications', 'QQQQQQQQ'), + ) diff --git a/os_net_config/objects.py b/os_net_config/objects.py index 2d4cb3c..c7389b1 100644 --- a/os_net_config/objects.py +++ b/os_net_config/objects.py @@ -24,6 +24,7 @@ from oslo_utils import strutils from os_net_config import common +from os_net_config import dcb_netlink from os_net_config import utils @@ -96,6 +97,8 @@ def object_from_json(json): return SriovVF.from_json(json) elif obj_type == "linux_tap": return LinuxTap.from_json(json) + elif obj_type == "dcb": + return Dcb.from_json(json) def _get_required_field(json, name, object_name, datatype=None): @@ -291,6 +294,33 @@ def from_json(json): return Address(ip_netmask) +class Dcb(object): + """Base class for DCB configuration""" + + def __init__(self, name, dscp2prio=[]): + self.name = name + self.dscp2prio = dscp2prio + self.pci_addr = utils.get_pci_address(name, False) + self.driver = utils.get_driver(name, False) + + @staticmethod + def from_json(json): + dscp2prio = [] + name = _get_required_field(json, 'name', 'Dcb') + dscp2prio_lst = _get_required_field(json, 'dscp2prio', 'Dcb') + for entry in dscp2prio_lst: + priority = _get_required_field(entry, 'priority', 'dscp2prio') + selector = entry.get('selector', + dcb_netlink.IEEE_8021QAZ_APP_SEL_DSCP) + protocol = _get_required_field(entry, 'protocol', 'dscp2prio') + dscp2prio_entry = {'selector': selector, + 'priority': priority, + 'protocol': protocol} + dscp2prio.append(dscp2prio_entry) + + return Dcb(name, dscp2prio) + + class RouteRule(object): """Base class for route rules.""" @@ -495,6 +525,14 @@ def from_json(json): opts = _BaseOpts.base_opts_from_json(json) ethtool_opts = json.get('ethtool_opts', None) linkdelay = json.get('linkdelay', None) + dcb_config_json = json.get('dcb') + if dcb_config_json: + dcb_config_json['name'] = name + dcb_config = Dcb.from_json(dcb_config_json) + common.update_dcb_map(ifname=name, pci_addr=dcb_config.pci_addr, + driver=dcb_config.driver, noop=False, + dscp2prio=dcb_config.dscp2prio) + return Interface(name, *opts, ethtool_opts=ethtool_opts, hotplug=hotplug, linkdelay=linkdelay) @@ -1632,6 +1670,15 @@ def from_json(json): if link_mode not in ['legacy', 'switchdev']: msg = 'Expecting link_mode to match legacy/switchdev' raise InvalidConfigException(msg) + + dcb_config_json = json.get('dcb') + if dcb_config_json: + dcb_config_json['name'] = name + dcb_config = Dcb.from_json(dcb_config_json) + common.update_dcb_map(ifname=name, pci_addr=dcb_config.pci_addr, + driver=dcb_config.driver, noop=False, + dscp2prio=dcb_config.dscp2prio) + opts = _BaseOpts.base_opts_from_json(json) return SriovPF(name, numvfs, *opts, promisc=promisc, link_mode=link_mode, ethtool_opts=ethtool_opts, diff --git a/os_net_config/schema.yaml b/os_net_config/schema.yaml index 3a0bf11..7af2b60 100644 --- a/os_net_config/schema.yaml +++ b/os_net_config/schema.yaml @@ -144,6 +144,25 @@ definitions: items: $ref: "#/definitions/address" minItems: 1 + dscp2prio: + type: object + properties: + priority: + $ref: "#/definitions/int_or_param" + protocol: + $ref: "#/definitions/int_or_param" + selector: + $ref: "#/definitions/int_or_param" + required: + - priority + - protocol + additionalProperties: False + + list_of_dscp2prio: + type: array + items: + $ref: "#/definitions/dscp2prio" + minItems: 0 route: type: object @@ -208,6 +227,24 @@ definitions: bonding_options: type: string + dscp_priority: + type: object + properties: + priotity: + $ref: "#/definitions/int_or_param" + protocol: + $ref: "#/definitions/int_or_param" + selector: + $ref: "#/definitions/int_or_param" + required: + - priotity + - protocol + + list_of_dscp_prio: + type: array + items: + $ref: "#/definitions/dscp_priority" + minItems: 0 ovs_options_string: type: string pattern: "^((?:[a-zA-Z][a-zA-Z0-9: _-]*)=(?:[a-zA-Z0-9:._-]+)[ ]*)+$" @@ -253,7 +290,30 @@ definitions: - $ref: "#/definitions/ovs_tunnel_type" - $ref: "#/definitions/param" + dcb: + type: object + properties: + dscp2prio: + $ref: "#/definitions/list_of_dscp2prio" + name: + $ref: "#/definitions/string_or_param" + required: + - dscp2prio + # os-net-config device types + dcb_config: + type: object + properties: + type: + enum: ["dcb"] + name: + $ref: "#/definitions/string_or_param" + dscp2prio: + $ref: "#/definitions/list_of_dscp2prio" + required: + - type + - name + - dscp2prio interface: type: object properties: @@ -298,6 +358,8 @@ definitions: $ref: "#/definitions/list_of_domain_name_string_or_domain_name_string" linkdelay: $ref: "#/definitions/int_or_param" + dcb: + $ref: "#/definitions/dcb" required: - type - name @@ -372,6 +434,8 @@ definitions: $ref: "#/definitions/bool_or_param" steering_mode: $ref: "#/definitions/sriov_steering_mode_or_param" + dcb: + $ref: "#/definitions/dcb" required: - type - name @@ -1589,4 +1653,5 @@ items: - $ref: "#/definitions/contrail_vrouter" - $ref: "#/definitions/contrail_vrouter_dpdk" - $ref: "#/definitions/linux_tap" + - $ref: "#/definitions/dcb_config" minItems: 1 diff --git a/os_net_config/tests/test_sriov_bind_config.py b/os_net_config/tests/test_sriov_bind_config.py index aad9e60..f838f00 100644 --- a/os_net_config/tests/test_sriov_bind_config.py +++ b/os_net_config/tests/test_sriov_bind_config.py @@ -19,9 +19,9 @@ import random +from os_net_config import common from os_net_config import sriov_bind_config from os_net_config.tests import base -from os_net_config import utils class TestSriovBindConfig(base.TestCase): @@ -48,6 +48,6 @@ def test_bind_vfs(self): os.makedirs(sriov_bind_config._PCI_DRIVER_BIND_FILE_PATH % {"driver": vfs_driver}) - utils.write_yaml_config(sriov_bind_config._SRIOV_BIND_CONFIG_FILE, - sriov_bind_pcis_map) + common.write_yaml_config(sriov_bind_config._SRIOV_BIND_CONFIG_FILE, + sriov_bind_pcis_map) sriov_bind_config.bind_vfs() diff --git a/os_net_config/tests/test_sriov_config.py b/os_net_config/tests/test_sriov_config.py index 988fc4a..b6e7ea8 100644 --- a/os_net_config/tests/test_sriov_config.py +++ b/os_net_config/tests/test_sriov_config.py @@ -24,7 +24,6 @@ from os_net_config import common from os_net_config import sriov_config from os_net_config.tests import base -from os_net_config import utils class TestSriovConfig(base.TestCase): @@ -354,7 +353,7 @@ def test_configure_sriov_pf(self): self._write_numvfs(ifname) self._action_order = [] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_config) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_config) sriov_config.configure_sriov_pf() self.assertEqual(exp_actions, self._action_order) f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r') @@ -402,7 +401,7 @@ def test_configure_sriov_pf_nicpart(self): self._write_numvfs(ifname) self._action_order = [] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_config) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_config) sriov_config.configure_sriov_pf() self.assertEqual(exp_actions, self._action_order) f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r') @@ -451,7 +450,7 @@ def test_configure_sriov_pf_non_nicpart(self): self._write_numvfs(ifname) self._action_order = [] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_config) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_config) sriov_config.configure_sriov_pf() self.assertEqual(exp_actions, self._action_order) f = open(sriov_config._UDEV_LEGACY_RULE_FILE, 'r') @@ -502,7 +501,7 @@ def test_configure_vdpa_pf(self): self._write_numvfs(ifname) self._action_order = [] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_config) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_config) sriov_config.configure_sriov_pf() self.assertEqual(exp_actions, self._action_order) self.assertEqual(10, sriov_config.get_numvfs('p2p1')) @@ -621,7 +620,7 @@ def run_ip_config_cmd_stub(*args, **kwargs): self.stub_out('os_net_config.sriov_config.run_ip_config_cmd', run_ip_config_cmd_stub) - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, vf_config) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, vf_config) sriov_config.configure_sriov_vf() for cmd in exp_cmds: diff --git a/os_net_config/tests/test_utils.py b/os_net_config/tests/test_utils.py index b089e35..935a8b0 100644 --- a/os_net_config/tests/test_utils.py +++ b/os_net_config/tests/test_utils.py @@ -198,7 +198,7 @@ def get_numvfs_stub(pf_name): get_numvfs_stub) pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy', 'name': 'eth1', 'numvfs': 10}] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) self.assertRaises(sriov_config.SRIOVNumvfsException, utils.update_sriov_pf_map, 'eth1', 20, False) @@ -210,7 +210,7 @@ def get_numvfs_stub(pf_name): pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy', 'name': 'eth1', 'numvfs': 10, 'promisc': 'on', 'vdpa': False}] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) utils.update_sriov_pf_map('eth1', 10, False, promisc='off') pf_final = [{'device_type': 'pf', 'link_mode': 'legacy', @@ -230,7 +230,7 @@ def get_numvfs_stub(pf_name): pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy', 'name': 'eth1', 'numvfs': 10, 'promisc': 'on', 'vdpa': False}] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) utils.update_sriov_pf_map('eth1', 10, False, vdpa=True) pf_final = [{'device_type': 'pf', 'link_mode': 'legacy', @@ -264,7 +264,7 @@ def get_numvfs_stub(pf_name): pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy', 'name': 'eth1', 'numvfs': 10, 'promisc': 'on', 'vdpa': False, 'lag_candidate': False}] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) utils.update_sriov_pf_map('eth1', 10, False, lag_candidate=True) pf_final = [{'device_type': 'pf', 'link_mode': 'legacy', @@ -285,7 +285,7 @@ def get_numvfs_stub(pf_name): pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy', 'name': 'eth1', 'numvfs': 10, 'promisc': 'on', 'vdpa': False}] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) utils.update_sriov_pf_map('eth1', 10, False, lag_candidate=True) pf_final = [{'device_type': 'pf', 'link_mode': 'legacy', @@ -306,7 +306,7 @@ def get_numvfs_stub(pf_name): pf_initial = [{'device_type': 'pf', 'link_mode': 'legacy', 'name': 'eth1', 'numvfs': 10, 'promisc': 'on', 'vdpa': False}] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, pf_initial) utils.update_sriov_pf_map('eth1', 10, False, lag_candidate=False) pf_final = [{'device_type': 'pf', 'link_mode': 'legacy', @@ -381,7 +381,7 @@ def test_update_sriov_vf_map_complete_new(self): def test_update_sriov_vf_map_exist(self): vf_initial = [{'device_type': 'vf', 'name': 'eth1_2', 'device': {"name": "eth1", "vfid": 2}}] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, vf_initial) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, vf_initial) utils.update_sriov_vf_map('eth1', 2, 'eth1_2', vlan_id=10, qos=5, spoofcheck="on", trust="on", state="enable", @@ -412,7 +412,7 @@ def test_update_sriov_vf_map_exist_complete(self): 'macaddr': 'AA:BB:CC:DD:EE:FF', 'promisc': 'off', 'pci_address': "0000:80:00.1"}] - utils.write_yaml_config(common.SRIOV_CONFIG_FILE, vf_initial) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, vf_initial) utils.update_sriov_vf_map('eth1', 2, 'eth1_2', vlan_id=100, qos=15, spoofcheck="off", trust="off", state="auto", @@ -684,7 +684,7 @@ def test_update_dpdk_map_exist(self): dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0', 'mac_address': '01:02:03:04:05:06', 'driver': 'vfio-pci'}] - utils.write_yaml_config(common.DPDK_MAPPING_FILE, dpdk_test) + common.write_yaml_config(common.DPDK_MAPPING_FILE, dpdk_test) utils._update_dpdk_map('eth1', '0000:03:00.0', '01:02:03:04:05:06', 'vfio-pci') @@ -697,7 +697,7 @@ def test_update_dpdk_map_exist(self): def test_update_dpdk_map_value_change(self): dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0', 'driver': 'vfio-pci'}] - utils.write_yaml_config(common.DPDK_MAPPING_FILE, dpdk_test) + common.write_yaml_config(common.DPDK_MAPPING_FILE, dpdk_test) dpdk_test = [{'name': 'eth1', 'pci_address': '0000:03:00.0', 'mac_address': '01:02:03:04:05:06', diff --git a/os_net_config/utils.py b/os_net_config/utils.py index 81bbe9b..3c88344 100644 --- a/os_net_config/utils.py +++ b/os_net_config/utils.py @@ -43,6 +43,23 @@ WantedBy=basic.target """ +# dcb_config service shall be created and enabled so that the various +# dcb configurations shall be done during reboot as well using +# dcb_config.py installed in path /usr/bin/os-net-config-dcb +_DCB_CONFIG_SERVICE_FILE = \ + '/etc/systemd/system/os-net-config-dcb-config.service' +_DCB_CONFIG_DEVICE_CONTENT = """[Unit] +Description=DCB configuration +After=NetworkManager + +[Service] +Type=oneshot +ExecStart=/usr/bin/os-net-config-dcb + +[Install] +WantedBy=basic.target +""" + # VPP startup operational configuration file. The content of this file will # be executed when VPP starts as if typed from CLI. _VPP_EXEC_FILE = '/etc/vpp/vpp-exec' @@ -69,18 +86,6 @@ def write_config(filename, data): f.write(str(data)) -def write_yaml_config(filepath, data): - ensure_directory_presence(filepath) - with open(filepath, 'w') as f: - yaml.safe_dump(data, f, default_flow_style=False) - - -def ensure_directory_presence(filepath): - dir_path = os.path.dirname(filepath) - if not os.path.exists(dir_path): - os.makedirs(dir_path) - - def is_active_nic(interface_name): return _is_available_nic(interface_name, True) @@ -296,6 +301,24 @@ def get_stored_pci_address(ifname, noop): 'ethtool' % ifname) +def get_driver(ifname, noop): + if not noop: + try: + out, err = processutils.execute('ethtool', '-i', ifname) + if not err: + for item in out.split('\n'): + if 'driver' in item: + return item.split(' ')[1] + except processutils.ProcessExecutionError: + # If ifname is already bound, then ethtool will not be able to + # list the device, in which case, binding is already done, proceed + # with scripts generation. + return + + else: + logger.info('Fetch the driver of the interface {ifname} using ethtool') + + def translate_ifname_to_pci_address(ifname, noop): pci_address = get_stored_pci_address(ifname, noop) if pci_address is None and not noop: @@ -365,7 +388,7 @@ def _update_dpdk_map(ifname, pci_address, mac_address, driver): new_item['driver'] = driver dpdk_map.append(new_item) - write_yaml_config(common.DPDK_MAPPING_FILE, dpdk_map) + common.write_yaml_config(common.DPDK_MAPPING_FILE, dpdk_map) def update_sriov_pf_map(ifname, numvfs, noop, promisc=None, @@ -404,7 +427,7 @@ def update_sriov_pf_map(ifname, numvfs, noop, promisc=None, new_item['lag_candidate'] = lag_candidate sriov_map.append(new_item) - write_yaml_config(common.SRIOV_CONFIG_FILE, sriov_map) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, sriov_map) def _set_vf_fields(vf_name, vlan_id, qos, spoofcheck, trust, state, macaddr, @@ -464,7 +487,7 @@ def update_sriov_vf_map(pf_name, vfid, vf_name, vlan_id=0, qos=0, _clear_empty_values(new_item) sriov_map.append(new_item) - write_yaml_config(common.SRIOV_CONFIG_FILE, sriov_map) + common.write_yaml_config(common.SRIOV_CONFIG_FILE, sriov_map) def _get_vf_name_from_map(pf_name, vfid): @@ -502,6 +525,23 @@ def nicpart_udev_rules_check(): fp.write(line) +def is_dcb_config_required(): + if os.path.isfile(common.DCB_CONFIG_FILE): + return True + return False + + +def configure_dcb_config_service(): + """Generate the os-net-config-dcb-config.service + + This service shall reconfigure the dcb configuration + during reboot of the nodes. + """ + with open(_DCB_CONFIG_SERVICE_FILE, 'w') as f: + f.write(_DCB_CONFIG_DEVICE_CONTENT) + processutils.execute('systemctl', 'enable', 'os-net-config-dcb-config') + + def _configure_sriov_config_service(): """Generate the sriov_config.service diff --git a/requirements.txt b/requirements.txt index ae0ac00..ceeb165 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ oslo.utils>=3.33.0 # Apache-2.0 PyYAML>=3.10.0 # MIT jsonschema>=3.2.0 # MIT pyudev>=0.16.1 # LGPLv2.1+ +pyroute2>=0.7.10 # Apache-2.0 GPL-2.0+ diff --git a/setup.cfg b/setup.cfg index d27ab8f..716dc92 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,4 +28,5 @@ packages = console_scripts = os-net-config = os_net_config.cli:main os-net-config-sriov = os_net_config.sriov_config:main + os-net-config-dcb = os_net_config.dcb_config:main os-net-config-sriov-bind = os_net_config.sriov_bind_config:main