Skip to content

Path improv #909

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 17, 2023
Merged

Path improv #909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
48df64b
pandas/engineobj: add a new rtn, in_subnet_series
ddutt Jul 15, 2023
d415b7b
Path: Map missing src to SVI
ddutt Jul 15, 2023
4ed54aa
Updated path tests based on new logic
ddutt Jul 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 33 additions & 5 deletions suzieq/engines/pandas/engineobj.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,15 +105,15 @@ def _is_any_in_list(self, column: pd.Series, values: List) -> pd.Series:
return column.apply(lambda x: any(v in x for v in values))

def _is_in_subnet(self, addr: pd.Series, net: str) -> pd.Series:
"""Check if the IP addresses in a Pandas dataframe
belongs to the given subnet
"""check which of the addresses belongs to a given subnet

Used to implement the prefix filter of arpnd/address/dhcp
Args:
addr (PandasObject): the collection of ip addresses to check
net: (str): network id of the subnet
addr (pd.Series): the pandas series of ip addresses to check
net: (str): the IP network to check the addresses against

Returns:
PandasObject: A collection of bool reporting the result
pd.Series: A collection of bool reporting the result
"""
network = ip_network(net)
if isinstance(addr.iloc[0], np.ndarray):
Expand All @@ -127,6 +127,34 @@ def _is_in_subnet(self, addr: pd.Series, net: str) -> pd.Series:
False if not a else ip_address(a.split("/")[0]) in network)
)

def _in_subnet_series(self, addr: str, net: pd.Series) -> pd.Series:
"""Check if an addr is in any of series' subnets

unlike is_in_subnet which checks if a pandas series of addresses
belongs in a given subnet, this routine checks if a given address
belongs to any of the provided pandas series of subnets. THis is
currently used in path to identify an SVI for a given address

Args:
addr (str): the address we're checking for
net: (pd.Series): the pandas series of subnets

Returns:
pd.Series: A collection of bool reporting the result
"""
address = ip_address(addr)
if isinstance(net.iloc[0], np.ndarray):
return net.apply(lambda x, network:
False if not x.any()
else any(address in ip_network(a, strict=False)
for a in x if a != '0.0.0.0/0'),
args=(address,))
else:
return net.apply(lambda a: (
False if not a or a == '0.0.0.0/0'
else address in ip_network(a, strict=False))
)

def _check_ipvers(self, addr: pd.Series, version: int) -> pd.Series:
"""Check if the IP version of addresses in a Pandas dataframe
correspond to the given version
Expand Down
58 changes: 44 additions & 14 deletions suzieq/engines/pandas/path.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from typing import Optional, List, Any, Iterable
from ipaddress import ip_network, ip_address
from collections import OrderedDict
from itertools import repeat
Expand All @@ -19,7 +20,7 @@ class PathObj(SqPandasEngine):
'''Backend class to handle manipulating virtual table, path, with pandas'''

@staticmethod
def table_name():
def table_name() -> str:
'''Table name'''
return 'path'

Expand Down Expand Up @@ -137,7 +138,34 @@ def _init_dfs(self, ns, source, dest):
self._srcnode_via_arp = True

if self._src_df.empty:
raise AttributeError(f"Invalid src {source}")
# See if we can find an mlag pair of devices that contains the SVI
# pandas has a bug that prevents us from using startswith with a
# tuple such as ("Vlan", "vlan", "irb.") directly instead of using
# the @svi_names trick
# pylint: disable=unused-variable
svi_names = tuple(["Vlan", "vlan", "irb."]) # noqa: F841
if ':' in source:
self._src_df = (
self._if_df.query(
f'@self._in_subnet_series("{source}", ip6AddressList)'
' and ifname.str.startswith(@svi_names)')
)
else:
self._src_df = (
self._if_df.query(
f'@self._in_subnet_series("{source}", ipAddressList)'
' and ifname.str.startswith(@svi_names)')
)
if not self._src_df.empty:
hosts = self._src_df.hostname.unique().tolist()
if len(hosts) > 2:
raise ValueError(
'source not in ARP and SVI on too many hosts')
self._srcnode_via_arp = True

if self._src_df.empty:
raise AttributeError(f"Unable to find starting node for {source}")

src_hostname = self._src_df.hostname.unique().tolist()[0]

if self._src_df.hostname.nunique() == 1 and len(self._src_df) > 1:
Expand Down Expand Up @@ -241,7 +269,7 @@ def _get_vrf(self, hostname: str, ifname: str, addr: str) -> str:

return vrf

def _find_fhr_df(self, device: str, ip: str) -> pd.DataFrame:
def _find_fhr_df(self, device: Optional[str], ip: str) -> pd.DataFrame:
"""Find Firstt Hop Router's iface DF for a given IP and device.
The logic in finding the next hop router is:
find the arp table entry corresponding to the IP provided;
Expand Down Expand Up @@ -317,12 +345,12 @@ def _get_if_vlan(self, device: str, ifname: str) -> int:
(self._if_df["ifname"] == ifname)]

if oif_df.empty:
return []
return -1

return oif_df.iloc[0]["vlan"]

def _get_l2_nexthop(self, device: str, vrf: str, dest: str,
macaddr: str, protocol: str) -> list:
macaddr: Optional[str], protocol: str) -> list:
"""Get the bridged/tunnel nexthops
We're passing protocol because we need to keep the return
match the other get nexthop function returns. We don't really
Expand Down Expand Up @@ -387,7 +415,7 @@ def _get_l2_nexthop(self, device: str, vrf: str, dest: str,

def _get_underlay_nexthop(self, hostname: str, vtep_list: list,
vrf_list: list,
is_overlay: bool) -> pd.DataFrame:
is_overlay: bool) -> List[Any]:
"""Return the underlay nexthop given the Vtep and VRF"""

# WARNING: This function is incomplete right now
Expand Down Expand Up @@ -486,7 +514,7 @@ def _handle_recursive_route(self, df: pd.DataFrame,
return df

def _get_nexthops(self, device: str, vrf: str, dest: str, is_l2: bool,
vtep: str, macaddr: str) -> list:
vtep: str, macaddr: str) -> Iterable:
"""Get nexthops (oif + IP + overlay) or just oif for given host/vrf.

The overlay is a bit indicating we're getting into overlay or not.
Expand Down Expand Up @@ -612,6 +640,7 @@ def _get_nh_with_peer(self, device: str, vrf: str, dest: str, is_l2: bool,
for (nhip, iface, overlay, l2hop, protocol,
timestamp) in new_nexthop_list:
df = pd.DataFrame()
arpdf = pd.DataFrame()
errormsg = ''
if l2hop and macaddr and not overlay:
if (not nhip or nhip == 'None') and iface:
Expand Down Expand Up @@ -721,7 +750,7 @@ def _get_nh_with_peer(self, device: str, vrf: str, dest: str, is_l2: bool,
'state!="failed"')
if not revarp_df.empty:
df = df.query(f'ifname == "{revarp_df.oif.iloc[0]}"')
df.apply(lambda x, nexthops:
df.apply(lambda x, nexthops: # type: ignore
nexthops.append((iface, x['hostname'],
x['ifname'], overlay,
l2hop, nhip,
Expand Down Expand Up @@ -761,8 +790,8 @@ def get(self, **kwargs) -> pd.DataFrame:
if not src or not dest:
raise AttributeError("Must specify trace source and dest")

srcvers = ip_network(src, strict=False)._version
dstvers = ip_network(dest, strict=False)._version
srcvers = ip_network(src, strict=False).version
dstvers = ip_network(dest, strict=False).version
if srcvers != dstvers:
raise AttributeError(
"Source and Dest MUST belong to same address familt")
Expand All @@ -771,7 +800,8 @@ def get(self, **kwargs) -> pd.DataFrame:
self._init_dfs(self.namespace, src, dest)

devices_iifs = OrderedDict()
src_mtu = None
src_mtu: int = MAX_MTU + 1
item = None
for i in range(len(self._src_df)):
item = self._src_df.iloc[i]
devices_iifs[f'{item.hostname}/'] = {
Expand All @@ -793,9 +823,9 @@ def get(self, **kwargs) -> pd.DataFrame:
"l3_visited_devices": set(),
"l2_visited_devices": set()
}
if src_mtu is None or (item.get('mtu', 0) < src_mtu):
src_mtu = item.get('mtu', 0)
if not dvrf:
if (src_mtu > MAX_MTU) or (item.get('mtu', 0) < src_mtu):
src_mtu = item.get('mtu', 0) # type: ignore
if not dvrf and item is not None:
dvrf = item['master']
if not dvrf:
dvrf = "default"
Expand Down
112 changes: 110 additions & 2 deletions tests/integration/sqcmds/cumulus-samples/path.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1959,9 +1959,111 @@ tests:
- command: path show --dest=172.16.2.104 --src=172.16.1.104 --namespace=dual-evpn
--format=json
data-directory: tests/data/parquet/
error:
error: '[{"error": "ERROR: Invalid src 172.16.1.104"}]'
marks: path show cumulus
output: '[{"pathid": 1, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01",
"iif": "vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false,
"mtuMatch": true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup":
"172.16.2.0/24", "vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1",
"hopError": "", "timestamp": 1616644822008}, {"pathid": 1, "hopCount": 1, "namespace":
"dual-evpn", "hostname": "spine01", "iif": "swp6", "oif": "swp3", "vrf": "default",
"isL2": true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216,
"protocol": "bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null,
"nexthopIp": "", "hopError": "", "timestamp": 1616644822008}, {"pathid": 1, "hopCount":
2, "namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp1", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
{"pathid": 2, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
"vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822167}, {"pathid": 2, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine01", "iif": "swp5", "oif": "swp3", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 2, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp1", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
{"pathid": 3, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01", "iif":
"vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822008}, {"pathid": 3, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine01", "iif": "swp6", "oif": "swp4", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 3, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp1", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983},
{"pathid": 4, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
"vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822167}, {"pathid": 4, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine01", "iif": "swp5", "oif": "swp4", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 4, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp1", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983},
{"pathid": 5, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01", "iif":
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822008}, {"pathid": 5, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine02", "iif": "swp6", "oif": "swp3", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 5, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp2", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
{"pathid": 6, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822167}, {"pathid": 6, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine02", "iif": "swp5", "oif": "swp3", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 6, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp2", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
{"pathid": 7, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01", "iif":
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822008}, {"pathid": 7, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine02", "iif": "swp6", "oif": "swp4", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 7, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp2", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983},
{"pathid": 8, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822167}, {"pathid": 8, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine02", "iif": "swp5", "oif": "swp4", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 8, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp2", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983}]'
- command: path show --dest=10.0.0.11 --src=10.0.0.14 --namespace=ospf-single --format=json
data-directory: tests/data/parquet/
marks: path show cumulus
Expand Down Expand Up @@ -3405,3 +3507,9 @@ tests:
marks: path top cumulus
output: '[{"hostname": "leaf04"}, {"hostname": "leaf04"}, {"hostname": "spine01"},
{"hostname": "leaf01"}, {"hostname": "spine02"}]'
- command: path show --dest=172.16.2.104 --src=172.16.21.104 --namespace=dual-evpn
--format=json
data-directory: tests/data/parquet/
error:
error: '[{"error": "ERROR: Unable to find starting node for 172.16.21.104"}]'
marks: path show cumulus
Loading