Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

sax parser filter generation generic logic #1047

Merged
merged 3 commits into from
Jun 12, 2020
Merged

sax parser filter generation generic logic #1047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
921315d
logic to create sax parser filter xml
Jun 12, 2020
5d7ff36
added test case for sax filter xml
Jun 12, 2020
d73843d
logic to create sax parser filter xml
Jun 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
48 changes: 25 additions & 23 deletions lib/jnpr/junos/factory/optable.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ def generate_sax_parser_input(obj):
for key, val in group_field_dict.items():
group_ele.append(E(val.get('xpath')))
parser_ingest.append(group_ele)
map_multilayer_fields = dict()
for i, item in enumerate(local_field_dict.items()):
# i is the index and item will be taple of field key and value
field_dict = item[1]
Expand All @@ -126,32 +125,35 @@ def generate_sax_parser_input(obj):
else:
xpath = field_dict.get('xpath')
# xpath can be multi level, for ex traffic-statistics/input-pps
# going in reverse order, for fields example.
# split xpath in 2 part, search for first part xpath, if exists, append later
# else continue, and finally add full xpath to parent
# min-delay: probe-test-global-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/min-delay
# max-delay: probe-test-global-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/max-delay
# avg-delay: probe-test-global-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/avg-delay
# positive-rtt-jitter: probe-test-global-results/probe-test-generic-results/probe-test-positive-round-trip-jitter/probe-summary-results/avg-delay
# loss-percentage: probe-test-global-results/probe-test-generic-results/loss-percentage
# current-min-delay: probe-last-test-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/min-delay
# current-max-delay: probe-last-test-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/max-delay
# current-avg-delay: probe-last-test-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/avg-delay
# current-positive-rtt-jitter: probe-last-test-results/probe-test-generic-results/probe-test-positive-round-trip-jitter/probe-summary-results/avg-delay
# current-loss-percentage: probe-last-test-results/probe-test-generic-results/loss-percentage
if '/' in xpath:
tags = xpath.split('/')
if tags[0] in map_multilayer_fields:
# cases where multiple fields got same parents
# fields:
# input-bytes: traffic-statistics/input-bytes
# output-bytes: traffic-statistics/output-bytes
# or
# fields:
# prefix-count: bgp-option-information/prefix-limit/prefix-count
# prefix-dummy: bgp-option-information/prefix-limit/prefix-dummy
tags_len = len(tags)
local_elem_to_add = E(tags[-1])
for i in range(tags_len, 0, -1):
xpath = "/".join(tags[:i])
local_obj = parser_ingest
for tag in tags[:-1]:
existing_elem = local_obj.xpath(tag)
if existing_elem:
local_obj = existing_elem[0]
else:
continue
else:
local_obj.append(E(tags[-1]))
elem_exists = local_obj.xpath(xpath)
if elem_exists:
xpath_exists = elem_exists[0]
xpath_exists.insert(1, local_elem_to_add)
break
if local_elem_to_add.tag != tags[i-1]:
local_elem_to_add = E(tags[i-1], local_elem_to_add)
else:
obj = E(tags[-1])
for tag in tags[:-1][::-1]:
obj = E(tag, obj)
map_multilayer_fields[tags[0]] = obj
parser_ingest.insert(i + 1, obj)
parser_ingest.insert(1, local_elem_to_add)
else:
parser_ingest.insert(i + 1, E(xpath))
# cases where item is something like
Expand Down
197 changes: 124 additions & 73 deletions tests/unit/factory/test_optable.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,78 +22,78 @@
from mock import patch


@attr('unit')
@attr("unit")
class TestFactoryOpTable(unittest.TestCase):

@patch('ncclient.manager.connect')
@patch("ncclient.manager.connect")
def setUp(self, mock_connect):
mock_connect.side_effect = self._mock_manager
self.dev = Device(host='1.1.1.1', user='rick', password='password123',
gather_facts=False)
self.dev = Device(
host="1.1.1.1", user="rick", password="password123", gather_facts=False
)
self.dev.open()
self.ppt = PhyPortStatsTable(self.dev)

@patch('jnpr.junos.Device.execute')
@patch("jnpr.junos.Device.execute")
def test_optable_get(self, mock_execute):
mock_execute.side_effect = self._mock_manager
self.ppt.get()
self.assertEqual(len(self.ppt), 2)

@patch('jnpr.junos.Device.execute')
@patch("jnpr.junos.Device.execute")
def test_optable_get_key(self, mock_execute):
mock_execute.side_effect = self._mock_manager
self.ppt.get('ge-0/0/0')
self.assertEqual(self.ppt.GET_KEY, 'interface_name')
self.ppt.get("ge-0/0/0")
self.assertEqual(self.ppt.GET_KEY, "interface_name")

def test_optable_path(self):
fname = 'local-get-interface-information.xml'
path = os.path.join(os.path.dirname(__file__),
'rpc-reply', fname)
fname = "local-get-interface-information.xml"
path = os.path.join(os.path.dirname(__file__), "rpc-reply", fname)
lppt = PhyPortStatsTable(path=path)
lppt.get()
self.assertEqual(len(lppt), 2)

def test_optable_xml(self):
fname = 'get-interface-information.xml'
fname = "get-interface-information.xml"
xml = self._read_file(fname)
lppt = PhyPortStatsTable(xml=xml)
lppt.get()
self.assertEqual(len(lppt), 2)

@patch('jnpr.junos.Device.execute')
@patch("jnpr.junos.Device.execute")
def test_optable_view_get(self, mock_execute):
mock_execute.side_effect = self._mock_manager
self.ppt.get()
v = self.ppt['ge-0/0/0']
self.assertEqual(v['rx_packets'], 1207)
v = self.ppt["ge-0/0/0"]
self.assertEqual(v["rx_packets"], 1207)

@patch('jnpr.junos.Device.execute')
@patch("jnpr.junos.Device.execute")
def test_optable_view_get_astype_bool(self, mock_execute):
mock_execute.side_effect = self._mock_manager
et = EthPortTable(self.dev)
et.get()
v = et['ge-0/0/0']
self.assertEqual(v['present'], True)
v = et["ge-0/0/0"]
self.assertEqual(v["present"], True)

@patch('jnpr.junos.Device.execute')
@patch("jnpr.junos.Device.execute")
def test_optable_view_get_astype_bool_regex(self, mock_execute):
mock_execute.side_effect = self._mock_manager
from jnpr.junos.op.bfd import BfdSessionTable

bfd = BfdSessionTable(self.dev)
bfd.get()
v = bfd['10.92.20.4']
self.assertEqual(v['no_absorb'], True)
v = bfd["10.92.20.4"]
self.assertEqual(v["no_absorb"], True)

@patch('jnpr.junos.Device.execute')
@patch("jnpr.junos.Device.execute")
def test_optable_view_get_unknown_field(self, mock_execute):
mock_execute.side_effect = self._mock_manager
self.ppt.get()

def bad(key):
v = self.ppt['ge-0/0/0']
v = self.ppt["ge-0/0/0"]
return v[key]

self.assertRaises(ValueError, bad, 'bunk')
self.assertRaises(ValueError, bad, "bunk")

def test_generate_sax_parser_fields_with_many_slash(self):
yaml_data = """
Expand All @@ -108,15 +108,63 @@ def test_generate_sax_parser_fields_with_many_slash(self):
prefix-count: bgp-option-information/prefix-limit/prefix-count
prefix-dummy: bgp-option-information/prefix-limit/prefix-dummy
"""
globals().update(FactoryLoader().load(yaml.load(yaml_data,
Loader=yaml.Loader)))
globals().update(FactoryLoader().load(yaml.load(yaml_data, Loader=yaml.Loader)))
tbl = bgpNeighborTable(self.dev)
data = generate_sax_parser_input(tbl)
self.assertEqual(data.tag, 'bgp-peer')
self.assertEqual(len(etree.tostring(data)), len(
b'<bgp-peer><peer-address/><bgp-option-information><prefix-limit>'
b'<prefix-count/><prefix-dummy/></prefix-limit>'
b'</bgp-option-information></bgp-peer>'))
self.assertEqual(data.tag, "bgp-peer")
self.assertEqual(
len(etree.tostring(data)),
len(
b"<bgp-peer><peer-address/><bgp-option-information><prefix-limit>"
b"<prefix-count/><prefix-dummy/></prefix-limit>"
b"</bgp-option-information></bgp-peer>"
),
)

def test_generate_sax_parser_fields_with_diff_child_xpaths(self):
yaml_data = """
---
twampProbeTable:
rpc: twamp-get-probe-results
item: probe-test-results
key: test-name
view: probeResultsView
probeResultsView:
fields:
min-delay: probe-test-global-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/min-delay
max-delay: probe-test-global-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/max-delay
avg-delay: probe-test-global-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/avg-delay
positive-rtt-jitter: probe-test-global-results/probe-test-generic-results/probe-test-positive-round-trip-jitter/probe-summary-results/avg-delay
loss-percentage: probe-test-global-results/probe-test-generic-results/loss-percentage
current-min-delay: probe-last-test-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/min-delay
current-max-delay: probe-last-test-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/max-delay
current-avg-delay: probe-last-test-results/probe-test-generic-results/probe-test-rtt/probe-summary-results/avg-delay
current-positive-rtt-jitter: probe-last-test-results/probe-test-generic-results/probe-test-positive-round-trip-jitter/probe-summary-results/avg-delay
current-loss-percentage: probe-last-test-results/probe-test-generic-results/loss-percentage
"""
globals().update(FactoryLoader().load(yaml.load(yaml_data, Loader=yaml.Loader)))
tbl = twampProbeTable(self.dev)
data = generate_sax_parser_input(tbl)
self.assertEqual(data.tag, "probe-test-results")
self.assertEqual(
len(etree.tostring(data)),
len(
b"<probe-test-results><test-name/><probe-last-test-results>"
b"<probe-test-generic-results><probe-test-rtt><probe-summary"
b"-results><min-delay/><avg-delay/><max-delay/></probe-summary"
b"-results></probe-test-rtt><loss-percentage/>"
b"<probe-test-positive-round-trip-jitter><probe-summary-results>"
b"<avg-delay/></probe-summary-results></probe-test-positive-round"
b"-trip-jitter></probe-test-generic-results></probe-last-test-resu"
b"lts><probe-test-global-results><probe-test-generic-results>"
b"<probe-test-rtt><probe-summary-results><min-delay/><avg-delay/>"
b"<max-delay/></probe-summary-results></probe-test-rtt>"
b"<loss-percentage/><probe-test-positive-round-trip-jitter>"
b"<probe-summary-results><avg-delay/></probe-summary-results>"
b"</probe-test-positive-round-trip-jitter></probe-test-generic-"
b"results></probe-test-global-results></probe-test-results>"
),
)

def test_generate_sax_parser_item_with_many_slash(self):
yaml_data = """
Expand All @@ -137,15 +185,18 @@ def test_generate_sax_parser_item_with_many_slash(self):
tmmaxallocbytes: tm-max-alloc-bytes
tmfunctioncalls: tm-function-calls
"""
globals().update(FactoryLoader().load(yaml.load(yaml_data,
Loader=yaml.Loader)))
globals().update(FactoryLoader().load(yaml.load(yaml_data, Loader=yaml.Loader)))
tbl = taskmallocdetail(self.dev)
data = generate_sax_parser_input(tbl)
self.assertEqual(data.tag, 'task-memory-malloc-usage-report')
self.assertEqual(len(etree.tostring(data)), len(
b'<task-memory-malloc-usage-report><task-malloc-list><task-malloc><tm-name/><t'
b'm-allocs/><tm-alloc-bytes/><tm-max-allocs/><tm-max-alloc-bytes/><tm-function'
b'-calls/></task-malloc></task-malloc-list></task-memory-malloc-usage-report>'))
self.assertEqual(data.tag, "task-memory-malloc-usage-report")
self.assertEqual(
len(etree.tostring(data)),
len(
b"<task-memory-malloc-usage-report><task-malloc-list><task-malloc><tm-name/><t"
b"m-allocs/><tm-alloc-bytes/><tm-max-allocs/><tm-max-alloc-bytes/><tm-function"
b"-calls/></task-malloc></task-malloc-list></task-memory-malloc-usage-report>"
),
)

def test_generate_sax_parser_same_parents_with_diff_fields(self):
yaml_data = """
Expand Down Expand Up @@ -183,23 +234,25 @@ def test_generate_sax_parser_same_parents_with_diff_fields(self):
tunnel-input-bytes: traffic-statistics/input-bytes
tunnel-output-bytes: traffic-statistics/output-bytes
"""
globals().update(FactoryLoader().load(yaml.load(yaml_data,
Loader=yaml.Loader)))
globals().update(FactoryLoader().load(yaml.load(yaml_data, Loader=yaml.Loader)))
tbl = VtepTable(self.dev)
data = generate_sax_parser_input(tbl)
self.assertEqual(data.tag, 'physical-interface')
self.assertEqual(len(etree.tostring(data)),len(
b'<physical-interface><name/><admin-status/><oper-status/>'
b'<link-level-type/><traffic-statistics><input-bytes/>'
b'<output-bytes/></traffic-statistics><input-error-list>'
b'<input-errors/></input-error-list><output-error-list>'
b'<output-errors/><carrier-transitions/></output-error-list>'
b'<logical-interface><name/><vtep-info><vtep-type/><vtep-address/>'
b'</vtep-info><traffic-statistics><input-bytes/><output-bytes/>'
b'</traffic-statistics></logical-interface></physical-interface>')
self.assertEqual(data.tag, "physical-interface")
self.assertEqual(
len(etree.tostring(data)),
len(
b"<physical-interface><name/><admin-status/><oper-status/>"
b"<link-level-type/><traffic-statistics><input-bytes/>"
b"<output-bytes/></traffic-statistics><input-error-list>"
b"<input-errors/></input-error-list><output-error-list>"
b"<output-errors/><carrier-transitions/></output-error-list>"
b"<logical-interface><name/><vtep-info><vtep-type/><vtep-address/>"
b"</vtep-info><traffic-statistics><input-bytes/><output-bytes/>"
b"</traffic-statistics></logical-interface></physical-interface>"
),
)

@patch('jnpr.junos.Device.execute')
@patch("jnpr.junos.Device.execute")
def test_key_pipe_delim_with_Null(self, mock_execute):
mock_execute.side_effect = self._mock_manager
yaml_data = """
Expand All @@ -214,14 +267,15 @@ def test_key_pipe_delim_with_Null(self, mock_execute):
fields:
running: { running: flag }
"""
globals().update(FactoryLoader().load(yaml.load(yaml_data,
Loader=yaml.Loader)))
globals().update(FactoryLoader().load(yaml.load(yaml_data, Loader=yaml.Loader)))
tbl = UTMStatusTable(self.dev)
data = tbl.get()
self.assertEqual(json.loads(data.to_json()),
{'node0': {'running': True}, 'node1': {'running': True}})
self.assertEqual(
json.loads(data.to_json()),
{"node0": {"running": True}, "node1": {"running": True}},
)

@patch('jnpr.junos.Device.execute')
@patch("jnpr.junos.Device.execute")
def test_key_pipe_delim_with_Null_use_Null(self, mock_execute):
mock_execute.side_effect = self._mock_manager
yaml_data = """
Expand All @@ -236,13 +290,12 @@ def test_key_pipe_delim_with_Null_use_Null(self, mock_execute):
fields:
running: { running: flag }
"""
globals().update(FactoryLoader().load(yaml.load(yaml_data,
Loader=yaml.Loader)))
globals().update(FactoryLoader().load(yaml.load(yaml_data, Loader=yaml.Loader)))
tbl = UTMStatusTable(self.dev)
data = tbl.get()
self.assertEqual(json.loads(data.to_json()), {'running': True})
self.assertEqual(json.loads(data.to_json()), {"running": True})

@patch('jnpr.junos.Device.execute')
@patch("jnpr.junos.Device.execute")
def test_key_and_item_pipe_delim_with_Null_use_Null(self, mock_execute):
mock_execute.side_effect = self._mock_manager
yaml_data = """
Expand All @@ -258,33 +311,31 @@ def test_key_and_item_pipe_delim_with_Null_use_Null(self, mock_execute):
fields:
running: { running: flag }
"""
globals().update(FactoryLoader().load(yaml.load(yaml_data,
Loader=yaml.Loader)))
globals().update(FactoryLoader().load(yaml.load(yaml_data, Loader=yaml.Loader)))
tbl = UTMStatusTable(self.dev)
data = tbl.get()
self.assertEqual(json.loads(data.to_json()), {'running': True})
self.assertEqual(json.loads(data.to_json()), {"running": True})

def _read_file(self, fname):
from ncclient.xml_ import NCElement

fpath = os.path.join(os.path.dirname(__file__),
'rpc-reply', fname)
fpath = os.path.join(os.path.dirname(__file__), "rpc-reply", fname)
foo = open(fpath).read()
reply = RPCReply(foo)
reply.parse()
rpc_reply = NCElement(reply, self.dev._conn.
_device_handler.transform_reply())\
._NCElement__doc[0]
rpc_reply = NCElement(
reply, self.dev._conn._device_handler.transform_reply()
)._NCElement__doc[0]
return rpc_reply

def _mock_manager(self, *args, **kwargs):
if kwargs:
if args and ('normalize' in kwargs or 'filter_xml' in kwargs):
return self._read_file(args[0].tag + '.xml')
device_params = kwargs['device_params']
if args and ("normalize" in kwargs or "filter_xml" in kwargs):
return self._read_file(args[0].tag + ".xml")
device_params = kwargs["device_params"]
device_handler = make_device_handler(device_params)
session = SSHSession(device_handler)
return Manager(session, device_handler)

if args:
return self._read_file(args[0].tag + '.xml')
return self._read_file(args[0].tag + ".xml")