From 084d6884ba4188a0d264884bd499dd1f45de71b8 Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Fri, 26 Jun 2020 16:32:35 -0700 Subject: [PATCH 01/17] Tests of FRR templates which rendered by sonic-cfggen --- .../docker-fpm-frr/frr/bgpd/bgpd.main.conf.j2 | 1 + .../data/sonic-cfggen/bgpd.conf.j2/all.conf | 76 ++++++++++ .../data/sonic-cfggen/bgpd.conf.j2/all.json | 33 +++++ .../sonic-cfggen/bgpd.main.conf.j2/all.conf | 56 ++++++++ .../sonic-cfggen/bgpd.main.conf.j2/all.json | 32 +++++ .../sonic-cfggen/bgpd.main.conf.j2/base.conf | 22 +++ .../sonic-cfggen/bgpd.main.conf.j2/base.json | 19 +++ .../bgpd.main.conf.j2/defaults.conf | 56 ++++++++ .../bgpd.main.conf.j2/defaults.json | 29 ++++ .../base.conf | 39 +++++ .../base.json | 42 ++++++ .../sonic-cfggen/common/daemons.common.conf | 10 ++ .../common/daemons.common.conf.json | 7 + .../data/sonic-cfggen/common/functions.conf | 1 + .../sonic-cfggen/common/functions.conf.json | 1 + .../data/sonic-cfggen/frr.conf.j2/all.conf | 91 ++++++++++++ .../data/sonic-cfggen/frr.conf.j2/all.json | 46 ++++++ .../tests/data/sonic-cfggen/isolate/isolate | 17 +++ .../data/sonic-cfggen/isolate/isolate.json | 11 ++ .../tests/data/sonic-cfggen/isolate/unisolate | 17 +++ .../data/sonic-cfggen/isolate/unisolate.json | 11 ++ .../data/sonic-cfggen/staticd/staticd.conf | 19 +++ .../sonic-cfggen/staticd/staticd.conf.json | 12 ++ .../staticd/staticd.default_route.conf | 4 + .../staticd/staticd.default_route.conf.json | 7 + .../tests/data/sonic-cfggen/tsa/isolate.conf | 5 + .../tests/data/sonic-cfggen/tsa/isolate.json | 9 ++ .../data/sonic-cfggen/tsa/unisolate.conf | 3 + .../data/sonic-cfggen/tsa/unisolate.json | 3 + .../data/sonic-cfggen/zebra/interfaces.conf | 14 ++ .../data/sonic-cfggen/zebra/interfaces.json | 10 ++ .../data/sonic-cfggen/zebra/set_src.conf | 8 ++ .../data/sonic-cfggen/zebra/set_src.json | 5 + .../tests/data/sonic-cfggen/zebra/zebra.conf | 36 +++++ .../data/sonic-cfggen/zebra/zebra.conf.json | 23 +++ src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 133 ++++++++++++++++++ 36 files changed, 908 insertions(+) create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.conf.j2/all.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.conf.j2/all.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/all.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/all.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/base.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/base.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/defaults.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/defaults.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.spine_chassis_frontend_router.conf.j2/base.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.spine_chassis_frontend_router.conf.j2/base.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/daemons.common.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/daemons.common.conf.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/functions.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/functions.conf.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/frr.conf.j2/all.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/frr.conf.j2/all.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/isolate create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/isolate.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/unisolate create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/unisolate.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.conf.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.default_route.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.default_route.conf.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/isolate.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/isolate.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/unisolate.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/unisolate.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/interfaces.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/interfaces.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/set_src.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/set_src.json create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/zebra.conf create mode 100644 src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/zebra.conf.json create mode 100644 src/sonic-bgpcfgd/tests/test_sonic-cfggen.py diff --git a/dockers/docker-fpm-frr/frr/bgpd/bgpd.main.conf.j2 b/dockers/docker-fpm-frr/frr/bgpd/bgpd.main.conf.j2 index 051cfa4e9dd9..ef98dcf9f16a 100644 --- a/dockers/docker-fpm-frr/frr/bgpd/bgpd.main.conf.j2 +++ b/dockers/docker-fpm-frr/frr/bgpd/bgpd.main.conf.j2 @@ -1,3 +1,4 @@ +{% from "common/functions.conf.j2" import get_ipv4_loopback_address, get_ipv6_loopback_address %} ! ! template: bgpd/bgpd.main.conf.j2 ! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.conf.j2/all.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.conf.j2/all.conf new file mode 100644 index 000000000000..166a5c64752c --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.conf.j2/all.conf @@ -0,0 +1,76 @@ +! +! template: bgpd/bgpd.conf.j2 +! +! =========== Managed by sonic-cfggen DO NOT edit manually! ==================== +! generated by templates/quagga/bgpd.conf.j2 with config DB data +! file: bgpd.conf +! +! template: common/daemons.common.conf.j2 +! +hostname new_hostname +password zebra +enable password zebra +! +log syslog informational +log facility local4 +! +! end of template: common/daemons.common.conf.j2! +agentx +! +! template: bgpd/bgpd.main.conf.j2 +! +! bgp multiple-instance +! +! BGP configuration +! +! TSA configuration +! +ip prefix-list PL_LoopbackV4 permit 55.55.55.55/32 +! +ipv6 prefix-list PL_LoopbackV6 permit fc00::/64 +! +route-map HIDE_INTERNAL permit 10 + set community local-AS +! +router bgp 55555 +! + bgp log-neighbor-changes + no bgp default ipv4-unicast +! + bgp bestpath as-path multipath-relax +! + bgp graceful-restart restart-time 480 + bgp graceful-restart + bgp graceful-restart preserve-fw-state +! + bgp router-id 55.55.55.55 +! + network 55.55.55.55/32 +! + address-family ipv6 + network fc00::1/64 + exit-address-family +! + network 10.10.10.1/24 + address-family ipv6 + network fc01::1/64 + exit-address-family +! + address-family ipv4 + redistribute connected route-map HIDE_INTERNAL + exit-address-family + address-family ipv6 + redistribute connected route-map HIDE_INTERNAL + exit-address-family +! + address-family ipv4 + maximum-paths 32 + exit-address-family + address-family ipv6 + maximum-paths 32 + exit-address-family +! +! end of template: bgpd/bgpd.main.conf.j2 +!! +! end of template: bgpd/bgpd.conf.j2 +! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.conf.j2/all.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.conf.j2/all.json new file mode 100644 index 000000000000..17e32589d84c --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.conf.j2/all.json @@ -0,0 +1,33 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "hostname": "new_hostname", + "bgp_asn": "55555", + "sub_role": "FrontEnd" + } + }, + "LOOPBACK_INTERFACE": { + "Loopback0|55.55.55.55/32": {}, + "Loopback0|fc00::1/128": {} + }, + "VLAN_INTERFACE": { + "Vlan10|10.10.10.1/24": {}, + "Vlan10|fc01::1/64": {} + }, + "constants": { + "bgp": { + "multipath_relax": { + "enabled": true + }, + "graceful_restart": { + "enabled": true, + "restart_time": 480 + }, + "maximum_paths": { + "enabled": true, + "ipv4": 32, + "ipv6": 32 + } + } + } +} diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/all.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/all.conf new file mode 100644 index 000000000000..8edbdb60a613 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/all.conf @@ -0,0 +1,56 @@ +! +! template: bgpd/bgpd.main.conf.j2 +! +! bgp multiple-instance +! +! BGP configuration +! +! TSA configuration +! +ip prefix-list PL_LoopbackV4 permit 55.55.55.55/32 +! +ipv6 prefix-list PL_LoopbackV6 permit fc00::/64 +! +route-map HIDE_INTERNAL permit 10 + set community local-AS +! +router bgp 55555 +! + bgp log-neighbor-changes + no bgp default ipv4-unicast +! + bgp bestpath as-path multipath-relax +! + bgp graceful-restart restart-time 480 + bgp graceful-restart + bgp graceful-restart preserve-fw-state +! + bgp router-id 55.55.55.55 +! + network 55.55.55.55/32 +! + address-family ipv6 + network fc00::1/64 + exit-address-family +! + network 10.10.10.1/24 + address-family ipv6 + network fc01::1/64 + exit-address-family +! + address-family ipv4 + redistribute connected route-map HIDE_INTERNAL + exit-address-family + address-family ipv6 + redistribute connected route-map HIDE_INTERNAL + exit-address-family +! + address-family ipv4 + maximum-paths 32 + exit-address-family + address-family ipv6 + maximum-paths 32 + exit-address-family +! +! end of template: bgpd/bgpd.main.conf.j2 +! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/all.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/all.json new file mode 100644 index 000000000000..36099674ac4e --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/all.json @@ -0,0 +1,32 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "bgp_asn": "55555", + "sub_role": "FrontEnd" + } + }, + "LOOPBACK_INTERFACE": { + "Loopback0|55.55.55.55/32": {}, + "Loopback0|fc00::1/128": {} + }, + "VLAN_INTERFACE": { + "Vlan10|10.10.10.1/24": {}, + "Vlan10|fc01::1/64": {} + }, + "constants": { + "bgp": { + "multipath_relax": { + "enabled": true + }, + "graceful_restart": { + "enabled": true, + "restart_time": 480 + }, + "maximum_paths": { + "enabled": true, + "ipv4": 32, + "ipv6": 32 + } + } + } +} diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/base.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/base.conf new file mode 100644 index 000000000000..97159456b6f9 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/base.conf @@ -0,0 +1,22 @@ +! +! template: bgpd/bgpd.main.conf.j2 +! +! bgp multiple-instance +! +! BGP configuration +! +! TSA configuration +! +ip prefix-list PL_LoopbackV4 permit 55.55.55.55/32 +! +router bgp 55555 +! + bgp log-neighbor-changes + no bgp default ipv4-unicast +! + bgp router-id 55.55.55.55 +! + network 55.55.55.55/32 +! +! end of template: bgpd/bgpd.main.conf.j2 +! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/base.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/base.json new file mode 100644 index 000000000000..692d4c78f0b9 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/base.json @@ -0,0 +1,19 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "bgp_asn": "55555", + "sub_role": "" + } + }, + "LOOPBACK_INTERFACE": { + "Loopback0|55.55.55.55/32": {}, + "Loopback1|fc00::1/128": {} + }, + "constants": { + "bgp": { + "multipath_relax": {}, + "graceful_restart": {}, + "maximum_paths": {} + } + } +} diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/defaults.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/defaults.conf new file mode 100644 index 000000000000..b444fcd7e559 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/defaults.conf @@ -0,0 +1,56 @@ +! +! template: bgpd/bgpd.main.conf.j2 +! +! bgp multiple-instance +! +! BGP configuration +! +! TSA configuration +! +ip prefix-list PL_LoopbackV4 permit 55.55.55.55/32 +! +ipv6 prefix-list PL_LoopbackV6 permit fc00::/64 +! +route-map HIDE_INTERNAL permit 10 + set community local-AS +! +router bgp 55555 +! + bgp log-neighbor-changes + no bgp default ipv4-unicast +! + bgp bestpath as-path multipath-relax +! + bgp graceful-restart restart-time 240 + bgp graceful-restart + bgp graceful-restart preserve-fw-state +! + bgp router-id 55.55.55.55 +! + network 55.55.55.55/32 +! + address-family ipv6 + network fc00::1/64 + exit-address-family +! + network 10.10.10.1/24 + address-family ipv6 + network fc01::1/64 + exit-address-family +! + address-family ipv4 + redistribute connected route-map HIDE_INTERNAL + exit-address-family + address-family ipv6 + redistribute connected route-map HIDE_INTERNAL + exit-address-family +! + address-family ipv4 + maximum-paths 64 + exit-address-family + address-family ipv6 + maximum-paths 64 + exit-address-family +! +! end of template: bgpd/bgpd.main.conf.j2 +! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/defaults.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/defaults.json new file mode 100644 index 000000000000..2db9bd771c84 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.main.conf.j2/defaults.json @@ -0,0 +1,29 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "bgp_asn": "55555", + "sub_role": "FrontEnd" + } + }, + "LOOPBACK_INTERFACE": { + "Loopback0|55.55.55.55/32": {}, + "Loopback0|fc00::1/128": {} + }, + "VLAN_INTERFACE": { + "Vlan10|10.10.10.1/24": {}, + "Vlan10|fc01::1/64": {} + }, + "constants": { + "bgp": { + "multipath_relax": { + "enabled": true + }, + "graceful_restart": { + "enabled": true + }, + "maximum_paths": { + "enabled": true + } + } + } +} diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.spine_chassis_frontend_router.conf.j2/base.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.spine_chassis_frontend_router.conf.j2/base.conf new file mode 100644 index 000000000000..581eb107ec23 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.spine_chassis_frontend_router.conf.j2/base.conf @@ -0,0 +1,39 @@ +! +! Vnet BGP instance +router bgp 555 vrf First + no bgp default ipv4-unicast + bgp log-neighbor-changes + bgp bestpath as-path multipath-relax + no bgp default ipv4-unicast + bgp graceful-restart restart-time 240 + bgp graceful-restart + bgp router-id 10.20.30.40 + neighbor 10.10.10.1 remote-as 10 + neighbor 10.10.10.1 description session1 + address-family ipv4 unicast + neighbor 10.10.10.1 activate + neighbor 10.10.10.1 soft-reconfiguration inbound + maximum-paths 64 + exit-address-family + address-family l2vpn evpn + advertise ipv4 unicast + exit-address-family +router bgp 555 vrf Second + no bgp default ipv4-unicast + bgp log-neighbor-changes + bgp bestpath as-path multipath-relax + no bgp default ipv4-unicast + bgp graceful-restart restart-time 240 + bgp graceful-restart + bgp router-id 10.20.30.40 + neighbor 20.20.20.1 remote-as 20 + neighbor 20.20.20.1 description session2 + address-family ipv4 unicast + neighbor 20.20.20.1 activate + neighbor 20.20.20.1 soft-reconfiguration inbound + maximum-paths 64 + exit-address-family + address-family l2vpn evpn + advertise ipv4 unicast + exit-address-family +! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.spine_chassis_frontend_router.conf.j2/base.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.spine_chassis_frontend_router.conf.j2/base.json new file mode 100644 index 000000000000..d6f09fb1139e --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/bgpd.spine_chassis_frontend_router.conf.j2/base.json @@ -0,0 +1,42 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "bgp_asn": "555" + } + }, + "LOOPBACK_INTERFACE": { + "Loopback0|10.20.30.40/32": {} + }, + "VNET": { + "First": { + "vni": 10 + }, + "Second": { + "vni": 20 + } + }, + "INTERFACE": { + "Ethernet0": { + "vnet_name": "First" + }, + "Ethernet0|10.10.10.10/24": {}, + "Ethernet8": { + "vnet_name": "Second" + }, + "Ethernet8|20.20.20.20/24": {}, + "Ethernet10": {}, + "Ethernet10|20.20.20.20/24": {} + }, + "BGP_NEIGHBOR": { + "10.10.10.1": { + "asn": "10", + "name": "session1", + "local_addr": "10.10.10.10" + }, + "20.20.20.1": { + "asn": "20", + "name": "session2", + "local_addr": "20.20.20.20" + } + } +} \ No newline at end of file diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/daemons.common.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/daemons.common.conf new file mode 100644 index 000000000000..14d7b99d07ed --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/daemons.common.conf @@ -0,0 +1,10 @@ +! template: common/daemons.common.conf.j2 +! +hostname test_hostname +password zebra +enable password zebra +! +log syslog informational +log facility local4 +! +! end of template: common/daemons.common.conf.j2 diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/daemons.common.conf.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/daemons.common.conf.json new file mode 100644 index 000000000000..8ef3e43694a7 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/daemons.common.conf.json @@ -0,0 +1,7 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "hostname": "test_hostname" + } + } +} diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/functions.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/functions.conf new file mode 100644 index 000000000000..8b137891791f --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/functions.conf @@ -0,0 +1 @@ + diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/functions.conf.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/functions.conf.json new file mode 100644 index 000000000000..9e26dfeeb6e6 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/common/functions.conf.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/frr.conf.j2/all.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/frr.conf.j2/all.conf new file mode 100644 index 000000000000..4c5e81f4f90d --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/frr.conf.j2/all.conf @@ -0,0 +1,91 @@ +! +! =========== Managed by sonic-cfggen DO NOT edit manually! ==================== +! generated by templates/frr.conf.j2 with config DB data +! file: frr.conf +! +! template: common/daemons.common.conf.j2 +! +hostname test_hostname +password zebra +enable password zebra +! +log syslog informational +log facility local4 +! +! end of template: common/daemons.common.conf.j2! +agentx +! +! Enable link-detect (default disabled) +interface Ethernet0 +link-detect +! +interface Ethernet4 +link-detect +! +interface PortChannel10 +link-detect +! +interface PortChannel20 +link-detect +! +!! +! +! set static default route to mgmt gateway as a backup to learned default +ip route 0.0.0.0/0 10.10.10.1 200 +!! +! +! template: bgpd/bgpd.main.conf.j2 +! +! bgp multiple-instance +! +! BGP configuration +! +! TSA configuration +! +ip prefix-list PL_LoopbackV4 permit 55.55.55.55/32 +! +ipv6 prefix-list PL_LoopbackV6 permit fc00::/64 +! +route-map HIDE_INTERNAL permit 10 + set community local-AS +! +router bgp 55555 +! + bgp log-neighbor-changes + no bgp default ipv4-unicast +! + bgp bestpath as-path multipath-relax +! + bgp graceful-restart restart-time 480 + bgp graceful-restart + bgp graceful-restart preserve-fw-state +! + bgp router-id 55.55.55.55 +! + network 55.55.55.55/32 +! + address-family ipv6 + network fc00::1/64 + exit-address-family +! + network 10.10.10.1/24 + address-family ipv6 + network fc01::1/64 + exit-address-family +! + address-family ipv4 + redistribute connected route-map HIDE_INTERNAL + exit-address-family + address-family ipv6 + redistribute connected route-map HIDE_INTERNAL + exit-address-family +! + address-family ipv4 + maximum-paths 32 + exit-address-family + address-family ipv6 + maximum-paths 32 + exit-address-family +! +! end of template: bgpd/bgpd.main.conf.j2 +!! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/frr.conf.j2/all.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/frr.conf.j2/all.json new file mode 100644 index 000000000000..d81eba2b0fe6 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/frr.conf.j2/all.json @@ -0,0 +1,46 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "bgp_asn": "55555", + "hostname": "test_hostname", + "sub_role": "FrontEnd" + } + }, + "INTERFACE": { + "Ethernet0|10.20.30.40/24": {}, + "Ethernet4|20.20.30.40/24": {} + }, + "PORTCHANNEL": { + "PortChannel10": {}, + "PortChannel20": {} + }, + "MGMT_INTERFACE": { + "eth0|10.10.10.10/24": { + "gwaddr": "10.10.10.1" + } + }, + "LOOPBACK_INTERFACE": { + "Loopback0|55.55.55.55/32": {}, + "Loopback0|fc00::1/128": {} + }, + "VLAN_INTERFACE": { + "Vlan10|10.10.10.1/24": {}, + "Vlan10|fc01::1/64": {} + }, + "constants": { + "bgp": { + "multipath_relax": { + "enabled": true + }, + "graceful_restart": { + "enabled": true, + "restart_time": 480 + }, + "maximum_paths": { + "enabled": true, + "ipv4": 32, + "ipv6": 32 + } + } + } +} \ No newline at end of file diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/isolate b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/isolate new file mode 100644 index 000000000000..74f107495298 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/isolate @@ -0,0 +1,17 @@ +#!/bin/bash +## vtysh only accepts script in stdin, so cannot be directly used in shebang +## Cut the tail of this script and feed vtysh stdin +sed -n -e '9,$p' < "$0" | vtysh "$@" +## Exit with vtysh return code +exit $? +## vtysh script start from next line, which line number MUST equal in 'sed' command above +configure terminal + router bgp 12345 + neighbor 10.20.30.40 route-map ISOLATE out + address-family ipv6 + neighbor fc00::1 route-map ISOLATE out + exit-address-family + exit +exit +clear ip bgp 10.20.30.40 soft out +clear ip bgp fc00::1 soft out diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/isolate.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/isolate.json new file mode 100644 index 000000000000..012b69ef3036 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/isolate.json @@ -0,0 +1,11 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "bgp_asn": "12345" + } + }, + "BGP_NEIGHBOR": { + "10.20.30.40": {}, + "fc00::1": {} + } +} diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/unisolate b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/unisolate new file mode 100644 index 000000000000..b9476d0b7107 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/unisolate @@ -0,0 +1,17 @@ +#!/bin/bash +## vtysh only accepts script in stdin, so cannot be directly used in shebang +## Cut the tail of this script and feed vtysh stdin +sed -n -e '9,$p' < "$0" | vtysh "$@" +## Exit with vtysh return code +exit $? +## vtysh script start from next line, which line number MUST equal in 'sed' command above +configure terminal + router bgp 12345 + no neighbor 10.20.30.40 route-map ISOLATE out + address-family ipv6 + no neighbor fc00::1 route-map ISOLATE out + exit-address-family + exit +exit +clear ip bgp 10.20.30.40 soft out +clear ip bgp fc00::1 soft out diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/unisolate.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/unisolate.json new file mode 100644 index 000000000000..012b69ef3036 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/isolate/unisolate.json @@ -0,0 +1,11 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "bgp_asn": "12345" + } + }, + "BGP_NEIGHBOR": { + "10.20.30.40": {}, + "fc00::1": {} + } +} diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.conf new file mode 100644 index 000000000000..522c0d36f296 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.conf @@ -0,0 +1,19 @@ +! +! =========== Managed by sonic-cfggen DO NOT edit manually! ==================== +! generated by templates/frr/staticd.conf.j2 using config DB data +! file: staticd.conf +! +! template: common/daemons.common.conf.j2 +! +hostname new_hostname +password zebra +enable password zebra +! +log syslog informational +log facility local4 +! +! end of template: common/daemons.common.conf.j2! +! +! set static default route to mgmt gateway as a backup to learned default +ip route 0.0.0.0/0 10.10.10.1 200 +!! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.conf.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.conf.json new file mode 100644 index 000000000000..37f660d101d6 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.conf.json @@ -0,0 +1,12 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "hostname": "new_hostname" + } + }, + "MGMT_INTERFACE": { + "eth0|10.10.10.10/24": { + "gwaddr": "10.10.10.1" + } + } +} diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.default_route.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.default_route.conf new file mode 100644 index 000000000000..11adb98ebc3c --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.default_route.conf @@ -0,0 +1,4 @@ +! +! set static default route to mgmt gateway as a backup to learned default +ip route 0.0.0.0/0 10.10.10.1 200 +! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.default_route.conf.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.default_route.conf.json new file mode 100644 index 000000000000..80cd218a790e --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/staticd/staticd.default_route.conf.json @@ -0,0 +1,7 @@ +{ + "MGMT_INTERFACE": { + "eth0|10.10.10.10/24": { + "gwaddr": "10.10.10.1" + } + } +} diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/isolate.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/isolate.conf new file mode 100644 index 000000000000..7bd155fcdb64 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/isolate.conf @@ -0,0 +1,5 @@ +route-map test_rm_name permit 2 + match ip address prefix-list PL_LoopbackV4 + set community 12345:555 +route-map test_rm_name deny 3 +! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/isolate.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/isolate.json new file mode 100644 index 000000000000..708e4013f131 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/isolate.json @@ -0,0 +1,9 @@ +{ + "constants": { + "bgp": { + "traffic_shift_community": "12345:555" + } + }, + "route_map_name": "test_rm_name", + "ip_version": "V4" +} \ No newline at end of file diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/unisolate.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/unisolate.conf new file mode 100644 index 000000000000..db53a69e7ec4 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/unisolate.conf @@ -0,0 +1,3 @@ +no route-map test_rm permit 2 +no route-map test_rm deny 3 +! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/unisolate.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/unisolate.json new file mode 100644 index 000000000000..da070431493d --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/tsa/unisolate.json @@ -0,0 +1,3 @@ +{ + "route_map_name": "test_rm" +} \ No newline at end of file diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/interfaces.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/interfaces.conf new file mode 100644 index 000000000000..919c35085b5c --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/interfaces.conf @@ -0,0 +1,14 @@ +! +! Enable link-detect (default disabled) +interface Ethernet0 +link-detect +! +interface Ethernet4 +link-detect +! +interface PortChannel10 +link-detect +! +interface PortChannel20 +link-detect +! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/interfaces.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/interfaces.json new file mode 100644 index 000000000000..e9c1bcba418d --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/interfaces.json @@ -0,0 +1,10 @@ +{ + "INTERFACE": { + "Ethernet0|10.20.30.40/24": {}, + "Ethernet4|20.20.30.40/24": {} + }, + "PORTCHANNEL": { + "PortChannel10": {}, + "PortChannel20": {} + } +} \ No newline at end of file diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/set_src.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/set_src.conf new file mode 100644 index 000000000000..b543d24e0023 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/set_src.conf @@ -0,0 +1,8 @@ +! +! Set ip source to loopback for bgp learned routes +! +route-map new_rm_name permit 10 + set src 10.20.30.40 +! +ipv4 protocol bgp route-map new_rm_name +! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/set_src.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/set_src.json new file mode 100644 index 000000000000..2e76d46fd106 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/set_src.json @@ -0,0 +1,5 @@ +{ + "rm_name": "new_rm_name", + "lo_ip": "10.20.30.40", + "ip_proto": "v4" +} \ No newline at end of file diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/zebra.conf b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/zebra.conf new file mode 100644 index 000000000000..f7b30f214d4d --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/zebra.conf @@ -0,0 +1,36 @@ +! +! =========== Managed by sonic-cfggen DO NOT edit manually! ==================== +! generated by templates/zebra/zebra.conf.j2 using config DB data +! file: zebra.conf +! +! template: common/daemons.common.conf.j2 +! +hostname new_hostname +password zebra +enable password zebra +! +log syslog informational +log facility local4 +! +! end of template: common/daemons.common.conf.j2! +! +vrf First +vni 10 +! +vrf Second +vni 20 +! +! Enable link-detect (default disabled) +interface Ethernet0 +link-detect +! +interface Ethernet4 +link-detect +! +interface PortChannel10 +link-detect +! +interface PortChannel20 +link-detect +! +!! diff --git a/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/zebra.conf.json b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/zebra.conf.json new file mode 100644 index 000000000000..9b63ff526fb9 --- /dev/null +++ b/src/sonic-bgpcfgd/tests/data/sonic-cfggen/zebra/zebra.conf.json @@ -0,0 +1,23 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "hostname": "new_hostname" + } + }, + "VNET": { + "First": { + "vni": 10 + }, + "Second": { + "vni": 20 + } + }, + "INTERFACE": { + "Ethernet0|10.20.30.40/24": {}, + "Ethernet4|20.20.30.40/24": {} + }, + "PORTCHANNEL": { + "PortChannel10": {}, + "PortChannel20": {} + } +} \ No newline at end of file diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py new file mode 100644 index 000000000000..065ecb815ccb --- /dev/null +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -0,0 +1,133 @@ +import os +import subprocess + + +from app.config import ConfigMgr +from .test_templates import compress_comments, write_result + + +TEMPLATE_PATH = os.path.abspath('../../dockers/docker-fpm-frr/frr') +DATA_PATH = "tests/data/sonic-cfggen/" +CONSTANTS_PATH = os.path.abspath('../../files/image_config/constants/constants.yml') + + +def run_test(name, template_path, json_path, match_path): + template_path = os.path.join(TEMPLATE_PATH, template_path) + json_path = os.path.join(DATA_PATH, json_path) + cfggen = os.path.abspath("../sonic-config-engine/sonic-cfggen") + command = [cfggen, "-T", TEMPLATE_PATH, "-t", template_path, "-y", json_path] + p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + assert p.returncode == 0, "sonic-cfggen for %s test returned %d code. stderr='%s'" % (name, p.returncode, stderr) + raw_generated_result = stdout + assert "None" not in raw_generated_result, "Test %s" % name + canonical_generated_result = ConfigMgr.to_canonical(raw_generated_result) + match_path = os.path.join(DATA_PATH, match_path) + # only for development write_result(match_path, raw_generated_result) + with open(match_path) as result_fp: + raw_saved_result = result_fp.read() + canonical_saved_result = ConfigMgr.to_canonical(raw_saved_result) + assert canonical_saved_result == canonical_generated_result, "Test %s" % name + + +def test_bgpd_main_conf_base(): + run_test("Base bgpd.main.conf.j2", + "bgpd/bgpd.main.conf.j2", + "bgpd.main.conf.j2/base.json", + "bgpd.main.conf.j2/base.conf") + +def test_bgpd_main_conf_comprehensive(): + run_test("Comprehensive bgpd.main.conf.j2", + "bgpd/bgpd.main.conf.j2", + "bgpd.main.conf.j2/all.json", + "bgpd.main.conf.j2/all.conf") + +def test_bgpd_main_conf_defaults(): + run_test("Defaults bgpd.main.conf.j2", + "bgpd/bgpd.main.conf.j2", + "bgpd.main.conf.j2/defaults.json", + "bgpd.main.conf.j2/defaults.conf") + +def test_tsa_isolate(): + run_test("tsa/bgpd.tsa.isolate.conf.j2", + "bgpd/tsa/bgpd.tsa.isolate.conf.j2", + "tsa/isolate.json", + "tsa/isolate.conf") + +def test_tsa_unisolate(): + run_test("tsa/bgpd.tsa.unisolate.conf.j2", + "bgpd/tsa/bgpd.tsa.unisolate.conf.j2", + "tsa/unisolate.json", + "tsa/unisolate.conf") + +def test_common_daemons(): + run_test("daemons.common.conf.j2", + "common/daemons.common.conf.j2", + "common/daemons.common.conf.json", + "common/daemons.common.conf") + +def test_common_functions(): + run_test("functions.conf.j2", + "common/functions.conf.j2", + "common/functions.conf.json", + "common/functions.conf") + +def test_staticd_default_route(): + run_test("staticd.default_route.conf.j2", + "staticd/staticd.default_route.conf.j2", + "staticd/staticd.default_route.conf.json", + "staticd/staticd.default_route.conf") + +def test_staticd(): + run_test("staticd.conf.j2", + "staticd/staticd.conf.j2", + "staticd/staticd.conf.json", + "staticd/staticd.conf") + +def test_zebra_interfaces(): + run_test("zebra.interfaces.conf.j2", + "zebra/zebra.interfaces.conf.j2", + "zebra/interfaces.json", + "zebra/interfaces.conf") + +def test_zebra_set_src(): + run_test("zebra.set_src.conf.j2", + "zebra/zebra.set_src.conf.j2", + "zebra/set_src.json", + "zebra/set_src.conf") + +def test_zebra(): + run_test("zebra.conf.j2", + "zebra/zebra.conf.j2", + "zebra/zebra.conf.json", + "zebra/zebra.conf") + +def test_isolate(): + run_test("isolate.j2", + "isolate.j2", + "isolate/isolate.json", + "isolate/isolate") + +def test_unisolate(): + run_test("unisolate.j2", + "unisolate.j2", + "isolate/unisolate.json", + "isolate/unisolate") + +def test_frr_conf(): + run_test("frr.conf.j2", + "frr.conf.j2", + "frr.conf.j2/all.json", + "frr.conf.j2/all.conf") + +def test_l3vpn_base(): + run_test("bgpd spine_chassis_frontend_router.conf.j2", + "bgpd/bgpd.spine_chassis_frontend_router.conf.j2", + "bgpd.spine_chassis_frontend_router.conf.j2/base.json", + "bgpd.spine_chassis_frontend_router.conf.j2/base.conf") + +def test_bgp_conf_all(): + run_test("bgpd/bgpd.conf", + "bgpd/bgpd.conf.j2", + "bgpd.conf.j2/all.json", + "bgpd.conf.j2/all.conf") From 32b5aaafcd7cd7d80bc6f87f5c1b0ad5b92340e7 Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 11:31:16 -0700 Subject: [PATCH 02/17] Check jinja2 version in the first test --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index 065ecb815ccb..ba045246133d 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -31,6 +31,10 @@ def run_test(name, template_path, json_path, match_path): def test_bgpd_main_conf_base(): + import jinja2 + ver = jinja2.__version__ + splitted_ver = ver.split('.') + assert int(splitted_ver[0]) >= 2 and int(splitted_ver[1]) >= 10 run_test("Base bgpd.main.conf.j2", "bgpd/bgpd.main.conf.j2", "bgpd.main.conf.j2/base.json", From 40de223b0db7415be9fab6fde9fd0b1917f591ef Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 11:48:53 -0700 Subject: [PATCH 03/17] Output jinja2 version --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index ba045246133d..df730bc77ed4 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -32,9 +32,7 @@ def run_test(name, template_path, json_path, match_path): def test_bgpd_main_conf_base(): import jinja2 - ver = jinja2.__version__ - splitted_ver = ver.split('.') - assert int(splitted_ver[0]) >= 2 and int(splitted_ver[1]) >= 10 + assert False, jinja2.__version__ run_test("Base bgpd.main.conf.j2", "bgpd/bgpd.main.conf.j2", "bgpd.main.conf.j2/base.json", From e999a3a7c98e88dd9becd72743e35830f95ae2bc Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 11:56:55 -0700 Subject: [PATCH 04/17] Output of version from sonic-cfggen --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 2 -- src/sonic-config-engine/sonic-cfggen | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index df730bc77ed4..065ecb815ccb 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -31,8 +31,6 @@ def run_test(name, template_path, json_path, match_path): def test_bgpd_main_conf_base(): - import jinja2 - assert False, jinja2.__version__ run_test("Base bgpd.main.conf.j2", "bgpd/bgpd.main.conf.j2", "bgpd.main.conf.j2/base.json", diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index c0f61d8a95f9..d97cfec8e95b 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -333,6 +333,7 @@ def main(): for attr in ['ip', 'network', 'prefixlen', 'netmask', 'broadcast']: env.filters[attr] = partial(prefix_attr, attr) template = env.get_template(template_file) + assert False, jinja2.__version__ print(template.render(sort_data(data))) if args.var != None: From c282fb9f78ef20a9c0caf2e0b01446430d900841 Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 12:03:36 -0700 Subject: [PATCH 05/17] Revert "Output of version from sonic-cfggen" This reverts commit e999a3a7c98e88dd9becd72743e35830f95ae2bc. --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 2 ++ src/sonic-config-engine/sonic-cfggen | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index 065ecb815ccb..df730bc77ed4 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -31,6 +31,8 @@ def run_test(name, template_path, json_path, match_path): def test_bgpd_main_conf_base(): + import jinja2 + assert False, jinja2.__version__ run_test("Base bgpd.main.conf.j2", "bgpd/bgpd.main.conf.j2", "bgpd.main.conf.j2/base.json", diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index d97cfec8e95b..c0f61d8a95f9 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -333,7 +333,6 @@ def main(): for attr in ['ip', 'network', 'prefixlen', 'netmask', 'broadcast']: env.filters[attr] = partial(prefix_attr, attr) template = env.get_template(template_file) - assert False, jinja2.__version__ print(template.render(sort_data(data))) if args.var != None: From 592d3bf1fa17a8310b67b8db8d2e45c5bda457af Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 12:20:08 -0700 Subject: [PATCH 06/17] Add environment to the call --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index df730bc77ed4..c2667f9b325b 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -16,7 +16,7 @@ def run_test(name, template_path, json_path, match_path): json_path = os.path.join(DATA_PATH, json_path) cfggen = os.path.abspath("../sonic-config-engine/sonic-cfggen") command = [cfggen, "-T", TEMPLATE_PATH, "-t", template_path, "-y", json_path] - p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ) stdout, stderr = p.communicate() assert p.returncode == 0, "sonic-cfggen for %s test returned %d code. stderr='%s'" % (name, p.returncode, stderr) raw_generated_result = stdout @@ -31,8 +31,6 @@ def run_test(name, template_path, json_path, match_path): def test_bgpd_main_conf_base(): - import jinja2 - assert False, jinja2.__version__ run_test("Base bgpd.main.conf.j2", "bgpd/bgpd.main.conf.j2", "bgpd.main.conf.j2/base.json", From f46d6ecea559bcb295f984b4ce9055f4cb69c736 Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 12:38:13 -0700 Subject: [PATCH 07/17] Extra debug efforts --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 2 ++ src/sonic-config-engine/sonic-cfggen | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index c2667f9b325b..4449d79291b3 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -15,6 +15,7 @@ def run_test(name, template_path, json_path, match_path): template_path = os.path.join(TEMPLATE_PATH, template_path) json_path = os.path.join(DATA_PATH, json_path) cfggen = os.path.abspath("../sonic-config-engine/sonic-cfggen") + #"/usr/bin/env", "python", command = [cfggen, "-T", TEMPLATE_PATH, "-t", template_path, "-y", json_path] p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ) stdout, stderr = p.communicate() @@ -31,6 +32,7 @@ def run_test(name, template_path, json_path, match_path): def test_bgpd_main_conf_base(): + assert False, os.environ["PYTHONPATH"] run_test("Base bgpd.main.conf.j2", "bgpd/bgpd.main.conf.j2", "bgpd.main.conf.j2/base.json", diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index c0f61d8a95f9..83f9eb592907 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -333,6 +333,8 @@ def main(): for attr in ['ip', 'network', 'prefixlen', 'netmask', 'broadcast']: env.filters[attr] = partial(prefix_attr, attr) template = env.get_template(template_file) + import os + assert False, os.environ["PYTHONPATH"] print(template.render(sort_data(data))) if args.var != None: From 7255af9e04f6703fda9f2b2e77f0080dc6dcdd2c Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 12:47:52 -0700 Subject: [PATCH 08/17] import in the right place --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 3 +-- src/sonic-config-engine/sonic-cfggen | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index 4449d79291b3..858d85c09c2d 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -17,7 +17,7 @@ def run_test(name, template_path, json_path, match_path): cfggen = os.path.abspath("../sonic-config-engine/sonic-cfggen") #"/usr/bin/env", "python", command = [cfggen, "-T", TEMPLATE_PATH, "-t", template_path, "-y", json_path] - p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ) + p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ) stdout, stderr = p.communicate() assert p.returncode == 0, "sonic-cfggen for %s test returned %d code. stderr='%s'" % (name, p.returncode, stderr) raw_generated_result = stdout @@ -32,7 +32,6 @@ def run_test(name, template_path, json_path, match_path): def test_bgpd_main_conf_base(): - assert False, os.environ["PYTHONPATH"] run_test("Base bgpd.main.conf.j2", "bgpd/bgpd.main.conf.j2", "bgpd.main.conf.j2/base.json", diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index 83f9eb592907..9b2a1ea41b79 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -27,6 +27,7 @@ from __future__ import print_function import lazy_re import sys +import os import os.path import argparse import yaml @@ -333,7 +334,6 @@ def main(): for attr in ['ip', 'network', 'prefixlen', 'netmask', 'broadcast']: env.filters[attr] = partial(prefix_attr, attr) template = env.get_template(template_file) - import os assert False, os.environ["PYTHONPATH"] print(template.render(sort_data(data))) From a062f7e1fb655c0768232ef09829028d94d452ac Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 12:59:09 -0700 Subject: [PATCH 09/17] Another --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index 858d85c09c2d..14bff87a43f1 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -17,7 +17,7 @@ def run_test(name, template_path, json_path, match_path): cfggen = os.path.abspath("../sonic-config-engine/sonic-cfggen") #"/usr/bin/env", "python", command = [cfggen, "-T", TEMPLATE_PATH, "-t", template_path, "-y", json_path] - p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ) + p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ) stdout, stderr = p.communicate() assert p.returncode == 0, "sonic-cfggen for %s test returned %d code. stderr='%s'" % (name, p.returncode, stderr) raw_generated_result = stdout From 415fad657813dc1036c8d459695a9870a082115e Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 13:08:05 -0700 Subject: [PATCH 10/17] Another variant --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 3 +-- src/sonic-config-engine/sonic-cfggen | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index 14bff87a43f1..6a63c8db5277 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -15,9 +15,8 @@ def run_test(name, template_path, json_path, match_path): template_path = os.path.join(TEMPLATE_PATH, template_path) json_path = os.path.join(DATA_PATH, json_path) cfggen = os.path.abspath("../sonic-config-engine/sonic-cfggen") - #"/usr/bin/env", "python", command = [cfggen, "-T", TEMPLATE_PATH, "-t", template_path, "-y", json_path] - p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ) + p = subprocess.Popen(" ".join(command), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() assert p.returncode == 0, "sonic-cfggen for %s test returned %d code. stderr='%s'" % (name, p.returncode, stderr) raw_generated_result = stdout diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index 9b2a1ea41b79..c0f61d8a95f9 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -27,7 +27,6 @@ from __future__ import print_function import lazy_re import sys -import os import os.path import argparse import yaml @@ -334,7 +333,6 @@ def main(): for attr in ['ip', 'network', 'prefixlen', 'netmask', 'broadcast']: env.filters[attr] = partial(prefix_attr, attr) template = env.get_template(template_file) - assert False, os.environ["PYTHONPATH"] print(template.render(sort_data(data))) if args.var != None: From 13fd86ed0833ed5a685ded165e9b62681272d9f6 Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 13:31:05 -0700 Subject: [PATCH 11/17] reorder --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 2 +- src/sonic-config-engine/sonic-cfggen | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index 6a63c8db5277..065ecb815ccb 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -16,7 +16,7 @@ def run_test(name, template_path, json_path, match_path): json_path = os.path.join(DATA_PATH, json_path) cfggen = os.path.abspath("../sonic-config-engine/sonic-cfggen") command = [cfggen, "-T", TEMPLATE_PATH, "-t", template_path, "-y", json_path] - p = subprocess.Popen(" ".join(command), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() assert p.returncode == 0, "sonic-cfggen for %s test returned %d code. stderr='%s'" % (name, p.returncode, stderr) raw_generated_result = stdout diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index c0f61d8a95f9..789573e13193 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -17,6 +17,17 @@ See usage string for detail description for arguments. from __future__ import print_function +import os +later = [] +new = [] +for d in os.environ["PYTHONPATH"].split(":"): + if d.startswith("/usr") and 'local' not in d: + later.append(d) + else: + new.append(d) +os.environ["PYTHONPATH"] = ":".join(new + later) + + # monkey patch re.compile to do lazy regular expression compilation. # This is done to improve import time of jinja2, yaml, natsort modules, because they # do many regexp compilation at import time, so it will speed up sonic-cfggen invocations From e6b077185cb3f4810bd43857da299f8372eace3a Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 13:42:51 -0700 Subject: [PATCH 12/17] Use pkg_resource --- src/sonic-config-engine/sonic-cfggen | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index 789573e13193..765d185322c6 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -17,15 +17,6 @@ See usage string for detail description for arguments. from __future__ import print_function -import os -later = [] -new = [] -for d in os.environ["PYTHONPATH"].split(":"): - if d.startswith("/usr") and 'local' not in d: - later.append(d) - else: - new.append(d) -os.environ["PYTHONPATH"] = ":".join(new + later) # monkey patch re.compile to do lazy regular expression compilation. @@ -37,6 +28,9 @@ os.environ["PYTHONPATH"] = ":".join(new + later) # FIXME: remove this once sonic-cfggen and templates dependencies are replaced with a faster approach import lazy_re +import pkg_resources +pkg_resources.require("jinja2>=2.10") + import sys import os.path import argparse From 0cdb69d225683ee08f1220b77c14386bb56b0eee Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 14:30:05 -0700 Subject: [PATCH 13/17] Remove the package --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 11 +++++++++++ src/sonic-config-engine/sonic-cfggen | 5 ----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index 065ecb815ccb..abfd313683e1 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -10,8 +10,19 @@ DATA_PATH = "tests/data/sonic-cfggen/" CONSTANTS_PATH = os.path.abspath('../../files/image_config/constants/constants.yml') +def run_cmd(cmd): + p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + assert p.returncode == 0, "cmd '%s' returned %d code. stderr='%s'" % (" ".join(cmd), p.returncode, stderr) + return stdout + +def remove_python_jinja2(): + out = run_cmd(["dpkg", "-l"]) + if 'python-jinja2' in out: + out = run_cmd(["dpkg", "-P", "python-jinja2"]) def run_test(name, template_path, json_path, match_path): + remove_python_jinja2() template_path = os.path.join(TEMPLATE_PATH, template_path) json_path = os.path.join(DATA_PATH, json_path) cfggen = os.path.abspath("../sonic-config-engine/sonic-cfggen") diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index 765d185322c6..c0f61d8a95f9 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -17,8 +17,6 @@ See usage string for detail description for arguments. from __future__ import print_function - - # monkey patch re.compile to do lazy regular expression compilation. # This is done to improve import time of jinja2, yaml, natsort modules, because they # do many regexp compilation at import time, so it will speed up sonic-cfggen invocations @@ -28,9 +26,6 @@ from __future__ import print_function # FIXME: remove this once sonic-cfggen and templates dependencies are replaced with a faster approach import lazy_re -import pkg_resources -pkg_resources.require("jinja2>=2.10") - import sys import os.path import argparse From 592cc52df91b1df23b295e817e49cba808867b29 Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 14:47:53 -0700 Subject: [PATCH 14/17] Add sudo --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index abfd313683e1..380e6833afc6 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -19,7 +19,7 @@ def run_cmd(cmd): def remove_python_jinja2(): out = run_cmd(["dpkg", "-l"]) if 'python-jinja2' in out: - out = run_cmd(["dpkg", "-P", "python-jinja2"]) + out = run_cmd(["sudo", "dpkg", "-P", "python-jinja2"]) def run_test(name, template_path, json_path, match_path): remove_python_jinja2() From 0dfec2984a8b1b5ea0028286066659f68ff8693f Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 15:01:50 -0700 Subject: [PATCH 15/17] Another --- src/sonic-bgpcfgd/tests/test_sonic-cfggen.py | 11 ----------- src/sonic-config-engine/sonic-cfggen | 2 ++ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py index 380e6833afc6..065ecb815ccb 100644 --- a/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py +++ b/src/sonic-bgpcfgd/tests/test_sonic-cfggen.py @@ -10,19 +10,8 @@ DATA_PATH = "tests/data/sonic-cfggen/" CONSTANTS_PATH = os.path.abspath('../../files/image_config/constants/constants.yml') -def run_cmd(cmd): - p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - assert p.returncode == 0, "cmd '%s' returned %d code. stderr='%s'" % (" ".join(cmd), p.returncode, stderr) - return stdout - -def remove_python_jinja2(): - out = run_cmd(["dpkg", "-l"]) - if 'python-jinja2' in out: - out = run_cmd(["sudo", "dpkg", "-P", "python-jinja2"]) def run_test(name, template_path, json_path, match_path): - remove_python_jinja2() template_path = os.path.join(TEMPLATE_PATH, template_path) json_path = os.path.join(DATA_PATH, json_path) cfggen = os.path.abspath("../sonic-config-engine/sonic-cfggen") diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index c0f61d8a95f9..7872c33bae10 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -17,6 +17,8 @@ See usage string for detail description for arguments. from __future__ import print_function +sys.path.insert(0, "/usr/local/lib/python2.7/dist-packages/jinja2") + # monkey patch re.compile to do lazy regular expression compilation. # This is done to improve import time of jinja2, yaml, natsort modules, because they # do many regexp compilation at import time, so it will speed up sonic-cfggen invocations From e76bd45bf7e048d5f909ab323d83beebe92e88d7 Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 15:25:42 -0700 Subject: [PATCH 16/17] Attempt --- src/sonic-config-engine/sonic-cfggen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index 7872c33bae10..ac1bea1968b1 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -17,7 +17,7 @@ See usage string for detail description for arguments. from __future__ import print_function -sys.path.insert(0, "/usr/local/lib/python2.7/dist-packages/jinja2") +sys.path.insert(0, "/usr/local/lib/python2.7/dist-packages") # monkey patch re.compile to do lazy regular expression compilation. # This is done to improve import time of jinja2, yaml, natsort modules, because they From 712b0ec64d29e09d97f62268b98f2f578d270922 Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Wed, 1 Jul 2020 15:37:11 -0700 Subject: [PATCH 17/17] Import --- src/sonic-config-engine/sonic-cfggen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index ac1bea1968b1..9979b5752ad1 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -16,7 +16,7 @@ See usage string for detail description for arguments. """ from __future__ import print_function - +import sys sys.path.insert(0, "/usr/local/lib/python2.7/dist-packages") # monkey patch re.compile to do lazy regular expression compilation.