From a1d03a491da5119d962f49ba484f38f17faea384 Mon Sep 17 00:00:00 2001 From: anish-n <44376847+anish-n@users.noreply.github.com> Date: Tue, 12 Jan 2021 14:41:02 -0800 Subject: [PATCH] [fgnhgorch] Match mode changes for Fine Grained ECMP (#1565) So that Fine Grained ECMP routes can be filtered via next-hop IPs(match_mode next-hop-based). The originally implemented solution filtered routes via the prefix, this continues to be supported via match_mode route-based. --- orchagent/fgnhgorch.cpp | 306 +++++++++------ orchagent/fgnhgorch.h | 25 +- orchagent/routeorch.cpp | 168 ++++++--- tests/test_fgnhg.py | 815 ++++++++++++++++++++++------------------ 4 files changed, 775 insertions(+), 539 deletions(-) diff --git a/orchagent/fgnhgorch.cpp b/orchagent/fgnhgorch.cpp index 61012bef630e..7077ff4790fc 100644 --- a/orchagent/fgnhgorch.cpp +++ b/orchagent/fgnhgorch.cpp @@ -30,6 +30,7 @@ FgNhgOrch::FgNhgOrch(DBConnector *db, DBConnector *appDb, DBConnector *stateDb, m_routeTable(appDb, APP_ROUTE_TABLE_NAME) { SWSS_LOG_ENTER(); + isFineGrainedConfigured = false; gPortsOrch->attach(this); } @@ -289,7 +290,7 @@ bool FgNhgOrch::createFineGrainedNextHopGroup(FGNextHopGroupEntry &syncd_fg_rout { SWSS_LOG_ERROR("Failed to query next hop group %s SAI_NEXT_HOP_GROUP_ATTR_REAL_SIZE, rv:%d", nextHops.to_string().c_str(), status); - if (!removeFineGrainedNextHopGroup(&syncd_fg_route_entry, fgNhgEntry)) + if (!removeFineGrainedNextHopGroup(&syncd_fg_route_entry)) { SWSS_LOG_ERROR("Failed to clean-up after next hop group real_size query failure"); } @@ -305,7 +306,7 @@ bool FgNhgOrch::createFineGrainedNextHopGroup(FGNextHopGroupEntry &syncd_fg_rout } -bool FgNhgOrch::removeFineGrainedNextHopGroup(FGNextHopGroupEntry *syncd_fg_route_entry, FgNhgEntry *fgNhgEntry) +bool FgNhgOrch::removeFineGrainedNextHopGroup(FGNextHopGroupEntry *syncd_fg_route_entry) { SWSS_LOG_ENTER(); sai_status_t status; @@ -331,43 +332,6 @@ bool FgNhgOrch::removeFineGrainedNextHopGroup(FGNextHopGroupEntry *syncd_fg_rout } -bool FgNhgOrch::createFineGrainedRouteEntry(FGNextHopGroupEntry &syncd_fg_route_entry, FgNhgEntry *fgNhgEntry, - sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const NextHopGroupKey &nextHops) -{ - SWSS_LOG_ENTER(); - sai_route_entry_t route_entry; - sai_attribute_t route_attr; - route_entry.vr_id = vrf_id; - route_entry.switch_id = gSwitchId; - copy(route_entry.destination, ipPrefix); - route_attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID; - route_attr.value.oid = syncd_fg_route_entry.next_hop_group_id; - sai_status_t status = sai_route_api->create_route_entry(&route_entry, 1, &route_attr); - if (status != SAI_STATUS_SUCCESS) - { - SWSS_LOG_ERROR("Failed to create route %s with next hop(s) %s", - ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); - - /* Clean up the newly created next hop group entry */ - if (!removeFineGrainedNextHopGroup(&syncd_fg_route_entry, fgNhgEntry)) - { - SWSS_LOG_ERROR("Failed to clean-up after route creation failure"); - } - return false; - } - - if (route_entry.destination.addr_family == SAI_IP_ADDR_FAMILY_IPV4) - { - gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE); - } - else - { - gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE); - } - return true; -} - - bool FgNhgOrch::validNextHopInNextHopGroup(const NextHopKey& nexthop) { SWSS_LOG_ENTER(); @@ -382,15 +346,22 @@ bool FgNhgOrch::validNextHopInNextHopGroup(const NextHopKey& nexthop) } FGNextHopGroupEntry *syncd_fg_route_entry = &(route_table.second); - auto prefix_entry = fgNhgPrefixes.find(route_table.first); - if (prefix_entry == fgNhgPrefixes.end()) + FgNhgEntry *fgNhgEntry = 0; + auto prefix_entry = m_fgNhgPrefixes.find(route_table.first); + if (prefix_entry == m_fgNhgPrefixes.end()) { - SWSS_LOG_ERROR("Hit unexpected condition for %s:%s where route_table exists but no fgNhgEntry found", - route_table.first.to_string().c_str(), syncd_fg_route_entry->nhg_key.to_string().c_str()); - return false; + auto member_entry = m_fgNhgNexthops.find(nexthop.ip_address); + if (member_entry == m_fgNhgNexthops.end()) + { + SWSS_LOG_ERROR("fgNhgOrch got a validNextHopInNextHopGroup for non-configured FG ECMP entry"); + return false; + } + fgNhgEntry = member_entry->second; + } + else + { + fgNhgEntry = prefix_entry->second; } - - FgNhgEntry *fgNhgEntry = prefix_entry->second; std::map nhopgroup_members_set; std::vector bank_member_changes( @@ -445,15 +416,23 @@ bool FgNhgOrch::invalidNextHopInNextHopGroup(const NextHopKey& nexthop) } FGNextHopGroupEntry *syncd_fg_route_entry = &(route_table.second); - auto prefix_entry = fgNhgPrefixes.find(route_table.first); - if (prefix_entry == fgNhgPrefixes.end()) + FgNhgEntry *fgNhgEntry = 0; + auto prefix_entry = m_fgNhgPrefixes.find(route_table.first); + if (prefix_entry == m_fgNhgPrefixes.end()) { - SWSS_LOG_ERROR("Hit unexpected condition for %s:%s where route_table exists but no fgNhgEntry found", - route_table.first.to_string().c_str(), syncd_fg_route_entry->nhg_key.to_string().c_str()); - return false; + auto member_entry = m_fgNhgNexthops.find(nexthop.ip_address); + if (member_entry == m_fgNhgNexthops.end()) + { + SWSS_LOG_ERROR("fgNhgOrch got an invalidNextHopInNextHopGroup for non-configured FG ECMP entry"); + return false; + } + fgNhgEntry = member_entry->second; + } + else + { + fgNhgEntry = prefix_entry->second; } - FgNhgEntry *fgNhgEntry = prefix_entry->second; std::map nhopgroup_members_set; std::vector bank_member_changes( @@ -1002,7 +981,7 @@ bool FgNhgOrch::setNewNhgMembers(FGNextHopGroupEntry &syncd_fg_route_entry, FgNh SWSS_LOG_ERROR("Failed to create next hop group %" PRIx64 " member %" PRIx64 ": %d", syncd_fg_route_entry.next_hop_group_id, next_hop_group_member_id, status); - if (!removeFineGrainedNextHopGroup(&syncd_fg_route_entry, fgNhgEntry)) + if (!removeFineGrainedNextHopGroup(&syncd_fg_route_entry)) { SWSS_LOG_ERROR("Failed to clean-up after next-hop member creation failure"); } @@ -1027,10 +1006,108 @@ bool FgNhgOrch::setNewNhgMembers(FGNextHopGroupEntry &syncd_fg_route_entry, FgNh } -bool FgNhgOrch::addRoute(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const NextHopGroupKey &nextHops) +bool FgNhgOrch::isRouteFineGrained(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const NextHopGroupKey &nextHops) +{ + SWSS_LOG_ENTER(); + + if (!isFineGrainedConfigured || (vrf_id != gVirtualRouterId)) + { + return false; + } + + FgNhgEntry *fgNhgEntry = 0; + set next_hop_set = nextHops.getNextHops(); + auto prefix_entry = m_fgNhgPrefixes.find(ipPrefix); + if (prefix_entry == m_fgNhgPrefixes.end()) + { + for (NextHopKey nhk : next_hop_set) + { + auto member_entry = m_fgNhgNexthops.find(nhk.ip_address); + if (member_entry == m_fgNhgNexthops.end()) + { + if (fgNhgEntry) + { + SWSS_LOG_WARN("Route %s:%s has some FG nhs, but %s is not, route is defaulted to non-fine grained ECMP", + ipPrefix.to_string().c_str(), nextHops.to_string().c_str(), nhk.to_string().c_str()); + } + return false; + } + + if (!fgNhgEntry) + { + fgNhgEntry = member_entry->second; + } + else + { + /* Case where fgNhgEntry is alredy found via previous nexthop + * We validate the it belongs to the same next-hop group set + */ + if (fgNhgEntry != member_entry->second) + { + SWSS_LOG_INFO("FG nh found across different FG_NH groups: %s expected %s, actual %s", + nhk.to_string().c_str(), fgNhgEntry->fg_nhg_name.c_str(), member_entry->second->fg_nhg_name.c_str()); + return false; + } + } + } + } + return true; +} + + +bool FgNhgOrch::syncdContainsFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix) +{ + if (!isFineGrainedConfigured || (vrf_id != gVirtualRouterId)) + { + return false; + } + + auto it_route_table = m_syncdFGRouteTables.find(vrf_id); + if (it_route_table == m_syncdFGRouteTables.end()) + { + return false; + } + + auto it_route = it_route_table->second.find(ipPrefix); + if (it_route == it_route_table->second.end()) + { + return false; + } + return true; +} + + +bool FgNhgOrch::setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const NextHopGroupKey &nextHops, + sai_object_id_t &next_hop_id, bool &prevNhgWasFineGrained) { SWSS_LOG_ENTER(); + /* default prevNhgWasFineGrained to true so that sai route is unaffected + * when we return early with success */ + prevNhgWasFineGrained = true; + FgNhgEntry *fgNhgEntry = 0; + set next_hop_set = nextHops.getNextHops(); + auto prefix_entry = m_fgNhgPrefixes.find(ipPrefix); + if (prefix_entry != m_fgNhgPrefixes.end()) + { + fgNhgEntry = prefix_entry->second; + } + else + { + for (NextHopKey nhk : next_hop_set) + { + auto member_entry = m_fgNhgNexthops.find(nhk.ip_address); + if (member_entry == m_fgNhgNexthops.end()) + { + SWSS_LOG_ERROR("fgNhgOrch got a route addition %s:%s for non-configured FG ECMP entry", + ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); + return false; + } + fgNhgEntry = member_entry->second; + break; + } + } + if (m_syncdFGRouteTables.find(vrf_id) != m_syncdFGRouteTables.end() && m_syncdFGRouteTables.at(vrf_id).find(ipPrefix) != m_syncdFGRouteTables.at(vrf_id).end() && m_syncdFGRouteTables.at(vrf_id).at(ipPrefix).nhg_key == nextHops) @@ -1044,11 +1121,6 @@ bool FgNhgOrch::addRoute(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const m_vrfOrch->increaseVrfRefCount(vrf_id); } - auto prefix_entry = fgNhgPrefixes.find(ipPrefix); - assert(prefix_entry != fgNhgPrefixes.end()); - FgNhgEntry *fgNhgEntry = prefix_entry->second; - - set next_hop_set = nextHops.getNextHops(); std::map nhopgroup_members_set; auto syncd_fg_route_entry_it = m_syncdFGRouteTables.at(vrf_id).find(ipPrefix); bool next_hop_to_add = false; @@ -1117,14 +1189,13 @@ bool FgNhgOrch::addRoute(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const } } - sai_object_id_t next_hop_id = m_neighOrch->getNextHopId(nhk); - nhopgroup_members_set[nhk] = next_hop_id; + sai_object_id_t nhid = m_neighOrch->getNextHopId(nhk); + nhopgroup_members_set[nhk] = nhid; } if (syncd_fg_route_entry_it != m_syncdFGRouteTables.at(vrf_id).end()) { FGNextHopGroupEntry *syncd_fg_route_entry = &(syncd_fg_route_entry_it->second); - /* Route exists, update FG ECMP group in SAI */ for (auto nhk : syncd_fg_route_entry->active_nexthops) { @@ -1149,6 +1220,7 @@ bool FgNhgOrch::addRoute(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const else { /* New route + nhg addition */ + prevNhgWasFineGrained = false; if (next_hop_to_add == false) { SWSS_LOG_INFO("There were no valid next-hops to add %s:%s", ipPrefix.to_string().c_str(), @@ -1168,11 +1240,6 @@ bool FgNhgOrch::addRoute(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const return false; } - if (!createFineGrainedRouteEntry(syncd_fg_route_entry, fgNhgEntry, vrf_id, ipPrefix, nextHops)) - { - return false; - } - m_syncdFGRouteTables[vrf_id][ipPrefix] = syncd_fg_route_entry; SWSS_LOG_NOTICE("Created route %s:%s", ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); @@ -1196,14 +1263,20 @@ bool FgNhgOrch::addRoute(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const } } + next_hop_id = m_syncdFGRouteTables[vrf_id][ipPrefix].next_hop_group_id; return true; } -bool FgNhgOrch::removeRoute(sai_object_id_t vrf_id, const IpPrefix &ipPrefix) +bool FgNhgOrch::removeFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix) { SWSS_LOG_ENTER(); + if (!isFineGrainedConfigured) + { + return true; + } + auto it_route_table = m_syncdFGRouteTables.find(vrf_id); if (it_route_table == m_syncdFGRouteTables.end()) { @@ -1220,32 +1293,8 @@ bool FgNhgOrch::removeRoute(sai_object_id_t vrf_id, const IpPrefix &ipPrefix) return true; } - auto prefix_entry = fgNhgPrefixes.find(ipPrefix); - assert(prefix_entry != fgNhgPrefixes.end()); - FgNhgEntry *fgNhgEntry = prefix_entry->second; - - sai_route_entry_t route_entry; - route_entry.vr_id = vrf_id; - route_entry.switch_id = gSwitchId; - copy(route_entry.destination, ipPrefix); - sai_status_t status = sai_route_api->remove_route_entry(&route_entry); - if (status != SAI_STATUS_SUCCESS) - { - SWSS_LOG_ERROR("Failed to remove route prefix:%s", ipPrefix.to_string().c_str()); - return false; - } - - if (route_entry.destination.addr_family == SAI_IP_ADDR_FAMILY_IPV4) - { - gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE); - } - else - { - gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE); - } - FGNextHopGroupEntry *syncd_fg_route_entry = &(it_route->second); - if (!removeFineGrainedNextHopGroup(syncd_fg_route_entry, fgNhgEntry)) + if (!removeFineGrainedNextHopGroup(syncd_fg_route_entry)) { SWSS_LOG_ERROR("Failed to clean-up fine grained ECMP SAI group"); return false; @@ -1331,6 +1380,7 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t) string key = kfvKey(t); string fg_nhg_name = key; auto fgNhg_entry = m_FgNhgs.find(fg_nhg_name); + FGMatchMode match_mode = ROUTE_BASED; if (op == SET_COMMAND) { @@ -1342,6 +1392,18 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t) { bucket_size = stoi(fvValue(i)); } + else if (fvField(i) == "match_mode") + { + if (fvValue(i) == "nexthop-based") + { + match_mode = NEXTHOP_BASED; + } + else if (fvValue(i) != "route-based") + { + SWSS_LOG_WARN("Received unsupported match_mode %s, defaulted to route-based", + fvValue(i).c_str()); + } + } } if (bucket_size == 0) @@ -1352,20 +1414,17 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t) if (fgNhg_entry != m_FgNhgs.end()) { - if (bucket_size != (fgNhg_entry->second).configured_bucket_size) - { - SWSS_LOG_WARN("Received request to change %s's bucket size to %d, unsupported operation, skipping", - fg_nhg_name.c_str(), bucket_size); - return true; - } + SWSS_LOG_WARN("FG_NHG %s already exists, ignoring", fg_nhg_name.c_str()); } else { FgNhgEntry fgNhgEntry; fgNhgEntry.configured_bucket_size = bucket_size; fgNhgEntry.fg_nhg_name = fg_nhg_name; - SWSS_LOG_INFO("Added new FG_NHG entry with configured_bucket_size %d", - fgNhgEntry.configured_bucket_size); + fgNhgEntry.match_mode = match_mode; + SWSS_LOG_NOTICE("Added new FG_NHG entry with bucket_size %d, match_mode: %'" PRIu8, + bucket_size, match_mode); + isFineGrainedConfigured = true; m_FgNhgs[fg_nhg_name] = fgNhgEntry; } } @@ -1382,7 +1441,6 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t) if (fgNhg_entry->second.prefixes.size() == 0 && fgNhg_entry->second.next_hops.size() == 0) { m_FgNhgs.erase(fgNhg_entry); - assert(m_FgNhgs.find(fg_nhg_name) == fgNhgPrefixes.end()); SWSS_LOG_INFO("Received delete call for valid entry with no further dependencies, deleting %s", fg_nhg_name.c_str()); } @@ -1392,6 +1450,10 @@ bool FgNhgOrch::doTaskFgNhg(const KeyOpFieldsValuesTuple & t) fg_nhg_name.c_str()); return false; } + if (m_FgNhgs.size() == 0) + { + isFineGrainedConfigured = false; + } } } return true; @@ -1404,11 +1466,11 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t) string op = kfvOp(t); string key = kfvKey(t); IpPrefix ip_prefix = IpPrefix(key); - auto prefix_entry = fgNhgPrefixes.find(ip_prefix); + auto prefix_entry = m_fgNhgPrefixes.find(ip_prefix); if (op == SET_COMMAND) { - if (prefix_entry != fgNhgPrefixes.end()) + if (prefix_entry != m_fgNhgPrefixes.end()) { SWSS_LOG_INFO("FG_NHG prefix already exists"); return true; @@ -1435,6 +1497,13 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t) return false; } + if (fgNhg_entry->second.match_mode == NEXTHOP_BASED) + { + SWSS_LOG_NOTICE("FG_NHG %s is configured as nexthop_based: FG_NHG_PREFIX is a no-op", + fg_nhg_name.c_str()); + return true; + } + /* delete regular ecmp handling for prefix */ sai_object_id_t vrf_id = gVirtualRouterId; NextHopGroupKey nhg = gRouteOrch->getSyncdRouteNhgKey(vrf_id, ip_prefix); @@ -1445,7 +1514,7 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t) { SWSS_LOG_INFO("Route does not exist in routeorch, don't need to migrate route to fgnhgorch"); fgNhg_entry->second.prefixes.push_back(ip_prefix); - fgNhgPrefixes[ip_prefix] = &(fgNhg_entry->second); + m_fgNhgPrefixes[ip_prefix] = &(fgNhg_entry->second); } else { @@ -1462,7 +1531,7 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t) /* Case where APP_DB route entry was present and the route delete was completed */ SWSS_LOG_INFO("Route removed in routeorch, now do an APP_DB addition"); fgNhg_entry->second.prefixes.push_back(ip_prefix); - fgNhgPrefixes[ip_prefix] = &(fgNhg_entry->second); + m_fgNhgPrefixes[ip_prefix] = &(fgNhg_entry->second); m_routeTable.set(ip_prefix.to_string(), generateRouteTableFromNhgKey(addCache->second)); m_fgPrefixAddCache.erase(addCache); SWSS_LOG_INFO("Performed APP_DB addition with prefix %s", ip_prefix.to_string().c_str()); @@ -1474,11 +1543,11 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t) } } SWSS_LOG_INFO("FG_NHG added for group %s, prefix %s", - fgNhgPrefixes[ip_prefix]->fg_nhg_name.c_str(), ip_prefix.to_string().c_str()); + m_fgNhgPrefixes[ip_prefix]->fg_nhg_name.c_str(), ip_prefix.to_string().c_str()); } else if (op == DEL_COMMAND) { - if (prefix_entry == fgNhgPrefixes.end()) + if (prefix_entry == m_fgNhgPrefixes.end()) { SWSS_LOG_INFO("FG_NHG prefix doesn't exists, ignore"); return true; @@ -1508,7 +1577,7 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t) } } - fgNhgPrefixes.erase(ip_prefix); + m_fgNhgPrefixes.erase(ip_prefix); } else { @@ -1532,7 +1601,7 @@ bool FgNhgOrch::doTaskFgNhgPrefix(const KeyOpFieldsValuesTuple & t) break; } } - fgNhgPrefixes.erase(ip_prefix); + m_fgNhgPrefixes.erase(ip_prefix); m_routeTable.set(ip_prefix.to_string(), generateRouteTableFromNhgKey(delCache->second)); SWSS_LOG_INFO("Perform APP_DB addition with prefix %s", ip_prefix.to_string().c_str()); @@ -1637,6 +1706,14 @@ bool FgNhgOrch::doTaskFgNhgMember(const KeyOpFieldsValuesTuple & t) } } + fgNhg_entry->second.next_hops[next_hop] = fg_nh_info; + + if (fgNhg_entry->second.match_mode == NEXTHOP_BASED) + { + SWSS_LOG_NOTICE("Add member %s as NEXTHOP_BASED", next_hop.to_string().c_str()); + m_fgNhgNexthops[next_hop] = &(fgNhg_entry->second); + } + /* query and check the next hop is valid in neighOrcch */ if (!m_neighOrch->hasNextHop(nhk)) { @@ -1648,11 +1725,13 @@ bool FgNhgOrch::doTaskFgNhgMember(const KeyOpFieldsValuesTuple & t) if (!validNextHopInNextHopGroup(nhk)) { cleanupIpInLinkToIpMap(link, next_hop, fgNhg_entry->second); + fgNhg_entry->second.next_hops.erase(next_hop); + m_fgNhgNexthops.erase(next_hop); SWSS_LOG_INFO("Failing validNextHopInNextHopGroup for %s", nhk.to_string().c_str()); return false; } } - fgNhg_entry->second.next_hops[next_hop] = fg_nh_info; + SWSS_LOG_INFO("FG_NHG member added for group %s, next-hop %s", fgNhg_entry->second.fg_nhg_name.c_str(), nhk.to_string().c_str()); } @@ -1679,12 +1758,13 @@ bool FgNhgOrch::doTaskFgNhgMember(const KeyOpFieldsValuesTuple & t) { string link = it->second.link; cleanupIpInLinkToIpMap(link, next_hop, fgnhg_it->second); - fgnhg_it->second.next_hops.erase(it); SWSS_LOG_INFO("FG_NHG member removed for group %s, next-hop %s", - fgnhg_it->second.fg_nhg_name.c_str(), next_hop.to_string().c_str()); + fgnhg_it->second.fg_nhg_name.c_str(), next_hop.to_string().c_str()); + break; } } + m_fgNhgNexthops.erase(next_hop); } return true; } diff --git a/orchagent/fgnhgorch.h b/orchagent/fgnhgorch.h index c096f7b90ebd..551d55ff29c2 100644 --- a/orchagent/fgnhgorch.h +++ b/orchagent/fgnhgorch.h @@ -52,6 +52,11 @@ typedef std::map FgPrefixOpCache; /* Map from link name to next-hop IP */ typedef std::unordered_map> Links; +enum FGMatchMode +{ + ROUTE_BASED, + NEXTHOP_BASED +}; /* Store the indices occupied by a bank */ typedef struct { @@ -68,10 +73,13 @@ typedef struct FgNhgEntry Links links; // Link to IP map for oper changes std::vector prefixes; // Prefix which desires FG behavior std::vector hash_bucket_indices; // The hash bucket indices for a bank + FGMatchMode match_mode; // Stores a match_mode from FGMatchModes } FgNhgEntry; /* Map from IP prefix to user configured FG NHG entries */ typedef std::map FgNhgPrefixes; +/* Map from IP address to user configured FG NHG entries */ +typedef std::map FgNhgMembers; /* Main structure to hold user configuration */ typedef std::map FgNhgs; @@ -89,14 +97,15 @@ typedef map WarmBootRecoveryMap; class FgNhgOrch : public Orch, public Observer { public: - FgNhgPrefixes fgNhgPrefixes; FgNhgOrch(DBConnector *db, DBConnector *appDb, DBConnector *stateDb, vector &tableNames, NeighOrch *neighOrch, IntfsOrch *intfsOrch, VRFOrch *vrfOrch); void update(SubjectType type, void *cntx); - bool addRoute(sai_object_id_t, const IpPrefix&, const NextHopGroupKey&); - bool removeRoute(sai_object_id_t, const IpPrefix&); + bool isRouteFineGrained(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const NextHopGroupKey &nextHops); + bool syncdContainsFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix); bool validNextHopInNextHopGroup(const NextHopKey&); bool invalidNextHopInNextHopGroup(const NextHopKey&); + bool setFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const NextHopGroupKey &nextHops, sai_object_id_t &next_hop_id, bool &prevNhgWasFineGrained); + bool removeFgNhg(sai_object_id_t vrf_id, const IpPrefix &ipPrefix); // warm reboot support bool bake() override; @@ -105,10 +114,16 @@ class FgNhgOrch : public Orch, public Observer NeighOrch *m_neighOrch; IntfsOrch *m_intfsOrch; VRFOrch *m_vrfOrch; + FgNhgs m_FgNhgs; FGRouteTables m_syncdFGRouteTables; + FgNhgMembers m_fgNhgNexthops; + FgNhgPrefixes m_fgNhgPrefixes; + bool isFineGrainedConfigured; + Table m_stateWarmRestartRouteTable; ProducerStateTable m_routeTable; + FgPrefixOpCache m_fgPrefixAddCache; FgPrefixOpCache m_fgPrefixDelCache; @@ -137,9 +152,7 @@ class FgNhgOrch : public Orch, public Observer const IpPrefix &ipPrefix, NextHopKey nextHop); bool createFineGrainedNextHopGroup(FGNextHopGroupEntry &syncd_fg_route_entry, FgNhgEntry *fgNhgEntry, const NextHopGroupKey &nextHops); - bool removeFineGrainedNextHopGroup(FGNextHopGroupEntry *syncd_fg_route_entry, FgNhgEntry *fgNhgEntry); - bool createFineGrainedRouteEntry(FGNextHopGroupEntry &syncd_fg_route_entry, FgNhgEntry *fgNhgEntry, - sai_object_id_t vrf_id, const IpPrefix &ipPrefix, const NextHopGroupKey &nextHops); + bool removeFineGrainedNextHopGroup(FGNextHopGroupEntry *syncd_fg_route_entry); vector generateRouteTableFromNhgKey(NextHopGroupKey nhg); void cleanupIpInLinkToIpMap(const string &link, const IpAddress &ip, FgNhgEntry &fgNhg_entry); diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index d8ba2b48249f..4eb830f8fc8c 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -1233,19 +1233,12 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) sai_object_id_t& vrf_id = ctx.vrf_id; IpPrefix& ipPrefix = ctx.ip_prefix; - if (m_fgNhgOrch->fgNhgPrefixes.find(ipPrefix) != m_fgNhgOrch->fgNhgPrefixes.end() - && vrf_id == gVirtualRouterId) - { - /* Only support the default vrf for Fine Grained ECMP */ - SWSS_LOG_INFO("Reroute %s:%s to fgNhgOrch", ipPrefix.to_string().c_str(), - nextHops.to_string().c_str()); - return m_fgNhgOrch->addRoute(vrf_id, ipPrefix, nextHops); - } - /* next_hop_id indicates the next hop id or next hop group id of this route */ sai_object_id_t next_hop_id = SAI_NULL_OBJECT_ID; bool overlay_nh = false; bool status = false; + bool curNhgIsFineGrained = false; + bool prevNhgWasFineGrained = false; if (m_syncdRoutes.find(vrf_id) == m_syncdRoutes.end()) { @@ -1260,9 +1253,23 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) auto it_route = m_syncdRoutes.at(vrf_id).find(ipPrefix); - /* The route is pointing to a next hop */ - if (nextHops.getSize() == 1) + if (m_fgNhgOrch->isRouteFineGrained(vrf_id, ipPrefix, nextHops)) + { + /* The route is pointing to a Fine Grained nexthop group */ + curNhgIsFineGrained = true; + /* We get 3 return values from setFgNhg: + * 1. success/failure: on addition/modification of nexthop group/members + * 2. next_hop_id: passed as a param to fn, used for sai route creation + * 3. prevNhgWasFineGrained: passed as a param to fn, used to determine transitions + * between regular and FG ECMP, this is an optimization to prevent multiple lookups */ + if (!m_fgNhgOrch->setFgNhg(vrf_id, ipPrefix, nextHops, next_hop_id, prevNhgWasFineGrained)) + { + return false; + } + } + else if (nextHops.getSize() == 1) { + /* The route is pointing to a next hop */ NextHopKey nexthop; if (overlay_nh) { @@ -1435,12 +1442,21 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops) gRouteBulker.set_entry_attribute(&object_statuses.back(), &route_entry, &route_attr); } - route_attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID; - route_attr.value.oid = next_hop_id; + if (curNhgIsFineGrained && prevNhgWasFineGrained) + { + /* Don't change route entry if the route is previously fine grained and new nhg is also fine grained. + * We already modifed sai nhg objs as part of setFgNhg to account for nhg change. */ + object_statuses.emplace_back(SAI_STATUS_SUCCESS); + } + else + { + route_attr.id = SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID; + route_attr.value.oid = next_hop_id; - /* Set the next hop ID to a new value */ - object_statuses.emplace_back(); - gRouteBulker.set_entry_attribute(&object_statuses.back(), &route_entry, &route_attr); + /* Set the next hop ID to a new value */ + object_statuses.emplace_back(); + gRouteBulker.set_entry_attribute(&object_statuses.back(), &route_entry, &route_attr); + } } return false; } @@ -1451,6 +1467,7 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey const sai_object_id_t& vrf_id = ctx.vrf_id; const IpPrefix& ipPrefix = ctx.ip_prefix; + bool isFineGrained = false; const auto& object_statuses = ctx.object_statuses; @@ -1463,9 +1480,14 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey /* next_hop_id indicates the next hop id or next hop group id of this route */ sai_object_id_t next_hop_id; - /* The route is pointing to a next hop */ - if (nextHops.getSize() == 1) + if (m_fgNhgOrch->isRouteFineGrained(vrf_id, ipPrefix, nextHops)) + { + /* Route is pointing to Fine Grained ECMP nexthop group */ + isFineGrained = true; + } + else if (nextHops.getSize() == 1) { + /* The route is pointing to a next hop */ NextHopKey nexthop; if(nextHops.is_overlay_nexthop()) { nexthop = NextHopKey(nextHops.to_string(), true); @@ -1508,7 +1530,51 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey auto it_status = object_statuses.begin(); auto it_route = m_syncdRoutes.at(vrf_id).find(ipPrefix); - if (it_route == m_syncdRoutes.at(vrf_id).end()) + if (isFineGrained) + { + if (it_route == m_syncdRoutes.at(vrf_id).end()) + { + /* First time route addition pointing to FG nhg */ + if (*it_status++ != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to create route %s with next hop(s) %s", + ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); + /* Clean up the newly created next hop group entry */ + m_fgNhgOrch->removeFgNhg(vrf_id, ipPrefix); + return false; + } + + if (ipPrefix.isV4()) + { + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV4_ROUTE); + } + else + { + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_ROUTE); + } + SWSS_LOG_INFO("FG Post create route %s with next hop(s) %s", + ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); + } + else + { + /* Route already exists */ + auto nh_entry = m_syncdNextHopGroups.find(it_route->second); + if (nh_entry != m_syncdNextHopGroups.end()) + { + /* Case where route was pointing to non-fine grained nhs in the past, + * and transitioned to Fine Grained ECMP */ + decreaseNextHopRefCount(it_route->second); + if (it_route->second.getSize() > 1 + && m_syncdNextHopGroups[it_route->second].ref_count == 0) + { + m_bulkNhgReducedRefCnt.emplace(it_route->second); + } + } + SWSS_LOG_INFO("FG Post set route %s with next hop(s) %s", + ipPrefix.to_string().c_str(), nextHops.to_string().c_str()); + } + } + else if (it_route == m_syncdRoutes.at(vrf_id).end()) { if (*it_status++ != SAI_STATUS_SUCCESS) { @@ -1564,16 +1630,24 @@ bool RouteOrch::addRoutePost(const RouteBulkContext& ctx, const NextHopGroupKey /* Increase the ref_count for the next hop (group) entry */ increaseNextHopRefCount(nextHops); - decreaseNextHopRefCount(it_route->second); - auto ol_nextHops = it_route->second; - if (it_route->second.getSize() > 1 - && m_syncdNextHopGroups[it_route->second].ref_count == 0) + if (m_fgNhgOrch->syncdContainsFgNhg(vrf_id, ipPrefix)) { - m_bulkNhgReducedRefCnt.emplace(it_route->second); - } else if (ol_nextHops.is_overlay_nexthop()){ + /* Remove FG nhg since prefix now points to standard nhg/nhs */ + m_fgNhgOrch->removeFgNhg(vrf_id, ipPrefix); + } + else + { + decreaseNextHopRefCount(it_route->second); + auto ol_nextHops = it_route->second; + if (it_route->second.getSize() > 1 + && m_syncdNextHopGroups[it_route->second].ref_count == 0) + { + m_bulkNhgReducedRefCnt.emplace(it_route->second); + } else if (ol_nextHops.is_overlay_nexthop()){ - SWSS_LOG_NOTICE("Update overlay Nexthop %s", ol_nextHops.to_string().c_str()); - removeOverlayNextHops(vrf_id, ol_nextHops); + SWSS_LOG_NOTICE("Update overlay Nexthop %s", ol_nextHops.to_string().c_str()); + removeOverlayNextHops(vrf_id, ol_nextHops); + } } SWSS_LOG_INFO("Post set route %s with next hop(s) %s", @@ -1593,14 +1667,6 @@ bool RouteOrch::removeRoute(RouteBulkContext& ctx) sai_object_id_t& vrf_id = ctx.vrf_id; IpPrefix& ipPrefix = ctx.ip_prefix; - if (m_fgNhgOrch->fgNhgPrefixes.find(ipPrefix) != m_fgNhgOrch->fgNhgPrefixes.end() - && vrf_id == gVirtualRouterId) - { - /* Only support the default vrf for Fine Grained ECMP */ - SWSS_LOG_INFO("Reroute %s to fgNhgOrch", ipPrefix.to_string().c_str()); - return m_fgNhgOrch->removeRoute(vrf_id, ipPrefix); - } - auto it_route_table = m_syncdRoutes.find(vrf_id); if (it_route_table == m_syncdRoutes.end()) { @@ -1710,20 +1776,28 @@ bool RouteOrch::removeRoutePost(const RouteBulkContext& ctx) } } - /* - * Decrease the reference count only when the route is pointing to a next hop. - */ - decreaseNextHopRefCount(it_route->second); + if (m_fgNhgOrch->syncdContainsFgNhg(vrf_id, ipPrefix)) + { + /* Delete Fine Grained nhg if the revmoved route pointed to it */ + m_fgNhgOrch->removeFgNhg(vrf_id, ipPrefix); + } + else + { + /* + * Decrease the reference count only when the route is pointing to a next hop. + */ + decreaseNextHopRefCount(it_route->second); - auto ol_nextHops = it_route->second; + auto ol_nextHops = it_route->second; - if (it_route->second.getSize() > 1 - && m_syncdNextHopGroups[it_route->second].ref_count == 0) - { - m_bulkNhgReducedRefCnt.emplace(it_route->second); - } else if (ol_nextHops.is_overlay_nexthop()){ - SWSS_LOG_NOTICE("Remove overlay Nexthop %s", ol_nextHops.to_string().c_str()); - removeOverlayNextHops(vrf_id, ol_nextHops); + if (it_route->second.getSize() > 1 + && m_syncdNextHopGroups[it_route->second].ref_count == 0) + { + m_bulkNhgReducedRefCnt.emplace(it_route->second); + } else if (ol_nextHops.is_overlay_nexthop()){ + SWSS_LOG_NOTICE("Remove overlay Nexthop %s", ol_nextHops.to_string().c_str()); + removeOverlayNextHops(vrf_id, ol_nextHops); + } } SWSS_LOG_INFO("Remove route %s with next hop(s) %s", diff --git a/tests/test_fgnhg.py b/tests/test_fgnhg.py index 6b6af86385fd..9e465935ada1 100644 --- a/tests/test_fgnhg.py +++ b/tests/test_fgnhg.py @@ -17,21 +17,19 @@ FG_NHG_MEMBER = 'FG_NHG_MEMBER' ROUTE_TB = "ROUTE_TABLE" ASIC_ROUTE_TB = "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY" +ASIC_NHG = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP" ASIC_NHG_MEMB = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER" ASIC_NH_TB = "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP" - def create_entry(db, table, key, pairs): db.create_entry(table, key, pairs) programmed_table = db.wait_for_entry(table,key) assert programmed_table != {} - def remove_entry(db, table, key): db.delete_entry(table, key) db.wait_for_deleted_entry(table,key) - def get_asic_route_key(asic_db, ipprefix): route_exists = False key = '' @@ -46,7 +44,6 @@ def get_asic_route_key(asic_db, ipprefix): assert route_exists return key - def validate_asic_nhg_fine_grained_ecmp(asic_db, ipprefix, size): def _access_function(): false_ret = (False, '') @@ -66,7 +63,7 @@ def _access_function(): return false_ret nhgid = fvs.get("SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID") - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nhgid) + fvs = asic_db.get_entry(ASIC_NHG, nhgid) if not fvs: return false_ret @@ -82,7 +79,6 @@ def _access_function(): failure_message="Fine Grained ECMP route not found") return result - def validate_asic_nhg_regular_ecmp(asic_db, ipprefix): def _access_function(): false_ret = (False, '') @@ -102,7 +98,7 @@ def _access_function(): return false_ret nhgid = fvs.get("SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID") - fvs = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP", nhgid) + fvs = asic_db.get_entry(ASIC_NHG, nhgid) if not fvs: return false_ret @@ -113,7 +109,6 @@ def _access_function(): _, result = wait_for_result(_access_function, failure_message="SAI_NEXT_HOP_GROUP_TYPE_DYNAMIC_UNORDERED_ECMP not found") return result - def get_nh_oid_map(asic_db): nh_oid_map = {} keys = asic_db.get_keys(ASIC_NH_TB) @@ -125,7 +120,6 @@ def get_nh_oid_map(asic_db): assert nh_oid_map != {} return nh_oid_map - def verify_programmed_fg_asic_db_entry(asic_db,nh_memb_exp_count,nh_oid_map,nhgid,bucket_size): def _access_function(): false_ret = (False, None) @@ -143,15 +137,19 @@ def _access_function(): return false_ret index = -1 nh_oid = "0" + memb_nhgid = "0" for key, val in fvs.items(): if key == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_INDEX": index = int(val) elif key == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID": nh_oid = val elif key == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID": - if nhgid != val: - print("Expected nhgid of " + nhgid + " but found " + val) - return false_ret + memb_nhgid = val + if memb_nhgid == "0": + print("memb_nhgid was not set") + return false_ret + if memb_nhgid != nhgid: + continue if (index == -1 or nh_oid == "0" or nh_oid_map.get(nh_oid,"NULL") == "NULL" or @@ -179,15 +177,12 @@ def _access_function(): status, result = wait_for_result(_access_function) assert status, f"Exact match not found: expected={nh_memb_exp_count}, received={result}" - return result - def shutdown_link(dvs, db, port): dvs.servers[port].runcmd("ip link set down dev eth0") == 0 db.wait_for_field_match("PORT_TABLE", "Ethernet%d" % (port * 4), {"oper_status": "down"}) - def startup_link(dvs, db, port): dvs.servers[port].runcmd("ip link set up dev eth0") == 0 db.wait_for_field_match("PORT_TABLE", "Ethernet%d" % (port * 4), {"oper_status": "up"}) @@ -203,14 +198,13 @@ def run_warm_reboot(dvs): dvs.runcmd(['sh', '-c', 'supervisorctl start neighsyncd']) dvs.runcmd(['sh', '-c', 'supervisorctl start restore_neighbors']) - # Enabling some extra logging for validating the order of orchagent - dvs.runcmd("swssloglevel -l INFO -c orchagent") - -def verify_programmed_fg_state_db_entry(state_db,nh_memb_exp_count): +def verify_programmed_fg_state_db_entry(state_db, fg_nhg_prefix, nh_memb_exp_count): memb_dict = nh_memb_exp_count keys = state_db.get_keys("FG_ROUTE_TABLE") - assert len(keys) != 0 + assert len(keys) != 0 for key in keys: + if key != fg_nhg_prefix: + continue fvs = state_db.get_entry("FG_ROUTE_TABLE", key) assert fvs != {} for key, value in fvs.items(): @@ -220,17 +214,15 @@ def verify_programmed_fg_state_db_entry(state_db,nh_memb_exp_count): for idx,memb in memb_dict.items(): assert memb == 0 - def validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size): + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size): state_db_entry_memb_exp_count = {} for ip, cnt in nh_memb_exp_count.items(): state_db_entry_memb_exp_count[ip + '@' + ip_to_if_map[ip]] = cnt verify_programmed_fg_asic_db_entry(asic_db,nh_memb_exp_count,nh_oid_map,nhgid,bucket_size) - verify_programmed_fg_state_db_entry(state_db, state_db_entry_memb_exp_count) - + verify_programmed_fg_state_db_entry(state_db, fg_nhg_prefix, state_db_entry_memb_exp_count) def program_route_and_validate_fine_grained_ecmp(app_db, asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size): @@ -248,265 +240,300 @@ def program_route_and_validate_fine_grained_ecmp(app_db, asic_db, state_db, ip_t fvs = swsscommon.FieldValuePairs([("nexthop", ips), ("ifname", ifs)]) ps.set(fg_nhg_prefix, fvs) validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - -def fgnhg_clean_up(config_db, asic_db, app_db, state_db, fg_nhg_name, fg_nhg_prefix, active_nhs): - # remove fgnhg prefix: The fine grained route should transition to regular ECMP/route - remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix) - - # Validate regular ECMP - validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix) - asic_db.wait_for_n_keys(ASIC_NHG_MEMB, active_nhs) - state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0) - - # clean up route entry - asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix) - ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB) - ps._del(fg_nhg_prefix) - asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key) - asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0) - - # Cleanup all FG, arp and interface - remove_entry(config_db, "FG_NHG", fg_nhg_name) - -class TestFineGrainedNextHopGroup(object): - def test_route_fgnhg(self, dvs, testlog): - app_db = dvs.get_app_db() - asic_db = dvs.get_asic_db() - config_db = dvs.get_config_db() - state_db = dvs.get_state_db() - fvs_nul = {"NULL": "NULL"} - NUM_NHs = 6 - fg_nhg_name = "fgnhg_v4" - fg_nhg_prefix = "2.2.2.0/24" - bucket_size = 60 - ip_to_if_map = {} - - fvs = {"bucket_size": str(bucket_size)} - create_entry(config_db, FG_NHG, fg_nhg_name, fvs) - + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + +def create_interface_n_fg_ecmp_config(dvs, nh_range_start, nh_range_end, fg_nhg_name): + ip_to_if_map = {} + app_db = dvs.get_app_db() + config_db = dvs.get_config_db() + fvs_nul = {"NULL": "NULL"} + for i in range(nh_range_start, nh_range_end): + if_name_key = "Ethernet" + str(i*4) + ip_pref_key = "Ethernet" + str(i*4) + "|10.0.0." + str(i*2) + "/31" + create_entry(config_db, IF_TB, if_name_key, fvs_nul) + create_entry(config_db, IF_TB, ip_pref_key, fvs_nul) + dvs.runcmd("config interface startup " + if_name_key) + shutdown_link(dvs, app_db, i) + startup_link(dvs, app_db, i) + bank = 1 + if i >= (nh_range_end - nh_range_start)/2: + bank = 0 + fvs = {"FG_NHG": fg_nhg_name, "bank": str(bank)} + create_entry(config_db, FG_NHG_MEMBER, "10.0.0." + str(1 + i*2), fvs) + ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key + dvs.runcmd("arp -s 10.0.0." + str(1 + i*2) + " 00:00:00:00:00:" + str(1 + i*2)) + return ip_to_if_map + +def remove_interface_n_fg_ecmp_config(dvs, nh_range_start, nh_range_end, fg_nhg_name): + app_db = dvs.get_app_db() + config_db = dvs.get_config_db() + for i in range(nh_range_start, nh_range_end): + if_name_key = "Ethernet" + str(i*4) + ip_pref_key = "Ethernet" + str(i*4) + "|10.0.0." + str(i*2) + "/31" + remove_entry(config_db, IF_TB, if_name_key) + remove_entry(config_db, IF_TB, ip_pref_key) + dvs.runcmd("config interface shutdown " + if_name_key) + shutdown_link(dvs, app_db, i) + remove_entry(config_db, FG_NHG_MEMBER, "10.0.0." + str(1 + i*2)) + remove_entry(config_db, FG_NHG, fg_nhg_name) + +def fine_grained_ecmp_base_test(dvs, match_mode): + app_db = dvs.get_app_db() + asic_db = dvs.get_asic_db() + config_db = dvs.get_config_db() + state_db = dvs.get_state_db() + fvs_nul = {"NULL": "NULL"} + NUM_NHs = 6 + fg_nhg_name = "fgnhg_v4" + fg_nhg_prefix = "2.2.2.0/24" + bucket_size = 60 + ip_to_if_map = {} + + fvs = {"bucket_size": str(bucket_size), "match_mode": match_mode} + create_entry(config_db, FG_NHG, fg_nhg_name, fvs) + + if match_mode == 'route-based': fvs = {"FG_NHG": fg_nhg_name} create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs) - for i in range(0,NUM_NHs): - if_name_key = "Ethernet" + str(i*4) - vlan_name_key = "Vlan" + str((i+1)*4) - ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31" - fvs = {"vlanid": str((i+1)*4)} - create_entry(config_db, VLAN_TB, vlan_name_key, fvs) - fvs = {"tagging_mode": "untagged"} - create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs) - create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul) - create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul) - dvs.runcmd("config interface startup " + if_name_key) - dvs.servers[i].runcmd("ip link set down dev eth0") == 0 - dvs.servers[i].runcmd("ip link set up dev eth0") == 0 - bank = 0 - if i >= NUM_NHs/2: - bank = 1 - fvs = {"FG_NHG": fg_nhg_name, "bank": str(bank), "link": if_name_key} - create_entry(config_db, FG_NHG_MEMBER, "10.0.0." + str(1 + i*2), fvs) - ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key - - # Wait for the software to receive the entries - time.sleep(1) - - ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB) - fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"), - ("ifname", "Vlan16,Vlan20,Vlan24")]) - ps.set(fg_nhg_prefix, fvs) - # No ASIC_DB entry we can wait for since ARP is not resolved yet, - # We just use sleep so that the sw receives this entry - time.sleep(1) + for i in range(0,NUM_NHs): + if_name_key = "Ethernet" + str(i*4) + vlan_name_key = "Vlan" + str((i+1)*4) + ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31" + fvs = {"vlanid": str((i+1)*4)} + create_entry(config_db, VLAN_TB, vlan_name_key, fvs) + fvs = {"tagging_mode": "untagged"} + create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs) + create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul) + create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul) + dvs.runcmd("config interface startup " + if_name_key) + dvs.servers[i].runcmd("ip link set down dev eth0") == 0 + dvs.servers[i].runcmd("ip link set up dev eth0") == 0 + bank = 0 + if i >= NUM_NHs/2: + bank = 1 + fvs = {"FG_NHG": fg_nhg_name, "bank": str(bank), "link": if_name_key} + create_entry(config_db, FG_NHG_MEMBER, "10.0.0." + str(1 + i*2), fvs) + ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key - adb = swsscommon.DBConnector(1, dvs.redis_sock, 0) - rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB) - keys = rtbl.getKeys() - found_route = False - for k in keys: - rt_key = json.loads(k) + # Wait for the software to receive the entries + time.sleep(1) - if rt_key['dest'] == fg_nhg_prefix: - found_route = True - break + ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB) + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"), + ("ifname", "Vlan16,Vlan20,Vlan24")]) + ps.set(fg_nhg_prefix, fvs) + # No ASIC_DB entry we can wait for since ARP is not resolved yet, + # We just use sleep so that the sw receives this entry + time.sleep(1) + + adb = swsscommon.DBConnector(1, dvs.redis_sock, 0) + rtbl = swsscommon.Table(adb, ASIC_ROUTE_TB) + keys = rtbl.getKeys() + found_route = False + for k in keys: + rt_key = json.loads(k) - # Since we didn't populate ARP yet, the route shouldn't be programmed - assert (found_route == False) + if rt_key['dest'] == fg_nhg_prefix: + found_route = True + break - asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB)) - dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01") - dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02") - dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03") - dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05") - dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06") - asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 5) + # Since we didn't populate ARP yet, the route shouldn't be programmed + assert (found_route == False) - asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size) - nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size) + asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB)) + dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01") + dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02") + dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03") + dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05") + dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06") + asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 5) - nh_oid_map = get_nh_oid_map(asic_db) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size) + nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size) - ### Test scenarios with bank 0 having 0 members up and only bank 1 having members - # ARP is not resolved for 10.0.0.7, so fg nhg should be created without 10.0.0.7 - nh_memb_exp_count = {"10.0.0.9":30,"10.0.0.11":30} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + nh_oid_map = get_nh_oid_map(asic_db) - # Resolve ARP for 10.0.0.7 - asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB)) - dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04") - asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 1) - nh_oid_map = get_nh_oid_map(asic_db) + ### Test scenarios with bank 0 having 0 members up and only bank 1 having members + # ARP is not resolved for 10.0.0.7, so fg nhg should be created without 10.0.0.7 + nh_memb_exp_count = {"10.0.0.9":30,"10.0.0.11":30} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Now that ARP was resolved, 10.0.0.7 should be added as a valid fg nhg member - nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + # Resolve ARP for 10.0.0.7 + asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB)) + dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04") + asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 1) + nh_oid_map = get_nh_oid_map(asic_db) - # Bring down 1 next hop in bank 1 - nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + # Now that ARP was resolved, 10.0.0.7 should be added as a valid fg nhg member + nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Bring down 2 next hop and bring up 1 next hop in bank 1 - nh_memb_exp_count = {"10.0.0.9":60} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + # Test warm reboot + run_warm_reboot(dvs) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size) + nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size) + nh_oid_map = get_nh_oid_map(asic_db) + nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Bring up 1 next hop in bank 1 - nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + # Bring down 1 next hop in bank 1 + nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Bring down 2 next hop and bring up 1 next hop in bank 1 + nh_memb_exp_count = {"10.0.0.9":60} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Bring up 1 next hop in bank 1 + nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Bring up some next-hops in bank 0 for the 1st time + nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Test warm reboot + run_warm_reboot(dvs) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size) + nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size) + nh_oid_map = get_nh_oid_map(asic_db) + nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Bring up some next-hops in bank 0 for the 1st time - nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + # Bring down 1 next-hop from bank 0, and 2 next-hops from bank 1 + nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.11":30} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Bring down 1 member and bring up 1 member in bank 0 at the same time + nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.3":15,"10.0.0.11":30} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Bring down 2 members and bring up 1 member in bank 0 at the same time + nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Bring up 2 members and bring down 1 member in bank 0 at the same time + nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.3":15,"10.0.0.11":30} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Bringup arbitrary # of next-hops from both banks at the same time + nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Bring all next-hops in bank 1 down + nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.5":20} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Make next-hop changes to bank 0 members, given bank 1 is still down + nh_memb_exp_count = {"10.0.0.1":30,"10.0.0.5":30} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Bringup 1 member in bank 1 again + nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.11":30} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Test 2nd,3rd memb up in bank + nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} + program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # bring all links down one by one + shutdown_link(dvs, app_db, 0) + shutdown_link(dvs, app_db, 1) + nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Bring down 1 next-hop from bank 0, and 2 next-hops from bank 1 - nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.11":30} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + shutdown_link(dvs, app_db, 2) + nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Bring down 1 member and bring up 1 member in bank 0 at the same time - nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.3":15,"10.0.0.11":30} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + shutdown_link(dvs, app_db, 3) + nh_memb_exp_count = {"10.0.0.9":30,"10.0.0.11":30} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Bring down 2 members and bring up 1 member in bank 0 at the same time - nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.11":30} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + shutdown_link(dvs, app_db, 4) + nh_memb_exp_count = {"10.0.0.11":60} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Bring up 2 members and bring down 1 member in bank 0 at the same time - nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.3":15,"10.0.0.11":30} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, - fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + # Bring down last link, there shouldn't be a crash or other bad orchagent state because of this + shutdown_link(dvs, app_db, 5) + # Nothing to check for in this case, sleep 1s for the shutdown to reach sw + time.sleep(1) - # Bringup arbitrary # of next-hops from both banks at the same time - nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + # bring all links up one by one + startup_link(dvs, app_db, 3) + startup_link(dvs, app_db, 4) + nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.9":30} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Bring all next-hops in bank 1 down - nh_memb_exp_count = {"10.0.0.1":20,"10.0.0.3":20,"10.0.0.5":20} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, - fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + startup_link(dvs, app_db, 5) + # Perform a route table update, Update the route to contain 10.0.0.3 as well, since Ethernet4 associated with it + # is link down, it should make no difference + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.3,10.0.0.5,10.0.0.7,10.0.0.9,10.0.0.11"), + ("ifname","Vlan4,Vlan8,Vlan12,Vlan16,Vlan20,Vlan24")]) + ps.set(fg_nhg_prefix, fvs) - # Make next-hop changes to bank 0 members, given bank 1 is still down - nh_memb_exp_count = {"10.0.0.1":30,"10.0.0.5":30} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + # 10.0.0.11 associated with newly brought up link 5 should be updated in FG ecmp + # 10.0.0.3 addition per above route table change should have no effect + nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Bringup 1 member in bank 1 again - nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.11":30} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + startup_link(dvs, app_db, 2) + nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Test 2nd,3rd memb up in bank - nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, + startup_link(dvs, app_db, 0) + nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # bring all links down one by one - shutdown_link(dvs, app_db, 0) - shutdown_link(dvs, app_db, 1) - nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - shutdown_link(dvs, app_db, 2) - nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - shutdown_link(dvs, app_db, 3) - nh_memb_exp_count = {"10.0.0.9":30,"10.0.0.11":30} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - shutdown_link(dvs, app_db, 4) - nh_memb_exp_count = {"10.0.0.11":60} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - # Bring down last link, there shouldn't be a crash or other bad orchagent state because of this - shutdown_link(dvs, app_db, 5) - # Nothing to check for in this case, sleep 1s for the shutdown to reach sw - time.sleep(1) - - # bring all links up one by one - startup_link(dvs, app_db, 3) - startup_link(dvs, app_db, 4) - nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.9":30} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - startup_link(dvs, app_db, 5) - # Perform a route table update, Update the route to contain 10.0.0.3 as well, since Ethernet4 associated with it - # is link down, it should make no difference - fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.3,10.0.0.5,10.0.0.7,10.0.0.9,10.0.0.11"), - ("ifname","Vlan4,Vlan8,Vlan12,Vlan16,Vlan20,Vlan24")]) - ps.set(fg_nhg_prefix, fvs) - - # 10.0.0.11 associated with newly brought up link 5 should be updated in FG ecmp - # 10.0.0.3 addition per above route table change should have no effect - nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - startup_link(dvs, app_db, 2) - nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - startup_link(dvs, app_db, 0) - nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - # remove fgnhg member - remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0.1") - nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + # remove fgnhg member + remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0.1") + nh_memb_exp_count = {"10.0.0.5":30,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # add fgnhg member - fvs = {"FG_NHG": fg_nhg_name, "bank": "0"} - create_entry(config_db, FG_NHG_MEMBER, "10.0.0.1", fvs) - nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} - validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + # add fgnhg member + fvs = {"FG_NHG": fg_nhg_name, "bank": "0"} + create_entry(config_db, FG_NHG_MEMBER, "10.0.0.1", fvs) + nh_memb_exp_count = {"10.0.0.1":15,"10.0.0.5":15,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - # Remove route - asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix) - ps._del(fg_nhg_prefix) + # Remove route + asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix) + ps._del(fg_nhg_prefix) - # validate routes and nhg member in asic db, route entry in state db are removed - asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key) - asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0) - state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0) + # validate routes and nhg member in asic db, route entry in state db are removed + asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0) + state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0) + if match_mode == 'route-based': remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix) # Nothing we can wait for in terms of db entries, we sleep here # to give the sw enough time to delete the entry @@ -533,33 +560,68 @@ def test_route_fgnhg(self, dvs, testlog): nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - fgnhg_clean_up(config_db, asic_db, app_db, state_db, fg_nhg_name, fg_nhg_prefix, 3) - - # clean up nh entries - for i in range(0,NUM_NHs): - if_name_key = "Ethernet" + str(i*4) - vlan_name_key = "Vlan" + str((i+1)*4) - ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31" - remove_entry(config_db, VLAN_IF_TB, ip_pref_key) - remove_entry(config_db, VLAN_IF_TB, vlan_name_key) - remove_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key) - remove_entry(config_db, VLAN_TB, vlan_name_key) - dvs.runcmd("config interface shutdown " + if_name_key) - dvs.servers[i].runcmd("ip link set down dev eth0") == 0 - remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2)) - - ### Create new set of entries with a greater number of FG members and - ### bigger bucket size such that the # of nhs are not divisible by - ### bucket size. Different physical interface type for dynamicitiy. - fg_nhg_name = "new_fgnhg_v4" + # remove fgnhg prefix: The fine grained route should transition to regular ECMP/route + remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix) + + # Validate regular ECMP + validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3) + state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0) + + # remove prefix entry + asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix) + ps._del(fg_nhg_prefix) + asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0) + + # Cleanup all FG, arp and interface + remove_entry(config_db, "FG_NHG", fg_nhg_name) + + for i in range(0,NUM_NHs): + if_name_key = "Ethernet" + str(i*4) + vlan_name_key = "Vlan" + str((i+1)*4) + ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31" + remove_entry(config_db, VLAN_IF_TB, ip_pref_key) + remove_entry(config_db, VLAN_IF_TB, vlan_name_key) + remove_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key) + remove_entry(config_db, VLAN_TB, vlan_name_key) + dvs.runcmd("config interface shutdown " + if_name_key) + dvs.servers[i].runcmd("ip link set down dev eth0") == 0 + remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2)) + + +class TestFineGrainedNextHopGroup(object): + def test_fgnhg_matchmode_route(self, dvs, testlog): + ''' + Test for match_mode route-based + ''' + fine_grained_ecmp_base_test(dvs, 'route-based') + + def test_fgnhg_matchmode_nexthop(self, dvs, testlog): + ''' + Test for match_mode nexthop-based + ''' + fine_grained_ecmp_base_test(dvs, 'nexthop-based') + + def test_fgnhg_more_nhs_nondiv_bucket_size(self, dvs, testlog): + ''' + Test Fine Grained ECMP with a greater number of FG members and + bigger bucket size, such that the no. of nhs are not divisible by + bucket size. Use a different physical interface type for dynamicitiy. + ''' + app_db = dvs.get_app_db() + asic_db = dvs.get_asic_db() + config_db = dvs.get_config_db() + state_db = dvs.get_state_db() + + fg_nhg_name = "fgnhg_v4" fg_nhg_prefix = "3.3.3.0/24" # Test with non-divisible bucket size bucket_size = 128 NUM_NHs = 10 - ip_to_if_map = {} nh_oid_map = {} # Initialize base config @@ -570,25 +632,11 @@ def test_route_fgnhg(self, dvs, testlog): create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs) asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB)) - for i in range(0,NUM_NHs): - if_name_key = "Ethernet" + str(i*4) - ip_pref_key = "Ethernet" + str(i*4) + "|10.0.0." + str(i*2) + "/31" - create_entry(config_db, IF_TB, if_name_key, fvs_nul) - create_entry(config_db, IF_TB, ip_pref_key, fvs_nul) - dvs.runcmd("config interface startup " + if_name_key) - shutdown_link(dvs, app_db, i) - startup_link(dvs, app_db, i) - bank = 1 - if i >= NUM_NHs/2: - bank = 0 - fvs = {"FG_NHG": fg_nhg_name, "bank": str(bank)} - create_entry(config_db, FG_NHG_MEMBER, "10.0.0." + str(1 + i*2), fvs) - ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key - dvs.runcmd("arp -s 10.0.0." + str(1 + i*2) + " 00:00:00:00:00:" + str(1 + i*2)) - + ip_to_if_map = create_interface_n_fg_ecmp_config(dvs, 0, NUM_NHs, fg_nhg_name) asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + NUM_NHs) # Program the route + ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB) fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.11"), ("ifname", "Ethernet0,Ethernet20")]) ps.set(fg_nhg_prefix, fvs) @@ -599,10 +647,10 @@ def test_route_fgnhg(self, dvs, testlog): nh_oid_map = get_nh_oid_map(asic_db) - # Test addition of route with 0 members in bank + # The route had been created with 0 members in bank nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.11":64} validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) # Add 2 nhs to both bank 0 and bank 1 nh_memb_exp_count = {"10.0.0.1":22,"10.0.0.3":21,"10.0.0.5":21,"10.0.0.11":22, @@ -643,126 +691,147 @@ def test_route_fgnhg(self, dvs, testlog): "10.0.0.9":13,"10.0.0.11":64} program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - fgnhg_clean_up(config_db, asic_db, app_db, state_db, fg_nhg_name, fg_nhg_prefix, 6) - # clean up nh entries - for i in range(0,NUM_NHs): - if_name_key = "Ethernet" + str(i*4) - ip_pref_key = "Ethernet" + str(i*4) + "|10.0.0." + str(i*2) + "/31" - remove_entry(config_db, IF_TB, ip_pref_key) - remove_entry(config_db, IF_TB, vlan_name_key) - dvs.runcmd("config interface shutdown " + if_name_key) - dvs.servers[i].runcmd("ip link set down dev eth0") == 0 - remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2)) - - def test_route_fgnhg_warm_reboot(self, dvs, testlog): - dvs.runcmd("swssloglevel -l INFO -c orchagent") + + # Remove route + # remove prefix entry + asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix) + ps._del(fg_nhg_prefix) + asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0) + asic_db.wait_for_n_keys(ASIC_NHG, 0) + + # cleanup all config + remove_interface_n_fg_ecmp_config(dvs, 0, NUM_NHs, fg_nhg_name) + remove_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix) + + def test_fgnhg_matchmode_nexthop_multi_route(self, dvs, testlog): + ''' + Test route/nh transitions to/from Fine Grained ECMP and Regular ECMP. + Create multiple prefixes pointing to the Fine Grained nhs and ensure + fine grained ECMP ASIC objects were created for this scenario as expected. + ''' app_db = dvs.get_app_db() asic_db = dvs.get_asic_db() config_db = dvs.get_config_db() state_db = dvs.get_state_db() fvs_nul = {"NULL": "NULL"} - NUM_NHs = 6 + fg_nhg_name = "fgnhg_v4" - fg_nhg_prefix = "2.2.2.0/24" - bucket_size = 60 - ip_to_if_map = {} + fg_nhg_prefix = "3.3.3.0/24" + # Test with non-divisible bucket size + bucket_size = 128 + NUM_NHs = 4 + NUM_NHs_non_fgnhg = 2 - fvs = {"bucket_size": str(bucket_size)} + nh_oid_map = {} + + # Initialize base config + fvs = {"bucket_size": str(bucket_size), "match_mode": "nexthop-based"} create_entry(config_db, FG_NHG, fg_nhg_name, fvs) - fvs = {"FG_NHG": fg_nhg_name} - create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs) + asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB)) + ip_to_if_map = create_interface_n_fg_ecmp_config(dvs, 0, NUM_NHs, fg_nhg_name) - for i in range(0,NUM_NHs): + # Create 2 more interface + IPs for non-fine grained ECMP validation + for i in range(NUM_NHs, NUM_NHs + NUM_NHs_non_fgnhg): if_name_key = "Ethernet" + str(i*4) - vlan_name_key = "Vlan" + str((i+1)*4) - ip_pref_key = vlan_name_key + "|10.0.0." + str(i*2) + "/31" - fvs = {"vlanid": str((i+1)*4)} - create_entry(config_db, VLAN_TB, vlan_name_key, fvs) - fvs = {"tagging_mode": "untagged"} - create_entry(config_db, VLAN_MEMB_TB, vlan_name_key + "|" + if_name_key, fvs) - create_entry(config_db, VLAN_IF_TB, vlan_name_key, fvs_nul) - create_entry(config_db, VLAN_IF_TB, ip_pref_key, fvs_nul) + ip_pref_key = "Ethernet" + str(i*4) + "|10.0.0." + str(i*2) + "/31" + create_entry(config_db, IF_TB, if_name_key, fvs_nul) + create_entry(config_db, IF_TB, ip_pref_key, fvs_nul) dvs.runcmd("config interface startup " + if_name_key) - dvs.servers[i].runcmd("ip link set down dev eth0") == 0 - dvs.servers[i].runcmd("ip link set up dev eth0") == 0 - bank = 0 - if i >= NUM_NHs/2: - bank = 1 - fvs = {"FG_NHG": fg_nhg_name, "bank": str(bank), "link": if_name_key} - create_entry(config_db, FG_NHG_MEMBER, "10.0.0." + str(1 + i*2), fvs) - ip_to_if_map["10.0.0." + str(1 + i*2)] = vlan_name_key + shutdown_link(dvs, app_db, i) + startup_link(dvs, app_db, i) + dvs.runcmd("arp -s 10.0.0." + str(1 + i*2) + " 00:00:00:00:00:" + str(1 + i*2)) - # Wait for the software to receive the entries - time.sleep(1) + asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + NUM_NHs + NUM_NHs_non_fgnhg) + # Program the route ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB) - fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"), - ("ifname", "Vlan16,Vlan20,Vlan24")]) + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.5"), + ("ifname", "Ethernet0,Ethernet8")]) ps.set(fg_nhg_prefix, fvs) - # No ASIC_DB entry we can wait for since ARP is not resolved yet, - # We just use sleep so that the sw receives this entry - time.sleep(1) - - asic_nh_count = len(asic_db.get_keys(ASIC_NH_TB)) - dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01") - dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02") - dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03") - dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04") - dvs.runcmd("arp -s 10.0.0.9 00:00:00:00:00:05") - dvs.runcmd("arp -s 10.0.0.11 00:00:00:00:00:06") - asic_db.wait_for_n_keys(ASIC_NH_TB, asic_nh_count + 6) + # Validate that the correct ASIC DB elements were setup per Fine Grained ECMP asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size) nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size) nh_oid_map = get_nh_oid_map(asic_db) - # Now that ARP was resolved, 10.0.0.7 should be added as a valid fg nhg member - nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} + # The route had been created with 0 members in bank + nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64} validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - - run_warm_reboot(dvs) - - asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size) - nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size) - - nh_oid_map = get_nh_oid_map(asic_db) - - nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} + fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + + # Add a 2nd prefix associated with the same set of next-hops + fg_nhg_prefix_2 = "5.5.5.0/16" + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.5"), + ("ifname", "Ethernet0,Ethernet8")]) + ps.set(fg_nhg_prefix_2, fvs) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size*2) + nhgid_2 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix_2, bucket_size) + nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64} validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, - nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - # Bring down 1 next hop in bank 1 - nh_memb_exp_count = {"10.0.0.7":30,"10.0.0.11":30} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, - fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - # Bring down 2 next hop and bring up 1 next hop in bank 1 - nh_memb_exp_count = {"10.0.0.9":60} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, - fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) - - # Bring up 1 next hop in bank 1 - nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, - fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + fg_nhg_prefix_2, nh_memb_exp_count, nh_oid_map, nhgid_2, bucket_size) + + # Add a 3rd prefix with a next-hop(10.0.0.9) not defined for FG ECMP + # Should end up as regular ECMP + fg_nhg_prefix_3 = "6.6.6.0/16" + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.5,10.0.0.9"), + ("ifname", "Ethernet0,Ethernet8,Ethernet16")]) + ps.set(fg_nhg_prefix_3, fvs) + validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix_3) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size*2 + 3) + # Remove the 10.0.0.9 next-hop, it should now transition to Fine Grained ECMP + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.5"), + ("ifname", "Ethernet0,Ethernet8")]) + ps.set(fg_nhg_prefix_3, fvs) + nhgid_3 = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix_3, bucket_size) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size*3) + nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, + fg_nhg_prefix_3, nh_memb_exp_count, nh_oid_map, nhgid_3, bucket_size) + # Add the 10.0.0.9 next-hop again, it should transition back to regular ECMP + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.5,10.0.0.9"), + ("ifname", "Ethernet0,Ethernet8,Ethernet16")]) + ps.set(fg_nhg_prefix_3, fvs) + validate_asic_nhg_regular_ecmp(asic_db, fg_nhg_prefix_3) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size*2 + 3) + # Delete the prefix + asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix_3) + ps._del(fg_nhg_prefix_3) + asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size*2) - # Bring up some next-hops in bank 0 for the 1st time - nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} + # Change FG nhs for one route, ensure that the other route nh is unaffected + nh_memb_exp_count = {"10.0.0.1":32,"10.0.0.3":32,"10.0.0.5":32,"10.0.0.7":32} program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) + nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, + fg_nhg_prefix_2, nh_memb_exp_count, nh_oid_map, nhgid_2, bucket_size) - run_warm_reboot(dvs) - + # Remove route + # remove prefix entry + asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix) + ps._del(fg_nhg_prefix) + asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key) asic_db.wait_for_n_keys(ASIC_NHG_MEMB, bucket_size) - nhgid = validate_asic_nhg_fine_grained_ecmp(asic_db, fg_nhg_prefix, bucket_size) + # Ensure that 2nd route is still here and then delete it + nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.5":64} + validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, + fg_nhg_prefix_2, nh_memb_exp_count, nh_oid_map, nhgid_2, bucket_size) + # Delete the 2nd route as well + asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix_2) + ps._del(fg_nhg_prefix_2) + asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key) + asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0) + asic_db.wait_for_n_keys(ASIC_NHG, 0) - nh_oid_map = get_nh_oid_map(asic_db) + # cleanup all entries + remove_interface_n_fg_ecmp_config(dvs, 0, NUM_NHs+NUM_NHs_non_fgnhg, fg_nhg_name) - nh_memb_exp_count = {"10.0.0.1":10,"10.0.0.3":10,"10.0.0.5":10,"10.0.0.7":10,"10.0.0.9":10,"10.0.0.11":10} - program_route_and_validate_fine_grained_ecmp(app_db.db_connection, asic_db, state_db, ip_to_if_map, - fg_nhg_prefix, nh_memb_exp_count, nh_oid_map, nhgid, bucket_size) \ No newline at end of file + +# Add Dummy always-pass test at end as workaroud +# for issue when Flaky fail on final test it invokes module tear-down before retrying +def test_nonflaky_dummy(): + pass