Support multiple protocols in fibagent

Summary:
After discussing with Saif Hasan, we found an easier way to do it. In linux, it used priority to determine which route to choose when there are multiple routes. And priority is the admin distance in some discussion or implementation online. (https://fburl.com/i4jibk4k, https://fburl.com/lpxi6b1i, https://fburl.com/13ftf6q4) here are some references I found online.

So, all we need to do is reflecting admin distance as priority and linux will choose the route accordingly.

Reviewed By: saifhhasan

Differential Revision: D15446677

fbshipit-source-id: c3eb10c6ac62c1717d6a8351a81f0acd6e3b5791

openr/if/Platform.thrift

@@ -133,15 +133,16 @@ service SystemService {
     throws (1: PlatformError error)
 }
 
-// static mapping of clientId => protocolId, priority
+// static mapping of clientId => protocolId, priority same of admin distance
 // For Open/R
 //    ClientId: 786 => ProtocolId: 99, Priority: 10
 // For BGP
 //    ClientId: 0 => ProtocolId: 253, Priority: 20
 // For TG breeze CLI client
 //    ClientId: 64 => ProtocolId: 64, Priority: 11
 const map<i16, i16> clientIdtoProtocolId = {786:99, 0:253, 64:64}
-const map<i16, i16> clientIdtoPriority = {786:10, 0:20, 64:11}
+const map<i16, i16> protocolIdtoPriority = {99:10, 253:20, 64:11}
+const i16 kUnknowProtAdminDistance = 255
 
 /**
  * Interface to on-box Fib.

openr/nl/NetlinkRoute.cpp

@@ -403,6 +403,17 @@ NetlinkRouteMessage::addRoute(const openr::fbnl::Route& route) {
     return status;
   };
 
+  // set up admin distance if priority.
+  if (route.getPriority()) {
+    const uint32_t adminDistance = route.getPriority().value();
+    const char* const adPtr = reinterpret_cast<const char*>(&adminDistance);
+    if ((status =
+             addAttributes(RTA_PRIORITY, adPtr, sizeof(uint32_t), msghdr_)) !=
+        ResultCode::SUCCESS) {
+      return status;
+    };
+  }
+
   return addNextHops(route);
 }
 

openr/nl/NetlinkSocket.cpp

@@ -6,6 +6,9 @@
  */
 
 #include <openr/nl/NetlinkSocket.h>
+#include <openr/if/gen-cpp2/Platform_constants.h>
+
+using pc = openr::thrift::Platform_constants;
 
 namespace {
 const folly::StringPiece kRouteObjectStr("route/route");
@@ -570,6 +573,19 @@ NetlinkSocket::doAddUpdateUnicastRoute(Route route) {
   // Create new set of nexthops to be programmed. Existing + New ones
   auto& unicastRoutes = unicastRoutesCache_[route.getProtocolId()];
   auto iter = unicastRoutes.find(dest);
+  // if user did not speicify priority
+  if (!route.getPriority()) {
+    const auto routePair =
+        openr::thrift::Platform_constants::protocolIdtoPriority().find(
+            route.getProtocolId());
+    if (routePair ==
+        openr::thrift::Platform_constants::protocolIdtoPriority().end()) {
+      route.setPriority(
+          openr::thrift::Platform_constants::kUnknowProtAdminDistance());
+    } else {
+      route.setPriority(routePair->second);
+    }
+  }
   // Same route
   if (iter != unicastRoutes.end() && iter->second == route) {
     return;

openr/nl/NetlinkTypes.cpp

@@ -41,6 +41,7 @@ RouteBuilder::loadFromObject(struct rtnl_route* obj) {
   setFlags(rtnl_route_get_flags(obj));
   setProtocolId(rtnl_route_get_protocol(obj));
   setType(rtnl_route_get_type(obj));
+  setPriority(rtnl_route_get_priority(obj));
   struct nl_addr* dst = rtnl_route_get_dst(obj);
 
   // Special handling for default routes
@@ -680,6 +681,11 @@ Route::buildAddrObject(const folly::CIDRNetwork& addr) {
   return nlAddr_;
 }
 
+void
+Route::setPriority(uint32_t priority) {
+  priority_ = priority;
+}
+
 /*=================================NextHop====================================*/
 
 NextHop

openr/nl/NetlinkTypes.h

@@ -350,6 +350,8 @@ class Route final {
 
   folly::Optional<std::string> getRouteIfName() const;
 
+  void setPriority(uint32_t priority);
+
   std::string str() const;
 
   /**

openr/nl/tests/NetlinkSocketTest.cpp

@@ -49,6 +49,9 @@ const std::string kVethNameY("vethTestY");
 // We will use this as the proto for our routes
 const uint8_t kAqRouteProtoId = 99;
 const uint8_t kAqRouteProtoId1 = 159;
+const uint32_t kAqRouteProtoIdPriority = 10;
+const uint32_t kAqRouteProtoId1Priority = 255;
+
 } // namespace
 
 // This fixture creates virtual interface (veths)
@@ -1451,6 +1454,247 @@ TEST_P(NetlinkSocketFixture, MultiProtocolUnicastTest) {
   EXPECT_EQ(0, count);
 }
 
+TEST_P(NetlinkSocketFixture, MultiProtocolUnicastTestDecisionTest) {
+  // V6 routes for protocol 99
+  folly::CIDRNetwork prefix1V6{folly::IPAddress("fc00:cafe:3::3"), 128};
+  folly::CIDRNetwork prefix2V6{folly::IPAddress("fc00:cafe:3::4"), 128};
+  auto nh1V6 = folly::IPAddress("fe80::1");
+  auto nh2V6 = folly::IPAddress("fe80::2");
+  int ifIndexX = rtnl_link_name2i(linkCache_, kVethNameX.c_str());
+  int ifIndexY = rtnl_link_name2i(linkCache_, kVethNameY.c_str());
+  // V4 routes for protocol 159
+  folly::CIDRNetwork prefix1V4{folly::IPAddress("192.168.0.11"), 32};
+  folly::CIDRNetwork prefix2V4{folly::IPAddress("192.168.0.12"), 32};
+  auto nh1V4 = folly::IPAddress("169.254.0.1");
+  auto nh2V4 = folly::IPAddress("169.254.0.2");
+
+  std::vector<folly::IPAddress> nextHopsV6{nh1V6};
+
+  auto routeFunc = [](struct nl_object * obj, void* arg) noexcept->void {
+    RouteCallbackContext* ctx = static_cast<RouteCallbackContext*>(arg);
+    struct rtnl_route* routeObj = reinterpret_cast<struct rtnl_route*>(obj);
+    RouteBuilder builder;
+    int protocol = rtnl_route_get_protocol(routeObj);
+    if (protocol == kAqRouteProtoId || protocol == kAqRouteProtoId1) {
+      ctx->results.emplace_back(builder.buildFromObject(routeObj));
+    }
+  };
+
+  // Add routes with single nextHop for protocol 99
+  netlinkSocket
+      ->addRoute(buildRoute(ifIndexX, kAqRouteProtoId, nextHopsV6, prefix1V6))
+      .get();
+  netlinkSocket
+      ->addRoute(buildRoute(ifIndexX, kAqRouteProtoId, nextHopsV6, prefix2V6))
+      .get();
+  auto routes = netlinkSocket->getCachedUnicastRoutes(kAqRouteProtoId).get();
+  EXPECT_EQ(2, routes.size());
+  ASSERT_EQ(1, routes.count(prefix1V6));
+  ASSERT_EQ(1, routes.count(prefix2V6));
+  const Route& rt1 = routes.at(prefix1V6);
+  const Route& rt2 = routes.at(prefix2V6);
+  EXPECT_EQ(1, rt1.getNextHops().size());
+  EXPECT_EQ(1, rt2.getNextHops().size());
+  EXPECT_TRUE(CompareNextHops(nextHopsV6, rt1));
+  EXPECT_TRUE(CompareNextHops(nextHopsV6, rt2));
+
+  // Adde routes for protocol 159
+  std::vector<folly::IPAddress> nextHops1V4{nh1V4};
+  std::vector<folly::IPAddress> nextHops1V6{nh2V6};
+  netlinkSocket
+      ->addRoute(buildRoute(ifIndexY, kAqRouteProtoId1, nextHops1V6, prefix1V6))
+      .get();
+  netlinkSocket
+      ->addRoute(buildRoute(ifIndexY, kAqRouteProtoId1, nextHops1V6, prefix2V6))
+      .get();
+  routes = netlinkSocket->getCachedUnicastRoutes(kAqRouteProtoId1).get();
+  EXPECT_EQ(2, routes.size());
+  ASSERT_EQ(1, routes.count(prefix1V6));
+  ASSERT_EQ(1, routes.count(prefix2V6));
+  const Route& rt3 = routes.at(prefix1V6);
+  const Route& rt4 = routes.at(prefix2V6);
+  EXPECT_EQ(1, rt3.getNextHops().size());
+  EXPECT_EQ(1, rt4.getNextHops().size());
+  EXPECT_TRUE(CompareNextHops(nextHops1V6, rt3));
+  EXPECT_TRUE(CompareNextHops(nextHops1V6, rt4));
+
+  // Check kernel kAqRouteProtoId should be selected
+  RouteCallbackContext ctx;
+  rtnlCacheCB(routeFunc, &ctx, routeCache_);
+  int count = 0;
+  int totalRoute = 0;
+  for (const auto& r : ctx.results) {
+    if (r.getDestination() == prefix1V6 &&
+        r.getProtocolId() == kAqRouteProtoId && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoIdPriority) {
+      count++;
+    }
+    if (r.getDestination() == prefix2V6 &&
+        r.getProtocolId() == kAqRouteProtoId && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoIdPriority) {
+      count++;
+    }
+    if (r.getDestination() == prefix1V6 &&
+        r.getProtocolId() == kAqRouteProtoId1 && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoId1Priority) {
+      count++;
+    }
+    if (r.getDestination() == prefix2V6 &&
+        r.getProtocolId() == kAqRouteProtoId1 && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoId1Priority) {
+      count++;
+    }
+    totalRoute++;
+  }
+  EXPECT_EQ(4, count);
+  EXPECT_EQ(4, totalRoute);
+
+  // Add same route again, should not affect result
+  netlinkSocket
+      ->addRoute(buildRoute(ifIndexX, kAqRouteProtoId, nextHopsV6, prefix1V6))
+      .get();
+
+  netlinkSocket
+      ->addRoute(buildRoute(ifIndexX, kAqRouteProtoId, nextHopsV6, prefix2V6))
+      .get();
+
+  // Check kernel kAqRouteProtoId should be selected
+  ctx.results.clear();
+  rtnlCacheCB(routeFunc, &ctx, routeCache_);
+  count = 0;
+  for (const auto& r : ctx.results) {
+    if (r.getDestination() == prefix1V6 &&
+        r.getProtocolId() == kAqRouteProtoId && r.getNextHops().size() == 1) {
+      count++;
+    }
+    if (r.getDestination() == prefix2V6 &&
+        r.getProtocolId() == kAqRouteProtoId && r.getNextHops().size() == 1) {
+      count++;
+    }
+  }
+  EXPECT_EQ(2, count);
+
+  // delete the route in routing table, system should choose the backup route
+  netlinkSocket
+      ->delRoute(buildRoute(ifIndexX, kAqRouteProtoId, nextHopsV6, prefix1V6))
+      .get();
+
+  netlinkSocket
+      ->delRoute(buildRoute(ifIndexX, kAqRouteProtoId, nextHopsV6, prefix2V6))
+      .get();
+
+  ctx.results.clear();
+  rtnlCacheCB(routeFunc, &ctx, routeCache_);
+  count = 0;
+  totalRoute = 0;
+  for (const auto& r : ctx.results) {
+    if (r.getDestination() == prefix1V6 &&
+        r.getProtocolId() == kAqRouteProtoId1 && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoId1Priority) {
+      count++;
+      LOG(INFO) << static_cast<int>(r.getProtocolId());
+    }
+    if (r.getDestination() == prefix2V6 &&
+        r.getProtocolId() == kAqRouteProtoId1 && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoId1Priority) {
+      count++;
+    }
+    totalRoute++;
+  }
+  EXPECT_EQ(2, count);
+  EXPECT_EQ(2, totalRoute);
+
+  // add route back to see if it replace
+  netlinkSocket
+      ->addRoute(buildRoute(ifIndexX, kAqRouteProtoId, nextHopsV6, prefix1V6))
+      .get();
+  netlinkSocket
+      ->addRoute(buildRoute(ifIndexX, kAqRouteProtoId, nextHopsV6, prefix2V6))
+      .get();
+
+  ctx.results.clear();
+  rtnlCacheCB(routeFunc, &ctx, routeCache_);
+  count = 0;
+  totalRoute = 0;
+  for (const auto& r : ctx.results) {
+    if (r.getDestination() == prefix1V6 &&
+        r.getProtocolId() == kAqRouteProtoId && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoIdPriority) {
+      count++;
+    }
+    if (r.getDestination() == prefix2V6 &&
+        r.getProtocolId() == kAqRouteProtoId && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoIdPriority) {
+      count++;
+    }
+    if (r.getDestination() == prefix1V6 &&
+        r.getProtocolId() == kAqRouteProtoId1 && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoId1Priority) {
+      count++;
+    }
+    if (r.getDestination() == prefix2V6 &&
+        r.getProtocolId() == kAqRouteProtoId1 && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoId1Priority) {
+      count++;
+    }
+    totalRoute++;
+  }
+  EXPECT_EQ(4, count);
+  EXPECT_EQ(4, totalRoute);
+
+  // delete protocol1 route
+  netlinkSocket
+      ->delRoute(buildRoute(ifIndexY, kAqRouteProtoId1, nextHops1V6, prefix1V6))
+      .get();
+
+  netlinkSocket
+      ->delRoute(buildRoute(ifIndexY, kAqRouteProtoId1, nextHops1V6, prefix2V6))
+      .get();
+
+  ctx.results.clear();
+  rtnlCacheCB(routeFunc, &ctx, routeCache_);
+  count = 0;
+  totalRoute = 0;
+  for (const auto& r : ctx.results) {
+    if (r.getDestination() == prefix1V6 &&
+        r.getProtocolId() == kAqRouteProtoId && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoIdPriority) {
+      count++;
+    }
+    if (r.getDestination() == prefix2V6 &&
+        r.getProtocolId() == kAqRouteProtoId && r.getNextHops().size() == 1 &&
+        r.getPriority().value() == kAqRouteProtoIdPriority) {
+      count++;
+    }
+    totalRoute++;
+  }
+  EXPECT_EQ(2, count);
+  EXPECT_EQ(2, totalRoute);
+
+  // delete protocol route
+  netlinkSocket
+      ->delRoute(buildRoute(ifIndexX, kAqRouteProtoId, nextHopsV6, prefix1V6))
+      .get();
+
+  netlinkSocket
+      ->delRoute(buildRoute(ifIndexX, kAqRouteProtoId, nextHopsV6, prefix2V6))
+      .get();
+
+  ctx.results.clear();
+  rtnlCacheCB(routeFunc, &ctx, routeCache_);
+  count = 0;
+
+  for (const auto& r : ctx.results) {
+    if (r.getDestination() == prefix1V6) {
+      count++;
+    }
+    if (r.getDestination() == prefix2V6) {
+      count++;
+    }
+  }
+  EXPECT_EQ(0, count);
+}
+
 // - Add different routes for different protocols
 // - Verify it is added,
 // - Update nh and then verify it is updated