Do not route workload and frontend networks via the gateway

If a Gateway does not allow hairpin traffic, it may drop our packets.
Instead, if the remote is accessible via link via ARP, (which it is
on the workload network and the frontend networks) route those
packets via the device instead.

ansible/roles/cloudinit/files/var/lib/vmware/ovf-to-cloud-init.sh

@@ -330,7 +330,10 @@ writeRouteTableConfig() {
     mac=$(getMacForNetwork "$network")
     ip=$(ovf-rpctool get.ovf "${3}")
     if [ "$ip" != "" ] && [ "$ip" != "null" ]; then
+        # Default Gateway
         echo "${id},${network},${mac},${ip},${gateway}" >> "/etc/vmware/route-tables.cfg"
+        # Link-scoped route
+        echo "${id},${network},${mac},${ip}" >> "/etc/vmware/route-tables.cfg"
     fi
 }
 

ansible/roles/vmware/files/var/lib/vmware/routetablectl.sh

@@ -138,17 +138,30 @@ function up_routes() {
     cfg_table_name="${line_parts[1]}"
     cfg_mac_addr="${line_parts[2]}"
     cfg_cidr="${line_parts[3]}"
-    cfg_gateway="${line_parts[4]}"
+    cfg_gateway=""
+
+    if [[ ${#line_parts[@]} == 5 ]]; then
+        cfg_gateway="${line_parts[4]}"
+    fi
+
     cfg_dev="$(dev_from_mac "${cfg_mac_addr}")"
     route_table_name="${RT_TABLE_NAME_PREFIX}${cfg_table_name}"
 
     # Create a new route table.
     echo2 "create new route table id=${cfg_table_id} name=${route_table_name}"
     printf '%d\t%s\n' "${cfg_table_id}" "${route_table_name}" >>"${RT_TABLES_FILE}"
 
-    # Create default route for new route table.
-    echo2 "create default route for ${route_table_name}"
-    ip route add table "${route_table_name}" default via "${cfg_gateway}" dev "${cfg_dev}" proto static
+    if [[ "${cfg_gateway}" == "" ]]; then
+        cfg_destination=$(python3 -c "import sys; import ipaddress; print(ipaddress.ip_network(sys.argv[1], strict=False))" "${cfg_cidr}")
+        host="$(echo "${cfg_cidr}" | cut -d/ -f 1)"
+        cmd="ip route add table ${route_table_name} ${cfg_destination} dev ${cfg_dev} proto kernel scope link src ${host}"
+        echo2 "create route with cmd: ${cmd}"
+        eval "${cmd}"
+    else
+        # Create default route for new route table.
+        echo2 "create default route for ${route_table_name}"
+        ip route add table "${route_table_name}" default via "${cfg_gateway}" dev "${cfg_dev}" proto static
+    fi
 
     # Create IP rule for new route table.
     echo2 "create IP rule for ${route_table_name}"

hack/test-route-programs.sh

@@ -102,6 +102,10 @@ function net_mac_addr() {
   docker network inspect --format='{{range .Containers}}{{.MacAddress}}{{end}}' "${1}"
 }
 
+function net_ip() {
+  docker network inspect --format='{{range .Containers}}{{.IPv4Address}}{{end}}' "${1}"
+}
+
 # Creates a Docker network if it does not exist.
 function net_create() {
   if [ -z "$(docker network ls -qf "Name=${1}")" ]; then
@@ -189,6 +193,9 @@ function start_haproxy() {
     #DOCKER_NET_1_MAC="$(net_mac_addr "${DOCKER_NET_1}")"
     DOCKER_NET_2_MAC="$(net_mac_addr "${DOCKER_NET_2}")"
     DOCKER_NET_3_MAC="$(net_mac_addr "${DOCKER_NET_3}")"
+
+    DOCKER_IP_NET_2="$(net_ip "${DOCKER_NET_2}")"
+    DOCKER_IP_NET_3="$(net_ip "${DOCKER_NET_3}")"
   fi
 }
 
@@ -311,7 +318,7 @@ function test_routetablectl() {
   test_prereqs
 
   # Create the config file.
-  #   <TableID>,<TableName>,<MACAddress>,<NetworkCIDR>,<Gateway4>
+  #   <TableID>,<TableName>,<MACAddress>,<Network IP (CIDR format)>,<Gateway4>
   TEMP_TEST=".$(date "+%s")"
   cat <<EOF >"${TEMP_TEST}"
 2,frontend,${DOCKER_NET_2_MAC},${DOCKER_NET_2_CIDR},${DOCKER_NET_2_GATEWAY}
@@ -331,8 +338,10 @@ set -o pipefail
 # Create the config file.
 #   <TableID>,<TableName>,<MACAddress>,<NetworkCIDR>,<Gateway4>
 cat <<EOD >/etc/vmware/route-tables.cfg
-2,frontend,${DOCKER_NET_2_MAC},${DOCKER_NET_2_CIDR},${DOCKER_NET_2_GATEWAY}
-3,workload,${DOCKER_NET_3_MAC},${DOCKER_NET_3_CIDR},${DOCKER_NET_3_GATEWAY}
+2,frontend,${DOCKER_NET_2_MAC},${DOCKER_IP_NET_2},${DOCKER_NET_2_GATEWAY}
+3,workload,${DOCKER_NET_3_MAC},${DOCKER_IP_NET_3},${DOCKER_NET_3_GATEWAY}
+2,frontend,${DOCKER_NET_2_MAC},${DOCKER_IP_NET_2}
+3,workload,${DOCKER_NET_3_MAC},${DOCKER_IP_NET_3}
 EOD
 
 # Run the program with a populated config file and expect no errors.