userspace: Improved packet drop statistics.

Currently OVS maintains explicit packet drop/error counters only on port
level.  Packets that are dropped as part of normal OpenFlow processing
are counted in flow stats of “drop” flows or as table misses in table
stats. These can only be interpreted by controllers that know the
semantics of the configured OpenFlow pipeline.  Without that knowledge,
it is impossible for an OVS user to obtain e.g. the total number of
packets dropped due to OpenFlow rules.

Furthermore, there are numerous other reasons for which packets can be
dropped by OVS slow path that are not related to the OpenFlow pipeline.
The generated datapath flow entries include a drop action to avoid
further expensive upcalls to the slow path, but subsequent packets
dropped by the datapath are not accounted anywhere.

Finally, the datapath itself drops packets in certain error situations.
Also, these drops are today not accounted for.This makes it difficult
for OVS users to monitor packet drop in an OVS instance and to alert a
management system in case of a unexpected increase of such drops.
Also OVS trouble-shooters face difficulties in analysing packet drops.

With this patch we implement following changes to address the issues
mentioned above.

1. Identify and account all the silent packet drop scenarios
2. Display these drops in ovs-appctl coverage/show

Co-authored-by: Rohith Basavaraja <rohith.basavaraja@gmail.com>
Co-authored-by: Keshav Gupta <keshugupta1@gmail.com>
Signed-off-by: Anju Thomas <anju.thomas@ericsson.com>
Signed-off-by: Rohith Basavaraja <rohith.basavaraja@gmail.com>
Signed-off-by: Keshav Gupta <keshugupta1@gmail.com>
Acked-by: Eelco Chaudron <echaudro@redhat.com
Acked-by: Ben Pfaff <blp@ovn.org>
Signed-off-by: Ilya Maximets <i.maximets@ovn.org>

Author: 3 people

datapath/linux/compat/include/linux/openvswitch.h

@@ -409,6 +409,28 @@ enum ovs_tunnel_key_attr {
 
 #define OVS_TUNNEL_KEY_ATTR_MAX (__OVS_TUNNEL_KEY_ATTR_MAX - 1)
 
+/**
+ * enum xlate_error -  Different types of error during translation
+ */
+
+#ifndef __KERNEL__
+enum xlate_error {
+	XLATE_OK = 0,
+	XLATE_BRIDGE_NOT_FOUND,
+	XLATE_RECURSION_TOO_DEEP,
+	XLATE_TOO_MANY_RESUBMITS,
+	XLATE_STACK_TOO_DEEP,
+	XLATE_NO_RECIRCULATION_CONTEXT,
+	XLATE_RECIRCULATION_CONFLICT,
+	XLATE_TOO_MANY_MPLS_LABELS,
+	XLATE_INVALID_TUNNEL_METADATA,
+	XLATE_UNSUPPORTED_PACKET_TYPE,
+	XLATE_CONGESTION_DROP,
+	XLATE_FORWARDING_DISABLED,
+	XLATE_MAX,
+};
+#endif
+
 /**
  * enum ovs_frag_type - IPv4 and IPv6 fragment type
  * @OVS_FRAG_TYPE_NONE: Packet is not a fragment.
@@ -965,6 +987,7 @@ struct check_pkt_len_arg {
  * @OVS_ACTION_ATTR_CHECK_PKT_LEN: Check the packet length and execute a set
  * of actions if greater than the specified packet length, else execute
  * another set of actions.
+ * @OVS_ACTION_ATTR_DROP: Explicit drop action.
  */
 
 enum ovs_action_attr {
@@ -997,6 +1020,7 @@ enum ovs_action_attr {
 #ifndef __KERNEL__
 	OVS_ACTION_ATTR_TUNNEL_PUSH,   /* struct ovs_action_push_tnl*/
 	OVS_ACTION_ATTR_TUNNEL_POP,    /* u32 port number. */
+	OVS_ACTION_ATTR_DROP,          /* u32 xlate_error. */
 #endif
 	__OVS_ACTION_ATTR_MAX,	      /* Nothing past this will be accepted
 				       * from userspace. */

lib/dpif-netdev.c

@@ -102,6 +102,17 @@ enum { MAX_METERS = 65536 };    /* Maximum number of meters. */
 enum { MAX_BANDS = 8 };         /* Maximum number of bands / meter. */
 enum { N_METER_LOCKS = 64 };    /* Maximum number of meters. */
 
+COVERAGE_DEFINE(datapath_drop_meter);
+COVERAGE_DEFINE(datapath_drop_upcall_error);
+COVERAGE_DEFINE(datapath_drop_lock_error);
+COVERAGE_DEFINE(datapath_drop_userspace_action_error);
+COVERAGE_DEFINE(datapath_drop_tunnel_push_error);
+COVERAGE_DEFINE(datapath_drop_tunnel_pop_error);
+COVERAGE_DEFINE(datapath_drop_recirc_error);
+COVERAGE_DEFINE(datapath_drop_invalid_port);
+COVERAGE_DEFINE(datapath_drop_invalid_tnl_port);
+COVERAGE_DEFINE(datapath_drop_rx_invalid_packet);
+
 /* Protects against changes to 'dp_netdevs'. */
 static struct ovs_mutex dp_netdev_mutex = OVS_MUTEX_INITIALIZER;
 
@@ -5770,7 +5781,7 @@ dp_netdev_run_meter(struct dp_netdev *dp, struct dp_packet_batch *packets_,
             band = &meter->bands[exceeded_band[j]];
             band->packet_count += 1;
             band->byte_count += dp_packet_size(packet);
-
+            COVERAGE_INC(datapath_drop_meter);
             dp_packet_delete(packet);
         } else {
             /* Meter accepts packet. */
@@ -6520,6 +6531,7 @@ dfc_processing(struct dp_netdev_pmd_thread *pmd,
 
         if (OVS_UNLIKELY(dp_packet_size(packet) < ETH_HEADER_LEN)) {
             dp_packet_delete(packet);
+            COVERAGE_INC(datapath_drop_rx_invalid_packet);
             continue;
         }
 
@@ -6640,6 +6652,7 @@ handle_packet_upcall(struct dp_netdev_pmd_thread *pmd,
                              put_actions);
     if (OVS_UNLIKELY(error && error != ENOSPC)) {
         dp_packet_delete(packet);
+        COVERAGE_INC(datapath_drop_upcall_error);
         return error;
     }
 
@@ -6770,6 +6783,7 @@ fast_path_processing(struct dp_netdev_pmd_thread *pmd,
         DP_PACKET_BATCH_FOR_EACH (i, packet, packets_) {
             if (OVS_UNLIKELY(!rules[i])) {
                 dp_packet_delete(packet);
+                COVERAGE_INC(datapath_drop_lock_error);
                 upcall_fail_cnt++;
             }
         }
@@ -7039,6 +7053,7 @@ dp_execute_userspace_action(struct dp_netdev_pmd_thread *pmd,
                                   actions->data, actions->size);
     } else if (should_steal) {
         dp_packet_delete(packet);
+        COVERAGE_INC(datapath_drop_userspace_action_error);
     }
 }
 
@@ -7053,6 +7068,7 @@ dp_execute_cb(void *aux_, struct dp_packet_batch *packets_,
     struct dp_netdev *dp = pmd->dp;
     int type = nl_attr_type(a);
     struct tx_port *p;
+    uint32_t packet_count, packets_dropped;
 
     switch ((enum ovs_action_attr)type) {
     case OVS_ACTION_ATTR_OUTPUT:
@@ -7094,6 +7110,9 @@ dp_execute_cb(void *aux_, struct dp_packet_batch *packets_,
                 dp_packet_batch_add(&p->output_pkts, packet);
             }
             return;
+        } else {
+            COVERAGE_ADD(datapath_drop_invalid_port,
+                         dp_packet_batch_size(packets_));
         }
         break;
 
@@ -7106,7 +7125,11 @@ dp_execute_cb(void *aux_, struct dp_packet_batch *packets_,
             break;
         }
         dp_packet_batch_apply_cutlen(packets_);
-        push_tnl_action(pmd, a, packets_);
+        packet_count = dp_packet_batch_size(packets_);
+        if (push_tnl_action(pmd, a, packets_)) {
+            COVERAGE_ADD(datapath_drop_tunnel_push_error,
+                         packet_count);
+        }
         return;
 
     case OVS_ACTION_ATTR_TUNNEL_POP:
@@ -7126,7 +7149,14 @@ dp_execute_cb(void *aux_, struct dp_packet_batch *packets_,
 
                 dp_packet_batch_apply_cutlen(packets_);
 
+                packet_count = dp_packet_batch_size(packets_);
                 netdev_pop_header(p->port->netdev, packets_);
+                packets_dropped =
+                   packet_count - dp_packet_batch_size(packets_);
+                if (packets_dropped) {
+                    COVERAGE_ADD(datapath_drop_tunnel_pop_error,
+                                 packets_dropped);
+                }
                 if (dp_packet_batch_is_empty(packets_)) {
                     return;
                 }
@@ -7141,6 +7171,11 @@ dp_execute_cb(void *aux_, struct dp_packet_batch *packets_,
                 (*depth)--;
                 return;
             }
+            COVERAGE_ADD(datapath_drop_invalid_tnl_port,
+                         dp_packet_batch_size(packets_));
+        } else {
+            COVERAGE_ADD(datapath_drop_recirc_error,
+                         dp_packet_batch_size(packets_));
         }
         break;
 
@@ -7185,6 +7220,8 @@ dp_execute_cb(void *aux_, struct dp_packet_batch *packets_,
 
             return;
         }
+        COVERAGE_ADD(datapath_drop_lock_error,
+                     dp_packet_batch_size(packets_));
         break;
 
     case OVS_ACTION_ATTR_RECIRC:
@@ -7208,6 +7245,8 @@ dp_execute_cb(void *aux_, struct dp_packet_batch *packets_,
             return;
         }
 
+        COVERAGE_ADD(datapath_drop_recirc_error,
+                     dp_packet_batch_size(packets_));
         VLOG_WARN("Packet dropped. Max recirculation depth exceeded.");
         break;
 
@@ -7365,6 +7404,7 @@ dp_execute_cb(void *aux_, struct dp_packet_batch *packets_,
     case OVS_ACTION_ATTR_POP_NSH:
     case OVS_ACTION_ATTR_CT_CLEAR:
     case OVS_ACTION_ATTR_CHECK_PKT_LEN:
+    case OVS_ACTION_ATTR_DROP:
     case __OVS_ACTION_ATTR_MAX:
         OVS_NOT_REACHED();
     }

lib/dpif.c

@@ -1283,6 +1283,7 @@ dpif_execute_helper_cb(void *aux_, struct dp_packet_batch *packets_,
     case OVS_ACTION_ATTR_CT_CLEAR:
     case OVS_ACTION_ATTR_UNSPEC:
     case OVS_ACTION_ATTR_CHECK_PKT_LEN:
+    case OVS_ACTION_ATTR_DROP:
     case __OVS_ACTION_ATTR_MAX:
         OVS_NOT_REACHED();
     }
@@ -1888,6 +1889,12 @@ dpif_supports_tnl_push_pop(const struct dpif *dpif)
     return dpif_is_netdev(dpif);
 }
 
+bool
+dpif_supports_explicit_drop_action(const struct dpif *dpif)
+{
+    return dpif_is_netdev(dpif);
+}
+
 /* Meters */
 void
 dpif_meter_get_features(const struct dpif *dpif,

lib/dpif.h

@@ -906,6 +906,7 @@ int dpif_get_pmds_for_port(const struct dpif * dpif, odp_port_t port_no,
 
 char *dpif_get_dp_version(const struct dpif *);
 bool dpif_supports_tnl_push_pop(const struct dpif *);
+bool dpif_supports_explicit_drop_action(const struct dpif *);
 
 /* Log functions. */
 struct vlog_module;

lib/odp-execute.c

@@ -25,6 +25,7 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "coverage.h"
 #include "dp-packet.h"
 #include "dpif.h"
 #include "netlink.h"
@@ -36,6 +37,72 @@
 #include "util.h"
 #include "csum.h"
 #include "conntrack.h"
+#include "openvswitch/vlog.h"
+
+VLOG_DEFINE_THIS_MODULE(odp_execute);
+COVERAGE_DEFINE(datapath_drop_sample_error);
+COVERAGE_DEFINE(datapath_drop_nsh_decap_error);
+COVERAGE_DEFINE(drop_action_of_pipeline);
+COVERAGE_DEFINE(drop_action_bridge_not_found);
+COVERAGE_DEFINE(drop_action_recursion_too_deep);
+COVERAGE_DEFINE(drop_action_too_many_resubmit);
+COVERAGE_DEFINE(drop_action_stack_too_deep);
+COVERAGE_DEFINE(drop_action_no_recirculation_context);
+COVERAGE_DEFINE(drop_action_recirculation_conflict);
+COVERAGE_DEFINE(drop_action_too_many_mpls_labels);
+COVERAGE_DEFINE(drop_action_invalid_tunnel_metadata);
+COVERAGE_DEFINE(drop_action_unsupported_packet_type);
+COVERAGE_DEFINE(drop_action_congestion);
+COVERAGE_DEFINE(drop_action_forwarding_disabled);
+
+static void
+dp_update_drop_action_counter(enum xlate_error drop_reason,
+                              int delta)
+{
+   static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
+
+   switch (drop_reason) {
+   case XLATE_OK:
+        COVERAGE_ADD(drop_action_of_pipeline, delta);
+        break;
+   case XLATE_BRIDGE_NOT_FOUND:
+        COVERAGE_ADD(drop_action_bridge_not_found, delta);
+        break;
+   case XLATE_RECURSION_TOO_DEEP:
+        COVERAGE_ADD(drop_action_recursion_too_deep, delta);
+        break;
+   case XLATE_TOO_MANY_RESUBMITS:
+        COVERAGE_ADD(drop_action_too_many_resubmit, delta);
+        break;
+   case XLATE_STACK_TOO_DEEP:
+        COVERAGE_ADD(drop_action_stack_too_deep, delta);
+        break;
+   case XLATE_NO_RECIRCULATION_CONTEXT:
+        COVERAGE_ADD(drop_action_no_recirculation_context, delta);
+        break;
+   case XLATE_RECIRCULATION_CONFLICT:
+        COVERAGE_ADD(drop_action_recirculation_conflict, delta);
+        break;
+   case XLATE_TOO_MANY_MPLS_LABELS:
+        COVERAGE_ADD(drop_action_too_many_mpls_labels, delta);
+        break;
+   case XLATE_INVALID_TUNNEL_METADATA:
+        COVERAGE_ADD(drop_action_invalid_tunnel_metadata, delta);
+        break;
+   case XLATE_UNSUPPORTED_PACKET_TYPE:
+        COVERAGE_ADD(drop_action_unsupported_packet_type, delta);
+        break;
+   case XLATE_CONGESTION_DROP:
+        COVERAGE_ADD(drop_action_congestion, delta);
+        break;
+   case XLATE_FORWARDING_DISABLED:
+        COVERAGE_ADD(drop_action_forwarding_disabled, delta);
+        break;
+   case XLATE_MAX:
+   default:
+        VLOG_ERR_RL(&rl, "Invalid Drop reason type: %d", drop_reason);
+   }
+}
 
 /* Masked copy of an ethernet address. 'src' is already properly masked. */
 static void
@@ -621,6 +688,7 @@ odp_execute_sample(void *dp, struct dp_packet *packet, bool steal,
         case OVS_SAMPLE_ATTR_PROBABILITY:
             if (random_uint32() >= nl_attr_get_u32(a)) {
                 if (steal) {
+                    COVERAGE_INC(datapath_drop_sample_error);
                     dp_packet_delete(packet);
                 }
                 return;
@@ -749,6 +817,7 @@ requires_datapath_assistance(const struct nlattr *a)
     case OVS_ACTION_ATTR_POP_NSH:
     case OVS_ACTION_ATTR_CT_CLEAR:
     case OVS_ACTION_ATTR_CHECK_PKT_LEN:
+    case OVS_ACTION_ATTR_DROP:
         return false;
 
     case OVS_ACTION_ATTR_UNSPEC:
@@ -965,6 +1034,7 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
                 if (pop_nsh(packet)) {
                     dp_packet_batch_refill(batch, packet, i);
                 } else {
+                    COVERAGE_INC(datapath_drop_nsh_decap_error);
                     dp_packet_delete(packet);
                 }
             }
@@ -989,6 +1059,14 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
             }
             break;
 
+        case OVS_ACTION_ATTR_DROP:{
+            const enum xlate_error *drop_reason = nl_attr_get(a);
+
+            dp_update_drop_action_counter(*drop_reason,
+                                          dp_packet_batch_size(batch));
+            dp_packet_delete_batch(batch, steal);
+            return;
+        }
         case OVS_ACTION_ATTR_OUTPUT:
         case OVS_ACTION_ATTR_TUNNEL_PUSH:
         case OVS_ACTION_ATTR_TUNNEL_POP:

lib/odp-util.c

@@ -141,6 +141,7 @@ odp_action_len(uint16_t type)
     case OVS_ACTION_ATTR_PUSH_NSH: return ATTR_LEN_VARIABLE;
     case OVS_ACTION_ATTR_POP_NSH: return 0;
     case OVS_ACTION_ATTR_CHECK_PKT_LEN: return ATTR_LEN_VARIABLE;
+    case OVS_ACTION_ATTR_DROP: return sizeof(uint32_t);
 
     case OVS_ACTION_ATTR_UNSPEC:
     case __OVS_ACTION_ATTR_MAX:
@@ -1238,6 +1239,9 @@ format_odp_action(struct ds *ds, const struct nlattr *a,
     case OVS_ACTION_ATTR_CHECK_PKT_LEN:
         format_odp_check_pkt_len_action(ds, a, portno_names);
         break;
+    case OVS_ACTION_ATTR_DROP:
+        ds_put_cstr(ds, "drop");
+        break;
     case OVS_ACTION_ATTR_UNSPEC:
     case __OVS_ACTION_ATTR_MAX:
     default:
@@ -2575,6 +2579,7 @@ odp_actions_from_string(const char *s, const struct simap *port_names,
     size_t old_size;
 
     if (!strcasecmp(s, "drop")) {
+        nl_msg_put_u32(actions, OVS_ACTION_ATTR_DROP, XLATE_OK);
         return 0;
     }
 

ofproto/ofproto-dpif-ipfix.c

@@ -3016,6 +3016,7 @@ dpif_ipfix_read_actions(const struct flow *flow,
         case OVS_ACTION_ATTR_POP_NSH:
         case OVS_ACTION_ATTR_CHECK_PKT_LEN:
         case OVS_ACTION_ATTR_UNSPEC:
+        case OVS_ACTION_ATTR_DROP:
         case __OVS_ACTION_ATTR_MAX:
         default:
             break;

ofproto/ofproto-dpif-sflow.c

@@ -1224,6 +1224,7 @@ dpif_sflow_read_actions(const struct flow *flow,
         case OVS_ACTION_ATTR_POP_NSH:
         case OVS_ACTION_ATTR_UNSPEC:
         case OVS_ACTION_ATTR_CHECK_PKT_LEN:
+        case OVS_ACTION_ATTR_DROP:
         case __OVS_ACTION_ATTR_MAX:
         default:
             break;