Pluggable packet module proposal

README.md

@@ -42,8 +42,8 @@ More information about Black, you find at
 The following software is required to run PTF:
 
  * Python 2.7 or 3.x
- * Scapy 2.4.5
  * six 1.16.0
+ * Scapy 2.4.5 (unless you provide another packet manipulation module)
  * pypcap (optional - VLAN tests will fail without this)
  * tcpdump (optional - Scapy will complain if it's missing)
 
@@ -54,6 +54,11 @@ To install minimal requirements execute:
 pip install -r requirements.txt
 ```
 
+The default packet manipulator tool for `ptf` is `Scapy`. To install it use:
+```text
+pip install scapy==2.4.5
+```
+
 To enable VLAN tests, you need to install `pypcap`:
 ```text
 pip install pypcap
@@ -187,6 +192,30 @@ error. A timeout can also be specified for each individual test case, using the
 `@testtimeout` decorator, which needs to be imported from `ptf.testutils`. This
 timeout takes precedence over the global timeout passed on the command line.
 
+## Pluggable packet manipulation module
+
+By default, `ptf` uses `Scapy` as the packet manipulation module, but it can 
+also operate on a different one. 
+
+Such module **must define/implement the same symbols**, as defined in `Scapy` 
+implementation of packet. Most of them are just names of most common frame 
+headers (Ether, IP, TCP, UDP, ...).
+
+The default implementation can be found in file 
+[/src/ptf/packet_scapy.py](/src/ptf/packet_scapy.py). It can be used as a 
+reference when implementing your own version.
+
+To use another packet manipulation module, one needs to 
+provide it as argument `-pmm` or `--packet-manipulation-module` when running the
+`ptf` binary.
+
+```text
+sudo ./ptf <other parameters> -pmm foo.packet_foo 
+```
+
+Please make sure that this module is loaded into the runtime before running 
+any tests.
+
 ---
 
 # Configuring PTF

example/mytests/switch.py

@@ -380,7 +380,7 @@ def runTest(self):
 
         # illustrates how to use a mask even if no impact here
         m = Mask(exp_pkt)
-        m.set_do_not_care_scapy(IP, 'ttl')
+        m.set_do_not_care_packet(IP, 'ttl')
         try:
             send_packet(self, 2, pkt)
             verify_packets(self, m, [1])

ptf

@@ -95,6 +95,8 @@ config_default = {
     "test_case_timeout": None,
     # Socket options
     "socket_recv_size": 4096,
+    # Packet manipulation provider module
+    "packet_manipulation_module": "ptf.packet_scapy",
     # Other configuration
     "port_map": None,
 }
@@ -239,6 +241,14 @@ be subtracted from the result by prefixing them with the '^' character.  """
 
     parser.add_argument("--pypath", dest="pypath", action="append")
 
+    parser.add_argument(
+        "-pmm",
+        "--packet-manipulation-module",
+        type=str,
+        help="Provide packet manipulation module which should be used "
+        "as a 'packet' one for other PTF modules",
+    )
+
     group = parser.add_argument_group("Test selection options")
     group.add_argument("-f", "--test-file", help="File of tests to run, one per line")
     group.add_argument(

ptf_nn/ptf_nn_test/test.py

@@ -105,7 +105,7 @@ def runTest(self):
         pkt1 = testutils.simple_udp_packet(eth_dst="00:11:11:11:11:11")
         pkt2 = testutils.simple_udp_packet(eth_dst="00:22:22:22:22:22")
         exp_pkt = Mask(pkt2)
-        exp_pkt.set_do_not_care_scapy(Ether, 'dst')
+        exp_pkt.set_do_not_care_packet(Ether, 'dst')
 
         testutils.send_packet(self, (0, 1), pkt1)
         print("Packet sent")

requirements.txt

@@ -1,2 +1 @@
-scapy==2.4.5
 six==1.16.0

src/ptf/dataplane.py

@@ -30,8 +30,7 @@
 from . import ptfutils
 from . import netutils
 from . import mask
-import scapy.packet
-import scapy.utils
+from . import packet
 from .pcap_writer import PcapWriter
 from six import StringIO
 
@@ -802,12 +801,12 @@ def format(self):
                 sys.stdout = StringIO()
 
                 print("========== RECEIVED ==========")
-                if isinstance(self.expected_packet, scapy.packet.Packet):
+                if isinstance(self.expected_packet, packet.Packet):
                     # Dissect this packet as if it were an instance of
                     # the expected packet's class.
-                    scapy.packet.ls(self.expected_packet.__class__(self.packet))
+                    packet.ls(self.expected_packet.__class__(self.packet))
                     print("--")
-                scapy.utils.hexdump(self.packet)
+                packet.hexdump(self.packet)
                 print("==============================")
 
                 return sys.stdout.getvalue()
@@ -850,30 +849,30 @@ def format(self):
 
                 if self.expected_packet is not None:
                     print("========== EXPECTED ==========")
-                    if isinstance(self.expected_packet, scapy.packet.Packet):
-                        scapy.packet.ls(self.expected_packet)
+                    if isinstance(self.expected_packet, packet.Packet):
+                        packet.ls(self.expected_packet)
                         print("--")
-                        scapy.utils.hexdump(self.expected_packet)
+                        packet.hexdump(self.expected_packet)
                     elif isinstance(self.expected_packet, mask.Mask):
                         print("Mask:")
                         print(self.expected_packet)
                     else:
-                        scapy.utils.hexdump(self.expected_packet)
+                        packet.hexdump(self.expected_packet)
 
                 print("========== RECEIVED ==========")
                 if self.recent_packets:
                     print(
                         "%d total packets. Displaying most recent %d packets:"
                         % (self.packet_count, len(self.recent_packets))
                     )
-                    for packet in self.recent_packets:
+                    for recent_packet in self.recent_packets:
                         print("------------------------------")
-                        if isinstance(self.expected_packet, scapy.packet.Packet):
+                        if isinstance(self.expected_packet, packet.Packet):
                             # Dissect this packet as if it were an instance of
                             # the expected packet's class.
-                            scapy.packet.ls(self.expected_packet.__class__(packet))
+                            packet.ls(self.expected_packet.__class__(recent_packet))
                             print("--")
-                        scapy.utils.hexdump(packet)
+                        packet.hexdump(recent_packet)
                 else:
                     print("%d total packets." % self.packet_count)
                 print("==============================")

src/ptf/mask.py

@@ -1,8 +1,8 @@
 from __future__ import print_function
+import warnings
 from six import StringIO
 import sys
-from scapy.utils import hexdump
-from . import packet as scapy
+from . import packet
 
 
 class Mask:
@@ -21,7 +21,7 @@ def set_do_not_care(self, offset, bitwidth):
             offsetb = idx % 8
             self.mask[offsetB] = self.mask[offsetB] & (~(1 << (7 - offsetb)))
 
-    def set_do_not_care_scapy(self, hdr_type, field_name):
+    def set_do_not_care_packet(self, hdr_type, field_name):
         if hdr_type not in self.exp_pkt:
             self.valid = False
             print("Unknown header type")
@@ -46,6 +46,14 @@ def set_do_not_care_scapy(self, hdr_type, field_name):
                 offset += bits
         self.set_do_not_care(hdr_offset * 8 + offset, bitwidth)
 
+    def set_do_not_care_scapy(self, hdr_type, field_name):
+        warnings.warn(
+            '"set_do_not_care_scapy" is going to be deprecated, please '
+            'switch to the new one: "set_do_not_care_packet"',
+            DeprecationWarning,
+        )
+        self.set_do_not_care_packet(hdr_type, field_name)
+
     def set_ignore_extra_bytes(self):
         self.ignore_extra_bytes = True
 
@@ -71,7 +79,7 @@ def __str__(self):
         assert self.valid
         old_stdout = sys.stdout
         sys.stdout = buffer = StringIO()
-        hexdump(self.exp_pkt)
+        packet.hexdump(self.exp_pkt)
         print("mask =", end=" ")
         for i in range(0, len(self.mask), 16):
             if i > 0:
@@ -84,14 +92,14 @@ def __str__(self):
 
 
 def utest():
-    p = scapy.Ether() / scapy.IP() / scapy.TCP()
+    p = packet.Ether() / packet.IP() / packet.TCP()
     m = Mask(p)
     assert m.pkt_match(p)
-    p1 = scapy.Ether() / scapy.IP() / scapy.TCP(sport=97)
+    p1 = packet.Ether() / packet.IP() / packet.TCP(sport=97)
     assert not m.pkt_match(p1)
-    m.set_do_not_care_scapy(scapy.TCP, "sport")
+    m.set_do_not_care_packet(packet.TCP, "sport")
     assert not m.pkt_match(p1)
-    m.set_do_not_care_scapy(scapy.TCP, "chksum")
+    m.set_do_not_care_packet(packet.TCP, "chksum")
     assert m.pkt_match(p1)
     exp_pkt = "\x01\x02\x03\x04\x05\x06"
     pkt = "\x01\x00\x00\x04\x05\x06\x07\x08"

src/ptf/packet.py

@@ -1,146 +1,24 @@
-# Distributed under the OpenFlow Software License (see LICENSE)
-# Copyright (c) 2010 The Board of Trustees of The Leland Stanford Junior University
-# Copyright (c) 2012, 2013 Big Switch Networks, Inc.
-"""
-Wrap scapy to satisfy pylint
+""" A pluggable packet module
+
+This module dynamically imports definitions from packet manipulation module,
+specified in config or provided as an agrument.
+The default one is Scapy, but one can develop its own packet manipulation framework and
+then, create an implementation of packet module for it (for Scapy it is packet_scapy.py)
 """
-import ptf
+from __future__ import print_function
 from ptf import config
-import sys
-import logging
-
-try:
-    import scapy.config
-    import scapy.route
-    import scapy.layers.l2
-    import scapy.layers.inet
-    import scapy.layers.dhcp
-    import scapy.layers.vxlan
-    import scapy.packet
-    import scapy.main
-    import scapy.fields
-
-    if not config.get("disable_ipv6", False):
-        import scapy.route6
-        import scapy.layers.inet6
-except ImportError:
-    sys.exit("Need to install scapy for packet parsing")
-
-Ether = scapy.layers.l2.Ether
-LLC = scapy.layers.l2.LLC
-SNAP = scapy.layers.l2.SNAP
-Dot1Q = scapy.layers.l2.Dot1Q
-GRE = scapy.layers.l2.GRE
-IP = scapy.layers.inet.IP
-IPOption = scapy.layers.inet.IPOption
-try:
-    ARP = scapy.layers.inet.ARP
-except AttributeError:
-    # Works with more recent versions of Scapy
-    ARP = scapy.layers.l2.ARP
-TCP = scapy.layers.inet.TCP
-UDP = scapy.layers.inet.UDP
-ICMP = scapy.layers.inet.ICMP
-DHCP = scapy.layers.dhcp.DHCP
-BOOTP = scapy.layers.dhcp.BOOTP
-PADDING = scapy.packet.Padding
-VXLAN = scapy.layers.vxlan.VXLAN
-
-BTH = None
-if not config.get("disable_rocev2", False):
-    try:
-        ptf.disable_logging()
-        scapy.main.load_contrib("roce")
-        BTH = scapy.contrib.roce.BTH
-        ptf.enable_logging()
-        logging.info("ROCEv2 support found in Scapy")
-    except:
-        ptf.enable_logging()
-        logging.warn("ROCEv2 support not found in Scapy")
-        pass
-
-if not config.get("disable_ipv6", False):
-    IPv6 = scapy.layers.inet6.IPv6
-    IPv6ExtHdrRouting = scapy.layers.inet6.IPv6ExtHdrRouting
-    ICMPv6Unknown = scapy.layers.inet6.ICMPv6Unknown
-    ICMPv6EchoRequest = scapy.layers.inet6.ICMPv6EchoRequest
-    ICMPv6MLReport = scapy.layers.inet6.ICMPv6MLReport
-
-ERSPAN = None
-ERSPAN_III = None
-PlatformSpecific = None
-if not config.get("disable_erspan", False):
-    try:
-        ptf.disable_logging()
-        scapy.main.load_contrib("erspan")
-        ERSPAN = scapy.contrib.erspan.ERSPAN
-        ERSPAN_III = scapy.contrib.erspan.ERSPAN_III
-        PlatformSpecific = scapy.contrib.erspan.ERSPAN_PlatformSpecific
-        ptf.enable_logging()
-        logging.info("ERSPAN support found in Scapy")
-    except:
-        ptf.enable_logging()
-        logging.warn("ERSPAN support not found in Scapy")
-        pass
-
-GENEVE = None
-if not config.get("disable_geneve", False):
-    try:
-        ptf.disable_logging()
-        scapy.main.load_contrib("geneve")
-        GENEVE = scapy.contrib.geneve.GENEVE
-        ptf.enable_logging()
-        logging.info("GENEVE support found in Scapy")
-    except:
-        ptf.enable_logging()
-        logging.warn("GENEVE support not found in Scapy")
-        pass
-
-MPLS = None
-if not config.get("disable_mpls", False):
-    try:
-        ptf.disable_logging()
-        scapy.main.load_contrib("mpls")
-        MPLS = scapy.contrib.mpls.MPLS
-        ptf.enable_logging()
-        logging.info("MPLS support found in Scapy")
-    except:
-        ptf.enable_logging()
-        logging.warn("MPLS support not found in Scapy")
-        pass
-
-NVGRE = None
-if not config.get("disable_nvgre", False):
-
-    class NVGRE(scapy.packet.Packet):
-        name = "NVGRE"
-        fields_desc = [
-            scapy.fields.BitField("chksum_present", 0, 1),
-            scapy.fields.BitField("routing_present", 0, 1),
-            scapy.fields.BitField("key_present", 1, 1),
-            scapy.fields.BitField("seqnum_present", 0, 1),
-            scapy.fields.BitField("reserved", 0, 9),
-            scapy.fields.BitField("version", 0, 3),
-            scapy.fields.XShortField("proto", 0x6558),
-            scapy.fields.ThreeBytesField("vsid", 0),
-            scapy.fields.XByteField("flowid", 0),
-        ]
 
-        def mysummary(self):
-            return self.sprintf("NVGRE (vni=%NVGRE.vsid%)")
+__module = __import__(
+    config.get("packet_manipulation_module", "ptf.packet_scapy"), fromlist=["*"]
+)
+__keys = []
 
-    scapy.packet.bind_layers(IP, NVGRE, proto=47)
-    scapy.packet.bind_layers(NVGRE, Ether)
+# import logic - everything from __all__ if provided, otherwise everything not starting
+# with underscore
+print("Using packet manipulation module: %s" % __module.__name__)
+if "__all__" in __module.__dict__:
+    __keys = __module.__dict__["__all__"]
+else:
+    __keys = [k for k in __module.__dict__ if not k.startswith("_")]
 
-IGMP = None
-if not config.get("disable_igmp", False):
-    try:
-        ptf.disable_logging()
-        scapy.main.load_contrib("igmp")
-        IGMP = scapy.contrib.igmp.IGMP
-        ptf.enable_logging()
-        logging.info("IGMP support found in Scapy")
-    except:
-        ptf.enable_logging()
-        logging.warn("IGMP support not found in Scapy")
-        pass
+locals().update({k: getattr(__module, k) for k in __keys})