Merge pull request #275 from michelsciortino/pcn-dynmon

New transparent service: Dynamic Monitor

Documentation/services/pcn-dynmon/dynmon.rst

@@ -0,0 +1,113 @@
+Dynmon service
+==============
+
+Dynmon is a transparent service that allows the dynamic injection of eBPF code in the linux kernel, enabling the monitoring of the network traffic and the collection and exportation of custom metrics.
+
+This service exploits the capabilities of Polycube to replace the eBPF code running in the dataplane and the use of eBPF maps to share data between the control plane and the data plane.
+
+Features
+--------
+- Transparent service, can be attached to any network interface and Polycube services
+- Support for the injection of any eBPF code at runtime
+- Support for eBPF maps content exportation through the REST interface as metrics
+- Support for two different exportation formats: JSON and OpenMetrics
+
+Limitations
+-----------
+- The OpenMetrics format does not support complex data structures, hence the maps are exported only if their value type is a simple type (structs and unions are not supported)
+- The OpenMetrics Histogram and Summary metrics are not yet supported
+
+How to use
+----------
+
+
+Creating the service
+^^^^^^^^^^^^^^^^^^^^
+::
+
+    #create the dynmon service instance
+    polycubectl dynmon add monitor
+
+
+Configuring the data plane
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+In order to configure the dataplane of the service, a configuration JSON object must be sent to the control plane; this action cannot be done through the **polycubectl** tool as it does not handle complex inputs.
+
+To send the data plane configuration to the control plane it is necessary to exploit the REST interface of the service, applying a ``PUT`` request to the ``/dataplane`` endpoint.
+
+Configuration examples can be found in the *examples* directory.
+
+
+Attaching to a interface
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+::
+
+    # Attach the service to a network interface
+    polycubectl attach monitor eno0
+
+    # Attach the service to a cube port
+    polycubectl attach monitor br1:toveth1
+
+
+Collecting metrics
+^^^^^^^^^^^^^^^^^^
+To collect the metrics of the service, two endpoints have been defined to enable the two possible exportation formats:
+
+- JSON format
+
+    ::
+
+        polycubectl monitor metrics show
+
+- OpenMetrics format
+
+    ::
+
+        polycubectl monitor open-metrics show
+
+
+Dynmon Injector Tool
+--------------------
+
+This tool allows the creation and the manipulation of a `dynmon` cube without using the standard `polycubectl` CLI.
+
+Install
+^^^^^^^
+Some dependencies are required for this tool to run:
+::
+    pip install -r requirements.txt
+
+
+Running the tool
+^^^^^^^^^^^^^^^^
+::
+
+    Usage: `dynmon_injector.py [-h] [-a ADDRESS] [-p PORT] [-v] cube_name peer_interface path_to_dataplane`
+    
+    positional arguments:
+    cube_name             indicates the name of the cube
+    peer_interface        indicates the network interface to connect the cube to
+    path_to_dataplane     indicates the path to the json file which contains the new dataplane configuration
+                          which contains the new dataplane code and the metadata associated to the exported metrics
+
+    optional arguments:
+    -h, --help                        show this help message and exit
+    -a ADDRESS, --address ADDRESS     set the polycube daemon ip address (default: localhost)
+    -p PORT, --port PORT              set the polycube daemon port (default: 9000)
+    -v, --version                     show program's version number and exit
+
+
+Usage examples
+^^^^^^^^^^^^^^
+::
+
+    basic usage:
+    ./dynmon_injector.py monitor_0 eno1 ../examples/packet_counter.json
+
+    setting custom ip address and port to contact the polycube daemon:
+    ./dynmon_injector.py -a 10.0.0.1 -p 5840 monitor_0 eno1 ../examples/packet_counter.json
+
+
+This tool creates a new `dynmon` cube with the given configuration and attaches it to the selected interface.
+
+If the monitor already exists, the tool checks if the attached interface is the same used previously; if not, it detaches the cube from the previous interface and attaches it to the new one; then, the selected dataplane is injected.

src/services/CMakeLists.txt

@@ -36,6 +36,7 @@ add_service(iptables pcn-iptables)
 add_service(transparenthelloworld pcn-transparent-helloworld)
 add_service(synflood pcn-synflood)
 add_service(packetcapture pcn-packetcapture)
+add_service(dynmon pcn-dynmon)
 
 # save string to create code that load the services
 SET_PROPERTY(GLOBAL PROPERTY LOAD_SERVICES_ ${LOAD_SERVICES})

src/services/pcn-dynmon/.swagger-codegen-ignore

@@ -0,0 +1,13 @@
+# Swagger Codegen Ignore
+# Generated by swagger-codegen https://github.com/swagger-api/swagger-codegen
+
+# Use this file to prevent files from being overwritten by the generator.
+
+.swagger-codegen-ignore
+
+src/*.cpp
+src/*.h
+
+!src/*Interface.h
+!src/*JsonObject.h
+!src/*JsonObject.cpp

src/services/pcn-dynmon/CMakeLists.txt

@@ -0,0 +1,5 @@
+cmake_minimum_required (VERSION 3.2)
+
+set (CMAKE_CXX_STANDARD 11)
+
+add_subdirectory(src)

src/services/pcn-dynmon/cmake/LoadFileAsVariable.cmake

@@ -0,0 +1,52 @@
+# Loads the contents of a file into a std::string variable
+#
+# It creates a header file in ${CMAKE_CURRENT_BINARY_DIR}/${file}.h
+# that wrapps the contents of the file in a std::string using the raw
+# string literal feature of C++11. The user needs to include that file
+# into the source code in order to see the variable.
+#
+# parameters:
+#  target: target to add a dependency on file
+#  file: file to be loaded
+#  variable_name: name variable where the file is loaded
+#
+# example:
+#  load_file_as_variable(my-lib resource.c my_resource)
+# Creates a resource.h in CMAKE_CURRENT_BINARY_DIR with a string variable
+# my_resource with the contents of resource.c
+# A dependency in resource.c is added to my-lib
+
+function(load_file_as_variable target file variable_name)
+    get_filename_component(file_name ${file} NAME_WE)
+    get_filename_component(file_dir ${file} DIRECTORY)
+
+    set(new_path ${file_dir}/${file_name}.h)
+
+    add_custom_command(
+            OUTPUT
+            ${CMAKE_CURRENT_BINARY_DIR}/${new_path}
+            COMMAND
+            mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/${file_dir}
+            COMMAND
+            echo "#pragma once" > ${CMAKE_CURRENT_BINARY_DIR}/${new_path}
+            COMMAND
+            echo "#include <string>" >> ${CMAKE_CURRENT_BINARY_DIR}/${new_path}
+            COMMAND
+            echo "const std::string ${variable_name} = R\"POLYCUBE_DP(" >> ${CMAKE_CURRENT_BINARY_DIR}/${new_path}
+            COMMAND
+            cat ${CMAKE_CURRENT_SOURCE_DIR}/${file} >> ${CMAKE_CURRENT_BINARY_DIR}/${new_path}
+            COMMAND
+            cmake -E echo ")POLYCUBE_DP\";" >> ${CMAKE_CURRENT_BINARY_DIR}/${new_path}
+            DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${file}
+            VERBATIM
+    )
+
+    string(REPLACE "/" "-" path_replaced ${new_path})
+
+    add_custom_target(
+            generate_${path_replaced}
+            DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${new_path}
+    )
+
+    add_dependencies(${target} generate_${path_replaced})
+endfunction()

src/services/pcn-dynmon/datamodel/dynmon.yang

@@ -0,0 +1,137 @@
+module dynmon {
+    yang-version 1.1;
+    namespace "http://polycube.network/dynmon";
+    prefix "dynmon";
+
+    import polycube-base {
+        prefix "polycube-base";
+    }
+    import polycube-transparent-base {
+        prefix "polycube-transparent-base";
+    }
+    import "ietf-inet-types" {
+        prefix "inet";
+    }
+    import "ietf-yang-types" {
+        prefix "yang";
+    }
+
+    organization
+      "Polycube open source project";
+    description
+      "YANG data model for the Polycube Dynamic Traffic Monitor transparent service";
+    polycube-base:service-description "Dynamic Traffic Monitor transparent service";
+    polycube-base:service-version "1.0";
+    polycube-base:service-name "dynmon";
+    polycube-base:service-min-kernel-version "4.14.0";
+
+    uses polycube-transparent-base:transparent-base-yang-module;
+
+    container dataplane {
+        description
+          "Running dataplane";
+        leaf name {
+            type string;
+            description
+              "eBPF program name";
+            polycube-base:init-only-config;
+        }
+        leaf code {
+            type string;
+            description
+              "eBPF source code";
+            polycube-base:init-only-config;
+        }
+        list metrics {
+            key "name";
+            description
+              "Exported Metric";
+            polycube-base:init-only-config;
+            leaf name {
+                type string;
+                description
+                  "Name of the metric (e.g., number of HTTP requests)";
+                polycube-base:init-only-config;
+            }
+            leaf map-name {
+                type string;
+                description
+                  "Corrisponding eBPF map name";
+                polycube-base:init-only-config;
+            }
+            container open-metrics-metadata {
+                presence
+                  "The metric will be exported with the OpenMetric format";
+                description
+                  "Open-Metrics metadata";
+                polycube-base:init-only-config;
+                leaf help {
+                    type string;
+                    description
+                      "Metric description";
+                    polycube-base:init-only-config;
+                }
+                leaf type {
+                    type enumeration{
+                        enum Counter;
+                        enum Gauge;
+                        enum Histogram;
+                        enum Summary;
+                        enum Untyped;
+                    }
+                    description
+                      "Metric type";
+                    polycube-base:init-only-config;
+                }
+                list labels {
+                    key "name";
+                    description
+                      "Label attached to the metric";
+                    polycube-base:init-only-config;
+                    leaf name {
+                        type string;
+                        description
+                          "Name of the label (e.g., 'method')";
+                        polycube-base:init-only-config;
+                    }
+                    leaf value {
+                        type string;
+                        description
+                          "Label value (e.g., 'POST')";
+                        polycube-base:init-only-config;
+                    }
+                }
+            }
+        }
+    }
+    list metrics {
+        config false;
+        key "name";
+        description
+          "Collected metrics";
+        leaf name {
+            config false;
+            type string;
+            description
+              "Name of the metric (e.g, number of HTTP requests)";
+        }
+        leaf value {
+            config false;
+            type string;
+            description
+              "Value of the metric";
+        }
+        leaf timestamp {
+            config false;
+            type int64;
+            description
+              "Timestamp";
+        }
+    }
+    leaf open-metrics {
+        config false;
+        type string;
+        description
+          "Collected metrics in OpenMetrics Format";
+    }
+}

src/services/pcn-dynmon/examples/README.md

@@ -0,0 +1,20 @@
+# Monitoring examples
+
+This folder contains a set of monitoring dataplanes, which can be used as examples for the Dynamic Monitoring service.
+
+- [packet_counter.json](packet_counter.json):
+  - the `packets_total` metric represents the number of packets that have traversed the attached network interface
+
+- [ntp_packets_counter.json](ntp_packets_counter.json):
+  - the `ntp_packets_total` metric represents the number of NTP packets that have traversed the attached network interface
+
+- [ntp_packets_ntp_mode_private_counters.json](ntp_packets_ntp_mode_private_counters.json):
+  - the `ntp_packets_total` metric represents the number of NTP packets that have traversed the attached network interface;
+  - the `ntp_mode_private_packets_total` metric represents the number of NTP packets with NTP_MODE = 7 (MODE_PRIVATE) that have traversed the attached network interface
+
+All counters are *incremental*, hence their values are monotonically increasing.
+
+
+Unfortunately the dataplane code (eBPF restricted C) contained in the above JSONs is not easy to read for a human, due to the formatting limitations of the JSON format. A more human friendly version can be produced by unescaping the code by using [this online free tool](https://www.freeformatter.com/json-escape.html).
+
+The same tool can be used also to escape any multiline code strings in order to create new valid injectable dataplanes.

src/services/pcn-dynmon/examples/ntp_packets_counter.json

@@ -0,0 +1,24 @@
+{
+    "name": "NTP Amplification BUA probe",
+    "code": "\r\n#include <uapi/linux/ip.h>\r\n#include <uapi/linux/udp.h>\r\n\r\n#define IP_PROTO_UDP 17\r\n#define NTP_PORT 123\r\n\r\nstruct eth_hdr {\r\n    __be64 dst : 48;\r\n    __be64 src : 48;\r\n    __be16 proto;\r\n} __attribute__((packed));\r\n\r\nBPF_ARRAY(NTP_PACKETS_COUNTER, uint64_t,1);\r\n\r\nstatic __always_inline\r\nint handle_rx(struct CTXTYPE *ctx, struct pkt_metadata *md) {\r\n    /*Parsing L2*/\r\n    void *data = (void *) (long) ctx->data;\r\n    void *data_end = (void *) (long) ctx->data_end;\r\n    struct eth_hdr *ethernet = data;\r\n    if (data + sizeof(*ethernet) > data_end)\r\n        return RX_OK;\r\n\r\n    if (ethernet->proto != bpf_htons(ETH_P_IP))\r\n        return RX_OK;\r\n\r\n    /*Parsing L3*/\r\n    struct iphdr *ip = data + sizeof(struct eth_hdr);\r\n    if (data + sizeof(struct eth_hdr) + sizeof(*ip) > data_end)\r\n        return RX_OK;\r\n    if ((int) ip->version != 4)\r\n        return RX_OK;\r\n\r\n    if (ip->protocol != IP_PROTO_UDP)\r\n        return RX_OK;\r\n\r\n    /*Parsing L4*/\r\n    uint8_t ip_header_len = 4 * ip->ihl;\r\n    struct udphdr *udp = data + sizeof(*ethernet) + ip_header_len;\r\n    if (data + sizeof(*ethernet) + ip_header_len + sizeof(*udp) > data_end)\r\n        return RX_OK;\r\n\r\n    if (udp->source == bpf_htons(NTP_PORT) || udp->dest == bpf_htons(NTP_PORT)) {\r\n        pcn_log(ctx, LOG_TRACE, \"%I:%P\\t-> %I:%P\", ip->saddr,udp->source,ip->daddr,udp->dest);\r\n        unsigned int key = 0;\r\n        uint64_t * ntp_pckts_counter = NTP_PACKETS_COUNTER.lookup(&key);\r\n        if (!ntp_pckts_counter)\r\n            pcn_log(ctx, LOG_ERR, \"[NTP_AMP_BUA] Unable to find NTP_PACKETS_COUNTER map\");\r\n        else\r\n            *ntp_pckts_counter+=1;\r\n    }\r\n\r\n    return RX_OK;\r\n}",
+    "metrics": [
+        {
+            "name": "ntp_packets_total",
+            "map-name": "NTP_PACKETS_COUNTER",
+            "open-metrics-metadata": {
+                "help": "This metric represents the number of NTP packets that has traveled through this probe.",
+                "type": "counter",
+                "labels": [
+                    {
+                        "name": "IP_PROTO",
+                        "value": "UDP"
+                    },
+                    {
+                        "name": "L4",
+                        "value": "NTP"
+                    }
+                ]
+            }
+        }
+    ]
+}