Syncup 6.0/release with master

.github/workflows/bcc-test.yml

@@ -0,0 +1,94 @@
+name: BCC Build and tests
+
+on: push
+
+jobs:
+  test_bcc:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-16.04, ubuntu-18.04] # 16.04.4 release has 4.15 kernel
+                                         # 18.04.3 release has 5.0.0 kernel
+        env:
+        - TYPE: Debug
+          PYTHON_TEST_LOGFILE: critical.log
+        - TYPE: Release
+          PYTHON_TEST_LOGFILE: critical.log
+    steps:
+    - uses: actions/checkout@v2
+    - name: System info
+      run: |
+        uname -a
+        ip addr
+    - name: Build docker container with all deps
+      run: |
+        docker build -t bcc-docker -f Dockerfile.tests .
+    - name: Run bcc build
+      env: ${{ matrix.env }}
+      run: |
+        /bin/bash -c \
+                   "docker run --privileged \
+                   --pid=host \
+                   -v $(pwd):/bcc \
+                   -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                   -v /lib/modules:/lib/modules:ro \
+                   -v /usr/src:/usr/src:ro \
+                   -v /usr/include/linux:/usr/include/linux:ro \
+                   bcc-docker \
+                   /bin/bash -c \
+                   'mkdir -p /bcc/build && cd /bcc/build && \
+                    cmake -DCMAKE_BUILD_TYPE=${TYPE} .. && make -j9'"
+    - name: Run bcc's cc tests
+      env: ${{ matrix.env }}
+      # tests are wrapped with `script` as a hack to get a TTY as github actions doesn't provide this
+      # see https://github.com/actions/runner/issues/241
+      run: |
+        script -e -c /bin/bash -c \
+        "docker run -ti \
+                    --privileged \
+                    --network=host \
+                    --pid=host \
+                    -v $(pwd):/bcc \
+                    -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                    -v /lib/modules:/lib/modules:ro \
+                    -v /usr/src:/usr/src:ro \
+                    -e CTEST_OUTPUT_ON_FAILURE=1 \
+                    bcc-docker \
+                    /bin/bash -c \
+                    '/bcc/build/tests/wrapper.sh \
+                        c_test_all sudo /bcc/build/tests/cc/test_libbcc'"
+
+    - name: Run all tests
+      env: ${{ matrix.env }}
+      run: |
+        script -e -c /bin/bash -c \
+        "docker run -ti \
+                    --privileged \
+                    --network=host \
+                    --pid=host \
+                    -v $(pwd):/bcc \
+                    -v /sys/kernel/debug:/sys/kernel/debug:rw \
+                    -v /lib/modules:/lib/modules:ro \
+                    -v /usr/src:/usr/src:ro \
+                    -e CTEST_OUTPUT_ON_FAILURE=1 \
+                    bcc-docker \
+                    /bin/bash -c \
+                    'cd /bcc/build && \
+                     make test PYTHON_TEST_LOGFILE=$PYTHON_TEST_LOGFILE ARGS=-V'"
+
+    - name: Check critical tests
+      env: ${{ matrix.env }}
+      run: |
+        critical_count=$(grep @mayFail tests/python/critical.log | wc -l)
+        echo "There were $critical_count critical tests skipped with @mayFail:"
+        grep -A2 @mayFail tests/python/critical.log
+
+    - uses: actions/upload-artifact@v1
+      with:
+        name: critical-tests-${{ matrix.env['TYPE'] }}-${{ matrix.os }}
+        path: tests/python/critical.log
+
+# To debug weird issues, you can add this step to be able to SSH to the test environment
+#     https://github.com/marketplace/actions/debugging-with-tmate
+#    - name: Setup tmate session
+#      uses: mxschmitt/action-tmate@v1

.gitignore

@@ -9,4 +9,6 @@
 /build/
 cmake-build-debug
 debian/**/*.log
+*critical.log
 obj-x86_64-linux-gnu
+examples/cgroupid/cgroupid

.travis.yml

@@ -1,8 +1,20 @@
-language: generic
-install:
-  - sudo apt-get install -y python-pip
-  - sudo pip install pep8
-script:
-  - set -euo pipefail
-  - ./scripts/check-helpers.sh
-  - ./scripts/py-style-check.sh
+language: python
+matrix:
+  include:
+    - name: "Check helpers on Python 2.7"
+      python: 2.7
+      script: ./scripts/check-helpers.sh
+    - name: "Python style check on Python 2.7"
+      python: 2.7
+      script: ./scripts/py-style-check.sh
+    - name: "flake8 lint on Python 2.7"      
+      python: 2.7
+      script: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+    - name: "flake8 lint on Python 3.7"
+      dist: xenial  # required for Python >= 3.7
+      python: 3.7
+      script: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+  allow_failures:
+    - name: "Check helpers on Python 2.7"
+before_install: pip install --upgrade pip
+install: pip install flake8

CMakeLists.txt

@@ -27,17 +27,27 @@ option(ENABLE_RTTI "Enable compiling with real time type information" OFF)
 option(ENABLE_LLVM_SHARED "Enable linking LLVM as a shared library" OFF)
 option(ENABLE_CLANG_JIT "Enable Loading BPF through Clang Frontend" ON)
 option(ENABLE_USDT "Enable User-level Statically Defined Tracing" ON)
+option(ENABLE_MAN "Build man pages" ON)
 CMAKE_DEPENDENT_OPTION(ENABLE_CPP_API "Enable C++ API" ON "ENABLE_USDT" OFF)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 
+if (CMAKE_USE_LIBBPF_PACKAGE)
+  find_package(LibBpf)
+endif()
+
 if(NOT PYTHON_ONLY AND ENABLE_CLANG_JIT)
 find_package(BISON)
 find_package(FLEX)
 find_package(LLVM REQUIRED CONFIG)
 message(STATUS "Found LLVM: ${LLVM_INCLUDE_DIRS} ${LLVM_PACKAGE_VERSION}")
 find_package(LibElf REQUIRED)
 
+if(CLANG_DIR)
+  set(CMAKE_FIND_ROOT_PATH "${CLANG_DIR}")
+  include_directories("${CLANG_DIR}/include")
+endif()
+
 # clang is linked as a library, but the library path searching is
 # primitively supported, unlike libLLVM
 set(CLANG_SEARCH "/opt/local/llvm/lib;/usr/lib/llvm-3.7/lib;${LLVM_LIBRARY_DIRS}")
@@ -85,7 +95,7 @@ if (USINGISYSTEM AND GCC_VERSION VERSION_LESS 6.0)
 endif()
 
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 
 endif(NOT PYTHON_ONLY AND ENABLE_CLANG_JIT)
 
@@ -96,7 +106,9 @@ add_subdirectory(src)
 add_subdirectory(introspection)
 if(ENABLE_CLANG_JIT)
 add_subdirectory(examples)
+if(ENABLE_MAN)
 add_subdirectory(man)
+endif(ENABLE_MAN)
 add_subdirectory(tests)
 add_subdirectory(tools)
 endif(ENABLE_CLANG_JIT)

Dockerfile.tests

@@ -0,0 +1,69 @@
+# Using ubuntu 19.04 for newer `unshare` command used in tests
+FROM ubuntu:19.04
+
+ARG LLVM_VERSION="8"
+ENV LLVM_VERSION=$LLVM_VERSION
+
+RUN apt-get update && apt-get install -y curl gnupg &&\
+    llvmRepository="\n\
+deb http://apt.llvm.org/disco/ llvm-toolchain-disco main\n\
+deb-src http://apt.llvm.org/disco/ llvm-toolchain-disco main\n\
+deb http://apt.llvm.org/disco/ llvm-toolchain-disco-${LLVM_VERSION} main\n\
+deb-src http://apt.llvm.org/disco/ llvm-toolchain-disco-${LLVM_VERSION} main\n" &&\
+    echo $llvmRepository >> /etc/apt/sources.list && \
+    curl -L https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+
+RUN apt-get update && apt-get install -y \
+      bison \
+      binutils-dev \
+      cmake \
+      flex \
+      g++ \
+      git \
+      kmod \
+      wget \
+      libelf-dev \
+      zlib1g-dev \
+      libiberty-dev \
+      libbfd-dev \
+      libedit-dev \
+      clang-${LLVM_VERSION} \
+      libclang-${LLVM_VERSION}-dev \
+      libclang-common-${LLVM_VERSION}-dev \
+      libclang1-${LLVM_VERSION} \
+      llvm-${LLVM_VERSION} \
+      llvm-${LLVM_VERSION}-dev \
+      llvm-${LLVM_VERSION}-runtime \
+      libllvm${LLVM_VERSION} \
+      systemtap-sdt-dev \
+      sudo \
+      iproute2 \
+      python3 \
+      python3-pip \
+      python-pip \
+      ethtool \
+      arping \
+      netperf \
+      iperf \
+      iputils-ping \
+      bridge-utils \
+      libtinfo5 \
+      libtinfo-dev
+
+RUN pip3 install pyroute2 netaddr
+RUN pip install pyroute2 netaddr
+
+# FIXME this is faster than building from source, but it seems there is a bug
+# in probing libruby.so rather than ruby binary
+#RUN apt-get update -qq && \
+#    apt-get install -y software-properties-common && \
+#    apt-add-repository ppa:brightbox/ruby-ng && \
+#    apt-get update -qq && apt-get install -y ruby2.6 ruby2.6-dev
+
+RUN wget -O ruby-install-0.7.0.tar.gz \
+         https://github.com/postmodern/ruby-install/archive/v0.7.0.tar.gz && \
+    tar -xzvf ruby-install-0.7.0.tar.gz && \
+    cd ruby-install-0.7.0/ && \
+    make install
+
+RUN ruby-install --system ruby 2.6.0 -- --enable-dtrace

FAQ.txt

@@ -11,7 +11,7 @@ A: make sure to 'make install' and add the directory
    export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
 
 Q: hello_world.py fails with:
-   ImportError: No module named bpf
+   ImportError: No module named bcc
 A: checkout "sudo make install" output to find out bpf package installation site,
    add it to the PYTHONPATH env variable before running the program.
    sudo bash -c 'PYTHONPATH=/usr/lib/python2.7/site-packages python examples/hello_world.py'
@@ -21,6 +21,22 @@ Q: hello_world.py still fails with:
    Exception: Failed to load BPF program hello
 A: sudo
 
+Q: hello_world.py fails with
+   bpf: Failed to load program: Operation not permitted
+   despite running as root, and strace shows each `bpf()` system call failing with an EPERM.
+A: The so-called Kernel lockdown might be the root cause. Try disabling it with the so-called
+   sysrq mechanism:
+       echo 1 > /proc/sys/kernel/sysrq
+       echo x > /proc/sysrq-trigger
+   Also see https://github.com/iovisor/bcc/issues/2525
+
+   If you have Secure Boot enabled you need to press Alt-PrintScr-x on the keyboard instead:
+   ```
+   This sysrq operation is disabled from userspace.
+   sysrq: Disabling Secure Boot restrictions
+   Lifting lockdown
+   ```
+
 Q: How do I fulfill the Linux kernel version requirement?
 A: You need to obtain a recent version of the Linux source code
    (please look at the README for the exact version), enable the