diff --git a/.gitattributes b/.gitattributes index a9d952d..c2cbd5d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ bin/bpftool filter=lfs diff=lfs merge=lfs -text +*.svg filter=lfs diff=lfs merge=lfs -text + diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fef86c..7f56400 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,8 +71,8 @@ write_basic_package_version_file( "${version_config}" COMPATIBILITY SameMajorVersion ) -# set(INCLUDE_INSTALL_DIR include/) -# set(LIB_INSTALL_DIR lib/ ) +set(INCLUDE_INSTALL_DIR include/) +set(LIB_INSTALL_DIR lib/ ) # Configure 'Config.cmake' # Use variables: # * TARGETS_EXPORT_NAME diff --git a/Makefile b/Makefile index eded8cd..bb79c6f 100644 --- a/Makefile +++ b/Makefile @@ -85,8 +85,8 @@ run-redirect-map-sample: all ## Build all and run BPF XDP redirect sample run-control-plane-tests: force-xdp-deload ## Run ControlPlaneTests ./tests/scripts/run_test ControlPlaneTests -run-session-manager-tests: force-xdp-deload ## Run SessionManagerTests - sudo bash ./tests/scripts/run_test UPFProgramTests +run-session-manager-tests: force-xdp-deload ## Run SessionManagerTests + sudo -E bash ./tests/scripts/run_test UPFProgramTests rerun: force-xdp-deload run ## Build all and run BPF XDP UPF @@ -94,7 +94,7 @@ dut-run: ## Run ControlPlaneTests on DUT sudo ./bin/ControlPlaneTests force-xdp-deload: ## Kill all and force deload XDP programs - sudo bash tests/scripts/force_xdp_deload + sudo -E bash tests/scripts/force_xdp_deload trex: ## Install, deploy configuration and run t-rex on remote server tests/scripts/install_trex_remote @@ -114,6 +114,9 @@ tmux: ## Create a test session using tmux docker-build: ## Build development image docker/build_docker +docker-build-ubuntu-18.04: ## Build development image + docker/build_docker ubuntu:18.04 + docker-run: ## Run development container docker/run_docker diff --git a/README.md b/README.md index d0f6f42..a408b86 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,59 @@ -User plane function using BPF and XDP for mobile core network (4G/5G). +# 5G/LTE User Plane using eBPF/XDP -## Usage -### Build Docker Image +An open source C++ library powered by eBPF/XDP for user plane in mobile core network (5G/LTE). -Copy the keys to jump server and run `make docker-build` -### Testing +This project is based on the following 3GPP Technical Specification: +- LTE; 5G; Interface between the Control Plane and the User Plane nodes (3GPP TS 29.244 version 16.5.0 Release 16) +- 5G; System architecture for the 5G System (5GS) (3GPP TS 23.501 version 16.5.0 Release 16) + +The main goal is to enable in-kernel fast packet processing in third-party UPF/5G or SPGWu/LTE components in order to: +1. Boost them for those which does not have any fast packet processing enabled, or +1. Co-locate them with other fast packet processing solutions (e.g. DPDK) + +## Design + +The library is divided in two paths: +- Slow path: An user space layer responsible to receive requests from the third-party UPF/SPGWu components to manage PFCP sessions and eBPF programs lifecycle +- Fast path: A kernel space layer representing by eBPF/XDP programs responsible to handle the user traffic + +The high level design is shown in figure below. + +drawing + +The library has a component, called `PFCP Sesssion Manager`, which is a C++ API responsible to manage PFCP (Packet Forwarding Control Protocol) sessions. For each session, there is an eBPF program that represents the PFCP context in the fast path. These programs are managed by `eBPF Program Manager` component. The fast path is composed by three main function: parser, traffic classifier and traffic forwarder. The image below shows this in more detail. + +drawing + +## Features + +As described in 3GPP TS 29.244, the Information Elements (IEs) are part of the PFCP context. The PFCP context is created by sending PFCP Session Establishment Request message. The main IEs supported in this project are: +- PDR (Packet Detection Rule) +- FAR (Forwarding Action Rule) -![setup-for-GPDU-debug-issue-2-upf-bpf](https://user-images.githubusercontent.com/42647168/86470179-4486cc80-bd11-11ea-8f55-fee848b12e11.png) +The logical data model between PFCP Session and IEs is shown in the image below. For more detail, see 3GPP TS 29.244 version 16.5.0 Release 16. -- Open terminal in `Linux #0` -- Configure veth pairs with `make config-veth-pair` in `Linux #0` -- Install the spdlog library with `make setup` -- Build, load and run program with `make run` in `Linux #0` -- Login remote Scapy `ssh -X navarro@192.168.15.7 "cd /work/mestrado/scapy/; sudo -S ./run_scapy"` -- Generate GTP G-PDU using [Scapy](https://github.com/secdev/scapy) with `gtpu-traffic-generator.py` in `Linux #1` -- Open other terminal tab in `Linux #0` -- Check logs: `sudo cat /sys/kernel/debug/tracing/trace | grep "GPDU"` in `Linux #0` +drawing + +## Usage + +Copy the keys to jump server and run `make docker-build` ### Make Commands -``` -- all Build all -- clean Clean all build files -- all-verbose Build all in verbose mode -- config-veth-pair Config veth pair. It must be run before `run-*` targets -- run-hello-world-samples Build all and run BPF XDP hello world sample -- run-redirect-map-sample Build all and run BPF XDP redirect sample -- run Build all and run BPF XDP UPF -- run-scapy Run scapy for packet manipulation -- force-xdp-deload Force deload XDP programs -``` -## Dependencies +## Main Dependencies -Upee +Core - libbpf -- libelf -- libz +- bpftool +- spdlog - clang >= version 3.4.0 - llvm >= version 3.7.1 - kernel-headers => version 5.3 -- bpftool +- cmake >= 3.16 Test - scapy v2.4.3 -- spdlog - gtest - sysstat - trex v2.86 @@ -71,8 +80,9 @@ The directory structure was created based on this [notes](https://blogs.oracle.c If you faced the problem below, create a symbolic from `libc.a -> liblibc.a` `No such file or directory: b'liblibc.a'` -## Contact +## Creator and Lead Developer -Thiago Navarro -Email: navarro.ime@gmail.com -Twitter: navarr0thiag0 +Thiago Arruda Navarro do Amaral +- Mail: +- GitHub: [@navarrothiago](https://github.com/navarrothiago/) +- Twitter: [@navarr0thiag0](https://twitter.com/navarr0thiag0) diff --git a/docker/Dockerfile b/docker/Dockerfile index c167e40..92110c9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,5 @@ -FROM ubuntu:20.04 +ARG BASE_DEV_IMAGE=ubuntu:20.04 +FROM $BASE_DEV_IMAGE ARG UNAME=oai-spgwu WORKDIR /tmp/$UNAME @@ -72,12 +73,11 @@ RUN apt-get update && \ apt-get install -y \ tmux -# It is comment due to ubuntu 20.04 base image. -# If you use another one, uncomment the lines below. -# RUN apt-get update && \ -# apt-get -y install libssl-dev && \ -# wget -c https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2.tar.gz -O - | tar -xz && \ -# cd cmake-3.19.2 && ./bootstrap && make && make install +# Ubuntu 20.04 already support cmake 3.19. +RUN if [ $BASE_DEV_IMAGE != "ubuntu:20.04" ]; then apt-get update && \ + apt-get -y install libssl-dev && \ + wget -c https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2.tar.gz -O - | tar -xz && \ + cd cmake-3.19.2 && ./bootstrap && make -j4 && make install; fi # Add your local bashrc ADD .bashrc /home/$USERNAME/ @@ -141,17 +141,16 @@ RUN apt-get update && \ RUN pip3 install ipykernel RUN pip3 install matplotlib -RUN pip3 install paramiko RUN pip3 install autopep8 RUN apt-get update && \ apt-get install -y \ rsync -ARG ENV_FILENAME="env.sh" -ADD ${ENV_FILENAME} ${ENV_FILENAME} +# TODO: pass as ARG +RUN echo "ENV_FILE=/workspaces/env.sh" >> /etc/environment # Omit if you want to keep the default as root. # USER $USERNAME -CMD ${ENV_FILENAME} && /bin/bash +CMD /bin/bash diff --git a/docker/build_docker b/docker/build_docker index 26d8015..f080246 100755 --- a/docker/build_docker +++ b/docker/build_docker @@ -9,6 +9,9 @@ main() { local -r context="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${context}"/../env.sh local env_file_name=env.sh + local base_dev_image="${1-ubuntu:20.04}" + + echo "Base image "${base_dev_image}"" # Copy configuration files and ssh keys to the docker context folder. cp "${GIT_CONFIG}" "${context}" @@ -20,13 +23,14 @@ main() { cp "${context}"/../env.sh "${context}" # Build dockerfile. - docker build --tag="${IMAGE}":"${VERSION}" --rm -f "${context}"/"${DOCKERFILE}" "${context}" \ + docker build --tag="${IMAGE_TAG}":"${IMAGE_VERSION}" --rm -f "${context}"/"${DOCKERFILE}" "${context}" \ --build-arg ENV_FILENAME="${env_file_name}" \ --build-arg DUT_SERVER_IP="${DUT_IP}" \ --build-arg TREX_SERVER_IP="${TREX_SERVER_IP}" \ --build-arg JUMP_SERVER_USERNAME="${JUMP_SERVER_USERNAME}" \ --build-arg JUMP_SERVER_IP="${JUMP_SERVER_IP}" \ - --build-arg JUMP_SERVER_PORT="${JUMP_SERVER_PORT}" + --build-arg JUMP_SERVER_PORT="${JUMP_SERVER_PORT}" \ + --build-arg BASE_DEV_IMAGE="${base_dev_image}" # Remove configuration files and ssh keys from the docker context folder. rm "${context}"/.gitconfig @@ -35,6 +39,10 @@ main() { rm "${context}"/"${SSH_CONFIG_FILE}" rm "${context}"/.bashrc rm "${context}"/env.sh + + echo + echo "Image "${IMAGE_TAG}" was build with base image "${base_dev_image}"" + echo } main "$@" diff --git a/docker/run_docker b/docker/run_docker index 786d75f..b39777b 100755 --- a/docker/run_docker +++ b/docker/run_docker @@ -12,7 +12,7 @@ main() { --env-file "${dirname}"/../env.sh \ --volume "${dirname}"/../:"${WORKSPACE}" \ --privileged \ - --workdir "${WORKSPACE}" "${IMAGE}":"${VERSION}" \ + --workdir "${WORKSPACE}" "${IMAGE_TAG}":"${IMAGE_VERSION}" \ /bin/bash exit 0 diff --git a/env.sh b/env.sh index 0c0df75..67ef704 100644 --- a/env.sh +++ b/env.sh @@ -9,8 +9,8 @@ NUM_THREADS=4 # Docker environment variable. USERNAME=oai-spgwu -IMAGE=upee -VERSION=v1.0 +IMAGE_TAG=upee +IMAGE_VERSION=v1.0 DOCKERFILE=Dockerfile SSH_FOLDER=~/.ssh SSH_PUBLIC_KEY_FILE=id_rsa.pub @@ -91,4 +91,4 @@ API_HTTP_PORT=80 # Programs name API_PROGRAM_NAME=api -PYTHONPATH=/workspaces/tests/trex/trex_client/interactive/ \ No newline at end of file +PYTHONPATH=/workspaces/tests/trex/trex_client/interactive/ diff --git a/img/up-ebpf-xdp-high-level.svg b/img/up-ebpf-xdp-high-level.svg new file mode 100644 index 0000000..f42d505 --- /dev/null +++ b/img/up-ebpf-xdp-high-level.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c778f1f239457a999e85b2dc3eb358aee79ab8d23987f225df1df4840e408a3a +size 19623 diff --git a/img/up-ebpf-xdp-high-level2.svg b/img/up-ebpf-xdp-high-level2.svg new file mode 100644 index 0000000..3f61d0c --- /dev/null +++ b/img/up-ebpf-xdp-high-level2.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:741fee9c8285098427df2ee4d6c6551db27bd17264af1a0ebaa072b69232a08e +size 26621 diff --git a/img/up-ebpf-xdp-ies.svg b/img/up-ebpf-xdp-ies.svg new file mode 100644 index 0000000..7420f55 --- /dev/null +++ b/img/up-ebpf-xdp-ies.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b48bdb2f1e6a78e56f5f7f1e8e36182e3ce22154422a24fd933bc3fcba733671 +size 45206 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f3320a2..963b2b6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -64,8 +64,8 @@ install( # Headers: # * ./*.h -> /include/*.h -# install( -# DIRECTORY "./" -# DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" -# FILES_MATCHING PATTERN "*.h" -# ) +install( + DIRECTORY "./" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + FILES_MATCHING PATTERN "*.h" +) diff --git a/src/programs/ProgramLifeCycle.hpp b/src/programs/ProgramLifeCycle.hpp index 4d99ed4..6e92edc 100644 --- a/src/programs/ProgramLifeCycle.hpp +++ b/src/programs/ProgramLifeCycle.hpp @@ -108,8 +108,10 @@ ProgramLifeCycle::ProgramLifeCycle(std::functionlink("xdp_entry_point", mUDPInterface.c_str()); + LOG_DBG("Link GTP interface to interface {}", mGTPInterface.c_str()) mpLifeCycle->link("xdp_entry_point", mGTPInterface.c_str()); } diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..fe845de --- /dev/null +++ b/tests/README.md @@ -0,0 +1,12 @@ +# Testing + +![setup-for-GPDU-debug-issue-2-upf-bpf](https://user-images.githubusercontent.com/42647168/86470179-4486cc80-bd11-11ea-8f55-fee848b12e11.png) + +- Open terminal in `Linux #0` +- Configure veth pairs with `make config-veth-pair` in `Linux #0` +- Install the spdlog library with `make setup` +- Build, load and run program with `make run` in `Linux #0` +- Login remote Scapy `ssh -X navarro@192.168.15.7 "cd /work/mestrado/scapy/; sudo -S ./run_scapy"` +- Generate GTP G-PDU using [Scapy](https://github.com/secdev/scapy) with `gtpu-traffic-generator.py` in `Linux #1` +- Open other terminal tab in `Linux #0` +- Check logs: `sudo cat /sys/kernel/debug/tracing/trace | grep "GPDU"` in `Linux #0` \ No newline at end of file diff --git a/tests/scripts/config_veth_pair b/tests/scripts/config_veth_pair index 46fed59..22eaf5e 100755 --- a/tests/scripts/config_veth_pair +++ b/tests/scripts/config_veth_pair @@ -12,7 +12,7 @@ create_namespace() { local -r filename="${dirname}/$(basename "${BASH_SOURCE[0]}")" # TODO navarrothiago - check if it is recommend to call env.sh here. - source "${dirname}"/../../env.sh + source "${ENV_FILE}" local nr_cpu=$(nproc) local namespace=$1 diff --git a/tests/scripts/deploy_package b/tests/scripts/deploy_package index bbbf433..a81b284 100755 --- a/tests/scripts/deploy_package +++ b/tests/scripts/deploy_package @@ -11,7 +11,7 @@ main() { local -r dirname="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" local -r filename="${dirname}/$(basename "${BASH_SOURCE[0]}")" - source "${dirname}"/../../env.sh + source "${ENV_FILE}" echo echo "Copy package with binary and libraries." diff --git a/tests/scripts/deploy_trex_config b/tests/scripts/deploy_trex_config index e009e67..152de89 100755 --- a/tests/scripts/deploy_trex_config +++ b/tests/scripts/deploy_trex_config @@ -10,7 +10,7 @@ main() { local -r dirname="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - source "${dirname}"/../../env.sh + source "${ENV_FILE}" scp -r "${LOCAL_CONFIG_DIR}" "${TREX_SERVER_SSH}":"${TREX_SERVER_UPLOAD_DIR}" scp -r "${LOCAL_TRAFFIC_DIR}" "${TREX_SERVER_SSH}":"${TREX_SERVER_UPLOAD_DIR}" diff --git a/tests/scripts/force_xdp_deload b/tests/scripts/force_xdp_deload index 81ede94..0c94392 100755 --- a/tests/scripts/force_xdp_deload +++ b/tests/scripts/force_xdp_deload @@ -7,7 +7,7 @@ main() { set -o nounset local -r dirname="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - source "${dirname}"/../../env.sh + source "${ENV_FILE}" local gtp_interface=${1:-veth0} local udp_interface=${2:-veth1} diff --git a/tests/scripts/install_trex_remote b/tests/scripts/install_trex_remote index aa5d5c6..c700f0a 100755 --- a/tests/scripts/install_trex_remote +++ b/tests/scripts/install_trex_remote @@ -11,7 +11,7 @@ main() { local -r dirname="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - source "${dirname}"/../../env.sh + source "${ENV_FILE}" # Execute local script (install_trex) on the server. diff --git a/tests/scripts/run_sample b/tests/scripts/run_sample index 4af973e..8988096 100755 --- a/tests/scripts/run_sample +++ b/tests/scripts/run_sample @@ -6,7 +6,7 @@ main() { set -o nounset local -r dirname="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - source "${dirname}"/../../env.sh + source "${ENV_FILE}" local sample_binary_name=$1 diff --git a/tests/scripts/run_test b/tests/scripts/run_test index 1dc4386..71fa09e 100755 --- a/tests/scripts/run_test +++ b/tests/scripts/run_test @@ -5,7 +5,7 @@ main() { set -o pipefail local -r dirname="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - source "${dirname}"/../../env.sh + source "${ENV_FILE}" local test_binary=$1 local gtp_interface=${2:-veth0} diff --git a/tests/scripts/run_trex_server b/tests/scripts/run_trex_server index 2ea7d15..d92fc27 100755 --- a/tests/scripts/run_trex_server +++ b/tests/scripts/run_trex_server @@ -10,7 +10,7 @@ main() { local -r dirname="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - source "${dirname}"/../../env.sh + source "${ENV_FILE}" echo echo "Run t-rex server on "${TREX_SERVER_IP}"." diff --git a/tests/scripts/run_trex_test_case b/tests/scripts/run_trex_test_case index 40a0b3c..0454a78 100755 --- a/tests/scripts/run_trex_test_case +++ b/tests/scripts/run_trex_test_case @@ -9,7 +9,7 @@ main() { set -o nounset local -r dirname="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - source "${dirname}"/../../env.sh + source "${ENV_FILE}" local test_case_file="${1}".py local run_trex_client_command="export PYTHONPATH='"${TREX_CLIENT_LIB_DIR}"'; python3 "${REMOTE_TEST_CASES_DIR}"/""${test_case_file}" diff --git a/tests/scripts/start_session b/tests/scripts/start_session index 2ad4567..b1d80c4 100755 --- a/tests/scripts/start_session +++ b/tests/scripts/start_session @@ -78,7 +78,7 @@ main() { local -r filename="${dirname}/$(basename "${BASH_SOURCE[0]}")" local -r session_name="upee" - source "${dirname}"/../../env.sh + source "${ENV_FILE}" # if [[ $# != 4 ]]; then # echo >&2 "TODO!! "