Skip to content

Commit

Permalink
chore(ci): merge bazel base and devcontainer - first pr (#12638)
Browse files Browse the repository at this point in the history 
* chore(ci): bazel-base is moved to .devcontainer

Signed-off-by: Nils Semmelrock <nils.semmelrock@tngtech.com>

* chore(ci): bazel base and devcontainer Dockerfiles are merged

Signed-off-by: Nils Semmelrock <nils.semmelrock@tngtech.com>

* chore(ci): bazel base and devcontainer build workflows are merged

Signed-off-by: Nils Semmelrock <nils.semmelrock@tngtech.com>

* chore(ci): new images are created with latestv2 tag - this will be reverted in a followup PR

Signed-off-by: Nils Semmelrock <nils.semmelrock@tngtech.com>
  • Loading branch information
@nstng
nstng committed May 16, 2022
1 parent c823aee commit 30bbebd
Show file tree
Hide file tree
Showing 11 changed files with 215 additions and 416 deletions.
233 changes: 48 additions & 185 deletions .devcontainer/Dockerfile
View file
@@ -1,4 +1,7 @@
FROM ubuntu:focal as magma-mme-builder
################################################################
# Devcontainer Image (for local development and CI)
################################################################
FROM ghcr.io/magma/magma/bazel-base:latestv2 as devcontainer

# [Option] Install zsh
ARG INSTALL_ZSH="true"
Expand Down Expand Up @@ -28,74 +31,50 @@ RUN apt-get update \
ENTRYPOINT [ "/usr/local/share/docker-init.sh" ]
CMD [ "sleep", "infinity" ]

ARG GIT_PROXY
ARG FEATURES=mme_oai
ARG PYTHON_VERSION=3.8
# PYTHON_VERSION must be in sync with "python.defaultInterpreterPath" and "python.analysis.extraPaths" in .devcontainer/devcontainer.json
ENV MAGMA_ROOT=/workspaces/magma
ENV BUILD_TYPE=Debug
ENV C_BUILD=/workspaces/magma/build/c
ENV DEBIAN_FRONTEND=noninteractive


RUN echo "Install general purpose packages" && \
apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
apt-transport-https \
apt-utils \
apt-get upgrade -y && \
apt-get install -y --no-install-recommends \
autoconf \
autoconf automake \
automake \
build-essential \
ca-certificates \
clang-11 \
clang-format-11 \
clang-tidy-11 \
curl \
g++ \
clangd-12 \
g++-9 \
gcc-9 \
# iputils-ping is a dependency of python services (e.g. magmad)
iputils-ping \
lcov \
gdb \
git \
gnupg \
lcov \
libclang-11-dev \
libgmp3-dev \
libpcap-dev \
libssl-dev \
libtool \
lld \
lldb \
llvm-11-dev \
make \
netbase \
ninja-build \
openjdk-8-jdk \
perl \
pkg-config \
python${PYTHON_VERSION} \
python3-pip \
python-is-python3 \
redis-server \
ruby \
rubygems \
ruby-dev \
software-properties-common \
tzdata \
unzip \
vim \
virtualenv=20.0.17-1ubuntu0.4 \
wget && \
gem install fpm && \
update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-11/bin/clang 10 && \
update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-11/bin/clang++ 10 && \
update-alternatives --install /usr/bin/clang-format clang-format /usr/lib/llvm-11/bin/clang-format 10 && \
update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/lib/llvm-11/bin/clang-tidy 10 && \
update-alternatives --install /usr/bin/clang-apply-replacements clang-apply-replacements /usr/lib/llvm-11/bin/clang-apply-replacements 10

RUN DEBIAN_FRONTEND=noninteractive apt-get install -y clangd-12
virtualenv=20.0.17-1ubuntu0.4 && \
gem install fpm && \
update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-11/bin/clang 10 && \
update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-11/bin/clang++ 10 && \
update-alternatives --install /usr/bin/clang-format clang-format /usr/lib/llvm-11/bin/clang-format 10 && \
update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/lib/llvm-11/bin/clang-tidy 10 && \
update-alternatives --install /usr/bin/clang-apply-replacements clang-apply-replacements /usr/lib/llvm-11/bin/clang-apply-replacements 10

# Install golang 1.18
WORKDIR /usr/local
Expand All @@ -104,67 +83,37 @@ RUN curl https://artifactory.magmacore.org/artifactory/generic/go1.18.linux-amd6
rm go1.18.linux-amd64.tar.gz
ENV PATH=$PATH:/usr/local/go/bin

# Install bazel
WORKDIR /usr/sbin
RUN wget --progress=dot:giga https://github.com/bazelbuild/bazelisk/releases/download/v1.10.0/bazelisk-linux-amd64 && \
chmod +x bazelisk-linux-amd64 && \
ln -s /usr/sbin/bazelisk-linux-amd64 /usr/sbin/bazel

RUN echo "Install 3rd party dependencies" && \
apt-get update && \
echo "Install CMake" && \
apt-get -y install cmake && \
echo "Install FMT lib requirements" && \
apt-get -y install libunwind8-dev libelf-dev libdwarf-dev bzip2 && \
apt-get -y install --no-install-recommends \
libunwind8-dev \
libelf-dev \
libdwarf-dev \
bzip2 && \
echo "Install Folly requirements" && \
apt-get -y install libboost-all-dev libevent-dev libdouble-conversion-dev \
libgoogle-glog-dev libgflags-dev libiberty-dev liblz4-dev liblzma-dev \
libsnappy-dev binutils-dev libjemalloc-dev libssl-dev pkg-config libunwind-dev && \
apt-get -y install --no-install-recommends \
liblz4-dev \
liblzma-dev \
libsnappy-dev \
binutils-dev \
libjemalloc-dev \
pkg-config \
libunwind-dev && \
echo "Install check for test support" && \
apt-get -y install check && \
apt-get -y install --no-install-recommends \
check && \
echo "Install gtest for test support" && \
apt-get -y install libgtest-dev && \
echo "Install FreeDiameter requirements" && \
apt-get -y install libsctp1 libsctp-dev libgcrypt-dev \
bison flex libidn11-dev && \
echo "Install libgtpnl requirements" && \
apt-get -y install libmnl-dev && \
apt-get -y install --no-install-recommends \
libgtest-dev && \
echo "Install Nettle requirements" && \
apt-get install -y libgoogle-glog-dev libconfig-dev libxml2-dev \
libyaml-cpp-dev nlohmann-json3-dev && \
echo "Install ZeroMQ" && \
apt-get install -y libczmq-dev && \
echo "Install dependencies for Sentry Native" && \
apt-get install -y libcurl4-openssl-dev

RUN ["/bin/bash", "-c", "if [[ -v GIT_PROXY ]]; then git config --global http.proxy $GIT_PROXY; fi"]

##### NETTLE and GNUTLS
# TODO Upgrade these - requires us to update our use of libnettle due to API migration.
# see https://gist.github.com/electronjoe/a899e4bfbc2904cb353444386296c38e
# Note the CFLAGS define below due to glibc deprecation of critical flag,
# see https://github.com/rdslw/openwrt/blob/e5d47f32131849a69a9267de51a30d6be1f0d0ac/tools/bison/patches/110-glibc-change-work-around.patch
RUN wget --progress=dot:giga https://ftp.gnu.org/gnu/nettle/nettle-2.5.tar.gz && \
tar -xf nettle-2.5.tar.gz && \
cd nettle-2.5 && \
mkdir build && \
cd build/ && \
../configure --disable-openssl --enable-shared --libdir=/usr/lib && \
make -j"$(nproc)" && \
make install && \
ldconfig -v && \
cd / && \
wget --progress=dot:giga https://www.gnupg.org/ftp/gcrypt/gnutls/v3.1/gnutls-3.1.23.tar.xz && \
tar xf gnutls-3.1.23.tar.xz && \
cd gnutls-3.1.23 && \
mkdir build && cd build && \
CFLAGS=-D_IO_ftrylockfile ../configure --libdir=/usr/lib --with-libnettle-prefix=/usr && \
make -j"$(nproc)" && \
make install && \
ldconfig -v && \
cd / && \
rm -rf nettle* && \
rm -rf gnutls*
apt-get -y install --no-install-recommends \
libxml2-dev \
libyaml-cpp-dev \
nlohmann-json3-dev && \
echo "GRPC and it's dependencies" && \
apt-get -y install --no-install-recommends \
grpc-dev

##### Useful for logfile modification e.g. pruning all /magma/... prefix from GCC warning logs
RUN GOBIN="/usr/bin/" go install github.com/ezekg/xo@0f7f076932dd
Expand Down Expand Up @@ -202,25 +151,12 @@ RUN git clone https://github.com/jupp0r/prometheus-cpp.git && \
make install && \
rm -rf /prometheus-cpp

##### Redis CPP
RUN git clone https://github.com/cpp-redis/cpp_redis.git && \
cd cpp_redis && \
git checkout bbe38a7f83de943ffcc90271092d689ae02b3489 && \
git submodule init && git submodule update && \
mkdir build && cd build && \
cmake .. -DCMAKE_BUILD_TYPE=Release && \
make -j"$(nproc)" && \
make install && \
rm -rf /cpp_redis

##### liblfds
# https://www.liblfds.org/mediawiki/index.php?title=r7.1.0:Building_Guide_(liblfds)
RUN git clone https://github.com/liblfds/liblfds.git && \
cd liblfds/liblfds/liblfds7.1.0/liblfds710/build/gcc_gnumake/ && \
make -j"$(nproc)" && \
make ar_install && \
cd / && \
rm -rf liblfds
# install magma dependencies
RUN apt-get install -y --no-install-recommends \
libtins-dev \
magma-cpp-redis \
magma-libfluid \
python3-aioeventlet

##### libgtpnl
# review https://github.com/OPENAIRINTERFACE/openair-cn/blob/master/build/tools/build_helper.gtpnl
Expand All @@ -235,38 +171,6 @@ RUN git clone https://git.osmocom.org/libgtpnl && \
cd / && \
rm -rf libgtpnl

##### asn1c
RUN git clone https://gitlab.eurecom.fr/oai/asn1c.git && \
cd asn1c && \
git checkout f12568d617dbf48497588f8e227d70388fa217c9 && \
autoreconf -iv && \
./configure && \
make -j"$(nproc)" && \
make install && \
ldconfig

## Install Fmt (Folly Dep)
RUN git clone https://github.com/fmtlib/fmt.git && cd fmt && \
mkdir _build && cd _build && \
cmake -DBUILD_SHARED_LIBS=ON .. && \
make -j"$(nproc)" && \
make install && \
cd / && \
rm -rf fmt

##### Facebook Folly C++ lib
# Note: "Because folly does not provide any ABI compatibility guarantees from
# commit to commit, we generally recommend building folly as a static library."
# Here we checkout the hash for v2021.02.22.00 (arbitrary recent version)
RUN git clone https://github.com/facebook/folly && cd folly && \
git checkout tags/v2021.02.15.00 && \
mkdir _build && cd _build && \
cmake -DBUILD_SHARED_LIBS=ON .. && \
make -j"$(nproc)" && \
make install && \
cd / && \
rm -rf folly

##### Build and install libgtest and gmock
RUN cd /usr/src/googletest && \
mkdir build && \
Expand All @@ -278,20 +182,6 @@ RUN cd /usr/src/googletest && \
make install && \
ldconfig -v

#### libtins is required to build the connection_tracker
# TODO(@themarwhal): this might not be hard to bazelify if we can generate the config
RUN git clone --branch v4.3 https://github.com/mfontanini/libtins.git && \
cd libtins && \
mkdir build && \
cd build && \
cmake ../ -DLIBTINS_ENABLE_CXX11=1 && \
make -j"$(nproc)" && \
make install && \
cd / && \
rm -rf libtins && \
# symlink to /usr/lib to match Magma VM
ln -s /usr/local/lib/libtins.so /usr/lib/libtins.so

###### Install Include What You Use for c/cpp header include fixup tooling
# Tag 0.15 tracks Clang 11.0 per https://github.com/include-what-you-use/include-what-you-use/tags
RUN git clone https://github.com/include-what-you-use/include-what-you-use && \
Expand All @@ -306,24 +196,6 @@ RUN git clone https://github.com/include-what-you-use/include-what-you-use && \
rm -rf include-what-you-use && \
rm -rf build_liwyu

#### libfluid is requird to build MME with OVS support
COPY third_party /tmp/third_party/
RUN /tmp/third_party/build/bin/libfluid_build.sh && \
find . -name magma-libfluid_0.1* -exec dpkg -i {} \; && \
rm -rf /tmp/*

##### FreeDiameter
COPY lte/gateway/c/core/oai/patches/ /tmp/
RUN git clone https://github.com/OPENAIRINTERFACE/opencord.org.freeDiameter.git freediameter && \
cd freediameter && \
patch -p1 < /tmp/0001-opencoord.org.freeDiameter.patch && \
mkdir build && \
cd build && \
cmake -DDISABLE_SCTP:BOOL=ON ../ && \
make -j"$(nproc)" && \
make install && \
rm -rf /freediameter /tmp/*

##### Go language server support for vscode
RUN GOBIN="/usr/bin/" go install -v golang.org/x/tools/gopls@v0.8.3

Expand All @@ -341,18 +213,9 @@ ENV PYTHON_VENV_EXECUTABLE=${PYTHON_VENV}/bin/python${PYTHON_VERSION}
RUN virtualenv --system-site-packages --python=/usr/bin/python${PYTHON_VERSION} ${PYTHON_VENV}
RUN ${PYTHON_VENV_EXECUTABLE} -m pip install --quiet --upgrade --no-cache-dir "setuptools==49.6.0"

### install python3-aioeventlet and bcc-tools from the magma apt repo
COPY orc8r/tools/ansible/roles/pkgrepo/files/jfrog.pub /tmp/
RUN apt-key add /tmp/jfrog.pub && \
echo "deb https://facebookconnectivity.jfrog.io/artifactory/list/dev-focal/ focal main" > /etc/apt/sources.list.d/fbc.list && \
apt-get update -y && \
apt-get install -y python3-aioeventlet bcc-tools && \
rm -f /tmp/* && \
rm -rf /var/lib/apt/lists/* # delete lists that are downloaded by apt-get update

# add patch that is missing in jfrog version of aioeventlet (it only comes with 1 of 2 patches)
COPY lte/gateway/deploy/roles/magma/files/patches/aioeventlet_fd_exception.patch /tmp/
RUN patch -N -s -f /usr/local/lib/python${PYTHON_VERSION}/dist-packages/aioeventlet.py < /tmp/aioeventlet_fd_exception.patch && \
RUN (patch -N -s -f /usr/local/lib/python${PYTHON_VERSION}/dist-packages/aioeventlet.py < /tmp/aioeventlet_fd_exception.patch) || ( true && echo "skipping aioeventlet patch since it was already applied") && \
rm -rf /tmp/*

### install eggs (lte, orc8r)
Expand Down Expand Up @@ -414,6 +277,6 @@ RUN for SWAGGER_SRC in lte orc8r; \
rm -r ${PYTHON_VENV}/${GEN_DIR}/${SWAGGER_SRC}/swagger/test; \
done

RUN ln -s $MAGMA_ROOT/bazel/bazelrcs/devcontainer.bazelrc /etc/bazelrc
RUN ln -fs $MAGMA_ROOT/bazel/bazelrcs/devcontainer.bazelrc /etc/bazelrc

WORKDIR $MAGMA_ROOT

0 comments on commit 30bbebd

Please sign in to comment.