From 6c194747309ad93aa4d4a8719c2c5b40a6311a41 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 16 Jul 2018 11:11:24 +0200 Subject: [PATCH] add Fedora 28 devel containers --- dev_env/fedora/base/28/Dockerfile | 190 ++++++++++++++++++ dev_env/fedora/base/28/build.sh | 3 + .../base/28/common/common/setup-projects.sh | 63 ++++++ .../fedora/base/28/common/setup-projects.sh | 63 ++++++ dev_env/fedora/base/28/setup-system.sh | 49 +++++ dev_env/fedora/buildbot/28/Dockerfile | 20 ++ dev_env/fedora/buildbot/28/build.sh | 1 + dev_env/fedora/buildbot/28/tpl-buildbot.tac | 39 ++++ 8 files changed, 428 insertions(+) create mode 100644 dev_env/fedora/base/28/Dockerfile create mode 100755 dev_env/fedora/base/28/build.sh create mode 100755 dev_env/fedora/base/28/common/common/setup-projects.sh create mode 100755 dev_env/fedora/base/28/common/setup-projects.sh create mode 100755 dev_env/fedora/base/28/setup-system.sh create mode 100644 dev_env/fedora/buildbot/28/Dockerfile create mode 100755 dev_env/fedora/buildbot/28/build.sh create mode 100644 dev_env/fedora/buildbot/28/tpl-buildbot.tac diff --git a/dev_env/fedora/base/28/Dockerfile b/dev_env/fedora/base/28/Dockerfile new file mode 100644 index 0000000..5ffa71d --- /dev/null +++ b/dev_env/fedora/base/28/Dockerfile @@ -0,0 +1,190 @@ +# container for rsyslog development +# creates the build environment +# Note: this image currently uses in-container git checkouts to +# build the "rsyslog libraries" - we do not have packages for them +FROM fedora:28 +# search for packages that contain : dnf whatprovides +RUN dnf -y update +RUN dnf -y install \ + autoconf \ + autoconf-archive \ + automake \ + make \ + bison \ + clang \ + clang-analyzer \ + compat-libstdc++-33 \ + curl \ + cyrus-sasl-devel \ + cyrus-sasl-lib \ + flex \ + gcc \ + gdb \ + git \ + gnutls-devel \ + hiredis-devel \ + java-1.8.0-openjdk \ + java-1.8.0-openjdk-devel \ + libcurl-devel \ + libdbi-dbd-mysql \ + libdbi-devel \ + libfaketime \ + libgcrypt-devel \ + libmaxminddb-devel \ + libnet libnet-devel \ + libtool \ + libuuid-devel \ + lsof \ + mysql-devel \ + nc \ + net-snmp-devel \ + net-tools \ + openssl-devel \ + postgresql-devel \ + python-devel \ + python-docutils \ + python-sphinx \ + qpid-proton-c-devel \ + redhat-rpm-config \ + snappy-devel \ + sudo \ + systemd-devel \ + tcl-devel \ + valgrind \ + wget \ + zlib-devel + # end of this RUN +#RUN dnf -y install epel-release && +RUN dnf -y install \ + czmq-devel \ + hiredis \ + hiredis-devel \ + libmaxminddb \ + libmaxminddb-devel \ + mongo-c-driver \ + mongo-c-driver-devel + # end of this RUN +WORKDIR /home/devel +VOLUME /rsyslog +RUN groupadd rsyslog \ + && adduser -g rsyslog -s /bin/bash rsyslog \ + && echo "rsyslog ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +ADD setup-system.sh setup-system.sh +ADD common/setup-projects.sh setup-projects.sh +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig \ + LD_LIBRARY_PATH=/usr/local/lib \ + LIBDIR_PATH=/usr/lib64 + +# bump dependency version below to trigger a dependency rebuild +# but not a full one (via --no-cache) +ENV DEP_VERSION=1 +# Helper projects and dependency build starts here +RUN mkdir helper-projects +# code style checker - not yet packaged +RUN cd helper-projects && \ + git clone https://github.com/rsyslog/codestyle && \ + cd codestyle && \ + gcc --std=c99 stylecheck.c -o stylecheck && \ + mv stylecheck /usr/bin/rsyslog_stylecheck && \ + cd .. && \ + rm -r codestyle && \ + cd .. + +# we need Guardtime libksi here, otherwise we cannot check the KSI component +RUN cd helper-projects && \ + git clone https://github.com/guardtime/libksi.git && \ + cd libksi && \ + autoreconf -fvi && \ + ./configure --libdir=/usr/lib64 && \ + make -j2 && \ + make install && \ + cd .. && \ + rm -r libksi && \ + cd .. +# we need the latest librdkafka as there as always required updates +RUN cd helper-projects && \ + git clone https://github.com/edenhill/librdkafka && \ + cd librdkafka && \ + (unset CFLAGS; ./configure --prefix=/usr $LIBDIR --CFLAGS="-g" ; make -j2) && \ + make install && \ + cd .. && \ +# Note: we do NOT delete the source as we may need it to +# uninstall (in case the user wants to go back to system-default) + cd .. + +# next ENV is specifically for running scan-build - so we do not need to +# change scripts if at a later time we can move on to a newer version +#ENV SCAN_BUILD=scan-build \ +# SCAN_BUILD_CC=clang-5.0 + +ENV RSYSLOG_CONFIGURE_OPTIONS \ + --enable-elasticsearch \ + --enable-elasticsearch-tests \ + --enable-gnutls \ + --enable-gssapi-krb5 \ + --enable-imczmq \ + --enable-imdiag \ + --enable-imfile \ + --enable-imjournal \ + --enable-imkafka \ + --enable-impstats \ + --enable-imptcp \ + --enable-kafka-tests \ + --enable-ksi-ls12 \ + --enable-libdbi \ + --enable-libfaketime \ + --enable-libgcrypt \ + --enable-mail \ + --enable-mmanon \ + --enable-mmaudit \ + --enable-mmcount \ + --enable-mmdblookup \ + --enable-mmfields \ + --enable-mmjsonparse \ + --enable-mmkubernetes \ + --enable-mmnormalize \ + --enable-mmpstrucdata \ + --enable-mmrm1stspace \ + --enable-mmsequence \ + --enable-mmsnmptrapd \ + --enable-mmutf8fix \ + --enable-mysql \ + --enable-omamqp1 \ + --enable-omczmq \ + --enable-omhiredis \ + --enable-omhiredis \ + --enable-omhttpfs \ + --enable-omjournal \ + --enable-omkafka \ + --enable-ommongodb \ + --enable-omprog \ + --enable-omrelp-default-port=13515 \ + --enable-omruleset \ + --enable-omstdout \ + --enable-omtcl=no \ + --enable-omudpspoof \ + --enable-omuxsock \ + --enable-openssl \ + --enable-pgsql \ + --enable-pmaixforwardedfrom \ + --enable-pmciscoios \ + --enable-pmcisconames \ + --enable-pmlastmsg \ + --enable-pmnormalize \ + --enable-pmnull \ + --enable-pmsnare \ + --enable-relp \ + --enable-snmp \ + --enable-usertools \ + --enable-valgrind \ + \ + --enable-testbench + +# build errors at the moment: --enable-kmsg +# --enable-mmgrok - no package + +# Install any needed packages +#RUN ./setup-system.sh +RUN ./setup-projects.sh +WORKDIR /rsyslog +USER rsyslog diff --git a/dev_env/fedora/base/28/build.sh b/dev_env/fedora/base/28/build.sh new file mode 100755 index 0000000..4941a3e --- /dev/null +++ b/dev_env/fedora/base/28/build.sh @@ -0,0 +1,3 @@ +#cp -r ../../../common/ common +docker build $1 -t rsyslog/rsyslog_dev_base_fedora:28 . +#rm -r common diff --git a/dev_env/fedora/base/28/common/common/setup-projects.sh b/dev_env/fedora/base/28/common/common/setup-projects.sh new file mode 100755 index 0000000..abcf5a3 --- /dev/null +++ b/dev_env/fedora/base/28/common/common/setup-projects.sh @@ -0,0 +1,63 @@ +#export SUDO= +#export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig +#export LD_LIBRARY_PATH=/usr/local/lib + +# end config settings +export CI_HOME=`pwd` +echo CI_HOME: $CI_HOME +mkdir $CI_HOME/proj +cd $CI_HOME/proj +git clone https://github.com/rsyslog/libestr.git +cd libestr +git pull +autoreconf -fvi && ./configure --prefix=/usr/local && make -j2 || exit $? +$SUDO make -j2 install || exit $? + +printf "\n\n================== libfastjson =======================\n\n" +cd $CI_HOME/proj +git clone https://github.com/rsyslog/libfastjson.git +cd libfastjson +git pull +autoreconf -fvi && ./configure --prefix=/usr/local --disable-journal && make -j2 || exit $? +$SUDO make -j2 install || exit $? + + +printf "\n\n================== liblogging =======================\n\n" +cd $CI_HOME/proj +git clone https://github.com/rsyslog/liblogging.git +cd liblogging +git pull +autoreconf -fvi && ./configure --prefix=/usr/local --disable-journal && make -j2 || exit $? +$SUDO make -j2 install || exit $? + +printf "\n\n================== liblognorm =======================\n\n" +cd $CI_HOME/proj +git clone https://github.com/rsyslog/liblognorm.git +cd liblognorm +git pull +autoreconf -fvi && ./configure --prefix=/usr/local && make -j2 || exit $? +$SUDO make -j2 install || exit $? + +printf "\n\n================== liblrelp =======================\n\n" +cd $CI_HOME/proj +git clone https://github.com/rsyslog/librelp.git +cd librelp +git pull +autoreconf -fvi && ./configure --prefix=/usr/local && make -j2 || exit $? +$SUDO make -j2 install || exit $? + + +### main project ### +printf "\n\n================== rsyslog =======================\n\n" +printf "This is primarily built to make sure that the helper\n" +printf "projects are correctly built and installed\n\n" + +cd $CI_HOME/proj +git clone https://github.com/rsyslog/rsyslog.git +cd rsyslog +git pull +mkdir utils +echo "./configure --prefix=/usr/local $RSYSLOG_CONFIGURE_OPTIONS" > utils/conf +chmod +x utils/conf + +autoreconf -fvi && utils/conf && make -j2 || exit $? diff --git a/dev_env/fedora/base/28/common/setup-projects.sh b/dev_env/fedora/base/28/common/setup-projects.sh new file mode 100755 index 0000000..0e4902a --- /dev/null +++ b/dev_env/fedora/base/28/common/setup-projects.sh @@ -0,0 +1,63 @@ +#export SUDO= +#export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig +#export LD_LIBRARY_PATH=/usr/local/lib + +# end config settings +export CI_HOME=`pwd` +echo CI_HOME: $CI_HOME +mkdir $CI_HOME/proj +cd $CI_HOME/proj +git clone https://github.com/rsyslog/libestr.git +cd libestr +git pull +autoreconf -fvi && ./configure --prefix=/usr/local && make -j2 || exit $? +$SUDO make -j2 install || exit $? + +printf "\n\n================== libfastjson =======================\n\n" +cd $CI_HOME/proj +git clone https://github.com/rsyslog/libfastjson.git +cd libfastjson +git pull +autoreconf -fvi && ./configure --prefix=/usr/local --disable-journal && make -j2 || exit $? +$SUDO make -j2 install || exit $? + + +printf "\n\n================== liblogging =======================\n\n" +cd $CI_HOME/proj +git clone https://github.com/rsyslog/liblogging.git +cd liblogging +git pull +autoreconf -fvi && ./configure --enable-compile-warnings=yes --prefix=/usr/local --disable-journal && make -j2 || exit $? +$SUDO make -j2 install || exit $? + +printf "\n\n================== liblognorm =======================\n\n" +cd $CI_HOME/proj +git clone https://github.com/rsyslog/liblognorm.git +cd liblognorm +git pull +autoreconf -fvi && ./configure --enable-compile-warnings=yes --prefix=/usr/local && make -j2 || exit $? +$SUDO make -j2 install || exit $? + +printf "\n\n================== liblrelp =======================\n\n" +cd $CI_HOME/proj +git clone https://github.com/rsyslog/librelp.git +cd librelp +git pull +autoreconf -fvi && ./configure --enable-compile-warnings=yes --prefix=/usr/local && make -j2 || exit $? +$SUDO make -j2 install || exit $? + + +### main project ### +printf "\n\n================== rsyslog =======================\n\n" +printf "This is primarily built to make sure that the helper\n" +printf "projects are correctly built and installed\n\n" + +cd $CI_HOME/proj +git clone https://github.com/rsyslog/rsyslog.git +cd rsyslog +git pull +mkdir utils +echo "./configure --enable-compile-warnings=yes --prefix=/usr/local $RSYSLOG_CONFIGURE_OPTIONS" > utils/conf +chmod +x utils/conf + +autoreconf -fvi && utils/conf && make -j2 || exit $? diff --git a/dev_env/fedora/base/28/setup-system.sh b/dev_env/fedora/base/28/setup-system.sh new file mode 100755 index 0000000..d2c24ab --- /dev/null +++ b/dev_env/fedora/base/28/setup-system.sh @@ -0,0 +1,49 @@ +set -v + +echo STEP: essentials +# prevent systemd from stealing our core files (bad for testbench) +#bash -c "echo core > /proc/sys/kernel/core_pattern" # does not work inside container + +# search for packages that contain : yum whatprovides + +# Now build some custom libraries for whom there are not packages +mkdir helper-projects +cd helper-projects + +if [ "$LIBDIR_PATH" != "" ]; then + export LIBDIR=--libdir=$LIBDIR_PATH +fi + +#set -e +# libgrok - not packaged, manual build also does not work ATM +#git clone https://github.com/jordansissel/grok +#cd grok +##autoreconf -fvi +##./configure --libdir=/usr/lib64 +#make +#make install +#cd .. +#rm -r grok + +# we need Guardtime libksi here, otherwise we cannot check the KSI component +git clone https://github.com/guardtime/libksi.git +cd libksi +autoreconf -fvi +./configure --libdir=/usr/lib64 +make -j2 +make install +cd .. +rm -r libksi + +# we need the latest librdkafka as there as always required updates +git clone https://github.com/edenhill/librdkafka +cd librdkafka +(unset CFLAGS; ./configure --prefix=/usr $LIBDIR --CFLAGS="-g" ; make -j2) +make install +cd .. +# Note: we do NOT delete the source as we may need it to +# uninstall (in case the user wants to go back to system-default) + +# finished installing helpers from source +cd .. + diff --git a/dev_env/fedora/buildbot/28/Dockerfile b/dev_env/fedora/buildbot/28/Dockerfile new file mode 100644 index 0000000..d8fdb6a --- /dev/null +++ b/dev_env/fedora/buildbot/28/Dockerfile @@ -0,0 +1,20 @@ +FROM rsyslog/rsyslog_dev_base_fedora:28 +USER root +RUN dnf install -y \ + python-devel \ + python-pip +RUN pip install buildbot-worker buildbot-slave +RUN groupadd -r buildbot && useradd -r -g buildbot buildbot +RUN mkdir /worker && chown buildbot:buildbot /worker +# Install your build-dependencies here ... +ENV WORKER_ENVIRONMENT_BLACKLIST=WORKER* +USER buildbot +WORKDIR /worker +RUN buildbot-worker create-worker . docker.rsyslog.com docker-ubuntu16 password +# the following script is directly from buildbot git repo and seems +# to be necessary at the moment. +# see https://github.com/buildbot/buildbot/issues/4179 +COPY tpl-buildbot.tac /worker/buildbot.tac +ENTRYPOINT ["/usr/local/bin/buildbot-worker"] +CMD ["start", "--nodaemon"] +VOLUME /worker diff --git a/dev_env/fedora/buildbot/28/build.sh b/dev_env/fedora/buildbot/28/build.sh new file mode 100755 index 0000000..6136daf --- /dev/null +++ b/dev_env/fedora/buildbot/28/build.sh @@ -0,0 +1 @@ +docker build $1 -t rsyslog/rsyslog_dev_buildbot_fedora:28 . diff --git a/dev_env/fedora/buildbot/28/tpl-buildbot.tac b/dev_env/fedora/buildbot/28/tpl-buildbot.tac new file mode 100644 index 0000000..5a3ff89 --- /dev/null +++ b/dev_env/fedora/buildbot/28/tpl-buildbot.tac @@ -0,0 +1,39 @@ +import fnmatch +import os +import sys + +from twisted.application import service +from twisted.python.log import FileLogObserver +from twisted.python.log import ILogObserver + +from buildbot_worker.bot import Worker + +# setup worker +basedir = os.path.abspath(os.path.dirname(__file__)) +application = service.Application('buildbot-worker') + + +application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit) +# and worker on the same process! +buildmaster_host = os.environ.get("BUILDMASTER", 'localhost') +port = int(os.environ.get("BUILDMASTER_PORT", 9989)) +workername = os.environ.get("WORKERNAME", 'docker') +passwd = os.environ.get("WORKERPASS") + +# delete the password from the environ so that it is not leaked in the log +blacklist = os.environ.get("WORKER_ENVIRONMENT_BLACKLIST", "WORKERPASS").split() +for name in list(os.environ.keys()): + for toremove in blacklist: + if fnmatch.fnmatch(name, toremove): + del os.environ[name] + +keepalive = 600 +umask = None +maxdelay = 300 +allow_shutdown = None +maxretries = 10 + +s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + allow_shutdown=allow_shutdown, maxRetries=maxretries) +s.setServiceParent(application)