From b2dca5818db7253c6eb4702b46efebf8c3ba46ae Mon Sep 17 00:00:00 2001 From: James Rigassio Date: Wed, 23 Sep 2020 13:00:38 -0700 Subject: [PATCH 1/4] base centos 8 image --- base/centos-8/Dockerfile | 22 +++++++++++++++++++ base/centos-8/install.sh | 46 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 base/centos-8/Dockerfile create mode 100755 base/centos-8/install.sh diff --git a/base/centos-8/Dockerfile b/base/centos-8/Dockerfile new file mode 100644 index 00000000..142c6b8d --- /dev/null +++ b/base/centos-8/Dockerfile @@ -0,0 +1,22 @@ +# Copyright 2020 Splunk +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM centos:8 +LABEL maintainer="support@splunk.com" + +ARG SCLOUD_URL +ENV SCLOUD_URL ${SCLOUD_URL} + +COPY install.sh /install.sh +RUN /install.sh && rm -rf /install.sh \ No newline at end of file diff --git a/base/centos-8/install.sh b/base/centos-8/install.sh new file mode 100755 index 00000000..d308fbc6 --- /dev/null +++ b/base/centos-8/install.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2020 Splunk +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 +export LANG=en_US.utf8 + +yum -y update && yum -y install wget sudo epel-release +yum -y install busybox ansible python-requests python-jmespath + +# Install scloud +wget -O /usr/bin/scloud.tar.gz ${SCLOUD_URL} +tar -xf /usr/bin/scloud.tar.gz -C /usr/bin/ +rm /usr/bin/scloud.tar.gz + +cd /bin +ln -s busybox killall +ln -s busybox netstat +ln -s busybox nslookup +ln -s busybox readline +ln -s busybox route +ln -s busybox syslogd +ln -s busybox traceroute +chmod u+s /bin/ping +groupadd sudo + +echo " +## Allows people in group sudo to run all commands +%sudo ALL=(ALL) ALL" >> /etc/sudoers + +# Clean +yum clean all +rm -rf /install.sh /anaconda-post.log /var/log/anaconda/* From eba1d9e86db86a330defc8531150617b8e6255df Mon Sep 17 00:00:00 2001 From: James Rigassio Date: Wed, 23 Sep 2020 15:31:08 -0700 Subject: [PATCH 2/4] py23 image centos 8 --- py23-image/centos-8/Dockerfile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 py23-image/centos-8/Dockerfile diff --git a/py23-image/centos-8/Dockerfile b/py23-image/centos-8/Dockerfile new file mode 100644 index 00000000..ddb1e8de --- /dev/null +++ b/py23-image/centos-8/Dockerfile @@ -0,0 +1,25 @@ +ARG SPLUNK_PRODUCT=splunk +FROM ${SPLUNK_PRODUCT}-centos-8:latest +USER root + +RUN yum -y update +RUN yum -y install gcc openssl-devel bzip2-devel libffi-devel python-pip +# manual installation of python 3.7 as default distro version is 3.6 +RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \ + && tar xzf Python-3.7.4.tgz \ + && cd Python-3.7.4 \ + && ./configure --enable-optimizations --prefix=/usr \ + && make install \ + && cd .. \ + && rm Python-3.7.4.tgz \ + && rm -r Python-3.7.4 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3.7 get-pip.py \ + && rm -f get-pip.py \ + # pip version is not automatically "fixed", unlike debian-based + && ln -sf /usr/bin/pip2 /usr/bin/pip \ + && ln -sf /usr/bin/pip3.7 /usr/bin/pip3 +RUN yum remove -y --setopt=tsflags=noscripts gcc openssl-devel bzip2-devel libffi-devel \ + && yum autoremove -y \ + && yum clean all +RUN pip3 --no-cache-dir install ansible requests \ No newline at end of file From 327141cfc5e37246872dfdb4038389d1c5756320 Mon Sep 17 00:00:00 2001 From: James Rigassio Date: Thu, 24 Sep 2020 18:22:03 -0700 Subject: [PATCH 3/4] py23 image and makefile centos 8 --- Makefile | 57 +++++++++++++++++++++++++++++----- base/centos-8/install.sh | 13 ++++++-- py23-image/centos-8/Dockerfile | 6 +++- 3 files changed, 66 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 3dadb9fb..d9ad80c1 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ ansible: @cat splunk-ansible/version.txt ##### Base images ##### -base: base-debian-9 base-debian-10 base-centos-7 base-redhat-8 base-windows-2016 +base: base-debian-9 base-debian-10 base-centos-7 base-centos-8 base-redhat-8 base-windows-2016 base-debian-10: docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} -t base-debian-10:${IMAGE_VERSION} ./base/debian-10 @@ -69,6 +69,9 @@ base-debian-9: base-centos-7: docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} -t base-centos-7:${IMAGE_VERSION} ./base/centos-7 +base-centos-8: + docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} -t base-centos-8:${IMAGE_VERSION} ./base/centos-8 + base-redhat-8: docker build ${DOCKER_BUILD_FLAGS} --build-arg SCLOUD_URL=${SCLOUD_URL} --label version=${SPLUNK_VERSION} -t base-redhat-8:${IMAGE_VERSION} ./base/redhat-8 @@ -76,7 +79,7 @@ base-windows-2016: docker build ${DOCKER_BUILD_FLAGS} -t base-windows-2016:${IMAGE_VERSION} ./base/windows-2016 ##### Minimal images ##### -minimal: minimal-debian-9 minimal-debian-10 minimal-centos-7 minimal-redhat-8 +minimal: minimal-debian-9 minimal-debian-10 minimal-centos-7 minimal-centos-8 minimal-redhat-8 minimal-debian-9: base-debian-9 docker build ${DOCKER_BUILD_FLAGS} \ @@ -99,6 +102,13 @@ minimal-centos-7: base-centos-7 --build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \ --target minimal -t minimal-centos-7:${IMAGE_VERSION} . +minimal-centos-8: base-centos-8 + docker build ${DOCKER_BUILD_FLAGS} \ + -f splunk/common-files/Dockerfile \ + --build-arg SPLUNK_BASE_IMAGE=base-centos-8 \ + --build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \ + --target minimal -t minimal-centos-8:${IMAGE_VERSION} . + minimal-redhat-8: base-redhat-8 docker build ${DOCKER_BUILD_FLAGS} \ -f splunk/common-files/Dockerfile \ @@ -107,7 +117,7 @@ minimal-redhat-8: base-redhat-8 --target minimal -t minimal-redhat-8:${IMAGE_VERSION} . ##### Bare images ##### -bare: bare-debian-9 bare-debian-10 bare-centos-7 bare-redhat-8 +bare: bare-debian-9 bare-debian-10 bare-centos-7 bare-centos-8 bare-redhat-8 bare-debian-9: base-debian-9 docker build ${DOCKER_BUILD_FLAGS} \ @@ -128,7 +138,14 @@ bare-centos-7: base-centos-7 -f splunk/common-files/Dockerfile \ --build-arg SPLUNK_BASE_IMAGE=base-centos-7 \ --build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \ - --target bare -t bare-centos-7:${IMAGE_VERSION} . + --target bare -t bare-centos-7:${IMAGE_VERSION} . + +bare-centos-8: base-centos-7 + docker build ${DOCKER_BUILD_FLAGS} \ + -f splunk/common-files/Dockerfile \ + --build-arg SPLUNK_BASE_IMAGE=base-centos-8 \ + --build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \ + --target bare -t bare-centos-8:${IMAGE_VERSION} . bare-redhat-8: base-redhat-8 docker build ${DOCKER_BUILD_FLAGS} \ @@ -138,7 +155,7 @@ bare-redhat-8: base-redhat-8 --target bare -t bare-redhat-8:${IMAGE_VERSION} . ##### Splunk images ##### -splunk: ansible splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-redhat-8 +splunk: ansible splunk-debian-9 splunk-debian-10 splunk-centos-7 splunk-centos-8 splunk-redhat-8 splunk-debian-9: base-debian-9 ansible docker build ${DOCKER_BUILD_FLAGS} \ @@ -161,6 +178,13 @@ splunk-centos-7: base-centos-7 ansible --build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \ -t splunk-centos-7:${IMAGE_VERSION} . +splunk-centos-8: base-centos-8 ansible + docker build ${DOCKER_BUILD_FLAGS} \ + -f splunk/common-files/Dockerfile \ + --build-arg SPLUNK_BASE_IMAGE=base-centos-8 \ + --build-arg SPLUNK_BUILD_URL=${SPLUNK_LINUX_BUILD_URL} \ + -t splunk-centos-8:${IMAGE_VERSION} . + splunk-redhat-8: base-redhat-8 ansible docker build ${DOCKER_BUILD_FLAGS} \ -f splunk/common-files/Dockerfile \ @@ -176,7 +200,7 @@ splunk-windows-2016: base-windows-2016 ansible -t splunk-windows-2016:${IMAGE_VERSION} . ##### UF images ##### -uf: ansible uf-debian-9 uf-debian-10 uf-centos-7 uf-redhat-8 +uf: ansible uf-debian-9 uf-debian-10 uf-centos-7 uf-centos-8 uf-redhat-8 ufbare-debian-9: base-debian-9 ansible docker build ${DOCKER_BUILD_FLAGS} \ @@ -213,6 +237,13 @@ uf-centos-7: base-centos-7 ansible --build-arg SPLUNK_BUILD_URL=${UF_LINUX_BUILD_URL} \ -t uf-centos-7:${IMAGE_VERSION} . +uf-centos-8: base-centos-8 ansible + docker build ${DOCKER_BUILD_FLAGS} \ + -f uf/common-files/Dockerfile \ + --build-arg SPLUNK_BASE_IMAGE=base-centos-8 \ + --build-arg SPLUNK_BUILD_URL=${UF_LINUX_BUILD_URL} \ + -t uf-centos-8:${IMAGE_VERSION} . + uf-redhat-8: base-redhat-8 ansible docker build ${DOCKER_BUILD_FLAGS} \ -f uf/common-files/Dockerfile \ @@ -229,7 +260,7 @@ uf-windows-2016: base-windows-2016 ansible ##### Python 3 support ##### -splunk-py23: splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-redhat-8 +splunk-py23: splunk-py23-debian-9 splunk-py23-debian-10 splunk-py23-centos-7 splunk-py23-centos-8 splunk-py23-redhat-8 splunk-py23-debian-9: splunk-debian-9 docker build ${DOCKER_BUILD_FLAGS} \ @@ -249,6 +280,12 @@ splunk-py23-centos-7: splunk-centos-7 --build-arg SPLUNK_PRODUCT=splunk \ -t splunk-py23-centos-7:${IMAGE_VERSION} . +splunk-py23-centos-8: splunk-centos-8 + docker build ${DOCKER_BUILD_FLAGS} \ + -f py23-image/centos-8/Dockerfile \ + --build-arg SPLUNK_PRODUCT=splunk \ + -t splunk-py23-centos-8:${IMAGE_VERSION} . + splunk-py23-redhat-8: splunk-redhat-8 docker build ${DOCKER_BUILD_FLAGS} \ -f py23-image/redhat-8/Dockerfile \ @@ -275,6 +312,12 @@ uf-py23-centos-7: uf-centos-7 --build-arg SPLUNK_PRODUCT=uf \ -t uf-py23-centos-7:${IMAGE_VERSION} . +uf-py23-centos-8: uf-centos-8 + docker build ${DOCKER_BUILD_FLAGS} \ + -f py23-image/centos-8/Dockerfile \ + --build-arg SPLUNK_PRODUCT=uf \ + -t uf-py23-centos-8:${IMAGE_VERSION} . + uf-py23-redhat-8: uf-redhat-8 docker build ${DOCKER_BUILD_FLAGS} \ -f py23-image/redhat-8/Dockerfile \ diff --git a/base/centos-8/install.sh b/base/centos-8/install.sh index d308fbc6..0e4f1c34 100755 --- a/base/centos-8/install.sh +++ b/base/centos-8/install.sh @@ -15,11 +15,17 @@ set -e +yum -y install glibc-locale-source glibc-langpack-en + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 export LANG=en_US.utf8 -yum -y update && yum -y install wget sudo epel-release -yum -y install busybox ansible python-requests python-jmespath +yum -y update && yum -y install wget sudo epel-release make +yum -y install ansible python3-requests python3-jmespath + +# Install busybox +wget -O /bin/busybox https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-`arch` +chmod +x /bin/busybox # Install scloud wget -O /usr/bin/scloud.tar.gz ${SCLOUD_URL} @@ -41,6 +47,9 @@ echo " ## Allows people in group sudo to run all commands %sudo ALL=(ALL) ALL" >> /etc/sudoers +# symlink for python3 +ln -s /bin/python3 /bin/python + # Clean yum clean all rm -rf /install.sh /anaconda-post.log /var/log/anaconda/* diff --git a/py23-image/centos-8/Dockerfile b/py23-image/centos-8/Dockerfile index ddb1e8de..0b0ebb8b 100644 --- a/py23-image/centos-8/Dockerfile +++ b/py23-image/centos-8/Dockerfile @@ -3,7 +3,8 @@ FROM ${SPLUNK_PRODUCT}-centos-8:latest USER root RUN yum -y update -RUN yum -y install gcc openssl-devel bzip2-devel libffi-devel python-pip +RUN yum -y install gcc openssl-devel bzip2-devel libffi-devel python3-pip python2 python2-pip + # manual installation of python 3.7 as default distro version is 3.6 RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \ && tar xzf Python-3.7.4.tgz \ @@ -19,6 +20,9 @@ RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \ # pip version is not automatically "fixed", unlike debian-based && ln -sf /usr/bin/pip2 /usr/bin/pip \ && ln -sf /usr/bin/pip3.7 /usr/bin/pip3 + # add python alias + # && ln -s /bin/python3 /bin/python + RUN yum remove -y --setopt=tsflags=noscripts gcc openssl-devel bzip2-devel libffi-devel \ && yum autoremove -y \ && yum clean all From 41df38e8c9a0ddd4945c2c559f67ae7972e1ff47 Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Fri, 25 Sep 2020 15:07:42 -0700 Subject: [PATCH 4/4] Support for createdefaults.py py2/py3 --- base/centos-7/Dockerfile | 2 +- base/centos-7/install.sh | 2 +- base/centos-8/Dockerfile | 2 +- base/centos-8/install.sh | 2 +- splunk/common-files/createdefaults.py | 13 +++++++---- uf/common-files/createdefaults.py | 31 ++++++++++++++++++++------- 6 files changed, 36 insertions(+), 16 deletions(-) diff --git a/base/centos-7/Dockerfile b/base/centos-7/Dockerfile index 9421d5f5..cd68c8ac 100644 --- a/base/centos-7/Dockerfile +++ b/base/centos-7/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2018 Splunk +# Copyright 2018-2020 Splunk # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/base/centos-7/install.sh b/base/centos-7/install.sh index 7de4088b..7309a0ed 100755 --- a/base/centos-7/install.sh +++ b/base/centos-7/install.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018 Splunk +# Copyright 2018-2020 Splunk # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/base/centos-8/Dockerfile b/base/centos-8/Dockerfile index 142c6b8d..c56bde27 100644 --- a/base/centos-8/Dockerfile +++ b/base/centos-8/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2020 Splunk +# Copyright 2018-2020 Splunk # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/base/centos-8/install.sh b/base/centos-8/install.sh index 0e4f1c34..d304275e 100755 --- a/base/centos-8/install.sh +++ b/base/centos-8/install.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 Splunk +# Copyright 2018-2020 Splunk # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/splunk/common-files/createdefaults.py b/splunk/common-files/createdefaults.py index 944c83c7..df5b60a5 100755 --- a/splunk/common-files/createdefaults.py +++ b/splunk/common-files/createdefaults.py @@ -1,5 +1,5 @@ #! /usr/bin/python -# Copyright 2018 Splunk +# Copyright 2018-2020 Splunk # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import six import sys import uuid import random @@ -32,10 +33,14 @@ def random_generator(size=24): # Use System Random for rng = random.SystemRandom() - bytes = [chr(rng.randrange(256)) for i in range(size)] - s = ''.join(bytes) + b = [chr(rng.randrange(256)) for i in range(size)] + s = ''.join(b) + if six.PY2: + s = base64.b64encode(s) + else: + s = base64.b64encode(s.encode()).decode() + return s - return base64.b64encode(s) # if there are no environment vars set, lets make some safe defaults if not splunk_hec_token: diff --git a/uf/common-files/createdefaults.py b/uf/common-files/createdefaults.py index fcf3da6b..df5b60a5 100755 --- a/uf/common-files/createdefaults.py +++ b/uf/common-files/createdefaults.py @@ -1,5 +1,5 @@ #! /usr/bin/python -# Copyright 2018 Splunk +# Copyright 2018-2020 Splunk # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +import six import sys import uuid import random @@ -25,15 +26,21 @@ splunk_hec_token = os.environ.get("SPLUNK_HEC_TOKEN", None) splunk_password = os.environ.get("SPLUNK_PASSWORD", None) splunk_idxc_secret = os.environ.get("SPLUNK_IDXC_SECRET", None) +splunk_idxc_pass4SymmKey = os.environ.get("SPLUNK_IDXC_PASS4SYMMKEY", None) splunk_shc_secret = os.environ.get("SPLUNK_SHC_SECRET", None) +splunk_shc_pass4SymmKey = os.environ.get("SPLUNK_SHC_PASS4SYMMKEY", None) def random_generator(size=24): # Use System Random for rng = random.SystemRandom() - bytes = [chr(rng.randrange(256)) for i in range(size)] - s = ''.join(bytes) + b = [chr(rng.randrange(256)) for i in range(size)] + s = ''.join(b) + if six.PY2: + s = base64.b64encode(s) + else: + s = base64.b64encode(s.encode()).decode() + return s - return base64.b64encode(s) # if there are no environment vars set, lets make some safe defaults if not splunk_hec_token: @@ -41,10 +48,18 @@ def random_generator(size=24): os.environ["SPLUNK_HEC_TOKEN"] = str(tempuuid) if not splunk_password: os.environ["SPLUNK_PASSWORD"] = random_generator() -if not splunk_idxc_secret: - os.environ["SPLUNK_IDXC_SECRET"] = random_generator() -if not splunk_shc_secret: - os.environ["SPLUNK_SHC_SECRET"] = random_generator() +if splunk_idxc_pass4SymmKey: + os.environ["SPLUNK_IDXC_PASS4SYMMKEY"] = os.environ["SPLUNK_IDXC_SECRET"] = splunk_idxc_pass4SymmKey +elif splunk_idxc_secret: + os.environ["SPLUNK_IDXC_PASS4SYMMKEY"] = os.environ["SPLUNK_IDXC_SECRET"] = splunk_idxc_secret +else: + os.environ["SPLUNK_IDXC_PASS4SYMMKEY"] = os.environ["SPLUNK_IDXC_SECRET"] = random_generator() +if splunk_shc_secret: + os.environ["SPLUNK_SHC_PASS4SYMMKEY"] = os.environ["SPLUNK_SHC_SECRET"] = splunk_shc_pass4SymmKey +elif splunk_shc_pass4SymmKey: + os.environ["SPLUNK_SHC_PASS4SYMMKEY"] = os.environ["SPLUNK_SHC_SECRET"] = splunk_shc_secret +else: + os.environ["SPLUNK_SHC_PASS4SYMMKEY"] = os.environ["SPLUNK_SHC_SECRET"] = random_generator() sys.argv.append("--write-to-stdout") import environ environ.main()