diff --git a/base/debian-10/install.sh b/base/debian-10/install.sh index 69606e16..1e2f7d0e 100755 --- a/base/debian-10/install.sh +++ b/base/debian-10/install.sh @@ -31,9 +31,11 @@ ln -sf /usr/share/zoneinfo/UTC /etc/localtime apt update # put back tools for customer support -apt-get install -y --no-install-recommends curl sudo libgssapi-krb5-2 busybox procps acl gcc libssl-dev libffi-dev python2-dev +apt-get install -y --no-install-recommends curl sudo libgssapi-krb5-2 busybox procps acl gcc libpython-dev libffi-dev libssl-dev apt-get install -y --no-install-recommends python-pip python-setuptools python-requests python-yaml pip --no-cache-dir install ansible +apt-get remove -y gcc libffi-dev libssl-dev libpython-dev +apt-get autoremove -y cd /bin ln -s busybox killall diff --git a/base/redhat-8/install.sh b/base/redhat-8/install.sh index f44c8ffb..73cc73cc 100755 --- a/base/redhat-8/install.sh +++ b/base/redhat-8/install.sh @@ -26,14 +26,14 @@ microdnf -y --nodocs install glibc-langpack-en #We get around the gen above by forcing the language install, and then point to it. export LANG=en_US.utf8 -microdnf -y --nodocs install wget sudo shadow-utils procps +microdnf -y --nodocs install wget sudo shadow-utils procps tar #install busybox direct from the multiarch since epel isn't availible yet for redhat8 wget -O /bin/busybox https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-`arch` chmod +x /bin/busybox -microdnf -y --nodocs install gcc redhat-rpm-config python2-devel libffi-devel openssl-devel tar +microdnf -y --nodocs install python2-pip python2-devel redhat-rpm-config gcc libffi-devel openssl-devel pip2 --no-cache-dir install requests ansible -microdnf -y remove gcc libffi-devel openssl-devel -microdnf clean all +microdnf -y remove gcc libffi-devel openssl-devel redhat-rpm-config python2-devel device-mapper-libs device-mapper cryptsetup-libs systemd systemd-pam dbus dbus-common dbus-daemon dbus-tools dbus-libs go-srpm-macros iptables-libs ocaml-srpm-macros openblas-srpm-macros qt5-srpm-macros perl-srpm-macros rust-srpm-macros ghc-srpm-macros platform-python python3-rpm-generators platform-python-setuptools python3-libs platform-python-pip python3-rpm-generators python3-rpm-macros elfutils-libs efi-srpm-macros zip unzip xkeyboard-config libxkbcommon redhat-rpm-config util-linux dwz file file-libs findutils iptables-libs diffutils annobin python-rpm-macros python-srpm-macros python2-devel python2-rpm-macros kmod-libs libfdisk libffi-devel libpcap libseccomp libutempter + cd /bin ln -s python2 python || true diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 551bbcf8..227d144e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,10 +2,11 @@ ## Navigation +* [8.0.0](#800) * [7.3.2](#732) * [7.3.1](#731) * [7.3.0](#730) -* [7.2.7](#728) +* [7.2.8](#728) * [7.2.7](#727) * [7.2.6](#726) * [7.2.5.1](#7251) @@ -18,6 +19,23 @@ --- +## 8.0.0 + +#### What's New? +* New Splunk Enterprise release of 8.0.0 + +#### docker-splunk changes: +* Bumping Splunk version. For details, see: https://docs.splunk.com/Documentation/Splunk/8.0.0/ReleaseNotes/Fixedissues +* Reduced base image size due to package management inflation +* Additional Python 2/Python 3 compatibility changes + +#### splunk-ansible changes: +* Increasing delay intervals to better handle different platforms +* Adding vars needed for Ansible Galaxy +* Bugfix for pre-playbook tasks not supporting URLs + +--- + ## 7.3.2 #### What's New? diff --git a/py23-image/centos-7/Dockerfile b/py23-image/centos-7/Dockerfile index a113bab0..4e907aff 100644 --- a/py23-image/centos-7/Dockerfile +++ b/py23-image/centos-7/Dockerfile @@ -3,5 +3,23 @@ FROM ${SPLUNK_PRODUCT}-centos-7:latest USER root RUN yum -y update -RUN yum -y install python36 python36-requests -RUN python3 -m ensurepip \ No newline at end of file +RUN yum -y install gcc openssl-devel bzip2-devel libffi-devel python-pip +# INFRA-15385: manual installation of python 3.7 as default distro version is 3.6 +RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \ + && tar xzf Python-3.7.4.tgz \ + && cd Python-3.7.4 \ + && ./configure --enable-optimizations --prefix=/usr \ + && make install \ + && cd .. \ + && rm Python-3.7.4.tgz \ + && rm -r Python-3.7.4 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3.7 get-pip.py \ + && rm -f get-pip.py \ + #pip version is not automatically "fixed", unlike debian-based + && ln -sf /usr/bin/pip2 /usr/bin/pip \ + && ln -sf /usr/bin/pip3.7 /usr/bin/pip3 +RUN yum remove -y --setopt=tsflags=noscripts gcc openssl-devel bzip2-devel libffi-devel \ + && yum autoremove -y \ + && yum clean all +RUN pip3 --no-cache-dir install ansible requests \ No newline at end of file diff --git a/py23-image/debian-10/Dockerfile b/py23-image/debian-10/Dockerfile index c190369f..218caf2c 100644 --- a/py23-image/debian-10/Dockerfile +++ b/py23-image/debian-10/Dockerfile @@ -2,6 +2,6 @@ ARG SPLUNK_PRODUCT=splunk FROM ${SPLUNK_PRODUCT}-debian-10:latest USER root -RUN apt update +RUN apt-get update RUN apt-get install -y --no-install-recommends python3 python3-pip python3-setuptools python3-requests python3-yaml RUN pip3 --no-cache-dir install ansible \ No newline at end of file diff --git a/py23-image/debian-9/Dockerfile b/py23-image/debian-9/Dockerfile index 159d497f..2a35ca19 100644 --- a/py23-image/debian-9/Dockerfile +++ b/py23-image/debian-9/Dockerfile @@ -3,4 +3,25 @@ FROM ${SPLUNK_PRODUCT}-debian-9:latest USER root RUN apt-get update -RUN apt-get install -y --no-install-recommends python3 python3-pip python3-requests \ No newline at end of file +RUN apt-get install -y gcc make \ + build-essential checkinstall libreadline-gplv2-dev libncursesw5-dev libssl-dev \ + libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev +# INFRA-15385: manual installation of python 3.7 as default distro version is 3.5 +RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \ + && tar xzf Python-3.7.4.tgz \ + && cd Python-3.7.4 \ + && ./configure --enable-optimizations --prefix=/usr \ + && make install \ + && cd .. \ + && rm Python-3.7.4.tgz \ + && rm -r Python-3.7.4 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3.7 get-pip.py \ + && rm -f get-pip.py \ + && ln -s /usr/share/pyshared/lsb_release.py /usr/lib/python3.7/site-packages/lsb_release.py +#removing all intermediate dependencies. All the stuff below comes up to 200+MB +RUN apt-get remove --purge -y gcc make build-essential checkinstall libreadline-gplv2-dev libncursesw5-dev \ + libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev \ + && apt autoremove -y \ + && apt autoclean +RUN pip3 --no-cache-dir install ansible requests \ No newline at end of file diff --git a/py23-image/redhat-8/Dockerfile b/py23-image/redhat-8/Dockerfile index 6d035fcd..21695cdb 100644 --- a/py23-image/redhat-8/Dockerfile +++ b/py23-image/redhat-8/Dockerfile @@ -2,6 +2,24 @@ ARG SPLUNK_PRODUCT=splunk FROM ${SPLUNK_PRODUCT}-redhat-8:latest USER root -RUN microdnf -y --nodocs install python3 -RUN alternatives --set python /usr/bin/python2 +RUN microdnf -y update \ + && microdnf -y install make gcc openssl-devel bzip2-devel libffi-devel +# INFRA-15385: manual installation of python 3.7 as default distro version is 3.6 +RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \ + && tar xzf Python-3.7.4.tgz \ + && cd Python-3.7.4 \ + && ./configure --enable-optimizations --prefix=/usr \ + && make install \ + && cd .. \ + && rm Python-3.7.4.tgz \ + && rm -r Python-3.7.4 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3.7 get-pip.py \ + && rm -f get-pip.py \ + #pip version is not automatically "fixed", unlike debian-based + && ln -sf /usr/bin/pip2 /usr/bin/pip \ + && ln -sf /usr/bin/pip3.7 /usr/bin/pip3 +#microdnf persists metadata which is a problem for removing packages. So have to clean first before removing. +RUN microdnf clean all \ + && microdnf remove -y gcc make openssl-devel bzip2-devel libffi-devel RUN pip3 -q --no-cache-dir install requests ansible \ No newline at end of file diff --git a/splunk/common-files/make-minimal-exclude.py b/splunk/common-files/make-minimal-exclude.py index 52f23513..36876764 100755 --- a/splunk/common-files/make-minimal-exclude.py +++ b/splunk/common-files/make-minimal-exclude.py @@ -9,7 +9,6 @@ */bin/*mongo* */3rdparty/Copyright-for-mongo* */bin/node* -*/bin/parsetest* */bin/pcregextest* */etc/*.lic* */etc/anonymizer* @@ -34,13 +33,15 @@ m = re.match(".*splunk-([0-9]+)\.([0-9]+)\.[0-9]+\.?[0-9]?-[0-9a-z]+-Linux-[0-9a-z_-]+.tgz", sys.argv[1]) if m and m.group(1): - if m.group(1) == "7": - print EXCLUDE_V7 + print(EXCLUDE_V7) + if int(m.group(1)) == 7: + print("*/bin/parsetest*") if int(m.group(2)) < 3: - print "*/etc/apps/framework*" - print "*/etc/apps/gettingstarted*" + print("*/etc/apps/framework*") + print("*/etc/apps/gettingstarted*") else: - print "*/etc/apps/splunk_metrics_workspace*" + print("*/etc/apps/splunk_metrics_workspace*") elif int(m.group(1)) > 7: - print EXCLUDE_V7 - print "*/etc/apps/splunk_metrics_workspace*" + print("*/etc/apps/splunk_metrics_workspace*") + if int(m.group(2)) < 1: + print("*/bin/parsetest*") diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index ab52442e..dc26794d 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -145,13 +145,14 @@ def get_number_of_containers(self, filename): yml = yaml.load(f) return len(yml["services"]) - def wait_for_containers(self, count, label=None, name=None): + def wait_for_containers(self, count, label=None, name=None, timeout=300): ''' NOTE: This helper method can only be used for `compose up` scenarios where self.project_name is defined ''' start = time.time() end = start - while end-start < 300: + # Wait + while end-start < timeout: filters = {} if name: filters["name"] = name @@ -182,8 +183,8 @@ def wait_for_containers(self, count, label=None, name=None): return True def handle_request_retry(self, method, url, kwargs): - RETRIES = 6 - IMPLICIT_WAIT = 3 + RETRIES = 10 + IMPLICIT_WAIT = 6 for n in range(RETRIES): try: self.logger.info("Attempt #{}: running {} against {} with kwargs {}".format(n+1, method, url, kwargs)) @@ -2125,7 +2126,7 @@ def test_compose_1idx3sh1cm1dep(self): container_count, rc = self.compose_up() assert rc == 0 # Wait for containers to come up - assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) + assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name), timeout=600) # Get container logs container_mapping = {"sh1": "sh", "sh2": "sh", "sh3": "sh", "cm1": "cm", "idx1": "idx", "dep1": "dep"} for container in container_mapping: @@ -2169,11 +2170,23 @@ def test_compose_1idx3sh1cm1dep(self): status, content = self.handle_request_retry("GET", url, kwargs) assert json.loads(content)["entry"][0]["content"]["preferred_captain"] == "1" # Search results won't return the correct results immediately :( - time.sleep(15) - search_providers, distinct_hosts = self.search_internal_distinct_hosts("sh1", password=self.password) - assert len(search_providers) == 2 - assert "idx1" in search_providers and "sh1" in search_providers - assert distinct_hosts == 6 + time.sleep(30) + RETRIES = 10 + IMPLICIT_WAIT = 6 + for n in range(RETRIES): + try: + self.logger.info("Attempt #{}: checking internal search host count".format(n+1)) + search_providers, distinct_hosts = self.search_internal_distinct_hosts("sh1", password=self.password) + assert len(search_providers) == 2 + assert "idx1" in search_providers and "sh1" in search_providers + assert distinct_hosts == 6 + break + except Exception as e: + self.logger.error("Attempt #{} error: {}".format(n+1, str(e))) + if n < RETRIES-1: + time.sleep(IMPLICIT_WAIT) + continue + raise e except Exception as e: self.logger.error(e) raise e