dev to master

base/debian-10/install.sh

@@ -31,9 +31,11 @@ ln -sf /usr/share/zoneinfo/UTC /etc/localtime
 apt update
 
 # put back tools for customer support
-apt-get install -y --no-install-recommends curl sudo libgssapi-krb5-2 busybox procps acl gcc libssl-dev libffi-dev python2-dev
+apt-get install -y --no-install-recommends curl sudo libgssapi-krb5-2 busybox procps acl gcc libpython-dev libffi-dev libssl-dev
 apt-get install -y --no-install-recommends python-pip python-setuptools python-requests python-yaml
 pip --no-cache-dir install ansible
+apt-get remove -y gcc libffi-dev libssl-dev libpython-dev
+apt-get autoremove -y
 
 cd /bin
 ln -s busybox killall

base/redhat-8/install.sh

@@ -26,14 +26,14 @@ microdnf -y --nodocs install glibc-langpack-en
 #We get around the gen above by forcing the language install, and then point to it.
 export LANG=en_US.utf8
 
-microdnf -y --nodocs install wget sudo shadow-utils procps
+microdnf -y --nodocs install wget sudo shadow-utils procps tar
 #install busybox direct from the multiarch since epel isn't availible yet for redhat8
 wget -O /bin/busybox https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-`arch`
 chmod +x /bin/busybox
-microdnf -y --nodocs install gcc redhat-rpm-config python2-devel libffi-devel openssl-devel tar
+microdnf -y --nodocs install python2-pip python2-devel redhat-rpm-config gcc libffi-devel openssl-devel
 pip2 --no-cache-dir install requests ansible
-microdnf -y remove gcc libffi-devel openssl-devel
-microdnf clean all
+microdnf -y remove gcc libffi-devel openssl-devel redhat-rpm-config python2-devel device-mapper-libs device-mapper cryptsetup-libs systemd systemd-pam dbus dbus-common dbus-daemon dbus-tools dbus-libs go-srpm-macros iptables-libs ocaml-srpm-macros openblas-srpm-macros qt5-srpm-macros perl-srpm-macros rust-srpm-macros ghc-srpm-macros platform-python python3-rpm-generators platform-python-setuptools python3-libs platform-python-pip python3-rpm-generators python3-rpm-macros elfutils-libs efi-srpm-macros zip unzip xkeyboard-config libxkbcommon redhat-rpm-config util-linux dwz file file-libs findutils iptables-libs diffutils annobin python-rpm-macros python-srpm-macros python2-devel python2-rpm-macros kmod-libs libfdisk libffi-devel libpcap libseccomp libutempter
+
 
 cd /bin
 ln -s python2 python || true

docs/CHANGELOG.md

@@ -2,10 +2,11 @@
 
 ## Navigation
 
+* [8.0.0](#800)
 * [7.3.2](#732)
 * [7.3.1](#731)
 * [7.3.0](#730)
-* [7.2.7](#728)
+* [7.2.8](#728)
 * [7.2.7](#727)
 * [7.2.6](#726)
 * [7.2.5.1](#7251)
@@ -18,6 +19,23 @@
 
 ---
 
+## 8.0.0
+
+#### What's New?
+* New Splunk Enterprise release of 8.0.0
+
+#### docker-splunk changes:
+* Bumping Splunk version. For details, see: https://docs.splunk.com/Documentation/Splunk/8.0.0/ReleaseNotes/Fixedissues
+* Reduced base image size due to package management inflation
+* Additional Python 2/Python 3 compatibility changes
+
+#### splunk-ansible changes:
+* Increasing delay intervals to better handle different platforms
+* Adding vars needed for Ansible Galaxy
+* Bugfix for pre-playbook tasks not supporting URLs
+
+---
+
 ## 7.3.2
 
 #### What's New?

py23-image/centos-7/Dockerfile

@@ -3,5 +3,23 @@ FROM ${SPLUNK_PRODUCT}-centos-7:latest
 USER root
 
 RUN yum -y update
-RUN yum -y install python36 python36-requests
-RUN python3 -m ensurepip
+RUN yum -y install gcc openssl-devel bzip2-devel libffi-devel python-pip
+# INFRA-15385: manual installation of python 3.7 as default distro version is 3.6
+RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \
+    && tar xzf Python-3.7.4.tgz \
+    && cd Python-3.7.4 \
+    && ./configure --enable-optimizations --prefix=/usr \
+    && make install \
+    && cd .. \
+    && rm Python-3.7.4.tgz \
+    && rm -r Python-3.7.4 \
+    && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
+    && python3.7 get-pip.py \
+    && rm -f get-pip.py \
+    #pip version is not automatically "fixed", unlike debian-based
+    && ln -sf /usr/bin/pip2 /usr/bin/pip \
+    && ln -sf /usr/bin/pip3.7 /usr/bin/pip3
+RUN yum remove -y --setopt=tsflags=noscripts gcc openssl-devel bzip2-devel libffi-devel \
+    && yum autoremove -y \
+    && yum clean all
+RUN pip3 --no-cache-dir install ansible requests

py23-image/debian-10/Dockerfile

@@ -2,6 +2,6 @@ ARG SPLUNK_PRODUCT=splunk
 FROM ${SPLUNK_PRODUCT}-debian-10:latest
 USER root
 
-RUN apt update
+RUN apt-get update
 RUN apt-get install -y --no-install-recommends python3 python3-pip python3-setuptools python3-requests python3-yaml
 RUN pip3 --no-cache-dir install ansible

py23-image/debian-9/Dockerfile

@@ -3,4 +3,25 @@ FROM ${SPLUNK_PRODUCT}-debian-9:latest
 USER root
 
 RUN apt-get update
-RUN apt-get install -y --no-install-recommends python3 python3-pip python3-requests
+RUN apt-get install -y gcc make \
+    build-essential checkinstall libreadline-gplv2-dev libncursesw5-dev libssl-dev \
+    libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev
+# INFRA-15385: manual installation of python 3.7 as default distro version is 3.5
+RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \
+    && tar xzf Python-3.7.4.tgz \
+    && cd Python-3.7.4 \
+    && ./configure --enable-optimizations --prefix=/usr \
+    && make install \
+    && cd .. \
+    && rm Python-3.7.4.tgz \
+    && rm -r Python-3.7.4 \
+    && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
+    && python3.7 get-pip.py \
+    && rm -f get-pip.py \
+    && ln -s /usr/share/pyshared/lsb_release.py /usr/lib/python3.7/site-packages/lsb_release.py
+#removing all intermediate dependencies. All the stuff below comes up to 200+MB
+RUN apt-get remove --purge -y gcc make build-essential checkinstall libreadline-gplv2-dev libncursesw5-dev \
+    libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev \
+    && apt autoremove -y \
+    && apt autoclean
+RUN pip3 --no-cache-dir install ansible requests

py23-image/redhat-8/Dockerfile

@@ -2,6 +2,24 @@ ARG SPLUNK_PRODUCT=splunk
 FROM ${SPLUNK_PRODUCT}-redhat-8:latest
 USER root
 
-RUN microdnf -y --nodocs install python3
-RUN alternatives --set python /usr/bin/python2
+RUN microdnf -y update \
+    && microdnf -y install make gcc openssl-devel bzip2-devel libffi-devel
+# INFRA-15385: manual installation of python 3.7 as default distro version is 3.6
+RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \
+    && tar xzf Python-3.7.4.tgz \
+    && cd Python-3.7.4 \
+    && ./configure --enable-optimizations --prefix=/usr \
+    && make install \
+    && cd .. \
+    && rm Python-3.7.4.tgz \
+    && rm -r Python-3.7.4 \
+    && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
+    && python3.7 get-pip.py \
+    && rm -f get-pip.py \
+    #pip version is not automatically "fixed", unlike debian-based
+    && ln -sf /usr/bin/pip2 /usr/bin/pip \
+    && ln -sf /usr/bin/pip3.7 /usr/bin/pip3
+#microdnf persists metadata which is a problem for removing packages. So have to clean first before removing. 
+RUN microdnf clean all \
+	&& microdnf remove -y gcc make openssl-devel bzip2-devel libffi-devel
 RUN pip3 -q --no-cache-dir install requests ansible

splunk/common-files/make-minimal-exclude.py

@@ -9,7 +9,6 @@
 */bin/*mongo*
 */3rdparty/Copyright-for-mongo*
 */bin/node*
-*/bin/parsetest*
 */bin/pcregextest*
 */etc/*.lic*
 */etc/anonymizer*
@@ -34,13 +33,15 @@
 m = re.match(".*splunk-([0-9]+)\.([0-9]+)\.[0-9]+\.?[0-9]?-[0-9a-z]+-Linux-[0-9a-z_-]+.tgz", sys.argv[1])
 
 if m and m.group(1):
-    if m.group(1) == "7":
-        print EXCLUDE_V7
+    print(EXCLUDE_V7)
+    if int(m.group(1)) == 7:
+        print("*/bin/parsetest*")
         if int(m.group(2)) < 3:
-            print "*/etc/apps/framework*"
-            print "*/etc/apps/gettingstarted*"
+            print("*/etc/apps/framework*")
+            print("*/etc/apps/gettingstarted*")
         else:
-            print "*/etc/apps/splunk_metrics_workspace*"
+            print("*/etc/apps/splunk_metrics_workspace*")
     elif int(m.group(1)) > 7:
-        print EXCLUDE_V7
-        print "*/etc/apps/splunk_metrics_workspace*"
+        print("*/etc/apps/splunk_metrics_workspace*")
+        if int(m.group(2)) < 1:
+            print("*/bin/parsetest*")

tests/test_docker_splunk.py

@@ -145,13 +145,14 @@ def get_number_of_containers(self, filename):
             yml = yaml.load(f)
         return len(yml["services"])
 
-    def wait_for_containers(self, count, label=None, name=None):
+    def wait_for_containers(self, count, label=None, name=None, timeout=300):
         '''
         NOTE: This helper method can only be used for `compose up` scenarios where self.project_name is defined
         '''
         start = time.time()
         end = start
-        while end-start < 300:
+        # Wait 
+        while end-start < timeout:
             filters = {}
             if name:
                 filters["name"] = name
@@ -182,8 +183,8 @@ def wait_for_containers(self, count, label=None, name=None):
         return True
 
     def handle_request_retry(self, method, url, kwargs):
-        RETRIES = 6
-        IMPLICIT_WAIT = 3
+        RETRIES = 10
+        IMPLICIT_WAIT = 6
         for n in range(RETRIES):
             try:
                 self.logger.info("Attempt #{}: running {} against {} with kwargs {}".format(n+1, method, url, kwargs))
@@ -2125,7 +2126,7 @@ def test_compose_1idx3sh1cm1dep(self):
             container_count, rc = self.compose_up()
             assert rc == 0
             # Wait for containers to come up
-            assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name))
+            assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name), timeout=600)
             # Get container logs
             container_mapping = {"sh1": "sh", "sh2": "sh", "sh3": "sh", "cm1": "cm", "idx1": "idx", "dep1": "dep"}
             for container in container_mapping:
@@ -2169,11 +2170,23 @@ def test_compose_1idx3sh1cm1dep(self):
                     status, content = self.handle_request_retry("GET", url, kwargs)
                     assert json.loads(content)["entry"][0]["content"]["preferred_captain"] == "1"
             # Search results won't return the correct results immediately :(
-            time.sleep(15)
-            search_providers, distinct_hosts = self.search_internal_distinct_hosts("sh1", password=self.password)
-            assert len(search_providers) == 2
-            assert "idx1" in search_providers and "sh1" in search_providers
-            assert distinct_hosts == 6
+            time.sleep(30)
+            RETRIES = 10
+            IMPLICIT_WAIT = 6
+            for n in range(RETRIES):
+                try:
+                    self.logger.info("Attempt #{}: checking internal search host count".format(n+1))
+                    search_providers, distinct_hosts = self.search_internal_distinct_hosts("sh1", password=self.password)
+                    assert len(search_providers) == 2
+                    assert "idx1" in search_providers and "sh1" in search_providers
+                    assert distinct_hosts == 6
+                    break
+                except Exception as e:
+                    self.logger.error("Attempt #{} error: {}".format(n+1, str(e)))
+                    if n < RETRIES-1:
+                        time.sleep(IMPLICIT_WAIT)
+                        continue
+                    raise e
         except Exception as e:
             self.logger.error(e)
             raise e