Merge pull request #498 from splunk/develop

Release/8.2.1

.circleci/config.yml

@@ -86,7 +86,7 @@ jobs:
             mkdir -p trivy-results
             for image in ./workspace_cache/*.tar; do
               [ -e "$image" ] || continue
-              ./trivy image --exit-code 1 --format template --template "@contrib/junit.tpl" -o trivy-results/`basename $image`.xml --ignore-unfixed --severity "HIGH,CRITICAL" --no-progress -i "$image"
+              ./trivy image --exit-code 1 --format template --vuln-type os --template "@contrib/junit.tpl" -o trivy-results/`basename $image`.xml --ignore-unfixed --severity "HIGH,CRITICAL" --no-progress -i "$image"
             done
       - store_test_results:
           path: trivy-results

.trivyignore

@@ -1,9 +1,3 @@
-# These are coming from the file below, but it's not actually installed in the container
-# /usr/lib/python3.7/site-packages/ansible_collections/netbox/netbox/poetry.lock
-CVE-2020-36242
-CVE-2020-14343
-CVE-2021-29510
-
 # Downgraded to LOW severity by the Product Security team as the packages are not actually
 # shipped with the release.
 # SPL-203200

Makefile

@@ -7,8 +7,8 @@ SPLUNK_ANSIBLE_BRANCH ?= develop
 SPLUNK_COMPOSE ?= cluster_absolute_unit.yaml
 # Set Splunk version/build parameters here to define downstream URLs and file names
 SPLUNK_PRODUCT := splunk
-SPLUNK_VERSION := 8.2.0
-SPLUNK_BUILD := e053ef3c985f
+SPLUNK_VERSION := 8.2.1
+SPLUNK_BUILD := ddff1c41e5cf
 ifeq ($(shell arch), s390x)
 	SPLUNK_ARCH = s390x
 else

README.md

@@ -9,7 +9,8 @@ Welcome to the official Splunk repository of Dockerfiles for building Splunk Ent
 ----
 
 > :warning: **DEPRECATION NOTICE**  
-We will no longer be releasing Debian images on Docker Hub after Spring of 2021.
+We are no longer releasing Debian images on Docker Hub as of May 2021 (Splunk Enterprise v8.2.0+).
+Red Hat images will continue to be published.
 
 ----
 

base/debian-10/install.sh

@@ -32,8 +32,8 @@ ln -sf /usr/share/zoneinfo/UTC /etc/localtime
 # Install utility packages
 apt-get install -y --no-install-recommends curl sudo libgssapi-krb5-2 busybox procps acl gcc make \
                                            libffi-dev libssl-dev make build-essential libbz2-dev \
-                                           wget xz-utils ca-certificates zlib1g-dev python3-apt \
-                                           p11-kit lz4
+                                           wget xz-utils ca-certificates zlib1g-dev python3-apt p11-kit lz4 \
+                                           libhogweed4=3.4.1-1+deb10u1 libgnutls30=3.6.7-4+deb10u7 libgcrypt20=1.8.4-5+deb10u1
 
 # Install Python and necessary packages
 PY_SHORT=${PYTHON_VERSION%.*}
@@ -65,7 +65,7 @@ cp apt_inst.cpython-37m-${ARCH}-linux-gnu.so apt_inst.so
 rm -rf /tmp/python3-apt
 # Install splunk-ansible dependencies
 cd /
-pip -q --no-cache-dir install six wheel requests cryptography==3.3.2 ansible jmespath --upgrade
+pip -q --no-cache-dir install six wheel requests cryptography==3.3.2 ansible==3.4.0 urllib3==1.26.5 jmespath --upgrade
 # Remove tests packaged in python libs
 find /usr/lib/ -depth \( -type d -a -not -wholename '*/ansible/plugins/test' -a \( -name test -o -name tests -o -name idle_test \) \) -exec rm -rf '{}' \;
 find /usr/lib/ -depth \( -type f -a -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) -exec rm -rf '{}' \;

base/debian-9/install.sh

@@ -32,14 +32,15 @@ ln -sf /usr/share/zoneinfo/UTC /etc/localtime
 # Install utility packages
 apt-get install -y --no-install-recommends curl sudo libgssapi-krb5-2 busybox procps acl gcc make \
                                            libffi-dev libssl-dev make build-essential libbz2-dev \
-                                           wget xz-utils ca-certificates zlib1g-dev lz4
+                                           wget xz-utils ca-certificates zlib1g-dev liblz4-dev
 
 # Install Python and necessary packages
 PY_SHORT=${PYTHON_VERSION%.*}
 wget -O /tmp/python.tgz https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
 wget -O /tmp/Python-gpg-sig-${PYTHON_VERSION}.tgz.asc https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz.asc
 apt-get install dirmngr -y
-gpg --keyserver pool.sks-keyservers.net --recv-keys $PYTHON_GPG_KEY_ID \
+gpg --keyserver keys.openpgp.org --recv-keys $PYTHON_GPG_KEY_ID \
+    || gpg --keyserver pool.sks-keyservers.net --recv-keys $PYTHON_GPG_KEY_ID \
     || gpg --keyserver pgp.mit.edu --recv-keys $PYTHON_GPG_KEY_ID \
     || gpg --keyserver keyserver.pgp.com --recv-keys $PYTHON_GPG_KEY_ID
 gpg --verify /tmp/Python-gpg-sig-${PYTHON_VERSION}.tgz.asc /tmp/python.tgz
@@ -65,7 +66,7 @@ cp apt_inst.cpython-35m-x86_64-linux-gnu.so apt_inst.so
 rm -rf /tmp/python3-apt
 # Install splunk-ansible dependencies
 cd /
-pip -q --no-cache-dir install six wheel requests cryptography==3.3.2 ansible jmespath --upgrade
+pip -q --no-cache-dir install six wheel requests cryptography==3.3.2 ansible==3.4.0 urllib3==1.26.5 jmespath --upgrade
 # Remove tests packaged in python libs
 find /usr/lib/ -depth \( -type d -a -not -wholename '*/ansible/plugins/test' -a \( -name test -o -name tests -o -name idle_test \) \) -exec rm -rf '{}' \;
 find /usr/lib/ -depth \( -type f -a -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) -exec rm -rf '{}' \;

base/redhat-8/Dockerfile

@@ -16,7 +16,7 @@
 # the container catalog moved from registry.access.redhat.com to registry.redhat.io
 # So at some point before they deprecate the old registry we have to make sure that
 # we have access to the new registry and change where we pull the ubi image from.
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3-291
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4-200.1622548483
 
 LABEL name="splunk" \
       maintainer="support@splunk.com" \

base/redhat-8/install.sh

@@ -35,7 +35,8 @@ microdnf -y --nodocs update gnutls kernel-headers librepo libnghttp2 tzdata nett
 PY_SHORT=${PYTHON_VERSION%.*}
 wget -O /tmp/python.tgz https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
 wget -O /tmp/Python-gpg-sig-${PYTHON_VERSION}.tgz.asc https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz.asc
-gpg --keyserver pool.sks-keyservers.net --recv-keys $PYTHON_GPG_KEY_ID \
+gpg --keyserver keys.openpgp.org --recv-keys $PYTHON_GPG_KEY_ID \
+    || gpg --keyserver pool.sks-keyservers.net --recv-keys $PYTHON_GPG_KEY_ID \
     || gpg --keyserver pgp.mit.edu --recv-keys $PYTHON_GPG_KEY_ID \
     || gpg --keyserver keyserver.pgp.com --recv-keys $PYTHON_GPG_KEY_ID
 gpg --verify /tmp/Python-gpg-sig-${PYTHON_VERSION}.tgz.asc /tmp/python.tgz
@@ -51,7 +52,7 @@ ln -sf /usr/bin/python${PY_SHORT} /usr/bin/python
 ln -sf /usr/bin/pip${PY_SHORT} /usr/bin/pip
 # Install splunk-ansible dependencies
 cd /
-pip -q --no-cache-dir install six wheel requests cryptography==3.3.2 ansible==3.4.0 jmespath --upgrade
+pip -q --no-cache-dir install six wheel requests cryptography==3.3.2 ansible==3.4.0 urllib3==1.26.5 jmespath --upgrade
 # Remove tests packaged in python libs
 find /usr/lib/ -depth \( -type d -a -not -wholename '*/ansible/plugins/test' -a \( -name test -o -name tests -o -name idle_test \) \) -exec rm -rf '{}' \;
 find /usr/lib/ -depth \( -type f -a -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) -exec rm -rf '{}' \;

docs/CHANGELOG.md

@@ -3,12 +3,14 @@
 ----
 
 > :warning: **DEPRECATION NOTICE**  
-We will no longer be releasing Debian images on Docker Hub after Spring of 2021.
+We are no longer releasing Debian images on Docker Hub as of May 2021 (Splunk Enterprise v8.2.0+).
+Red Hat images will continue to be published.
 
 ----
 
 ## Navigation
 
+* [8.2.1](#821)
 * [8.2.0](#820)
 * [8.1.4](#814)
 * [8.1.3](#813)
@@ -57,10 +59,26 @@ We will no longer be releasing Debian images on Docker Hub after Spring of 2021.
 
 ---
 
+## 8.2.1
+
+#### What's New?
+* Releasing new images to support Splunk Enterprise release.
+
+#### docker-splunk changes:
+* Bumping Splunk version. For details, see [Fixed issues for 8.2.1](https://docs.splunk.com/Documentation/Splunk/8.2.1/ReleaseNotes/Fixedissues)
+* Updated teardown for more graceful shutdown
+* Bugfixes and documentation updates
+
+#### splunk-ansible changes:
+* Added support for installing apps locally on a Cluster Manager or Deployer instance using `apps_location_local`
+
+---
+
 ## 8.2.0
 
 #### What's New?
 * Releasing new images to support Splunk Enterprise release.
+* Deprecated Debian image releases. Red Hat images will continue to be published to Docker Hub and the Red Hat Container Catalog.
 
 #### docker-splunk changes:
 * Bumping Splunk version. For details, see [Release Notes for 8.2.0](https://docs.splunk.com/Documentation/Splunk/8.2.0/ReleaseNotes/)

docs/STORAGE_OPTIONS.md

@@ -1,8 +1,8 @@
 ## Data Storage ##
-This section will cover examples of different options for configuring data persistence. This includes both indexed data and configuration items. Splunk only supports data persistence to volumes mounted outside of the container. Data persistence for folders inside of the container is not supported. The following are intended as only as examples and unofficial guidelines.
+This section will cover examples of different options for configuring data persistence. This includes both indexed data and configuration items. Splunk only supports data persistence to volumes mounted outside of the container. Data persistence for folders inside of the container is not supported. The following are intended only as examples and unofficial guidelines.
 
 ### Storing indexes and search artifacts ###
-Splunk Enterprise, by default, Splunk Enterprise uses the var directory for indexes, search artifacts, etc. In the public image, the Splunk Enterprise home directory is /opt/splunk, and the indexes are configured to run under var/. If you want to persist the indexed data, then mount an external directory into the container under this folder.
+By default, Splunk Enterprise uses the var directory for indexes, search artifacts, etc. In the public image, the Splunk Enterprise home directory is /opt/splunk, and the indexes are configured to run under var/. If you want to persist the indexed data, then mount an external directory into the container under this folder.
 
 If you do not want to modify or persist any configuration changes made outside of what has been defined in the docker image file, then use the following steps for your service.
 
@@ -13,10 +13,10 @@ docker volume create so1-var
 ```
 See Docker's official documentation for more complete instructions and additional options.
 
-#### Step 2: Define the docker compose YAML  and start the service####
+#### Step 2: Define the docker-compose YAML  and start the service ####
 Using the Docker Compose format, save the following contents into a docker-compose.yml file:
 
-```
+```yaml
 version: "3.6"
 
 networks:
@@ -46,7 +46,7 @@ services:
       - so1-var:/opt/splunk/var
 ```
 
-This mounts only the contents of /opt/splunk/var, so anything outside of this folder will not persist. Any configuration changes will not remain when the container exits. Note that changes will persist between starting and stopping a container. See the Docker documentation for more discussion on the difference between starting, stopping, and exiting if the difference between them is unclear.
+This mounts only the contents of /opt/splunk/var, so anything outside this folder will not persist. Any configuration changes will not remain when the container exits. Note that changes will persist between starting and stopping a container. See the Docker documentation for more discussion on the difference between starting, stopping, and exiting if the difference between them is unclear.
 
 In the same directory as `docker-compose.yml`, run the following command to start the service.
 ```
@@ -61,21 +61,21 @@ docker volume inspect so1-var
 The output of that command should list where the data is stored.
 
 ### Storing indexes, search artifacts, and configuration changes ###
-In this section, we build off of the previous example to save the configuration as well. This can make it easier to save modified configurations, but simultaneously allows configuration drift to occur. If you want to keep configuration drift from happening, but still want to be able to persist some of the data, you can save off the specific "local" folders that you want the data to be persisted for (such as etc/system/local). However, be careful when doing this because you will both know what folders you need to save off and the number of volumes can proliferate rapidly - depending on the deployment. Please take the "Administrating Splunk" through Splunk Education prior to attempting this configuration.
+In this section, we build off of the previous example to save the configuration as well. This can make it easier to save modified configurations, but simultaneously allows configuration drift to occur. If you want to keep configuration drift from happening, but still want to persist some of the data, you can save off the specific "local" folders that you want the data to be persisted for (such as etc/system/local). However, be careful when doing this because you will both know what folders you need to save off and the number of volumes can increase rapidly - depending on the deployment. Please take the "Administrating Splunk" through Splunk Education before attempting this configuration.
 
-In these examples, we will assume that the entire etc folder is being mounted into the container.
+We will assume that the entire /etc folder is being mounted into the container in these examples.
 
 #### Step 1: Create a named volume ####
 Again, create a simple named volume in your Docker environment, run the following command
-```
+```shell
 docker volume create so1-etc
 ```
 See Docker's official documentation for more complete instructions and additional options.
 
 #### Step 2: Define the Docker Compose YAML ####
 Notice that this differs from the previous example by adding in the so1-etc volume references. In the following example, save the following data into a file named `docker-compose.yml`.
 
-```
+```yaml
 version: "3.6"
 
 networks:
@@ -104,19 +104,19 @@ services:
       - 8089
     volumes:
       - so1-var:/opt/splunk/var
-	    - so1-etc:/opt/splunk/etc
+      - so1-etc:/opt/splunk/etc
 ```
 
 In the same directory as `docker-compose.yml`, run the following command to start the service:
-```
+```shell
 docker-compose up
 ```
 
-When the volume is mounted the data will persist after the container exits. If a container has exited and restarted, but no data shows up, then check the volume definition and verify that the container did not create a new volume or that the volume mounted is in the same location.
+When the volume is mounted, the data will persist after the container exits. If a container has exited and restarted, but no data shows up, check the volume definition and verify that the container did not create a new volume or that the volume mounted is in the same location.
 
 #### Viewing the contents of the volume ####
-To view the etc directory outside of the container run one or both of the commands
-```
+To view the /etc directory outside of the container, run one or both of the commands
+```shell
 docker volume inspect so1-etc
 ```
 The output of that command should list the directory associated with the volume mount.
@@ -135,7 +135,7 @@ In the same yaml file you initially used to deploy Splunk instances, update the
 
 Below is an example yaml with `SPLUNK_UPGRADE=true`:
 
-```
+```yaml
 version: "3.6"
 
 networks:
@@ -165,13 +165,13 @@ services:
       - 8089
     volumes:
       - so1-var:/opt/splunk/var
-	    - so1-etc:/opt/splunk/etc
+      - so1-etc:/opt/splunk/etc
 ```
 
 #### Step 3: Deploy your containers using the updated yaml ####
-Similar to how you initially deployed your containers, run the command with the updated yaml that contains a reference to the new image and SPLUNK_UPGRADE=true in the environment. Make sure that you do NOT destroy previously existing network and volumes. After running the command with the yaml file, your containers should be recreated with the new version of Splunk and persisted data properly mounted to /opt/splunk/var and /opt/splunk/etc.
+Like how you initially deployed your containers, run the command with the updated yaml containing a reference to the new image and SPLUNK_UPGRADE=true in the environment. Make sure that you do NOT destroy previously existing networks and volumes. After running the command with the yaml file, your containers should be recreated with the new version of Splunk and persisted data properly mounted to /opt/splunk/var and /opt/splunk/etc.
 
 #### Different types of volumes ####
-Using named volume is recommended so it is easier to attach and detach volumes to different Splunk instances while persisting your data. If you use anonymous volumes, Docker gives them random and unique names so you can still reuse anonymous volumes on different containers. If you use bind mounts, make sure that the mounts are setup properly to persist /opt/splunk/var and opt/splunk/etc. Starting new containers without proper mounts will result in a loss of your data.
+Using named volume is recommended because it is easier to attach and detach volumes to different Splunk instances while persisting your data. If you use anonymous volumes, Docker gives them random and unique names so you can still reuse anonymous volumes on other containers. If you use bind mounts, make sure that the mounts are set up correctly to persist /opt/splunk/var and opt/splunk/etc. Starting new containers without proper mounts will result in a loss of your data.
 
 See [Create and manage volumes](https://docs.docker.com/storage/volumes/#create-and-manage-volumes) in the Docker documentation for more information.

py23-image/centos-7/Dockerfile

@@ -22,4 +22,4 @@ RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \
 RUN yum remove -y --setopt=tsflags=noscripts gcc openssl-devel bzip2-devel libffi-devel \
     && yum autoremove -y \
     && yum clean all
-RUN pip3 --no-cache-dir install ansible requests
+RUN pip3 --no-cache-dir install ansible==3.4.0 requests==2.25.1 pyyaml==5.4.1 jmespath==0.10.0

py23-image/centos-8/Dockerfile

@@ -26,5 +26,5 @@ RUN wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz \
 RUN yum remove -y --setopt=tsflags=noscripts gcc openssl-devel bzip2-devel libffi-devel \
     && yum autoremove -y \
     && yum clean all
-RUN pip3 --no-cache-dir install ansible requests \
-    && pip --no-cache-dir install pyyaml requests
+RUN pip3 --no-cache-dir install ansible==3.4.0 requests==2.25.1 pyyaml==5.4.1 jmespath==0.10.0 \
+    && pip --no-cache-dir install requests==2.25.1 pyyaml==5.4.1 jmespath==0.10.0

py23-image/debian-10/Dockerfile

@@ -3,8 +3,9 @@ FROM ${SPLUNK_PRODUCT}-debian-10:latest
 USER root
 
 RUN apt-get update -y \
-    && apt-get install -y --no-install-recommends libpython-dev ansible python-pip python-requests python-jmespath python-yaml \
+    && apt-get install -y --no-install-recommends libpython-dev python-pip python-requests python-jmespath python-yaml \
     && ln -sf /usr/bin/python3.7 /usr/bin/python3 \
     && ln -sf /usr/bin/pip3.7 /usr/bin/pip3 \
     && ln -sf /usr/bin/python3.7 /usr/bin/python \
-    && ln -sf /usr/bin/pip3.7 /usr/bin/pip
+    && ln -sf /usr/bin/pip3.7 /usr/bin/pip \
+    && pip3 install --upgrade ansible==3.4.0 requests==2.25.1 pyyaml==5.4.1 jmespath==0.10.0

py23-image/debian-9/Dockerfile

@@ -3,8 +3,9 @@ FROM ${SPLUNK_PRODUCT}-debian-9:latest
 USER root
 
 RUN apt-get update -y \
-    && apt-get install -y --no-install-recommends libpython-dev ansible python-pip python-requests python-jmespath python-yaml \
+    && apt-get install -y --no-install-recommends libpython-dev python-pip python-requests python-jmespath python-yaml \
     && ln -sf /usr/bin/python3.7 /usr/bin/python3 \
     && ln -sf /usr/bin/pip3.7 /usr/bin/pip3 \
     && ln -sf /usr/bin/python3.7 /usr/bin/python \
-    && ln -sf /usr/bin/pip3.7 /usr/bin/pip
+    && ln -sf /usr/bin/pip3.7 /usr/bin/pip \
+    && pip3 install --upgrade ansible==3.4.0 requests==2.25.1 pyyaml==5.4.1 jmespath==0.10.0