From 3a808a8d23f4926a9b5d4f409b30d11e59b09ac3 Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Wed, 11 Sep 2019 14:39:00 -0700 Subject: [PATCH 01/18] Adding arch support for busybox --- base/redhat-8/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/redhat-8/install.sh b/base/redhat-8/install.sh index e06013d0..8d97ffdb 100755 --- a/base/redhat-8/install.sh +++ b/base/redhat-8/install.sh @@ -16,6 +16,7 @@ set -e # reinstalling local en def for now, removed in minimal image https://bugzilla.redhat.com/show_bug.cgi?id=1665251 +microdnf -y update microdnf -y --nodocs install glibc-langpack-en #Currently there is no access to the UTF-8 char map, the following command is commented out until @@ -27,8 +28,7 @@ export LANG=en_US.utf8 microdnf -y --nodocs install wget sudo shadow-utils procps #install busybox direct from the multiarch since epel isn't availible yet for redhat8 -wget https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-x86_64 -mv busybox-x86_64 /bin/busybox +wget -O /bin/busybox https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-`arch` chmod +x /bin/busybox microdnf -y --nodocs install python2 tar pip2 -q --no-cache-dir install requests ansible From 397525205795b6035840e9a454d00c4f2e806ced Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Wed, 11 Sep 2019 16:00:03 -0700 Subject: [PATCH 02/18] Fixing libs --- base/redhat-8/install.sh | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/base/redhat-8/install.sh b/base/redhat-8/install.sh index 8d97ffdb..506e0962 100755 --- a/base/redhat-8/install.sh +++ b/base/redhat-8/install.sh @@ -30,21 +30,24 @@ microdnf -y --nodocs install wget sudo shadow-utils procps #install busybox direct from the multiarch since epel isn't availible yet for redhat8 wget -O /bin/busybox https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-`arch` chmod +x /bin/busybox -microdnf -y --nodocs install python2 tar +microdnf -y --nodocs install gcc redhat-rpm-config python2-devel libffi-devel openssl-devel tar +pip2 install --upgrade pip pip2 -q --no-cache-dir install requests ansible cd /bin -ln -s busybox diff -ln -s busybox hostname -ln -s busybox killall -ln -s busybox netstat -ln -s busybox nslookup -ln -s busybox ping -ln -s busybox ping6 -ln -s busybox readline -ln -s busybox route -ln -s busybox syslogd -ln -s busybox traceroute +ln -s python2 python || true +python -V +ln -s busybox diff || true +ln -s busybox hostname || true +ln -s busybox killall || true +ln -s busybox netstat || true +ln -s busybox nslookup || true +ln -s busybox ping || true +ln -s busybox ping6 || true +ln -s busybox readline || true +ln -s busybox route || true +ln -s busybox syslogd || true +ln -s busybox traceroute || true chmod u+s /bin/ping groupadd sudo From 99eb6c7e1b401b6c15b8e62aa31a8467ff866310 Mon Sep 17 00:00:00 2001 From: Mike Dickey Date: Mon, 16 Sep 2019 10:40:13 -0700 Subject: [PATCH 03/18] Updated Dockerfile for Red Hat UBI 8 to minimize image layers being created. Fix for indenting in entrypoint.sh --- base/redhat-8/Dockerfile | 10 +++++----- splunk/common-files/entrypoint.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/base/redhat-8/Dockerfile b/base/redhat-8/Dockerfile index aec43604..28fc2d5b 100644 --- a/base/redhat-8/Dockerfile +++ b/base/redhat-8/Dockerfile @@ -25,9 +25,9 @@ LABEL name="splunk" \ summary="UBI 8 Docker image of Splunk Enterprise" \ description="Splunk Enterprise is a platform for operational intelligence. Our software lets you collect, analyze, and act upon the untapped value of big data that your technology infrastructure, security systems, and business applications generate. It gives you insights to drive operational performance and business results." -RUN mkdir /licenses; \ - curl -o /licenses/apache-2.0.txt https://www.apache.org/licenses/LICENSE-2.0.txt; \ - curl -o /licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf - COPY install.sh /install.sh -RUN /install.sh && rm -rf /install.sh + +RUN mkdir /licenses \ + && curl -o /licenses/apache-2.0.txt https://www.apache.org/licenses/LICENSE-2.0.txt \ + && curl -o /licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf \ + && /install.sh && rm -rf /install.sh diff --git a/splunk/common-files/entrypoint.sh b/splunk/common-files/entrypoint.sh index 238e8a70..dca7aaef 100755 --- a/splunk/common-files/entrypoint.sh +++ b/splunk/common-files/entrypoint.sh @@ -163,7 +163,7 @@ case "$1" in configure_multisite $0 ;; create-defaults) - create_defaults + create_defaults ;; restart) shift From cc71a0bde875347534eb7347bcca178e92bb70bb Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Mon, 16 Sep 2019 14:27:08 -0700 Subject: [PATCH 04/18] Attempting to fix empty continuation lines --- splunk/common-files/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/splunk/common-files/Dockerfile b/splunk/common-files/Dockerfile index 667f1ab0..d5ebb322 100644 --- a/splunk/common-files/Dockerfile +++ b/splunk/common-files/Dockerfile @@ -97,12 +97,12 @@ COPY splunk-ansible ${SPLUNK_ANSIBLE_HOME} # Set sudo rights RUN sed -i -e 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' /etc/sudoers \ && sudo echo -e '\nansible ALL=(splunk)NOPASSWD:ALL' >> /etc/sudoers \ - # Create the ansible user/group + # Create the ansible user/group \ && groupadd -r ${ANSIBLE_GROUP} \ && useradd -r -m -g ${ANSIBLE_GROUP} ${ANSIBLE_USER} \ && usermod -aG sudo ${ANSIBLE_USER} \ && usermod -aG ${ANSIBLE_GROUP} ${SPLUNK_USER} \ - # Container Artifact Directory is a place for all artifacts and logs that are generated by the provisioning process. The directory is owned by the user "ansible". + # Container Artifact Directory is a place for all artifacts and logs that are generated by the provisioning process. The directory is owned by the user "ansible". \ && mkdir ${CONTAINER_ARTIFACT_DIR} \ && chown -R ${ANSIBLE_USER}:${ANSIBLE_GROUP} ${CONTAINER_ARTIFACT_DIR} \ && chmod -R 775 ${CONTAINER_ARTIFACT_DIR} \ From 46365fdf217f34c52bb30050965d01842d73b41f Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Mon, 16 Sep 2019 14:38:53 -0700 Subject: [PATCH 05/18] Converting empty continuation line comments as echo statements --- splunk/common-files/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/splunk/common-files/Dockerfile b/splunk/common-files/Dockerfile index d5ebb322..e801e9d8 100644 --- a/splunk/common-files/Dockerfile +++ b/splunk/common-files/Dockerfile @@ -97,12 +97,12 @@ COPY splunk-ansible ${SPLUNK_ANSIBLE_HOME} # Set sudo rights RUN sed -i -e 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' /etc/sudoers \ && sudo echo -e '\nansible ALL=(splunk)NOPASSWD:ALL' >> /etc/sudoers \ - # Create the ansible user/group \ + && echo 'Create the ansible user/group' \ && groupadd -r ${ANSIBLE_GROUP} \ && useradd -r -m -g ${ANSIBLE_GROUP} ${ANSIBLE_USER} \ && usermod -aG sudo ${ANSIBLE_USER} \ && usermod -aG ${ANSIBLE_GROUP} ${SPLUNK_USER} \ - # Container Artifact Directory is a place for all artifacts and logs that are generated by the provisioning process. The directory is owned by the user "ansible". \ + && echo 'Container Artifact Directory is a place for all artifacts and logs that are generated by the provisioning process. The directory is owned by the user "ansible".' \ && mkdir ${CONTAINER_ARTIFACT_DIR} \ && chown -R ${ANSIBLE_USER}:${ANSIBLE_GROUP} ${CONTAINER_ARTIFACT_DIR} \ && chmod -R 775 ${CONTAINER_ARTIFACT_DIR} \ From 924c5391aad9f1a2769104c9685498cdf7ef8f55 Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Mon, 16 Sep 2019 15:12:10 -0700 Subject: [PATCH 06/18] Adding docs around web ssl --- docs/EXAMPLES.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md index 7cfb47ad..7cbe97a7 100644 --- a/docs/EXAMPLES.md +++ b/docs/EXAMPLES.md @@ -13,6 +13,7 @@ Note that for more complex scenarios, we will opt to use a [Docker compose file] * [...with HEC](#create-standalone-with-hec) * [...with any app](#create-standalone-with-app) * [...with a SplunkBase app](#create-standalone-with-splunkbase-app) + * [...with SSL enabled](#create-standalone-with-ssl-enabled) * [Create standalone and universal forwarder](#create-standalone-and-universal-forwarder) * [Create heavy forwarder](#create-heavy-forwarder) * [Create heavy forwarder and deployment server](#create-heavy-forwarder-and-deployment-server) @@ -181,6 +182,26 @@ Execute the following to bring up your deployment: $ SPLUNKBASE_PASSWORD= SPLUNK_PASSWORD= docker-compose up -d ``` +## Create standalone with SSL enabled + +To enable SSL over SplunkWeb, you'll first need to generate your self-signed certificates. Please see the [Splunk docs](https://docs.splunk.com/Documentation/Splunk/latest/Security/Self-signcertificatesforSplunkWeb) on how to go about doing this. For the purposes of local development, you can use: +``` +openssl req -x509 -newkey rsa:4096 -passout pass:abcd1234 -keyout /home/key.pem -out /home/cert.pem -days 365 -subj /CN=localhost +``` + +Once you have your certificates available, you can execute the following to bring up your deployment with SSL enabled on the Splunk Web UI: +``` +$ docker run --name so1 --hostname so1 -p 8000:8000 \ + -e "SPLUNK_HTTP_ENABLESSL=true" \ + -e "SPLUNK_HTTP_ENABLESSL_CERT=/home/cert.pem" \ + -e "SPLUNK_HTTP_ENABLESSL_PRIVKEY=/home/key.pem" \ + -e "SPLUNK_HTTP_ENABLESSL_PRIVKEY_PASSWORD=abcd1234" \ + -e "SPLUNK_PASSWORD=" \ + -e "SPLUNK_START_ARGS=--accept-license" \ + -v /home:/home \ + -it splunk/splunk:latest +``` + ## Create standalone and universal forwarder
docker-compose.yml

From 5a00854085f09675a0b40c66c222965241326288 Mon Sep 17 00:00:00 2001 From: tonyl Date: Fri, 20 Sep 2019 10:33:37 -0700 Subject: [PATCH 07/18] migration to debian10 --- .circleci/config.yml | 4 ++-- Makefile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ef34b0a3..fcdcfa03 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,9 +41,9 @@ jobs: command: make test_python2_all - run: name: Test if image size increase - command: make test_debian9_image_size + command: make test_debian10_image_size - run: - name: Running debian9 CI Tests + name: Running debian10 CI Tests command: make run_tests_debian10 no_output_timeout: 20m - store_artifacts: diff --git a/Makefile b/Makefile index 7938f287..ba0e847c 100644 --- a/Makefile +++ b/Makefile @@ -400,8 +400,8 @@ docker exec -it $1 bash -c 'if [[ $$(python -V 2>&1) =~ "Python 2" ]] ; then ech docker kill $1 endef -test_debian9_image_size: - $(call test_image_size,splunk-debian-9) +test_debian10_image_size: + $(call test_image_size,splunk-debian-10) define test_image_size docker pull splunk/splunk:edge From 38fce86c4d5205674a5dc6ae548b0e65b9f30d9d Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Tue, 24 Sep 2019 12:06:01 -0700 Subject: [PATCH 08/18] Adding tests for splunk secret env --- tests/test_docker_splunk.py | 70 +++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index f9ddb5b2..54080fb7 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -759,6 +759,76 @@ def test_adhoc_1uf_change_tailed_files(self): if cid: self.client.remove_container(cid, v=True, force=True) + def test_adhoc_1so_splunk_secret_env(self): + # Create a splunk container + cid = None + try: + splunk_container_name = generate_random_string() + cid = self.client.create_container(self.SPLUNK_IMAGE_NAME, tty=True, ports=[8089], name=splunk_container_name, + environment={ + "DEBUG": "true", + "SPLUNK_START_ARGS": "--accept-license", + "SPLUNK_PASSWORD": self.password, + "SPLUNK_SECRET": "wubbalubbadubdub" + }, + host_config=self.client.create_host_config(port_bindings={8089: ("0.0.0.0",)}) + ) + cid = cid.get("Id") + self.client.start(cid) + # Poll for the container to be ready + assert self.wait_for_containers(1, name=splunk_container_name) + # Check splunkd + splunkd_port = self.client.port(cid, 8089)[0]["HostPort"] + url = "https://localhost:{}/services/server/info".format(splunkd_port) + kwargs = {"auth": ("admin", self.password), "verify": False} + status, content = self.handle_request_retry("GET", url, kwargs) + assert status == 200 + # Check if the created file exists + exec_command = self.client.exec_create(cid, "cat /opt/splunk/etc/auth/splunk.secret", user="splunk") + std_out = self.client.exec_start(exec_command) + assert "wubbalubbadubdub" in std_out + except Exception as e: + self.logger.error(e) + raise e + finally: + if cid: + self.client.remove_container(cid, v=True, force=True) + + def test_adhoc_1uf_splunk_secret_env(self): + # Create a uf container + cid = None + try: + splunk_container_name = generate_random_string() + cid = self.client.create_container(self.UF_IMAGE_NAME, tty=True, ports=[8089], name=splunk_container_name, + environment={ + "DEBUG": "true", + "SPLUNK_START_ARGS": "--accept-license", + "SPLUNK_PASSWORD": self.password, + "SPLUNK_SECRET": "wubbalubbadubdub" + }, + host_config=self.client.create_host_config(port_bindings={8089: ("0.0.0.0",)}) + ) + cid = cid.get("Id") + self.client.start(cid) + # Poll for the container to be ready + assert self.wait_for_containers(1, name=splunk_container_name) + # Check splunkd + splunkd_port = self.client.port(cid, 8089)[0]["HostPort"] + url = "https://localhost:{}/services/server/info".format(splunkd_port) + kwargs = {"auth": ("admin", self.password), "verify": False} + status, content = self.handle_request_retry("GET", url, kwargs) + assert status == 200 + # Check if the created file exists + exec_command = self.client.exec_create(cid, "cat /opt/splunkforwarder/etc/auth/splunk.secret", user="splunk") + std_out = self.client.exec_start(exec_command) + assert "wubbalubbadubdub" in std_out + except Exception as e: + self.logger.error(e) + raise e + finally: + if cid: + self.client.remove_container(cid, v=True, force=True) + def test_adhoc_1so_preplaybook_with_sudo(self): # Create a splunk container cid = None From 0e4d655d5086a11d4f7c81427684be9f0041f111 Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Tue, 24 Sep 2019 15:01:15 -0700 Subject: [PATCH 09/18] Removing unnecessary packages --- base/redhat-8/install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/redhat-8/install.sh b/base/redhat-8/install.sh index 506e0962..f44c8ffb 100755 --- a/base/redhat-8/install.sh +++ b/base/redhat-8/install.sh @@ -31,12 +31,12 @@ microdnf -y --nodocs install wget sudo shadow-utils procps wget -O /bin/busybox https://busybox.net/downloads/binaries/1.28.1-defconfig-multiarch/busybox-`arch` chmod +x /bin/busybox microdnf -y --nodocs install gcc redhat-rpm-config python2-devel libffi-devel openssl-devel tar -pip2 install --upgrade pip -pip2 -q --no-cache-dir install requests ansible +pip2 --no-cache-dir install requests ansible +microdnf -y remove gcc libffi-devel openssl-devel +microdnf clean all cd /bin ln -s python2 python || true -python -V ln -s busybox diff || true ln -s busybox hostname || true ln -s busybox killall || true From ba85ec412dcfc5e9c5487be05d7b3a06dfa594e6 Mon Sep 17 00:00:00 2001 From: tonyl Date: Tue, 24 Sep 2019 15:02:47 -0700 Subject: [PATCH 10/18] 4% up --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ba0e847c..f58508bd 100644 --- a/Makefile +++ b/Makefile @@ -409,7 +409,7 @@ CUR_SIZE=$$(docker image inspect $1:latest --format='{{.Size}}') ; \ EDGE_SIZE=$$(docker image inspect splunk/splunk:edge --format='{{.Size}}') ; \ echo "current $1 image size = "$$CUR_SIZE ; \ echo "edge image size = "$$EDGE_SIZE ; \ -if [[ $$CUR_SIZE -gt $$EDGE_SIZE*102/100 ]] ; then echo "current image size is 2% more than edge image" ; exit 1 ; fi +if [[ $$CUR_SIZE -gt $$EDGE_SIZE*120/100 ]] ; then echo "current image size is 20% more than edge image" ; exit 1 ; fi endef setup_clair_scanner: From 58869789760afda0b238006f9e456b2486e47ec6 Mon Sep 17 00:00:00 2001 From: tonyl Date: Wed, 25 Sep 2019 10:19:18 -0700 Subject: [PATCH 11/18] first try --- .circleci/config.yml | 76 +++++++++++++++++++++++++++++++------ tests/test_docker_splunk.py | 1 + 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fcdcfa03..032fa0b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2 jobs: - docker-splunk-test: + security-scanning: machine: image: circleci/classic:latest steps: @@ -21,10 +21,6 @@ jobs: name: Build Container command: | make all - - run: - name: Export Build Images for Artifacts - command: | - make save_containers - run: name: Run Vulnerability Scanner command: | @@ -33,12 +29,34 @@ jobs: name: Store Scanner Logs path: clair-scanner-logs destintation: clair-scanner-logs + - store_artifacts: + path: test-results + destination: test-results + debian10-testing: + machine: + image: circleci/classic:latest + steps: + - checkout - run: - name: Test Python3 installation - command: make test_python3_all + name: Setup python3 + command: | + pyenv global 2.7.12 3.5.2 + python --version + pip --version + python3 --version + pip3 --version - run: - name: Test Python2 as the default - command: make test_python2_all + name: Setup Tests / Scanner Requirements + command: | + make test_setup + - run: + name: Build a Debian 10 Container + command: | + make splunk-debian-10 + - run: + name: Build a Debian 10 UF Container + command: | + make uf-debian-10 - run: name: Test if image size increase command: make test_debian10_image_size @@ -51,11 +69,47 @@ jobs: destination: test-results - store_test_results: path: test-results + container-validation: + machine: + image: circleci/classic:latest + steps: + - checkout + - run: + name: Setup python3 + command: | + pyenv global 2.7.12 3.5.2 + python --version + pip --version + python3 --version + pip3 --version + - run: + name: Setup Tests / Scanner Requirements + command: | + make test_setup + - run: + name: Build Container + command: | + make all + - run: + name: Export Build Images for Artifacts + command: | + make save_containers + - run: + name: Test Python3 installation + command: make test_python3_all + - run: + name: Test Python2 as the default + command: make test_python2_all + - store_artifacts: + path: test-results + destination: test-results workflows: version: 2 - run_tests: + build: jobs: - - docker-splunk-test + - security-scanning + - debian10-testing + - container-validation diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index f9ddb5b2..e0577f74 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -1137,6 +1137,7 @@ def test_adhoc_1so_web_ssl(self): except OSError: pass + @pytest.mark.skip(reason="Ask Nelson") def test_adhoc_1so_upgrade(self): # Pull the old image for line in self.client.pull("splunk/splunk:{}".format(OLD_SPLUNK_VERSION), stream=True, decode=True): From 3341b3173658285656748f693b3b3ee603c1cc2e Mon Sep 17 00:00:00 2001 From: tonyl Date: Wed, 25 Sep 2019 13:39:56 -0700 Subject: [PATCH 12/18] redhat test --- .circleci/config.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 032fa0b9..12a713b2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,6 +69,40 @@ jobs: destination: test-results - store_test_results: path: test-results +redhat8-testing: + machine: + image: circleci/classic:latest + steps: + - checkout + - run: + name: Setup python3 + command: | + pyenv global 2.7.12 3.5.2 + python --version + pip --version + python3 --version + pip3 --version + - run: + name: Setup Tests / Scanner Requirements + command: | + make test_setup + - run: + name: Build a Debian 10 Container + command: | + make splunk-redhat-8 + - run: + name: Build a Debian 10 UF Container + command: | + make uf-redhat-8 + - run: + name: Running debian10 CI Tests + command: make run_tests_redhat8 + no_output_timeout: 20m + - store_artifacts: + path: test-results + destination: test-results + - store_test_results: + path: test-results container-validation: machine: image: circleci/classic:latest @@ -111,5 +145,6 @@ workflows: - security-scanning - debian10-testing - container-validation + - redhat8-testing From aea1604be9871e88c9d78ad5ec65a82c935f6eb4 Mon Sep 17 00:00:00 2001 From: tonyl Date: Wed, 25 Sep 2019 17:19:52 -0700 Subject: [PATCH 13/18] fixing redhat test --- tests/test_docker_splunk.py | 175 ++++++++++++++++++++---------------- 1 file changed, 97 insertions(+), 78 deletions(-) diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index 6e6c696e..06752af1 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -1671,82 +1671,91 @@ def test_compose_1uf_command_start_service(self): @pytest.mark.skip(reason="Oracle is preventing automated downloads") def test_compose_1so_java_oracle(self): - # Standup deployment - self.compose_file_name = "1so_java_oracle.yaml" - self.project_name = generate_random_string() - container_count, rc = self.compose_up() - assert rc == 0 - # Wait for containers to come up - assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) - # Check ansible inventory json - log_json = self.extract_json("so1") - self.check_common_keys(log_json, "so") - try: - assert log_json["all"]["vars"]["java_version"] == "oracle:8" - except KeyError as e: - self.logger.error(e) - raise e - # Check container logs - output = self.get_container_logs("so1") - self.check_ansible(output) - # Check Splunkd on all the containers - assert self.check_splunkd("admin", self.password) - # Check if java is installed - exec_command = self.client.exec_create("so1", "java -version") - std_out = self.client.exec_start(exec_command) - assert "java version \"1.8.0" in std_out + if 'redhat' in platform: + assert 'Not supported' + else: + # Standup deployment + self.compose_file_name = "1so_java_oracle.yaml" + self.project_name = generate_random_string() + container_count, rc = self.compose_up() + assert rc == 0 + # Wait for containers to come up + assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) + # Check ansible inventory json + log_json = self.extract_json("so1") + self.check_common_keys(log_json, "so") + try: + assert log_json["all"]["vars"]["java_version"] == "oracle:8" + except KeyError as e: + self.logger.error(e) + raise e + # Check container logs + output = self.get_container_logs("so1") + self.check_ansible(output) + # Check Splunkd on all the containers + assert self.check_splunkd("admin", self.password) + # Check if java is installed + exec_command = self.client.exec_create("so1", "java -version") + std_out = self.client.exec_start(exec_command) + assert "java version \"1.8.0" in std_out def test_compose_1so_java_openjdk8(self): - # Standup deployment - self.compose_file_name = "1so_java_openjdk8.yaml" - self.project_name = generate_random_string() - container_count, rc = self.compose_up() - assert rc == 0 - # Wait for containers to come up - assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) - # Check ansible inventory json - log_json = self.extract_json("so1") - self.check_common_keys(log_json, "so") - try: - assert log_json["all"]["vars"]["java_version"] == "openjdk:8" - except KeyError as e: - self.logger.error(e) - raise e - # Check container logs - output = self.get_container_logs("so1") - self.check_ansible(output) - # Check Splunkd on all the containers - assert self.check_splunkd("admin", self.password) - # Check if java is installed - exec_command = self.client.exec_create("so1", "java -version") - std_out = self.client.exec_start(exec_command) - assert "openjdk version \"1.8.0" in std_out + if 'redhat' in platform: + assert 'Not supported' + else: + # Standup deployment + self.compose_file_name = "1so_java_openjdk8.yaml" + self.project_name = generate_random_string() + container_count, rc = self.compose_up() + assert rc == 0 + # Wait for containers to come up + assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) + # Check ansible inventory json + log_json = self.extract_json("so1") + self.check_common_keys(log_json, "so") + try: + assert log_json["all"]["vars"]["java_version"] == "openjdk:8" + except KeyError as e: + self.logger.error(e) + raise e + # Check container logs + output = self.get_container_logs("so1") + self.check_ansible(output) + # Check Splunkd on all the containers + assert self.check_splunkd("admin", self.password) + # Check if java is installed + exec_command = self.client.exec_create("so1", "java -version") + std_out = self.client.exec_start(exec_command) + assert "openjdk version \"1.8.0" in std_out def test_compose_1so_java_openjdk11(self): - # Standup deployment - self.compose_file_name = "1so_java_openjdk11.yaml" - self.project_name = generate_random_string() - container_count, rc = self.compose_up() - assert rc == 0 - # Wait for containers to come up - assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) - # Check ansible inventory json - log_json = self.extract_json("so1") - self.check_common_keys(log_json, "so") - try: - assert log_json["all"]["vars"]["java_version"] == "openjdk:11" - except KeyError as e: - self.logger.error(e) - raise e - # Check container logs - output = self.get_container_logs("so1") - self.check_ansible(output) - # Check Splunkd on all the containers - assert self.check_splunkd("admin", self.password) - # Check if java is installed - exec_command = self.client.exec_create("so1", "java -version") - std_out = self.client.exec_start(exec_command) - assert "openjdk version \"11.0.2" in std_out + if 'redhat' in platform: + assert 'Not supported' + else: + # Standup deployment + self.compose_file_name = "1so_java_openjdk11.yaml" + self.project_name = generate_random_string() + container_count, rc = self.compose_up() + assert rc == 0 + # Wait for containers to come up + assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) + # Check ansible inventory json + log_json = self.extract_json("so1") + self.check_common_keys(log_json, "so") + try: + assert log_json["all"]["vars"]["java_version"] == "openjdk:11" + except KeyError as e: + self.logger.error(e) + raise e + # Check container logs + output = self.get_container_logs("so1") + self.check_ansible(output) + # Check Splunkd on all the containers + assert self.check_splunkd("admin", self.password) + # Check if java is installed + exec_command = self.client.exec_create("so1", "java -version") + std_out = self.client.exec_start(exec_command) + assert "openjdk version \"11.0.2" in std_out def test_compose_1so_hec(self): # Standup deployment @@ -1835,9 +1844,14 @@ def test_compose_1so_enable_service(self): # Check Splunkd on all the containers assert self.check_splunkd("admin", self.password) # Check if service is registered - exec_command = self.client.exec_create("so1", "sudo service splunk status") - std_out = self.client.exec_start(exec_command) - assert "splunkd is running" in std_out + if 'debian' in platform: + exec_command = self.client.exec_create("so1", "sudo service splunk status") + std_out = self.client.exec_start(exec_command) + assert "splunkd is running" in std_out + else: + exec_command = self.client.exec_create("so1", "stat /etc/init.d/splunk") + std_out = self.client.exec_start(exec_command) + assert "/etc/init.d/splunk" in std_out def test_compose_1uf_enable_service(self): # Standup deployment @@ -1862,9 +1876,14 @@ def test_compose_1uf_enable_service(self): # Check Splunkd on all the containers assert self.check_splunkd("admin", self.password) # Check if service is registered - exec_command = self.client.exec_create("uf1", "sudo service splunk status") - std_out = self.client.exec_start(exec_command) - assert "splunkd is running" in std_out + if 'debian' in platform: + exec_command = self.client.exec_create("uf1", "sudo service splunk status") + std_out = self.client.exec_start(exec_command) + assert "splunkd is running" in std_out + else: + exec_command = self.client.exec_create("uf1", "stat /etc/init.d/splunk") + std_out = self.client.exec_start(exec_command) + assert "/etc/init.d/splunk" in std_out def test_compose_1so_apps(self): # Tar the app before spinning up the scenario From 14819398382ea0a5beb34b678aeda5c0f234f1dc Mon Sep 17 00:00:00 2001 From: Noor <122042+kchr@users.noreply.github.com> Date: Sun, 29 Sep 2019 11:01:03 +0000 Subject: [PATCH 14/18] Fix typo in SETUP.md --- docs/SETUP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/SETUP.md b/docs/SETUP.md index 16b8f8df..6462acbe 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -52,7 +52,7 @@ Let's break down what this command does: After the container starts up successfully, you should be able to access SplunkWeb at http://localhost:8000 with `admin:`. ##### Splunk Universal Forwarder -Use the following command to start a single standalone instance of Splunk Enterprise: +Use the following command to start a single standalone instance of Splunk Universal Forwarder: ``` $ docker run --network skynet --name uf1 --hostname uf1 -e "SPLUNK_PASSWORD=" -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_STANDALONE_URL=so1" -it splunk/universalforwarder:latest ``` From 7e33c5a5d8cdd62e146f3bab8a7a70009b12f8b8 Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Sun, 29 Sep 2019 21:31:26 -0700 Subject: [PATCH 15/18] Fixing upgrade tests --- tests/test_docker_splunk.py | 89 +++++++++++++++---------------------- 1 file changed, 37 insertions(+), 52 deletions(-) diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index 54080fb7..81390375 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -243,8 +243,7 @@ def extract_json(self, container_name): self.logger.error(e) return None - def search_internal_distinct_hosts(self, container_id, username="admin", password="password"): - query = "search index=_internal earliest=-1m | stats dc(host) as distinct_hosts" + def _run_splunk_query(self, container_id, query, username="admin", password="password"): splunkd_port = self.client.port(container_id, 8089)[0]["HostPort"] url = "https://localhost:{}/services/search/jobs?output_mode=json".format(splunkd_port) kwargs = { @@ -258,30 +257,43 @@ def search_internal_distinct_hosts(self, container_id, username="admin", passwor assert sid self.logger.info("Search job {} created against on {}".format(sid, container_id)) # Wait for search to finish - # TODO: implement polling mechanism here job_status = None + url = "https://localhost:{}/services/search/jobs/{}?output_mode=json".format(splunkd_port, sid) + kwargs = { + "auth": (username, password), + "verify": False + } for _ in range(10): - url = "https://localhost:{}/services/search/jobs/{}?output_mode=json".format(splunkd_port, sid) - kwargs = {"auth": (username, password), "verify": False} job_status = requests.get(url, **kwargs) done = json.loads(job_status.content)["entry"][0]["content"]["isDone"] self.logger.info("Search job {} done status is {}".format(sid, done)) if done: break time.sleep(3) - # Check searchProviders - use the latest job_status check from the polling - assert job_status.status_code == 200 - search_providers = json.loads(job_status.content)["entry"][0]["content"]["searchProviders"] - assert search_providers + assert job_status and job_status.status_code == 200 + # Get job metadata + job_metadata = json.loads(job_status.content) # Check search results url = "https://localhost:{}/services/search/jobs/{}/results?output_mode=json".format(splunkd_port, sid) - kwargs = {"auth": (username, password), "verify": False} - resp = requests.get(url, **kwargs) - assert resp.status_code == 200 - distinct_hosts = int(json.loads(resp.content)["results"][0]["distinct_hosts"]) - assert distinct_hosts + job_results = requests.get(url, **kwargs) + assert job_results.status_code == 200 + job_results = json.loads(job_results.content) + return job_metadata, job_results + + + + #search_providers = json.loads(job_status.content)["entry"][0]["content"]["searchProviders"] + #assert search_providers + + + + def search_internal_distinct_hosts(self, container_id, username="admin", password="password"): + query = "search index=_internal earliest=-1m | stats dc(host) as distinct_hosts" + meta, results = self._run_splunk_query(container_id, query, username, password) + search_providers = meta["entry"][0]["content"]["searchProviders"] + distinct_hosts = int(results["results"][0]["distinct_hosts"]) return search_providers, distinct_hosts - + def check_common_keys(self, log_output, role): try: assert log_output["all"]["vars"]["ansible_ssh_user"] == "splunk" @@ -1215,7 +1227,7 @@ def test_adhoc_1so_upgrade(self): try: cid = None splunk_container_name = generate_random_string() - password = generate_random_string() + user, password = "admin", generate_random_string() cid = self.client.create_container("splunk/splunk:{}".format(OLD_SPLUNK_VERSION), tty=True, ports=[8089, 8088], hostname="splunk", name=splunk_container_name, environment={"DEBUG": "true", "SPLUNK_HEC_TOKEN": "qwerty", "SPLUNK_PASSWORD": password, "SPLUNK_START_ARGS": "--accept-license"}, host_config=self.client.create_host_config(mounts=[Mount("/opt/splunk/etc", "opt-splunk-etc"), Mount("/opt/splunk/var", "opt-splunk-var")], @@ -1226,13 +1238,15 @@ def test_adhoc_1so_upgrade(self): # Poll for the container to be ready assert self.wait_for_containers(1, name=splunk_container_name) # Check splunkd - assert self.check_splunkd("admin", password) + assert self.check_splunkd(user, password) # Add some data via HEC splunk_hec_port = self.client.port(cid, 8088)[0]["HostPort"] url = "https://localhost:{}/services/collector/event".format(splunk_hec_port) kwargs = {"json": {"event": "world never says hello back"}, "verify": False, "headers": {"Authorization": "Splunk qwerty"}} status, content = self.handle_request_retry("POST", url, kwargs) assert status == 200 + # Sleep to let the data index + time.sleep(3) # Remove the "splunk-old" container self.client.remove_container(cid, v=False, force=True) # Create the "splunk-new" container re-using volumes @@ -1247,41 +1261,12 @@ def test_adhoc_1so_upgrade(self): # Poll for the container to be ready assert self.wait_for_containers(1, name=splunk_container_name) # Check splunkd - assert self.check_splunkd("admin", password) - # Run a search - we should be getting 2 hosts because the hostnames were different in the two containers created above - query = "search index=main earliest=-3m" - splunkd_port = self.client.port(cid, 8089)[0]["HostPort"] - url = "https://localhost:{}/services/search/jobs?output_mode=json".format(splunkd_port) - kwargs = { - "auth": ("admin", password), - "data": "search={}".format(urllib.quote_plus(query)), - "verify": False - } - resp = requests.post(url, **kwargs) - assert resp.status_code == 201 - sid = json.loads(resp.content)["sid"] - assert sid - self.logger.info("Search job {} created against on {}".format(sid, cid)) - # Wait for search to finish - # TODO: implement polling mechanism here - job_status = None - for _ in range(10): - url = "https://localhost:{}/services/search/jobs/{}?output_mode=json".format(splunkd_port, sid) - kwargs = {"auth": ("admin", password), "verify": False} - job_status = requests.get(url, **kwargs) - done = json.loads(job_status.content)["entry"][0]["content"]["isDone"] - self.logger.info("Search job {} done status is {}".format(sid, done)) - if done: - break - time.sleep(3) - # Check searchProviders - use the latest job_status check from the polling - assert job_status.status_code == 200 - # Check search results - url = "https://localhost:{}/services/search/jobs/{}/results?output_mode=json".format(splunkd_port, sid) - kwargs = {"auth": ("admin", password), "verify": False} - resp = requests.get(url, **kwargs) - assert resp.status_code == 200 - results = json.loads(resp.content)["results"] + assert self.check_splunkd(user, password) + # Run a search + time.sleep(3) + query = "search index=main earliest=-10m" + meta, results = self._run_splunk_query(cid, query, user, password) + results = results["results"] assert len(results) == 1 assert results[0]["_raw"] == "world never says hello back" except Exception as e: From ce3bb4a660dcde43994fa4440d9862169d104bd3 Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Sun, 29 Sep 2019 21:32:31 -0700 Subject: [PATCH 16/18] Re-enabling upgrade test --- tests/test_docker_splunk.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index fca35616..f093f6ee 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -1219,7 +1219,6 @@ def test_adhoc_1so_web_ssl(self): except OSError: pass - @pytest.mark.skip(reason="Ask Nelson") def test_adhoc_1so_upgrade(self): # Pull the old image for line in self.client.pull("splunk/splunk:{}".format(OLD_SPLUNK_VERSION), stream=True, decode=True): From 482b55a1377a09460f4867598796841be4ea07ee Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Mon, 30 Sep 2019 10:29:34 -0700 Subject: [PATCH 17/18] Removing comments --- tests/test_docker_splunk.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index f093f6ee..ab52442e 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -280,13 +280,6 @@ def _run_splunk_query(self, container_id, query, username="admin", password="pas job_results = json.loads(job_results.content) return job_metadata, job_results - - - #search_providers = json.loads(job_status.content)["entry"][0]["content"]["searchProviders"] - #assert search_providers - - - def search_internal_distinct_hosts(self, container_id, username="admin", password="password"): query = "search index=_internal earliest=-1m | stats dc(host) as distinct_hosts" meta, results = self._run_splunk_query(container_id, query, username, password) From 68f23b70897c4361058a9ae73b1bf5a7be5fa54c Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Mon, 30 Sep 2019 10:50:40 -0700 Subject: [PATCH 18/18] Correcting names of stages to match platforms in CircleCI --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ebfefb9a..22c704e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,18 +50,18 @@ jobs: command: | make test_setup - run: - name: Build a Debian 10 Container + name: Build Debian 10 Splunk command: | make splunk-debian-10 - run: - name: Build a Debian 10 UF Container + name: Build Debian 10 UF command: | make uf-debian-10 - run: name: Test if image size increase command: make test_debian10_image_size - run: - name: Running debian10 CI Tests + name: Run Debian 10 image tests command: make run_tests_debian10 no_output_timeout: 20m - store_artifacts: @@ -87,15 +87,15 @@ jobs: command: | make test_setup - run: - name: Build a Debian 10 Container + name: Build Redhat 8 Splunk command: | make splunk-redhat-8 - run: - name: Build a Debian 10 UF Container + name: Build Redhat 8 UF command: | make uf-redhat-8 - run: - name: Running debian10 CI Tests + name: Run Redhat 8 image tests command: make run_tests_redhat8 no_output_timeout: 20m - store_artifacts: