From c21449fe1ba7acdf3a2472860915a11760c9884c Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Thu, 7 Nov 2019 14:21:20 -0800 Subject: [PATCH 1/2] Adding tests for cluster master repl factor + search factor setting --- tests/test_docker_splunk.py | 123 ++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index 189268e8..18b67cd8 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -1988,6 +1988,129 @@ def test_compose_1uf1so(self): assert search_providers[0] == "so1" assert distinct_hosts == 2 + def test_compose_3idx1cm_default_repl_factor(self): + # Generate default.yml + cid = self.client.create_container(self.SPLUNK_IMAGE_NAME, tty=True, command="create-defaults") + self.client.start(cid.get("Id")) + output = self.get_container_logs(cid.get("Id")) + self.client.remove_container(cid.get("Id"), v=True, force=True) + # Get the password + password = re.search(" password: (.*)", output).group(1).strip() + assert password + # Write the default.yml to a file + with open(os.path.join(SCENARIOS_DIR, "defaults", "default.yml"), "w") as f: + f.write(output) + # Standup deployment + try: + self.compose_file_name = "3idx1cm.yaml" + self.project_name = generate_random_string() + container_count, rc = self.compose_up() + assert rc == 0 + # Wait for containers to come up + assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name), timeout=600) + # Get container logs + container_mapping = {"cm1": "cm", "idx1": "idx", "idx2": "idx", "idx3": "idx"} + for container in container_mapping: + # Check ansible version & configs + ansible_logs = self.get_container_logs(container) + self.check_ansible(ansible_logs) + # Check values in log output + inventory_json = self.extract_json(container) + self.check_common_keys(inventory_json, container_mapping[container]) + try: + assert inventory_json["splunk_indexer"]["hosts"] == ["idx1", "idx2", "idx3"] + assert inventory_json["splunk_cluster_master"]["hosts"] == ["cm1"] + except KeyError as e: + self.logger.error(e) + raise e + # Check Splunkd on all the containers + assert self.check_splunkd("admin", self.password) + # Make sure apps are installed, and shcluster is setup properly + containers = self.client.containers(filters={"label": "com.docker.compose.project={}".format(self.project_name)}) + assert len(containers) == 4 + for container in containers: + container_name = container["Names"][0].strip("/") + splunkd_port = self.client.port(container["Id"], 8089)[0]["HostPort"] + if container_name == "cm1": + # Check the replication factor & search factor + url = "https://localhost:{}/services/cluster/config/config?output_mode=json".format(splunkd_port) + kwargs = {"auth": ("admin", self.password), "verify": False} + status, content = self.handle_request_retry("GET", url, kwargs) + assert status == 200 + assert json.loads(content)["entry"][0]["content"]["replication_factor"] == 3 + assert json.loads(content)["entry"][0]["content"]["search_factor"] == 3 + except Exception as e: + self.logger.error(e) + raise e + finally: + try: + os.remove(os.path.join(SCENARIOS_DIR, "defaults", "default.yml")) + except OSError as e: + pass + + def test_compose_3idx1cm_custom_repl_factor(self): + # Generate default.yml + cid = self.client.create_container(self.SPLUNK_IMAGE_NAME, tty=True, command="create-defaults") + self.client.start(cid.get("Id")) + output = self.get_container_logs(cid.get("Id")) + self.client.remove_container(cid.get("Id"), v=True, force=True) + # Get the password + password = re.search(" password: (.*)", output).group(1).strip() + assert password + # Change repl factor & search factor + output = re.sub(r' replication_factor: 3', r''' replication_factor: 2''', output) + output = re.sub(r' search_factor: 3', r''' search_factor: 1''', output) + # Write the default.yml to a file + with open(os.path.join(SCENARIOS_DIR, "defaults", "default.yml"), "w") as f: + f.write(output) + # Standup deployment + try: + self.compose_file_name = "3idx1cm.yaml" + self.project_name = generate_random_string() + container_count, rc = self.compose_up() + assert rc == 0 + # Wait for containers to come up + assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name), timeout=600) + # Get container logs + container_mapping = {"cm1": "cm", "idx1": "idx", "idx2": "idx", "idx3": "idx"} + for container in container_mapping: + # Check ansible version & configs + ansible_logs = self.get_container_logs(container) + self.check_ansible(ansible_logs) + # Check values in log output + inventory_json = self.extract_json(container) + self.check_common_keys(inventory_json, container_mapping[container]) + try: + assert inventory_json["splunk_indexer"]["hosts"] == ["idx1", "idx2", "idx3"] + assert inventory_json["splunk_cluster_master"]["hosts"] == ["cm1"] + except KeyError as e: + self.logger.error(e) + raise e + # Check Splunkd on all the containers + assert self.check_splunkd("admin", self.password) + # Make sure apps are installed, and shcluster is setup properly + containers = self.client.containers(filters={"label": "com.docker.compose.project={}".format(self.project_name)}) + assert len(containers) == 4 + for container in containers: + container_name = container["Names"][0].strip("/") + splunkd_port = self.client.port(container["Id"], 8089)[0]["HostPort"] + if container_name == "cm1": + # Check the replication factor & search factor + url = "https://localhost:{}/services/cluster/config/config?output_mode=json".format(splunkd_port) + kwargs = {"auth": ("admin", self.password), "verify": False} + status, content = self.handle_request_retry("GET", url, kwargs) + assert status == 200 + assert json.loads(content)["entry"][0]["content"]["replication_factor"] == 2 + assert json.loads(content)["entry"][0]["content"]["search_factor"] == 1 + except Exception as e: + self.logger.error(e) + raise e + finally: + try: + os.remove(os.path.join(SCENARIOS_DIR, "defaults", "default.yml")) + except OSError as e: + pass + def test_compose_1cm_smartstore(self): # Generate default.yml cid = self.client.create_container(self.SPLUNK_IMAGE_NAME, tty=True, command="create-defaults") From c2e9ddae14b28b88ea3755b3be222912b64dec37 Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Thu, 7 Nov 2019 16:55:41 -0800 Subject: [PATCH 2/2] Forgot to commit new test scenario --- test_scenarios/3idx1cm.yaml | 100 ++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 test_scenarios/3idx1cm.yaml diff --git a/test_scenarios/3idx1cm.yaml b/test_scenarios/3idx1cm.yaml new file mode 100644 index 00000000..02560fe3 --- /dev/null +++ b/test_scenarios/3idx1cm.yaml @@ -0,0 +1,100 @@ +version: "3.6" + +networks: + splunknet: + driver: bridge + attachable: true + +services: + cm1: + networks: + splunknet: + aliases: + - cm1 + image: ${SPLUNK_IMAGE:-splunk/splunk:latest} + command: start + hostname: cm1 + container_name: cm1 + environment: + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_INDEXER_URL=idx1,idx2,idx3 + - SPLUNK_CLUSTER_MASTER_URL=cm1 + - SPLUNK_ROLE=splunk_cluster_master + - SPLUNK_LICENSE_URI + - DEBUG=true + - SPLUNK_PASSWORD + ports: + - 8000 + - 8089 + volumes: + - ./defaults:/tmp/defaults + + idx1: + networks: + splunknet: + aliases: + - idx1 + image: ${SPLUNK_IMAGE:-splunk/splunk:latest} + command: start + hostname: idx1 + container_name: idx1 + environment: + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_INDEXER_URL=idx1,idx2,idx3 + - SPLUNK_CLUSTER_MASTER_URL=cm1 + - SPLUNK_ROLE=splunk_indexer + - SPLUNK_LICENSE_URI + - DEBUG=true + - SPLUNK_PASSWORD + ports: + - 8000 + - 8089 + volumes: + - ./defaults:/tmp/defaults + + idx2: + networks: + splunknet: + aliases: + - idx2 + image: ${SPLUNK_IMAGE:-splunk/splunk:latest} + command: start + hostname: idx2 + container_name: idx2 + environment: + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_INDEXER_URL=idx1,idx2,idx3 + - SPLUNK_CLUSTER_MASTER_URL=cm1 + - SPLUNK_ROLE=splunk_indexer + - SPLUNK_LICENSE_URI + - DEBUG=true + - SPLUNK_PASSWORD + ports: + - 8000 + - 8089 + volumes: + - ./defaults:/tmp/defaults + + idx3: + networks: + splunknet: + aliases: + - idx3 + image: ${SPLUNK_IMAGE:-splunk/splunk:latest} + command: start + hostname: idx3 + container_name: idx3 + environment: + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_INDEXER_URL=idx1,idx2,idx3 + - SPLUNK_CLUSTER_MASTER_URL=cm1 + - SPLUNK_ROLE=splunk_indexer + - SPLUNK_LICENSE_URI + - DEBUG=true + - SPLUNK_PASSWORD + ports: + - 8000 + - 8089 + volumes: + - ./defaults:/tmp/defaults + \ No newline at end of file