From 4b8e4fe208b3006c881d8d1f71824b7e0875e18b Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Mon, 14 Oct 2019 21:54:34 -0700 Subject: [PATCH 1/4] Allowing longer timeout for shc scenario --- tests/test_docker_splunk.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index ab52442e..ac0a2ef1 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -145,13 +145,14 @@ def get_number_of_containers(self, filename): yml = yaml.load(f) return len(yml["services"]) - def wait_for_containers(self, count, label=None, name=None): + def wait_for_containers(self, count, label=None, name=None, timeout=300): ''' NOTE: This helper method can only be used for `compose up` scenarios where self.project_name is defined ''' start = time.time() end = start - while end-start < 300: + # Wait + while end-start < timeout: filters = {} if name: filters["name"] = name @@ -2125,7 +2126,7 @@ def test_compose_1idx3sh1cm1dep(self): container_count, rc = self.compose_up() assert rc == 0 # Wait for containers to come up - assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) + assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name), timeout=600) # Get container logs container_mapping = {"sh1": "sh", "sh2": "sh", "sh3": "sh", "cm1": "cm", "idx1": "idx", "dep1": "dep"} for container in container_mapping: @@ -2169,7 +2170,7 @@ def test_compose_1idx3sh1cm1dep(self): status, content = self.handle_request_retry("GET", url, kwargs) assert json.loads(content)["entry"][0]["content"]["preferred_captain"] == "1" # Search results won't return the correct results immediately :( - time.sleep(15) + time.sleep(30) search_providers, distinct_hosts = self.search_internal_distinct_hosts("sh1", password=self.password) assert len(search_providers) == 2 assert "idx1" in search_providers and "sh1" in search_providers From f53a95802ba2b95d5b6510f8662e95a7341cce89 Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Tue, 15 Oct 2019 13:31:44 -0700 Subject: [PATCH 2/4] Increasing retries --- tests/test_docker_splunk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index ac0a2ef1..a92b0e20 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -183,8 +183,8 @@ def wait_for_containers(self, count, label=None, name=None, timeout=300): return True def handle_request_retry(self, method, url, kwargs): - RETRIES = 6 - IMPLICIT_WAIT = 3 + RETRIES = 10 + IMPLICIT_WAIT = 6 for n in range(RETRIES): try: self.logger.info("Attempt #{}: running {} against {} with kwargs {}".format(n+1, method, url, kwargs)) From 4087e5345bc08107553f7b4f59d07523f629d26c Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Tue, 15 Oct 2019 15:35:44 -0700 Subject: [PATCH 3/4] Adding retries around search query --- tests/test_docker_splunk.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index a92b0e20..15eda5e4 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -2171,10 +2171,26 @@ def test_compose_1idx3sh1cm1dep(self): assert json.loads(content)["entry"][0]["content"]["preferred_captain"] == "1" # Search results won't return the correct results immediately :( time.sleep(30) - search_providers, distinct_hosts = self.search_internal_distinct_hosts("sh1", password=self.password) - assert len(search_providers) == 2 - assert "idx1" in search_providers and "sh1" in search_providers - assert distinct_hosts == 6 + RETRIES = 10 + IMPLICIT_WAIT = 6 + for n in range(RETRIES): + try: + self.logger.info("Attempt #{}: checking internal search host count".format(n+1)) + search_providers, distinct_hosts = self.search_internal_distinct_hosts("sh1", password=self.password) + assert len(search_providers) == 2 + assert "idx1" in search_providers + assert "cm1" in search_providers + assert "dep1" in search_providers + assert "sh1" in search_providers + assert "sh2" in search_providers + assert "sh3" in search_providers + assert distinct_hosts == 6 + except Exception as e: + self.logger.error("Attempt #{} error: {}".format(n+1, str(e))) + if n < RETRIES-1: + time.sleep(IMPLICIT_WAIT) + continue + raise e except Exception as e: self.logger.error(e) raise e From 4aad52d82f0aa83a66f5771a39c6d2e7fb3f7cc4 Mon Sep 17 00:00:00 2001 From: Nelson Wang Date: Tue, 15 Oct 2019 16:20:17 -0700 Subject: [PATCH 4/4] Bad assertion change --- tests/test_docker_splunk.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index 15eda5e4..dc26794d 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -2178,13 +2178,9 @@ def test_compose_1idx3sh1cm1dep(self): self.logger.info("Attempt #{}: checking internal search host count".format(n+1)) search_providers, distinct_hosts = self.search_internal_distinct_hosts("sh1", password=self.password) assert len(search_providers) == 2 - assert "idx1" in search_providers - assert "cm1" in search_providers - assert "dep1" in search_providers - assert "sh1" in search_providers - assert "sh2" in search_providers - assert "sh3" in search_providers + assert "idx1" in search_providers and "sh1" in search_providers assert distinct_hosts == 6 + break except Exception as e: self.logger.error("Attempt #{} error: {}".format(n+1, str(e))) if n < RETRIES-1: