diff --git a/tests/test_docker_splunk.py b/tests/test_docker_splunk.py index ab52442e..dc26794d 100644 --- a/tests/test_docker_splunk.py +++ b/tests/test_docker_splunk.py @@ -145,13 +145,14 @@ def get_number_of_containers(self, filename): yml = yaml.load(f) return len(yml["services"]) - def wait_for_containers(self, count, label=None, name=None): + def wait_for_containers(self, count, label=None, name=None, timeout=300): ''' NOTE: This helper method can only be used for `compose up` scenarios where self.project_name is defined ''' start = time.time() end = start - while end-start < 300: + # Wait + while end-start < timeout: filters = {} if name: filters["name"] = name @@ -182,8 +183,8 @@ def wait_for_containers(self, count, label=None, name=None): return True def handle_request_retry(self, method, url, kwargs): - RETRIES = 6 - IMPLICIT_WAIT = 3 + RETRIES = 10 + IMPLICIT_WAIT = 6 for n in range(RETRIES): try: self.logger.info("Attempt #{}: running {} against {} with kwargs {}".format(n+1, method, url, kwargs)) @@ -2125,7 +2126,7 @@ def test_compose_1idx3sh1cm1dep(self): container_count, rc = self.compose_up() assert rc == 0 # Wait for containers to come up - assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) + assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name), timeout=600) # Get container logs container_mapping = {"sh1": "sh", "sh2": "sh", "sh3": "sh", "cm1": "cm", "idx1": "idx", "dep1": "dep"} for container in container_mapping: @@ -2169,11 +2170,23 @@ def test_compose_1idx3sh1cm1dep(self): status, content = self.handle_request_retry("GET", url, kwargs) assert json.loads(content)["entry"][0]["content"]["preferred_captain"] == "1" # Search results won't return the correct results immediately :( - time.sleep(15) - search_providers, distinct_hosts = self.search_internal_distinct_hosts("sh1", password=self.password) - assert len(search_providers) == 2 - assert "idx1" in search_providers and "sh1" in search_providers - assert distinct_hosts == 6 + time.sleep(30) + RETRIES = 10 + IMPLICIT_WAIT = 6 + for n in range(RETRIES): + try: + self.logger.info("Attempt #{}: checking internal search host count".format(n+1)) + search_providers, distinct_hosts = self.search_internal_distinct_hosts("sh1", password=self.password) + assert len(search_providers) == 2 + assert "idx1" in search_providers and "sh1" in search_providers + assert distinct_hosts == 6 + break + except Exception as e: + self.logger.error("Attempt #{} error: {}".format(n+1, str(e))) + if n < RETRIES-1: + time.sleep(IMPLICIT_WAIT) + continue + raise e except Exception as e: self.logger.error(e) raise e