Skip to content

Commit

Permalink
Merge pull request #55890 from ClickHouse/tavplubix-patch-9
Browse files Browse the repository at this point in the history
Update stress.py
  • Loading branch information
tavplubix committed Oct 24, 2023
2 parents 62b1549 + 6167bf4 commit 2619eec
Showing 1 changed file with 6 additions and 1 deletion.
7 changes: 6 additions & 1 deletion tests/ci/stress.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,11 @@ def compress_stress_logs(output_path: Path, files_prefix: str) -> None:


def call_with_retry(query: str, timeout: int = 30, retry_count: int = 5) -> None:
logging.info("Running command: %s", str(query))
for i in range(retry_count):
code = call(query, shell=True, stderr=STDOUT, timeout=timeout)
if code != 0:
logging.info("Command returend %s, retrying", str(code))
time.sleep(i)
else:
break
Expand All @@ -132,8 +134,11 @@ def prepare_for_hung_check(drop_databases: bool) -> bool:
# However, it obstruct checking for hung queries.
logging.info("Will terminate gdb (if any)")
call_with_retry("kill -TERM $(pidof gdb)")
call_with_retry("tail --pid=$(pidof gdb) -f /dev/null")
# Sometimes there is a message `Child process was stopped by signal 19` in logs after stopping gdb
call_with_retry("kill -CONT $(lsof -ti:9000)")
call_with_retry(
"kill -CONT $(cat /var/run/clickhouse-server/clickhouse-server.pid)"
)

# ThreadFuzzer significantly slows down server and causes false-positive hung check failures
call_with_retry("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'")
Expand Down

0 comments on commit 2619eec

Please sign in to comment.