From 5335fa0dcc95a6392a2bded7ea8193a1d05f8975 Mon Sep 17 00:00:00 2001 From: Zach Toogood Date: Mon, 13 Jan 2025 15:10:19 +0000 Subject: [PATCH] CI: Rewrite startup_checks to be reactive --- .github/workflows/build.yml | 2 + tools/ci/startup_checks.py | 157 ++++++++++++++++++++++++++++-------- 2 files changed, 124 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 682fdef88b8..ea1913ca318 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -437,6 +437,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 1 + submodules: recursive # For navmeshes - uses: actions/download-artifact@v4 with: name: linux_executables @@ -720,6 +721,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 1 + submodules: recursive # For navmeshes - uses: actions/download-artifact@v4 with: name: windows_modules_executables diff --git a/tools/ci/startup_checks.py b/tools/ci/startup_checks.py index 513a1838c83..837fda2e9be 100644 --- a/tools/ci/startup_checks.py +++ b/tools/ci/startup_checks.py @@ -1,57 +1,144 @@ #!/usr/bin/python -import io import platform import subprocess import time import signal +import threading +from queue import Queue, Empty + +TEN_MINUTES_IN_SECONDS = 600 +CHECK_INTERVAL_SECONDS = 5 + + +def kill_all(processes): + """Send SIGTERM to all running processes.""" + for proc in processes: + if proc.poll() is None: # still running + proc.send_signal(signal.SIGTERM) + + +def reader_thread(proc, output_queue): + """ + Reads lines from proc.stdout and puts them into the output_queue + along with a reference to the proc. + + When the process ends (stdout is closed), push a (proc, None) + to indicate it's done. + """ + with proc.stdout: + for line in proc.stdout: + # 'line' already in string form since we use text=True + output_queue.put((proc, line)) + # Signal that this proc has ended + output_queue.put((proc, None)) + def main(): print("Running exe startup checks...({})".format(platform.system())) - p0 = subprocess.Popen( - ["xi_connect", "--log", "connect-server.log"], stdout=subprocess.PIPE - ) - p1 = subprocess.Popen( - ["xi_search", "--log", "search-server.log"], stdout=subprocess.PIPE - ) - p2 = subprocess.Popen( - ["xi_map", "--log", "game-server.log", "--load_all"], stdout=subprocess.PIPE - ) - p3 = subprocess.Popen( - ["xi_world", "--log", "world-server.log"], stdout=subprocess.PIPE + # Start the processes + # Use text=True (or universal_newlines=True) so we get strings instead of bytes. + processes = [ + subprocess.Popen( + ["xi_connect", "--log", "connect-server.log"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ), + subprocess.Popen( + ["xi_search", "--log", "search-server.log"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ), + subprocess.Popen( + ["xi_map", "--log", "game-server.log", "--load_all"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ), + subprocess.Popen( + ["xi_world", "--log", "world-server.log"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ), + ] + + # Keep track of which processes have reported "ready to work" + ready_status = {proc: False for proc in processes} + + # Create a queue to receive stdout lines from all processes + output_queue = Queue() + + # Start a reading thread for each process + threads = [] + for proc in processes: + t = threading.Thread( + target=reader_thread, args=(proc, output_queue), daemon=True + ) + t.start() + threads.append(t) + + print( + f"Polling process output every {CHECK_INTERVAL_SECONDS}s for up to {TEN_MINUTES_IN_SECONDS}s..." ) + start_time = time.time() - print("Sleeping for 5 minutes...") + while True: + # If we've hit the timeout (10 minutes), fail + if time.time() - start_time > TEN_MINUTES_IN_SECONDS: + print("Timed out waiting for all processes to become ready.") + kill_all(processes) + exit(-1) - time.sleep(300) + # Poll the queue for new lines + # We'll keep pulling until it's empty (non-blocking) + while True: + try: + proc, line = output_queue.get_nowait() + except Empty: + break # No more lines at the moment - print("Checking logs and killing exes...") + # If line is None, that means this proc ended + if line is None: + # If the process ended but wasn't marked ready => error + if not ready_status[proc]: + print( + f"ERROR: {proc.args[0]} exited before it was 'ready to work'." + ) + kill_all(processes) + exit(-1) + else: + # We have an actual line of output + line_str = line.strip() + print(f"[{proc.args[0]}] {line_str}") - has_seen_output = False - error = False - for proc in {p0, p1, p2, p3}: - print(proc.args[0]) - proc.send_signal(signal.SIGTERM) - for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): - print(line.replace("\n", "")) - has_seen_output = True - if ( - "error" in line.lower() - or "warning" in line.lower() - or "crash" in line.lower() - ): - print("^^^") - error = True + # Check for error or warning text + lower_line = line_str.lower() + if any(x in lower_line for x in ["error", "warning", "crash"]): + print("^^^ Found error or warning in output.") + kill_all(processes) + print("Killing all processes and exiting with error.") + exit(-1) - if not has_seen_output: - print("ERROR: Did not get any output!") + # Check for "ready to work" + if "ready to work" in lower_line: + print(f"==> {proc.args[0]} is ready!") + ready_status[proc] = True - if error or not has_seen_output: - exit(-1) + # Check if all processes are marked ready + if all(ready_status.values()): + print( + "All processes reached 'ready to work'! Killing them and exiting successfully." + ) + kill_all(processes) + exit(0) - time.sleep(5) + # Sleep until next poll + time.sleep(CHECK_INTERVAL_SECONDS) if __name__ == "__main__": main()