From d6fe4085393927859919d0aaa6cf9c66af8dd9de Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Tue, 9 Oct 2018 13:42:10 +0100 Subject: [PATCH 1/4] Spalloc without fork --- spalloc/job.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/spalloc/job.py b/spalloc/job.py index e47a8f4a0..e87fb9fde 100644 --- a/spalloc/job.py +++ b/spalloc/job.py @@ -3,7 +3,7 @@ from collections import namedtuple import logging -import subprocess +import os import time import sys @@ -11,6 +11,7 @@ from .config import read_config, SEARCH_PATH from .states import JobState from ._utils import time_left, timed_out, make_timeout +import traceback logger = logging.getLogger(__name__) @@ -299,10 +300,10 @@ def __init__(self, *args, **kwargs): logger.info("Created spalloc job %d", self.id) # Set-up and start background keepalive thread - self._keepalive_process = subprocess.Popen(map(str, [ + self._keepalive_process = os.popen(" ".join(map(str, [ sys.executable, "-m", "spalloc._keepalive_process", hostname, port, self.id, self._keepalive, self._timeout, - self._reconnect_delay]), stdin=subprocess.PIPE) + self._reconnect_delay])), "w") def __enter__(self): """ Convenience context manager for common case where a new job is to @@ -389,9 +390,11 @@ def close(self): :py:meth:`.destroy` instead. """ # Stop background thread - if self._keepalive_process.poll() is None: - self._keepalive_process.communicate(input="exit\n".encode("ascii")) - self._keepalive_process.wait() + try: + self._keepalive_process.write("exit\n".encode("ascii")) + self._keepalive_process.close() + except Exception: + traceback.print_exc() # Disconnect self._client.close() From b03a58cd0dd7238093d5d05f376579bb1afd97c5 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Mon, 3 Dec 2018 09:37:23 +0000 Subject: [PATCH 2/4] Update keepalive to pause until read done, to (possibly) avoid memerror --- spalloc/_keepalive_process.py | 3 +++ spalloc/job.py | 20 +++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/spalloc/_keepalive_process.py b/spalloc/_keepalive_process.py index c358db5d4..49135b0fc 100644 --- a/spalloc/_keepalive_process.py +++ b/spalloc/_keepalive_process.py @@ -1,6 +1,7 @@ """ A script for keeping Spalloc jobs alive, intended to only ever be run\ from the Spalloc client itself. """ +from __future__ import print_function import sys import threading from spalloc.protocol_client import ProtocolClient, ProtocolTimeoutError @@ -65,6 +66,8 @@ def keep_job_alive(hostname, port, job_id, keepalive_period, timeout, if __name__ == "__main__": + print("KEEPALIVE") + sys.stdout.flush() if len(sys.argv) != 7: sys.stderr.write( "wrong # args: should be '" + sys.argv[0] + " hostname port " diff --git a/spalloc/job.py b/spalloc/job.py index 6f36d8c7e..67a77f7b4 100644 --- a/spalloc/job.py +++ b/spalloc/job.py @@ -3,14 +3,13 @@ from collections import namedtuple import logging -import os +import subprocess import time import sys from .protocol_client import ProtocolClient, ProtocolTimeoutError from .config import read_config, SEARCH_PATH from .states import JobState from ._utils import time_left, timed_out, make_timeout -import traceback logger = logging.getLogger(__name__) @@ -299,10 +298,15 @@ def __init__(self, *args, **kwargs): logger.info("Created spalloc job %d", self.id) # Set-up and start background keepalive thread - self._keepalive_process = os.popen(" ".join(map(str, [ + self._keepalive_process = subprocess.Popen(map(str, [ sys.executable, "-m", "spalloc._keepalive_process", hostname, port, self.id, self._keepalive, self._timeout, - self._reconnect_delay])), "w") + self._reconnect_delay]), stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + first_line = self._keepalive_process.stdout.readline().strip() + if first_line != "KEEPALIVE": + raise Exception("Keepalive process wrote odd line: {}".format( + first_line)) def __enter__(self): """ Convenience context manager for common case where a new job is to @@ -389,11 +393,9 @@ def close(self): :py:meth:`.destroy` instead. """ # Stop background thread - try: - self._keepalive_process.write("exit\n".encode("ascii")) - self._keepalive_process.close() - except Exception: - traceback.print_exc() + if self._keepalive_process.poll() is None: + self._keepalive_process.communicate(input="exit\n".encode("ascii")) + self._keepalive_process.wait() # Disconnect self._client.close() From 1b0d12859a0187cb4724e05e4b15778fb4bc8a28 Mon Sep 17 00:00:00 2001 From: Andrew Rowley Date: Mon, 10 Dec 2018 15:30:19 +0000 Subject: [PATCH 3/4] Allow use of the debugger --- spalloc/job.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spalloc/job.py b/spalloc/job.py index 67a77f7b4..5f811eabf 100644 --- a/spalloc/job.py +++ b/spalloc/job.py @@ -304,6 +304,8 @@ def __init__(self, *args, **kwargs): self._reconnect_delay]), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) first_line = self._keepalive_process.stdout.readline().strip() + if first_line.startswith("pydev debugger"): + first_line = self._keepalive_process.stdout.readline().strip() if first_line != "KEEPALIVE": raise Exception("Keepalive process wrote odd line: {}".format( first_line)) From adae6a8eb4788a05a02bd8b30fe194228d389396 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 22 Jan 2019 13:55:27 +0000 Subject: [PATCH 4/4] python3.6 fix --- spalloc/job.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spalloc/job.py b/spalloc/job.py index 5f811eabf..33f523ec1 100644 --- a/spalloc/job.py +++ b/spalloc/job.py @@ -304,9 +304,11 @@ def __init__(self, *args, **kwargs): self._reconnect_delay]), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) first_line = self._keepalive_process.stdout.readline().strip() - if first_line.startswith("pydev debugger"): + if first_line.startswith(b"pydev debugger"): first_line = self._keepalive_process.stdout.readline().strip() - if first_line != "KEEPALIVE": + if len(first_line) == 0: + first_line = self._keepalive_process.stdout.readline().strip() + if first_line != b"KEEPALIVE": raise Exception("Keepalive process wrote odd line: {}".format( first_line))