From 4bbab83d6d718595cfa60a243b0c42e284fba4d8 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 1 Jul 2018 23:10:50 -0400 Subject: [PATCH 1/4] Require wurlitzer This is needed to handle redirection from C DRMAA implementations to the Python logger. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index be956c9..9ea832b 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ def readme(): license="BSD", keywords="python grid hpc drmaa", url="https://github.com/pygridtools/drmaa-python", + install_requires=["wurlitzer"], tests_require='nose', test_suite='nose.collector', classifiers=["Development Status :: 4 - Beta", From e6109af768a3702a52e4a1ea38037460d30eef64 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 27 Jul 2018 17:04:39 -0400 Subject: [PATCH 2/4] Setup top-level logger for drmaa Make sure that we have some way configured for logging from `drmaa` generally. --- drmaa/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drmaa/__init__.py b/drmaa/__init__.py index 411e344..1602692 100644 --- a/drmaa/__init__.py +++ b/drmaa/__init__.py @@ -32,6 +32,8 @@ from __future__ import absolute_import, print_function, unicode_literals +import logging + from .const import (ATTR_BUFFER, BLOCK_EMAIL, CONTACT_BUFFER, control_action_to_string, DEADLINE_TIME, DRM_SYSTEM_BUFFER, DRMAA_IMPLEMENTATION_BUFFER, DURATION_HLIMIT, @@ -98,3 +100,8 @@ 'SUBMISSION_STATE_ACTIVE', 'SUBMISSION_STATE_HOLD', 'TIMEOUT_NO_WAIT', 'TIMEOUT_WAIT_FOREVER', 'TRANSFER_FILES', 'V_ARGV', 'V_EMAIL', 'V_ENV', 'WCT_HLIMIT', 'WCT_SLIMIT', 'WD'] + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +logger.addHandler(logging.NullHandler()) From a20cb51cc5fc035cc7f01712e9e5af5b45556598 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 27 Jul 2018 17:05:21 -0400 Subject: [PATCH 3/4] Setup logging in `drmaa.helpers` This is needed for logging C stdout and stderr messages. --- drmaa/helpers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drmaa/helpers.py b/drmaa/helpers.py index 7ba0ff0..45479c5 100644 --- a/drmaa/helpers.py +++ b/drmaa/helpers.py @@ -26,6 +26,7 @@ from __future__ import absolute_import, print_function, unicode_literals +import logging import sys from collections import namedtuple from ctypes import (byref, c_uint, create_string_buffer, POINTER, pointer, @@ -55,6 +56,11 @@ _BUFLEN = ATTR_BUFFER +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +logger.addHandler(logging.NullHandler()) + + class BoolConverter(object): """Helper class to convert to/from bool attributes.""" From 6a9ebb957c4cdf5c100a8ac975a1a8ba99ae334c Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 27 Jul 2018 19:35:44 -0400 Subject: [PATCH 4/4] Capture `stdout`/`stderr` from C and log it To ensure that information printed from the underlying DRMAA implementation is not lost, capture the C `stdout` and `stderr` information and log it using our logger. That way this can be handled like any other logging information in a cluster environment. --- drmaa/helpers.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drmaa/helpers.py b/drmaa/helpers.py index 45479c5..5a34066 100644 --- a/drmaa/helpers.py +++ b/drmaa/helpers.py @@ -32,6 +32,8 @@ from ctypes import (byref, c_uint, create_string_buffer, POINTER, pointer, sizeof) +from wurlitzer import pipes + from drmaa.const import ATTR_BUFFER, ENCODING, NO_MORE_ELEMENTS from drmaa.errors import error_buffer from drmaa.wrappers import (drmaa_attr_names_t, drmaa_attr_values_t, @@ -305,8 +307,19 @@ def c(f, *args): A helper function wrapping calls to the C DRMAA functions with error managing code. """ - return f(*(args + (error_buffer, sizeof(error_buffer)))) + with pipes() as (stdout, stderr): + result = f(*(args + (error_buffer, sizeof(error_buffer)))) + + for line in stdout.readline(): + if line: + logger.debug(line) + + for line in stderr.readline(): + if line: + logger.debug(line) + + return result def string_vector(v): vlen = len(v)