From b59313f6a67c64e0203ad8fb12aa92664beaaa77 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Wed, 26 Dec 2012 22:43:54 -0500 Subject: [PATCH 1/2] enh: config option to control commandline terminal output --- CHANGES | 1 + doc/users/config_file.rst | 8 +++ nipype/interfaces/base.py | 94 +++++++++++++++++++--------- nipype/interfaces/tests/test_base.py | 21 +++++++ nipype/pipeline/utils.py | 3 +- nipype/utils/config.py | 1 + 6 files changed, 97 insertions(+), 31 deletions(-) diff --git a/CHANGES b/CHANGES index ed2cbb4b8f..f817ce5447 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,7 @@ Next release ============ +* ENH: Allows controlling how commandline interface terminal output is handled. Release 0.7.0 (Dec 18, 2012) ============================ diff --git a/doc/users/config_file.rst b/doc/users/config_file.rst index e59303994c..673b07ab10 100644 --- a/doc/users/config_file.rst +++ b/doc/users/config_file.rst @@ -114,6 +114,14 @@ Execution version information is available. Please notify developers or submit a patch. +*commandline_output (CommandLine interfaces only)* + Controls how terminal output from a command line is handled. There are 4 + possible options. (default: 'stream' for current compatibility) + `stream`: line by line streaming to terminal + `allatonce`: streams after command has terminated + `file`: writes output to stdout.nipype and stderr.nipype + `none`: suppresses all output + Example ~~~~~~~ diff --git a/nipype/interfaces/base.py b/nipype/interfaces/base.py index 04c0412f27..a65428dd3a 100644 --- a/nipype/interfaces/base.py +++ b/nipype/interfaces/base.py @@ -1038,42 +1038,76 @@ def run_command(runtime, timeout=0.01): shell=True, cwd=runtime.cwd, env=runtime.environ) - streams = [ - Stream('stdout', proc.stdout), - Stream('stderr', proc.stderr) - ] + result = {} + if config.get('execution', 'commandline_output').lower() == 'stream': + streams = [ + Stream('stdout', proc.stdout), + Stream('stderr', proc.stderr) + ] - def _process(drain=0): - try: - res = select.select(streams, [], [], timeout) - except select.error, e: - iflogger.info(str(e)) - if e[0] == errno.EINTR: - return + def _process(drain=0): + try: + res = select.select(streams, [], [], timeout) + except select.error, e: + iflogger.info(str(e)) + if e[0] == errno.EINTR: + return + else: + raise else: - raise - else: - for stream in res[0]: - stream.read(drain) - - while proc.returncode is None: - proc.poll() - _process() - runtime.returncode = proc.returncode - _process(drain=1) + for stream in res[0]: + stream.read(drain) + + while proc.returncode is None: + proc.poll() + _process() + runtime.returncode = proc.returncode + _process(drain=1) + + # collect results, merge and return + temp = [] + for stream in streams: + rows = stream._rows + temp += rows + result[stream._name] = [r[2] for r in rows] + temp.sort() + result['merged'] = [r[1] for r in temp] + elif config.get('execution', 'commandline_output').lower() == 'allatonce': + stdout, stderr = proc.communicate() + result['stdout'] = stdout.split('\n') + result['stderr'] = stderr.split('\n') + result['merged'] = '' + elif config.get('execution', 'commandline_output').lower() == 'file': + errfile = os.path.join(runtime.cwd, 'stderr.nipype') + outfile = os.path.join(runtime.cwd, 'stdout.nipype') + stderr = open(errfile, 'wt') + stdout = open(outfile, 'wt') + proc = subprocess.Popen(runtime.cmdline, + stdout=stdout, + stderr=stderr, + shell=True, + cwd=runtime.cwd, + env=runtime.environ) + ret_code = proc.wait() + stderr.flush() + stdout.flush() + result['stdout'] = [line.strip() for line in open(outfile).readlines()] + result['stderr'] = [line.strip() for line in open(errfile).readlines()] + result['merged'] = '' + elif config.get('execution', 'commandline_output').lower() == 'none': + proc.communicate() + result['stdout'] = [] + result['stderr'] = [] + result['merged'] = '' + else: + raise ValueError(("Unknown input for config['execution']" + "['commandline_output']. Should be one of 'stream', " + "'allatonce', 'file', or 'none'")) - # collect results, merge and return - result = {} - temp = [] - for stream in streams: - rows = stream._rows - temp += rows - result[stream._name] = [r[2] for r in rows] - temp.sort() - result['merged'] = [r[1] for r in temp] runtime.stderr = '\n'.join(result['stderr']) runtime.stdout = '\n'.join(result['stdout']) runtime.merged = result['merged'] + runtime.returncode = proc.returncode return runtime diff --git a/nipype/interfaces/tests/test_base.py b/nipype/interfaces/tests/test_base.py index f4c7a9d327..fc55e6da63 100644 --- a/nipype/interfaces/tests/test_base.py +++ b/nipype/interfaces/tests/test_base.py @@ -461,3 +461,24 @@ def test_Commandline_environ(): ci3.inputs.environ = {'DISPLAY' : ':2'} res = ci3.run() yield assert_equal, res.runtime.environ['DISPLAY'], ':2' + + +def test_CommandLine_output(): + from nipype import config + config.set_default_config() + config.set('execution', 'commandline_output', 'foo') + ci = nib.CommandLine(command='ls -l') + yield assert_raises, ValueError, ci.run + + tmp_infile = setup_file() + tmpd, name = os.path.split(tmp_infile) + pwd = os.getcwd() + os.chdir(tmpd) + yield assert_true, os.path.exists(tmp_infile) + config.set('execution', 'commandline_output', 'allatonce') + ci = nib.CommandLine(command='ls -l') + res = ci.run() + yield assert_equal, res.runtime.merged, '' + yield assert_true, name in res.runtime.stdout + os.chdir(pwd) + teardown_file(tmpd) diff --git a/nipype/pipeline/utils.py b/nipype/pipeline/utils.py index 48f7bf3acb..39bd55d28f 100644 --- a/nipype/pipeline/utils.py +++ b/nipype/pipeline/utils.py @@ -670,7 +670,8 @@ def clean_working_directory(outputs, cwd, inputs, needed_outputs, config, input_files.extend(walk_outputs(inputdict)) needed_files += [path for path, type in input_files if type == 'f'] for extra in ['_0x*.json', 'provenance.xml', 'pyscript*.m', - 'command.txt', 'result*.pklz', '_inputs.pklz', '_node.pklz']: + 'command.txt', 'result*.pklz', '_inputs.pklz', '_node.pklz', + 'stderr.nipype', 'stdout.nipype']: needed_files.extend(glob(os.path.join(cwd, extra))) if files2keep: needed_files.extend(filename_to_list(files2keep)) diff --git a/nipype/utils/config.py b/nipype/utils/config.py index a4ee0bbc79..ee728a5cc0 100644 --- a/nipype/utils/config.py +++ b/nipype/utils/config.py @@ -46,6 +46,7 @@ stop_on_first_rerun = false use_relative_paths = false stop_on_unknown_version = false +commandline_output = stream [check] interval = 1209600 From 8353efe5e95981a9d8296f58a4e42b8c9db8f8f7 Mon Sep 17 00:00:00 2001 From: Satrajit Ghosh Date: Wed, 26 Dec 2012 22:48:06 -0500 Subject: [PATCH 2/2] tst: added tests for all options --- nipype/interfaces/tests/test_base.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nipype/interfaces/tests/test_base.py b/nipype/interfaces/tests/test_base.py index fc55e6da63..e5e9a75d6a 100644 --- a/nipype/interfaces/tests/test_base.py +++ b/nipype/interfaces/tests/test_base.py @@ -480,5 +480,17 @@ def test_CommandLine_output(): res = ci.run() yield assert_equal, res.runtime.merged, '' yield assert_true, name in res.runtime.stdout + config.set('execution', 'commandline_output', 'file') + ci = nib.CommandLine(command='ls -l') + res = ci.run() + yield assert_true, 'stdout.nipype' in res.runtime.stdout + config.set('execution', 'commandline_output', 'none') + ci = nib.CommandLine(command='ls -l') + res = ci.run() + yield assert_equal, res.runtime.stdout, '' + config.set('execution', 'commandline_output', 'stream') + ci = nib.CommandLine(command='ls -l') + res = ci.run() + yield assert_true, 'stdout.nipype' in res.runtime.stdout os.chdir(pwd) teardown_file(tmpd)