From 80fcc919de4566c8f05c77308c845001d5c1f69d Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 1 Apr 2014 00:46:16 +0200 Subject: [PATCH 01/15] base code for new AddCSVRow interface --- nipype/algorithms/misc.py | 136 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index ff6c3d6cb0..f35e82fa28 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1147,6 +1147,142 @@ def _list_outputs(self): return outputs +class AddCSVRowInputSpec(TraitedSpec): + in_file = traits.File(mandatory=True, desc='Input comma-separated value (CSV) files') + cols = traits.Int(desc='Number of columns') + field_headings = traits.List(traits.Str(), mandatory=True, + desc='Heading list of available field to be added.') + + float_trait = traits.Float( 0.0, argstr='%.5f' ) + int_trait = traits.Int( 0, argstr='%d' ) + str_trait = traits.Str( '', argstr='"%s"') + row_trait = traits.Either( int_trait, float_trait, str_trait ) + new_fields = traits.List( row_trait, mandatory=True, desc='List of new values in row') + +class AddCSVRowOutputSpec(TraitedSpec): + csv_file = File(desc='Output CSV file containing rows ') + + +class AddCSVRow(BaseInterface): + """ + Short interface to add an extra row to a text file + + Example + ------- + + >>> import nipype.algorithms.misc as misc + >>> addrow = misc.AddCSVRow() + >>> addrow.inputs.in_file = 'degree.csv' + >>> addrow.inputs.field_headings = [ 'id', 'group', 'age', 'degree' ] + >>> addrow.inputs.new_fields = [ 'S400', 'male', '25', '10.5' ] + >>> addrow.run() # doctest: +SKIP + """ + input_spec = AddCSVRowInputSpec + output_spec = AddCSVRowOutputSpec + + def _run_interface(self, runtime): + cols = 0 + headings = [] + + if not isdefined( self.inputs.cols ) and not isdefined( self.inputs.field_headings ): + iflogger.error( 'Either number of cols or field headings is required' ) + + if isdefined( self.inputs.cols ) and isdefined( self.inputs.field_headings ): + if( len( self.inputs.field_headings ) != self.inputs.cols ): + iflogger.error( 'Number of cols and length of field headings list should match' ) + else: + cols = self.inputs.cols + headings = self.inputs.field_headings + + if isdefined( self.inputs.cols ) and not isdefined( self.inputs.field_headings ): + cols = self.inputs.cols + iflogger.warn( 'No column headers were set.') + + if not isdefined( self.inputs.cols ) and isdefined( self.inputs.field_headings ): + cols = len( self.inputs.field_headings ) + + if cols == 0: + iflogger.error( 'Number of cols and length of field headings must be > 0' ) + + if len( self.inputs.new_fields ) != cols: + iflogger.error( 'Wrong length of fields, does not match number of cols' ) + + with open(self.inputs.in_file, 'r+') as in_file: + lines = in_file.readlines() + + if len(headings)>0 and (len(lines) == 0 or lines[0] == ''): + hdr = [ '"%s"' % h for h in self.inputs.field_headings ] + hdrstr = ",".join(hdr) + lines.insert( 0, hdrstr ) + + print self.inputs.new_fields.items() + + metadata = dict(argstr=lambda t: t is not None) + for name, spec in sorted(self.inputs.traits(**metadata).items()): + print name + value = getattr(self.inputs, name) + if not value is None: + print value + #arg = self._format_row( name, spec, value ) + + + + #newrow = ",".join( self.inputs.new_fields ) + #lines.append( newrow ) + #in_file.writelines( lines ) + + return runtime + + + + def _format_row(self, name, trait_spec, value): + """A helper function for _run_interface + """ + argstr = trait_spec.argstr + iflogger.debug('%s_%s' % (name, str(value))) + if trait_spec.is_trait_type(traits.Bool) and "%" not in argstr: + if value: + # Boolean options have no format string. Just append options + # if True. + return argstr + else: + return None + # traits.Either turns into traits.TraitCompound and does not have any + # inner_traits + elif trait_spec.is_trait_type(traits.List) \ + or (trait_spec.is_trait_type(traits.TraitCompound) + and isinstance(value, list)): + # This is a bit simple-minded at present, and should be + # construed as the default. If more sophisticated behavior + # is needed, it can be accomplished with metadata (e.g. + # format string for list member str'ification, specifying + # the separator, etc.) + + # Depending on whether we stick with traitlets, and whether or + # not we beef up traitlets.List, we may want to put some + # type-checking code here as well + sep = trait_spec.sep + if sep is None: + sep = ' ' + if argstr.endswith('...'): + + # repeatable option + # --id %d... will expand to + # --id 1 --id 2 --id 3 etc.,. + argstr = argstr.replace('...', '') + return sep.join([argstr % elt for elt in value]) + else: + return argstr % sep.join(str(elt) for elt in value) + else: + # Append options using format string. + return argstr % value + + def _list_outputs(self): + outputs = self.output_spec().get() + outputs['csv_file'] = self.inputs.in_file + return outputs + + class CalculateNormalizedMomentsInputSpec(TraitedSpec): timeseries_file = File( exists=True, mandatory=True, From a9713908424caa4684a681dbb5ff0ce44d181e24 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 1 Apr 2014 02:13:41 +0200 Subject: [PATCH 02/15] little advances --- nipype/algorithms/misc.py | 51 ++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index f35e82fa28..175b8cb25a 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1152,12 +1152,9 @@ class AddCSVRowInputSpec(TraitedSpec): cols = traits.Int(desc='Number of columns') field_headings = traits.List(traits.Str(), mandatory=True, desc='Heading list of available field to be added.') - - float_trait = traits.Float( 0.0, argstr='%.5f' ) - int_trait = traits.Int( 0, argstr='%d' ) - str_trait = traits.Str( '', argstr='"%s"') - row_trait = traits.Either( int_trait, float_trait, str_trait ) - new_fields = traits.List( row_trait, mandatory=True, desc='List of new values in row') + new_fields = traits.List( traits.Any(), mandatory=True, desc='List of new values in row', separator=',') + col_width = traits.Int( 7, mandatory=True, usedefault=True, desc='column width' ) + float_dec = traits.Int( 4, mandatory=True, usedefault=True, desc='decimals' ) class AddCSVRowOutputSpec(TraitedSpec): csv_file = File(desc='Output CSV file containing rows ') @@ -1200,6 +1197,7 @@ def _run_interface(self, runtime): if not isdefined( self.inputs.cols ) and isdefined( self.inputs.field_headings ): cols = len( self.inputs.field_headings ) + headings = self.inputs.field_headings if cols == 0: iflogger.error( 'Number of cols and length of field headings must be > 0' ) @@ -1207,29 +1205,42 @@ def _run_interface(self, runtime): if len( self.inputs.new_fields ) != cols: iflogger.error( 'Wrong length of fields, does not match number of cols' ) - with open(self.inputs.in_file, 'r+') as in_file: + col_width = self.inputs.col_width + float_dec = self.inputs.float_dec + + with open(self.inputs.in_file, 'a+') as in_file: lines = in_file.readlines() + + if len(lines)>0 and lines[0]=='\n': + lines.pop() + + print len(lines) - if len(headings)>0 and (len(lines) == 0 or lines[0] == ''): + if (len(headings)>0) and (len(lines)==0): hdr = [ '"%s"' % h for h in self.inputs.field_headings ] hdrstr = ",".join(hdr) - lines.insert( 0, hdrstr ) + lines = [ hdrstr+'\n' ] + + if not lines[-1] or lines[-1]=='\n': + lines.pop() - print self.inputs.new_fields.items() - - metadata = dict(argstr=lambda t: t is not None) + row_data = [] + metadata = dict(separator=lambda t: t is not None) for name, spec in sorted(self.inputs.traits(**metadata).items()): - print name - value = getattr(self.inputs, name) - if not value is None: - print value - #arg = self._format_row( name, spec, value ) + values = getattr(self.inputs, name) + + for v in values: + argstr = '{:>%d}' % col_width + if type(v)=='float': + argstr = '{:>%d.%df}' % ( col_width, float_dec ) + row_data.append( argstr.format(v) ) + newrow = ",".join( row_data ) + lines.append( newrow+'\n' ) - #newrow = ",".join( self.inputs.new_fields ) - #lines.append( newrow ) - #in_file.writelines( lines ) + print "".join( lines ) + in_file.write( "".join(lines) ) return runtime From bd58815cf7028e13e3512a24d2cfbe9e96cccb10 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 1 Apr 2014 15:05:12 +0200 Subject: [PATCH 03/15] first working version --- nipype/algorithms/misc.py | 72 +++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index 175b8cb25a..599b6ac1b2 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1153,8 +1153,8 @@ class AddCSVRowInputSpec(TraitedSpec): field_headings = traits.List(traits.Str(), mandatory=True, desc='Heading list of available field to be added.') new_fields = traits.List( traits.Any(), mandatory=True, desc='List of new values in row', separator=',') - col_width = traits.Int( 7, mandatory=True, usedefault=True, desc='column width' ) - float_dec = traits.Int( 4, mandatory=True, usedefault=True, desc='decimals' ) + col_width = traits.Int( 9, mandatory=True, usedefault=True, desc='column width' ) + float_dec = traits.Int( 6, mandatory=True, usedefault=True, desc='decimals' ) class AddCSVRowOutputSpec(TraitedSpec): csv_file = File(desc='Output CSV file containing rows ') @@ -1176,10 +1176,13 @@ class AddCSVRow(BaseInterface): """ input_spec = AddCSVRowInputSpec output_spec = AddCSVRowOutputSpec + _hdrstr = None def _run_interface(self, runtime): cols = 0 headings = [] + col_width = self.inputs.col_width + float_dec = self.inputs.float_dec if not isdefined( self.inputs.cols ) and not isdefined( self.inputs.field_headings ): iflogger.error( 'Either number of cols or field headings is required' ) @@ -1205,42 +1208,45 @@ def _run_interface(self, runtime): if len( self.inputs.new_fields ) != cols: iflogger.error( 'Wrong length of fields, does not match number of cols' ) - col_width = self.inputs.col_width - float_dec = self.inputs.float_dec + if len(headings)>0: + argstr = '{:>%d}' % col_width + hdr = [ argstr.format( '"' + h + '"') for h in self.inputs.field_headings ] + self._hdrstr = ",".join(hdr) + '\n' - with open(self.inputs.in_file, 'a+') as in_file: - lines = in_file.readlines() - - if len(lines)>0 and lines[0]=='\n': - lines.pop() - print len(lines) + if op.exists( self.inputs.in_file ): + with open(self.inputs.in_file, 'a+') as in_file: + lines = in_file.readlines() - if (len(headings)>0) and (len(lines)==0): - hdr = [ '"%s"' % h for h in self.inputs.field_headings ] - hdrstr = ",".join(hdr) - lines = [ hdrstr+'\n' ] - - if not lines[-1] or lines[-1]=='\n': + if len(lines)>0 and lines[0]=='\n': lines.pop() - row_data = [] - metadata = dict(separator=lambda t: t is not None) - for name, spec in sorted(self.inputs.traits(**metadata).items()): - values = getattr(self.inputs, name) - - for v in values: - argstr = '{:>%d}' % col_width - if type(v)=='float': - argstr = '{:>%d.%df}' % ( col_width, float_dec ) - - row_data.append( argstr.format(v) ) - - newrow = ",".join( row_data ) - lines.append( newrow+'\n' ) - - print "".join( lines ) - in_file.write( "".join(lines) ) + if (len(headings)>0) and (len(lines)==0): + lines.insert(0, self._hdrstr ) + in_file.write( "".join(lines) ) + else: + with open(self.inputs.in_file, 'w+') as in_file: + in_file.write( self._hdrstr ) + + + row_data = [] + metadata = dict(separator=lambda t: t is not None) + for name, spec in sorted(self.inputs.traits(**metadata).items()): + values = getattr(self.inputs, name) + for v in values: + argstr = '{:>%d}' % col_width + if type(v) is float: + argstr = '{:>%d.%df}' % ( col_width, float_dec ) + if type(v) is str: + v = '"' + v + '"' + row_data.append( argstr.format(v) ) + newrow = ",".join( row_data ) + '\n' + + with open(self.inputs.in_file, 'r+') as in_file: + in_file.seek(-2, 2) + if in_file.read(2) == '\n\n': + in_file.seek(-1, 1) + in_file.write( newrow ) return runtime From fd99ffd797e758ed0ddd0beb4d5299b21d5e7e17 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 3 Apr 2014 16:13:09 +0200 Subject: [PATCH 04/15] more permissive treatment of rows with extra fields --- nipype/algorithms/misc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index 599b6ac1b2..c1515e4b26 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1206,7 +1206,9 @@ def _run_interface(self, runtime): iflogger.error( 'Number of cols and length of field headings must be > 0' ) if len( self.inputs.new_fields ) != cols: - iflogger.error( 'Wrong length of fields, does not match number of cols' ) + iflogger.warn( 'Wrong length of fields (%d), does not match number of \ + cols (%d)' % (len(self.inputs.new_fields), cols ) ) + cols = len( self.inputs.new_fields ) if len(headings)>0: argstr = '{:>%d}' % col_width From 5620bbe24059e366184cba20e614b58a99ddced2 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 8 Apr 2014 23:36:43 +0200 Subject: [PATCH 05/15] make specs for #829 --- .../algorithms/tests/test_auto_AddCSVRow.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 nipype/algorithms/tests/test_auto_AddCSVRow.py diff --git a/nipype/algorithms/tests/test_auto_AddCSVRow.py b/nipype/algorithms/tests/test_auto_AddCSVRow.py new file mode 100644 index 0000000000..74b0a86818 --- /dev/null +++ b/nipype/algorithms/tests/test_auto_AddCSVRow.py @@ -0,0 +1,35 @@ +# AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT +from nipype.testing import assert_equal +from nipype.algorithms.misc import AddCSVRow + +def test_AddCSVRow_inputs(): + input_map = dict(col_width=dict(mandatory=True, + usedefault=True, + ), + cols=dict(), + field_headings=dict(mandatory=True, + ), + float_dec=dict(mandatory=True, + usedefault=True, + ), + in_file=dict(mandatory=True, + ), + new_fields=dict(mandatory=True, + separator=',', + ), + ) + inputs = AddCSVRow.input_spec() + + for key, metadata in input_map.items(): + for metakey, value in metadata.items(): + yield assert_equal, getattr(inputs.traits()[key], metakey), value + +def test_AddCSVRow_outputs(): + output_map = dict(csv_file=dict(), + ) + outputs = AddCSVRow.output_spec() + + for key, metadata in output_map.items(): + for metakey, value in metadata.items(): + yield assert_equal, getattr(outputs.traits()[key], metakey), value + From 49d326e8808777b2c496851d650a396dc133c619 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 10 Apr 2014 15:18:55 +0200 Subject: [PATCH 06/15] New AddCSVRow using pandas --- nipype/algorithms/misc.py | 167 +++++++++++--------------------------- 1 file changed, 47 insertions(+), 120 deletions(-) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index c1515e4b26..235f14d623 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -30,7 +30,8 @@ from ..interfaces.base import (BaseInterface, traits, TraitedSpec, File, InputMultiPath, OutputMultiPath, - BaseInterfaceInputSpec, isdefined) + BaseInterfaceInputSpec, isdefined, + DynamicTraitedSpec ) from ..utils.filemanip import fname_presuffix, split_filename iflogger = logging.getLogger('interface') @@ -1147,19 +1148,23 @@ def _list_outputs(self): return outputs -class AddCSVRowInputSpec(TraitedSpec): +class AddCSVRowInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): in_file = traits.File(mandatory=True, desc='Input comma-separated value (CSV) files') - cols = traits.Int(desc='Number of columns') - field_headings = traits.List(traits.Str(), mandatory=True, - desc='Heading list of available field to be added.') - new_fields = traits.List( traits.Any(), mandatory=True, desc='List of new values in row', separator=',') - col_width = traits.Int( 9, mandatory=True, usedefault=True, desc='column width' ) - float_dec = traits.Int( 6, mandatory=True, usedefault=True, desc='decimals' ) + _outputs = traits.Dict( traits.Any, value={}, usedefault=True ) + + def __setattr__(self, key, value): + if key not in self.copyable_trait_names(): + if not isdefined(value): + super(AddCSVRowInputSpec, self).__setattr__(key, value) + self._outputs[key] = value + else: + if key in self._outputs: + self._outputs[key] = value + super(AddCSVRowInputSpec, self).__setattr__(key, value) class AddCSVRowOutputSpec(TraitedSpec): csv_file = File(desc='Output CSV file containing rows ') - class AddCSVRow(BaseInterface): """ Short interface to add an extra row to a text file @@ -1169,133 +1174,55 @@ class AddCSVRow(BaseInterface): >>> import nipype.algorithms.misc as misc >>> addrow = misc.AddCSVRow() - >>> addrow.inputs.in_file = 'degree.csv' - >>> addrow.inputs.field_headings = [ 'id', 'group', 'age', 'degree' ] - >>> addrow.inputs.new_fields = [ 'S400', 'male', '25', '10.5' ] + >>> addrow.inputs.in_file = 'scores.csv' + >>> addrow.inputs.si = 0.74 + >>> addrow.inputs.di = 0.93 + >>> addrow.subject_id = 'S400' + >>> addrow.inputs.list_of_values = [ 0.4, 0.7, 0.3 ] >>> addrow.run() # doctest: +SKIP """ input_spec = AddCSVRowInputSpec output_spec = AddCSVRowOutputSpec - _hdrstr = None - def _run_interface(self, runtime): - cols = 0 - headings = [] - col_width = self.inputs.col_width - float_dec = self.inputs.float_dec + def __init__(self, infields=None, force_run=True, **kwargs): + super(AddCSVRow, self).__init__(**kwargs) + undefined_traits = {} + self._infields = infields - if not isdefined( self.inputs.cols ) and not isdefined( self.inputs.field_headings ): - iflogger.error( 'Either number of cols or field headings is required' ) + if infields: + for key in infields: + self.inputs.add_trait( key, traits.Any ) + self.inputs._outputs[key] = Undefined + undefined_traits[key] = Undefined + self.inputs.trait_set( trait_change_notify=False, **undefined_traits ) - if isdefined( self.inputs.cols ) and isdefined( self.inputs.field_headings ): - if( len( self.inputs.field_headings ) != self.inputs.cols ): - iflogger.error( 'Number of cols and length of field headings list should match' ) - else: - cols = self.inputs.cols - headings = self.inputs.field_headings + if force_run: + self._always_run = True - if isdefined( self.inputs.cols ) and not isdefined( self.inputs.field_headings ): - cols = self.inputs.cols - iflogger.warn( 'No column headers were set.') - - if not isdefined( self.inputs.cols ) and isdefined( self.inputs.field_headings ): - cols = len( self.inputs.field_headings ) - headings = self.inputs.field_headings - - if cols == 0: - iflogger.error( 'Number of cols and length of field headings must be > 0' ) - - if len( self.inputs.new_fields ) != cols: - iflogger.warn( 'Wrong length of fields (%d), does not match number of \ - cols (%d)' % (len(self.inputs.new_fields), cols ) ) - cols = len( self.inputs.new_fields ) + def _run_interface(self, runtime): + import pandas as pd - if len(headings)>0: - argstr = '{:>%d}' % col_width - hdr = [ argstr.format( '"' + h + '"') for h in self.inputs.field_headings ] - self._hdrstr = ",".join(hdr) + '\n' + input_dict = {} + for key, val in self.inputs._outputs.items(): + # expand lists to several columns + if isinstance(val, list): + for i,v in enumerate(val): + input_dict['%s_%d' % (key,i)]=v + else: + input_dict[key] = val - if op.exists( self.inputs.in_file ): - with open(self.inputs.in_file, 'a+') as in_file: - lines = in_file.readlines() + df = pd.DataFrame([input_dict]) - if len(lines)>0 and lines[0]=='\n': - lines.pop() + if op.exists(self.inputs.in_file): + formerdf = pd.read_csv(self.inputs.in_file, index_col=0) + df = pd.concat( [formerdf, df], ignore_index=True ) - if (len(headings)>0) and (len(lines)==0): - lines.insert(0, self._hdrstr ) - in_file.write( "".join(lines) ) - else: - with open(self.inputs.in_file, 'w+') as in_file: - in_file.write( self._hdrstr ) - - - row_data = [] - metadata = dict(separator=lambda t: t is not None) - for name, spec in sorted(self.inputs.traits(**metadata).items()): - values = getattr(self.inputs, name) - for v in values: - argstr = '{:>%d}' % col_width - if type(v) is float: - argstr = '{:>%d.%df}' % ( col_width, float_dec ) - if type(v) is str: - v = '"' + v + '"' - row_data.append( argstr.format(v) ) - newrow = ",".join( row_data ) + '\n' - - with open(self.inputs.in_file, 'r+') as in_file: - in_file.seek(-2, 2) - if in_file.read(2) == '\n\n': - in_file.seek(-1, 1) - in_file.write( newrow ) + with open(self.inputs.in_file, 'w') as f: + df.to_csv(f) return runtime - - - def _format_row(self, name, trait_spec, value): - """A helper function for _run_interface - """ - argstr = trait_spec.argstr - iflogger.debug('%s_%s' % (name, str(value))) - if trait_spec.is_trait_type(traits.Bool) and "%" not in argstr: - if value: - # Boolean options have no format string. Just append options - # if True. - return argstr - else: - return None - # traits.Either turns into traits.TraitCompound and does not have any - # inner_traits - elif trait_spec.is_trait_type(traits.List) \ - or (trait_spec.is_trait_type(traits.TraitCompound) - and isinstance(value, list)): - # This is a bit simple-minded at present, and should be - # construed as the default. If more sophisticated behavior - # is needed, it can be accomplished with metadata (e.g. - # format string for list member str'ification, specifying - # the separator, etc.) - - # Depending on whether we stick with traitlets, and whether or - # not we beef up traitlets.List, we may want to put some - # type-checking code here as well - sep = trait_spec.sep - if sep is None: - sep = ' ' - if argstr.endswith('...'): - - # repeatable option - # --id %d... will expand to - # --id 1 --id 2 --id 3 etc.,. - argstr = argstr.replace('...', '') - return sep.join([argstr % elt for elt in value]) - else: - return argstr % sep.join(str(elt) for elt in value) - else: - # Append options using format string. - return argstr % value - def _list_outputs(self): outputs = self.output_spec().get() outputs['csv_file'] = self.inputs.in_file From 8c0626c7d33c3fc56179c027441eabbb439ae7d8 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 10 Apr 2014 15:26:11 +0200 Subject: [PATCH 07/15] Added pandas as requirement in documentation --- nipype/algorithms/misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index 235f14d623..a78ea387e1 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1168,6 +1168,7 @@ class AddCSVRowOutputSpec(TraitedSpec): class AddCSVRow(BaseInterface): """ Short interface to add an extra row to a text file + Requires pandas - http://pandas.pydata.org/ Example ------- From 56e13315b3fefb8140eb20c5d022fcdb6fdc5d13 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 10 Apr 2014 15:28:49 +0200 Subject: [PATCH 08/15] Added pandas try except before execution --- nipype/algorithms/misc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index a78ea387e1..595a2884e3 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1201,7 +1201,11 @@ def __init__(self, infields=None, force_run=True, **kwargs): self._always_run = True def _run_interface(self, runtime): - import pandas as pd + try: + import pandas as pd + except ImportError: + raise ImportError('This interface requires pandas (http://pandas.pydata.org/) to run.') + input_dict = {} From 96462b2d6780649f2a3f3bf662888e957f9e2b5d Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 10 Apr 2014 15:46:47 +0200 Subject: [PATCH 09/15] Updated auto test --- nipype/algorithms/tests/test_auto_AddCSVRow.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/nipype/algorithms/tests/test_auto_AddCSVRow.py b/nipype/algorithms/tests/test_auto_AddCSVRow.py index 74b0a86818..09b2c58adb 100644 --- a/nipype/algorithms/tests/test_auto_AddCSVRow.py +++ b/nipype/algorithms/tests/test_auto_AddCSVRow.py @@ -3,20 +3,13 @@ from nipype.algorithms.misc import AddCSVRow def test_AddCSVRow_inputs(): - input_map = dict(col_width=dict(mandatory=True, - usedefault=True, - ), - cols=dict(), - field_headings=dict(mandatory=True, + input_map = dict(_outputs=dict(usedefault=True, ), - float_dec=dict(mandatory=True, + ignore_exception=dict(nohash=True, usedefault=True, ), in_file=dict(mandatory=True, ), - new_fields=dict(mandatory=True, - separator=',', - ), ) inputs = AddCSVRow.input_spec() From 916ddfe8e6f33e2530ac533bfa03368476697de4 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Fri, 11 Apr 2014 13:07:03 +0200 Subject: [PATCH 10/15] Finished AddCSVRow Methods _outputs and _add_output_traits were missing to provide a fully IOBase-like interface. --- nipype/algorithms/misc.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index 595a2884e3..ec74a83151 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1233,6 +1233,12 @@ def _list_outputs(self): outputs['csv_file'] = self.inputs.in_file return outputs + def _outputs(self): + return self._add_output_traits(super(AddCSVRow, self)._outputs()) + + def _add_output_traits(self, base): + return base + class CalculateNormalizedMomentsInputSpec(TraitedSpec): timeseries_file = File( From de373e497dd1da916b89c414d604004339ad33e0 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 27 May 2014 17:50:39 +0200 Subject: [PATCH 11/15] Added warning in constructor and documentation --- nipype/algorithms/misc.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index ec74a83151..e5b5d277c0 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1166,10 +1166,19 @@ class AddCSVRowOutputSpec(TraitedSpec): csv_file = File(desc='Output CSV file containing rows ') class AddCSVRow(BaseInterface): - """ - Short interface to add an extra row to a text file + """Simple interface to add an extra row to a csv file + + .. warning:: + + This interface is not thread-safe in multi-proc mode when + writing the output file. + + + .. note:: + Requires pandas - http://pandas.pydata.org/ + Example ------- @@ -1186,6 +1195,7 @@ class AddCSVRow(BaseInterface): output_spec = AddCSVRowOutputSpec def __init__(self, infields=None, force_run=True, **kwargs): + warnings.warn('AddCSVRow is not thread-safe in multi-processor execution') super(AddCSVRow, self).__init__(**kwargs) undefined_traits = {} self._infields = infields From 7e4d434bc782f7cd46b4654df1a062c484539020 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 28 May 2014 10:27:51 +0200 Subject: [PATCH 12/15] Fixing failing test --- nipype/algorithms/misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index e5b5d277c0..05f76051be 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1195,6 +1195,7 @@ class AddCSVRow(BaseInterface): output_spec = AddCSVRowOutputSpec def __init__(self, infields=None, force_run=True, **kwargs): + import warnings warnings.warn('AddCSVRow is not thread-safe in multi-processor execution') super(AddCSVRow, self).__init__(**kwargs) undefined_traits = {} From 4d3e36871cf53297c623c592c3f41f881f4ad993 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Wed, 25 Jun 2014 12:11:49 +0200 Subject: [PATCH 13/15] Added file-locking to interface Now, the interface is thread-safe using lockfile. It is included in documentation, and also a warning is issued when the module is not available or could not be imported. --- nipype/algorithms/misc.py | 41 ++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index 05f76051be..ad3b646470 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -26,13 +26,13 @@ import itertools import scipy.stats as stats -from .. import logging +from nipype import logging -from ..interfaces.base import (BaseInterface, traits, TraitedSpec, File, +from nipype.interfaces.base import (BaseInterface, traits, TraitedSpec, File, InputMultiPath, OutputMultiPath, BaseInterfaceInputSpec, isdefined, DynamicTraitedSpec ) -from ..utils.filemanip import fname_presuffix, split_filename +from nipype.utils.filemanip import fname_presuffix, split_filename iflogger = logging.getLogger('interface') @@ -1168,15 +1168,12 @@ class AddCSVRowOutputSpec(TraitedSpec): class AddCSVRow(BaseInterface): """Simple interface to add an extra row to a csv file - .. warning:: + .. note:: Requires `pandas `_ - This interface is not thread-safe in multi-proc mode when - writing the output file. - - - .. note:: - - Requires pandas - http://pandas.pydata.org/ + .. warning:: Multi-platform thread-safe execution is possible with + `lockfile `_. Please recall that (1) + this module is alpha software; and (2) it should be installed for thread-safe writing. + If lockfile is not installed, then the interface is not thread-safe. Example @@ -1193,10 +1190,10 @@ class AddCSVRow(BaseInterface): """ input_spec = AddCSVRowInputSpec output_spec = AddCSVRowOutputSpec + _have_lock = False + _lock = None def __init__(self, infields=None, force_run=True, **kwargs): - import warnings - warnings.warn('AddCSVRow is not thread-safe in multi-processor execution') super(AddCSVRow, self).__init__(**kwargs) undefined_traits = {} self._infields = infields @@ -1217,6 +1214,15 @@ def _run_interface(self, runtime): except ImportError: raise ImportError('This interface requires pandas (http://pandas.pydata.org/) to run.') + try: + import lockfile as pl + self._have_lock = True + except ImportError: + import warnings + warnings.warn(('Python module lockfile was not found: AddCSVRow will not be thread-safe ' + 'in multi-processor execution')) + + input_dict = {} @@ -1230,6 +1236,12 @@ def _run_interface(self, runtime): df = pd.DataFrame([input_dict]) + if self._have_lock: + self._lock = pl.FileLock(self.inputs.in_file) + + # Acquire lock + self._lock.acquire() + if op.exists(self.inputs.in_file): formerdf = pd.read_csv(self.inputs.in_file, index_col=0) df = pd.concat( [formerdf, df], ignore_index=True ) @@ -1237,6 +1249,9 @@ def _run_interface(self, runtime): with open(self.inputs.in_file, 'w') as f: df.to_csv(f) + if self._have_lock: + self._lock.release() + return runtime def _list_outputs(self): From 8103357ccf9e0b175115a0cb6966bdc41baeef01 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Thu, 24 Jul 2014 12:31:19 +0200 Subject: [PATCH 14/15] Added fix to inputs check of workflows to allow ... ... dynamic input traits --- nipype/pipeline/engine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nipype/pipeline/engine.py b/nipype/pipeline/engine.py index e3a7ad2757..b7b873b5aa 100644 --- a/nipype/pipeline/engine.py +++ b/nipype/pipeline/engine.py @@ -190,6 +190,8 @@ def _check_outputs(self, parameter): return hasattr(self.outputs, parameter) def _check_inputs(self, parameter): + if isinstance(self.inputs, DynamicTraitedSpec): + return True return hasattr(self.inputs, parameter) def _verify_name(self, name): From cda31e92dd1359643c9ca679c9a894d8bddfca64 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Mon, 28 Jul 2014 13:06:57 +0200 Subject: [PATCH 15/15] Minor fixes, still needed some further tests --- CHANGES | 1 + nipype/algorithms/misc.py | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index b21025bfdb..272afb6057 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,7 @@ Next Release ============ +* ENH: New miscelaneous interface: AddCSVRow * API: Interfaces to external packages are no longer available in the top-level ``nipype`` namespace, and must be imported directly (e.g. ``from nipype.interfaces import fsl``). * ENH: New ANTs interface: ApplyTransformsToPoints * ENH: New FreeSurfer workflow: create_skullstripped_recon_flow() diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index ad3b646470..88bab1f3a6 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1190,20 +1190,20 @@ class AddCSVRow(BaseInterface): """ input_spec = AddCSVRowInputSpec output_spec = AddCSVRowOutputSpec - _have_lock = False - _lock = None def __init__(self, infields=None, force_run=True, **kwargs): super(AddCSVRow, self).__init__(**kwargs) undefined_traits = {} self._infields = infields + self._have_lock = False + self._lock = None if infields: for key in infields: self.inputs.add_trait( key, traits.Any ) self.inputs._outputs[key] = Undefined undefined_traits[key] = Undefined - self.inputs.trait_set( trait_change_notify=False, **undefined_traits ) + self.inputs.trait_set(trait_change_notify=False, **undefined_traits ) if force_run: self._always_run = True @@ -1222,10 +1222,7 @@ def _run_interface(self, runtime): warnings.warn(('Python module lockfile was not found: AddCSVRow will not be thread-safe ' 'in multi-processor execution')) - - input_dict = {} - for key, val in self.inputs._outputs.items(): # expand lists to several columns if isinstance(val, list):