From c5278a918b1d1fedb93015b421490bd1cdc10009 Mon Sep 17 00:00:00 2001 From: Chris Filo Gorgolewski Date: Wed, 8 Jan 2014 16:22:57 +0100 Subject: [PATCH 1/6] test for datafinder --- CHANGES | 5 +++++ nipype/interfaces/tests/test_io.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGES b/CHANGES index 7e64d98601..85f884e34e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,8 @@ +Release 0.9.2 (January 8, 2014) +============ + +* FIX: DataFind was broken due to a typo + Release 0.9.1 (December 25, 2013) ============ diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 504c273678..54fcd30c13 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -171,6 +171,24 @@ def test_datasink_copydir(): shutil.rmtree(outdir) shutil.rmtree(pth) +def test_datafinder_copydir(): + outdir = mkdtemp() + open(os.path.join(outdir, "findme.txt"), 'a').close() + open(os.path.join(outdir, "dontfindme"), 'a').close() + open(os.path.join(outdir, "findmetoo.txt"), 'a').close() + + from nipype.interfaces.io import DataFinder + df = DataFinder() + df.inputs.root_paths = outdir + df.inputs.match_regex = '.txt' + result = df.run() + expected = ["findme.txt", "findmetoo.txt"] + for path, expected_fname in zip(result.outputs.out_paths, expected): + _, fname = os.path.split(path) + yield assert_equal, fname, expected_fname + + shutil.rmtree(outdir) + def test_freesurfersource(): fss = nio.FreeSurferSource() From f4b2c0247e108ff4de0fc80a9b439bdbf62cc0ab Mon Sep 17 00:00:00 2001 From: Chris Filo Gorgolewski Date: Wed, 8 Jan 2014 17:19:58 +0100 Subject: [PATCH 2/6] added sorting --- CHANGES | 3 ++- nipype/interfaces/io.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 85f884e34e..cbb66efb31 100644 --- a/CHANGES +++ b/CHANGES @@ -1,7 +1,8 @@ Release 0.9.2 (January 8, 2014) ============ -* FIX: DataFind was broken due to a typo +* FIX: DataFinder was broken due to a typo +* FIX: Order of DataFinder outputs was not guaranteed, it's human sorted now Release 0.9.1 (December 25, 2013) ============ diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 9057ed83b8..a9bc08f57f 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -829,6 +829,13 @@ def _run_interface(self, runtime): self.result[key] = vals[0] if not self.result: raise RuntimeError("Regular expression did not match any files!") + + #sort all keys acording to out_paths + for key in self.result.keys(): + if key == "out_paths": + continue + self.result[key] = [x for (_,x) in human_order_sorted(zip(self.result["out_paths"], self.result[key]))] + return runtime def _list_outputs(self): From bd5e4fee5c3768669837f33d8ea2fadfccaff222 Mon Sep 17 00:00:00 2001 From: Chris Filo Gorgolewski Date: Wed, 8 Jan 2014 17:33:55 +0100 Subject: [PATCH 3/6] added test for sorting, fixed tuples --- nipype/interfaces/tests/test_io.py | 8 +++++--- nipype/utils/misc.py | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 54fcd30c13..5f67d9bfba 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -180,12 +180,14 @@ def test_datafinder_copydir(): from nipype.interfaces.io import DataFinder df = DataFinder() df.inputs.root_paths = outdir - df.inputs.match_regex = '.txt' + df.inputs.match_regex = '.+/(?P.+).txt' result = df.run() expected = ["findme.txt", "findmetoo.txt"] for path, expected_fname in zip(result.outputs.out_paths, expected): - _, fname = os.path.split(path) - yield assert_equal, fname, expected_fname + _, fname = os.path.split(path) + yield assert_equal, fname, expected_fname + + yield assert_equal, result.outputs.basename, ["findme", "findmetoo"] shutil.rmtree(outdir) diff --git a/nipype/utils/misc.py b/nipype/utils/misc.py index 6c7aa216c5..4a6102297a 100644 --- a/nipype/utils/misc.py +++ b/nipype/utils/misc.py @@ -17,6 +17,8 @@ def atoi(text): return int(text) if text.isdigit() else text def natural_keys(text): + if isinstance(text,tuple): + text = text[0] return [ atoi(c) for c in re.split('(\d+)', text) ] return sorted(l, key=natural_keys) From 43cfb4d490fda5e141ae9cec912ee8aae52bcd8f Mon Sep 17 00:00:00 2001 From: Chris Filo Gorgolewski Date: Wed, 8 Jan 2014 17:42:48 +0100 Subject: [PATCH 4/6] PEP8 and fixed out_paths sort --- nipype/interfaces/io.py | 1 + nipype/utils/misc.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index a9bc08f57f..868fa481e8 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -831,6 +831,7 @@ def _run_interface(self, runtime): raise RuntimeError("Regular expression did not match any files!") #sort all keys acording to out_paths + self.result["out_paths"] = human_order_sorted(self.result["out_paths"]) for key in self.result.keys(): if key == "out_paths": continue diff --git a/nipype/utils/misc.py b/nipype/utils/misc.py index 4a6102297a..113049437a 100644 --- a/nipype/utils/misc.py +++ b/nipype/utils/misc.py @@ -17,7 +17,7 @@ def atoi(text): return int(text) if text.isdigit() else text def natural_keys(text): - if isinstance(text,tuple): + if isinstance(text, tuple): text = text[0] return [ atoi(c) for c in re.split('(\d+)', text) ] From 63eaeb5d476c5db253df1ba57da242a6ed8feb88 Mon Sep 17 00:00:00 2001 From: Chris Filo Gorgolewski Date: Wed, 8 Jan 2014 17:56:19 +0100 Subject: [PATCH 5/6] order matters when you do sorting --- nipype/interfaces/io.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 868fa481e8..3f46c9866d 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -831,12 +831,11 @@ def _run_interface(self, runtime): raise RuntimeError("Regular expression did not match any files!") #sort all keys acording to out_paths - self.result["out_paths"] = human_order_sorted(self.result["out_paths"]) for key in self.result.keys(): if key == "out_paths": continue self.result[key] = [x for (_,x) in human_order_sorted(zip(self.result["out_paths"], self.result[key]))] - + self.result["out_paths"] = human_order_sorted(self.result["out_paths"]) return runtime def _list_outputs(self): From a0fc11611b8c344b93af7f0dfd79efe0a943cce0 Mon Sep 17 00:00:00 2001 From: Brendan Moloney Date: Mon, 13 Jan 2014 14:25:22 -0800 Subject: [PATCH 6/6] TST+BF: Improve DataFinder tests, fix issue with unpack_single and sorting --- nipype/interfaces/io.py | 18 +++++++----- nipype/interfaces/tests/test_io.py | 45 ++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 3f46c9866d..7f14b0e796 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -827,15 +827,19 @@ def _run_interface(self, runtime): ): for key, vals in self.result.iteritems(): self.result[key] = vals[0] + else: + #sort all keys acording to out_paths + for key in self.result.keys(): + if key == "out_paths": + continue + sort_tuples = human_order_sorted(zip(self.result["out_paths"], + self.result[key])) + self.result[key] = [x for (_, x) in sort_tuples] + self.result["out_paths"] = human_order_sorted(self.result["out_paths"]) + if not self.result: raise RuntimeError("Regular expression did not match any files!") - - #sort all keys acording to out_paths - for key in self.result.keys(): - if key == "out_paths": - continue - self.result[key] = [x for (_,x) in human_order_sorted(zip(self.result["out_paths"], self.result[key]))] - self.result["out_paths"] = human_order_sorted(self.result["out_paths"]) + return runtime def _list_outputs(self): diff --git a/nipype/interfaces/tests/test_io.py b/nipype/interfaces/tests/test_io.py index 5f67d9bfba..07fcb89040 100644 --- a/nipype/interfaces/tests/test_io.py +++ b/nipype/interfaces/tests/test_io.py @@ -171,27 +171,68 @@ def test_datasink_copydir(): shutil.rmtree(outdir) shutil.rmtree(pth) + def test_datafinder_copydir(): outdir = mkdtemp() open(os.path.join(outdir, "findme.txt"), 'a').close() open(os.path.join(outdir, "dontfindme"), 'a').close() + open(os.path.join(outdir, "dontfindmealsotxt"), 'a').close() open(os.path.join(outdir, "findmetoo.txt"), 'a').close() + open(os.path.join(outdir, "ignoreme.txt"), 'a').close() + open(os.path.join(outdir, "alsoignore.txt"), 'a').close() from nipype.interfaces.io import DataFinder df = DataFinder() df.inputs.root_paths = outdir - df.inputs.match_regex = '.+/(?P.+).txt' + df.inputs.match_regex = '.+/(?P.+)\.txt' + df.inputs.ignore_regexes = ['ignore'] result = df.run() expected = ["findme.txt", "findmetoo.txt"] for path, expected_fname in zip(result.outputs.out_paths, expected): _, fname = os.path.split(path) yield assert_equal, fname, expected_fname - + yield assert_equal, result.outputs.basename, ["findme", "findmetoo"] shutil.rmtree(outdir) +def test_datafinder_depth(): + outdir = mkdtemp() + os.makedirs(os.path.join(outdir, '0', '1', '2', '3')) + + from nipype.interfaces.io import DataFinder + df = DataFinder() + df.inputs.root_paths = os.path.join(outdir, '0') + for min_depth in range(4): + for max_depth in range(min_depth, 4): + df.inputs.min_depth = min_depth + df.inputs.max_depth = max_depth + result = df.run() + expected = [str(x) for x in range(min_depth, max_depth + 1)] + for path, exp_fname in zip(result.outputs.out_paths, expected): + _, fname = os.path.split(path) + yield assert_equal, fname, exp_fname + + shutil.rmtree(outdir) + + +def test_datafinder_unpack(): + outdir = mkdtemp() + single_res = os.path.join(outdir, "findme.txt") + open(single_res, 'a').close() + open(os.path.join(outdir, "dontfindme"), 'a').close() + + from nipype.interfaces.io import DataFinder + df = DataFinder() + df.inputs.root_paths = outdir + df.inputs.match_regex = '.+/(?P.+)\.txt' + df.inputs.unpack_single = True + result = df.run() + print result.outputs.out_paths + yield assert_equal, result.outputs.out_paths, single_res + + def test_freesurfersource(): fss = nio.FreeSurferSource() yield assert_equal, fss.inputs.hemi, 'both'