From 4c1c44a08bde0db88fbc716234aab74a301e8b43 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Fri, 16 Nov 2018 16:34:07 +0300 Subject: [PATCH 01/27] Added command line arguments tests. --- .../pyDKB/dataflow/stage/tests/__init__.py | 0 .../stage/tests/test_ProcessorStage.py | 137 ++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 Utils/Dataflow/pyDKB/dataflow/stage/tests/__init__.py create mode 100644 Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/__init__.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py new file mode 100644 index 000000000..c442cd979 --- /dev/null +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python + +""" +Tests for pyDKB.dataflow.stage.ProcessorStage. +Usage: 'python -m unittest discover' from .. +(directory with pyDKB.dataflow.stage code). +""" + + +import os +import sys +import unittest + + +# Relative import inside of pyDKB prevents the use of simple 'import pyDKB'. +try: + base_dir = os.path.dirname(__file__) # Directory with this file + dkb_dir = os.path.join(base_dir, os.pardir) # stage directory + dkb_dir = os.path.join(dkb_dir, os.pardir) # dataflow directory + dkb_dir = os.path.join(dkb_dir, os.pardir) # pyDKB's directory + dkb_dir = os.path.join(dkb_dir, os.pardir) # pyDKB's parent directory + sys.path.append(dkb_dir) + import pyDKB +except Exception, err: + sys.stderr.write("(ERROR) Failed to import pyDKB library: %s\n" % err) + sys.exit(1) + + +class ProcessorStageArgsTestCase(unittest.TestCase): + default_args = { + 'mode': 'f', + 'config': None, + 'eom': '\n', + 'eop': '', + 'source': 'f', + 'dest': 'f', + 'input_dir': os.curdir, + 'output_dir': 'out', + 'hdfs': False, + 'input_files': [] + } + + def setUp(self): + self.stage = pyDKB.dataflow.stage.ProcessorStage() + + def tearDown(self): + self.stage = None + + def check_args(self, args): + for a in args: + self.assertEqual(getattr(self.stage.ARGS, a), args[a]) + + def test_default(self): + self.stage.parse_args('') + self.check_args(self.default_args) + + def test_hdfs(self): + self.stage.parse_args(['--hdfs']) + args = dict(self.default_args) + args['hdfs'] = True + self.check_args(args) + + def test_eom(self): + self.stage.parse_args(['-e', '\t']) + args = dict(self.default_args) + args['eom'] = '\t' + self.check_args(args) + + def test_eop(self): + self.stage.parse_args(['-E', '\t']) + args = dict(self.default_args) + args['eop'] = '\t' + self.check_args(args) + + def test_input_dir(self): + self.stage.parse_args(['--input-dir', 'something']) + args = dict(self.default_args) + args['input_dir'] = 'something' + self.check_args(args) + + def test_i(self): + self.stage.parse_args(['-i', 'something']) + args = dict(self.default_args) + args['input_dir'] = 'something' + self.check_args(args) + + def test_output_dir(self): + self.stage.parse_args(['--output-dir', 'something']) + args = dict(self.default_args) + args['output_dir'] = 'something' + self.check_args(args) + + def test_o(self): + self.stage.parse_args(['-o', 'something']) + args = dict(self.default_args) + args['output_dir'] = 'something' + self.check_args(args) + + +args_to_add = { + 'source': ['f', 's', 'h'], + 'dest': ['f', 's', 'h'], + 'mode': ['f', 's', 'm'], +} + + +def add_arg(arg, val, short=False): + def f(self): + if short: + self.stage.parse_args(['-' + arg[0], val]) + else: + self.stage.parse_args(['--' + arg, val]) + args = dict(self.default_args) + args[arg] = val + self.check_args(args) + if short: + setattr(ProcessorStageArgsTestCase, 'test_%s_%s' % (arg[0], val), f) + else: + setattr(ProcessorStageArgsTestCase, 'test_%s_%s' % (arg, val), f) + + +for a in args_to_add: + for v in args_to_add[a]: + add_arg(a, v) + add_arg(a, v, True) + + +test_cases = ( + ProcessorStageArgsTestCase, +) + + +def load_tests(loader, tests, pattern): + suite = unittest.TestSuite() + for case in test_cases: + suite.addTest(loader.loadTestsFromTestCase(case)) + return suite From e1343c50ce5fb3a6964fce455fda9c5dbaf8e6c6 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Mon, 19 Nov 2018 16:45:19 +0300 Subject: [PATCH 02/27] Updated a check_args method. Now the name of an argument that causes assertion failure should be obtainable. --- .../pyDKB/dataflow/stage/tests/test_ProcessorStage.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index c442cd979..6821f9da6 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -48,7 +48,13 @@ def tearDown(self): def check_args(self, args): for a in args: - self.assertEqual(getattr(self.stage.ARGS, a), args[a]) + # Such kind of testing does not display the argument's name, + # hence the "a + '_'" addition. + if isinstance(args[a], str): + self.assertEqual(a + '_' + getattr(self.stage.ARGS, a), + a + '_' + args[a]) + else: + self.assertEqual(getattr(self.stage.ARGS, a), args[a]) def test_default(self): self.stage.parse_args('') From 1022253d95c7065fd7874a454f7210a8f45bb2e1 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Mon, 19 Nov 2018 16:54:26 +0300 Subject: [PATCH 03/27] Reworked mode tests. This includes mode dictionary creation, removal of 'mode' member from args_to_add dictionary and repositioning of the latter for the sake of consistency. --- .../stage/tests/test_ProcessorStage.py | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 6821f9da6..36f66ce8e 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -26,6 +26,20 @@ sys.exit(1) +args_to_add = { + 'source': ['f', 's', 'h'], + 'dest': ['f', 's', 'h'], +} + + +# mode: source, dest, EOM, EOP +modes = { + 's': ['s', 's', '\n', '\0'], + 'f': ['f', 'f', '\n', ''], + 'm': ['s', 's', '\n', ''], +} + + class ProcessorStageArgsTestCase(unittest.TestCase): default_args = { 'mode': 'f', @@ -103,11 +117,6 @@ def test_o(self): self.check_args(args) -args_to_add = { - 'source': ['f', 's', 'h'], - 'dest': ['f', 's', 'h'], - 'mode': ['f', 's', 'm'], -} def add_arg(arg, val, short=False): @@ -125,10 +134,32 @@ def f(self): setattr(ProcessorStageArgsTestCase, 'test_%s_%s' % (arg, val), f) +def add_mode(val, short=False): + def f(self): + if short: + self.stage.parse_args(['-m', val]) + else: + self.stage.parse_args(['--mode', val]) + args = dict(self.default_args) + args['mode'] = val + args['source'] = modes[val][0] + args['dest'] = modes[val][1] + args['eom'] = modes[val][2] + args['eop'] = modes[val][3] + self.check_args(args) + if short: + setattr(ProcessorStageArgsTestCase, 'test_m_%s' % (val), f) + else: + setattr(ProcessorStageArgsTestCase, 'test_mode_%s' % (val), f) + + for a in args_to_add: for v in args_to_add[a]: add_arg(a, v) add_arg(a, v, True) +for m in modes: + add_mode(m) + add_mode(m, True) test_cases = ( From 088e180c51c167b420a2b13c97ed0c8a7837dcae Mon Sep 17 00:00:00 2001 From: Evildoor Date: Mon, 19 Nov 2018 16:55:32 +0300 Subject: [PATCH 04/27] Added testing for arguments' interactions. --- .../stage/tests/test_ProcessorStage.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 36f66ce8e..089470e4f 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -78,6 +78,8 @@ def test_hdfs(self): self.stage.parse_args(['--hdfs']) args = dict(self.default_args) args['hdfs'] = True + args['source'] = 'h' + args['dest'] = 'h' self.check_args(args) def test_eom(self): @@ -116,7 +118,54 @@ def test_o(self): args['output_dir'] = 'something' self.check_args(args) +# hdfs >> source-dest >> mode + def test_override_hdfs_m_s(self): + self.stage.parse_args(['-m', 's', '--hdfs']) + args = dict(self.default_args) + args['hdfs'] = True + args['source'] = 'h' + args['dest'] = 'h' + self.check_args(args) + + def test_override_hdfs_m_f(self): + self.stage.parse_args(['-m', 'f', '--hdfs']) + args = dict(self.default_args) + args['hdfs'] = True + args['source'] = 'h' + args['dest'] = 'h' + self.check_args(args) + + def test_override_hdfs_m_m(self): + self.stage.parse_args(['-m', 'm', '--hdfs']) + args = dict(self.default_args) + args['hdfs'] = True + args['source'] = 'h' + args['dest'] = 'h' + self.check_args(args) + def test_override_hdfs_mode_s(self): + self.stage.parse_args(['--mode', 's', '--hdfs']) + args = dict(self.default_args) + args['hdfs'] = True + args['source'] = 'h' + args['dest'] = 'h' + self.check_args(args) + + def test_override_hdfs_mode_f(self): + self.stage.parse_args(['--mode', 'f', '--hdfs']) + args = dict(self.default_args) + args['hdfs'] = True + args['source'] = 'h' + args['dest'] = 'h' + self.check_args(args) + + def test_override_hdfs_mode_m(self): + self.stage.parse_args(['--mode', 'm', '--hdfs']) + args = dict(self.default_args) + args['hdfs'] = True + args['source'] = 'h' + args['dest'] = 'h' + self.check_args(args) def add_arg(arg, val, short=False): @@ -153,10 +202,58 @@ def f(self): setattr(ProcessorStageArgsTestCase, 'test_mode_%s' % (val), f) +def add_override_hdfs(arg, val, short=False): + def f(self): + if short: + self.stage.parse_args(['--hdfs', '-' + arg[0], val]) + else: + self.stage.parse_args(['--hdfs', '--' + arg, val]) + args = dict(self.default_args) + args['hdfs'] = True + args['source'] = 'h' + args['dest'] = 'h' + self.check_args(args) + if short: + setattr(ProcessorStageArgsTestCase, + 'test_override_hdfs_%s_%s' % (arg[0], val), f) + else: + setattr(ProcessorStageArgsTestCase, + 'test_override_hdfs_%s_%s' % (arg, val), f) + + +def add_override_mode(arg, val, mode_val, short=False): + def f(self): + if short: + self.stage.parse_args(['-' + arg[0], val, '--mode', mode_val]) + else: + self.stage.parse_args(['--' + arg, val, '--mode', mode_val]) + args = dict(self.default_args) + args['mode'] = mode_val + args['source'] = modes[mode_val][0] + args['dest'] = modes[mode_val][1] + args['eom'] = modes[mode_val][2] + args['eop'] = modes[mode_val][3] + args[arg] = val + self.check_args(args) + if short: + setattr(ProcessorStageArgsTestCase, + 'test_override_%s_%s_mode_%s' % (arg[0], val, mode_val), f) + else: + setattr(ProcessorStageArgsTestCase, + 'test_override_%s_%s_mode_%s' % (arg, val, mode_val), f) + + for a in args_to_add: for v in args_to_add[a]: add_arg(a, v) add_arg(a, v, True) + add_override_hdfs(a, v) + add_override_hdfs(a, v, True) + for m in ('s', 'f', 'm'): + add_override_mode(a, v, m) + add_override_mode(a, v, m, True) + + for m in modes: add_mode(m) add_mode(m, True) From 339c3ca2e97c27a19612344878284fb23528c72a Mon Sep 17 00:00:00 2001 From: Evildoor Date: Tue, 20 Nov 2018 15:23:30 +0300 Subject: [PATCH 05/27] Improved check_args method. --- .../pyDKB/dataflow/stage/tests/test_ProcessorStage.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 089470e4f..aea415948 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -63,12 +63,8 @@ def tearDown(self): def check_args(self, args): for a in args: # Such kind of testing does not display the argument's name, - # hence the "a + '_'" addition. - if isinstance(args[a], str): - self.assertEqual(a + '_' + getattr(self.stage.ARGS, a), - a + '_' + args[a]) - else: - self.assertEqual(getattr(self.stage.ARGS, a), args[a]) + # hence the {a: ...} addition. + self.assertEqual({a: getattr(self.stage.ARGS, a)}, {a: args[a]}) def test_default(self): self.stage.parse_args('') From 1a56cd0342bd5ac973bc1078afa8661c955d63f3 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Tue, 20 Nov 2018 16:23:31 +0300 Subject: [PATCH 06/27] Removed unnecesary redefinition. --- .../Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index aea415948..7b5d93af0 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -245,7 +245,7 @@ def f(self): add_arg(a, v, True) add_override_hdfs(a, v) add_override_hdfs(a, v, True) - for m in ('s', 'f', 'm'): + for m in modes: add_override_mode(a, v, m) add_override_mode(a, v, m, True) From dcbdeab52a5912920b211a4883806ad400acbf0a Mon Sep 17 00:00:00 2001 From: Evildoor Date: Tue, 20 Nov 2018 16:25:31 +0300 Subject: [PATCH 07/27] Reworked modes handling. This way the code should be more flexible and easier to read. --- .../stage/tests/test_ProcessorStage.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 7b5d93af0..46cae87b6 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -32,11 +32,10 @@ } -# mode: source, dest, EOM, EOP modes = { - 's': ['s', 's', '\n', '\0'], - 'f': ['f', 'f', '\n', ''], - 'm': ['s', 's', '\n', ''], + 's': {'mode': 's', 'source': 's', 'dest': 's', 'eom': '\n', 'eop': '\0'}, + 'f': {'mode': 'f', 'source': 'f', 'dest': 'f', 'eom': '\n', 'eop': ''}, + 'm': {'mode': 'm', 'source': 's', 'dest': 's', 'eom': '\n', 'eop': ''}, } @@ -186,11 +185,7 @@ def f(self): else: self.stage.parse_args(['--mode', val]) args = dict(self.default_args) - args['mode'] = val - args['source'] = modes[val][0] - args['dest'] = modes[val][1] - args['eom'] = modes[val][2] - args['eop'] = modes[val][3] + args.update(modes[val]) self.check_args(args) if short: setattr(ProcessorStageArgsTestCase, 'test_m_%s' % (val), f) @@ -224,11 +219,7 @@ def f(self): else: self.stage.parse_args(['--' + arg, val, '--mode', mode_val]) args = dict(self.default_args) - args['mode'] = mode_val - args['source'] = modes[mode_val][0] - args['dest'] = modes[mode_val][1] - args['eom'] = modes[mode_val][2] - args['eop'] = modes[mode_val][3] + args.update(modes[mode_val]) args[arg] = val self.check_args(args) if short: From 3eaa413d3c7aa6269fc65bdce5ffa69b0bb4cf5c Mon Sep 17 00:00:00 2001 From: Evildoor Date: Tue, 20 Nov 2018 16:45:20 +0300 Subject: [PATCH 08/27] Added long versions for eop and eom testing. --- .../dataflow/stage/tests/test_ProcessorStage.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 46cae87b6..d1d7bbda8 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -77,18 +77,30 @@ def test_hdfs(self): args['dest'] = 'h' self.check_args(args) - def test_eom(self): + def test_e(self): self.stage.parse_args(['-e', '\t']) args = dict(self.default_args) args['eom'] = '\t' self.check_args(args) - def test_eop(self): + def test_end_of_message(self): + self.stage.parse_args(['--end-of-message', '\t']) + args = dict(self.default_args) + args['eom'] = '\t' + self.check_args(args) + + def test_E(self): self.stage.parse_args(['-E', '\t']) args = dict(self.default_args) args['eop'] = '\t' self.check_args(args) + def test_end_of_process(self): + self.stage.parse_args(['--end-of-process', '\t']) + args = dict(self.default_args) + args['eop'] = '\t' + self.check_args(args) + def test_input_dir(self): self.stage.parse_args(['--input-dir', 'something']) args = dict(self.default_args) From 1565684a8132409e1a6de9ce777bcc768525f869 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Tue, 20 Nov 2018 16:46:41 +0300 Subject: [PATCH 09/27] Improved consistency: short version before long. --- .../stage/tests/test_ProcessorStage.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index d1d7bbda8..f651b84b7 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -101,26 +101,26 @@ def test_end_of_process(self): args['eop'] = '\t' self.check_args(args) - def test_input_dir(self): - self.stage.parse_args(['--input-dir', 'something']) + def test_i(self): + self.stage.parse_args(['-i', 'something']) args = dict(self.default_args) args['input_dir'] = 'something' self.check_args(args) - def test_i(self): - self.stage.parse_args(['-i', 'something']) + def test_input_dir(self): + self.stage.parse_args(['--input-dir', 'something']) args = dict(self.default_args) args['input_dir'] = 'something' self.check_args(args) - def test_output_dir(self): - self.stage.parse_args(['--output-dir', 'something']) + def test_o(self): + self.stage.parse_args(['-o', 'something']) args = dict(self.default_args) args['output_dir'] = 'something' self.check_args(args) - def test_o(self): - self.stage.parse_args(['-o', 'something']) + def test_output_dir(self): + self.stage.parse_args(['--output-dir', 'something']) args = dict(self.default_args) args['output_dir'] = 'something' self.check_args(args) @@ -244,18 +244,18 @@ def f(self): for a in args_to_add: for v in args_to_add[a]: - add_arg(a, v) add_arg(a, v, True) - add_override_hdfs(a, v) + add_arg(a, v) add_override_hdfs(a, v, True) + add_override_hdfs(a, v) for m in modes: - add_override_mode(a, v, m) add_override_mode(a, v, m, True) + add_override_mode(a, v, m) for m in modes: - add_mode(m) add_mode(m, True) + add_mode(m) test_cases = ( From be922c9b1ff3a2a06d29b6108afc08983a8bfc5f Mon Sep 17 00:00:00 2001 From: Evildoor Date: Tue, 20 Nov 2018 16:51:51 +0300 Subject: [PATCH 10/27] Added input files test. --- .../pyDKB/dataflow/stage/tests/test_ProcessorStage.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index f651b84b7..12cb24a5f 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -174,6 +174,12 @@ def test_override_hdfs_mode_m(self): args['dest'] = 'h' self.check_args(args) + def test_input_files(self): + self.stage.parse_args(['something', 'something_else']) + args = dict(self.default_args) + args['input_files'] = ['something', 'something_else'] + self.check_args(args) + def add_arg(arg, val, short=False): def f(self): From aa90f7c979a10b3b8ddcebe05d8667cadb7c93d5 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Tue, 20 Nov 2018 17:01:27 +0300 Subject: [PATCH 11/27] Removed short version of override tests. Both short and long versions of defining an argument are doing the same thing and are processed by argparse. Therefore, it is presumed that testing both versions of defining each argument is enough and overriding tests can be done just for one version. --- .../stage/tests/test_ProcessorStage.py | 56 +++---------------- 1 file changed, 8 insertions(+), 48 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 12cb24a5f..9646430dc 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -126,30 +126,6 @@ def test_output_dir(self): self.check_args(args) # hdfs >> source-dest >> mode - def test_override_hdfs_m_s(self): - self.stage.parse_args(['-m', 's', '--hdfs']) - args = dict(self.default_args) - args['hdfs'] = True - args['source'] = 'h' - args['dest'] = 'h' - self.check_args(args) - - def test_override_hdfs_m_f(self): - self.stage.parse_args(['-m', 'f', '--hdfs']) - args = dict(self.default_args) - args['hdfs'] = True - args['source'] = 'h' - args['dest'] = 'h' - self.check_args(args) - - def test_override_hdfs_m_m(self): - self.stage.parse_args(['-m', 'm', '--hdfs']) - args = dict(self.default_args) - args['hdfs'] = True - args['source'] = 'h' - args['dest'] = 'h' - self.check_args(args) - def test_override_hdfs_mode_s(self): self.stage.parse_args(['--mode', 's', '--hdfs']) args = dict(self.default_args) @@ -211,51 +187,35 @@ def f(self): setattr(ProcessorStageArgsTestCase, 'test_mode_%s' % (val), f) -def add_override_hdfs(arg, val, short=False): +def add_override_hdfs(arg, val): def f(self): - if short: - self.stage.parse_args(['--hdfs', '-' + arg[0], val]) - else: - self.stage.parse_args(['--hdfs', '--' + arg, val]) + self.stage.parse_args(['--hdfs', '--' + arg, val]) args = dict(self.default_args) args['hdfs'] = True args['source'] = 'h' args['dest'] = 'h' self.check_args(args) - if short: - setattr(ProcessorStageArgsTestCase, - 'test_override_hdfs_%s_%s' % (arg[0], val), f) - else: - setattr(ProcessorStageArgsTestCase, - 'test_override_hdfs_%s_%s' % (arg, val), f) + setattr(ProcessorStageArgsTestCase, + 'test_override_hdfs_%s_%s' % (arg, val), f) -def add_override_mode(arg, val, mode_val, short=False): +def add_override_mode(arg, val, mode_val): def f(self): - if short: - self.stage.parse_args(['-' + arg[0], val, '--mode', mode_val]) - else: - self.stage.parse_args(['--' + arg, val, '--mode', mode_val]) + self.stage.parse_args(['--' + arg, val, '--mode', mode_val]) args = dict(self.default_args) args.update(modes[mode_val]) args[arg] = val self.check_args(args) - if short: - setattr(ProcessorStageArgsTestCase, - 'test_override_%s_%s_mode_%s' % (arg[0], val, mode_val), f) - else: - setattr(ProcessorStageArgsTestCase, - 'test_override_%s_%s_mode_%s' % (arg, val, mode_val), f) + setattr(ProcessorStageArgsTestCase, + 'test_override_%s_%s_mode_%s' % (arg, val, mode_val), f) for a in args_to_add: for v in args_to_add[a]: add_arg(a, v, True) add_arg(a, v) - add_override_hdfs(a, v, True) add_override_hdfs(a, v) for m in modes: - add_override_mode(a, v, m, True) add_override_mode(a, v, m) From cf56b0c0606b69872876c70d0cea733bb1bdf9fe Mon Sep 17 00:00:00 2001 From: Evildoor Date: Tue, 20 Nov 2018 17:06:26 +0300 Subject: [PATCH 12/27] Moved override_hdfs_mode definition into a loop. Note: new version includes mode args check that was missing. --- .../stage/tests/test_ProcessorStage.py | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 9646430dc..8df5b3b18 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -125,31 +125,6 @@ def test_output_dir(self): args['output_dir'] = 'something' self.check_args(args) -# hdfs >> source-dest >> mode - def test_override_hdfs_mode_s(self): - self.stage.parse_args(['--mode', 's', '--hdfs']) - args = dict(self.default_args) - args['hdfs'] = True - args['source'] = 'h' - args['dest'] = 'h' - self.check_args(args) - - def test_override_hdfs_mode_f(self): - self.stage.parse_args(['--mode', 'f', '--hdfs']) - args = dict(self.default_args) - args['hdfs'] = True - args['source'] = 'h' - args['dest'] = 'h' - self.check_args(args) - - def test_override_hdfs_mode_m(self): - self.stage.parse_args(['--mode', 'm', '--hdfs']) - args = dict(self.default_args) - args['hdfs'] = True - args['source'] = 'h' - args['dest'] = 'h' - self.check_args(args) - def test_input_files(self): self.stage.parse_args(['something', 'something_else']) args = dict(self.default_args) @@ -187,6 +162,7 @@ def f(self): setattr(ProcessorStageArgsTestCase, 'test_mode_%s' % (val), f) +# hdfs >> source-dest >> mode def add_override_hdfs(arg, val): def f(self): self.stage.parse_args(['--hdfs', '--' + arg, val]) @@ -210,6 +186,19 @@ def f(self): 'test_override_%s_%s_mode_%s' % (arg, val, mode_val), f) +def add_override_hdfs_mode(val): + def f(self): + self.stage.parse_args(['--hdfs', '--mode', val]) + args = dict(self.default_args) + args.update(modes[val]) + args['hdfs'] = True + args['source'] = 'h' + args['dest'] = 'h' + self.check_args(args) + setattr(ProcessorStageArgsTestCase, + 'test_override_hdfs_mode_%s' % (val), f) + + for a in args_to_add: for v in args_to_add[a]: add_arg(a, v, True) @@ -222,6 +211,7 @@ def f(self): for m in modes: add_mode(m, True) add_mode(m) + add_override_hdfs_mode(m) test_cases = ( From 8a37c7da079b169afb8058962002da0c074d36b4 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Wed, 21 Nov 2018 14:25:01 +0300 Subject: [PATCH 13/27] Improved readability in testing hdfs overrides. In current version the line of code is pointless because arg is either 'source' or 'dest' which are redefined in the next few lines. However, the code reader may not know that. Furthermore, this line may be of use if it would be necessary to test some other arguments' interaction with hdfs. --- Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 8df5b3b18..c5674b17b 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -167,6 +167,7 @@ def add_override_hdfs(arg, val): def f(self): self.stage.parse_args(['--hdfs', '--' + arg, val]) args = dict(self.default_args) + args[arg] = val args['hdfs'] = True args['source'] = 'h' args['dest'] = 'h' From 324978c0aeccb9e3d1ff4c94f1f045e856043b89 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Fri, 23 Nov 2018 15:01:14 +0300 Subject: [PATCH 14/27] Moved repeating args definition into setUp(). tearDown() and check_args() were modified to accomodate the changes. --- .../stage/tests/test_ProcessorStage.py | 104 ++++++++---------- 1 file changed, 46 insertions(+), 58 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index c5674b17b..565998b77 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -55,81 +55,74 @@ class ProcessorStageArgsTestCase(unittest.TestCase): def setUp(self): self.stage = pyDKB.dataflow.stage.ProcessorStage() + self.args = dict(self.default_args) def tearDown(self): self.stage = None + self.args = None - def check_args(self, args): - for a in args: + def check_args(self): + for a in self.args: # Such kind of testing does not display the argument's name, # hence the {a: ...} addition. - self.assertEqual({a: getattr(self.stage.ARGS, a)}, {a: args[a]}) + self.assertEqual({a: getattr(self.stage.ARGS, a)}, + {a: self.args[a]}) def test_default(self): self.stage.parse_args('') - self.check_args(self.default_args) + self.check_args() def test_hdfs(self): self.stage.parse_args(['--hdfs']) - args = dict(self.default_args) - args['hdfs'] = True - args['source'] = 'h' - args['dest'] = 'h' - self.check_args(args) + self.args['hdfs'] = True + self.args['source'] = 'h' + self.args['dest'] = 'h' + self.check_args() def test_e(self): self.stage.parse_args(['-e', '\t']) - args = dict(self.default_args) - args['eom'] = '\t' - self.check_args(args) + self.args['eom'] = '\t' + self.check_args() def test_end_of_message(self): self.stage.parse_args(['--end-of-message', '\t']) - args = dict(self.default_args) - args['eom'] = '\t' - self.check_args(args) + self.args['eom'] = '\t' + self.check_args() def test_E(self): self.stage.parse_args(['-E', '\t']) - args = dict(self.default_args) - args['eop'] = '\t' - self.check_args(args) + self.args['eop'] = '\t' + self.check_args() def test_end_of_process(self): self.stage.parse_args(['--end-of-process', '\t']) - args = dict(self.default_args) - args['eop'] = '\t' - self.check_args(args) + self.args['eop'] = '\t' + self.check_args() def test_i(self): self.stage.parse_args(['-i', 'something']) - args = dict(self.default_args) - args['input_dir'] = 'something' - self.check_args(args) + self.args['input_dir'] = 'something' + self.check_args() def test_input_dir(self): self.stage.parse_args(['--input-dir', 'something']) - args = dict(self.default_args) - args['input_dir'] = 'something' - self.check_args(args) + self.args['input_dir'] = 'something' + self.check_args() def test_o(self): self.stage.parse_args(['-o', 'something']) - args = dict(self.default_args) - args['output_dir'] = 'something' - self.check_args(args) + self.args['output_dir'] = 'something' + self.check_args() def test_output_dir(self): self.stage.parse_args(['--output-dir', 'something']) - args = dict(self.default_args) - args['output_dir'] = 'something' - self.check_args(args) + self.args['output_dir'] = 'something' + self.check_args() def test_input_files(self): self.stage.parse_args(['something', 'something_else']) - args = dict(self.default_args) - args['input_files'] = ['something', 'something_else'] - self.check_args(args) + self.args['input_files'] = ['something', 'something_else'] + self.check_args() def add_arg(arg, val, short=False): @@ -138,9 +131,8 @@ def f(self): self.stage.parse_args(['-' + arg[0], val]) else: self.stage.parse_args(['--' + arg, val]) - args = dict(self.default_args) - args[arg] = val - self.check_args(args) + self.args[arg] = val + self.check_args() if short: setattr(ProcessorStageArgsTestCase, 'test_%s_%s' % (arg[0], val), f) else: @@ -153,9 +145,8 @@ def f(self): self.stage.parse_args(['-m', val]) else: self.stage.parse_args(['--mode', val]) - args = dict(self.default_args) - args.update(modes[val]) - self.check_args(args) + self.args.update(modes[val]) + self.check_args() if short: setattr(ProcessorStageArgsTestCase, 'test_m_%s' % (val), f) else: @@ -166,12 +157,11 @@ def f(self): def add_override_hdfs(arg, val): def f(self): self.stage.parse_args(['--hdfs', '--' + arg, val]) - args = dict(self.default_args) - args[arg] = val - args['hdfs'] = True - args['source'] = 'h' - args['dest'] = 'h' - self.check_args(args) + self.args[arg] = val + self.args['hdfs'] = True + self.args['source'] = 'h' + self.args['dest'] = 'h' + self.check_args() setattr(ProcessorStageArgsTestCase, 'test_override_hdfs_%s_%s' % (arg, val), f) @@ -179,10 +169,9 @@ def f(self): def add_override_mode(arg, val, mode_val): def f(self): self.stage.parse_args(['--' + arg, val, '--mode', mode_val]) - args = dict(self.default_args) - args.update(modes[mode_val]) - args[arg] = val - self.check_args(args) + self.args.update(modes[mode_val]) + self.args[arg] = val + self.check_args() setattr(ProcessorStageArgsTestCase, 'test_override_%s_%s_mode_%s' % (arg, val, mode_val), f) @@ -190,12 +179,11 @@ def f(self): def add_override_hdfs_mode(val): def f(self): self.stage.parse_args(['--hdfs', '--mode', val]) - args = dict(self.default_args) - args.update(modes[val]) - args['hdfs'] = True - args['source'] = 'h' - args['dest'] = 'h' - self.check_args(args) + self.args.update(modes[val]) + self.args['hdfs'] = True + self.args['source'] = 'h' + self.args['dest'] = 'h' + self.check_args() setattr(ProcessorStageArgsTestCase, 'test_override_hdfs_mode_%s' % (val), f) From fdb498a4c6b8dab638dfe0afa77aad7b466da51a Mon Sep 17 00:00:00 2001 From: Evildoor Date: Fri, 14 Dec 2018 15:23:04 +0300 Subject: [PATCH 15/27] Added tests for config argument. The argument differs from the others because its type is not string - therefore, argparse performs some manipulations with the supplied file, and these manipulations have to be taken into consideration. --- .../stage/tests/test_ProcessorStage.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 565998b77..468f3f9fb 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -9,6 +9,7 @@ import os import sys +import tempfile import unittest @@ -203,8 +204,49 @@ def f(self): add_override_hdfs_mode(m) +class ProcessorStageConfigArgTestCase(unittest.TestCase): + def setUp(self): + self.stage = pyDKB.dataflow.stage.ProcessorStage() + self.fake_config = tempfile.NamedTemporaryFile(dir='.') + + def tearDown(self): + self.stage = None + if not self.fake_config.closed: + self.fake_config.close() + self.fake_config = None + + def test_correct_c(self): + self.stage.parse_args(['-c', self.fake_config.name]) + self.assertIsNotNone(getattr(self.stage.ARGS, 'config')) + + def test_correct_config(self): + self.stage.parse_args(['--config', self.fake_config.name]) + self.assertIsNotNone(getattr(self.stage.ARGS, 'config')) + + def test_missing_c(self): + self.fake_config.close() + with self.assertRaises(SystemExit) as e: + self.stage.parse_args(['-c', self.fake_config.name]) + + def test_missing_config(self): + self.fake_config.close() + with self.assertRaises(SystemExit) as e: + self.stage.parse_args(['--config', self.fake_config.name]) + + def test_unreadable_c(self): + os.chmod(self.fake_config.name, 0300) + with self.assertRaises(SystemExit) as e: + self.stage.parse_args(['-c', self.fake_config.name]) + + def test_unreadable_config(self): + os.chmod(self.fake_config.name, 0300) + with self.assertRaises(SystemExit) as e: + self.stage.parse_args(['--config', self.fake_config.name]) + + test_cases = ( ProcessorStageArgsTestCase, + ProcessorStageConfigArgTestCase, ) From 42877e29abcb4b2c3396fd83d277e92dff0c8f68 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Mon, 17 Dec 2018 15:27:09 +0300 Subject: [PATCH 16/27] Improved config tests. Argparse module throws SystemExit when something is wrong with the supplied arguments, regardless of the problem's type. Implemented a function to intercept such behaviour and retrieve argparse' message. Analysis of the message is used to check that the encountered problem is the expected one. --- .../stage/tests/test_ProcessorStage.py | 60 ++++++++++++++++--- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 468f3f9fb..0f58d971b 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -7,6 +7,7 @@ """ +import cStringIO import os import sys import tempfile @@ -27,6 +28,35 @@ sys.exit(1) +def isolate_function_error(f, *args): + """ Silence and retrieve the function's error message. + + The function is expected to throw a SystemExit when run with + specific arguments. Error stream is redirected into a string during the + function's execution, and the resulting messages can be analyzed. + + :param f: function to execute + :type f: function + :param args: arguments to execute function with + :type args: list + + :return: + :rtype: + """ + buf = cStringIO.StringIO() + temp_err = sys.stderr + sys.stderr = buf + try: + result = f(*args) + except SystemExit: + result = None + sys.stderr = temp_err + buf.seek(0) + msg = buf.read() + buf.close() + return [msg, result] + + args_to_add = { 'source': ['f', 's', 'h'], 'dest': ['f', 's', 'h'], @@ -225,23 +255,37 @@ def test_correct_config(self): def test_missing_c(self): self.fake_config.close() - with self.assertRaises(SystemExit) as e: - self.stage.parse_args(['-c', self.fake_config.name]) + [msg, result] = isolate_function_error(self.stage.parse_args, + ['-c', self.fake_config.name]) + err = "[Errno 2] No such file or directory: '%s'" %\ + self.fake_config.name + self.assertIn(err, msg) def test_missing_config(self): self.fake_config.close() - with self.assertRaises(SystemExit) as e: - self.stage.parse_args(['--config', self.fake_config.name]) + [msg, result] = isolate_function_error(self.stage.parse_args, + ['--config', + self.fake_config.name]) + err = "[Errno 2] No such file or directory: '%s'" %\ + self.fake_config.name + self.assertIn(err, msg) def test_unreadable_c(self): os.chmod(self.fake_config.name, 0300) - with self.assertRaises(SystemExit) as e: - self.stage.parse_args(['-c', self.fake_config.name]) + [msg, result] = isolate_function_error(self.stage.parse_args, + ['-c', self.fake_config.name]) + err = "[Errno 13] Permission denied: '%s'" %\ + self.fake_config.name + self.assertIn(err, msg) def test_unreadable_config(self): os.chmod(self.fake_config.name, 0300) - with self.assertRaises(SystemExit) as e: - self.stage.parse_args(['--config', self.fake_config.name]) + [msg, result] = isolate_function_error(self.stage.parse_args, + ['--config', + self.fake_config.name]) + err = "[Errno 13] Permission denied: '%s'" %\ + self.fake_config.name + self.assertIn(err, msg) test_cases = ( From 9dc9aa0a6d7a6b0ebc0c6ba9434420484bc1bdb9 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Tue, 18 Dec 2018 16:00:40 +0300 Subject: [PATCH 17/27] Added 'incorrect' tests for mode, source and dest. These arguments have a fixed sets of allowed values. The tests are checking that the code correctly displays error information when an unexpected argument value is supplied. --- .../stage/tests/test_ProcessorStage.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 0f58d971b..64f598227 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -170,6 +170,24 @@ def f(self): setattr(ProcessorStageArgsTestCase, 'test_%s_%s' % (arg, val), f) +def add_arg_incorrect(arg, short=False): + if short: + val = 'i' + args = ['-' + arg[0], val] + fname = 'test_%s_%s' % (arg[0], val) + else: + val = 'incorrect' + args = ['--' + arg, val] + fname = 'test_%s_%s' % (arg, val) + + def f(self): + [msg, result] = isolate_function_error(self.stage.parse_args, args) + err = "error: argument -%s/--%s: invalid choice: '%s'" % (arg[0], + arg, val) + self.assertIn(err, msg) + setattr(ProcessorStageArgsTestCase, fname, f) + + def add_mode(val, short=False): def f(self): if short: @@ -226,6 +244,8 @@ def f(self): add_override_hdfs(a, v) for m in modes: add_override_mode(a, v, m) + add_arg_incorrect(a, True) + add_arg_incorrect(a) for m in modes: @@ -234,6 +254,10 @@ def f(self): add_override_hdfs_mode(m) +add_arg_incorrect('mode', True) +add_arg_incorrect('mode') + + class ProcessorStageConfigArgTestCase(unittest.TestCase): def setUp(self): self.stage = pyDKB.dataflow.stage.ProcessorStage() From 19ca40977f579865857ca08962d1e1727d00878b Mon Sep 17 00:00:00 2001 From: Evildoor Date: Thu, 20 Dec 2018 15:39:14 +0300 Subject: [PATCH 18/27] Improved the functions with 'short' argument. The functions now perform 'short'-related manipulations by defining varialbes and using them in functions instead of having two different calls of the same function. --- .../stage/tests/test_ProcessorStage.py | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 64f598227..529dd4d88 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -157,17 +157,18 @@ def test_input_files(self): def add_arg(arg, val, short=False): + if short: + args = ['-' + arg[0], val] + fname = 'test_%s_%s' % (arg[0], val) + else: + args = ['--' + arg, val] + fname = 'test_%s_%s' % (arg, val) + def f(self): - if short: - self.stage.parse_args(['-' + arg[0], val]) - else: - self.stage.parse_args(['--' + arg, val]) + self.stage.parse_args(args) self.args[arg] = val self.check_args() - if short: - setattr(ProcessorStageArgsTestCase, 'test_%s_%s' % (arg[0], val), f) - else: - setattr(ProcessorStageArgsTestCase, 'test_%s_%s' % (arg, val), f) + setattr(ProcessorStageArgsTestCase, fname, f) def add_arg_incorrect(arg, short=False): @@ -189,17 +190,18 @@ def f(self): def add_mode(val, short=False): + if short: + args = ['-m', val] + fname = 'test_m_%s' % (val) + else: + args = ['--mode', val] + fname = 'test_mode_%s' % (val) + def f(self): - if short: - self.stage.parse_args(['-m', val]) - else: - self.stage.parse_args(['--mode', val]) + self.stage.parse_args(args) self.args.update(modes[val]) self.check_args() - if short: - setattr(ProcessorStageArgsTestCase, 'test_m_%s' % (val), f) - else: - setattr(ProcessorStageArgsTestCase, 'test_mode_%s' % (val), f) + setattr(ProcessorStageArgsTestCase, fname, f) # hdfs >> source-dest >> mode From c3e481030ec0736b58642ecfdd1f28b8237ec8f8 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Wed, 23 Jan 2019 15:13:31 +0300 Subject: [PATCH 19/27] Updated tests for correct config. Now the tests check that the parameter in question is a file, rather than not None. --- .../pyDKB/dataflow/stage/tests/test_ProcessorStage.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 529dd4d88..76441dabc 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -273,11 +273,13 @@ def tearDown(self): def test_correct_c(self): self.stage.parse_args(['-c', self.fake_config.name]) - self.assertIsNotNone(getattr(self.stage.ARGS, 'config')) + isfile = isinstance(getattr(self.stage.ARGS, 'config'), file) + self.assertTrue(isfile) def test_correct_config(self): self.stage.parse_args(['--config', self.fake_config.name]) - self.assertIsNotNone(getattr(self.stage.ARGS, 'config')) + isfile = isinstance(getattr(self.stage.ARGS, 'config'), file) + self.assertTrue(isfile) def test_missing_c(self): self.fake_config.close() From f795e2411170a40c8f4af6182ea15c26df934899 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Fri, 26 Apr 2019 15:50:08 +0200 Subject: [PATCH 20/27] Adapt tests to pyDKB changes. - Replace stage.parse_args() with stage.configure(). - Update tests to set input_dir if source is either stream or hdfs. - Add an input file name into most tests - 'file' is now a default source, and it demands an input file. Exceptions: test_i/test_input_dir - all files in the input dir will be taken, so input file is not mandatory; all tests including --hdfs; test_default - its role is to simulate an execution with no command line arguments. - Change test_default() to catch the error about missing input file. - Change add_override_mode() - file source/destination is not allowed in map-reduce mode, meaning that such combination raises an error that must be caught. - Define stage.configure() arguments in list args in some tests to follow the 80 symbols line length restriction without stretching a single code line into 3+ lines. --- .../stage/tests/test_ProcessorStage.py | 102 +++++++++++------- 1 file changed, 65 insertions(+), 37 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 76441dabc..65df4d9c5 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -100,73 +100,85 @@ def check_args(self): {a: self.args[a]}) def test_default(self): - self.stage.parse_args('') - self.check_args() + [msg, result] = isolate_function_error(self.stage.configure, []) + self.assertIn('No input files specified.', msg) def test_hdfs(self): - self.stage.parse_args(['--hdfs']) + self.stage.configure(['--hdfs']) self.args['hdfs'] = True self.args['source'] = 'h' self.args['dest'] = 'h' + self.args['input_dir'] = '/user/DKB/' self.check_args() def test_e(self): - self.stage.parse_args(['-e', '\t']) + self.stage.configure(['-e', '\t', 'something']) self.args['eom'] = '\t' + self.args['input_files'] = ['something'] self.check_args() def test_end_of_message(self): - self.stage.parse_args(['--end-of-message', '\t']) + self.stage.configure(['--end-of-message', '\t', 'something']) self.args['eom'] = '\t' + self.args['input_files'] = ['something'] self.check_args() def test_E(self): - self.stage.parse_args(['-E', '\t']) + self.stage.configure(['-E', '\t', 'something']) self.args['eop'] = '\t' + self.args['input_files'] = ['something'] self.check_args() def test_end_of_process(self): - self.stage.parse_args(['--end-of-process', '\t']) + self.stage.configure(['--end-of-process', '\t', 'something']) self.args['eop'] = '\t' + self.args['input_files'] = ['something'] self.check_args() def test_i(self): - self.stage.parse_args(['-i', 'something']) + self.stage.configure(['-i', 'something']) self.args['input_dir'] = 'something' self.check_args() def test_input_dir(self): - self.stage.parse_args(['--input-dir', 'something']) + self.stage.configure(['--input-dir', 'something']) self.args['input_dir'] = 'something' self.check_args() def test_o(self): - self.stage.parse_args(['-o', 'something']) + self.stage.configure(['-o', 'something', 'something']) self.args['output_dir'] = 'something' + self.args['input_files'] = ['something'] self.check_args() def test_output_dir(self): - self.stage.parse_args(['--output-dir', 'something']) + self.stage.configure(['--output-dir', 'something', 'something']) self.args['output_dir'] = 'something' + self.args['input_files'] = ['something'] self.check_args() def test_input_files(self): - self.stage.parse_args(['something', 'something_else']) + self.stage.configure(['something', 'something_else']) self.args['input_files'] = ['something', 'something_else'] self.check_args() def add_arg(arg, val, short=False): if short: - args = ['-' + arg[0], val] + args = ['-' + arg[0], val, 'something'] fname = 'test_%s_%s' % (arg[0], val) else: - args = ['--' + arg, val] + args = ['--' + arg, val, 'something'] fname = 'test_%s_%s' % (arg, val) def f(self): - self.stage.parse_args(args) + self.stage.configure(args) self.args[arg] = val + self.args['input_files'] = ['something'] + if self.args['source'] == 's': + self.args['input_dir'] = None + elif self.args['source'] == 'h': + self.args['input_dir'] = '/user/DKB/' self.check_args() setattr(ProcessorStageArgsTestCase, fname, f) @@ -191,15 +203,18 @@ def f(self): def add_mode(val, short=False): if short: - args = ['-m', val] + args = ['-m', val, 'something'] fname = 'test_m_%s' % (val) else: - args = ['--mode', val] + args = ['--mode', val, 'something'] fname = 'test_mode_%s' % (val) def f(self): - self.stage.parse_args(args) + self.stage.configure(args) self.args.update(modes[val]) + self.args['input_files'] = ['something'] + if val != 'f': + self.args['input_dir'] = None self.check_args() setattr(ProcessorStageArgsTestCase, fname, f) @@ -207,33 +222,48 @@ def f(self): # hdfs >> source-dest >> mode def add_override_hdfs(arg, val): def f(self): - self.stage.parse_args(['--hdfs', '--' + arg, val]) + self.stage.configure(['--hdfs', '--' + arg, val]) self.args[arg] = val self.args['hdfs'] = True self.args['source'] = 'h' self.args['dest'] = 'h' + self.args['input_dir'] = '/user/DKB/' self.check_args() setattr(ProcessorStageArgsTestCase, 'test_override_hdfs_%s_%s' % (arg, val), f) def add_override_mode(arg, val, mode_val): - def f(self): - self.stage.parse_args(['--' + arg, val, '--mode', mode_val]) - self.args.update(modes[mode_val]) - self.args[arg] = val - self.check_args() + if mode_val == 'm' and val == 'f': + def f(self): + args = ['--' + arg, val, '--mode', mode_val, 'something'] + [msg, result] = isolate_function_error(self.stage.configure, args) + err = "File source/destination is not allowed in map-reduce mode." + self.assertIn(err, msg) + else: + def f(self): + self.stage.configure(['--' + arg, val, '--mode', mode_val, + 'something']) + self.args.update(modes[mode_val]) + self.args[arg] = val + if self.args['source'] == 's': + self.args['input_dir'] = None + elif self.args['source'] == 'h': + self.args['input_dir'] = '/user/DKB/' + self.args['input_files'] = ['something'] + self.check_args() setattr(ProcessorStageArgsTestCase, 'test_override_%s_%s_mode_%s' % (arg, val, mode_val), f) def add_override_hdfs_mode(val): def f(self): - self.stage.parse_args(['--hdfs', '--mode', val]) + self.stage.configure(['--hdfs', '--mode', val]) self.args.update(modes[val]) self.args['hdfs'] = True self.args['source'] = 'h' self.args['dest'] = 'h' + self.args['input_dir'] = '/user/DKB/' self.check_args() setattr(ProcessorStageArgsTestCase, 'test_override_hdfs_mode_%s' % (val), f) @@ -272,45 +302,43 @@ def tearDown(self): self.fake_config = None def test_correct_c(self): - self.stage.parse_args(['-c', self.fake_config.name]) + self.stage.configure(['-c', self.fake_config.name, 'something']) isfile = isinstance(getattr(self.stage.ARGS, 'config'), file) self.assertTrue(isfile) def test_correct_config(self): - self.stage.parse_args(['--config', self.fake_config.name]) + self.stage.configure(['--config', self.fake_config.name, 'something']) isfile = isinstance(getattr(self.stage.ARGS, 'config'), file) self.assertTrue(isfile) def test_missing_c(self): self.fake_config.close() - [msg, result] = isolate_function_error(self.stage.parse_args, - ['-c', self.fake_config.name]) + args = ['-c', self.fake_config.name, 'something'] + [msg, result] = isolate_function_error(self.stage.configure, args) err = "[Errno 2] No such file or directory: '%s'" %\ self.fake_config.name self.assertIn(err, msg) def test_missing_config(self): self.fake_config.close() - [msg, result] = isolate_function_error(self.stage.parse_args, - ['--config', - self.fake_config.name]) + args = ['--config', self.fake_config.name, 'something'] + [msg, result] = isolate_function_error(self.stage.configure, args) err = "[Errno 2] No such file or directory: '%s'" %\ self.fake_config.name self.assertIn(err, msg) def test_unreadable_c(self): os.chmod(self.fake_config.name, 0300) - [msg, result] = isolate_function_error(self.stage.parse_args, - ['-c', self.fake_config.name]) + args = ['-c', self.fake_config.name, 'something'] + [msg, result] = isolate_function_error(self.stage.configure, args) err = "[Errno 13] Permission denied: '%s'" %\ self.fake_config.name self.assertIn(err, msg) def test_unreadable_config(self): os.chmod(self.fake_config.name, 0300) - [msg, result] = isolate_function_error(self.stage.parse_args, - ['--config', - self.fake_config.name]) + args = ['--config', self.fake_config.name, 'something'] + [msg, result] = isolate_function_error(self.stage.configure, args) err = "[Errno 13] Permission denied: '%s'" %\ self.fake_config.name self.assertIn(err, msg) From 77ef71b2b943661a275ed0576f63dd50a708e5bf Mon Sep 17 00:00:00 2001 From: Evildoor Date: Mon, 29 Apr 2019 12:21:04 +0200 Subject: [PATCH 21/27] Add empty EOM tests. If EOM is set to empty string, EOP will be set to '\n', unless overridden. --- .../dataflow/stage/tests/test_ProcessorStage.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 65df4d9c5..5e14e6577 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -135,6 +135,20 @@ def test_end_of_process(self): self.args['input_files'] = ['something'] self.check_args() + def test_e_empty(self): + self.stage.configure(['-e', '', 'something']) + self.args['eom'] = '' + self.args['eop'] = '\n' + self.args['input_files'] = ['something'] + self.check_args() + + def test_e_empty_E_override(self): + self.stage.configure(['-e', '', '-E', '', 'something']) + self.args['eom'] = '' + self.args['eop'] = '' + self.args['input_files'] = ['something'] + self.check_args() + def test_i(self): self.stage.configure(['-i', 'something']) self.args['input_dir'] = 'something' From 1d4feb2f255150497c9ae3f061b5c2f93e0acc8e Mon Sep 17 00:00:00 2001 From: Evildoor Date: Mon, 29 Apr 2019 12:48:03 +0200 Subject: [PATCH 22/27] Change value used to trigger an error. It is possible that value 'i' will become valid in the future, while 'incorrect' is much 'safer' in this regard - all of the values are single letters. --- .../Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 5e14e6577..83ac096d1 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -199,7 +199,7 @@ def f(self): def add_arg_incorrect(arg, short=False): if short: - val = 'i' + val = 'incorrect' args = ['-' + arg[0], val] fname = 'test_%s_%s' % (arg[0], val) else: From b5d684283e519fdbd2572f0e3eaa0e0e21314fcd Mon Sep 17 00:00:00 2001 From: Evildoor Date: Mon, 29 Apr 2019 13:09:26 +0200 Subject: [PATCH 23/27] Rename a variable to improve readability. --- .../pyDKB/dataflow/stage/tests/test_ProcessorStage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 83ac096d1..1b456c89a 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -71,7 +71,7 @@ def isolate_function_error(f, *args): class ProcessorStageArgsTestCase(unittest.TestCase): - default_args = { + expected_args = { 'mode': 'f', 'config': None, 'eom': '\n', @@ -86,7 +86,7 @@ class ProcessorStageArgsTestCase(unittest.TestCase): def setUp(self): self.stage = pyDKB.dataflow.stage.ProcessorStage() - self.args = dict(self.default_args) + self.args = dict(self.expected_args) def tearDown(self): self.stage = None From 38a383ec5903d98cbdd6c665598de8c781770d89 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Mon, 29 Apr 2019 13:27:43 +0200 Subject: [PATCH 24/27] Add raw strings processing test. --- .../pyDKB/dataflow/stage/tests/test_ProcessorStage.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 1b456c89a..c57ff01af 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -149,6 +149,13 @@ def test_e_empty_E_override(self): self.args['input_files'] = ['something'] self.check_args() + def test_raw_strings(self): + self.stage.configure(['-e', r'\t', '-E', r'\n', 'something']) + self.args['eom'] = '\t' + self.args['eop'] = '\n' + self.args['input_files'] = ['something'] + self.check_args() + def test_i(self): self.stage.configure(['-i', 'something']) self.args['input_dir'] = 'something' From c99f4c605124bd8814d9b05a95b2b990018cccea Mon Sep 17 00:00:00 2001 From: Evildoor Date: Mon, 29 Apr 2019 13:32:21 +0200 Subject: [PATCH 25/27] Replace repeating assignments with dict.update(). --- .../stage/tests/test_ProcessorStage.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index c57ff01af..9bee58777 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -63,6 +63,13 @@ def isolate_function_error(f, *args): } +hdfs_args = { + 'hdfs': True, + 'source': 'h', + 'dest': 'h', +} + + modes = { 's': {'mode': 's', 'source': 's', 'dest': 's', 'eom': '\n', 'eop': '\0'}, 'f': {'mode': 'f', 'source': 'f', 'dest': 'f', 'eom': '\n', 'eop': ''}, @@ -105,9 +112,7 @@ def test_default(self): def test_hdfs(self): self.stage.configure(['--hdfs']) - self.args['hdfs'] = True - self.args['source'] = 'h' - self.args['dest'] = 'h' + self.args.update(hdfs_args) self.args['input_dir'] = '/user/DKB/' self.check_args() @@ -245,9 +250,7 @@ def add_override_hdfs(arg, val): def f(self): self.stage.configure(['--hdfs', '--' + arg, val]) self.args[arg] = val - self.args['hdfs'] = True - self.args['source'] = 'h' - self.args['dest'] = 'h' + self.args.update(hdfs_args) self.args['input_dir'] = '/user/DKB/' self.check_args() setattr(ProcessorStageArgsTestCase, @@ -281,9 +284,7 @@ def add_override_hdfs_mode(val): def f(self): self.stage.configure(['--hdfs', '--mode', val]) self.args.update(modes[val]) - self.args['hdfs'] = True - self.args['source'] = 'h' - self.args['dest'] = 'h' + self.args.update(hdfs_args) self.args['input_dir'] = '/user/DKB/' self.check_args() setattr(ProcessorStageArgsTestCase, From e1df893bc0e893eee6a626da385df7b4bcb25790 Mon Sep 17 00:00:00 2001 From: Evildoor Date: Mon, 29 Apr 2019 16:50:51 +0200 Subject: [PATCH 26/27] Add missing return value description. --- .../pyDKB/dataflow/stage/tests/test_ProcessorStage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 9bee58777..9e33c971f 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -40,8 +40,9 @@ def isolate_function_error(f, *args): :param args: arguments to execute function with :type args: list - :return: - :rtype: + :return: list with two members, first one is the error message, + second one is the function's return + :rtype: list """ buf = cStringIO.StringIO() temp_err = sys.stderr From 7dad261296de13f085e759f371a4dab808ff204d Mon Sep 17 00:00:00 2001 From: Evildoor Date: Tue, 30 Apr 2019 11:55:48 +0200 Subject: [PATCH 27/27] Divide test_ProcessorStage.py per how-to: tests. - Move common function to separate file lib.py. --- .../pyDKB/dataflow/stage/tests/lib.py | 36 ++++++ .../stage/tests/test_ProcessorStage.py | 116 ++---------------- .../stage/tests/test_ProcessorStage_config.py | 87 +++++++++++++ 3 files changed, 133 insertions(+), 106 deletions(-) create mode 100644 Utils/Dataflow/pyDKB/dataflow/stage/tests/lib.py create mode 100644 Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage_config.py diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/lib.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/lib.py new file mode 100644 index 000000000..732c1cf94 --- /dev/null +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/lib.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +""" Common functions for testing ProcessorStage. """ + +import cStringIO +import sys + + +def isolate_function_error(f, *args): + """ Silence and retrieve the function's error message. + + The function is expected to throw a SystemExit when run with + specific arguments. Error stream is redirected into a string during the + function's execution, and the resulting messages can be analyzed. + + :param f: function to execute + :type f: function + :param args: arguments to execute function with + :type args: list + + :return: list with two members, first one is the error message, + second one is the function's return + :rtype: list + """ + buf = cStringIO.StringIO() + temp_err = sys.stderr + sys.stderr = buf + try: + result = f(*args) + except SystemExit: + result = None + sys.stderr = temp_err + buf.seek(0) + msg = buf.read() + buf.close() + return [msg, result] diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py index 9e33c971f..1c8dc731a 100644 --- a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py @@ -1,18 +1,16 @@ #!/usr/bin/env python """ -Tests for pyDKB.dataflow.stage.ProcessorStage. +Tests for pyDKB.dataflow.stage.ProcessorStage.configure(). Usage: 'python -m unittest discover' from .. (directory with pyDKB.dataflow.stage code). """ - -import cStringIO import os import sys -import tempfile import unittest +from lib import isolate_function_error # Relative import inside of pyDKB prevents the use of simple 'import pyDKB'. try: @@ -28,36 +26,6 @@ sys.exit(1) -def isolate_function_error(f, *args): - """ Silence and retrieve the function's error message. - - The function is expected to throw a SystemExit when run with - specific arguments. Error stream is redirected into a string during the - function's execution, and the resulting messages can be analyzed. - - :param f: function to execute - :type f: function - :param args: arguments to execute function with - :type args: list - - :return: list with two members, first one is the error message, - second one is the function's return - :rtype: list - """ - buf = cStringIO.StringIO() - temp_err = sys.stderr - sys.stderr = buf - try: - result = f(*args) - except SystemExit: - result = None - sys.stderr = temp_err - buf.seek(0) - msg = buf.read() - buf.close() - return [msg, result] - - args_to_add = { 'source': ['f', 's', 'h'], 'dest': ['f', 's', 'h'], @@ -78,7 +46,7 @@ def isolate_function_error(f, *args): } -class ProcessorStageArgsTestCase(unittest.TestCase): +class Case(unittest.TestCase): expected_args = { 'mode': 'f', 'config': None, @@ -207,7 +175,7 @@ def f(self): elif self.args['source'] == 'h': self.args['input_dir'] = '/user/DKB/' self.check_args() - setattr(ProcessorStageArgsTestCase, fname, f) + setattr(Case, fname, f) def add_arg_incorrect(arg, short=False): @@ -225,7 +193,7 @@ def f(self): err = "error: argument -%s/--%s: invalid choice: '%s'" % (arg[0], arg, val) self.assertIn(err, msg) - setattr(ProcessorStageArgsTestCase, fname, f) + setattr(Case, fname, f) def add_mode(val, short=False): @@ -243,7 +211,7 @@ def f(self): if val != 'f': self.args['input_dir'] = None self.check_args() - setattr(ProcessorStageArgsTestCase, fname, f) + setattr(Case, fname, f) # hdfs >> source-dest >> mode @@ -254,8 +222,7 @@ def f(self): self.args.update(hdfs_args) self.args['input_dir'] = '/user/DKB/' self.check_args() - setattr(ProcessorStageArgsTestCase, - 'test_override_hdfs_%s_%s' % (arg, val), f) + setattr(Case, 'test_override_hdfs_%s_%s' % (arg, val), f) def add_override_mode(arg, val, mode_val): @@ -277,8 +244,7 @@ def f(self): self.args['input_dir'] = '/user/DKB/' self.args['input_files'] = ['something'] self.check_args() - setattr(ProcessorStageArgsTestCase, - 'test_override_%s_%s_mode_%s' % (arg, val, mode_val), f) + setattr(Case, 'test_override_%s_%s_mode_%s' % (arg, val, mode_val), f) def add_override_hdfs_mode(val): @@ -288,8 +254,7 @@ def f(self): self.args.update(hdfs_args) self.args['input_dir'] = '/user/DKB/' self.check_args() - setattr(ProcessorStageArgsTestCase, - 'test_override_hdfs_mode_%s' % (val), f) + setattr(Case, 'test_override_hdfs_mode_%s' % (val), f) for a in args_to_add: @@ -313,68 +278,7 @@ def f(self): add_arg_incorrect('mode') -class ProcessorStageConfigArgTestCase(unittest.TestCase): - def setUp(self): - self.stage = pyDKB.dataflow.stage.ProcessorStage() - self.fake_config = tempfile.NamedTemporaryFile(dir='.') - - def tearDown(self): - self.stage = None - if not self.fake_config.closed: - self.fake_config.close() - self.fake_config = None - - def test_correct_c(self): - self.stage.configure(['-c', self.fake_config.name, 'something']) - isfile = isinstance(getattr(self.stage.ARGS, 'config'), file) - self.assertTrue(isfile) - - def test_correct_config(self): - self.stage.configure(['--config', self.fake_config.name, 'something']) - isfile = isinstance(getattr(self.stage.ARGS, 'config'), file) - self.assertTrue(isfile) - - def test_missing_c(self): - self.fake_config.close() - args = ['-c', self.fake_config.name, 'something'] - [msg, result] = isolate_function_error(self.stage.configure, args) - err = "[Errno 2] No such file or directory: '%s'" %\ - self.fake_config.name - self.assertIn(err, msg) - - def test_missing_config(self): - self.fake_config.close() - args = ['--config', self.fake_config.name, 'something'] - [msg, result] = isolate_function_error(self.stage.configure, args) - err = "[Errno 2] No such file or directory: '%s'" %\ - self.fake_config.name - self.assertIn(err, msg) - - def test_unreadable_c(self): - os.chmod(self.fake_config.name, 0300) - args = ['-c', self.fake_config.name, 'something'] - [msg, result] = isolate_function_error(self.stage.configure, args) - err = "[Errno 13] Permission denied: '%s'" %\ - self.fake_config.name - self.assertIn(err, msg) - - def test_unreadable_config(self): - os.chmod(self.fake_config.name, 0300) - args = ['--config', self.fake_config.name, 'something'] - [msg, result] = isolate_function_error(self.stage.configure, args) - err = "[Errno 13] Permission denied: '%s'" %\ - self.fake_config.name - self.assertIn(err, msg) - - -test_cases = ( - ProcessorStageArgsTestCase, - ProcessorStageConfigArgTestCase, -) - - def load_tests(loader, tests, pattern): suite = unittest.TestSuite() - for case in test_cases: - suite.addTest(loader.loadTestsFromTestCase(case)) + suite.addTest(loader.loadTestsFromTestCase(Case)) return suite diff --git a/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage_config.py b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage_config.py new file mode 100644 index 000000000..11fa218ff --- /dev/null +++ b/Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage_config.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +""" +Tests for pyDKB.dataflow.stage.ProcessorStage.configure()'s config argument. +Usage: 'python -m unittest discover' from .. +(directory with pyDKB.dataflow.stage code). +""" + +import os +import sys +import tempfile +import unittest + +from lib import isolate_function_error + +# Relative import inside of pyDKB prevents the use of simple 'import pyDKB'. +try: + base_dir = os.path.dirname(__file__) # Directory with this file + dkb_dir = os.path.join(base_dir, os.pardir) # stage directory + dkb_dir = os.path.join(dkb_dir, os.pardir) # dataflow directory + dkb_dir = os.path.join(dkb_dir, os.pardir) # pyDKB's directory + dkb_dir = os.path.join(dkb_dir, os.pardir) # pyDKB's parent directory + sys.path.append(dkb_dir) + import pyDKB +except Exception, err: + sys.stderr.write("(ERROR) Failed to import pyDKB library: %s\n" % err) + sys.exit(1) + + +class Case(unittest.TestCase): + def setUp(self): + self.stage = pyDKB.dataflow.stage.ProcessorStage() + self.fake_config = tempfile.NamedTemporaryFile(dir='.') + + def tearDown(self): + self.stage = None + if not self.fake_config.closed: + self.fake_config.close() + self.fake_config = None + + def test_correct_c(self): + self.stage.configure(['-c', self.fake_config.name, 'something']) + isfile = isinstance(getattr(self.stage.ARGS, 'config'), file) + self.assertTrue(isfile) + + def test_correct_config(self): + self.stage.configure(['--config', self.fake_config.name, 'something']) + isfile = isinstance(getattr(self.stage.ARGS, 'config'), file) + self.assertTrue(isfile) + + def test_missing_c(self): + self.fake_config.close() + args = ['-c', self.fake_config.name, 'something'] + [msg, result] = isolate_function_error(self.stage.configure, args) + err = "[Errno 2] No such file or directory: '%s'" %\ + self.fake_config.name + self.assertIn(err, msg) + + def test_missing_config(self): + self.fake_config.close() + args = ['--config', self.fake_config.name, 'something'] + [msg, result] = isolate_function_error(self.stage.configure, args) + err = "[Errno 2] No such file or directory: '%s'" %\ + self.fake_config.name + self.assertIn(err, msg) + + def test_unreadable_c(self): + os.chmod(self.fake_config.name, 0300) + args = ['-c', self.fake_config.name, 'something'] + [msg, result] = isolate_function_error(self.stage.configure, args) + err = "[Errno 13] Permission denied: '%s'" %\ + self.fake_config.name + self.assertIn(err, msg) + + def test_unreadable_config(self): + os.chmod(self.fake_config.name, 0300) + args = ['--config', self.fake_config.name, 'something'] + [msg, result] = isolate_function_error(self.stage.configure, args) + err = "[Errno 13] Permission denied: '%s'" %\ + self.fake_config.name + self.assertIn(err, msg) + + +def load_tests(loader, tests, pattern): + suite = unittest.TestSuite() + suite.addTest(loader.loadTestsFromTestCase(Case)) + return suite