Skip to content

Commit

Permalink
Divide test_ProcessorStage.py per how-to: tests.
Browse files Browse the repository at this point in the history
- Move common function to separate file lib.py.
  • Loading branch information
Evildoor committed Apr 30, 2019
1 parent e1df893 commit 6f81f0b
Show file tree
Hide file tree
Showing 3 changed files with 134 additions and 106 deletions.
37 changes: 37 additions & 0 deletions Utils/Dataflow/pyDKB/dataflow/stage/tests/lib.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/usr/bin/env python

""" Common functions for testing ProcessorStage. """

import cStringIO
import sys


def isolate_function_error(f, *args):
""" Silence and retrieve the function's error message.
The function is expected to throw a SystemExit when run with
specific arguments. Error stream is redirected into a string during the
function's execution, and the resulting messages can be analyzed.
:param f: function to execute
:type f: function
:param args: arguments to execute function with
:type args: list
:return: list with two members, first one is the error message,
second one is the function's return
:rtype: list
"""
buf = cStringIO.StringIO()
temp_err = sys.stderr
sys.stderr = buf
try:
result = f(*args)
except SystemExit:
result = None
sys.stderr = temp_err
buf.seek(0)
msg = buf.read()
buf.close()
return [msg, result]

116 changes: 10 additions & 106 deletions Utils/Dataflow/pyDKB/dataflow/stage/tests/test_ProcessorStage.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
#!/usr/bin/env python

"""
Tests for pyDKB.dataflow.stage.ProcessorStage.
Tests for pyDKB.dataflow.stage.ProcessorStage.configure().
Usage: 'python -m unittest discover' from ..
(directory with pyDKB.dataflow.stage code).
"""


import cStringIO
import os
import sys
import tempfile
import unittest

from lib import isolate_function_error

# Relative import inside of pyDKB prevents the use of simple 'import pyDKB'.
try:
Expand All @@ -28,36 +26,6 @@
sys.exit(1)


def isolate_function_error(f, *args):
""" Silence and retrieve the function's error message.
The function is expected to throw a SystemExit when run with
specific arguments. Error stream is redirected into a string during the
function's execution, and the resulting messages can be analyzed.
:param f: function to execute
:type f: function
:param args: arguments to execute function with
:type args: list
:return: list with two members, first one is the error message,
second one is the function's return
:rtype: list
"""
buf = cStringIO.StringIO()
temp_err = sys.stderr
sys.stderr = buf
try:
result = f(*args)
except SystemExit:
result = None
sys.stderr = temp_err
buf.seek(0)
msg = buf.read()
buf.close()
return [msg, result]


args_to_add = {
'source': ['f', 's', 'h'],
'dest': ['f', 's', 'h'],
Expand All @@ -78,7 +46,7 @@ def isolate_function_error(f, *args):
}


class ProcessorStageArgsTestCase(unittest.TestCase):
class Case(unittest.TestCase):
expected_args = {
'mode': 'f',
'config': None,
Expand Down Expand Up @@ -207,7 +175,7 @@ def f(self):
elif self.args['source'] == 'h':
self.args['input_dir'] = '/user/DKB/'
self.check_args()
setattr(ProcessorStageArgsTestCase, fname, f)
setattr(Case, fname, f)


def add_arg_incorrect(arg, short=False):
Expand All @@ -225,7 +193,7 @@ def f(self):
err = "error: argument -%s/--%s: invalid choice: '%s'" % (arg[0],
arg, val)
self.assertIn(err, msg)
setattr(ProcessorStageArgsTestCase, fname, f)
setattr(Case, fname, f)


def add_mode(val, short=False):
Expand All @@ -243,7 +211,7 @@ def f(self):
if val != 'f':
self.args['input_dir'] = None
self.check_args()
setattr(ProcessorStageArgsTestCase, fname, f)
setattr(Case, fname, f)


# hdfs >> source-dest >> mode
Expand All @@ -254,8 +222,7 @@ def f(self):
self.args.update(hdfs_args)
self.args['input_dir'] = '/user/DKB/'
self.check_args()
setattr(ProcessorStageArgsTestCase,
'test_override_hdfs_%s_%s' % (arg, val), f)
setattr(Case, 'test_override_hdfs_%s_%s' % (arg, val), f)


def add_override_mode(arg, val, mode_val):
Expand All @@ -277,8 +244,7 @@ def f(self):
self.args['input_dir'] = '/user/DKB/'
self.args['input_files'] = ['something']
self.check_args()
setattr(ProcessorStageArgsTestCase,
'test_override_%s_%s_mode_%s' % (arg, val, mode_val), f)
setattr(Case, 'test_override_%s_%s_mode_%s' % (arg, val, mode_val), f)


def add_override_hdfs_mode(val):
Expand All @@ -288,8 +254,7 @@ def f(self):
self.args.update(hdfs_args)
self.args['input_dir'] = '/user/DKB/'
self.check_args()
setattr(ProcessorStageArgsTestCase,
'test_override_hdfs_mode_%s' % (val), f)
setattr(Case, 'test_override_hdfs_mode_%s' % (val), f)


for a in args_to_add:
Expand All @@ -313,68 +278,7 @@ def f(self):
add_arg_incorrect('mode')


class ProcessorStageConfigArgTestCase(unittest.TestCase):
def setUp(self):
self.stage = pyDKB.dataflow.stage.ProcessorStage()
self.fake_config = tempfile.NamedTemporaryFile(dir='.')

def tearDown(self):
self.stage = None
if not self.fake_config.closed:
self.fake_config.close()
self.fake_config = None

def test_correct_c(self):
self.stage.configure(['-c', self.fake_config.name, 'something'])
isfile = isinstance(getattr(self.stage.ARGS, 'config'), file)
self.assertTrue(isfile)

def test_correct_config(self):
self.stage.configure(['--config', self.fake_config.name, 'something'])
isfile = isinstance(getattr(self.stage.ARGS, 'config'), file)
self.assertTrue(isfile)

def test_missing_c(self):
self.fake_config.close()
args = ['-c', self.fake_config.name, 'something']
[msg, result] = isolate_function_error(self.stage.configure, args)
err = "[Errno 2] No such file or directory: '%s'" %\
self.fake_config.name
self.assertIn(err, msg)

def test_missing_config(self):
self.fake_config.close()
args = ['--config', self.fake_config.name, 'something']
[msg, result] = isolate_function_error(self.stage.configure, args)
err = "[Errno 2] No such file or directory: '%s'" %\
self.fake_config.name
self.assertIn(err, msg)

def test_unreadable_c(self):
os.chmod(self.fake_config.name, 0300)
args = ['-c', self.fake_config.name, 'something']
[msg, result] = isolate_function_error(self.stage.configure, args)
err = "[Errno 13] Permission denied: '%s'" %\
self.fake_config.name
self.assertIn(err, msg)

def test_unreadable_config(self):
os.chmod(self.fake_config.name, 0300)
args = ['--config', self.fake_config.name, 'something']
[msg, result] = isolate_function_error(self.stage.configure, args)
err = "[Errno 13] Permission denied: '%s'" %\
self.fake_config.name
self.assertIn(err, msg)


test_cases = (
ProcessorStageArgsTestCase,
ProcessorStageConfigArgTestCase,
)


def load_tests(loader, tests, pattern):
suite = unittest.TestSuite()
for case in test_cases:
suite.addTest(loader.loadTestsFromTestCase(case))
suite.addTest(loader.loadTestsFromTestCase(Case))
return suite
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
#!/usr/bin/env python

"""
Tests for pyDKB.dataflow.stage.ProcessorStage.configure()'s config argument.
Usage: 'python -m unittest discover' from ..
(directory with pyDKB.dataflow.stage code).
"""

import os
import sys
import tempfile
import unittest

from lib import isolate_function_error

# Relative import inside of pyDKB prevents the use of simple 'import pyDKB'.
try:
base_dir = os.path.dirname(__file__) # Directory with this file
dkb_dir = os.path.join(base_dir, os.pardir) # stage directory
dkb_dir = os.path.join(dkb_dir, os.pardir) # dataflow directory
dkb_dir = os.path.join(dkb_dir, os.pardir) # pyDKB's directory
dkb_dir = os.path.join(dkb_dir, os.pardir) # pyDKB's parent directory
sys.path.append(dkb_dir)
import pyDKB
except Exception, err:
sys.stderr.write("(ERROR) Failed to import pyDKB library: %s\n" % err)
sys.exit(1)


class Case(unittest.TestCase):
def setUp(self):
self.stage = pyDKB.dataflow.stage.ProcessorStage()
self.fake_config = tempfile.NamedTemporaryFile(dir='.')

def tearDown(self):
self.stage = None
if not self.fake_config.closed:
self.fake_config.close()
self.fake_config = None

def test_correct_c(self):
self.stage.configure(['-c', self.fake_config.name, 'something'])
isfile = isinstance(getattr(self.stage.ARGS, 'config'), file)
self.assertTrue(isfile)

def test_correct_config(self):
self.stage.configure(['--config', self.fake_config.name, 'something'])
isfile = isinstance(getattr(self.stage.ARGS, 'config'), file)
self.assertTrue(isfile)

def test_missing_c(self):
self.fake_config.close()
args = ['-c', self.fake_config.name, 'something']
[msg, result] = isolate_function_error(self.stage.configure, args)
err = "[Errno 2] No such file or directory: '%s'" %\
self.fake_config.name
self.assertIn(err, msg)

def test_missing_config(self):
self.fake_config.close()
args = ['--config', self.fake_config.name, 'something']
[msg, result] = isolate_function_error(self.stage.configure, args)
err = "[Errno 2] No such file or directory: '%s'" %\
self.fake_config.name
self.assertIn(err, msg)

def test_unreadable_c(self):
os.chmod(self.fake_config.name, 0300)
args = ['-c', self.fake_config.name, 'something']
[msg, result] = isolate_function_error(self.stage.configure, args)
err = "[Errno 13] Permission denied: '%s'" %\
self.fake_config.name
self.assertIn(err, msg)

def test_unreadable_config(self):
os.chmod(self.fake_config.name, 0300)
args = ['--config', self.fake_config.name, 'something']
[msg, result] = isolate_function_error(self.stage.configure, args)
err = "[Errno 13] Permission denied: '%s'" %\
self.fake_config.name
self.assertIn(err, msg)


def load_tests(loader, tests, pattern):
suite = unittest.TestSuite()
suite.addTest(loader.loadTestsFromTestCase(Case))
return suite

0 comments on commit 6f81f0b

Please sign in to comment.