Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Fixed: WorkflowManager.output() acts differently when "count frequenc…
…y" is False

This is for table-like outputs only: CSV, Stata, Excel, and HTML.

Previously, output() would fail if called with table-like output formats but the
"count frequency" setting were False. Now, it outputs the unaggregated results,
one file per piece, quasi-LilyPond-style.

This is required for "moment-by-moment" output in the Counterpoint Web App.
  • Loading branch information
crantila committed Dec 14, 2014
1 parent 67754c5 commit 856c608
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 48 deletions.
1 change: 1 addition & 0 deletions VERSION_HISTORY.md
Expand Up @@ -7,6 +7,7 @@ the top of the file.
* 2.1.0:
- Python 3 compatibility.
- Importing no longer uses/stores cached pickle files.
- WorkflowManager.output() produces many table-format results if "count frequency" is False
* 2.0.0:
- Analyzers must now accept and return DataFrame objects.
- Require pandas 0.14.1 at minimum.
Expand Down
2 changes: 1 addition & 1 deletion run_tests.py
Expand Up @@ -79,7 +79,7 @@
# WorkflowManager
test_workflow.WORKFLOW_TESTS,
test_workflow.FILTER_DATA_FRAME,
test_workflow.EXPORT,
test_workflow.MAKE_TABLE,
test_workflow.EXTRA_PAIRS,
test_workflow.SETTINGS,
test_workflow.OUTPUT,
Expand Down
104 changes: 76 additions & 28 deletions vis/tests/test_workflow.py
Expand Up @@ -351,12 +351,12 @@ def test_output_4(self):
except RuntimeError as run_err:
self.assertEqual(WorkflowManager._NO_RESULTS_ERROR, run_err.args[0])

@mock.patch('vis.workflow.WorkflowManager._export')
def test_output_5(self, mock_export):
@mock.patch('vis.workflow.WorkflowManager._make_table')
def test_output_5(self, mock_table):
"""ensure output() calls export() as required"""
# 1: prepare
export_path = 'the_path'
mock_export.return_value = export_path
mock_table.return_value = export_path
test_wc = WorkflowManager([])
test_wc._previous_exp = 'intervals'
test_wc._data = [1 for _ in xrange(20)]
Expand All @@ -367,7 +367,7 @@ def test_output_5(self, mock_export):
actual = test_wc.output('Excel', path)
# 3: check
self.assertEqual(export_path, actual)
mock_export.assert_called_once_with(*expected_args)
mock_table.assert_called_once_with(*expected_args)


@mock.patch('vis.workflow.WorkflowManager._filter_dataframe')
Expand Down Expand Up @@ -701,42 +701,90 @@ def test_extra_pairs_5(self):
self.assertSequenceEqual(list(expected.columns), list(actual.columns))


class Export(TestCase):
"""Tests for WorkflowManager._export()"""
class MakeTable(TestCase):
"""Tests for WorkflowManager._make_table()"""

@mock.patch('vis.workflow.WorkflowManager._filter_dataframe')
def test_export_1(self, mock_fdf):
"""the method works as expected for CSV, Excel, and Stata when _result is a DataFrame"""
def test_table_1(self, mock_fdf):
'''
_make_table():
- "count frequency" is True
- file extension on "pathname" with period
'''
test_wm = WorkflowManager([])
test_wm.settings(None, 'count frequency', True) # just to be 100% clear
mock_fdf.return_value = mock.MagicMock(spec_set=pandas.DataFrame)
test_wm._result = mock_fdf.return_value # to avoid a RuntimeError
test_wm._export('CSV', 'test_path') # pylint: disable=protected-access
test_wm._export('Excel', 'test_path') # pylint: disable=protected-access
test_wm._export('Stata', 'test_path') # pylint: disable=protected-access
test_wm._export('HTML', 'test_path') # pylint: disable=protected-access
top_x = None
threshold = None
pathname = 'test_path'

test_wm._make_table('CSV', pathname + '.csv', top_x, threshold) # pylint: disable=protected-access
test_wm._make_table('Excel', pathname + '.xlsx', top_x, threshold) # pylint: disable=protected-access
test_wm._make_table('Stata', pathname + '.dta', top_x, threshold) # pylint: disable=protected-access
test_wm._make_table('HTML', pathname + '.html', top_x, threshold) # pylint: disable=protected-access

mock_fdf.return_value.to_csv.assert_called_once_with('test_path.csv')
mock_fdf.return_value.to_stata.assert_called_once_with('test_path.dta')
mock_fdf.return_value.to_excel.assert_called_once_with('test_path.xlsx')
mock_fdf.return_value.to_html.assert_called_once_with('test_path.html')
self.assertSequenceEqual([mock.call(top_x=None, threshold=None) for _ in xrange(4)],
mock_fdf.call_args_list)

@mock.patch('vis.workflow.WorkflowManager._filter_dataframe')
def test_export_2(self, mock_fdf):
"""test_export_1() with a valid extension already on"""
def test_table_2(self):
'''
_make_table():
- "count frequency" is False
- file extension not on "pathname"
- there is only one IndexedPiece
'''
test_wm = WorkflowManager([])
mock_fdf.return_value = mock.MagicMock(spec_set=pandas.DataFrame)
test_wm._result = mock_fdf.return_value # to avoid a RuntimeError
test_wm._export('CSV', 'test_path.csv') # pylint: disable=protected-access
test_wm._export('Excel', 'test_path.xlsx') # pylint: disable=protected-access
test_wm._export('Stata', 'test_path.dta') # pylint: disable=protected-access
test_wm._export('HTML', 'test_path.html') # pylint: disable=protected-access
test_wm._result.to_csv.assert_called_once_with('test_path.csv')
test_wm._result.to_stata.assert_called_once_with('test_path.dta')
test_wm._result.to_excel.assert_called_once_with('test_path.xlsx')
test_wm._result.to_html.assert_called_once_with('test_path.html')
self.assertSequenceEqual([mock.call(top_x=None, threshold=None) for _ in xrange(4)],
mock_fdf.call_args_list)
test_wm.settings(None, 'count frequency', False)
test_wm._result = [mock.MagicMock(spec_set=pandas.DataFrame)]
test_wm._data = ['boop' for _ in xrange(len(test_wm._result))]
top_x = None
threshold = None
pathname = 'test_path'

test_wm._make_table('CSV', pathname, top_x, threshold) # pylint: disable=protected-access
test_wm._make_table('Excel', pathname, top_x, threshold) # pylint: disable=protected-access
test_wm._make_table('Stata', pathname, top_x, threshold) # pylint: disable=protected-access
test_wm._make_table('HTML', pathname, top_x, threshold) # pylint: disable=protected-access

for i in xrange(len(test_wm._result)):
test_wm._result[i].to_csv.assert_called_once_with(pathname + '.csv')
test_wm._result[i].to_excel.assert_called_once_with(pathname + '.xlsx')
test_wm._result[i].to_stata.assert_called_once_with(pathname + '.dta')
test_wm._result[i].to_html.assert_called_once_with(pathname + '.html')

def test_table_3(self):
'''
_make_table():
- "count frequency" is False
- file extension not on "pathname"
- there are several IndexedPiece objects
'''
test_wm = WorkflowManager([])
test_wm.settings(None, 'count frequency', False)
test_wm._result = [mock.MagicMock(spec_set=pandas.DataFrame) for _ in xrange(5)]
test_wm._data = ['boop' for _ in xrange(len(test_wm._result))]
top_x = None
threshold = None
pathname = 'test_path'

test_wm._make_table('CSV', pathname, top_x, threshold) # pylint: disable=protected-access
test_wm._make_table('Excel', pathname, top_x, threshold) # pylint: disable=protected-access
test_wm._make_table('Stata', pathname, top_x, threshold) # pylint: disable=protected-access
test_wm._make_table('HTML', pathname, top_x, threshold) # pylint: disable=protected-access

for i in xrange(len(test_wm._result)):
test_wm._result[i].to_csv.assert_called_once_with('{}-{}{}'.format(pathname, i, '.csv'))
test_wm._result[i].to_excel.assert_called_once_with('{}-{}{}'.format(pathname, i, '.xlsx'))
test_wm._result[i].to_stata.assert_called_once_with('{}-{}{}'.format(pathname, i, '.dta'))
test_wm._result[i].to_html.assert_called_once_with('{}-{}{}'.format(pathname, i, '.html'))


class FilterDataFrame(TestCase):
Expand Down Expand Up @@ -946,7 +994,7 @@ def test_unique_combos_2(self):
#-------------------------------------------------------------------------------------------------#
WORKFLOW_TESTS = TestLoader().loadTestsFromTestCase(WorkflowTests)
FILTER_DATA_FRAME = TestLoader().loadTestsFromTestCase(FilterDataFrame)
EXPORT = TestLoader().loadTestsFromTestCase(Export)
MAKE_TABLE = TestLoader().loadTestsFromTestCase(MakeTable)
EXTRA_PAIRS = TestLoader().loadTestsFromTestCase(ExtraPairs)
SETTINGS = TestLoader().loadTestsFromTestCase(Settings)
OUTPUT = TestLoader().loadTestsFromTestCase(Output)
Expand Down
72 changes: 53 additions & 19 deletions vis/workflow.py
Expand Up @@ -755,6 +755,9 @@ def output(self, instruction, pathname=None, top_x=None, threshold=None):
.. note:: For LiliyPond output, you must have called :meth:`run` with ``count frequency``
set to ``False``.
.. note:: If ``count frequency`` is set to ``False`` for CSV, Stata, Excel, or HTML output,
the ``top_x`` and ``threshold`` parameters are ignored.
:parameter str instruction: The type of visualization to output.
:parameter str pathname: The pathname for the output. The default is
``'test_output/output_result``. Do not include a file-type "extension," since we add
Expand All @@ -770,6 +773,7 @@ def output(self, instruction, pathname=None, top_x=None, threshold=None):
:returns: The pathname(s) of the outputted visualization(s). Requesting a histogram always
returns a single string; requesting a score (or some scores) always returns a list.
The other formats will return a list if the ``count frequency`` setting is ``False``.
:rtype: str or [str]
:raises: :exc:`RuntimeError` for unrecognized instructions.
Expand Down Expand Up @@ -807,8 +811,11 @@ def output(self, instruction, pathname=None, top_x=None, threshold=None):

# handle instructions
if instruction in ('CSV', 'Stata', 'Excel', 'HTML'):
# these will be done by export()
return self._export(instruction, pathname, top_x, threshold)
pathnames = self._make_table(instruction, pathname, top_x, threshold)
if 1 == len(pathnames):
return pathnames[0]
else:
return pathnames # TODO: test this
elif instruction == 'LilyPond':
return self._make_lilypond(pathname)
elif instruction == 'histogram' or instruction == 'R histogram':
Expand Down Expand Up @@ -897,30 +904,57 @@ def _make_lilypond(self, pathname=None):

return pathnames

def _export(self, form, pathname=None, top_x=None, threshold=None):
"""
Arguments as per :meth:`output`. You should always use :meth:`output`.
def _make_table(self, form, pathname, top_x, threshold):
"""
Output a table-style result. Called by :meth:`output`.
# filter the results
export_me = self._filter_dataframe(top_x=top_x, threshold=threshold)
:param st form: Either 'CSV', 'Stata', 'Excel', or 'HTML', depending on the desired output
format.
:param str pathname: As in :meth:`output`.
:param int top_x: As in :meth:`output`.
:param int threshold: As in :meth:`output`.
:returns: The pathname(s) of the outputted files.
:rtype: list of str
.. note:: If ``count frequency`` is ``False``, the ``top_x`` and ``threshold`` parameters
are ignored.
"""

# key is the instruction; value is (extension, export_method)
directory = {'CSV': ('.csv', export_me.to_csv),
'Stata': ('.dta', export_me.to_stata),
'Excel': ('.xlsx', export_me.to_excel),
'HTML': ('.html', export_me.to_html)}
directory = {'CSV': ('.csv', 'to_csv'),
'Stata': ('.dta', 'to_stata'),
'Excel': ('.xlsx', 'to_excel'),
'HTML': ('.html', 'to_html')}

# ensure we have an output path
pathname = 'test_output/no_path' if pathname is None else six.u(pathname)
# ensure there's a file extension
if directory[form][0] != pathname[-1 * len(directory[form][0]):]:
pathname += directory[form][0]
# set file extension and the method to call for output
file_ext, output_meth = directory[form]

# call the to_whatever() method
directory[form][1](pathname)
# ensure the pathname doesn't have a file extension
if pathname.endswith(file_ext):
pathname = pathname[:(-1 * len(file_ext))]

return pathname
pathnames = []

if self.settings(None, 'count frequency'):
# filter the results
export_me = self._filter_dataframe(top_x=top_x, threshold=threshold)
pathnames.append('{}{}'.format(pathname, file_ext))
getattr(export_me, output_meth)(pathnames[-1])
else:
enum = True if (len(self._data) > 1 and not self.settings(None, 'count frequency')) else False
for i in xrange(len(self._data)):
# append piece index to pathname, if there are many pieces
if enum:
pathnames.append('{}-{}{}'.format(pathname, i, file_ext))
# call the method that actually outputs the result
getattr(self._result[i], output_meth)(pathnames[-1])
else:
pathnames.append('{}{}'.format(pathname, file_ext))
# call the method that actually outputs the result
getattr(self._result[i], output_meth)(pathnames[-1])

return pathnames

def metadata(self, index, field, value=None):
"""
Expand Down

0 comments on commit 856c608

Please sign in to comment.