Tests: update tests for Importer

Also turns aggregated_pieces tests back on and adds a test to
noterest.

run_tests.py

@@ -95,11 +95,12 @@
              test_aggregator.COLUMN_AGGREGATOR_SUITE,
              test_barchart.R_BAR_CHART_SUITE,
              # test_dendrogram.DENDROGRAM_SUITE, # This test suite is commented out so that we can remove our SciPy dependency.
-             # IndexedPiece and AggregatedPieces
+             # Importer, IndexedPiece, and AggregatedPieces
+             test_aggregated_pieces.IMPORTER_SUITE,
              test_indexed_piece.INDEXED_PIECE_SUITE_A,
              test_indexed_piece.INDEXED_PIECE_PARTS_TITLES,
              test_indexed_piece.INDEXED_PIECE_SUITE_C,
-             # test_aggregated_pieces.AGGREGATED_PIECES_SUITE,
+             test_aggregated_pieces.AGGREGATED_PIECES_SUITE,
              # NB: Most of these WorkflowManager tests pass but they are commented out because the WorkflowManager is deprecated.
              # # WorkflowManager 
              # test_workflow.WORKFLOW_TESTS,  # FutureWarning: sort(columns) is depracated, use sort_values(by=...)

vis/models/aggregated_pieces.py

@@ -78,7 +78,7 @@ class Metadata(object):
         """
         __slots__ = ('composers', 'dates', 'date_range', 'titles', 'locales', 'pathnames')
 
-    def __init__(self, pieces=None, metafiles=None):
+    def __init__(self, pieces=None, metafile=None):
         """
         :param pieces: The IndexedPieces to collect.
         :type pieces: list of :class:`~vis.models.indexed_piece.IndexedPiece`
@@ -94,7 +94,7 @@ def init_metadata():
 
         super(AggregatedPieces, self).__init__()
         self._pieces = pieces if pieces is not None else []
-        self._metafiles = metafiles if metafiles is not None else []
+        self._metafile = metafile if metafile is not None else []
         self._metadata = {}
         init_metadata()
         # Multi-key dictionary for combined_experimenter calls to get_data()
@@ -166,7 +166,8 @@ def _fetch_metadata(self, field):
         # locales: list of all the locales in the IndexedPieces
         elif 'locales' == field:
             post = [p.metadata('locale_of_composition') for p in self._pieces]
-
+        elif 'pathnames' == field:
+            post = [p._pathname for p in self._pieces]
         if post is not None:
             self._metadata[field] = post
         return post

vis/models/indexed_piece.py

@@ -292,30 +292,33 @@ def _import_file(pathname, metafile=None):
 
     return score
 
-def _import_directory(directory):
+def _import_directory(directory, metafile=None):
 
     pieces = [] # a list of the pieces being imported
-
-    for root, dirs, files in os.walk(directory):
-        if len(files) == 0:
-            raise RuntimeError(vis.models.aggregated_piece.AggregatedPieces._NO_FILES)
-
-        # remove ds_stores
-        if '.DS_Store' in files:
-            files.remove('.DS_Store')
-
-        # attach meta files if they exist
-        if 'meta' in files:
-            meta = root + '/meta'
-            files.remove('meta')
-        else:
-            meta = None
-
-        for file in files:
-            if len(file) > 1 and file[:2] == '._': # filter out hidden files if they show up
-                continue
-            path = root + '/' + file
-            pieces.extend(_import_file(pathname=path, metafile=meta))
+    meta = metafile
+
+    if isinstance(directory, list):
+        file_paths = directory
+
+    else: # the `directory` argument is the pathname of a directory
+        file_paths = []
+        for root, dirs, files in os.walk(directory):
+            for f in files:
+                if f == '.DS_Store': # exclude ds_stores
+                    continue
+                if len(f) > 1 and f[:2] == '._': # filter out hidden files if they show up
+                    continue
+                if f == 'meta': # attach meta files if they exist
+                    meta = root + '/meta'
+                    continue
+                file_paths.append('/'.join((root, f)))
+
+    if not file_paths:
+        raise RuntimeError(vis.models.aggregated_piece.AggregatedPieces._NO_FILES)
+
+    for path in file_paths:
+        # use extend rather than append because it could import as a multi-movement opus
+        pieces.extend(_import_file(pathname=path, metafile=meta))
 
     return (pieces, meta)
 
@@ -332,25 +335,24 @@ def Importer(location, metafile=None):
     :rtype: A new :class:`IndexedPiece` or :class:`AggregatedPieces` object.
     """
     pieces = []
-    meta = None
-
-    # index piece if it is a file or a link
-    if os.path.isfile(location):
-        pieces.extend(_import_file(location))
 
     # load directory of pieces
-    elif os.path.isdir(location):
-        directory_return = _import_directory(location)
+    if isinstance(location, list) or os.path.isdir(location):
+        directory_return = _import_directory(location, metafile)
         pieces.extend(directory_return[0])
-        meta = directory_return[1]
+        metafile = directory_return[1]
+
+    # index piece if it is a file or a link
+    elif os.path.isfile(location):
+        pieces.extend(_import_file(location))
 
     else:
         raise RuntimeError(self._UNKNOWN_INPUT)
 
     if len(pieces) == 1: # there was a single piece that imported as a score (not an opus)
         return(pieces[0]) # this returns an IndexedPiece object
     else: # there were multiple pieces or a single piece that imported as an opus
-        return(AggregatedPieces(pieces=pieces, metafiles=meta))
+        return(AggregatedPieces(pieces=pieces, metafile=metafile))
 
 
 class OpusWarning(RuntimeWarning):

vis/scripts/ngram_test.py

@@ -1,7 +1,7 @@
 from vis.analyzers.indexers import noterest, interval, ngram, meter, dissonance #, active_voices
 from vis.models import indexed_piece
-from vis.models.indexed_piece import IndexedPiece
-from vis.models.aggregated_pieces import AggregatedPieces
+# from vis.models.indexed_piece import IndexedPiece
+# from vis.models.aggregated_pieces import AggregatedPieces
 import pandas
 import pdb
 import time
@@ -17,7 +17,10 @@
 # piece_path = '/home/amor/Code/vis-framework/vis/scripts/Morley_Duets/7 Miraculous loves wounding.xml' # example of IR = quarter note
 chopin_prelude = '/home/amor/Code/vis-framework/vis/scripts/prelude28-20.mid'
 
+# Senfl_Buchner folder:
+folder = '/home/amor/Code/vis-framework/vis/scripts/Senfl motets'
 # Senfl Pieces:
+piece_path = '/home/amor/Code/vis-framework/vis/scripts/Senfl motets/No03_LaudateDominum_ns_final.xml'
 # piece_path = '/home/amor/Code/vis-framework/vis/scripts/Senfl_Buchner/AssumptaEst_normal.xml'
 # piece_path = '/home/amor/Code/vis-framework/vis/scripts/Senfl_Buchner/DumSteteritis_normal.xml'
 # piece_path = '/home/amor/Code/vis-framework/vis/scripts/Senfl_Buchner/No01_Converte_nos Kopie.xml'
@@ -39,24 +42,23 @@
            'terminator': ['Rest'], 'open-ended': False, 'brackets': False, 'align': 'right'}
 n_setts_5 = {'n': 3, 'continuer': 'P1', 'horizontal': 'lowest', 'vertical': 'all',
            'terminator': ['Rest'], 'open-ended': False, 'brackets': False}
-test = indexed_piece.ImportScore(piece_path)
-pdb.set_trace()
-test.get_data('du')
-ip = IndexedPiece(piece_path)
-ps = ip._get_m21_objs()
+# ip = indexed_piece.Importer(piece_path)
+t0 = time.time()
+ag = indexed_piece.Importer(folder)
+# ps = ip._get_part_streams()
 # bs = ip._get_beat_strength()
 # dr = ip._get_duration()
 # ms = ip._get_measure()
-nr = ip._get_noterest()
-vz = ip._get_vertical_interval(h_setts2)
-hz = ip._get_horizontal_interval(v_setts)
-t0 = time.time()
+# nr = ip._get_noterest()
+# vz = ip._get_vertical_interval(h_setts2)
+# hz = ip._get_horizontal_interval(v_setts)
+# recursed = ip._get_m21_objs()
 # mnr = ip._get_multistop()
 # # vz = ip._get_vertical_interval(v_setts)
 # av = active_voices.ActiveVoicesIndexer(nr).run()
-ng = ngram.NGramIndexer((vz, hz), n_setts_4).run()
+# ng = ngram.NGramIndexer((vz, hz), n_setts_4).run()
 t1 = time.time()
-ng2 = ngram.NGramIndexer((vz, hz), n_setts_5).run()
+# ng2 = ngram.NGramIndexer((vz, hz), n_setts_5).run()
 # t2 = time.time()
 print(t1-t0)
 # print(t2-t1)

vis/tests/bwv2_integration_tests.py

@@ -29,7 +29,7 @@
 import os
 from unittest import TestCase, TestLoader
 import pandas
-from vis.models.indexed_piece import ImportScore, IndexedPiece
+from vis.models.indexed_piece import Importer, IndexedPiece
 from vis.analyzers.indexers import noterest, interval, ngram
 from vis.analyzers.experimenters import frequency
 from vis import workflow
@@ -125,7 +125,7 @@ def test_ngrams_1(self):
         """Ngram integration test."""
         expected = AllVoiceIntervalNGrams.series_maker(AllVoiceIntervalNGrams.two_grams)
         expected = pandas.DataFrame({('ngram.NGramIndexer', '0,3 1,3 2,3 : 3'): expected})
-        ind_piece = ImportScore(os.path.join(VIS_PATH, 'tests', 'corpus', 'bwv2.xml'))
+        ind_piece = Importer(os.path.join(VIS_PATH, 'tests', 'corpus', 'bwv2.xml'))
         setts = {'quality': False, 'simple': False, 'horiz_attach_later': True}
         horiz_ints = ind_piece.get_data('horizontal_interval', settings=setts)
         vert_ints = ind_piece.get_data('vertical_interval', settings=setts)

vis/tests/bwv603_integration_tests.py

@@ -29,7 +29,7 @@
 import os
 from unittest import TestCase, TestLoader
 import pandas
-from vis.models.indexed_piece import ImportScore, IndexedPiece
+from vis.models.indexed_piece import Importer, IndexedPiece
 from vis.analyzers.indexers import noterest, interval, ngram
 from vis.analyzers.experimenters import frequency
 from vis import workflow
@@ -105,7 +105,7 @@ def test_ngrams_3(self):
         v_setts = {'quality': False, 'simple or compound': 'compound', 'directed': True}
         h_setts = {'quality': False, 'horiz_attach_later': True, 'simple or compound': False, 'directed': True}
         n_setts = {'n': 2, 'horizontal': [('3',)], 'vertical': [('0,3', '1,3', '2,3')], 'brackets': True, 'continuer': '1'}
-        ind_piece = ImportScore(os.path.join(VIS_PATH, 'tests', 'corpus', 'bwv603.xml'))
+        ind_piece = Importer(os.path.join(VIS_PATH, 'tests', 'corpus', 'bwv603.xml'))
         vt = ind_piece.get_data('vertical_interval', settings=v_setts)
         hz = ind_piece.get_data('horizontal_interval', settings=h_setts)
         actual = ind_piece.get_data('ngram', data=(vt, hz), settings=n_setts)

vis/tests/test_active_voices.py

@@ -94,30 +94,30 @@ def test_init2(self):
     def test_active(self):
         """tests that it gives the right results with no settings and that the get_data()
         method properly calls active_voices"""
-        ip = IndexedPiece('phony pathname')
+        ip = IndexedPiece()
         ip._analyses['noterest'] = self.NOTES
         actual = ip.get_data('active_voices')
         self.assertTrue(actual.equals(self.EXPECTED))
 
     def test_with_data(self):
         """tests that it gives the right results with no settings and that the get_data()
         method properly calls active_voices"""
-        ip = IndexedPiece('phony pathname')
+        ip = IndexedPiece()
         actual = ip.get_data('active_voices', data=self.NOTES)
         self.assertTrue(actual.equals(self.EXPECTED))
 
     def test_attacked(self):
         """tests that it gives the right results with attacked set to true"""
         settings = {'attacked': True}
-        ip = IndexedPiece('phony pathname')
+        ip = IndexedPiece()
         ip._analyses['noterest'] = self.NOTES
         actual = ip.get_data('active_voices', settings=settings)
         self.assertTrue(actual.equals(self.ATT_EXPECTED))
 
     def test_show(self):
         """tests that the ``show_all`` argument works when True"""
         settings = {'show_all': True}
-        ip = IndexedPiece('phony pathname')
+        ip = IndexedPiece()
         ip._analyses['noterest'] = self.NOTES
         actual = ip.get_data('active_voices', settings=settings)
         self.assertTrue(actual.equals(self.SHOW_EXPECTED))