Merge pull request #80 from RSGInc/multi-file-link_daily

Multi file link daily
RSGInc · Jan 15, 2017 · 507896c · 507896c
2 parents e475cdd + b90cf54
commit 507896c
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 21 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -13,7 +13,7 @@ install:
 - conda update -q conda
 - conda info -a
 - |
-  conda create -q -c synthicity -n test-environment python=$TRAVIS_PYTHON_VERSION cytoolz ipython-notebook jinja2 matplotlib numpy pandas pandana patsy pip pytables pytest pyyaml scipy statsmodels toolz
+  conda create -q -c synthicity -n test-environment python=$TRAVIS_PYTHON_VERSION cytoolz ipython-notebook jinja2 matplotlib numpy pandas pandana patsy pip pytables pytest pyyaml scipy statsmodels toolz=0.8.0
 - source activate test-environment
 - pip install bottle orca prettytable simplejson zbox
 - pip install openmatrix

diff --git a/bca4abm/processors/link.py b/bca4abm/processors/link.py
@@ -82,12 +82,13 @@ def add_tables_to_locals(data_dir, settings, settings_tag, locals_dict):
     return locals_dict
 
 
-def eval_link_spec(link_spec, link_file_names, data_dir, link_file_column_map,
+def eval_link_spec(link_spec, link_file_names, data_dir,
+                   link_file_column_map, link_index_fields,
                    settings, settings_tag, trace_tag=None, trace_od=None):
 
-    # we accept a single string as well as a list of strings
+    # accept a single string as well as a dict of {suffix: filename}
     if isinstance(link_file_names, str):
-        link_file_names = [link_file_names]
+        link_file_names = {"": link_file_names}
 
     locals_dict = bca.assign_variables_locals(settings, settings_tag)
 
@@ -99,20 +100,25 @@ def eval_link_spec(link_spec, link_file_names, data_dir, link_file_column_map,
 
         link_data_subdir = 'base-data' if scenario == 'base' else 'build-data'
 
-        for i in range(len(link_file_names)):
-            if i == 0:
-                links_df = read_csv_file(data_dir=os.path.join(data_dir, link_data_subdir),
-                                         file_name=link_file_names[0],
-                                         column_map=link_file_column_map)
-            else:
-                links_df_add = read_csv_file(data_dir=os.path.join(data_dir, link_data_subdir),
-                                             file_name=link_file_names[i],
-                                             column_map=link_file_column_map)
-                link_index_fields = settings['link_daily_index_fields']
-                links_df = links_df.set_index(link_index_fields, drop=False)
-                links_df_add = links_df_add.set_index(link_index_fields, drop=False)
-                suffix = "_" + link_file_names[i].replace(".csv", "")
-                links_df = links_df.join(links_df_add, how="outer", rsuffix=suffix)
+        df_list = []
+        for suffix, link_file_name in link_file_names.iteritems():
+
+            df = read_csv_file(data_dir=os.path.join(data_dir, link_data_subdir),
+                               file_name=link_file_name,
+                               column_map=link_file_column_map)
+
+            if link_index_fields:
+                df.set_index(link_index_fields, drop=True, inplace=True)
+            if suffix:
+                df = df.add_suffix("_" + suffix)
+
+            df_list.append(df)
+
+        links_df = pd.concat(df_list, axis=1)
+
+        # copy index fields into columns
+        if link_index_fields:
+            links_df = links_df.reset_index().set_index(link_index_fields, drop=False)
 
         if trace_od:
             od_column = settings.get('%s_od_column' % settings_tag, None)
@@ -164,10 +170,13 @@ def link_processor(link_manifest, link_spec, settings, data_dir):
 
     for row in link_manifest.itertuples(index=True):
 
+        link_index_fields = None
+
         row_results = eval_link_spec(link_spec,
                                      row.link_file_name,
                                      data_dir,
                                      settings.get('link_table_column_map', None),
+                                     link_index_fields,
                                      settings,
                                      settings_tag='link')
 
@@ -207,6 +216,7 @@ def link_daily_processor(link_daily_spec, settings, data_dir, trace_od):
                              link_daily_file_names,
                              data_dir,
                              settings.get('link_daily_table_column_map', None),
+                             settings.get('link_daily_index_fields', None),
                              settings,
                              settings_tag='link_daily',
                              trace_tag='link_daily',

diff --git a/bca4abm/tests/test_4step/configs/link_daily.csv b/bca4abm/tests/test_4step/configs/link_daily.csv
@@ -1,6 +1,8 @@
 Silos,Description,Target,Expression
 ,,_auto_volume,links.svol + links.hvol
 ,,_med_truck_volume,links.mtvol
+# check that the suffix is appended
+,,_med_truck_volume,links.mtvol_dupe
 ,,_heavy_truck_volume,links.htvol
 ,,_total_truck_volume,_med_truck_volume + _heavy_truck_volume
 ,,_total_volume,_auto_volume + _total_truck_volume

diff --git a/bca4abm/tests/test_4step/configs/settings.yaml b/bca4abm/tests/test_4step/configs/settings.yaml
@@ -99,7 +99,8 @@ aggregate_data_manifest_column_map:
 
 
 link_daily_file_names:
-  - linksMD1.csv
+  "": linksMD1.csv
+  dupe: linksMD1.csv
 
 link_daily_table_column_map:
   i: i

diff --git a/example_4step/configs/settings.yaml b/example_4step/configs/settings.yaml
@@ -140,8 +140,8 @@ locals_aggregate_od:
 #### Link Daily Processor
 
 link_daily_file_names:
-  - linksMD1.csv
-  - linksPM2.csv
+  "": linksMD1.csv
+  linksPM2: linksPM2.csv
 
 link_daily_index_fields:
   - i