Merge pull request ActivitySim#4 from RSGInc/task_4

Read CSVs (and better index names)

.gitignore

@@ -65,3 +65,5 @@ docs/_build/
 *.cprof
 *.lprof
 
+# macOS
+*.DS_Store

MANIFEST.in

@@ -2,3 +2,7 @@ include ez_setup.py
 include README.rst
 include activitysim\abm\test\data\mtc_asim.h5
 include activitysim\abm\test\data\skims.omx
+include activitysim\abm\test\data\households.csv
+include activitysim\abm\test\data\persons.csv
+include activitysim\abm\test\data\land_use.csv
+include activitysim\abm\test\data\override_hh_ids.csv

activitysim/abm/models/initialize.py

@@ -23,6 +23,22 @@
 from activitysim.abm.tables import shadow_pricing
 
 
+# We are using the naming conventions in the mtc_asim.h5 example
+# file for our default list. This provides backwards compatibility
+# with previous versions of ActivitySim in which only 'input_store'
+# is given in the settings file.
+DEFAULT_TABLE_LIST = [
+    {'tablename': 'households',
+     'h5_tablename': 'households',
+     'index_col': 'household_id'},
+    {'tablename': 'persons',
+     'h5_tablename': 'persons',
+     'index_col': 'person_id'},
+    {'tablename': 'land_use',
+     'h5_tablename': 'land_use_taz',
+     'index_col': 'TAZ'}
+]
+
 logger = logging.getLogger(__name__)
 
 
@@ -109,6 +125,18 @@ def preload_injectables():
     inject.add_step('write_data_dictionary', write_data_dictionary)
     inject.add_step('write_tables', write_tables)
 
+    table_list = config.setting('input_table_list')
+
+    # default ActivitySim table names and indices
+    if table_list is None:
+        logger.warn(
+            "No 'input_table_list' found in settings. This will be a "
+            "required setting in upcoming versions of ActivitySim.")
+
+        new_settings = inject.get_injectable('settings')
+        new_settings['input_table_list'] = DEFAULT_TABLE_LIST
+        inject.add_injectable('settings', new_settings)
+
     t0 = tracing.print_elapsed_time()
 
     # FIXME - still want to do this?

activitysim/abm/tables/__init__.py

@@ -3,8 +3,6 @@
 
 from __future__ import absolute_import
 
-from . import input_store
-
 from . import households
 from . import persons
 from . import landuse

activitysim/abm/tables/households.py

@@ -15,7 +15,7 @@
 from activitysim.core import pipeline
 from activitysim.core import inject
 
-from .input_store import read_input_table
+from activitysim.core.input import read_input_table
 
 logger = logging.getLogger(__name__)
 
@@ -86,8 +86,6 @@ def households(households_sample_size, override_hh_ids, trace_hh_id):
 
     logger.info("loaded households %s" % (df.shape,))
 
-    df.index.name = 'household_id'
-
     # FIXME - pathological knowledge of name of chunk_id column used by chunked_choosers_by_chunk_id
     assert 'chunk_id' not in df.columns
     df['chunk_id'] = pd.Series(list(range(len(df))), df.index)

activitysim/abm/tables/landuse.py

@@ -8,20 +8,18 @@
 import logging
 
 from activitysim.core import inject
-from .input_store import read_input_table
+from activitysim.core.input import read_input_table
 
 logger = logging.getLogger(__name__)
 
 
 @inject.table()
 def land_use():
 
-    df = read_input_table("land_use_taz")
+    df = read_input_table("land_use")
 
     logger.info("loaded land_use %s" % (df.shape,))
 
-    df.index.name = 'TAZ'
-
     # replace table function with dataframe
     inject.add_table('land_use', df)
 

activitysim/abm/tables/persons.py

@@ -11,7 +11,7 @@
 from activitysim.core import inject
 from activitysim.core import tracing
 
-from .input_store import read_input_table
+from activitysim.core.input import read_input_table
 
 logger = logging.getLogger(__name__)
 
@@ -34,8 +34,6 @@ def persons(households, trace_hh_id):
 
     logger.info("loaded persons %s" % (df.shape,))
 
-    df.index.name = 'person_id'
-
     # replace table function with dataframe
     inject.add_table('persons', df)
 

activitysim/abm/test/configs/settings.yaml

@@ -1,9 +1,26 @@
 inherit_settings: True
 
-#input data store and skims
-input_store: mtc_asim.h5
+# input skims
 skims_file: skims.omx
 
+# input tables
+input_table_list:
+  - tablename: households
+    filename: households.csv
+    index_col: household_id
+    column_map:
+      HHID: household_id
+  - tablename: persons
+    filename: persons.csv
+    index_col: person_id
+    column_map:
+      PERID: person_id
+  - tablename: land_use
+    filename: land_use.csv
+    index_col: TAZ
+    column_map:
+      ZONE: TAZ
+
 #number of households to simulate
 households_sample_size:  100
 # simulate all households