merge master into develop

activitysim/__init__.py

@@ -1,4 +1,4 @@
 # ActivitySim
 # See full license in LICENSE.txt.
 
-__version__ = version = '0.1dev'
+__version__ = version = '0.3dev'

activitysim/defaults/misc.py → activitysim/abm/misc.py

@@ -10,46 +10,14 @@
 import pandas as pd
 import yaml
 
+from activitysim.core import pipeline
+
 warnings.filterwarnings('ignore', category=pd.io.pytables.PerformanceWarning)
 pd.options.mode.chained_assignment = None
 
 logger = logging.getLogger(__name__)
 
 
-@orca.injectable()
-def configs_dir():
-    if not os.path.exists('configs'):
-        raise RuntimeError("configs_dir: directory does not exist")
-    return 'configs'
-
-
-@orca.injectable()
-def data_dir():
-    if not os.path.exists('data'):
-        raise RuntimeError("data_dir: directory does not exist")
-    return 'data'
-
-
-@orca.injectable()
-def output_dir():
-    if not os.path.exists('output'):
-        raise RuntimeError("output_dir: directory does not exist")
-    return 'output'
-
-
-@orca.injectable()
-def extensions_dir():
-    if not os.path.exists('extensions'):
-        raise extensions_dir("output_dir: directory does not exist")
-    return 'extensions'
-
-
-@orca.injectable()
-def settings(configs_dir):
-    with open(os.path.join(configs_dir, 'settings.yaml')) as f:
-        return yaml.load(f)
-
-
 @orca.injectable(cache=True)
 def store(data_dir, settings):
     if 'store' not in settings:
@@ -59,7 +27,11 @@ def store(data_dir, settings):
     if not os.path.exists(fname):
         logger.error("store file not found: %s" % fname)
         raise RuntimeError("store file not found: %s" % fname)
-    return pd.HDFStore(fname, mode='r')
+
+    file = pd.HDFStore(fname, mode='r')
+    pipeline.close_on_exit(file, fname)
+
+    return file
 
 
 @orca.injectable(cache=True)

activitysim/defaults/models/accessibility.py → activitysim/abm/models/accessibility.py

@@ -8,10 +8,9 @@
 import pandas as pd
 import numpy as np
 
-from activitysim import asim_eval as asim_eval
-from activitysim import tracing
-
-from .util.misc import read_model_settings, get_model_constants
+from activitysim.core import assign
+from activitysim.core import tracing
+from activitysim.core import config
 
 
 logger = logging.getLogger(__name__)
@@ -43,7 +42,7 @@ def __init__(self, skim_dict, omx, length, transpose=False):
     def __getitem__(self, key):
         """
         accessor to return flattened skim array with specified key
-        flattened array will have length length*length and will match tiled OD df used by asim_eval
+        flattened array will have length length*length and will match tiled OD df used by assign
 
         this allows the skim array to be accessed from expressions as
         skim['DISTANCE'] or skim[('SOVTOLL_TIME', 'MD')]
@@ -66,12 +65,12 @@ def __getitem__(self, key):
 @orca.injectable()
 def accessibility_spec(configs_dir):
     f = os.path.join(configs_dir, 'accessibility.csv')
-    return asim_eval.read_assignment_spec(f)
+    return assign.read_assignment_spec(f)
 
 
 @orca.injectable()
 def accessibility_settings(configs_dir):
-    return read_model_settings(configs_dir, 'accessibility.yaml')
+    return config.read_model_settings(configs_dir, 'accessibility.yaml')
 
 
 @orca.step()
@@ -97,7 +96,7 @@ def compute_accessibility(settings, accessibility_spec,
 
     logger.info("Running compute_accessibility")
 
-    constants = get_model_constants(accessibility_settings)
+    constants = config.get_model_constants(accessibility_settings)
     land_use_columns = accessibility_settings.get('land_use_columns', [])
 
     land_use_df = land_use.to_frame()
@@ -131,8 +130,8 @@ def compute_accessibility(settings, accessibility_spec,
     if constants is not None:
         locals_d.update(constants)
 
-    results, trace_results = asim_eval.assign_variables(accessibility_spec, od_df, locals_d,
-                                                        trace_rows=trace_od_rows)
+    results, trace_results, trace_assigned_locals \
+        = assign.assign_variables(accessibility_spec, od_df, locals_d, trace_rows=trace_od_rows)
     accessibility_df = pd.DataFrame(index=land_use.index)
     for column in results.columns:
         data = np.asanyarray(results[column])
@@ -159,5 +158,8 @@ def compute_accessibility(settings, accessibility_spec,
                              slicer='NONE',
                              warn_if_empty=True)
 
+            if trace_assigned_locals:
+                tracing.write_locals(df, file_name="accessibility_locals")
+
         tracing.trace_df(orca.get_table('persons_merged').to_frame(), "persons_merged",
                          warn_if_empty=True)

activitysim/defaults/models/auto_ownership.py → activitysim/abm/models/auto_ownership.py

@@ -6,10 +6,10 @@
 
 import orca
 
-from activitysim import activitysim as asim
-from activitysim import tracing
-from .util.misc import add_dependent_columns
-from .util.misc import read_model_settings, get_logit_model_settings, get_model_constants
+from activitysim.core import simulate as asim
+from activitysim.core import tracing
+from activitysim.core import pipeline
+from activitysim.core import config
 
 
 logger = logging.getLogger(__name__)
@@ -23,7 +23,7 @@ def auto_ownership_spec(configs_dir):
 
 @orca.injectable()
 def auto_ownership_settings(configs_dir):
-    return read_model_settings(configs_dir, 'auto_ownership.yaml')
+    return config.read_model_settings(configs_dir, 'auto_ownership.yaml')
 
 
 @orca.step()
@@ -38,8 +38,8 @@ def auto_ownership_simulate(households_merged,
 
     logger.info("Running auto_ownership_simulate with %d households" % len(households_merged))
 
-    nest_spec = get_logit_model_settings(auto_ownership_settings)
-    constants = get_model_constants(auto_ownership_settings)
+    nest_spec = config.get_logit_model_settings(auto_ownership_settings)
+    constants = config.get_model_constants(auto_ownership_settings)
 
     choices = asim.simple_simulate(
         choosers=households_merged.to_frame(),
@@ -53,7 +53,7 @@ def auto_ownership_simulate(households_merged,
 
     orca.add_column('households', 'auto_ownership', choices)
 
-    add_dependent_columns('households', 'households_autoown')
+    pipeline.add_dependent_columns('households', 'households_autoown')
 
     if trace_hh_id:
         trace_columns = ['auto_ownership'] + orca.get_table('households_autoown').columns

activitysim/defaults/models/cdap.py → activitysim/abm/models/cdap.py

@@ -7,14 +7,13 @@
 import orca
 import pandas as pd
 
-from activitysim import activitysim as asim
-from activitysim import tracing
+from activitysim.core import simulate as asim
+from activitysim.core import tracing
+from activitysim.core import pipeline
+from activitysim.core import config
 
-from .util.misc import read_model_settings, get_model_constants
-from .util.misc import add_dependent_columns
-
-from activitysim.cdap import cdap
 
+from .util.cdap import run_cdap
 
 logger = logging.getLogger(__name__)
 
@@ -26,7 +25,7 @@ def cdap_settings(configs_dir):
     cdap_indiv_spec and cdap_fixed_relative_proportions
     """
 
-    return read_model_settings(configs_dir, 'cdap.yaml')
+    return config.read_model_settings(configs_dir, 'cdap.yaml')
 
 
 @orca.injectable()
@@ -83,18 +82,18 @@ def cdap_simulate(persons_merged,
 
     persons_df = persons_merged.to_frame()
 
-    constants = get_model_constants(cdap_settings)
+    constants = config.get_model_constants(cdap_settings)
 
     logger.info("Running cdap_simulate with %d persons" % len(persons_df.index))
 
-    choices = cdap.run_cdap(persons=persons_df,
-                            cdap_indiv_spec=cdap_indiv_spec,
-                            cdap_interaction_coefficients=cdap_interaction_coefficients,
-                            cdap_fixed_relative_proportions=cdap_fixed_relative_proportions,
-                            locals_d=constants,
-                            chunk_size=hh_chunk_size,
-                            trace_hh_id=trace_hh_id,
-                            trace_label='cdap')
+    choices = run_cdap(persons=persons_df,
+                       cdap_indiv_spec=cdap_indiv_spec,
+                       cdap_interaction_coefficients=cdap_interaction_coefficients,
+                       cdap_fixed_relative_proportions=cdap_fixed_relative_proportions,
+                       locals_d=constants,
+                       chunk_size=hh_chunk_size,
+                       trace_hh_id=trace_hh_id,
+                       trace_label='cdap')
 
     tracing.print_summary('cdap_activity', choices.cdap_activity, value_counts=True)
 
@@ -104,8 +103,8 @@ def cdap_simulate(persons_merged,
     orca.add_column("persons", "cdap_activity", choices.cdap_activity)
     orca.add_column("persons", "cdap_rank", choices.cdap_rank)
 
-    add_dependent_columns("persons", "persons_cdap")
-    add_dependent_columns("households", "households_cdap")
+    pipeline.add_dependent_columns("persons", "persons_cdap")
+    pipeline.add_dependent_columns("households", "households_cdap")
 
     if trace_hh_id:
 

activitysim/defaults/models/create_trips.py → activitysim/abm/models/create_trips.py

@@ -8,11 +8,10 @@
 import pandas as pd
 import numpy as np
 
-from activitysim.util import reindex
+from activitysim.core.util import reindex
 
-from activitysim import activitysim as asim
-from activitysim import tracing
-from activitysim import pipeline
+from activitysim.core import tracing
+from activitysim.core import pipeline
 
 logger = logging.getLogger(__name__)
 

activitysim/defaults/models/destination.py → activitysim/abm/models/destination.py

@@ -8,10 +8,10 @@
 import pandas as pd
 import numpy as np
 
-from activitysim import activitysim as asim
-from activitysim import tracing
+from activitysim.core import simulate as asim
+from activitysim.core import tracing
+from activitysim.core import config
 
-from .util.misc import read_model_settings, get_logit_model_settings, get_model_constants
 
 logger = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@ def destination_choice_spec(configs_dir):
 
 @orca.injectable()
 def destination_choice_settings(configs_dir):
-    return read_model_settings(configs_dir, 'destination_choice.yaml')
+    return config.read_model_settings(configs_dir, 'destination_choice.yaml')
 
 
 @orca.step()
@@ -47,7 +47,7 @@ def destination_choice(non_mandatory_tours_merged,
     alternatives = destination_size_terms.to_frame()
     spec = destination_choice_spec.to_frame()
 
-    constants = get_model_constants(destination_choice_settings)
+    constants = config.get_model_constants(destination_choice_settings)
 
     sample_size = destination_choice_settings["SAMPLE_SIZE"]
 

activitysim/defaults/models/mandatory_scheduling.py → activitysim/abm/models/mandatory_scheduling.py

@@ -7,18 +7,18 @@
 import orca
 import pandas as pd
 
-from activitysim import activitysim as asim
-from activitysim import tracing
-from .util.vectorize_tour_scheduling import vectorize_tour_scheduling
+from activitysim.core import simulate as asim
+from activitysim.core import tracing
+from activitysim.core import config
 
-from .util.misc import read_model_settings, get_model_constants
+from .util.vectorize_tour_scheduling import vectorize_tour_scheduling
 
 logger = logging.getLogger(__name__)
 
 
 @orca.injectable()
 def mandatory_scheduling_settings(configs_dir):
-    return read_model_settings(configs_dir, 'mandatory_scheduling.yaml')
+    return config.read_model_settings(configs_dir, 'mandatory_scheduling.yaml')
 
 
 @orca.table()
@@ -65,7 +65,7 @@ def mandatory_scheduling(mandatory_tours_merged,
     tours = mandatory_tours_merged.to_frame()
     alts = tdd_alts.to_frame()
 
-    constants = get_model_constants(mandatory_scheduling_settings)
+    constants = config.get_model_constants(mandatory_scheduling_settings)
 
     school_spec = tdd_school_spec.to_frame()
     school_tours = tours[tours.tour_type == "school"]

activitysim/defaults/models/mandatory_tour_frequency.py → activitysim/abm/models/mandatory_tour_frequency.py

@@ -8,14 +8,12 @@
 import pandas as pd
 import yaml
 
-from activitysim import activitysim as asim
-from activitysim import tracing
-from .util.tour_frequency import process_mandatory_tours
-from activitysim import pipeline
-
-from .util.misc import read_model_settings, get_logit_model_settings, get_model_constants
-from .util.misc import add_dependent_columns
+from activitysim.core import simulate as asim
+from activitysim.core import tracing
+from activitysim.core import pipeline
+from activitysim.core import config
 
+from .util.tour_frequency import process_mandatory_tours
 
 logger = logging.getLogger(__name__)
 
@@ -28,7 +26,7 @@ def mandatory_tour_frequency_spec(configs_dir):
 
 @orca.injectable()
 def mandatory_tour_frequency_settings(configs_dir):
-    return read_model_settings(configs_dir, 'mandatory_tour_frequency.yaml')
+    return config.read_model_settings(configs_dir, 'mandatory_tour_frequency.yaml')
 
 
 @orca.step()
@@ -46,8 +44,8 @@ def mandatory_tour_frequency(persons_merged,
     choosers = choosers[choosers.cdap_activity == 'M']
     logger.info("Running mandatory_tour_frequency with %d persons" % len(choosers))
 
-    nest_spec = get_logit_model_settings(mandatory_tour_frequency_settings)
-    constants = get_model_constants(mandatory_tour_frequency_settings)
+    nest_spec = config.get_logit_model_settings(mandatory_tour_frequency_settings)
+    constants = config.get_model_constants(mandatory_tour_frequency_settings)
 
     choices = asim.simple_simulate(
         choosers,
@@ -65,7 +63,7 @@ def mandatory_tour_frequency(persons_merged,
     tracing.print_summary('mandatory_tour_frequency', choices, value_counts=True)
 
     orca.add_column("persons", "mandatory_tour_frequency", choices)
-    add_dependent_columns("persons", "persons_mtf")
+    pipeline.add_dependent_columns("persons", "persons_mtf")
 
     create_mandatory_tours_table()