Develop popsim mp support

activitysim/abm/models/accessibility.py

@@ -19,7 +19,7 @@
 
 
 @inject.step()
-def compute_accessibility(accessibility, network_los, land_use, trace_od):
+def compute_accessibility(land_use, accessibility, network_los, trace_od):
 
     """
     Compute accessibility for each zone in land use file using expressions from accessibility_spec
@@ -42,6 +42,9 @@ def compute_accessibility(accessibility, network_los, land_use, trace_od):
     assignment_spec = assign.read_assignment_spec(config.config_file_path('accessibility.csv'))
 
     accessibility_df = accessibility.to_frame()
+    if len(accessibility_df.columns) > 0:
+        logger.warning(f"accessibility table is not empty. Columns:{list(accessibility_df.columns)}")
+        raise RuntimeError(f"accessibility table is not empty.")
 
     constants = config.get_model_constants(model_settings)
 

activitysim/abm/models/cdap.py

@@ -34,7 +34,8 @@ def cdap_simulate(persons_merged, persons, households,
 
     trace_label = 'cdap'
     model_settings = config.read_model_settings('cdap.yaml')
-    person_type_map = model_settings.get('PERSON_TYPE_MAP', {})
+    person_type_map = model_settings.get('PERSON_TYPE_MAP', None)
+    assert person_type_map is not None, f"Expected to find PERSON_TYPE_MAP setting in cdap.yaml"
     cdap_indiv_spec = simulate.read_model_spec(file_name=model_settings['INDIV_AND_HHSIZE1_SPEC'])
 
     # Rules and coefficients for generating interaction specs for different household sizes

activitysim/abm/models/initialize.py

@@ -91,10 +91,8 @@ def initialize_landuse():
 
     annotate_tables(model_settings, trace_label)
 
-    # create accessibility (only required if multiprocessing wants to slice accessibility)
-    land_use = pipeline.get_table('land_use')
-    accessibility_df = pd.DataFrame(index=land_use.index)
-    pipeline.replace_table("accessibility", accessibility_df)
+    # instantiate accessibility (must be checkpointed to be be used to slice accessibility)
+    accessibility = pipeline.get_table('accessibility')
 
 
 @inject.step()

activitysim/abm/tables/__init__.py

@@ -3,6 +3,7 @@
 from . import households
 from . import persons
 from . import landuse
+from . import accessibility
 from . import skims
 from . import tours
 from . import size_terms

activitysim/abm/tables/accessibility.py

@@ -0,0 +1,39 @@
+# ActivitySim
+# See full license in LICENSE.txt.
+import logging
+
+import pandas as pd
+
+from activitysim.core import inject
+from activitysim.core.input import read_input_table
+
+logger = logging.getLogger(__name__)
+
+
+@inject.table()
+def accessibility(land_use):
+    """
+    If 'accessibility' is in input_tables list, then read it in,
+    otherwise create skeleton table with same index as landuse.
+
+    This allows loading of pre-computed accessibility table, which is particularly useful
+    for single-process small household sample runs when there are many zones in landuse
+
+    skeleton table only required if multiprocessing wants to slice accessibility,
+    otherwise it will simply be replaced when accessibility model is run
+    """
+
+    accessibility_df = read_input_table("accessibility", required=False)
+
+    if accessibility_df is None:
+        accessibility_df = pd.DataFrame(index=land_use.index)
+        logger.info("created placeholder accessibility table %s" % (accessibility_df.shape,))
+    else:
+        assert accessibility_df.sort_index().index.equals(land_use.to_frame().sort_index().index), \
+            f"loaded accessibility table index does not match index of land_use table"
+        logger.info("loaded land_use %s" % (accessibility_df.shape,))
+
+    # replace table function with dataframe
+    inject.add_table('accessibility', accessibility_df)
+
+    return accessibility_df

activitysim/abm/tables/landuse.py

@@ -2,6 +2,8 @@
 # See full license in LICENSE.txt.
 import logging
 
+import pandas as pd
+
 from activitysim.core import inject
 from activitysim.core.input import read_input_table
 

activitysim/abm/tables/shadow_pricing.py

@@ -566,50 +566,6 @@ def block_name(model_selector):
     return model_selector
 
 
-def get_shadow_pricing_info():
-    """
-    return dict with info about dtype and shapes of desired and modeled size tables
-
-    block shape is (num_zones, num_segments + 1)
-
-
-    Returns
-    -------
-    shadow_pricing_info: dict
-        dtype: <sp_dtype>,
-        block_shapes: dict {<model_selector>: <block_shape>}
-    """
-
-    land_use = inject.get_table('land_use')
-    size_terms = inject.get_injectable('size_terms')
-
-    shadow_settings = config.read_model_settings('shadow_pricing.yaml')
-
-    # shadow_pricing_models is dict of {<model_selector>: <model_name>}
-    shadow_pricing_models = shadow_settings.get('shadow_pricing_models', {})
-
-    blocks = OrderedDict()
-    for model_selector in shadow_pricing_models:
-
-        sp_rows = len(land_use)
-        sp_cols = len(size_terms[size_terms.model_selector == model_selector])
-
-        # extra tally column for TALLY_CHECKIN and TALLY_CHECKOUT semaphores
-        blocks[block_name(model_selector)] = (sp_rows, sp_cols + 1)
-
-    sp_dtype = np.int64
-
-    shadow_pricing_info = {
-        'dtype': sp_dtype,
-        'block_shapes': blocks,
-    }
-
-    for k in shadow_pricing_info:
-        logger.debug("shadow_pricing_info %s: %s" % (k, shadow_pricing_info.get(k)))
-
-    return shadow_pricing_info
-
-
 def buffers_for_shadow_pricing(shadow_pricing_info):
     """
     Allocate shared_data buffers for multiprocess shadow pricing
@@ -730,9 +686,7 @@ def load_shadow_price_calculator(model_settings):
 
         # - shadow_pricing_info
         shadow_pricing_info = inject.get_injectable('shadow_pricing_info', None)
-        if shadow_pricing_info is None:
-            shadow_pricing_info = get_shadow_pricing_info()
-            inject.add_injectable('shadow_pricing_info', shadow_pricing_info)
+        assert shadow_pricing_info is not None
 
         # - extract data buffer and reshape as numpy array
         data, lock = \
@@ -840,3 +794,57 @@ def add_size_tables():
             scaled_size = raw_size
 
         inject.add_table(size_table_name(model_selector), scaled_size)
+
+
+def get_shadow_pricing_info():
+    """
+    return dict with info about dtype and shapes of desired and modeled size tables
+
+    block shape is (num_zones, num_segments + 1)
+
+
+    Returns
+    -------
+    shadow_pricing_info: dict
+        dtype: <sp_dtype>,
+        block_shapes: dict {<model_selector>: <block_shape>}
+    """
+
+    land_use = inject.get_table('land_use')
+    size_terms = inject.get_injectable('size_terms')
+
+    shadow_settings = config.read_model_settings('shadow_pricing.yaml')
+
+    # shadow_pricing_models is dict of {<model_selector>: <model_name>}
+    shadow_pricing_models = shadow_settings.get('shadow_pricing_models', {})
+
+    blocks = OrderedDict()
+    for model_selector in shadow_pricing_models:
+
+        sp_rows = len(land_use)
+        sp_cols = len(size_terms[size_terms.model_selector == model_selector])
+
+        # extra tally column for TALLY_CHECKIN and TALLY_CHECKOUT semaphores
+        blocks[block_name(model_selector)] = (sp_rows, sp_cols + 1)
+
+    sp_dtype = np.int64
+
+    shadow_pricing_info = {
+        'dtype': sp_dtype,
+        'block_shapes': blocks,
+    }
+
+    for k in shadow_pricing_info:
+        logger.debug("shadow_pricing_info %s: %s" % (k, shadow_pricing_info.get(k)))
+
+    return shadow_pricing_info
+
+
+@inject.injectable(cache=True)
+def shadow_pricing_info():
+
+    # when multiprocessing with shared data mp_tasks has to call network_los methods
+    # get_shadow_pricing_info() and buffers_for_shadow_pricing()
+    logger.debug("loading shadow_pricing_info injectable")
+
+    return get_shadow_pricing_info()

activitysim/abm/tables/skims.py

@@ -5,7 +5,7 @@
 
 from activitysim.core import los
 from activitysim.core import inject
-
+from activitysim.core import config
 
 from activitysim.core.pathbuilder import TransitVirtualPathBuilder
 
@@ -39,3 +39,16 @@ def network_los(network_los_preload):
 @inject.injectable(cache=True)
 def skim_dict(network_los):
     return network_los.get_default_skim_dict()
+
+
+@inject.injectable()
+def log_settings():
+
+    # abm settings to log on startup
+    return [
+        'households_sample_size',
+        'chunk_size',
+        'multiprocess',
+        'num_processes',
+        'resume_after',
+    ]

activitysim/abm/test/data/cached_accessibility.csv

@@ -0,0 +1,26 @@
+zone_id,auPkRetail,auPkTotal,auOpRetail,auOpTotal,trPkRetail,trPkTotal,trOpRetail,trOpTotal,nmRetail,nmTotal
+2,9.316897918969254,12.61346094178568,9.304626990654363,12.604208995044406,7.511300904234601,10.950045514776265,7.427060033663283,10.76310190871255,8.142716869603856,11.724185979755147
+1,9.316494246422568,12.61517574315289,9.30743678664037,12.607849392261432,7.764263497215064,11.145248189788484,7.69308601648347,11.037285956909018,8.137360927784435,11.72624220917462
+3,9.293216972412948,12.580014459851448,9.286241654126094,12.574901923298121,7.340975248764847,10.78760844956674,7.252677859156825,10.574953610768503,8.050369141519473,11.478912551444546
+4,9.357349507455009,12.630894215587896,9.34824855856986,12.623585760796873,7.8733268245042565,11.22417118506541,7.814365228731915,11.135415935503294,8.371197500397228,11.7752306986044
+5,9.343550555725637,12.585069455568574,9.333262167175757,12.574553584249722,7.589355611211902,11.082549740994516,7.549557403733535,11.02796499800084,8.318059258671834,11.431764411398849
+6,9.271350280266805,12.523449297481994,9.265762312325014,12.519697722182595,7.313872482530061,10.504310979906819,7.068341268659587,10.25178992198404,7.83824123593578,11.023737612069544
+7,9.29319438854463,12.528401453550224,9.286372808530611,12.520416143845779,7.641910047999711,10.805002738627607,7.607878383964711,10.752509699164879,8.016914788053809,11.10880473926608
+8,9.267844254343885,12.49714602566128,9.262133093242994,12.489886062198453,7.54693384078048,10.834136328674168,7.501423798794656,10.77932009280521,7.981950488551539,11.052152831140146
+9,9.189502970549922,12.426036497501059,9.18403503221456,12.415459798818294,7.1887514829393275,10.303186204562111,7.149056558862683,10.260609520047622,7.415629777091658,10.758663414411654
+10,9.186004126767736,12.403890115167114,9.180762007450875,12.39634439278583,7.379335812398483,10.548674760263006,7.306521923298712,10.495921687099003,7.567826158130381,10.694411506898852
+11,9.320092663930184,12.519242155343258,9.315095032027857,12.511758193169229,7.455701967440773,10.87560132590152,7.348368031699475,10.762777837534243,8.228286574614595,11.171156624491507
+12,9.351590564549332,12.600777209765274,9.340287123569157,12.590072566154266,7.945965142396818,11.20437497188151,7.846325586111904,11.074533951389993,8.420517844196304,11.61897257056201
+13,9.347595785718323,12.610940363517232,9.337328586879362,12.60159046628016,7.767893952496161,11.121006481369408,7.691841587260776,11.012476509709911,8.422746480387373,11.742390105950426
+14,9.327287575225915,12.61272183638518,9.319522377122373,12.605842846570052,7.982914455823258,11.205704183942983,7.914738300790072,11.09630521314994,8.293606227119808,11.73659301352507
+15,9.284935168568785,12.581337458176352,9.27779827535493,12.57546325604843,7.656614098578269,10.9970755596348,7.574397447118308,10.914272265432203,8.00048744376541,11.541814464299403
+16,9.312158663142275,12.554715353239278,9.309851535862029,12.554250345219318,7.016153642361786,10.534220861272349,6.945206169705918,10.442447057629572,8.247303259214123,11.373742452328104
+17,9.252513241068934,12.480891476278531,9.25151293367599,12.47931539012377,6.661199575340683,9.84475302896639,6.562683873514392,9.735317917445975,7.667141682301395,10.785216325380155
+18,9.249360247756258,12.438990592515587,9.248961615230405,12.440034822845453,7.085929678533842,10.252687951615647,6.9977559941155825,10.137302150915072,7.596636098405335,10.414585329688187
+19,9.169029469573758,12.357455424191105,9.169914328186714,12.359583892671727,6.088623501170164,9.295992033901246,5.9074690665597505,9.10121719553157,7.088693449393891,9.8650009995902
+20,9.221742596618885,12.420066329040457,9.218863350318259,12.41597698216911,6.63665285998053,9.801904452298267,6.590820450001259,9.753201500136925,7.49419708276564,10.367677840955208
+21,9.321915722714706,12.515866528079302,9.325176439957023,12.518960672606589,7.4289965576337975,10.580037608031049,7.342733628487911,10.454321465284288,8.108435987481963,11.011608265602703
+22,9.229652192356095,12.543187236023256,9.219600134334192,12.535205058018846,7.150923913256277,10.713897541538804,6.941083933766704,10.463984535369464,7.637908292039161,11.319586649838495
+23,9.116149708261213,12.433017916414748,9.107848447001038,12.426054885294798,6.211644503927987,9.770780596054532,6.1182229539325945,9.687798719977543,6.887639592088898,10.656699626013532
+24,9.243796729875907,12.550974057334248,9.23008642020084,12.541349602973465,7.322741788274842,10.850763510641398,7.115121241312532,10.577146926988949,7.713627680945117,11.346711466451557
+25,9.198261982819503,12.494596320746052,9.191437063620823,12.490871930253187,7.29664609502182,10.729604503562992,7.18036563628942,10.549488538701693,7.616518018582319,11.016222759405112