From 8e1fbcd54475bb8db89a1a1fbb987579d1ec8fe2 Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Thu, 18 Apr 2024 23:21:41 -0400 Subject: [PATCH] BayDAG All Contributions (#834) * get all disaggregate accessibility values * updated settings to work with Pydantic * Squashed commit of the following: commit 1670dc96cb57a85c9e35a2dffc08d0e79b0f92d5 Author: David Hensle Date: Fri Mar 8 15:53:37 2024 -0800 updated settings to work with Pydantic commit ea25a669d339485c6975569769c85e8ef9f05d25 Author: David Hensle Date: Fri Dec 15 16:30:41 2023 -0800 get all disaggregate accessibility values * Squashed commit of the following: commit d05f77c76602d41c28041776865f12cbb536a6c4 Author: David Hensle Date: Mon Mar 11 15:00:28 2024 -0700 fixing person_rank call commit f252006a94ab14acd65aab303be9a6274c7b981d Author: David Hensle Date: Fri Dec 15 16:35:37 2023 -0800 CDAP estimation with joint component * Squashed commit of the following: commit 90b0b8c26b64bb72b5dffdd251df48b3618ec563 Author: David Hensle Date: Fri Mar 8 14:05:43 2024 -0800 Only want joint tours for joint tour dest model -- got dropped in rebase * Squashed commit of the following: commit 807b0af22b796674971d58bbb9ff689fb19771cc Author: David Hensle Date: Fri Dec 15 16:40:26 2023 -0800 Alt name for jtfc estimation mode * Squashed commit of the following: commit 5518ca4584050995d7e59b8eed27bef8a8877f3f Author: David Hensle Date: Fri Dec 15 16:41:59 2023 -0800 error out of infinite loop * Squashed commit of the following: commit 0e6919bb0b3339651aff442234e56c6a4fa11bd8 Author: David Hensle Date: Fri Mar 8 13:23:13 2024 -0800 blacken commit 807106866d0c3bfdadc670f644686e72df8cdc26 Author: David Hensle Date: Fri Dec 15 16:59:32 2023 -0800 sampling in EDB for location choice commit 150d593777df5345a930dd468084d08fa8fbd622 Author: David Hensle Date: Fri Dec 15 16:59:32 2023 -0800 sampling in EDB for location choice * Squashed commit of the following: commit 48be6f4f567e27117379766fa33ce8ace8b334f6 Author: David Hensle Date: Thu Mar 7 13:56:38 2024 -0800 replace orca inject with state object commit 4201d36ed7678a844a8a7dd3c904edef4620398f Author: David Hensle Date: Fri Dec 15 17:44:06 2023 -0800 landuse available in trip destination commit 0630659b71df02d1e16cc643dbf1c9cf8c56282a Author: David Hensle Date: Fri Dec 15 17:02:59 2023 -0800 landuse and reindex in loc choice * Squashed commit of the following: commit d1f4db82fb6f10d36ecb2cbbb773bd43ac9bdac5 Author: David Hensle Date: Fri Mar 8 15:09:38 2024 -0800 call ALT_DEST_COL_NAME as attribute commit fe0c284f9d1edc2a4219e26aff77325214c0544d Author: David Hensle Date: Fri Mar 8 14:11:35 2024 -0800 fix Nones to None typo commit b955295166836374d37d9e59618cf22e54af54e7 Author: David Hensle Date: Mon Dec 18 11:56:54 2023 -0800 blacken commit cdcca9185c5e3d057dd7ba0afef7e94134d01dd6 Author: David Hensle Date: Fri Dec 15 17:12:02 2023 -0800 mode choice logsum extraction * Squashed commit of the following: commit 504b974574e62683621e695aa4f10b5364f15cda Merge: ac69af88 a40fa16a Author: David Hensle Date: Mon Mar 18 09:21:05 2024 -0700 Merge branch 'BayDAG_cont10_nmtf_avail_time_windows' of https://github.com/SANDAG/activitysim into BayDAG_cont10_nmtf_avail_time_windows commit ac69af88348b4dd790745930c3fe109f91d589fe Author: David Hensle Date: Fri Mar 8 14:15:31 2024 -0800 remove bad path to annotate.py commit a40fa16a15aac8240d4b69ef2dcd834fcecdc29c Author: David Hensle Date: Fri Mar 8 14:15:31 2024 -0800 remove bad path to annotate.py commit 5a54dc5b0fe7a45eb51d4b367ded7924a696cf2f Merge: 481238b3 79980b86 Author: David Hensle Date: Fri Mar 8 13:38:45 2024 -0800 Merge branch 'BayDAG_cont10_nmtf_avail_time_windows' of https://github.com/SANDAG/activitysim into BayDAG_cont10_nmtf_avail_time_windows commit 481238b30e176c5be3778885602788aaf413eb43 Author: David Hensle Date: Fri Mar 8 13:37:37 2024 -0800 blacken commit 3aa3f46e914aa51163646f88cf9058cb4f40696a Author: David Hensle Date: Fri Dec 15 17:16:51 2023 -0800 NMTF person available periods commit 79980b86397e7283598e1b040abd1e30e75693f7 Author: David Hensle Date: Fri Dec 15 17:16:51 2023 -0800 NMTF person available periods * Squashed commit of the following: commit c4394056409f22b5f916fd8da2ba4088f9ee6185 Author: David Hensle Date: Mon Mar 18 10:05:23 2024 -0700 blacken commit d9392f3899177f4a91a7275e974a425cc7441ec8 Author: David Hensle Date: Mon Mar 18 09:33:09 2024 -0700 adding missed columns necessary for no school escorting commit 4470dd4d8fc7f289183e7b0e8654694934519ae7 Author: David Hensle Date: Mon Mar 11 14:10:42 2024 -0700 updating to work with Pydantic and State object commit cf0a663e293d6d8eae174b3b7c1c31f0b7a4db38 Author: David Hensle Date: Fri Mar 8 13:48:42 2024 -0800 blacken commit 5c1524f2a42c1f088238e0aaf38ca83f4b58cc0b Author: David Hensle Date: Fri Dec 15 17:42:15 2023 -0800 School escorting estimation updates Most changes are needed to avoid crash if estimation run actually had no school escorting in the input data * Squashed commit of the following: commit f30a9ea28ffd3839543688209185ecee15362bb3 Author: David Hensle Date: Tue Mar 12 10:28:11 2024 -0700 settings in write_omnibus_table commit e8144af44f70b15880c1b12ea651180d44d7c9d1 Author: David Hensle Date: Fri Dec 15 17:59:19 2023 -0800 comments allowed in stop freq spec commit ece02a884809fad04ed6938337d54aba91502bce Author: David Hensle Date: Fri Dec 15 17:53:12 2023 -0800 estimation mode usability * Squashed commit of the following: commit 74212e4a64d69e86618b40b6b2f04e84ced71e00 Author: David Hensle Date: Tue Mar 12 14:56:17 2024 -0700 nmtf larch av arg commit 6a3a3322632adfde8429d074b25db251a35d8d75 Author: David Hensle Date: Fri Dec 15 17:54:47 2023 -0800 decrease larch load time for nmtf * Squashed commit of the following: commit 2b248ce54539ec20d3f5b10772d8c773790824c9 Author: David Hensle Date: Mon Dec 18 13:13:23 2023 -0800 blacken commit d8af5b8d7cdac89e8e8c9d1a698b60263d776f94 Author: David Hensle Date: Fri Dec 15 17:58:28 2023 -0800 adding calls to larch for new simple simulate models * KEEP_COLS setting * working with sandag abm3 example * adding required settings in cdap test * Default value for KEEP_COLS * parking locations in write trip matrices * avoiding NaN in transit_pass_subsidy result * fix bug in which joint utilities not in probabilities cdap * putting origin and destination rewrite in correct location * blacken * removing arc test * transit pass subsidy filter_col bug fix * comment for persons_merged issue 853 --- activitysim/abm/models/auto_ownership.py | 5 +++- activitysim/abm/models/cdap.py | 2 ++ activitysim/abm/models/initialize.py | 4 +++ .../models/non_mandatory_tour_frequency.py | 3 ++ .../abm/models/transit_pass_subsidy.py | 13 +++++++-- activitysim/abm/models/util/annotate.py | 2 ++ activitysim/abm/models/util/cdap.py | 29 +++++++------------ .../abm/models/util/test/configs/cdap.yaml | 3 ++ activitysim/abm/tables/persons.py | 3 +- activitysim/core/configuration/logit.py | 4 +-- 10 files changed, 43 insertions(+), 25 deletions(-) diff --git a/activitysim/abm/models/auto_ownership.py b/activitysim/abm/models/auto_ownership.py index eefaa40a0..1f6f84648 100644 --- a/activitysim/abm/models/auto_ownership.py +++ b/activitysim/abm/models/auto_ownership.py @@ -37,6 +37,8 @@ def auto_ownership_simulate( state: workflow.State, households: pd.DataFrame, households_merged: pd.DataFrame, + # FIXME: persons_merged not used but included, see #853 + persons_merged: pd.DataFrame, model_settings: AutoOwnershipSettings | None = None, model_settings_file_name: str = "auto_ownership.yaml", trace_label: str = "auto_ownership_simulate", @@ -76,6 +78,7 @@ def auto_ownership_simulate( locals_d.update(constants) expressions.assign_columns( + state, df=choosers, model_settings=preprocessor_settings, locals_dict=locals_d, @@ -119,7 +122,7 @@ def auto_ownership_simulate( ) if model_settings.annotate_households: - annotate.annotate_households(model_settings, trace_label) + annotate.annotate_households(state, model_settings, trace_label) if trace_hh_id: state.tracing.trace_df(households, label="auto_ownership", warn_if_empty=True) diff --git a/activitysim/abm/models/cdap.py b/activitysim/abm/models/cdap.py index c99817fde..6776c06c7 100644 --- a/activitysim/abm/models/cdap.py +++ b/activitysim/abm/models/cdap.py @@ -34,6 +34,8 @@ class CdapSettings(PydanticReadable, extra="forbid"): FIXED_RELATIVE_PROPORTIONS_SPEC: str = "cdap_fixed_relative_proportions.csv" ADD_JOINT_TOUR_UTILITY: bool = False JOINT_TOUR_COEFFICIENTS: str = "cdap_joint_tour_coefficients.csv" + JOINT_TOUR_USEFUL_COLUMNS: list[str] | None = None + """Columns to include from the persons table that will be need to calculate household joint tour utility.""" annotate_persons: PreprocessorSettings | None = None annotate_households: PreprocessorSettings | None = None COEFFICIENTS: Path diff --git a/activitysim/abm/models/initialize.py b/activitysim/abm/models/initialize.py index 8e0c758c0..af1ba079b 100644 --- a/activitysim/abm/models/initialize.py +++ b/activitysim/abm/models/initialize.py @@ -178,6 +178,10 @@ def initialize_households( suffixes = disaggregate_accessibility.disaggregate_suffixes(state) shadow_pricing.add_size_tables(state, suffixes) + # create disaggregate_accessibility table if not model was run + if state.is_table("proto_disaggregate_accessibility"): + disaggregate_accessibility.disaggregate_accessibility(state) + # - preload person_windows person_windows = state.get_dataframe("person_windows") chunk_sizer.log_df(trace_label, "person_windows", person_windows) diff --git a/activitysim/abm/models/non_mandatory_tour_frequency.py b/activitysim/abm/models/non_mandatory_tour_frequency.py index f43a523f0..5ec2c9407 100644 --- a/activitysim/abm/models/non_mandatory_tour_frequency.py +++ b/activitysim/abm/models/non_mandatory_tour_frequency.py @@ -334,6 +334,9 @@ def non_mandatory_tour_frequency( choices_list.append(choices) + # FIXME only want to keep actual purposes, adding cols in alts will mess this up + # this is complicated by canonical_ids calculated based on alts if not specified explicitly + # thus, adding column to input alts will change IDs and break estimation mode.... del alternatives["tot_tours"] # del tot_tours column we added above # The choice value 'non_mandatory_tour_frequency' assigned by interaction_simulate diff --git a/activitysim/abm/models/transit_pass_subsidy.py b/activitysim/abm/models/transit_pass_subsidy.py index e21be89f9..0f71279cd 100644 --- a/activitysim/abm/models/transit_pass_subsidy.py +++ b/activitysim/abm/models/transit_pass_subsidy.py @@ -28,6 +28,9 @@ class TransitPassSubsidySettings(LogitComponentSettings, extra="forbid"): preprocessor: PreprocessorSettings | None = None """Setting for the preprocessor.""" + CHOOSER_FILTER_COLUMN_NAME: str | None = None + """Column name which selects choosers. If None, all persons are choosers.""" + @workflow.step def transit_pass_subsidy( @@ -48,7 +51,6 @@ def transit_pass_subsidy( ) choosers = persons_merged - logger.info("Running %s with %d persons", trace_label, len(choosers)) estimator = estimation.manager.begin_estimation(state, "transit_pass_subsidy") @@ -69,6 +71,11 @@ def transit_pass_subsidy( trace_label=trace_label, ) + filter_col = model_settings.CHOOSER_FILTER_COLUMN_NAME + if filter_col is not None: + choosers = choosers[choosers[filter_col]] + logger.info("Running %s with %d persons", trace_label, len(choosers)) + model_spec = state.filesystem.read_model_spec(model_settings.SPEC) coefficients_df = state.filesystem.read_model_coefficients(model_settings) model_spec = simulate.eval_coefficients( @@ -103,7 +110,9 @@ def transit_pass_subsidy( estimator.write_override_choices(choices) estimator.end_estimation() - persons["transit_pass_subsidy"] = choices.reindex(persons.index) + persons["transit_pass_subsidy"] = ( + choices.reindex(persons.index).fillna(0).astype(int) + ) state.add_table("persons", persons) diff --git a/activitysim/abm/models/util/annotate.py b/activitysim/abm/models/util/annotate.py index 69958b1ab..5c0f62821 100644 --- a/activitysim/abm/models/util/annotate.py +++ b/activitysim/abm/models/util/annotate.py @@ -36,6 +36,7 @@ def annotate_households( locals_dict = {} households = state.get_dataframe("households") expressions.assign_columns( + state, df=households, model_settings=model_settings.get("annotate_households"), locals_dict=locals_dict, @@ -64,6 +65,7 @@ def annotate_persons( locals_dict = {} persons = state.get_dataframe("persons") expressions.assign_columns( + state, df=persons, model_settings=model_settings.get("annotate_persons"), locals_dict=locals_dict, diff --git a/activitysim/abm/models/util/cdap.py b/activitysim/abm/models/util/cdap.py index 635d37a82..f52713ebf 100644 --- a/activitysim/abm/models/util/cdap.py +++ b/activitysim/abm/models/util/cdap.py @@ -221,8 +221,10 @@ def individual_utilities( indiv_utils[useful_columns] = persons[useful_columns] # add attributes for joint tour utility - model_settings = state.filesystem.read_model_settings("cdap.yaml") - additional_useful_columns = model_settings.get("JOINT_TOUR_USEFUL_COLUMNS", None) + from activitysim.abm.models.cdap import CdapSettings + + model_settings = CdapSettings.read_settings_file(state.filesystem, "cdap.yaml") + additional_useful_columns = model_settings.JOINT_TOUR_USEFUL_COLUMNS if additional_useful_columns is not None: indiv_utils[additional_useful_columns] = persons[additional_useful_columns] @@ -850,8 +852,10 @@ def hh_choosers(state: workflow.State, indiv_utils, hhsize): merge_cols = [_hh_id_, _ptype_, "M", "N", "H"] # add attributes for joint tour utility - model_settings = state.filesystem.read_model_settings("cdap.yaml") - additional_merge_cols = model_settings.get("JOINT_TOUR_USEFUL_COLUMNS", None) + from activitysim.abm.models.cdap import CdapSettings + + model_settings = CdapSettings.read_settings_file(state.filesystem, "cdap.yaml") + additional_merge_cols = model_settings.JOINT_TOUR_USEFUL_COLUMNS if additional_merge_cols is not None: merge_cols.extend(additional_merge_cols) @@ -972,7 +976,6 @@ def household_activity_choices( if len(utils.index) == 0: return pd.Series(dtype="float64") - probs = logit.utils_to_probs(state, utils, trace_label=trace_label) # calculate joint tour utility if add_joint_tour_utility & (hhsize > 1): # calculate joint utils @@ -996,6 +999,8 @@ def household_activity_choices( # add joint util to util utils = utils.add(joint_tour_utils) + probs = logit.utils_to_probs(state, utils, trace_label=trace_label) + # select an activity pattern alternative for each household based on probability # result is a series indexed on _hh_index_ with the (0 based) index of the column from probs idx_choices, rands = logit.make_choices(state, probs, trace_label=trace_label) @@ -1278,20 +1283,6 @@ def _run_cdap( persons["cdap_activity"] = person_choices chunk_sizer.log_df(trace_label, "persons", persons) - # return household joint tour flag - if add_joint_tour_utility: - hh_activity_choices = hh_activity_choices.to_frame(name="hh_choices") - hh_activity_choices["has_joint_tour"] = hh_activity_choices["hh_choices"].apply( - lambda x: 1 if "J" in x else 0 - ) - - # return household joint tour flag - if add_joint_tour_utility: - hh_activity_choices = hh_activity_choices.to_frame(name="hh_choices") - hh_activity_choices["has_joint_tour"] = hh_activity_choices["hh_choices"].apply( - lambda x: 1 if "J" in x else 0 - ) - # return household joint tour flag if add_joint_tour_utility: hh_activity_choices = hh_activity_choices.to_frame(name="hh_choices") diff --git a/activitysim/abm/models/util/test/configs/cdap.yaml b/activitysim/abm/models/util/test/configs/cdap.yaml index f20d2979a..e2efdfa88 100644 --- a/activitysim/abm/models/util/test/configs/cdap.yaml +++ b/activitysim/abm/models/util/test/configs/cdap.yaml @@ -6,3 +6,6 @@ PERSON_TYPE_MAP: - 6 - 7 - 8 + +INDIV_AND_HHSIZE1_SPEC: cdap_indiv_and_hhsize1.csv +COEFFICIENTS: cdap_coefficients.csv \ No newline at end of file diff --git a/activitysim/abm/tables/persons.py b/activitysim/abm/tables/persons.py index 8825ebe84..d5ab67fb5 100644 --- a/activitysim/abm/tables/persons.py +++ b/activitysim/abm/tables/persons.py @@ -99,7 +99,8 @@ def persons_merged( households, left_on="household_id", ) - if disaggregate_accessibility is not None and not disaggregate_accessibility.empty: + if state.is_table("disaggregate_accessibility"): + disaggregate_accessibility = state.get_table("disaggregate_accessibility") persons = simple_table_join( persons, disaggregate_accessibility, diff --git a/activitysim/core/configuration/logit.py b/activitysim/core/configuration/logit.py index 22e14e49f..9c93d9f97 100644 --- a/activitysim/core/configuration/logit.py +++ b/activitysim/core/configuration/logit.py @@ -212,7 +212,7 @@ class TourLocationComponentSettings(LocationComponentSettings, extra="forbid"): DEST_CHOICE_SAMPLE_TABLE_NAME: str | None = None CHOOSER_TABLE_NAME: str | None = None CHOOSER_SEGMENT_COLUMN_NAME: str | None = None - SEGMENT_IDS: dict[str, int] | None = None + SEGMENT_IDS: dict[str, int] | dict[str, str] | dict[str, bool] | None = None SHADOW_PRICE_TABLE: str | None = None MODELED_SIZE_TABLE: str | None = None annotate_persons: PreprocessorSettings | None = None @@ -220,7 +220,7 @@ class TourLocationComponentSettings(LocationComponentSettings, extra="forbid"): SIMULATE_CHOOSER_COLUMNS: list[str] | None = None ALT_DEST_COL_NAME: str LOGSUM_TOUR_PURPOSE: str | dict[str, str] | None = None - MODEL_SELECTOR: Literal["workplace", "school", None] = None + MODEL_SELECTOR: str | None = None SAVED_SHADOW_PRICE_TABLE_NAME: str | None = None CHOOSER_ID_COLUMN: str = "person_id"