BayDAG Contribution #13: Estimation Mode Usability (#779)

* estimation mode usability * comments allowed in stop freq spec * settings in write_omnibus_table
ActivitySim · Apr 1, 2024 · 0a119ad · 0a119ad
1 parent 216c81a
commit 0a119ad
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 6 deletions.
diff --git a/activitysim/abm/models/stop_frequency.py b/activitysim/abm/models/stop_frequency.py
@@ -275,7 +275,7 @@ def stop_frequency(
             print(f"survey_trips_not_in_trips\n{survey_trips_not_in_trips}")
             different = True
         trips_not_in_survey_trips = trips[~trips.index.isin(survey_trips.index)]
-        if len(survey_trips_not_in_trips) > 0:
+        if len(trips_not_in_survey_trips) > 0:
             print(f"trips_not_in_survey_trips\n{trips_not_in_survey_trips}")
             different = True
         assert not different

diff --git a/activitysim/core/estimation.py b/activitysim/core/estimation.py
@@ -220,6 +220,15 @@ def write_omnibus_table(self):
         if len(self.omnibus_tables) == 0:
             return
 
+        settings = self.state.filesystem.read_model_settings(
+            ESTIMATION_SETTINGS_FILE_NAME, mandatory=False
+        )
+
+        edbs_to_skip = settings.get("SKIP_BUNDLE_WRITE_FOR", [])
+        if self.bundle_name in edbs_to_skip:
+            self.debug(f"Skipping write to disk for {self.bundle_name}")
+            return
+
         for omnibus_table, table_names in self.omnibus_tables.items():
             self.debug(
                 "write_omnibus_table: %s table_names: %s" % (omnibus_table, table_names)
@@ -237,12 +246,19 @@ def write_omnibus_table(self):
                 1 if omnibus_table in self.omnibus_tables_append_columns else 0
             )
 
-            df = pd.concat([self.tables[t] for t in table_names], axis=concat_axis)
+            if len(table_names) == 0:
+                # empty tables
+                df = pd.DataFrame()
+            else:
+                df = pd.concat([self.tables[t] for t in table_names], axis=concat_axis)
+
+            self.debug(f"sorting tables: {table_names}")
             df.sort_index(ascending=True, inplace=True, kind="mergesort")
 
             file_path = self.output_file_path(omnibus_table, "csv")
             assert not os.path.isfile(file_path)
 
+            self.debug(f"writing table: {file_path}")
             df.to_csv(file_path, mode="a", index=True, header=True)
 
             self.debug("write_omnibus_choosers: %s" % file_path)

diff --git a/activitysim/estimation/larch/location_choice.py b/activitysim/estimation/larch/location_choice.py
@@ -144,6 +144,9 @@ def _file_exists(filename):
         .set_index("segment")
     )
     size_spec = size_spec.loc[:, size_spec.max() > 0]
+    assert (
+        len(size_spec) > 0
+    ), f"Empty size_spec, is model_selector {SIZE_TERM_SELECTOR} in your size term file?"
 
     size_coef = size_coefficients_from_spec(size_spec)
 
@@ -294,6 +297,9 @@ def split(a, n):
     else:
         av = 1
 
+    assert len(x_co) > 0, "Empty chooser dataframe"
+    assert len(x_ca_1) > 0, "Empty alternatives dataframe"
+
     d = DataFrames(co=x_co, ca=x_ca_1, av=av)
 
     m = Model(dataservice=d)

diff --git a/activitysim/estimation/larch/stop_frequency.py b/activitysim/estimation/larch/stop_frequency.py
@@ -42,8 +42,7 @@ def stop_frequency_data(
         seg_purpose = seg_["primary_purpose"]
         seg_subdir = Path(os.path.join(edb_directory, seg_purpose))
         segment_coef[seg_["primary_purpose"]] = pd.read_csv(
-            seg_subdir / seg_["COEFFICIENTS"],
-            index_col="coefficient_name",
+            seg_subdir / seg_["COEFFICIENTS"], index_col="coefficient_name", comment="#"
         )
 
     for seg in segments:
@@ -89,13 +88,13 @@ def stop_frequency_data(
         seg_purpose = seg["primary_purpose"]
         seg_subdir = Path(os.path.join(edb_directory, seg_purpose))
         coeffs_ = pd.read_csv(
-            seg_subdir / seg["COEFFICIENTS"], index_col="coefficient_name"
+            seg_subdir / seg["COEFFICIENTS"], index_col="coefficient_name", comment="#"
         )
         coeffs_.index = pd.Index(
             [f"{i}_{seg_purpose}" for i in coeffs_.index], name="coefficient_name"
         )
         seg_coefficients.append(coeffs_)
-        spec = pd.read_csv(seg_subdir / "stop_frequency_SPEC_.csv")
+        spec = pd.read_csv(seg_subdir / "stop_frequency_SPEC_.csv", comment="#")
         spec = remove_apostrophes(spec, ["Label"])
         # spec.iloc[:, 3:] = spec.iloc[:, 3:].applymap(lambda x: f"{x}_{seg_purpose}" if not pd.isna(x) else x)
         seg_spec.append(spec)