auto offest_preprocessing detection, ARC tests

activitysim/core/input.py

@@ -167,6 +167,11 @@ def read_from_table_info(table_info):
                 df[colname] = df[colname].apply(remapper.get)
                 if keep_columns:
                     keep_columns.append(f"_original_{colname}")
+                if tablename == 'land_use' and colname == canonical_index_col:
+                    # We need to keep track if we have recoded the land_use
+                    # table's index to zero-based, as we need to disable offset
+                    # processing for legacy skim access.
+                    config.override_setting("offset_preprocessing", True)
             else:
                 source_table, lookup_col = recode_instruction.split(".")
                 parent_table = inject.get_table(source_table)

activitysim/examples/example_arc/test/configs/settings.yaml

@@ -38,3 +38,5 @@ output_tables:
   sort: True
   tables:
     - trips
+
+recode_pipeline_columns: False

activitysim/examples/example_arc/test/configs_recode/network_los.yaml

@@ -0,0 +1,6 @@
+inherit_settings: True
+
+# read cached skims (using numpy memmap) from output directory (memmap is faster than omx )
+read_skim_cache: False
+# write memmapped cached skims to output directory after reading from omx, for use in subsequent runs
+write_skim_cache: False

activitysim/examples/example_arc/test/configs_recode/settings.yaml

@@ -0,0 +1,30 @@
+inherit_settings: True
+
+# EXAMPLE_HAS_WARNINGS
+strict: False
+
+# number of households to simulate
+households_sample_size:  10
+chunk_size: 0
+
+# - shadow pricing global switches
+use_shadow_pricing: False
+
+# turn writing of sample_tables on and off for all models
+# (if True, tables will be written if DEST_CHOICE_SAMPLE_TABLE_NAME is specified in individual model settings)
+want_dest_choice_sample_tables: False
+
+cleanup_pipeline_after_run: True
+
+output_tables:
+  h5_store: False
+  action: include
+  prefix: final_
+  sort: True
+  tables:
+    - tablename: trips
+      decode_columns:
+        origin: land_use.zone_id
+        destination: land_use.zone_id
+
+recode_pipeline_columns: True

activitysim/examples/example_arc/test/configs_sharrow/network_los.yaml

@@ -0,0 +1,6 @@
+inherit_settings: True
+
+# read cached skims (using numpy memmap) from output directory (memmap is faster than omx )
+read_skim_cache: False
+# write memmapped cached skims to output directory after reading from omx, for use in subsequent runs
+write_skim_cache: False

activitysim/examples/example_arc/test/configs_sharrow/settings.yaml

@@ -0,0 +1,31 @@
+inherit_settings: True
+
+# EXAMPLE_HAS_WARNINGS
+strict: False
+
+# number of households to simulate
+households_sample_size:  10
+chunk_size: 0
+
+# - shadow pricing global switches
+use_shadow_pricing: False
+
+# turn writing of sample_tables on and off for all models
+# (if True, tables will be written if DEST_CHOICE_SAMPLE_TABLE_NAME is specified in individual model settings)
+want_dest_choice_sample_tables: False
+
+cleanup_pipeline_after_run: True
+
+output_tables:
+  h5_store: False
+  action: include
+  prefix: final_
+  sort: True
+  tables:
+    - tablename: trips
+      decode_columns:
+        origin: land_use.zone_id
+        destination: land_use.zone_id
+
+sharrow: require
+recode_pipeline_columns: True

activitysim/examples/example_arc/test/test_arc.py

@@ -2,10 +2,10 @@
 # See full license in LICENSE.txt.
 import os
 import subprocess
-import pkg_resources
 
 import pandas as pd
 import pandas.testing as pdt
+import pkg_resources
 
 from activitysim.core import inject
 
@@ -15,7 +15,7 @@ def teardown_function(func):
     inject.reinject_decorated_tables()
 
 
-def test_arc():
+def _test_arc(recode=False, sharrow=False):
     def example_path(dirname):
         resource = os.path.join("examples", "example_arc", dirname)
         return pkg_resources.resource_filename("activitysim", resource)
@@ -34,12 +34,30 @@ def regress():
 
     file_path = os.path.join(os.path.dirname(__file__), "simulation.py")
 
-    subprocess.run(
-        [
-            "coverage",
-            "run",
-            "-a",
-            file_path,
+    if recode:
+        run_args = [
+            "-c",
+            test_path("configs_recode"),
+            "-c",
+            example_path("configs"),
+            "-d",
+            example_path("data"),
+            "-o",
+            test_path("output"),
+        ]
+    elif sharrow:
+        run_args = [
+            "-c",
+            test_path("configs_sharrow"),
+            "-c",
+            example_path("configs"),
+            "-d",
+            example_path("data"),
+            "-o",
+            test_path("output"),
+        ]
+    else:
+        run_args = [
             "-c",
             test_path("configs"),
             "-c",
@@ -48,13 +66,27 @@ def regress():
             example_path("data"),
             "-o",
             test_path("output"),
-        ],
-        check=True,
-    )
+        ]
+
+    subprocess.run(["coverage", "run", "-a", file_path] + run_args, check=True)
 
     regress()
 
 
+def test_arc():
+    _test_arc()
+
+
+def test_arc_recode():
+    _test_arc(recode=True)
+
+
+def test_arc_sharrow():
+    _test_arc(sharrow=True)
+
+
 if __name__ == "__main__":
 
-    test_arc()
+    _test_arc()
+    _test_arc(recode=True)
+    _test_arc(sharrow=True)

activitysim/examples/example_mtc/configs/settings.yaml

@@ -263,4 +263,3 @@ household_median_value_of_time:
   3: 10.44
   4: 12.86
 
-offset_preprocessing: True  # set to true if home TAZ's are recoded to zero-based

activitysim/examples/example_mtc/test/configs/settings.yaml

@@ -25,4 +25,3 @@ output_tables:
     - trips
 
 recode_pipeline_columns: False
-offset_preprocessing: False

activitysim/examples/example_mtc/test/configs_chunkless/settings.yaml

@@ -26,4 +26,3 @@ output_tables:
     - trips
 
 recode_pipeline_columns: False
-offset_preprocessing: False

activitysim/examples/example_mtc/test/configs_mp/settings.yaml

@@ -31,4 +31,3 @@ output_tables:
     - trips
 
 recode_pipeline_columns: False
-offset_preprocessing: False

activitysim/examples/example_mtc/test/configs_recode/settings.yaml

@@ -28,4 +28,3 @@ output_tables:
         destination: land_use.zone_id
 
 recode_pipeline_columns: True
-offset_preprocessing: True

activitysim/examples/example_mtc/test/configs_sharrow/settings.yaml

@@ -29,4 +29,3 @@ output_tables:
 
 sharrow: require
 recode_pipeline_columns: True
-offset_preprocessing: True

activitysim/examples/example_sandag/configs_1_zone/settings.yaml

@@ -191,4 +191,3 @@ household_median_value_of_time:
   3: 10.44
   4: 12.86
 
-offset_preprocessing: True  # set to true if home TAZ's are recoded to zero-based

activitysim/examples/example_semcog/configs/settings.yaml

@@ -173,4 +173,3 @@ household_median_value_of_time:
   3: 10.44
   4: 12.86
 
-offset_preprocessing: True  # set to true if home TAZ's are recoded to zero-based

activitysim/workflows/sharrow-contrast/_contrast_runner.yaml

@@ -164,7 +164,6 @@ run-legacy:
       payload:
         inherit_settings: True
         recode_pipeline_columns: False
-        offset_preprocessing: False
         cache_dir: cache
         households_sample_size: '{main_n_households}'
         trace_hh_id: '{trace_hh_id}'