Contingency plan code updates

.gitignore

@@ -18,5 +18,7 @@ configs/hlcm_renter_no_unplaced.yaml
 configs/hlcm_owner_no_unplaced.yaml
 configs/hlcm_renter_lowincome_no_unplaced.yaml
 configs/hlcm_owner_lowincome_no_unplaced.yaml
+**/*[Cc][Oo][Ss][Tt][Aa][Rr]*.csv
 .DS_Store
-site/*
+site/*
+*.twbr

README.md

@@ -1,5 +1,4 @@
 Bay Area UrbanSim (BAUS) Implementation
-=======
 
 This is the UrbanSim implementation for the Bay Area. Documentation for the UrbanSim framework is available [here](https://udst.github.io/urbansim/). All documentation for Bay Area Urbansim is at: http://bayareametro.github.io/bayarea_urbansim/main/
 

baus.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
 import os
+import pathlib
 import sys
 import time
 import traceback
@@ -68,6 +69,8 @@
 
 run_setup = orca.get_injectable("run_setup")
 run_name = orca.get_injectable("run_name")
+outputs_dir = pathlib.Path(orca.get_injectable("outputs_dir"))
+outputs_dir.mkdir(parents=True, exist_ok=True)
 
 
 def run_models(MODE):
@@ -405,6 +408,8 @@ def get_simulation_visualization_models():
 print("numpy version: %s" % np.__version__)
 print("pandas version: %s" % pd.__version__)
 
+print("SLACK: {}".format(SLACK))
+print("MODE: {}".format(MODE))
 
 if SLACK and MODE == "simulation":
     slack_start_message = f'Starting simulation {run_name} on host {host}'
@@ -433,4 +438,4 @@ def get_simulation_visualization_models():
     response = client.chat_postMessage(channel=slack_channel,text=slack_completion_message)
 
 
-print("Finished", time.ctime())         
+print("Finished", time.ctime())         

baus/datasources.py

@@ -182,7 +182,7 @@ def initial_year():
 
 @orca.injectable()
 def initial_summary_year():
-    return 2015
+    return 2020
 
 
 @orca.injectable()
@@ -611,11 +611,25 @@ def reprocess_dev_projects(df):
 
 
 # shared between demolish and build tables below
-def get_dev_projects_table(parcels):
+def get_dev_projects_table(parcels, run_setup):
     df = pd.read_csv(os.path.join(orca.get_injectable("inputs_dir"), 
-                     "basis_inputs/parcels_buildings_agents/2021_0309_1939_development_projects.csv"), 
+                     "basis_inputs/parcels_buildings_agents/development_pipeline_with_basis_buildings_nov2023.csv"), 
                      dtype={'PARCEL_ID': np.int64, 'geom_id':   np.int64})
     df = reprocess_dev_projects(df)
+
+    # Optionally - if flag set to use housing element pipeline, load that and append:
+    if run_setup.get('use_housing_element_pipeline',False):
+
+
+        he_pipe = pd.read_csv(os.path.join(orca.get_injectable("inputs_dir"), 
+                                           "basis_inputs/parcels_buildings_agents",
+                                           "he_pipeline_updated_dec2023.csv"),
+                                           dtype={'parcel_id': np.int64})
+        he_pipe = he_pipe.rename(columns={'parcel_id':'PARCEL_ID'})
+
+        he_pipe = he_pipe['geom_id'] = parcel_id_to_geom_id(he_pipe.PARCEL_ID)
+        df = pd.concat([df,he_pipe],axis=0)
+
     orca.add_injectable("devproj_len", len(df))
 
     df = df.dropna(subset=['geom_id'])
@@ -636,16 +650,16 @@ def get_dev_projects_table(parcels):
 
 
 @orca.table(cache=True)
-def demolish_events(parcels):
-    df = get_dev_projects_table(parcels)
+def demolish_events(parcels, run_setup):
+    df = get_dev_projects_table(parcels, run_setup)
 
     # keep demolish and build records
     return df[df.action.isin(["demolish", "build"])]
 
 
 @orca.table(cache=True)
-def development_projects(parcels, mapping):
-    df = get_dev_projects_table(parcels)
+def development_projects(parcels, mapping, run_setup):
+    df = get_dev_projects_table(parcels, run_setup)
 
     for col in [
             'residential_sqft', 'residential_price', 'non_residential_rent']:
@@ -688,6 +702,7 @@ def development_projects(parcels, mapping):
     return df
 
 
+
 def print_error_if_not_available(store, table):
     if table not in store:
         raise Exception(

baus/subsidies.py

@@ -55,8 +55,12 @@ def coffer(account_strategies):
 
 
 @orca.step()
-def preserve_affordable(year, base_year, preservation, residential_units, taz_geography, buildings, parcels_geography):
+def preserve_affordable(year, base_year, preservation, residential_units, taz_geography,
+                        buildings, parcels_geography, initial_summary_year):
 
+    if not year > initial_summary_year:
+        return
+
     # join several geography columns to units table so that we can apply units
     res_units = residential_units.to_frame()
     bldgs = buildings.to_frame()

baus/summaries/affordable_housing_summaries.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 import os
+import pathlib
 import orca
 import pandas as pd
 from baus import datasources
@@ -55,8 +56,9 @@ def deed_restricted_units_summary(run_name, parcels, buildings, year, initial_su
     region_dr["h5_dr_units"] = buildings.h5_dr_units.sum()
     region_dr["cs_dr_units"] = buildings.cs_dr_units.sum()
 
-    region_dr.to_csv(os.path.join(orca.get_injectable("outputs_dir"), 
-                                  "affordable_housing_summaries/{}_region_dr_summary_{}.csv").format(run_name, year))
+    affhousum_output_dir =  pathlib.Path(orca.get_injectable("outputs_dir")) / "affordable_housing_summaries"
+    affhousum_output_dir.mkdir(parents=True, exist_ok=True)
+    region_dr.to_csv(affhousum_output_dir / "{}_region_dr_summary_{}.csv".format(run_name, year))
 
     #### geographic deed restricted units summary ####
     geographies = ['juris', 'superdistrict', 'county']
@@ -91,8 +93,10 @@ def deed_restricted_units_summary(run_name, parcels, buildings, year, initial_su
 
         summary_table.index.name = geography
         summary_table = summary_table.sort_index()
-        summary_table.fillna(0).to_csv(os.path.join(orca.get_injectable("outputs_dir"), "affordable_housing_summaries/{}_{}_dr_summary_{}.csv").\
-                                          format(run_name, geography, year))
+
+        affhousum_output_dir =  pathlib.Path(orca.get_injectable("outputs_dir")) / "affordable_housing_summaries"
+        affhousum_output_dir.mkdir(parents=True, exist_ok=True)
+        summary_table.fillna(0).to_csv(affhousum_output_dir / "{}_{}_dr_summary_{}.csv".format(run_name, geography, year))
 
 
 @orca.step()
@@ -129,4 +133,7 @@ def deed_restricted_units_growth_summary(year, initial_summary_year, final_year,
                                                dr_growth[col+"_"+str(initial_summary_year)+"_share"])
 
         dr_growth = dr_growth.fillna(0)
-        dr_growth.to_csv(os.path.join(orca.get_injectable("outputs_dir"), "affordable_housing_summaries/{}_{}_dr_growth.csv").format(run_name, geography))
+
+        affhousum_output_dir =  pathlib.Path(orca.get_injectable("outputs_dir")) / "affordable_housing_summaries"
+        affhousum_output_dir.mkdir(parents=True, exist_ok=True)
+        dr_growth.to_csv(affhousum_output_dir / "{}_{}_dr_growth.csv".format(run_name, geography))

baus/summaries/core_summaries.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 import os
+import pathlib
 import orca
 import pandas as pd
 
@@ -41,7 +42,9 @@ def parcel_summary(run_name, parcels, buildings, households, jobs, year, initial
     df["totemp"] = jobs_df.groupby('parcel_id').size()
 
     df = df.fillna(0)
-    df.to_csv(os.path.join(orca.get_injectable("outputs_dir"), "core_summaries/%s_parcel_summary_%d.csv" % (run_name, year)))
+    coresum_output_dir = pathlib.Path(orca.get_injectable("outputs_dir")) / "core_summaries"
+    coresum_output_dir.mkdir(parents=True, exist_ok=True)
+    df.to_csv(coresum_output_dir / f"{run_name}_parcel_summary_{year}.csv")
 
 
 @orca.step()
@@ -50,10 +53,13 @@ def parcel_growth_summary(year, run_name, initial_summary_year, final_year):
     if year != final_year:
         return
 
-    df1 = pd.read_csv(os.path.join(orca.get_injectable("outputs_dir"), "core_summaries/%s_parcel_summary_%d.csv" %
-                        (run_name, initial_summary_year)), index_col="parcel_id")
-    df2 = pd.read_csv(os.path.join(orca.get_injectable("outputs_dir"), "core_summaries/%s_parcel_summary_%d.csv" %
-                        (run_name, final_year)), index_col="parcel_id")
+    coresum_output_dir = pathlib.Path(orca.get_injectable("outputs_dir")) / "core_summaries"
+    initial_parcel_file = coresum_output_dir / f"{run_name}_parcel_summary_{initial_summary_year}.csv"
+    print("initial_parcel_file resolve:{} exists:{}".format(initial_parcel_file.resolve(), initial_parcel_file.exists()))
+
+    df1 = pd.read_csv(initial_parcel_file, index_col="parcel_id")
+    df2 = pd.read_csv(coresum_output_dir / f"{run_name}_parcel_summary_{final_year}.csv",
+                      index_col="parcel_id")
 
     for col in df1.columns:
         if col in ["geom_id", "x", "y"]:
@@ -65,7 +71,10 @@ def parcel_growth_summary(year, run_name, initial_summary_year, final_year):
 
         df1[col] = df2[col] - df1[col]
 
-    df1.to_csv(os.path.join(orca.get_injectable("outputs_dir"), "core_summaries/{}_parcel_growth.csv").format(run_name))
+    coresum_output_dir = pathlib.Path(orca.get_injectable("outputs_dir")) / "core_summaries"
+    coresum_output_dir.mkdir(parents=True, exist_ok=True)
+    df1.to_csv(coresum_output_dir / f"{run_name}_parcel_growth.csv")
+
 
 
 @orca.step()
@@ -81,7 +90,9 @@ def building_summary(run_name, parcels, buildings, year, initial_summary_year, f
                  'preserved_units', 'subsidized_units', 'job_spaces', 'source'])
 
     df = df.fillna(0)
-    df.to_csv(os.path.join(orca.get_injectable("outputs_dir"), "core_summaries/%s_building_summary_%d.csv" % (run_name, year)))
+    coresum_output_dir = pathlib.Path(orca.get_injectable("outputs_dir")) / "core_summaries"
+    coresum_output_dir.mkdir(parents=True, exist_ok=True)
+    df.to_csv(coresum_output_dir / "{}_building_summary_{}.csv".format(run_name, year))
 
 
 @orca.step()
@@ -106,7 +117,9 @@ def new_buildings_summary(run_name, parcels, buildings, year, final_year):
     df["run_name"] = run_name
 
     df = df.fillna(0)
-    df.to_csv(os.path.join(orca.get_injectable("outputs_dir"), "core_summaries/%s_new_buildings_summary.csv" % (run_name)))
+    coresum_output_dir = pathlib.Path(orca.get_injectable("outputs_dir")) / "core_summaries"
+    coresum_output_dir.mkdir(parents=True, exist_ok=True)
+    df.to_csv(coresum_output_dir / "{}_new_buildings_summary.csv".format(run_name))
 
 
 @orca.step()
@@ -157,20 +170,22 @@ def interim_zone_output(run_name, households, buildings, residential_units, parc
     zones['residential_rent'] = residential_units.groupby('zone_id').unit_residential_rent.quantile()
     zones['non_residential_rent'] = buildings.groupby('zone_id').non_residential_rent.quantile()
 
-    zones.to_csv(os.path.join(orca.get_injectable("outputs_dir"), "core_summaries/%s_interim_zone_output_%d.csv" % (run_name, year)))
+    coresum_output_dir = pathlib.Path(orca.get_injectable("outputs_dir")) / "core_summaries"
+    coresum_output_dir.mkdir(parents=True, exist_ok=True)
+    zones.to_csv(coresum_output_dir / "{}_interim_zone_output_{}.csv".format(run_name, year))
 
     # now add all interim zone output to a single dataframe
 
     zones = zones.add_suffix("_"+str(year))
 
-    if year == initial_summary_year:
-        all_years = pd.DataFrame(index=zones.index)
-    else:
+    try:
         all_years = orca.get_table("interim_zone_output_all").to_frame()
+    except KeyError:
+        all_years = pd.DataFrame(index=zones.index)
 
     all_years = all_years.merge(zones, left_index=True, right_index=True)
     orca.add_table("interim_zone_output_all", all_years)
 
     if year == final_year:
         all_years.to_csv(os.path.join(orca.get_injectable("outputs_dir"), 
-                                      "core_summaries/%s_interim_zone_output_allyears.csv" % (run_name)))
+                                      "core_summaries/%s_interim_zone_output_allyears.csv" % (run_name)))

baus/summaries/geographic_summaries.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 import os
+import pathlib
 import orca
 import pandas as pd
 from baus import datasources
@@ -15,8 +16,9 @@
 def geographic_summary(parcels, households, jobs, buildings, year, superdistricts_geography,
                        initial_summary_year, interim_summary_year, final_year, run_name):  
 
-    if year not in [initial_summary_year, interim_summary_year, final_year]:
-         return
+    # Commenting this out so we get geographic summaries for all years - DSL 2023-08-31
+    # if year not in [initial_summary_year, interim_summary_year, final_year]:
+    #      return
 
     households_df = orca.merge_tables('households', [parcels, buildings, households],
         columns=['juris', 'superdistrict', 'county', 'subregion', 'base_income_quartile',])
@@ -50,7 +52,9 @@ def geographic_summary(parcels, households, jobs, buildings, year, superdistrict
 
     # non-residential buildings
     region['non_residential_sqft'] = buildings_df.non_residential_sqft.sum()
-    region.to_csv(os.path.join(orca.get_injectable("outputs_dir"), "geographic_summaries/{}_region_summary_{}.csv").format(run_name, year))
+    geosum_output_dir = pathlib.Path(orca.get_injectable("outputs_dir")) / "geographic_summaries"
+    geosum_output_dir.mkdir(parents=True, exist_ok=True)
+    region.to_csv(geosum_output_dir / "{}_region_summary_{}.csv".format(run_name, year))
 
     #### summarize by sub-regional geography ####
     geographies = ['juris', 'superdistrict', 'county', 'subregion']

baus/summaries/hazards_summaries.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 import os
+import pathlib
 import orca
 import pandas as pd
 from urbansim.utils import misc
@@ -59,7 +60,9 @@ def hazards_slr_summary(run_setup, run_name, year):
     for empsix in ['AGREMPN', 'MWTEMPN', 'RETEMPN', 'FPSEMPN', 'HEREMPN', 'OTHEMPN']:
         slr_summary["impacted_jobs_"+str(empsix)] = (unplaced_jobs_tot["empsix"] == empsix).sum()
 
-    slr_summary.to_csv(os.path.join(orca.get_injectable("outputs_dir"), "hazards_summaries/%s_slr_summary_%d.csv" % (run_name, year)))
+    hazsumm_output_dir = pathlib.Path(orca.get_injectable("outputs_dir")) / "hazards_summaries"
+    hazsumm_output_dir.mkdir(parents=True, exist_ok=True)
+    slr_summary.to_csv(hazsumm_output_dir / "{}_slr_summary_{}.csv".format(run_name, year))
 
 
 @orca.step()
@@ -72,12 +75,12 @@ def hazards_eq_summary(run_setup, run_name, year, parcels, buildings):
         return
 
     code = orca.get_injectable("code").to_frame()
-    code.value_counts().to_csv(os.path.join(orca.get_injectable("outputs_dir"), "hazards_summaries/%s_eq_codes_summary_%d.csv"
-                                    % (run_name, year)))
+    hazsumm_output_dir = pathlib.Path(orca.get_injectable("outputs_dir")) / "hazards_summaries"
+    hazsumm_output_dir.mkdir(parents=True, exist_ok=True)
+    code.value_counts().to_csv(hazsumm_output_dir / "{}_eq_codes_summary_{}.csv".format(run_name, year))
 
     fragilities = orca.get_injectable("fragilities").to_frame()
-    fragilities.value_counts().to_csv(os.path.join(orca.get_injectable("outputs_dir"), "hazards_summaries/%s_eq_fragilities_summary_%d.csv"
-                                    % (run_name, year)))
+    fragilities.value_counts().to_csv(hazsumm_output_dir / "{}_eq_fragilities_summary_{}.csv".format(run_name, year))
 
     eq_summary = pd.DataFrame(index=[0])
 
@@ -105,17 +108,15 @@ def hazards_eq_summary(run_setup, run_name, year, parcels, buildings):
     for empsix in ['AGREMPN', 'MWTEMPN', 'RETEMPN', 'FPSEMPN', 'HEREMPN', 'OTHEMPN']:
         eq_summary["impacted_jobs_"+str(empsix)] = (jobs_unplaced_eq["empsix"] == empsix).sum()
 
-    eq_summary.to_csv(os.path.join(orca.get_injectable("outputs_dir"), "hazards_summaries/%s_eq_summary_%d.csv"
-                                    % (run_name, year)))
+    eq_summary.to_csv(hazsumm_output_dir / "{}_eq_summary_{}.csv".format(run_name, year))
 
     # print out demolished buildings by TAZ
     eq_demolish_taz = misc.reindex(parcels.zone_id, eq_demolish.parcel_id)
     eq_demolish['taz'] = eq_demolish_taz
     eq_demolish['count'] = 1
     eq_demolish = eq_demolish.drop(['parcel_id', 'year_built', 'redfin_sale_year'], axis=1)
     eq_demolish = eq_demolish.groupby(['taz']).sum()
-    eq_demolish.to_csv(os.path.join(orca.get_injectable("outputs_dir"), "hazards_summaries/%s_eq_demolish_buildings_%d.csv"
-                                    % (run_name, year)))
+    eq_demolish.to_csv(hazsumm_output_dir / "{}_eq_demolish_buildings_{}.csv".format(run_name, year))
 
     # print out retrofit buildings by TAZ
     if not run_setup['eq_mitigation']:
@@ -127,8 +128,7 @@ def hazards_eq_summary(run_setup, run_name, year, parcels, buildings):
     retrofit_bldgs_tot = retrofit_bldgs_tot[['taz', 'residential_units', 'residential_sqft',
         'non_residential_sqft', 'building_sqft', 'stories','redfin_sale_price', 'non_residential_rent',
         'deed_restricted_units', 'residential_price', 'count']]
-    retrofit_bldgs_tot.to_csv(os.path.join(
-                orca.get_injectable("outputs_dir"), "hazards_summaries/%s_eq_retrofit_buildings_%d.csv" % (run_name, year)))
+    retrofit_bldgs_tot.to_csv(hazsumm_output_dir / "{}_eq_retrofit_buildings_{}.csv".format(run_name, year))
 
     # print out buildings by TAZ around earthquake years
     if year not in [2030, 2035, 2050]:
@@ -140,5 +140,4 @@ def hazards_eq_summary(run_setup, run_name, year, parcels, buildings):
     buildings = buildings[['taz', 'count', 'residential_units', 'residential_sqft', 'non_residential_sqft',
                         'building_sqft', 'stories', 'redfin_sale_price', 'non_residential_rent', 'deed_restricted_units',
                         'residential_price']]
-    buildings.to_csv(os.path.join(
-        orca.get_injectable("outputs_dir"), "hazards_summaries/%s_eq_buildings_list_%d.csv" % (run_name, year)))
+    buildings.to_csv(hazsumm_output_dir / "{}_eq_buildings_list_{}.csv".format(run_name, year))