Merge branch 'dev' into spot-fix-ferc-exploder

catalyst-cooperative · Jun 19, 2023 · cde26b0 · cde26b0
2 parents 6e8c6cf + 733e90e
commit cde26b0
Show file tree

Hide file tree

Showing 9 changed files with 186 additions and 232 deletions.
diff --git a/...start_over_with_new_pk_in_sales_eia861.py → ...e_start_over_with_new_pk_in_emissions_.py b/...start_over_with_new_pk_in_sales_eia861.py → ...e_start_over_with_new_pk_in_emissions_.py
diff --git a/migrations/versions/88d9201ae4c4_update_schema_for_ferc1_exploder.py b/migrations/versions/88d9201ae4c4_update_schema_for_ferc1_exploder.py
diff --git a/pyproject.toml b/pyproject.toml
@@ -20,7 +20,7 @@ dependencies = [
     "addfips>=0.4,<0.5",
     "alembic>=1.10.3,<1.12",
     "catalystcoop.dbfread>=3.0,<3.1",
-    "catalystcoop.ferc-xbrl-extractor==0.8.2",
+    "catalystcoop.ferc-xbrl-extractor==0.8.3",
     "coloredlogs>=14.0,<15.1",  # Dagster requires 14.0
     "dagit>=1.2.2,<1.4",  # 1.2.2 is first version to support Python 3.11
     "dagster>=1.2.2,<1.4",  # 1.2.2 is first version to support Python 3.11

diff --git a/src/pudl/metadata/codes.py b/src/pudl/metadata/codes.py
@@ -264,7 +264,7 @@
                     "ishikawajima_harima",
                     "Ishikawajima-Harima Heavy Industries  (IHI Corp)",
                 ),
-                ("IS" "innovative_steam_technology", "Innovative Steam Technology"),
+                ("IS", "innovative_steam_technology", "Innovative Steam Technology"),
                 ("JO", "joy_manufacturing", "Joy Manufacturing"),
                 (
                     "JZ",
@@ -352,6 +352,95 @@
         "code_fixes": {},
         "ignored_codes": ["NA", "IN"],
     },
+    "emission_control_equipment_types_eia": {
+        "df": pd.DataFrame(
+            columns=["code", "label", "description"],
+            data=[
+                (
+                    "ACI",
+                    "activated_carbon_injection",
+                    "Activated carbon injection system",
+                ),
+                ("BP", "baghouse_pulse", "Baghouse (fabric filter), pulse"),
+                (
+                    "BR",
+                    "baghouse_reverse_air",
+                    "Baghouse (fabric filter), reverse air",
+                ),
+                (
+                    "BS",
+                    "baghouse_shake_deflate",
+                    "Baghouse (fabric filter), shake and deflate",
+                ),
+                ("CD", "circulating_dry_scrubber", "Circulating dry scrubber"),
+                (
+                    "DSI",
+                    "dry_sorbent_injection",
+                    "Dry sorbent (powder) injection type",
+                ),
+                (
+                    "EC",
+                    "electrostatic_cold_conditioned",
+                    "Electrostatic precipitator, cold side, with flue gas conditioning",
+                ),
+                (
+                    "EH",
+                    "electrostatic_hot_conditioned",
+                    "Electrostatic precipitator, hot side, with flue gas conditioning",
+                ),
+                (
+                    "EK",
+                    "electrostatic_cold_unconditioned",
+                    "Electrostatic precipitator, cold side, without flue gas conditioning",
+                ),
+                (
+                    "EW",
+                    "electrostatic_hot_unconditioned",
+                    "Electrostatic precipitator, hot side, without flue gas conditioning",
+                ),
+                (
+                    "JB",
+                    "wet_scrubber_jet_bubbling",
+                    "Jet bubbling reactor (wet) scrubber",
+                ),
+                ("LIJ", "lime_injection", "Lime injection"),
+                ("LNB", "low_nox_burner", "Low NOx burner"),
+                (
+                    "MA",
+                    "wet_scrubber_mechanical",
+                    "Mechanically aided type (wet) scrubber",
+                ),
+                ("MC", "multiple_cyclone", "Multiple cyclone"),
+                ("OT", "other", "Other"),
+                ("PA", "wet_scrubber_packed", "Packed type (wet) scrubber"),
+                ("SC", "single_cyclone", "Single cyclone"),
+                (
+                    "SD",
+                    "spray_dryer",
+                    "Spray dryer type / dry FGD / semi-dry FGD",
+                ),
+                (
+                    "SN",
+                    "selective_noncatalytic_reduction",
+                    "Selective noncatalytic reduction",
+                ),
+                ("SP", "wet_scrubber_spray", "Spray type (wet) scrubber"),
+                (
+                    "SR",
+                    "selective_catalytic_reduction",
+                    "Selective catalytic reduction",
+                ),
+                ("TR", "wet_scrubber_tray", "Tray type (wet) scrubber"),
+                ("VE", "wet_scrubber_venturi", "Venturi type (wet) scrubber."),
+            ],
+        ).convert_dtypes(),
+        "code_fixes": {
+            "SR-2": "SR",
+            "sn": "SN",
+            "LN": "LNB",
+        },
+        "ignored_codes": ["HRSG1", "HRSG2", "FGD", "OV"],
+    },
     "firing_types_eia": {
         "df": pd.DataFrame(
             columns=["code", "label", "description"],

diff --git a/src/pudl/metadata/fields.py b/src/pudl/metadata/fields.py
@@ -632,10 +632,11 @@
     "emission_control_equipment_cost": {
         "type": "number",
         "description": "The total cost to install a piece of emission control equipment.",
+        "unit": "USD",
     },
-    "emission_control_equipment_type": {
+    "emission_control_equipment_type_code": {
         "type": "string",
-        "description": "The type of emission control equipment installed.",
+        "description": "Short code indicating the type of emission control equipment installed.",
     },
     "emission_control_operating_date": {
         "type": "date",

diff --git a/src/pudl/metadata/resources/eia.py b/src/pudl/metadata/resources/eia.py
@@ -101,6 +101,20 @@
         "etl_group": "static_eia",
         "field_namespace": "eia",
     },
+    "emission_control_equipment_types_eia": {
+        "description": "A coding table describing the type of emissions control equipment installed on a boiler.",
+        "schema": {
+            "fields": ["code", "label", "description"],
+            "primary_key": ["code"],
+            "foreign_key_rules": {
+                "fields": [["emission_control_equipment_type_code"]],
+            },
+        },
+        "encoder": CODE_METADATA["emission_control_equipment_types_eia"],
+        "sources": ["eia860"],
+        "etl_group": "static_eia",
+        "field_namespace": "eia",
+    },
     "firing_types_eia": {
         "description": "A coding table describing different boiler firing types in the EIA-860.",
         "schema": {

diff --git a/src/pudl/metadata/resources/eia860.py b/src/pudl/metadata/resources/eia860.py
@@ -489,7 +489,7 @@
                 "plant_id_eia",
                 "emission_control_id_pudl",
                 "data_maturity",
-                "emission_control_equipment_type",
+                "emission_control_equipment_type_code",
                 "operational_status_code",
                 "mercury_control_id_eia",
                 "nox_control_id_eia",
@@ -526,7 +526,7 @@
                 "utility_name_eia",
                 "emission_control_id_pudl",
                 "data_maturity",
-                "emission_control_equipment_type",
+                "emission_control_equipment_type_code",
                 "operational_status_code",
                 "operational_status",
                 "mercury_control_id_eia",

diff --git a/src/pudl/package_data/eia860/column_maps/emissions_control_equipment.csv b/src/pudl/package_data/eia860/column_maps/emissions_control_equipment.csv
@@ -3,7 +3,7 @@ utility_id_eia,utility_id,utility_id,utility_id,utility_id,utility_id,utility_id
 utility_name_eia,utility_name,utility_name,utility_name,utility_name,utility_name,utility_name,utility_name,utility_name,utility_name
 plant_id_eia,plant_code,plant_code,plant_code,plant_code,plant_code,plant_code,plant_code,plant_code,plant_code
 plant_name_eia,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name,plant_name
-emission_control_equipment_type,equipment_type,equipment_type,equipment_type,equipment_type,equipment_type,equipment_type,equipment_type,equipment_type,equipment_type
+emission_control_equipment_type_code,equipment_type,equipment_type,equipment_type,equipment_type,equipment_type,equipment_type,equipment_type,equipment_type,equipment_type
 particulate_control_id_eia,particulate_matter_control_id,particulate_matter_control_id,particulate_matter_control_id,particulate_matter_control_id,particulate_matter_control_id,particulate_matter_control_id,particulate_matter_control_id,particulate_matter_control_id,particulate_matter_control_id
 so2_control_id_eia,so2_control_id,so2_control_id,so2_control_id,so2_control_id,so2_control_id,so2_control_id,so2_control_id,so2_control_id,so2_control_id
 nox_control_id_eia,nox_control_id,nox_control_id,nox_control_id,nox_control_id,nox_control_id,nox_control_id,nox_control_id,nox_control_id,nox_control_id

diff --git a/src/pudl/transform/eia860.py b/src/pudl/transform/eia860.py
@@ -860,6 +860,11 @@ def clean_emissions_control_equipment_eia860(
         "emission_control_equipment_cost",
     ] = 3200
 
+    # Convert thousands of dollars to dollars:
+    emce_df.loc[:, "emission_control_equipment_cost"] = (
+        1000.0 * emce_df["emission_control_equipment_cost"]
+    )
+
     emce_df = (
         pudl.metadata.classes.Package.from_resource_ids()
         .get_resource("emissions_control_equipment_eia860")