Fixing up utc offset

EPAENERGYSTAR · Jun 2, 2020 · 03fc625 · 03fc625
1 parent 45679b2
commit 03fc625
Show file tree

Hide file tree

Showing 56 changed files with 1,297,554 additions and 1,297,608 deletions.
diff --git a/scripts/thermostat_converter.py b/scripts/thermostat_converter.py
@@ -2,7 +2,7 @@
 import csv
 import glob
 import os
-
+import dateutil.parser
 from datetime import datetime, timedelta
 
 FIELDNAMES = [
@@ -11,18 +11,52 @@
         'heat_runtime_stg1', 'heat_runtime_stg2', 'heat_equiv_runtime',
         'emergency_heat_runtime',
         'auxiliary_heat_runtime',
-        'heating_setpoint', 'cooling_setpoint',
         'temp_in',
         ]
 
 
+def normalize_utc_offset(utc_offset):
+    """
+    Normalizes the UTC offset
+    Returns the UTC offset based on the string passed in.
+
+    Parameters
+    ----------
+    utc_offset : str
+        String representation of the UTC offset
+
+    Returns
+    -------
+    datetime timdelta offset
+    """
+    try:
+        if int(utc_offset) == 0:
+            utc_offset = "+0"
+        delta = dateutil.parser.parse(
+            "2000-01-01T00:00:00" + str(utc_offset)).tzinfo.utcoffset(None)
+        return delta
+
+    except (ValueError, TypeError, AttributeError) as e:
+        raise TypeError("Invalid UTC offset: {} ({})".format(
+           utc_offset,
+           e))
+
+
 def main():
     """ This script will convert over old thermostat data files to the new
     EPAThermostat 2.0 format. It should be used for testing only, and is not
     intended for data submissions.
     """
 
     hour_interval = timedelta(hours=1)
+
+    metadata = {}
+    for filename in glob.iglob('metadata*.csv'):
+        with open(filename) as metadata_csv:
+            metadata_reader = csv.DictReader(metadata_csv)
+            for row in metadata_reader:
+                metadata[row['interval_data_filename']] = row
+
     for filename in glob.iglob('thermostat_*.csv'):
         input_filename = filename
         output_filename = os.path.join("new", filename)
@@ -35,10 +69,12 @@ def main():
             with open(input_filename) as csvfile:
                 thermostat_csv = csv.DictReader(csvfile)
                 for row in thermostat_csv:
+                    utc_offset = normalize_utc_offset(metadata[input_filename]['utc_offset'])
                     current_datetime = datetime.strptime(row['date'], '%Y-%m-%d')
                     for hour in range(0, 24):
+
                         current_row = {}
-                        current_row['datetime'] = current_datetime + (hour_interval * hour)
+                        current_row['datetime'] = current_datetime + (hour_interval * hour) - utc_offset
 
                         current_row['cool_runtime_stg2'] = None
 
@@ -69,11 +105,6 @@ def main():
                         header = 'temp_in_%02d' % hour
                         current_row['temp_in'] = row[header]
 
-                        header = 'heating_setpoint_%02d' % hour
-                        current_row['heating_setpoint'] = row[header]
-
-                        header = 'cooling_setpoint_%02d' % hour
-                        current_row['cooling_setpoint'] = row[header]
                         csv_out.writerow(current_row)
 
 

diff --git a/tests/data/metadata.csv b/tests/data/metadata.csv
@@ -1,36 +1,36 @@
-thermostat_id,heat_type,heat_stage,cool_type,cool_stage,zipcode,utc_offset,interval_data_filename
-064ec4bc-5317-4ab1-a1e7-a214d1acd74a,none,,central,single_speed,33155,-7,thermostat_064ec4bc-5317-4ab1-a1e7-a214d1acd74a.csv
-d28b18d0-4dc0-4010-ae3e-58ab19b5fd61,none,,central,single_speed,78660,-7,thermostat_d28b18d0-4dc0-4010-ae3e-58ab19b5fd61.csv
-959325a4-3347-4b43-aa24-2f1bd55195ee,heat_pump_no_electric_backup,single_stage,heat_pump,,39565,-7,thermostat_959325a4-3347-4b43-aa24-2f1bd55195ee.csv
-feadf000-44c3-45fc-8a67-177414868ae7,none,,central,single_speed,85234,-7,thermostat_feadf000-44c3-45fc-8a67-177414868ae7.csv
-e44e30f7-4937-45dd-84f5-b9f78f77ce3a,none,,central,single_speed,78840,-7,thermostat_e44e30f7-4937-45dd-84f5-b9f78f77ce3a.csv
-a59238c1-9e4b-4ae7-80ff-9c5de4152bd8,none,,central,single_speed,36047,-7,thermostat_a59238c1-9e4b-4ae7-80ff-9c5de4152bd8.csv
-e839fe3c-b4af-43a8-bcd4-209241f1612c,non_heat_pump,single_stage,central,single_speed,30331,-7,thermostat_e839fe3c-b4af-43a8-bcd4-209241f1612c.csv
-54aeb85f-aa8c-4e32-afb7-4617c08446a5,none,,central,single_speed,88009,-7,thermostat_54aeb85f-aa8c-4e32-afb7-4617c08446a5.csv
-8465829e-df0d-449e-97bf-96317c24dec3,heat_pump_electric_backup,single_stage,heat_pump,,62223,-7,thermostat_8465829e-df0d-449e-97bf-96317c24dec3.csv
-566eed02-e92f-4ce8-afc2-8d224dc6c61e,non_heat_pump,single_stage,none,,87005,-7,thermostat_566eed02-e92f-4ce8-afc2-8d224dc6c61e.csv
-d36e20f2-e6bb-4caf-9d53-6183b7c08fe1,heat_pump_electric_backup,single_stage,heat_pump,,88061,-7,thermostat_d36e20f2-e6bb-4caf-9d53-6183b7c08fe1.csv
-a7ed7c85-dc4c-4041-acb3-b42139fa5ecb,non_heat_pump,single_stage,none,,98260,-7,thermostat_a7ed7c85-dc4c-4041-acb3-b42139fa5ecb.csv
-582b15ff-3ae3-4c05-b401-5e6b3bb96688,heat_pump_electric_backup,single_stage,heat_pump,,98031,-7,thermostat_582b15ff-3ae3-4c05-b401-5e6b3bb96688.csv
-2f3852ae-efbe-42f0-8100-6f6183baeede,heat_pump_electric_backup,single_stage,heat_pump,,03854,-7,thermostat_2f3852ae-efbe-42f0-8100-6f6183baeede.csv
-8850a9e0-b917-40ab-bb94-e03286575d73,non_heat_pump,single_stage,none,,84123,-7,thermostat_8850a9e0-b917-40ab-bb94-e03286575d73.csv
-f4080332-7bc7-40be-a4d1-26311b5ef726,heat_pump_electric_backup,single_stage,heat_pump,,04469,-7,thermostat_f4080332-7bc7-40be-a4d1-26311b5ef726.csv
-879b43f2-a02d-499a-9d3a-f44cdc78c2ae,non_heat_pump,single_stage,none,,83276,-7,thermostat_879b43f2-a02d-499a-9d3a-f44cdc78c2ae.csv
-9a168b59-6d9c-4b1d-a8af-2a765afbed92,non_heat_pump,single_stage,none,,80461,-7,thermostat_9a168b59-6d9c-4b1d-a8af-2a765afbed92.csv
-3d7310ca-ef32-470a-a2aa-a18d7740ed07,non_heat_pump,single_stage,none,,99771,-7,thermostat_3d7310ca-ef32-470a-a2aa-a18d7740ed07.csv
-2d3b385b-1ed1-4658-b551-5728a213f03d,heat_pump_no_electric_backup,single_stage,heat_pump,,95460,-7,thermostat_2d3b385b-1ed1-4658-b551-5728a213f03d.csv
-b5c68765-4c68-4ff9-98ae-5a2da9a5e517,non_heat_pump,single_stage,central,single_speed,94945,-7,thermostat_b5c68765-4c68-4ff9-98ae-5a2da9a5e517.csv
-b2a305d1-9df1-4387-a89c-d702f6a9ba3e,heat_pump_electric_backup,single_stage,heat_pump,,95019,-7,thermostat_b2a305d1-9df1-4387-a89c-d702f6a9ba3e.csv
-a16d8fd5-95c9-41d7-a29e-86ef5adb9893,non_heat_pump,single_stage,central,single_speed,95035,-7,thermostat_a16d8fd5-95c9-41d7-a29e-86ef5adb9893.csv
-da25834c-3c93-4dda-902e-58e986f5e8f5,non_heat_pump,single_stage,central,single_speed,93430,-7,thermostat_da25834c-3c93-4dda-902e-58e986f5e8f5.csv
-3766e958-03ec-4d96-83c2-7eeb87c74c90,non_heat_pump,single_stage,central,single_speed,90073,-7,thermostat_3766e958-03ec-4d96-83c2-7eeb87c74c90.csv
-e5a25daa-c774-48cf-b112-057ebedda3c2,heat_pump_electric_backup,single_stage,heat_pump,,92147,-7,thermostat_e5a25daa-c774-48cf-b112-057ebedda3c2.csv
-fe01324c-e625-4b6c-862f-5a963f52c82a,non_heat_pump,single_stage,central,single_speed,92603,-7,thermostat_fe01324c-e625-4b6c-862f-5a963f52c82a.csv
-1ab401c3-8588-4568-b012-e5d22941edad,heat_pump_no_electric_backup,single_stage,heat_pump,,90048,-7,thermostat_1ab401c3-8588-4568-b012-e5d22941edad.csv
-c19345f4-14c9-4aee-9abc-5a9c67016586,heat_pump_electric_backup,single_stage,heat_pump,,92082,-7,thermostat_c19345f4-14c9-4aee-9abc-5a9c67016586.csv
-eab16216-83c6-42ec-96f0-ee523c159966,non_heat_pump,single_stage,central,single_speed,95979,-7,thermostat_eab16216-83c6-42ec-96f0-ee523c159966.csv
-04fa68f1-12c4-45ab-a924-a19c51c8a72b,heat_pump_electric_backup,single_stage,heat_pump,,95816,-7,thermostat_04fa68f1-12c4-45ab-a924-a19c51c8a72b.csv
-e787b7a4-d09e-4bb0-b9b1-95032d44199e,non_heat_pump,single_stage,central,single_speed,93313,-7,thermostat_e787b7a4-d09e-4bb0-b9b1-95032d44199e.csv
-f6b8d9fa-f189-4b47-9d8e-3785c1a06bbe,none,,central,single_speed,92309,-7,thermostat_f6b8d9fa-f189-4b47-9d8e-3785c1a06bbe.csv
-a2889532-7fb0-4e63-a85b-7fbcbe0f7fe0,none,,central,single_speed,92201,-7,thermostat_a2889532-7fb0-4e63-a85b-7fbcbe0f7fe0.csv
-4b82fc6c-9beb-4d64-b5bf-3d9b85003273,heat_pump_no_electric_backup,single_stage,heat_pump,,96128,-7,thermostat_4b82fc6c-9beb-4d64-b5bf-3d9b85003273.csv
+thermostat_id,heat_type,heat_stage,cool_type,cool_stage,zipcode,interval_data_filename
+064ec4bc-5317-4ab1-a1e7-a214d1acd74a,none,,central,single_speed,33155,thermostat_064ec4bc-5317-4ab1-a1e7-a214d1acd74a.csv
+d28b18d0-4dc0-4010-ae3e-58ab19b5fd61,none,,central,single_speed,78660,thermostat_d28b18d0-4dc0-4010-ae3e-58ab19b5fd61.csv
+959325a4-3347-4b43-aa24-2f1bd55195ee,heat_pump_no_electric_backup,single_stage,heat_pump,,39565,thermostat_959325a4-3347-4b43-aa24-2f1bd55195ee.csv
+feadf000-44c3-45fc-8a67-177414868ae7,none,,central,single_speed,85234,thermostat_feadf000-44c3-45fc-8a67-177414868ae7.csv
+e44e30f7-4937-45dd-84f5-b9f78f77ce3a,none,,central,single_speed,78840,thermostat_e44e30f7-4937-45dd-84f5-b9f78f77ce3a.csv
+a59238c1-9e4b-4ae7-80ff-9c5de4152bd8,none,,central,single_speed,36047,thermostat_a59238c1-9e4b-4ae7-80ff-9c5de4152bd8.csv
+e839fe3c-b4af-43a8-bcd4-209241f1612c,non_heat_pump,single_stage,central,single_speed,30331,thermostat_e839fe3c-b4af-43a8-bcd4-209241f1612c.csv
+54aeb85f-aa8c-4e32-afb7-4617c08446a5,none,,central,single_speed,88009,thermostat_54aeb85f-aa8c-4e32-afb7-4617c08446a5.csv
+8465829e-df0d-449e-97bf-96317c24dec3,heat_pump_electric_backup,single_stage,heat_pump,,62223,thermostat_8465829e-df0d-449e-97bf-96317c24dec3.csv
+566eed02-e92f-4ce8-afc2-8d224dc6c61e,non_heat_pump,single_stage,none,,87005,thermostat_566eed02-e92f-4ce8-afc2-8d224dc6c61e.csv
+d36e20f2-e6bb-4caf-9d53-6183b7c08fe1,heat_pump_electric_backup,single_stage,heat_pump,,88061,thermostat_d36e20f2-e6bb-4caf-9d53-6183b7c08fe1.csv
+a7ed7c85-dc4c-4041-acb3-b42139fa5ecb,non_heat_pump,single_stage,none,,98260,thermostat_a7ed7c85-dc4c-4041-acb3-b42139fa5ecb.csv
+582b15ff-3ae3-4c05-b401-5e6b3bb96688,heat_pump_electric_backup,single_stage,heat_pump,,98031,thermostat_582b15ff-3ae3-4c05-b401-5e6b3bb96688.csv
+2f3852ae-efbe-42f0-8100-6f6183baeede,heat_pump_electric_backup,single_stage,heat_pump,,03854,thermostat_2f3852ae-efbe-42f0-8100-6f6183baeede.csv
+8850a9e0-b917-40ab-bb94-e03286575d73,non_heat_pump,single_stage,none,,84123,thermostat_8850a9e0-b917-40ab-bb94-e03286575d73.csv
+f4080332-7bc7-40be-a4d1-26311b5ef726,heat_pump_electric_backup,single_stage,heat_pump,,04469,thermostat_f4080332-7bc7-40be-a4d1-26311b5ef726.csv
+879b43f2-a02d-499a-9d3a-f44cdc78c2ae,non_heat_pump,single_stage,none,,83276,thermostat_879b43f2-a02d-499a-9d3a-f44cdc78c2ae.csv
+9a168b59-6d9c-4b1d-a8af-2a765afbed92,non_heat_pump,single_stage,none,,80461,thermostat_9a168b59-6d9c-4b1d-a8af-2a765afbed92.csv
+3d7310ca-ef32-470a-a2aa-a18d7740ed07,non_heat_pump,single_stage,none,,99771,thermostat_3d7310ca-ef32-470a-a2aa-a18d7740ed07.csv
+2d3b385b-1ed1-4658-b551-5728a213f03d,heat_pump_no_electric_backup,single_stage,heat_pump,,95460,thermostat_2d3b385b-1ed1-4658-b551-5728a213f03d.csv
+b5c68765-4c68-4ff9-98ae-5a2da9a5e517,non_heat_pump,single_stage,central,single_speed,94945,thermostat_b5c68765-4c68-4ff9-98ae-5a2da9a5e517.csv
+b2a305d1-9df1-4387-a89c-d702f6a9ba3e,heat_pump_electric_backup,single_stage,heat_pump,,95019,thermostat_b2a305d1-9df1-4387-a89c-d702f6a9ba3e.csv
+a16d8fd5-95c9-41d7-a29e-86ef5adb9893,non_heat_pump,single_stage,central,single_speed,95035,thermostat_a16d8fd5-95c9-41d7-a29e-86ef5adb9893.csv
+da25834c-3c93-4dda-902e-58e986f5e8f5,non_heat_pump,single_stage,central,single_speed,93430,thermostat_da25834c-3c93-4dda-902e-58e986f5e8f5.csv
+3766e958-03ec-4d96-83c2-7eeb87c74c90,non_heat_pump,single_stage,central,single_speed,90073,thermostat_3766e958-03ec-4d96-83c2-7eeb87c74c90.csv
+e5a25daa-c774-48cf-b112-057ebedda3c2,heat_pump_electric_backup,single_stage,heat_pump,,92147,thermostat_e5a25daa-c774-48cf-b112-057ebedda3c2.csv
+fe01324c-e625-4b6c-862f-5a963f52c82a,non_heat_pump,single_stage,central,single_speed,92603,thermostat_fe01324c-e625-4b6c-862f-5a963f52c82a.csv
+1ab401c3-8588-4568-b012-e5d22941edad,heat_pump_no_electric_backup,single_stage,heat_pump,,90048,thermostat_1ab401c3-8588-4568-b012-e5d22941edad.csv
+c19345f4-14c9-4aee-9abc-5a9c67016586,heat_pump_electric_backup,single_stage,heat_pump,,92082,thermostat_c19345f4-14c9-4aee-9abc-5a9c67016586.csv
+eab16216-83c6-42ec-96f0-ee523c159966,non_heat_pump,single_stage,central,single_speed,95979,thermostat_eab16216-83c6-42ec-96f0-ee523c159966.csv
+04fa68f1-12c4-45ab-a924-a19c51c8a72b,heat_pump_electric_backup,single_stage,heat_pump,,95816,thermostat_04fa68f1-12c4-45ab-a924-a19c51c8a72b.csv
+e787b7a4-d09e-4bb0-b9b1-95032d44199e,non_heat_pump,single_stage,central,single_speed,93313,thermostat_e787b7a4-d09e-4bb0-b9b1-95032d44199e.csv
+f6b8d9fa-f189-4b47-9d8e-3785c1a06bbe,none,,central,single_speed,92309,thermostat_f6b8d9fa-f189-4b47-9d8e-3785c1a06bbe.csv
+a2889532-7fb0-4e63-a85b-7fbcbe0f7fe0,none,,central,single_speed,92201,thermostat_a2889532-7fb0-4e63-a85b-7fbcbe0f7fe0.csv
+4b82fc6c-9beb-4d64-b5bf-3d9b85003273,heat_pump_no_electric_backup,single_stage,heat_pump,,96128,thermostat_4b82fc6c-9beb-4d64-b5bf-3d9b85003273.csv
diff --git a/tests/data/metadata_20181022.csv b/tests/data/metadata_20181022.csv
@@ -1,2 +1,2 @@
-thermostat_id,heat_type,heat_stage,cool_type,cool_stage,zipcode,utc_offset,interval_data_filename
-411999234671,none,,central,single_stage,33155,-7,411999234671.csv
+thermostat_id,heat_type,heat_stage,cool_type,cool_stage,zipcode,interval_data_filename
+411999234671,none,,central,single_stage,33155,411999234671.csv