Merge pull request #1914 from SEED-platform/gb-meter-import-fix

GreenButton meter import unit reading fixes and improvements
SEED-platform · Jul 11, 2019 · 04c106f · 04c106f
2 parents a3cce7e + 3632934
commit 04c106f
Show file tree

Hide file tree

Showing 3 changed files with 181 additions and 34 deletions.
diff --git a/seed/lib/mcm/reader.py b/seed/lib/mcm/reader.py
@@ -134,57 +134,94 @@ def data(self):
 
     def _parse_type_and_unit(self, raw_data):
         """
-        Uses the kind and uom/powerOfTenMultiplier to parse type and
-        raw unit, respectively.
-
-        For the given type, it first scans the valid units for that type to see
-        if the raw unit (including prefix) can be matched exactly to one of those
-        valid units.
-
-        If an exact match is not found, it scans those valid units again to find
-        an approximate match for the raw base unit (without the prefix). If an
-        approximate match is found, the prefix/powerOfTenMultiplier is used to
-        calculate the multiplier needed to convert readings from the
-        raw unit (including prefix) to the valid unit found as an approximate match.
+        Parses raw XML to read the kind and uom/powerOfTenMultiplier. Using
+        those, an attempt is made to validate the type and unit as a combination
+        that the application accepts.
+
+        The if the type and unit are parsable and valid, they are returned,
+        otherwise, None is returned as applicable.
         """
         kind_entry = raw_data['feed']['entry'][0]
         kind = kind_entry['content']['UsagePoint']['ServiceCategory']['kind']
         type = self.kind_codes.get(int(kind), None)
 
-        uom_entry = raw_data['feed']['entry'][2]
-        uom = uom_entry['content']['ReadingType']['uom']
+        if type is None:
+            return None, None, 1
+
+        uom_entry = raw_data['feed']['entry'][2]['content']['ReadingType']
+        uom = uom_entry['uom']
         raw_base_unit = self.uom_codes.get(int(uom), '')
 
-        power_of_ten_multiplier = int(uom_entry['content']['ReadingType']['powerOfTenMultiplier'])
-        raw_prefix_unit = self.power_of_ten_codes.get(power_of_ten_multiplier, None)
+        power_of_ten_multiplier = int(uom_entry['powerOfTenMultiplier'])
+
+        resulting_unit, multiplier = self._parse_valid_unit_and_multiplier(
+            type,
+            power_of_ten_multiplier,
+            raw_base_unit
+        )
+
+        return type, resulting_unit, multiplier
+
+    def _parse_valid_unit_and_multiplier(self, type, power_of_ten_multiplier, raw_base_unit):
+        """
+        Parses valid/accepted unit and multiplier using the given type and raw
+        base unit. The powerOfTenMultiplier is used to find the raw unit prefix
+        which is used along with the raw base unit to try to accomplish this.
+
+        The 3 scenarios accounted for are the following in order:
+        - prefix and base == valid unit (or "starts with")
+        - base == valid unit
+        - base similar to a valid unit
+
+        In the first scenario, no multiplier is needed since the provided readings
+        are already given in a unit that's understood by the application.
+
+        In the second scenario, the multiplier is generated directly from the
+        powerOfTenMultiplier without any adjustments.
 
-        raw_unit = "{}{}".format(raw_prefix_unit, raw_base_unit)
+        In the last scenario, the multiplier is generated directly from the
+        powerOfTenMultiplier with an adjustment made to convert the readings
+        into a unit understood by the application.
 
+        If none of these scenarios return a validated unit, None, 1 is returned.
+        """
         valid_units_for_type = self._thermal_factors[type].keys()
 
-        exact_match_unit = next(
-            (key for key in valid_units_for_type if key.startswith(raw_unit)),
+        raw_prefix_unit = self.power_of_ten_codes.get(power_of_ten_multiplier, None)
+        raw_full_unit = "{}{}".format(raw_prefix_unit, raw_base_unit)
+
+        # Check if the raw full unit is an exact match (or left match) with a known valid unit
+        exact_match_full_unit = next(
+            (key for key in valid_units_for_type if key.startswith(raw_full_unit)),
             None
         )
+        if exact_match_full_unit is not None:
+            return exact_match_full_unit, 1
 
-        resulting_unit = None
-        multiplier = 1
-        if exact_match_unit is not None:
-            resulting_unit = exact_match_unit
-        else:
-            approx_base_unit_match = next(
-                (key for key in valid_units_for_type if raw_base_unit in key),
-                None
-            )
-            if approx_base_unit_match is not None:
-                factor_prefix = approx_base_unit_match[0]
+        # Check if just the base unit is an exact match with a known valid unit
+        base_unit_only_match = next(
+            (key for key in valid_units_for_type if raw_base_unit == key),
+            None
+        )
+        if base_unit_only_match is not None:
+            multiplier = 10**(power_of_ten_multiplier)
+            return base_unit_only_match, multiplier
 
-                # an exact match is expected for factor_prefix - if not, this should error
-                multiplier = 10**(power_of_ten_multiplier - self.thermal_factor_prefixes[factor_prefix])
+        # Check if just the base unit is similar to a known valid unit
+        approx_match_base_unit = next(
+            (key for key in valid_units_for_type if raw_base_unit in key),
+            None
+        )
+        if approx_match_base_unit is not None:
+            # this assumes the prefix is one character long
+            factor_prefix = approx_match_base_unit[0]
 
-                resulting_unit = approx_base_unit_match
+            # an exact match is expected for factor_prefix - if not, this should error
+            multiplier = 10**(power_of_ten_multiplier - self.thermal_factor_prefixes[factor_prefix])
 
-        return type, resulting_unit, multiplier
+            return approx_match_base_unit, multiplier
+
+        return None, 1
 
 
 class GeoJSONParser(object):

diff --git a/seed/lib/mcm/tests/test_data/greenbutton/example-GreenButton-data-gas-therms--3.xml b/seed/lib/mcm/tests/test_data/greenbutton/example-GreenButton-data-gas-therms--3.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<feed xmlns="http://www.w3.org/2005/Atom" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://naesb.org/espi espi.xsd">
+  <id>urn:uuid:5762c9e8-4e65-3b0c-83b3-7874683f3dbe</id>
+  <link href="/v1/espi_third_party_batch_feed" rel="self">
+  </link>
+  <title type="text">Opower ESPI Third Party Batch Feed v1</title>
+  <updated>2012-04-06T20:04:14.983Z</updated>
+  <entry>
+    <id>urn:uuid:7c0ea8fe-646f-32fa-b415-37b370e8edee</id>
+    <link href="/v1/User/6150855/UsagePoint/409483" rel="self">
+    </link>
+    <link href="/v1/User/6150855/UsagePoint/409483/MeterReading/1" rel="related">
+    </link>
+    <title type="text">A10S Large Est Values</title>
+    <updated>2012-04-06T20:04:14.983Z</updated>
+    <published>2011-11-30T12:00:00.000Z</published>
+    <content type="xml">
+      <UsagePoint xmlns="http://naesb.org/espi">
+        <ServiceCategory>
+          <kind>1</kind>
+        </ServiceCategory>
+      </UsagePoint>
+    </content>
+  </entry>
+  <entry>
+    <id>urn:uuid:64cfa7a1-aae7-305a-8d73-19f29f52a0b0</id>
+    <link href="/v1/User/6150855/UsagePoint/409483/MeterReading/1" rel="self">
+    </link>
+    <link href="/v1/ReadingType/1" rel="related">
+    </link>
+    <link href="/v1/User/6150855/UsagePoint/409483/MeterReading/1/IntervalBlock/1" rel="related">
+    </link>
+    <updated>2012-04-06T20:04:14.983Z</updated>
+    <published>2011-11-30T12:00:00.000Z</published>
+    <content type="xml">
+      <MeterReading xmlns="http://naesb.org/espi">
+      </MeterReading>
+    </content>
+  </entry>
+  <entry>
+    <id>urn:uuid:4e1226d5-5172-3fdf-adf6-4001aee94849</id>
+    <link href="/v1/ReadingType/1" rel="self">
+    </link>
+    <updated>2012-04-06T20:04:14.983Z</updated>
+    <published>2011-11-30T12:00:00.000Z</published>
+    <content type="xml">
+      <ReadingType xmlns="http://naesb.org/espi">
+        <powerOfTenMultiplier>-3</powerOfTenMultiplier>
+        <uom>169</uom>
+      </ReadingType>
+    </content>
+  </entry>
+  <entry>
+    <id>urn:uuid:e50a62c2-aa08-3348-9f17-fe7893558949</id>
+    <link href="/v1/User/6150855/UsagePoint/409483/MeterReading/1/IntervalBlock/1" rel="self">
+    </link>
+    <content type="xml">
+      <IntervalBlock xmlns="http://naesb.org/espi">
+        <interval>
+          <duration>34383600</duration>
+          <start>1299387600</start>
+        </interval>
+        <IntervalReading>
+          <timePeriod>
+            <duration>900</duration>
+            <start>1299387600</start>
+          </timePeriod>
+          <value>1790</value>
+        </IntervalReading>
+        <IntervalReading>
+          <timePeriod>
+            <duration>900</duration>
+            <start>1299388500</start>
+          </timePeriod>
+          <value>1792</value>
+        </IntervalReading>
+      </IntervalBlock>
+    </content>
+  </entry>
+</feed>
diff --git a/seed/lib/mcm/tests/test_reader_greenbuttonparser.py b/seed/lib/mcm/tests/test_reader_greenbuttonparser.py
@@ -10,6 +10,7 @@
 
 class GreenButtonParserTest(TestCase):
     def test_data_property_can_handle_electricity_wh(self):
+        # Case when powerOfTenMultiplier + base unit = exact match of known unit
         file_path = os.path.dirname(os.path.abspath(__file__)) + "/test_data/greenbutton/example-GreenButton-data-electricity-wh.xml"
         file = open(file_path, "r", encoding="utf-8")
         parser = GreenButtonParser(file)
@@ -36,6 +37,7 @@ def test_data_property_can_handle_electricity_wh(self):
         self.assertEqual(parser.data, expectation)
 
     def test_data_property_can_handle_gas_MBtu(self):
+        # Different case when powerOfTenMultiplier + base unit = exact match of known unit
         file_path = os.path.dirname(os.path.abspath(__file__)) + "/test_data/greenbutton/example-GreenButton-data-gas-MBtu.xml"
         file = open(file_path, "r", encoding="utf-8")
         parser = GreenButtonParser(file)
@@ -62,6 +64,7 @@ def test_data_property_can_handle_gas_MBtu(self):
         self.assertEqual(parser.data, expectation)
 
     def test_data_property_can_handle_gas_J_with_power_of_ten_of_negative_3(self):
+        # Case when base unit approximated and powerOfTenMultiplier used as conversion
         file_path = os.path.dirname(os.path.abspath(__file__)) + "/test_data/greenbutton/example-GreenButton-data-gas-J--3.xml"
         file = open(file_path, "r", encoding="utf-8")
         parser = GreenButtonParser(file)
@@ -87,6 +90,33 @@ def test_data_property_can_handle_gas_J_with_power_of_ten_of_negative_3(self):
 
         self.assertEqual(parser.data, expectation)
 
+    def test_data_property_can_handle_gas_therms_with_power_of_ten_of_negative_3(self):
+        # Case when only base unit == exact match and powerOfTenMultiplier used as conversion
+        file_path = os.path.dirname(os.path.abspath(__file__)) + "/test_data/greenbutton/example-GreenButton-data-gas-therms--3.xml"
+        file = open(file_path, "r", encoding="utf-8")
+        parser = GreenButtonParser(file)
+
+        expectation = [
+            {
+                'start_time': 1299387600,
+                'source_id': 'User/6150855/UsagePoint/409483/MeterReading/1/IntervalBlock/1',
+                'duration': 900,
+                'Meter Type': 'Natural Gas',
+                'Usage Units': 'therms',
+                'Usage/Quantity': 1790.0 / 10**3,
+            },
+            {
+                'start_time': 1299388500,
+                'source_id': 'User/6150855/UsagePoint/409483/MeterReading/1/IntervalBlock/1',
+                'duration': 900,
+                'Meter Type': 'Natural Gas',
+                'Usage Units': 'therms',
+                'Usage/Quantity': 1792.0 / 10**3,
+            }
+        ]
+
+        self.assertEqual(parser.data, expectation)
+
     def test_data_property_can_handle_invalid_energy_type_of_time(self):
         file_path = os.path.dirname(os.path.abspath(__file__)) + "/test_data/greenbutton/example-GreenButton-data-invalid-time-service-kind.xml"
         file = open(file_path, "r", encoding="utf-8")