Split NEM13 days for some output types

README.md

@@ -44,7 +44,7 @@ Most importantly, you will want to get the energy data itself:
 
 ```python
 > for nmi in m.readings:
->     for channel in m.readings[nmi]:
+>     for suffix in m.readings[nmi]:
 >         for reading in m.readings[nmi][suffix][-1:]:
 >             print(reading)
 Reading(t_start=datetime.datetime(2004, 4, 17, 23, 30), t_end=datetime.datetime(2004, 4, 18, 0, 0), read_value=14.733, uom='kWh', quality_method='S14', event='', val_start=None, val_end=None)

nemreader/outputs.py

@@ -11,6 +11,7 @@
 from pathlib import Path
 from .nem_objects import Reading
 from .nem_reader import read_nem_file
+from .split_days import split_multiday_reads
 
 log = logging.getLogger(__name__)
 
@@ -24,11 +25,17 @@ def nmis_in_file(file_name) -> Generator[Tuple[str, List[str]], None, None]:
 
 
 def flatten_rows(
-    nmi_transactions: Dict[str, list], nmi_readings: Dict[str, List[Reading]]
+    nmi_transactions: Dict[str, list],
+    nmi_readings: Dict[str, List[Reading]],
+    split_days: bool = False,
 ) -> Tuple[List[str], List[list]]:
     """ Create flattened list of NMI reading data """
 
     channels = list(nmi_transactions.keys())
+    if split_days:
+        # Split any readings that are >24 hours
+        for ch in channels:
+            nmi_readings[ch] = list(split_multiday_reads(nmi_readings[ch]))
 
     headings = ["period_start", "period_end"]
     for channel in channels:
@@ -58,7 +65,7 @@ def flatten_rows(
     return headings, rows
 
 
-def output_as_data_frames(file_name):
+def output_as_data_frames(file_name, split_days: bool = True):
     """ Return list of data frames for each NMI """
 
     import pandas as pd
@@ -67,7 +74,8 @@ def output_as_data_frames(file_name):
     nmis = list(m.readings.keys())
     data_frames = []
     for nmi in nmis:
-        headings, rows = flatten_rows(m.transactions[nmi], m.readings[nmi])
+        nmi_readings = m.readings[nmi]
+        headings, rows = flatten_rows(m.transactions[nmi], nmi_readings, split_days)
         nmi_df = pd.DataFrame(data=rows, columns=headings)
         data_frames.append((nmi, nmi_df))
 
@@ -122,8 +130,16 @@ def flatten_and_group_rows(
 
     channels = list(nmi_transactions.keys())
 
-    rows = []
+    # Datastream suffix starting with a number are Accumulated Metering Data (NEM13)
+    # Ensure no reading exceeds 24 hours
+    split_required = False
+    for ch in channels:
+        if ch[0].isdigit():
+            split_required = True
+    if split_required:
+        nmi_readings = split_multiday_reads(nmi_readings)
 
+    rows = []
     for ch in channels:
         date_totals = {}
         date_qualities = {}

nemreader/split_days.py

@@ -0,0 +1,55 @@
+from typing import Iterable, Generator
+from datetime import timedelta
+from .nem_objects import Reading
+
+
+def split_multiday_reads(
+    readings: Iterable[Reading],
+) -> Generator[Reading, None, None]:
+    """ Split readings into daily intervals if they exceed 24 hours """
+    for r in readings:
+        interval_s = int((r.t_end - r.t_start).total_seconds())
+        interval_days = interval_s / 60 / 60 / 24
+        if interval_days <= 1.0:
+            # Don't need to do anything
+            yield r
+        else:
+            for start, end, val in split_reading_into_days(
+                r.t_start, r.t_end, r.read_value
+            ):
+                yield Reading(
+                    start,
+                    end,
+                    val,
+                    r.uom,
+                    r.meter_serial_number,
+                    r.quality_method,
+                    r.event_code,
+                    r.event_desc,
+                    None,
+                    None,
+                )
+
+
+def split_reading_into_days(start, end, val):
+    """ Split a single reading into even daily readings """
+    total_secs = (end - start).total_seconds()
+    # Split first day into single day
+    next_day = start.replace(hour=0, minute=0, second=0) + timedelta(days=1)
+    fd_secs = (next_day - start).total_seconds()
+    fd_val = val * (fd_secs / total_secs)
+    yield start, next_day, fd_val
+
+    # Generate the rest of the days
+    period_start = next_day
+    period_end = period_start + timedelta(days=1)
+    if period_end >= end:
+        period_end = end
+    while period_start < end:
+        if period_end > end:
+            period_end = end
+        period_secs = (period_end - period_start).total_seconds()
+        period_val = val * (period_secs / total_secs)
+        yield period_start, period_end, period_val,
+        period_start += timedelta(days=1)
+        period_end += timedelta(days=1)

nemreader/version.py

@@ -1 +1 @@
-__version__ = "0.5.3"
+__version__ = "0.6"

tests/test_file_outputs.py

@@ -24,3 +24,13 @@ def test_data_frame_output():
     for nmi, df in output_dfs:
         assert type(nmi) == str
         assert df["quality_method"][0] == "A"
+
+
+def test_data_frame_output_nem13():
+    """ Create a pandas dataframe """
+    file_name = "examples/unzipped/Example_NEM13_forward_estimate.csv"
+    output_dfs = output_as_data_frames(file_name)
+    for nmi, df in output_dfs:
+        print(df.head())
+        assert type(nmi) == str
+        assert df["quality_method"][0] == "E64"

tests/test_open_examples.py

@@ -12,7 +12,7 @@ def test_unzipped_examples():
             continue
         test_file = os.path.join(test_path, file_name)
         meter_data = nr.read_nem_file(test_file)
-        assert meter_data.header.version_header == "NEM12"
+        assert meter_data.header.version_header in ["NEM12", "NEM13"]
 
 
 def test_nem12_examples():