Merge pull request #21 from aguinane/develop

v0.5.2

Example_NEM12_incomplete_interval_daily_totals.csv

@@ -0,0 +1,4 @@
+nmi,meter_sn,day,channel,day_total,uom,quality_method
+VABD000163,METSER123,20040202,E1,53.32799999999995,kWh,A
+VABD000163,METSER123,20040201,Q1,106.6559999999999,kVArh,A
+VABD000163,METSER123,20040202,Q1,106.6559999999999,kVArh,A

README.md

@@ -51,3 +51,36 @@ period_start,period_end,E1,Q1,quality_method
 2004-02-01 00:30:00,2004-02-01 01:00:00,1.111,2.222,A
 ...
 ```
+
+## Charting
+
+You can easily chart the usage data using pandas:
+
+```python
+import matplotlib.pyplot as plt
+from nemreader import output_as_data_frames
+
+# Setup Pandas DataFrame
+dfs = output_as_data_frames("examples/nem12/NEM12#000000000000002#CNRGYMDP#NEMMCO.zip")
+nmi, df = dfs[0] # Return data for first NMI in file
+df.set_index("period_start", inplace=True)
+
+# Chart time of day profile
+hourly = df.groupby([(df.index.hour)]).sum()
+plot = hourly.plot(title=nmi, kind="bar", y=["E1"])
+plt.show()
+```
+
+!["Time of day plot"](docs/plot_profile.png)
+
+Or even generate a calendar with daily usage totals:
+
+```python
+# Chart daily usage calendar
+import pandas as pd
+import calmap
+plot = calmap.calendarplot(pd.Series(df.E1), daylabels="MTWTFSS")
+plt.show()
+```
+
+!["Calendar Plot"](docs/plot_cal.png)

examples/invalid/Example_NEM12_incomplete_interval.csv

@@ -0,0 +1,8 @@
+100,NEM12,200405011135,MDA1,Ret1
+200,VABD000163,E1Q1,1,E1,N1,METSER123,kWh,30,
+300,20040201,V,,,20040202120025,
+300,20040202,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,1.111,A,,,20040202120025,20040202142516
+200,VABD000163,E1Q1,2,Q1,,METSER123,kVArh,30,
+300,20040201,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,A,,,20040202120025,
+300,20040202,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,2.222,A,,,20040202120025,
+900

nemreader/cli.py

@@ -23,8 +23,13 @@ def cli():
 @click.option("-v", "--verbose", is_flag=True, help="Will print verbose messages.")
 def list_nmis(nemfile, verbose):
     """ Output list of NMIs in NEM file """
+
     if verbose:
-        logging.basicConfig(level="DEBUG", format=LOG_FORMAT)
+        log_level = "DEBUG"
+    else:
+        log_level = "WARNING"
+    logging.basicConfig(level=log_level, format=LOG_FORMAT)
+
     nmis = list(nmis_in_file(nemfile))
     click.echo("The following NMI[suffix] exist in this file:")
     for nmi, suffixes in nmis:
@@ -48,7 +53,10 @@ def output(nemfile, verbose, outdir):
     NEMFILE is the name of the file to parse.
     """
     if verbose:
-        logging.basicConfig(level="DEBUG", format=LOG_FORMAT)
+        log_level = "DEBUG"
+    else:
+        log_level = "WARNING"
+    logging.basicConfig(level=log_level, format=LOG_FORMAT)
     for fname in output_as_csv(nemfile, output_dir=outdir):
         click.echo(f"Created {fname}")
 
@@ -69,6 +77,9 @@ def output_daily(nemfile, verbose, outdir):
     NEMFILE is the name of the file to parse.
     """
     if verbose:
-        logging.basicConfig(level="DEBUG", format=LOG_FORMAT)
+        log_level = "DEBUG"
+    else:
+        log_level = "WARNING"
+    logging.basicConfig(level=log_level, format=LOG_FORMAT)
     fname = output_as_daily_csv(nemfile, output_dir=outdir)
     click.echo(f"Created {fname}")

nemreader/nem_reader.py

@@ -29,7 +29,7 @@ def read_nem_file(file_path: str, ignore_missing_header=False) -> NEMFile:
     """ Read in NEM file and return meter readings named tuple
 
     :param file_path: The NEM file to process
-    :param ignore_missing_header: Whether to continue parsing if missing header. 
+    :param ignore_missing_header: Whether to continue parsing if missing header.
                                   Will assume NEM12 format.
     :returns: The file that was created
     """
@@ -156,9 +156,13 @@ def parse_nem12_rows(
 
         elif record_indicator == 300:
             num_intervals = int(24 * 60 / nmi_d.interval_length)
-            assert len(row) > num_intervals, "Incomplete 300 Row in {}".format(
-                file_name
-            )
+            assert len(row) > 1, f"Invalid 300 Row in {file_name}"
+            if len(row) < num_intervals + 2:
+                record_date = row[1]
+                msg = "Skipping 300 record for %s %s %s. "
+                msg += "It does not have the expected %s intervals"
+                log.error(msg, nmi_d.nmi, nmi_d.nmi_suffix, record_date, num_intervals)
+                continue
             interval_record = parse_300_row(
                 row, nmi_d.interval_length, nmi_d.uom, nmi_d.meter_serial_number
             )

nemreader/outputs.py

@@ -94,19 +94,24 @@ def output_as_csv(file_name, output_dir="."):
         last_date = rows[-1][1]
         output_file = "{}_{}_transposed.csv".format(nmi, last_date.strftime("%Y%m%d"))
         output_path = output_dir / output_file
-        with open(output_path, "w", newline="") as csvfile:
-            cwriter = csv.writer(
-                csvfile, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL
-            )
-            cwriter.writerow(headings)
-            for row in rows:
-                cwriter.writerow(row)
-
-        log.debug("Created %s", output_path)
+        save_to_csv(headings, rows, output_path)
         output_paths.append(output_path)
     return output_paths
 
 
+def save_to_csv(headings: List[str], rows: List[list], output_path):
+    """ save data to csv file """
+    with open(output_path, "w", newline="") as csvfile:
+        cwriter = csv.writer(
+            csvfile, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL
+        )
+        cwriter.writerow(headings)
+        for row in rows:
+            cwriter.writerow(row)
+    log.debug("Created %s", output_path)
+    return output_path
+
+
 def flatten_and_group_rows(
     nmi: str,
     nmi_transactions: Dict[str, list],
@@ -185,14 +190,6 @@ def output_as_daily_csv(file_name, output_dir="."):
         for row in rows:
             all_rows.append(row)
 
-    with open(output_path, "w", newline="") as csvfile:
-        cwriter = csv.writer(
-            csvfile, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL
-        )
-        cwriter.writerow(headings)
-        for row in all_rows:
-            cwriter.writerow(row)
-
-    log.debug("Created %s", output_path)
+    save_to_csv(headings, all_rows, output_path)
 
     return output_path

nemreader/version.py

@@ -1 +1 @@
-__version__ = "0.5.1"
+__version__ = "0.5.2"

setup.py

@@ -35,7 +35,7 @@
 setup(
     name="nemreader",
     version=about["__version__"],
-    description="Parse NEM12 (interval metering data) and NEM13 (accumulated metering data) data files ",
+    description="Parse NEM12 and NEM13 metering data files",
     packages=find_packages(exclude=["contrib", "docs", "tests*"]),
     include_package_data=True,
     long_description=long_description,

tests/test_actual_interval.py

@@ -1,11 +1,4 @@
-""" Test Suite
-"""
-
 import pytest
-import os
-import sys
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 import nemreader as nr
 
 

tests/test_cli.py

@@ -1,9 +1,5 @@
-import os
-import sys
 import pytest
 from click.testing import CliRunner
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 from nemreader.cli import cli
 
 

tests/test_consumption_data.py

@@ -2,10 +2,6 @@
 """
 
 import pytest
-import os
-import sys
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 import nemreader as nr
 
 

tests/test_file_outputs.py

@@ -1,9 +1,3 @@
-""" Test Suite """
-
-import os
-import sys
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 from nemreader import output_as_csv
 from nemreader import output_as_daily_csv
 from nemreader import output_as_data_frames

tests/test_header_datestamp.py

@@ -1,11 +1,5 @@
-""" Test Suite
-"""
-
 import os
-import sys
 import datetime
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 import nemreader as nr
 
 

tests/test_powercor.py → tests/test_incomplete_files.py

@@ -1,25 +1,26 @@
-""" Test Suite
-"""
-
 import pytest
-import os
-import sys
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 import nemreader as nr
 
 
 def test_correct_NMIs():
     meter_data = nr.read_nem_file(
-        "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True
+        "examples/invalid/Example_NEM12_powercor.csv", ignore_missing_header=True
+    )
+    assert len(meter_data.readings) == 1
+    assert "VABD000163" in meter_data.readings
+
+
+def test_incomplete_interval_row():
+    meter_data = nr.read_nem_file(
+        "examples/invalid/Example_NEM12_incomplete_interval.csv"
     )
     assert len(meter_data.readings) == 1
     assert "VABD000163" in meter_data.readings
 
 
 def test_correct_channels():
     meter_data = nr.read_nem_file(
-        "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True
+        "examples/invalid/Example_NEM12_powercor.csv", ignore_missing_header=True
     )
     readings = meter_data.readings["VABD000163"]
     assert len(readings) == 2
@@ -29,7 +30,7 @@ def test_correct_channels():
 
 def test_correct_records():
     meter_data = nr.read_nem_file(
-        "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True
+        "examples/invalid/Example_NEM12_powercor.csv", ignore_missing_header=True
     )
     readings = meter_data.readings["VABD000163"]
 
@@ -44,12 +45,12 @@ def test_correct_records():
 
 
 def test_zipped_load():
-    meter_data = nr.read_nem_file(
-        "examples/nem12/Example_NEM12_powercor.csv.zip", ignore_missing_header=True
+    nr.read_nem_file(
+        "examples/invalid/Example_NEM12_powercor.csv.zip", ignore_missing_header=True
     )
 
 
 def test_missing_fields():
-    meter_data = nr.read_nem_file(
-        "examples/unzipped/Example_NEM12_powercor.csv", ignore_missing_header=True
+    nr.read_nem_file(
+        "examples/invalid/Example_NEM12_powercor.csv", ignore_missing_header=True
     )

tests/test_multiple_quality.py

@@ -1,11 +1,4 @@
-""" Test Suite
-"""
-
 import pytest
-import os
-import sys
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 import nemreader as nr
 
 

tests/test_open_examples.py

@@ -1,11 +1,4 @@
-""" Test Suite
-"""
-
-import pytest
 import os
-import sys
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 import nemreader as nr