Merge 1313f40 into 0103bf8

.github/workflows/pythonpackage.yml

@@ -1,11 +1,11 @@
 name: Python package
 
 on: ["pull_request"]
-
 jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
@@ -24,3 +24,11 @@ jobs:
         run: |
           pip install pytest pytest-cov
           pytest --cov-report=xml
+      - name: Coveralls
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+        run: |
+          pip install coveralls
+          coverage run --source=nemwriter -m pytest tests/
+          coveralls

nemreader/nem_reader.py

@@ -31,7 +31,7 @@
 class NEMFile:
     """An NEM file object"""
 
-    def __init__(self, file_path, fileobj=None, strict: bool=False) -> None:
+    def __init__(self, file_path, fileobj=None, strict: bool = False) -> None:
         self.file_path = file_path
         self.fileobj = fileobj
         self.strict = strict
@@ -111,16 +111,16 @@ def nem_data(self) -> NEMData:
         except zipfile.BadZipFile:
             """Not a zip"""
             if not isinstance(self.fileobj, io.IOBase):
-                datafile = open(self.file_path)
+                datafile = open(self.file_path)  # noqa: SIM115
             else:
                 """If we've been given a binary IO stream change it"""
                 if not isinstance(self.fileobj, io.TextIOBase):
                     datafile = self.fileobj.read().decode("utf-8")
                 else:
                     datafile = self.fileobj
             reads = self.parse_nem_file(datafile, file_name=self.file_path)
-                
-        for nmi in reads.transactions.keys():
+
+        for nmi in reads.transactions:
             self._nmis.add(nmi)
             suffixes = list(reads.transactions[nmi].keys())
             self._nmi_channels[nmi] = suffixes
@@ -194,24 +194,30 @@ def get_pivot_data_frame(
         for col in df.columns:
             if "value_" in col:
                 new_names[col] = col[6:]
+
             if "quality" in col:
-                if not quality:
+                perc_nans = df[col].isna().sum() / len(df[col])
+                if not quality and perc_nans < 0.5:
                     new_names[col] = "quality"
+                    quality += 1
                 else:
                     del df[col]
-                quality += 1
+
             if "evt_code" in col:
-                if not evt_code:
+                perc_nans = df[col].isna().sum() / len(df[col])
+                if not evt_code and perc_nans < 0.5:
                     new_names[col] = "evt_code"
+                    evt_code += 1
                 else:
                     del df[col]
-                evt_code += 1
+
             if "evt_desc" in col:
-                if not evt_desc:
+                perc_nans = df[col].isna().sum() / len(df[col])
+                if not evt_desc and perc_nans < 0.5:
                     new_names[col] = "evt_desc"
+                    evt_desc += 1
                 else:
                     del df[col]
-                evt_desc += 1
         df.rename(columns=new_names, inplace=True)
         return df
 
@@ -281,7 +287,8 @@ def parse_nem12_rows(nem_list: Iterable, file_name=None) -> NEMReadings:
                 # try to keep parsing anyway.
                 if observed_900_records:
                     log.warning(
-                        f"Found multiple end of data (900) rows on lines {observed_900_records}"
+                        "Found multiple end of data (900) rows on lines %s",
+                        observed_900_records,
                     )
 
                 observed_900_records.append(row_num)
@@ -449,12 +456,14 @@ def parse_200_row(row: list) -> NmiDetails:
 def parse_250_row(row: list) -> BasicMeterData:
     """Parse basic meter data record (250)
 
-        RecordIndicator,NMI,NMIConfiguration,RegisterID,NMISuffix,MDMDataStreamIdentifier,MeterSeri
-    alNumber,DirectionIndicator,PreviousRegisterRead,PreviousRegisterReadDateTime,PreviousQuali
-    tyMethod,PreviousReasonCode,PreviousReasonDescription,CurrentRegisterRead,CurrentRegister
-    ReadDateTime,CurrentQualityMethod,CurrentReasonCode,CurrentReasonDescription,Quantity,U
-    OM,NextScheduledReadDate,UpdateDateTime,MSATSLoadDateTime
-        Example: 250,1234567890,1141,01,11,11,METSER66,E,000021.2,20031001103230,A,,,000534.5,20040201100030,E64,77,,343.5,kWh,20040509, 20040202125010,20040203000130
+    RecordIndicator,NMI,NMIConfiguration,RegisterID,NMISuffix,
+    MDMDataStreamIdentifier,MeterSerialNumber,DirectionIndicator,
+    PreviousRegisterRead,PreviousRegisterReadDateTime,PreviousQualityMethod,
+    PreviousReasonCode,PreviousReasonDescription,CurrentRegisterRead,
+    CurrentRegisterReadDateTime,CurrentQualityMethod,CurrentReasonCode,
+    CurrentReasonDescription,Quantity,UOM,
+    NextScheduledReadDate,UpdateDateTime,MSATSLoadDateTime
+
     """
     return BasicMeterData(
         row[1],
@@ -508,8 +517,8 @@ def parse_300_row(
     if len(row) < num_intervals + num_required_non_reading_fields:
         num_rows = len(row) - num_required_non_reading_fields
         raise ValueError(
-            "Unexpected number of values in 300 row: " +
-            f"{num_rows} readings for {interval}min intervals"
+            "Unexpected number of values in 300 row: "
+            + f"{num_rows} readings for {interval}min intervals"
         )
     quality_method = row[last_interval]
 
@@ -593,17 +602,17 @@ def parse_400_row(row: list, interval_length: int) -> tuple:
     start_interval = int(row[1])
     end_interval = int(row[2])
     if end_interval < start_interval:
-        raise ValueError(
-            f"End interval {end_interval} is earlier than start interval {start_interval} in 400 row."
-        )
+        msg = f"End interval {end_interval} is earlier than "
+        msg += f"start interval {start_interval} in 400 row."
+        raise ValueError(msg)
     if not (0 < start_interval <= num_intervals):
-        raise ValueError(
-            f"Invalid start interval {start_interval} in 400 row. Expecting {num_intervals} intervals."
-        )
+        msg = f"Invalid start interval {start_interval} in 400 row."
+        msg += " Expecting {num_intervals} intervals."
+        raise ValueError(msg)
     if not (0 < end_interval <= num_intervals):
-        raise ValueError(
-            f"Invalid end interval {end_interval} in 400 row. Expecting {num_intervals} intervals."
-        )
+        msg = f"Invalid end interval {end_interval} in 400 row."
+        msg += f"Expecting {num_intervals} intervals."
+        raise ValueError(msg)
 
     return EventRecord(int(row[1]), int(row[2]), row[3], row[4], row[5])
 

nemreader/output_db.py

@@ -187,7 +187,7 @@ def calc_nmi_daily_summary(db_path: Path, nmi: str):
     for ch in channels:
         if ch[0] not in ["B", "E"]:
             continue  # Skip other channels
-        feed_in = True if ch[0] == "B" else False
+        feed_in = ch[0] == "B"
         for read in get_nmi_readings(db_path, nmi, ch):
             day = read.start.strftime("%Y-%m-%d")
             if feed_in:
@@ -230,7 +230,8 @@ def extend_sqlite(db_path: Path) -> None:
     db.create_view(
         "combined_readings",
         """
-    SELECT nmi, t_start, t_end, SUM(CASE WHEN substr(channel,1,1) = 'B' THEN -1 * value ELSE value END) as value
+    SELECT nmi, t_start, t_end, 
+    SUM(CASE WHEN substr(channel,1,1) = 'B' THEN -1 * value ELSE value END) as value
     FROM readings
     GROUP BY nmi, t_start, t_end
     ORDER BY 1, 2

nemreader/split_days.py

@@ -55,7 +55,11 @@ def split_reading_into_days(start, end, val):
             period_end = end
         period_secs = (period_end - period_start).total_seconds()
         period_val = val * (period_secs / total_secs)
-        yield period_start, period_end, period_val,
+        yield (
+            period_start,
+            period_end,
+            period_val,
+        )
         period_start += timedelta(days=1)
         period_end += timedelta(days=1)
 

nemreader/version.py

@@ -1 +1 @@
-__version__ = "0.9.0"
+__version__ = "0.9.1"

pyproject.toml

@@ -13,7 +13,6 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.8",
     "Operating System :: OS Independent",
 ]
 keywords = ["energy", "NEM12", "NEM13"]

requirements.txt

@@ -7,34 +7,44 @@ click==8.1.7
     #   typer
 click-default-group==1.2.4
     # via sqlite-utils
-coverage==7.4.4
+coverage==7.5.3
     # via pytest-cov
-exceptiongroup==1.2.0
+exceptiongroup==1.2.1
     # via pytest
 iniconfig==2.0.0
     # via pytest
-mypy==1.9.0
+markdown-it-py==3.0.0
+    # via rich
+mdurl==0.1.2
+    # via markdown-it-py
+mypy==1.10.0
 mypy-extensions==1.0.0
     # via mypy
 numpy==1.26.4
     # via pandas
 packaging==24.0
     # via pytest
-pandas==2.2.1
-pluggy==1.4.0
+pandas==2.2.2
+pluggy==1.5.0
     # via
     #   pytest
     #   sqlite-utils
-pytest==8.1.1
+pygments==2.18.0
+    # via rich
+pytest==8.2.1
     # via pytest-cov
-pytest-cov==4.1.0
+pytest-cov==5.0.0
 python-dateutil==2.9.0.post0
     # via
     #   pandas
     #   sqlite-utils
 pytz==2024.1
     # via pandas
-ruff==0.3.4
+rich==13.7.1
+    # via typer
+ruff==0.4.6
+shellingham==1.5.4
+    # via typer
 six==1.16.0
     # via python-dateutil
 sqlite-fts4==1.0.3
@@ -47,8 +57,8 @@ tomli==2.0.1
     #   coverage
     #   mypy
     #   pytest
-typer==0.9.0
-typing-extensions==4.10.0
+typer==0.12.3
+typing-extensions==4.12.0
     # via
     #   mypy
     #   typer

tests/test_actual_interval.py

@@ -3,7 +3,7 @@
 from nemreader import NEMFile
 
 
-def test_correct_NMIs():
+def test_correct_nmis():
     nf = NEMFile("examples/unzipped/Example_NEM12_actual_interval.csv", strict=True)
     meter_data = nf.nem_data()
     assert len(meter_data.readings) == 1

tests/test_dataframe_pivot.py

@@ -21,3 +21,17 @@ def test_pivot_dataframe():
     for suffix in ["t_start", "E1", "E2", "V1", "quality", "evt_code", "evt_desc"]:
         assert suffix in df.columns
     assert len(df) == 48
+
+
+def test_pivot_dataframe_partialchannel():
+    """Test example where some days are missing from first channel in file"""
+    nf = NEMFile("examples/unzipped/Example_NEM12_partialchannel.csv", strict=True)
+
+    # The intervals are different, so we have 48 30min + 144 10min = 192 rows
+    df = nf.get_pivot_data_frame(set_interval=30)
+    for suffix in ["t_start", "B1", "E1", "quality", "evt_code", "evt_desc"]:
+        assert suffix in df.columns
+    assert len(df) == 1488
+    for col in ["t_start", "E1", "quality", "evt_code", "evt_desc"]:  # only B1 has nans
+        perc_nans = df[col].isna().sum() / len(df[col])
+        assert perc_nans == 0.0

tests/test_header_datestamp.py

@@ -3,7 +3,7 @@
 from nemreader import NEMFile
 
 
-def test_Date12_parse():
+def test_date12_parse():
     """test that Date12 is parsed correctly"""
     # 200402070911 = 2004-02-07, 09:11
     nf = NEMFile("examples/unzipped/Example_NEM12_multiple_meters.csv", strict=True)

tests/test_incomplete_files.py

@@ -3,7 +3,7 @@
 from nemreader import NEMFile
 
 
-def test_correct_NMIs():
+def test_correct_nmis_powercor():
     nf = NEMFile("examples/invalid/Example_NEM12_powercor.csv", strict=False)
     meter_data = nf.nem_data()
     assert len(meter_data.readings) == 1

tests/test_open_examples.py

@@ -26,11 +26,12 @@ def test_blob_load():
             continue
         test_filename = os.path.join(test_path, file_name)
         """Blob load here"""
-        with open(test_filename, 'rb') as test_file:
+        with open(test_filename, "rb") as test_file:
             nf = NEMFile(test_file, strict=True)
             meter_data = nf.nem_data()
             assert meter_data.header.version_header in ["NEM12", "NEM13"]
 
+
 def test_nem12_examples():
     """Open and parse zipped NEM12 example files"""
     skips = [