Merge pull request #38 from matt-telstra/35-performance

.gitignore

@@ -97,3 +97,7 @@ ENV/
 /*.csv
 !examples/*.csv
 *.db
+
+# Mac metadata
+.DS_Store
+

nemreader/nem_objects.py

@@ -1,9 +1,6 @@
 from datetime import datetime
 from typing import Dict, List, NamedTuple, Optional
 
-from pydantic import BaseModel
-
-
 class HeaderRecord(NamedTuple):
     """Header record (100)"""
 
@@ -112,21 +109,34 @@ class B2BDetails13(NamedTuple):
     current_trans_code: str
     current_ret_service_order: str
 
-
-class NEMReadings(BaseModel):
+class NEMReadings:
     """Represents a meter reading"""
 
     readings: Dict[str, Dict[str, List[Reading]]]
     transactions: Dict[str, Dict[str, list]]
 
+    def __init__(self, readings, transactions):
+        self.readings = readings
+        self.transactions = transactions
 
-class NEMData(BaseModel):
+    class Config:
+        copy_on_model_validation = 'shallow' # faster
+
+class NEMData:
     """Represents a meter reading"""
 
     header: HeaderRecord
     readings: Dict[str, Dict[str, List[Reading]]]
     transactions: Dict[str, Dict[str, list]]
 
+    def __init__(self, header, readings, transactions):
+        self.header = header
+        self.readings = readings
+        self.transactions = transactions
+
     @property
     def nmis(self) -> List[str]:
         return list(self.transactions.keys())
+
+    class Config:
+        copy_on_model_validation = 'shallow' # faster

nemreader/nem_reader.py

@@ -94,28 +94,28 @@ def nem_data(self) -> NEMData:
                     # Zip file is open in binary mode
                     # So decode then convert back to list
                     nmi_file = csv_text.read().decode("utf-8").splitlines()
-                    reads = self.parse_nem_file(nmi_file, file_name=csv_file)
-                    for nmi in reads.transactions.keys():
-                        self.nmis.add(nmi)
-                        suffixes = list(reads.transactions[nmi].keys())
-                        self.nmi_channels[nmi] = suffixes
-                    return NEMData(
-                        header=self.header,
-                        readings=reads.readings,
-                        transactions=reads.transactions,
-                    )
+                reads = self.parse_nem_file(nmi_file, file_name=csv_file)
+                for nmi in reads.transactions.keys():
+                    self.nmis.add(nmi)
+                    suffixes = list(reads.transactions[nmi].keys())
+                    self.nmi_channels[nmi] = suffixes
+                return NEMData(
+                    header=self.header,
+                    readings=reads.readings,
+                    transactions=reads.transactions,
+                )
 
         with open(self.file_path) as nmi_file:
             reads = self.parse_nem_file(nmi_file)
-            for nmi in reads.transactions.keys():
-                self.nmis.add(nmi)
-                suffixes = list(reads.transactions[nmi].keys())
-                self.nmi_channels[nmi] = suffixes
-            return NEMData(
-                header=self.header,
-                readings=reads.readings,
-                transactions=reads.transactions,
-            )
+        for nmi in reads.transactions.keys():
+            self.nmis.add(nmi)
+            suffixes = list(reads.transactions[nmi].keys())
+            self.nmi_channels[nmi] = suffixes
+        return NEMData(
+            header=self.header,
+            readings=reads.readings,
+            transactions=reads.transactions,
+        )
 
     def get_data_frame(
         self, split_days: bool = False, set_interval: Optional[int] = None

pyproject.toml

@@ -19,7 +19,7 @@ classifiers = [
 keywords = ["energy", "NEM12", "NEM13"]
 requires-python = ">=3.8"
 dynamic = ["version", "description"]
-dependencies = ["pandas", "pydantic", "sqlite_utils", "typer"]
+dependencies = ["pandas", "sqlite_utils", "typer"]
 
 [project.optional-dependencies]
 test = ["pytest >=2.7.3", "pytest-cov", "mypy"]

requirements.txt

@@ -58,8 +58,6 @@ pluggy==1.0.0
     # via pytest
 pycodestyle==2.10.0
     # via flake8
-pydantic==1.10.2
-    # via nemreader (pyproject.toml)
 pyflakes==3.0.1
     # via flake8
 pytest==7.2.0
@@ -91,6 +89,4 @@ tomli==2.0.1
 typer==0.7.0
     # via nemreader (pyproject.toml)
 typing-extensions==4.4.0
-    # via
-    #   mypy
-    #   pydantic
+    # via mypy