Merge pull request pvcaptest#99 from bt-/load-data-tweaks

Improves load_data

- Adds reading site data from a file
- Adds check for any files loaded when loading from a dir

CHANGELOG.md

@@ -5,7 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 [Unreleased]
+### Added
+- `load_data` can read site location and system information for generating clear sky irradiance from a json or yaml file.
 
+### Changed
+- `load_data` now checks if any individual files were loaded when loading multiple files from a directory.
 
 [0.12.0]: https://github.com/pvcaptest/pvcaptest/compare/v0.11.2...v0.12.0
 ## [0.12.0] - 2023-08-27

src/captest/io.py

@@ -1,4 +1,5 @@
 # this file is formatted with black
+import copy
 import dateutil
 import datetime
 from pathlib import Path
@@ -416,12 +417,18 @@ def load(self, extension="csv", verbose=True, print_errors=False, **kwargs):
                         print(err)
                     failed_to_load_count += 1
                     continue
-            (
-                self.loaded_files,
-                self.common_freq,
-                self.file_frequencies,
-            ) = self._reindex_loaded_files()
-            data = self._join_files()
+            if len(self.loaded_files) == 0:
+                warnings.warn(
+                    "No files were loaded. Check that file_reader is working")
+            elif len(self.loaded_files) > 1:
+                (
+                    self.loaded_files,
+                    self.common_freq,
+                    self.file_frequencies,
+                ) = self._reindex_loaded_files()
+                data = self._join_files()
+            elif len(self.loaded_files) == 1:
+                data = list(self.loaded_files.values())[0]
             data.index.name = "Timestamp"
             self.data = data
         else:
@@ -492,13 +499,13 @@ def load_data(
         By default will create a new index for the data using the earliest datetime,
         latest datetime, and the most frequent time interval ensuring there are no
         missing intervals.
-    site : dict, default None
-        Pass a dictionary containing site data, which will be used to generate
-        modeled clear sky ghi and poa values. The clear sky irradiance values are
-        added to the data and the column_groups attribute is updated to include these
-        two irradiance columns. The site data dictionary should be
-        {sys: {system data}, loc: {location data}}. See the capdata.csky documentation
-        for the format of the system data and location data.
+    site : dict or str, default None
+        Pass a dictionary or path to a json or yaml file containing site data, which
+        will be used to generate modeled clear sky ghi and poa values. The clear sky
+        irradiance values are added to the data and the column_groups attribute is
+        updated to include these two irradiance columns. The site data dictionary should
+        be {sys: {system data}, loc: {location data}}. See the capdata.csky
+        documentation for the format of the system data and location data.
     column_groups_template : bool, default False
         If True, will call `CapData.data_columns_to_excel` to save a file to use to
         manually create column groupings at `path`.
@@ -537,10 +544,19 @@ def load_data(
         elif (p.suffix == '.xlsx') or (p.suffix == '.xls'):
             cd.column_groups = cg.ColumnGroups(load_excel_column_groups(group_columns))
     if site is not None:
-        cd.data = csky(cd.data, loc=site['loc'], sys=site['sys'])
-        cd.data_filtered = cd.data.copy()
-        cd.column_groups['irr-poa-clear_sky'] = ['poa_mod_csky']
-        cd.column_groups['irr-ghi-clear_sky'] = ['ghi_mod_csky']
+        if isinstance(site, str):
+            path_to_site = Path(site)
+            if path_to_site.is_file():
+                if path_to_site.suffix == ".json":
+                    site = util.read_json(site)
+                if (path_to_site.suffix == ".yaml") or (path_to_site.suffix == ".yml"):
+                    site = util.read_yaml(site)
+                cd.site = copy.deepcopy(site)
+        if isinstance(site, dict):
+            cd.data = csky(cd.data, loc=site['loc'], sys=site['sys'])
+            cd.data_filtered = cd.data.copy()
+            cd.column_groups['irr-poa-clear_sky'] = ['poa_mod_csky']
+            cd.column_groups['irr-ghi-clear_sky'] = ['ghi_mod_csky']
     cd.trans_keys = list(cd.column_groups.keys())
     cd.set_plot_attributes()
     if column_groups_template:

tests/data/site_loc_orientation.json

@@ -0,0 +1,16 @@
+{
+    "loc": {
+        "latitude": 41.2,
+        "longitude": -71.5,
+        "altitude": 90,
+        "tz": "America/New_York"
+    },
+    "sys": {
+        "axis_tilt": 0,
+        "axis_azimuth": 180,
+        "max_angle": 60,
+        "backtrack": true,
+        "gcr": 0.40,
+        "albedo": 0.2
+    }
+}

tests/data/site_loc_orientation.yaml

@@ -0,0 +1,12 @@
+loc:
+  altitude: 90
+  latitude: 41.2
+  longitude: -71.5
+  tz: America/New_York
+sys:
+  albedo: 0.2
+  axis_azimuth: 180
+  axis_tilt: 0
+  backtrack: true
+  gcr: 0.4
+  max_angle: 60

tests/data/site_loc_orientation.yml

@@ -0,0 +1,12 @@
+loc:
+  altitude: 90
+  latitude: 41.2
+  longitude: -71.5
+  tz: America/New_York
+sys:
+  albedo: 0.2
+  axis_azimuth: 180
+  axis_tilt: 0
+  backtrack: true
+  gcr: 0.4
+  max_angle: 60

tests/test_CapData.py

@@ -37,7 +37,7 @@
 pytest tests/test_CapData.py::TestCapDataEmpty::test_capdata_empty
 
 To create a test coverage report (html output) with pytest:
-pytest --cov-report html --cov=captest tests/
+pytest --cov-report html --cov=src/captest tests/
 
 pytest fixtures meas, location_and_system, nrel, pvsyst, pvsyst_irr_filter, and
 nrel_clear_sky are in the ./tests/conftest.py file.

tests/test_io.py

@@ -795,6 +795,44 @@ def test_kwargs_pass_to_read_csv(self, tmp_path):
         assert isinstance(cd.data, pd.DataFrame)
         assert isinstance(cd.data.index, pd.DatetimeIndex)
 
+    def test_load_site_data_from_json_file(self):
+        """
+        Test loading site data from a json file.
+        """
+        cd = load_data(
+            path="./tests/data/example_measured_data.csv",
+            site="./tests/data/site_loc_orientation.json",
+        )
+        assert "ghi_mod_csky" in cd.data.columns
+        assert "poa_mod_csky" in cd.data.columns
+        assert "poa_mod_csky" in cd.data_filtered.columns
+        assert isinstance(cd.site, dict)
+
+    def test_load_site_data_from_yaml_file(self):
+        """
+        Test loading site data from a yaml file with 'yaml' extension.
+        """
+        cd = load_data(
+            path="./tests/data/example_measured_data.csv",
+            site="./tests/data/site_loc_orientation.yaml",
+        )
+        assert "ghi_mod_csky" in cd.data.columns
+        assert "poa_mod_csky" in cd.data.columns
+        assert "poa_mod_csky" in cd.data_filtered.columns
+        assert isinstance(cd.site, dict)
+
+    def test_load_site_data_from_yml_file(self):
+        """
+        Test loading site data from a yaml file with 'yml' extension.
+        """
+        cd = load_data(
+            path="./tests/data/example_measured_data.csv",
+            site="./tests/data/site_loc_orientation.yml",
+        )
+        assert "ghi_mod_csky" in cd.data.columns
+        assert "poa_mod_csky" in cd.data.columns
+        assert "poa_mod_csky" in cd.data_filtered.columns
+        assert isinstance(cd.site, dict)
 
 class TestLoadDataMethods(unittest.TestCase):
     """Test for load data methods without setup."""