Merge pull request #321 from jaredthomas68/resourcefiles

Unify resource files location

.gitignore

@@ -10,10 +10,10 @@ log/
 # needed for h2 test inputs: tests/analysis/results/
 *_disp.txt
 
-# Results
-resource_files/wind
-resource_files/solar
-resource_files/grid
+# Resource files (force add ones that are really needed in the repository)
+hopp/simulation/resource_files/wind
+hopp/simulation/resource_files/solar
+hopp/simulation/resource_files/grid
 
 .idea/*
 .idea/workspace.xml

alt_dev/run_alt.py

@@ -62,8 +62,8 @@ def init_simulation_pv():
 
     # Load in weather and price data files
     solar_file = Path(
-        __file__).parent.parent / "resource_files" / "solar" / WEATHER_FILE #"Beni_Miha" / "659265_32.69_10.90_2019.csv"
-    grid_file = Path(__file__).parent.parent / "resource_files" / "grid" / PRICE_FILE #"tunisia_est_grid_prices.csv"
+        __file__).parent.parent / "hopp" / "simulation" / "resource_files" / "solar" / WEATHER_FILE #"Beni_Miha" / "659265_32.69_10.90_2019.csv"
+    grid_file = Path(__file__).parent.parent / "hopp" / "simulation" / "resource_files" / "grid" / PRICE_FILE #"tunisia_est_grid_prices.csv"
 
     # Combine the data into a site definition
     site_info = SiteInfo(site_data, solar_resource_file=solar_file, grid_resource_file=grid_file)
@@ -151,8 +151,8 @@ def init_simulation_csp():
 
     # Load in weather and price data files
     solar_file = Path(
-        __file__).parent.parent / "resource_files" / "solar" / WEATHER_FILE #"Beni_Miha" / "659265_32.69_10.90_2019.csv"
-    grid_file = Path(__file__).parent.parent / "resource_files" / "grid" / PRICE_FILE #"tunisia_est_grid_prices.csv"
+        __file__).parent.parent / "hopp" / "simulation" / "resource_files" / "solar" / WEATHER_FILE #"Beni_Miha" / "659265_32.69_10.90_2019.csv"
+    grid_file = Path(__file__).parent.parent / "hopp" / "simulation" / "resource_files" / "grid" / PRICE_FILE #"tunisia_est_grid_prices.csv"
 
     # Combine the data into a site definition
     site_info = SiteInfo(site_data, solar_resource_file=solar_file, grid_resource_file=grid_file)
@@ -228,8 +228,8 @@ def init_simulation_hybrid():
 
     # Load in weather and price data files
     solar_file = Path(
-        __file__).parent.parent / "resource_files" / "solar" / WEATHER_FILE #"Beni_Miha" / "659265_32.69_10.90_2019.csv"
-    grid_file = Path(__file__).parent.parent / "resource_files" / "grid" / PRICE_FILE #"tunisia_est_grid_prices.csv"
+        __file__).parent.parent / "hopp" / "simulation" / "resource_files" / "solar" / WEATHER_FILE #"Beni_Miha" / "659265_32.69_10.90_2019.csv"
+    grid_file = Path(__file__).parent.parent / "hopp" / "simulation" / "resource_files" / "grid" / PRICE_FILE #"tunisia_est_grid_prices.csv"
 
     # Combine the data into a site definition
     site_info = SiteInfo(site_data, solar_resource_file=solar_file, grid_resource_file=grid_file)

hopp/simulation/technologies/layout/flicker_mismatch.py

@@ -169,7 +169,7 @@ def _setup_irradiance(self):
         pv_model.SystemDesign.gcr = .1
         if self.solar_resource_data is None:
             filename = str(self.lat) + "_" + str(self.lon) + "_psmv3_60_2012.csv"
-            weather_path = ROOT_DIR.parent / "resource_files" / "solar" / filename
+            weather_path = ROOT_DIR / "simulation" / "resource_files" / "solar" / filename
             if not weather_path.is_file():
                 SolarResource(self.lat, self.lon, year=2012)
                 if not weather_path.is_file():

hopp/simulation/technologies/resource/elec_prices.py

@@ -31,7 +31,7 @@ def __init__(self, lat, lon, year, path_resource="", filepath=""):
         self.path_resource = os.path.join(self.path_resource, 'grid')
 
         if filepath == "":
-            home_dir = Path(__file__).parent.parent.parent.parent.parent.absolute()
+            home_dir = Path(__file__).parent.parent.parent.absolute()
             filepath = os.path.join(
                 str(home_dir),
                 "resource_files",

hopp/simulation/technologies/resource/solar_resource.py

@@ -35,7 +35,7 @@ def __init__(
         lat: float, 
         lon: float, 
         year: int, 
-        path_resource: Union[str, Path] = ROOT_DIR.parent / "resource_files", 
+        path_resource: Union[str, Path] = ROOT_DIR / "simulation" / "resource_files", 
         filepath: Union[str, Path] ="", 
         use_api: bool = False,
         **kwargs

hopp/simulation/technologies/resource/wind_resource.py

@@ -29,7 +29,7 @@ def __init__(
         lon: float, 
         year: int, 
         wind_turbine_hub_ht: float, 
-        path_resource: Union[str, Path] = ROOT_DIR.parent / "resource_files", 
+        path_resource: Union[str, Path] = ROOT_DIR / "simulation" / "resource_files", 
         filepath: Union[str, Path] ="", 
         source: str ="WTK", 
         use_api: bool = False,

hopp/type_dec.py

@@ -20,10 +20,9 @@
 import os.path
 
 from hopp.utilities.log import hybrid_logger as logger
-
+from hopp import ROOT_DIR
 ### Define general data types used throughout
 
-hopp_path = Path(__file__).parent.parent
 hopp_float_type = np.float64
 hopp_int_type = np.int_
 
@@ -73,7 +72,7 @@ def resource_file_converter(resource_file: str) -> Union[Path, str]:
         return ""
 
     # Check the path relative to the hopp directory for the resource file and return if it exists
-    resource_file_path = str(hopp_path / resource_file)
+    resource_file_path = str(ROOT_DIR / "simulation" / resource_file)
     resolved_path = convert_to_path(resource_file_path)
     file_exists = os.path.isfile(resolved_path)
     if file_exists:

setup.py

@@ -26,7 +26,9 @@
     *base_path.glob("simulation/technologies/csp/pySSC_daotk/libs/*"),
     *base_path.glob("simulation/technologies/csp/pySSC_daotk/tower_data/*"),
     *base_path.glob("simulation/technologies/csp/pySSC_daotk/trough_data/*"),
-    *base_path.glob("simulation/technologies/dispatch/cbc_solver/cbc-win64/*")
+    *base_path.glob("simulation/technologies/dispatch/cbc_solver/cbc-win64/*"),
+    *base_path.glob("simulation/resource_files/*"),
+    *base_path.glob("simulation/resource_files/*/*")
 ]
 
 package_data = {

tests/hopp/test_battery_dispatch.py

@@ -12,10 +12,10 @@
 from hopp.simulation.technologies.dispatch import SimpleBatteryDispatch
 from hopp.simulation.technologies.dispatch.hybrid_dispatch_builder_solver import HybridDispatchBuilderSolver, HybridDispatchOptions
 from hopp.simulation.technologies.financial.custom_financial_model import CustomFinancialModel
+from hopp import ROOT_DIR
 
-
-solar_resource_file = Path(__file__).absolute().parent.parent.parent / "resource_files" / "solar" / "35.2018863_-101.945027_psmv3_60_2012.csv"
-wind_resource_file = Path(__file__).absolute().parent.parent.parent / "resource_files" / "wind" / "35.2018863_-101.945027_windtoolkit_2012_60min_80m_100m.srw"
+solar_resource_file = ROOT_DIR / "simulation" / "resource_files" / "solar" / "35.2018863_-101.945027_psmv3_60_2012.csv"
+wind_resource_file = ROOT_DIR / "simulation" / "resource_files" / "wind" / "35.2018863_-101.945027_windtoolkit_2012_60min_80m_100m.srw"
 site = SiteInfo(flatirons_site, solar_resource_file=solar_resource_file, wind_resource_file=wind_resource_file)
 
 default_fin_config = {

tests/hopp/test_clustering.py

@@ -39,13 +39,13 @@ def parse_wind_file(filename, height=None):
 
 def test_parse_wind_file():
     # Test single height wind file
-    single_height_file = f"{ROOT_DIR.parent}/resource_files/wind/35.2018863_-101.945027_windtoolkit_2012_60min_100m.srw"
+    single_height_file = f"{ROOT_DIR}/simulation/resource_files/wind/35.2018863_-101.945027_windtoolkit_2012_60min_100m.srw"
     wind_data = parse_wind_file(single_height_file)
     assert len(wind_data) == 8760
     assert sum(wind_data) == approx(75760, 1e-4)
 
     # Test multiple-height wind file
-    multiple_height_file = f"{ROOT_DIR.parent}/resource_files/wind/35.2018863_-101.945027_windtoolkit_2012_60min_80m_100m.srw"
+    multiple_height_file = f"{ROOT_DIR}/simulation/resource_files/wind/35.2018863_-101.945027_windtoolkit_2012_60min_80m_100m.srw"
     wind_data = parse_wind_file(multiple_height_file)       # don't specify height -> should return first height
     assert len(wind_data) == 8760
     assert sum(wind_data) == approx(72098, 1e-4)
@@ -60,7 +60,7 @@ def test_parse_wind_file():
 def test_minimum_specification():
     clusterer = clustering.Clustering(
         power_sources=['tower'],
-        solar_resource_file=f"{ROOT_DIR.parent}/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv")
+        solar_resource_file=f"{ROOT_DIR}/simulation/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv")
     clusterer.run_clustering()
     n_clusters = len(clusterer.clusters['count'])
     assert n_clusters == 20
@@ -73,7 +73,7 @@ def test_minimum_specification():
 def test_alternate_solar_file():
     clusterer = clustering.Clustering(
         power_sources=['tower'],
-        solar_resource_file=f"{ROOT_DIR.parent}/resource_files/solar/34.865371_-116.783023_psmv3_60_tmy.csv")
+        solar_resource_file=f"{ROOT_DIR}/simulation/resource_files/solar/34.865371_-116.783023_psmv3_60_tmy.csv")
     clusterer.run_clustering()
     n_clusters = len(clusterer.clusters['count'])
     assert n_clusters == 20
@@ -84,7 +84,7 @@ def test_alternate_solar_file():
 
 
 def test_default_weights_and_divisions():
-    solar_file = f"{ROOT_DIR.parent}/resource_files/solar/34.865371_-116.783023_psmv3_60_tmy.csv"
+    solar_file = f"{ROOT_DIR}/simulation/resource_files/solar/34.865371_-116.783023_psmv3_60_tmy.csv"
     clusterer = clustering.Clustering(
         power_sources=['trough'],
         solar_resource_file=solar_file)
@@ -198,7 +198,7 @@ def test_default_weights_and_divisions():
 def test_too_high_clusters():
     clusterer = clustering.Clustering(
         power_sources=['tower'],
-        solar_resource_file=f"{ROOT_DIR.parent}/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv")
+        solar_resource_file=f"{ROOT_DIR}/simulation/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv")
     clusterer.n_cluster = 1000                              # make unachievably high
     clusterer.run_clustering()
     n_clusters = len(clusterer.clusters['count'])
@@ -210,7 +210,7 @@ def test_too_high_clusters():
 def test_various_simulation_days():
     clusterer = clustering.Clustering(
         power_sources=['tower'],
-        solar_resource_file=f"{ROOT_DIR.parent}/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv")
+        solar_resource_file=f"{ROOT_DIR}/simulation/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv")
 
     clusterer.ndays = 1
     clusterer.run_clustering()
@@ -232,7 +232,7 @@ def test_various_simulation_days():
 
 
 def test_custom_weights_and_divisions():
-    solar_file = f"{ROOT_DIR.parent}/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv"
+    solar_file = f"{ROOT_DIR}/simulation/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv"
     clusterer = clustering.Clustering(
         power_sources=['tower'],
         solar_resource_file=solar_file)
@@ -261,7 +261,7 @@ def test_custom_weights_and_divisions():
 
 def test_initial_state_heuristics():
     # Battery heuristics
-    solar_file = f"{ROOT_DIR.parent}/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv"
+    solar_file = f"{ROOT_DIR}/simulation/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv"
     clusterer = clustering.Clustering(
         power_sources=['tower', 'pv', 'battery'],
         solar_resource_file=solar_file)
@@ -302,8 +302,8 @@ def test_initial_state_heuristics():
 def test_price_parameter():
     clusterer = clustering.Clustering(
         power_sources=['tower'],
-        solar_resource_file=f"{ROOT_DIR.parent}/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv",
-        price_data=parse_price_data(f"{ROOT_DIR.parent}/resource_files/grid/pricing-data-2015-IronMtn-002_factors.csv"))
+        solar_resource_file=f"{ROOT_DIR}/simulation/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv",
+        price_data=parse_price_data(f"{ROOT_DIR}/simulation/resource_files/grid/pricing-data-2015-IronMtn-002_factors.csv"))
     clusterer.run_clustering()
     n_clusters = len(clusterer.clusters['count'])
     assert n_clusters == 20
@@ -316,7 +316,7 @@ def test_price_parameter():
 def test_wind_defaults():
     clusterer = clustering.Clustering(
         power_sources=['wind'],
-        solar_resource_file=f"{ROOT_DIR.parent}/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv")
+        solar_resource_file=f"{ROOT_DIR}/simulation/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv")
     clusterer.run_clustering()
     n_clusters = len(clusterer.clusters['count'])
     assert n_clusters == 20
@@ -329,8 +329,8 @@ def test_wind_defaults():
 def test_wind_resource_parameter():
     clusterer = clustering.Clustering(
         power_sources=['wind'],
-        solar_resource_file=f"{ROOT_DIR.parent}/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv",
-        wind_resource_data=parse_wind_file(f"{ROOT_DIR.parent}/resource_files/wind/35.2018863_-101.945027_windtoolkit_2012_60min_100m.srw"))
+        solar_resource_file=f"{ROOT_DIR}/simulation/resource_files/solar/35.2018863_-101.945027_psmv3_60_2012.csv",
+        wind_resource_data=parse_wind_file(f"{ROOT_DIR}/simulation/resource_files/wind/35.2018863_-101.945027_windtoolkit_2012_60min_100m.srw"))
     clusterer.run_clustering()
     n_clusters = len(clusterer.clusters['count'])
     assert n_clusters == 20
@@ -344,7 +344,7 @@ def test_annual_array_from_cluster_exemplars():
     # Run clustering on weather file
     clusterer = clustering.Clustering(
         power_sources=['tower'],
-        solar_resource_file=f"{ROOT_DIR.parent}/resource_files/solar/34.865371_-116.783023_psmv3_60_tmy.csv")
+        solar_resource_file=f"{ROOT_DIR}/simulation/resource_files/solar/34.865371_-116.783023_psmv3_60_tmy.csv")
     clusterer.ndays = 2
     clusterer.run_clustering()
 
@@ -385,7 +385,7 @@ def test_cluster_avgs_from_timeseries():
     # Run clustering on weather file
     clusterer = clustering.Clustering(
         power_sources=['tower'],
-        solar_resource_file=f"{ROOT_DIR.parent}/resource_files/solar/34.865371_-116.783023_psmv3_60_tmy.csv")
+        solar_resource_file=f"{ROOT_DIR}/simulation/resource_files/solar/34.865371_-116.783023_psmv3_60_tmy.csv")
     clusterer.ndays = 2
     clusterer.run_clustering()
 

tests/hopp/test_dispatch.py

@@ -28,6 +28,7 @@
 
 from tests.hopp.utils import create_default_site_info
 from hopp.utilities import load_yaml
+from hopp import ROOT_DIR
 
 
 @pytest.fixture
@@ -350,7 +351,7 @@ def test_wave_dispatch():
 		"tz": -7,
 	}
 
-    wave_resource_file = Path(__file__).absolute().parent.parent.parent / "resource_files" / "wave" / "Wave_resource_timeseries.csv"
+    wave_resource_file = ROOT_DIR / "simulation" / "resource_files" / "wave" / "Wave_resource_timeseries.csv"
     site = SiteInfo(data, solar=False, wind=False, wave=True, wave_resource_file=wave_resource_file)
 
     mhk_yaml_path = Path(__file__).absolute().parent.parent.parent / "tests" / "hopp" / "inputs" / "wave" / "wave_device.yaml"

tests/hopp/test_flicker.py

@@ -3,6 +3,7 @@
 import csv
 from hopp.simulation.technologies.layout.shadow_flicker import *
 from hopp.simulation.technologies.sites import flatirons_site
+from hopp import ROOT_DIR
 
 sys.path.append('..')
 verts = flatirons_site['site_boundaries']['verts']
@@ -13,7 +14,7 @@ def sun_info(n):
 
     # get DNI from csv
     irrad = []
-    with open('../../resource_files/solar/39.7555_-105.2211_psmv3_60_2012.csv') as csv_f:
+    with open(f'{ROOT_DIR}/simulation/resource_files/solar/39.7555_-105.2211_psmv3_60_2012.csv') as csv_f:
         reader = csv.reader(csv_f, delimiter=',')
         for row in reader:
             try:

tests/hopp/test_hybrid.py

@@ -38,7 +38,7 @@ def site():
 
 
 wave_resource_file = (
-    ROOT_DIR.parent / "resource_files" / "wave" / "Wave_resource_timeseries.csv"
+    ROOT_DIR / "simulation" / "resource_files" / "wave" / "Wave_resource_timeseries.csv"
 )
 
 

tests/hopp/test_resource_download.py

@@ -21,7 +21,7 @@
 with open(DEFAULT_WIND_RESOURCE_FILE, 'r') as f:
     wind_body = f.read()
 
-solar_file = ROOT_DIR.parent / "resource_files" / "solar" / "39.7555_-105.2211_psmv3_60_2012.csv"
+solar_file = ROOT_DIR / "simulation" / "resource_files" / "solar" / "39.7555_-105.2211_psmv3_60_2012.csv"
 
 with open(solar_file, 'r') as f:
     solar_body = f.read()

tests/hopp/test_site_info.py

@@ -12,15 +12,15 @@
 from hopp import ROOT_DIR
 
 solar_resource_file = os.path.join(
-    ROOT_DIR.parent, "resource_files", "solar", 
+    ROOT_DIR, "simulation", "resource_files", "solar", 
     "35.2018863_-101.945027_psmv3_60_2012.csv"
 )
 wind_resource_file = os.path.join(
-    ROOT_DIR.parent, "resource_files", "wind", 
+    ROOT_DIR, "simulation", "resource_files", "wind", 
     "35.2018863_-101.945027_windtoolkit_2012_60min_80m_100m.srw"
 )
 grid_resource_file = os.path.join(
-    ROOT_DIR.parent, "resource_files", "grid", 
+    ROOT_DIR, "simulation", "resource_files", "grid", 
     "pricing-data-2015-IronMtn-002_factors.csv"
 )
 kml_filepath = Path(__file__).absolute().parent / "layout_example.kml"

tests/hopp/test_solar_wind.py

@@ -3,6 +3,7 @@
 import csv
 from hopp.simulation.technologies.layout.shadow_flicker import *
 from hopp.simulation.technologies.sites import flatirons_site
+from hopp import ROOT_DIR
 
 sys.path.append('..')
 verts = flatirons_site['site_boundaries']['verts']
@@ -13,7 +14,7 @@ def sun_info(n):
 
     # get DNI from csv
     irrad = []
-    with open('../../resource_files/solar/39.7555_-105.2211_psmv3_60_2012.csv') as csv_f:
+    with open(f'{ROOT_DIR}/simulation/resource_files/solar/39.7555_-105.2211_psmv3_60_2012.csv') as csv_f:
         reader = csv.reader(csv_f, delimiter=',')
         for row in reader:
             try:

tests/hopp/test_utility_rate.py

@@ -10,7 +10,7 @@
 from hopp.utilities.keys import get_developer_nrel_gov_key
 
 path = os.path.dirname(os.path.abspath(__file__))
-path_rates = ROOT_DIR.parent / "resource_files"
+path_rates = ROOT_DIR / "simulation" / "resource_files"
 
 # these rates sometimes mysteriously disappear from URDB fyi
 urdb_label = "5ca4d1175457a39b23b3d45e"  # https://openei.org/apps/IURDB/rate/view/5ca4d1175457a39b23b3d45e
@@ -34,7 +34,7 @@ def test_urdb_url(urdb):
 @responses.activate
 def test_urdb_response(urdb):
     # instead of making a real call, we'll stub using a saved JSON response
-    file = str(ROOT_DIR.parent / "resource_files" / "utility_rate_response.json")
+    file = str(ROOT_DIR / "simulation" / "resource_files" / "utility_rate_response.json")
     with open(file, 'r') as f:
         body = json.load(f)
     responses.add(

tests/hopp/test_wave.py

@@ -7,6 +7,7 @@
 from hopp.simulation.technologies.financial.mhk_cost_model import MHKCostModelInputs
 from hopp.simulation.technologies.financial.custom_financial_model import CustomFinancialModel
 from hopp.utilities import load_yaml
+from hopp import ROOT_DIR
 
 # TODO: I'm seeing this copied around in tests, let's refactor to a module
 default_fin_config = {
@@ -40,7 +41,7 @@ def site():
 		"year": 2010,
 		"tz": -7,
 	}
-	wave_resource_file = Path(__file__).absolute().parent.parent.parent / "resource_files" / "wave" / "Wave_resource_timeseries.csv"
+	wave_resource_file = ROOT_DIR / "simulation" / "resource_files" / "wave" / "Wave_resource_timeseries.csv"
 	site = SiteInfo(data, solar=False, wind=False, wave=True, wave_resource_file=wave_resource_file)
 
 	return site

tests/hopp/utils.py

@@ -2,8 +2,8 @@
 from hopp.simulation.technologies.sites import SiteInfo, flatirons_site
 
 # default resource files
-DEFAULT_SOLAR_RESOURCE_FILE = ROOT_DIR.parent / "resource_files" / "solar" / "35.2018863_-101.945027_psmv3_60_2012.csv"
-DEFAULT_WIND_RESOURCE_FILE = ROOT_DIR.parent / "resource_files" / "wind" / "35.2018863_-101.945027_windtoolkit_2012_60min_80m_100m.srw"
+DEFAULT_SOLAR_RESOURCE_FILE = ROOT_DIR / "simulation" / "resource_files" / "solar" / "35.2018863_-101.945027_psmv3_60_2012.csv"
+DEFAULT_WIND_RESOURCE_FILE = ROOT_DIR / "simulation" / "resource_files" / "wind" / "35.2018863_-101.945027_windtoolkit_2012_60min_80m_100m.srw"
 
 # default configuration for `CustomFinancialModel`
 DEFAULT_FIN_CONFIG = {