From 5bf9302870b231f19ce9e692793b038eb813c678 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Thu, 30 Sep 2021 10:54:19 +0100
Subject: [PATCH 1/8] add test add frame work

---
 .../data_sources/gsp/gsp_data_source.py           | 13 +++++++++++++
 tests/data_sources/gsp/test_gsp_metadata.py       | 15 ++++++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/nowcasting_dataset/data_sources/gsp/gsp_data_source.py b/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
index 10325943..87e0884f 100644
--- a/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
+++ b/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
@@ -393,3 +393,16 @@ def load_solar_gsp_data(
     gsp_power_df.columns = [int(col) for col in gsp_power_df.columns]
 
     return gsp_power_df
+
+
+def normalize_gsp_data(gsp_power: pd.DataFrame):
+
+    # load installed capacity
+
+    # merged with gsp power
+
+    # check that no gsp dont have installed capacity
+
+    # normalize by installed capacity
+
+    return gsp_power
diff --git a/tests/data_sources/gsp/test_gsp_metadata.py b/tests/data_sources/gsp/test_gsp_metadata.py
index 9b4ad2f7..ed8e27bc 100644
--- a/tests/data_sources/gsp/test_gsp_metadata.py
+++ b/tests/data_sources/gsp/test_gsp_metadata.py
@@ -8,7 +8,10 @@
     get_gsp_metadata_from_eso,
     get_gsp_shape_from_eso,
 )
-from nowcasting_dataset.data_sources.gsp.pvlive import load_pv_gsp_raw_data_from_pvlive
+from nowcasting_dataset.data_sources.gsp.pvlive import (
+    load_pv_gsp_raw_data_from_pvlive,
+    get_installed_capacity,
+)
 
 
 def test_get_gsp_metadata_from_eso():
@@ -122,3 +125,13 @@ def test_load_gsp_raw_data_from_pvlive_many_gsp():
     assert len(gsp_pv_df) == (48 + 1) * 10
     assert "datetime_gmt" in gsp_pv_df.columns
     assert "generation_mw" in gsp_pv_df.columns
+
+
+def test_get_installed_capacity():
+
+    installed_capacity = get_installed_capacity(maximum_number_of_gsp=10)
+
+    assert len(installed_capacity) == 10
+    assert "installedcapacity_mwp" == installed_capacity.name
+    assert installed_capacity.iloc[0] == 342.02623
+    assert installed_capacity.iloc[9] == 308.00432

From dcdfe415493f5d471c4827ebcc66dfa7b7f8929e Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Thu, 30 Sep 2021 10:59:06 +0100
Subject: [PATCH 2/8] add method to get installed capacity for each gsp

---
 nowcasting_dataset/data_sources/gsp/pvlive.py | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/nowcasting_dataset/data_sources/gsp/pvlive.py b/nowcasting_dataset/data_sources/gsp/pvlive.py
index 7648f0af..6f4f14bd 100644
--- a/nowcasting_dataset/data_sources/gsp/pvlive.py
+++ b/nowcasting_dataset/data_sources/gsp/pvlive.py
@@ -2,6 +2,8 @@
 import logging
 import pandas as pd
 from pvlive_api import PVLive
+from typing import Optional
+import pytz
 
 from nowcasting_dataset.data_sources.gsp.eso import get_list_of_gsp_ids
 
@@ -87,3 +89,50 @@ def load_pv_gsp_raw_data_from_pvlive(
     gsp_data_df["datetime_gmt"] = gsp_data_df["datetime_gmt"].dt.tz_localize(None)
 
     return gsp_data_df
+
+
+def get_installed_capacity(
+    start: Optional[datetime] = datetime(2021, 1, 1, tzinfo=pytz.utc),
+    maximum_number_of_gsp: Optional[int] = None,
+) -> pd.Series:
+    """
+    Get the installed capacity of each gsp
+
+    This can take ~30 seconds for getting the full list
+
+    Args:
+        start: optional datetime when the installed cpapcity is collected
+        maximum_number_of_gsp: Truncate list of GSPs to be no larger than this number of GSPs.
+            Set to None to disable truncation.
+
+    Returns: pd.Series of installed capacity indexed by gsp_id
+
+    """
+
+    logger.debug(f"Getting all installed capacity at {start}")
+
+    # get a lit of gsp ids
+    gsp_ids = get_list_of_gsp_ids(maximum_number_of_gsp=maximum_number_of_gsp)
+
+    # setup pv Live class, although here we are getting historic data
+    pvl = PVLive()
+
+    # loop over gsp_id to get installed capacity
+    data = []
+    for gsp_id in gsp_ids:
+        d = pvl.at_time(
+            start,
+            entity_type="gsp",
+            extra_fields="installedcapacity_mwp",
+            dataframe=True,
+            entity_id=gsp_id,
+        )
+        data.append(d)
+
+    # join data together
+    data_df = pd.concat(data)
+
+    # set gsp_id as index
+    data_df.set_index("gsp_id", inplace=True)
+
+    return data_df["installedcapacity_mwp"]

From d149f315789ecf27c622bf0a59dbfc3e904707d6 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Thu, 30 Sep 2021 11:26:24 +0100
Subject: [PATCH 3/8] normalise generation data on-the-fly

---
 .../data_sources/gsp/gsp_data_source.py       |  5 +-
 nowcasting_dataset/data_sources/gsp/pvlive.py | 11 ++-
 tests/data_sources/gsp/test_gsp_metadata.py   | 66 --------------
 tests/data_sources/gsp/test_gsp_pvlive.py     | 88 +++++++++++++++++++
 4 files changed, 101 insertions(+), 69 deletions(-)
 create mode 100644 tests/data_sources/gsp/test_gsp_pvlive.py

diff --git a/nowcasting_dataset/data_sources/gsp/gsp_data_source.py b/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
index 87e0884f..b6b35f5f 100644
--- a/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
+++ b/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
@@ -54,6 +54,8 @@ class GSPDataSource(ImageDataSource):
     get_center: bool = True
     # the maximum number of gsp's to be loaded for data sample
     n_gsp_per_example: int = DEFAULT_N_GSP_PER_EXAMPLE
+    # scale from zero to one
+    do_scale_0_to_1: bool = False
 
     def __post_init__(self, image_size_pixels: int, meters_per_pixel: int):
         """
@@ -93,7 +95,8 @@ def load(self):
         )
 
         # scale from 0 to 1
-        self.gsp_power = scale_to_0_to_1(self.gsp_power)
+        if self.do_scale_0_to_1:
+            self.gsp_power = scale_to_0_to_1(self.gsp_power)
 
         logger.debug(f"There are {len(self.gsp_power.columns)} GSP")
 
diff --git a/nowcasting_dataset/data_sources/gsp/pvlive.py b/nowcasting_dataset/data_sources/gsp/pvlive.py
index 6f4f14bd..a7ffd6c8 100644
--- a/nowcasting_dataset/data_sources/gsp/pvlive.py
+++ b/nowcasting_dataset/data_sources/gsp/pvlive.py
@@ -13,7 +13,7 @@
 
 
 def load_pv_gsp_raw_data_from_pvlive(
-    start: datetime, end: datetime, number_of_gsp: int = None
+    start: datetime, end: datetime, number_of_gsp: int = None, normalize_data: bool = True
 ) -> pd.DataFrame:
     """
     Load raw pv gsp data from pvlive. Note that each gsp is loaded separately. Also the data is loaded in 30 day chunks.
@@ -21,6 +21,7 @@ def load_pv_gsp_raw_data_from_pvlive(
         start: the start date for gsp data to load
         end: the end date for gsp data to load
         number_of_gsp: The number of gsp to load. Note that on 2021-09-01 there were 338 to load.
+        normalize_data: Option to normalize the generation according to installed capacity
 
     Returns: Data frame of time series of gsp data. Shows PV data for each GSP from {start} to {end}
 
@@ -58,7 +59,7 @@ def load_pv_gsp_raw_data_from_pvlive(
                     end=end_chunk,
                     entity_type="gsp",
                     entity_id=gsp_id,
-                    extra_fields="",
+                    extra_fields="installedcapacity_mwp",
                     dataframe=True,
                 )
             )
@@ -74,6 +75,12 @@ def load_pv_gsp_raw_data_from_pvlive(
         one_gsp_data_df = pd.concat(one_gsp_data_df)
         one_gsp_data_df = one_gsp_data_df.sort_values(by=["gsp_id", "datetime_gmt"])
 
+        # normalize
+        if normalize_data:
+            one_gsp_data_df["generation_mw"] = (
+                one_gsp_data_df["generation_mw"] / one_gsp_data_df["installedcapacity_mwp"]
+            )
+
         # append to longer list
         gsp_data_df.append(one_gsp_data_df)
 
diff --git a/tests/data_sources/gsp/test_gsp_metadata.py b/tests/data_sources/gsp/test_gsp_metadata.py
index ed8e27bc..9b4e5cae 100644
--- a/tests/data_sources/gsp/test_gsp_metadata.py
+++ b/tests/data_sources/gsp/test_gsp_metadata.py
@@ -1,17 +1,10 @@
-from datetime import datetime
-
 import geopandas as gpd
 import pandas as pd
-import pytz
 
 from nowcasting_dataset.data_sources.gsp.eso import (
     get_gsp_metadata_from_eso,
     get_gsp_shape_from_eso,
 )
-from nowcasting_dataset.data_sources.gsp.pvlive import (
-    load_pv_gsp_raw_data_from_pvlive,
-    get_installed_capacity,
-)
 
 
 def test_get_gsp_metadata_from_eso():
@@ -76,62 +69,3 @@ def test_get_pv_gsp_shape_from_eso():
     assert "RegionID" in gsp_shapes.columns
     assert "RegionName" in gsp_shapes.columns
     assert "geometry" in gsp_shapes.columns
-
-
-def test_load_gsp_raw_data_from_pvlive_one_gsp_one_day():
-    """
-    Test that one gsp system data can be loaded, just for one day
-    """
-
-    start = datetime(2019, 1, 1, tzinfo=pytz.utc)
-    end = datetime(2019, 1, 2, tzinfo=pytz.utc)
-
-    gsp_pv_df = load_pv_gsp_raw_data_from_pvlive(start=start, end=end, number_of_gsp=1)
-
-    assert isinstance(gsp_pv_df, pd.DataFrame)
-    assert len(gsp_pv_df) == (48 + 1)
-    assert "datetime_gmt" in gsp_pv_df.columns
-    assert "generation_mw" in gsp_pv_df.columns
-
-
-def test_load_gsp_raw_data_from_pvlive_one_gsp():
-    """
-    Test that one gsp system data can be loaded
-    """
-
-    start = datetime(2019, 1, 1, tzinfo=pytz.utc)
-    end = datetime(2019, 3, 1, tzinfo=pytz.utc)
-
-    gsp_pv_df = load_pv_gsp_raw_data_from_pvlive(start=start, end=end, number_of_gsp=1)
-
-    assert isinstance(gsp_pv_df, pd.DataFrame)
-    assert len(gsp_pv_df) == (48 * 59 + 1)
-    # 30 days in january, 29 days in february, plus one for the first timestamp in march
-    assert "datetime_gmt" in gsp_pv_df.columns
-    assert "generation_mw" in gsp_pv_df.columns
-
-
-def test_load_gsp_raw_data_from_pvlive_many_gsp():
-    """
-    Test that one gsp system data can be loaded
-    """
-
-    start = datetime(2019, 1, 1, tzinfo=pytz.utc)
-    end = datetime(2019, 1, 2, tzinfo=pytz.utc)
-
-    gsp_pv_df = load_pv_gsp_raw_data_from_pvlive(start=start, end=end, number_of_gsp=10)
-
-    assert isinstance(gsp_pv_df, pd.DataFrame)
-    assert len(gsp_pv_df) == (48 + 1) * 10
-    assert "datetime_gmt" in gsp_pv_df.columns
-    assert "generation_mw" in gsp_pv_df.columns
-
-
-def test_get_installed_capacity():
-
-    installed_capacity = get_installed_capacity(maximum_number_of_gsp=10)
-
-    assert len(installed_capacity) == 10
-    assert "installedcapacity_mwp" == installed_capacity.name
-    assert installed_capacity.iloc[0] == 342.02623
-    assert installed_capacity.iloc[9] == 308.00432
diff --git a/tests/data_sources/gsp/test_gsp_pvlive.py b/tests/data_sources/gsp/test_gsp_pvlive.py
new file mode 100644
index 00000000..30e37d52
--- /dev/null
+++ b/tests/data_sources/gsp/test_gsp_pvlive.py
@@ -0,0 +1,88 @@
+from datetime import datetime
+
+import pandas as pd
+import pytz
+
+from nowcasting_dataset.data_sources.gsp.pvlive import (
+    load_pv_gsp_raw_data_from_pvlive,
+    get_installed_capacity,
+)
+
+
+def test_load_gsp_raw_data_from_pvlive_one_gsp_one_day():
+    """
+    Test that one gsp system data can be loaded, just for one day
+    """
+
+    start = datetime(2019, 1, 1, tzinfo=pytz.utc)
+    end = datetime(2019, 1, 2, tzinfo=pytz.utc)
+
+    gsp_pv_df = load_pv_gsp_raw_data_from_pvlive(start=start, end=end, number_of_gsp=1)
+
+    assert isinstance(gsp_pv_df, pd.DataFrame)
+    assert len(gsp_pv_df) == (48 + 1)
+    assert "datetime_gmt" in gsp_pv_df.columns
+    assert "generation_mw" in gsp_pv_df.columns
+
+
+def test_load_gsp_raw_data_from_pvlive_one_gsp_one_day_not_normalised():
+    """
+    Test that one gsp system data can be loaded, just for one day, and is normalized correctly
+    """
+
+    # pick a summer day
+    start = datetime(2019, 6, 21, tzinfo=pytz.utc)
+    end = datetime(2019, 6, 22, tzinfo=pytz.utc)
+
+    gsp_pv_df = load_pv_gsp_raw_data_from_pvlive(
+        start=start, end=end, number_of_gsp=1, normalize_data=False
+    )
+    assert gsp_pv_df["generation_mw"].max() > 1
+
+    gsp_pv_df = load_pv_gsp_raw_data_from_pvlive(
+        start=start, end=end, number_of_gsp=1, normalize_data=True
+    )
+    assert gsp_pv_df["generation_mw"].max() <= 1
+
+
+def test_load_gsp_raw_data_from_pvlive_one_gsp():
+    """a
+    Test that one gsp system data can be loaded
+    """
+
+    start = datetime(2019, 1, 1, tzinfo=pytz.utc)
+    end = datetime(2019, 3, 1, tzinfo=pytz.utc)
+
+    gsp_pv_df = load_pv_gsp_raw_data_from_pvlive(start=start, end=end, number_of_gsp=1)
+
+    assert isinstance(gsp_pv_df, pd.DataFrame)
+    assert len(gsp_pv_df) == (48 * 59 + 1)
+    # 30 days in january, 29 days in february, plus one for the first timestamp in march
+    assert "datetime_gmt" in gsp_pv_df.columns
+    assert "generation_mw" in gsp_pv_df.columns
+
+
+def test_load_gsp_raw_data_from_pvlive_many_gsp():
+    """
+    Test that one gsp system data can be loaded
+    """
+
+    start = datetime(2019, 1, 1, tzinfo=pytz.utc)
+    end = datetime(2019, 1, 2, tzinfo=pytz.utc)
+
+    gsp_pv_df = load_pv_gsp_raw_data_from_pvlive(start=start, end=end, number_of_gsp=10)
+
+    assert isinstance(gsp_pv_df, pd.DataFrame)
+    assert len(gsp_pv_df) == (48 + 1) * 10
+    assert "datetime_gmt" in gsp_pv_df.columns
+    assert "generation_mw" in gsp_pv_df.columns
+
+
+def test_get_installed_capacity():
+
+    installed_capacity = get_installed_capacity(maximum_number_of_gsp=10)
+
+    assert len(installed_capacity) == 10
+    assert "installedcapacity_mwp" == installed_capacity.name
+    assert installed_capacity.iloc[0] == 342.02623
+    assert installed_capacity.iloc[9] == 308.00432

From 3201a35eda0f96fe8dee4117d96e4e0e41be168f Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Thu, 30 Sep 2021 11:47:33 +0100
Subject: [PATCH 4/8] update v of gsp data

---
 nowcasting_dataset/config/gcp.yaml                  |  2 +-
 nowcasting_dataset/config/on_premises.yaml          |  2 +-
 .../data_sources/gsp/gsp_data_source.py             | 13 -------------
 tests/data_sources/get_test_data.py                 |  2 +-
 4 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/nowcasting_dataset/config/gcp.yaml b/nowcasting_dataset/config/gcp.yaml
index 92574e07..4f5ded46 100644
--- a/nowcasting_dataset/config/gcp.yaml
+++ b/nowcasting_dataset/config/gcp.yaml
@@ -6,7 +6,7 @@ input_data:
   satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr
   solar_pv_data_filename: gs://solar-pv-nowcasting-data/PV/PVOutput.org/UK_PV_timeseries_batch.nc
   solar_pv_metadata_filename: gs://solar-pv-nowcasting-data/PV/PVOutput.org/UK_PV_metadata.csv
-  gsp_zarr_path: gs://solar-pv-nowcasting-data/PV/PVOutput.org/PV/GSP/v0/pv_gsp.zarr
+  gsp_zarr_path: gs://solar-pv-nowcasting-data/PV/PVOutput.org/PV/GSP/v1/pv_gsp.zarr
   topographic_filename: gs://solar-pv-nowcasting-data/Topographic/europe_dem_1km_osgb.tif
 output_data:
   filepath: gs://solar-pv-nowcasting-data/prepared_ML_training_data/v6/
diff --git a/nowcasting_dataset/config/on_premises.yaml b/nowcasting_dataset/config/on_premises.yaml
index 984cc132..20cf9931 100644
--- a/nowcasting_dataset/config/on_premises.yaml
+++ b/nowcasting_dataset/config/on_premises.yaml
@@ -7,7 +7,7 @@ input_data:
   solar_pv_path:
   solar_pv_data_filename: /storage/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/raw/PV/PVOutput.org/UK_PV_timeseries_batch.nc
   solar_pv_metadata_filename: /storage/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/raw/PV/PVOutput.org/UK_PV_metadata.csv
-  gsp_zarr_path: /storage/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/raw/PV/GSP/v0/pv_gsp.zarr
+  gsp_zarr_path: /storage/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/raw/PV/GSP/v1/pv_gsp.zarr
 output_data:
   filepath: /storage/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/prepared_ML_training_data/v7/
 process:
diff --git a/nowcasting_dataset/data_sources/gsp/gsp_data_source.py b/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
index b6b35f5f..00708821 100644
--- a/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
+++ b/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
@@ -396,16 +396,3 @@ def load_solar_gsp_data(
     gsp_power_df.columns = [int(col) for col in gsp_power_df.columns]
 
     return gsp_power_df
-
-
-def normalize_gsp_data(gsp_power: pd.DataFrame):
-
-    # load installed capacity
-
-    # merged with gsp power
-
-    # check that no gsp dont have installed capacity
-
-    # normalize by installed capacity
-
-    return gsp_power
diff --git a/tests/data_sources/get_test_data.py b/tests/data_sources/get_test_data.py
index 2369b786..c038bfe4 100644
--- a/tests/data_sources/get_test_data.py
+++ b/tests/data_sources/get_test_data.py
@@ -81,7 +81,7 @@
 # ### GSP data
 
 gsp = GSPDataSource(
-    filename="gs://solar-pv-nowcasting-data/PV/GSP/v0/pv_gsp.zarr",
+    filename="gs://solar-pv-nowcasting-data/PV/GSP/v1/pv_gsp.zarr",
     start_dt=start_dt,
     end_dt=end_dt,
     history_minutes=30,

From e8e3b34276f3d5d850875ea7eedde04954afc064 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Thu, 30 Sep 2021 12:12:12 +0100
Subject: [PATCH 5/8] update scripts and load method for making data to save
 installed capacity too,

---
 .../data_sources/gsp/gsp_data_source.py       |  9 +++++++++
 scripts/get_raw_pv_gsp_data.py                | 19 +++++++++++++++----
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/nowcasting_dataset/data_sources/gsp/gsp_data_source.py b/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
index 00708821..413554db 100644
--- a/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
+++ b/nowcasting_dataset/data_sources/gsp/gsp_data_source.py
@@ -383,7 +383,16 @@ def load_solar_gsp_data(
     # Open data - it may be quicker to open byte file first, but decided just to keep it like this at the moment
     gsp_power = xr.open_dataset(filename, engine="zarr")
     gsp_power = gsp_power.sel(datetime_gmt=slice(start_dt, end_dt))
+
+    # only take generation data
+    gsp_power = gsp_power.generation_mw
+
+    # make dataframe with index datetime_gmt and columns og gsp_id
     gsp_power_df = gsp_power.to_dataframe()
+    gsp_power_df.reset_index(inplace=True)
+    gsp_power_df = gsp_power_df.pivot(
+        index="datetime_gmt", columns="gsp_id", values="generation_mw"
+    )
 
     # Save memory
     del gsp_power
diff --git a/scripts/get_raw_pv_gsp_data.py b/scripts/get_raw_pv_gsp_data.py
index 5406d053..32b56c28 100755
--- a/scripts/get_raw_pv_gsp_data.py
+++ b/scripts/get_raw_pv_gsp_data.py
@@ -11,6 +11,7 @@
 import yaml
 import os
 import numcodecs
+import xarray as xr
 
 from nowcasting_dataset.data_sources.gsp.pvlive import load_pv_gsp_raw_data_from_pvlive
 from pathlib import Path
@@ -36,11 +37,21 @@
 data_df = load_pv_gsp_raw_data_from_pvlive(start=start, end=end)
 
 # pivot to index as datetime_gmt, and columns as gsp_id
-data_df = data_df.pivot(index="datetime_gmt", columns="gsp_id", values="generation_mw")
-data_df.columns = [str(col) for col in data_df.columns]
+data_generation = data_df.pivot(index="datetime_gmt", columns="gsp_id", values="generation_mw")
+data_generation.columns = [str(col) for col in data_generation.columns]
+data_generation_xarray = xr.DataArray(
+    data_generation, name="generation_mw", dims=["datetime_gmt", "gsp_id"]
+)
 
-# change to xarray
-data_xarray = data_df.to_xarray()
+data_capacity = data_df.pivot(
+    index="datetime_gmt", columns="gsp_id", values="installedcapacity_mwp"
+)
+data_capacity.columns = [str(col) for col in data_capacity.columns]
+data_capacity_xarray = xr.DataArray(
+    data_capacity, name="installedcapacity_mwp", dims=["datetime_gmt", "gsp_id"]
+)
+
+data_xarray = xr.merge([data_generation_xarray, data_capacity_xarray])
 
 # save config to file
 with open(os.path.join(LOCAL_TEMP_PATH, "configuration.yaml"), "w+") as f:

From 1c07a78ecfcbbf6dc6d986365204c8d498a85726 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Thu, 30 Sep 2021 12:26:22 +0100
Subject: [PATCH 6/8] update test data

---
 nowcasting_dataset/dataset/split/__init__.py  |   0
 tests/data/gsp/test.zarr/.zmetadata           | 487 ++----------------
 tests/data/gsp/test.zarr/1/.zattrs            |   5 -
 tests/data/gsp/test.zarr/1/0                  | Bin 688 -> 0 bytes
 tests/data/gsp/test.zarr/10/.zarray           |  20 -
 tests/data/gsp/test.zarr/10/0                 | Bin 699 -> 0 bytes
 tests/data/gsp/test.zarr/100/.zattrs          |   5 -
 tests/data/gsp/test.zarr/100/0                | Bin 685 -> 0 bytes
 tests/data/gsp/test.zarr/101/.zattrs          |   5 -
 tests/data/gsp/test.zarr/101/0                | Bin 685 -> 0 bytes
 tests/data/gsp/test.zarr/102/.zarray          |  20 -
 tests/data/gsp/test.zarr/102/.zattrs          |   5 -
 tests/data/gsp/test.zarr/102/0                | Bin 686 -> 0 bytes
 tests/data/gsp/test.zarr/103/.zarray          |  20 -
 tests/data/gsp/test.zarr/103/.zattrs          |   5 -
 tests/data/gsp/test.zarr/103/0                | Bin 685 -> 0 bytes
 tests/data/gsp/test.zarr/104/.zarray          |  20 -
 tests/data/gsp/test.zarr/104/.zattrs          |   5 -
 tests/data/gsp/test.zarr/104/0                | Bin 691 -> 0 bytes
 tests/data/gsp/test.zarr/105/.zarray          |  20 -
 tests/data/gsp/test.zarr/105/.zattrs          |   5 -
 tests/data/gsp/test.zarr/105/0                | Bin 688 -> 0 bytes
 tests/data/gsp/test.zarr/106/.zarray          |  20 -
 tests/data/gsp/test.zarr/106/.zattrs          |   5 -
 tests/data/gsp/test.zarr/106/0                | Bin 671 -> 0 bytes
 tests/data/gsp/test.zarr/107/.zarray          |  20 -
 tests/data/gsp/test.zarr/107/.zattrs          |   5 -
 tests/data/gsp/test.zarr/107/0                | Bin 685 -> 0 bytes
 tests/data/gsp/test.zarr/108/.zarray          |  20 -
 tests/data/gsp/test.zarr/108/.zattrs          |   5 -
 tests/data/gsp/test.zarr/108/0                | Bin 683 -> 0 bytes
 tests/data/gsp/test.zarr/109/.zarray          |  20 -
 tests/data/gsp/test.zarr/109/.zattrs          |   5 -
 tests/data/gsp/test.zarr/109/0                | Bin 674 -> 0 bytes
 tests/data/gsp/test.zarr/11/.zarray           |  20 -
 tests/data/gsp/test.zarr/11/.zattrs           |   5 -
 tests/data/gsp/test.zarr/11/0                 | Bin 694 -> 0 bytes
 tests/data/gsp/test.zarr/110/.zarray          |  20 -
 tests/data/gsp/test.zarr/110/.zattrs          |   5 -
 tests/data/gsp/test.zarr/110/0                | Bin 683 -> 0 bytes
 tests/data/gsp/test.zarr/111/.zarray          |  20 -
 tests/data/gsp/test.zarr/111/.zattrs          |   5 -
 tests/data/gsp/test.zarr/111/0                | Bin 683 -> 0 bytes
 tests/data/gsp/test.zarr/112/.zarray          |  20 -
 tests/data/gsp/test.zarr/112/.zattrs          |   5 -
 tests/data/gsp/test.zarr/112/0                | Bin 140 -> 0 bytes
 tests/data/gsp/test.zarr/113/.zarray          |  20 -
 tests/data/gsp/test.zarr/113/.zattrs          |   5 -
 tests/data/gsp/test.zarr/113/0                | Bin 681 -> 0 bytes
 tests/data/gsp/test.zarr/114/.zarray          |  20 -
 tests/data/gsp/test.zarr/114/.zattrs          |   5 -
 tests/data/gsp/test.zarr/114/0                | Bin 140 -> 0 bytes
 tests/data/gsp/test.zarr/115/.zarray          |  20 -
 tests/data/gsp/test.zarr/115/.zattrs          |   5 -
 tests/data/gsp/test.zarr/115/0                | Bin 677 -> 0 bytes
 tests/data/gsp/test.zarr/116/.zarray          |  20 -
 tests/data/gsp/test.zarr/116/.zattrs          |   5 -
 tests/data/gsp/test.zarr/116/0                | Bin 684 -> 0 bytes
 tests/data/gsp/test.zarr/datetime_gmt/.zarray |   6 +-
 tests/data/gsp/test.zarr/datetime_gmt/.zattrs |   4 +-
 tests/data/gsp/test.zarr/datetime_gmt/0       | Bin 336 -> 138 bytes
 .../test.zarr/{1 => generation_mw}/.zarray    |  10 +-
 .../data/gsp/test.zarr/generation_mw/.zattrs  |   6 +
 tests/data/gsp/test.zarr/generation_mw/0.0    | Bin 0 -> 2630 bytes
 .../gsp/test.zarr/{101 => gsp_id}/.zarray     |  10 +-
 .../data/gsp/test.zarr/{10 => gsp_id}/.zattrs |   4 +-
 tests/data/gsp/test.zarr/gsp_id/0             | Bin 0 -> 71 bytes
 .../{100 => installedcapacity_mwp}/.zarray    |  10 +-
 .../test.zarr/installedcapacity_mwp/.zattrs   |   6 +
 .../gsp/test.zarr/installedcapacity_mwp/0.0   | Bin 0 -> 218 bytes
 tests/data_sources/get_test_data.py           |  27 +-
 71 files changed, 80 insertions(+), 925 deletions(-)
 create mode 100644 nowcasting_dataset/dataset/split/__init__.py
 delete mode 100644 tests/data/gsp/test.zarr/1/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/1/0
 delete mode 100644 tests/data/gsp/test.zarr/10/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/10/0
 delete mode 100644 tests/data/gsp/test.zarr/100/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/100/0
 delete mode 100644 tests/data/gsp/test.zarr/101/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/101/0
 delete mode 100644 tests/data/gsp/test.zarr/102/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/102/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/102/0
 delete mode 100644 tests/data/gsp/test.zarr/103/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/103/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/103/0
 delete mode 100644 tests/data/gsp/test.zarr/104/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/104/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/104/0
 delete mode 100644 tests/data/gsp/test.zarr/105/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/105/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/105/0
 delete mode 100644 tests/data/gsp/test.zarr/106/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/106/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/106/0
 delete mode 100644 tests/data/gsp/test.zarr/107/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/107/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/107/0
 delete mode 100644 tests/data/gsp/test.zarr/108/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/108/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/108/0
 delete mode 100644 tests/data/gsp/test.zarr/109/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/109/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/109/0
 delete mode 100644 tests/data/gsp/test.zarr/11/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/11/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/11/0
 delete mode 100644 tests/data/gsp/test.zarr/110/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/110/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/110/0
 delete mode 100644 tests/data/gsp/test.zarr/111/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/111/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/111/0
 delete mode 100644 tests/data/gsp/test.zarr/112/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/112/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/112/0
 delete mode 100644 tests/data/gsp/test.zarr/113/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/113/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/113/0
 delete mode 100644 tests/data/gsp/test.zarr/114/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/114/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/114/0
 delete mode 100644 tests/data/gsp/test.zarr/115/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/115/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/115/0
 delete mode 100644 tests/data/gsp/test.zarr/116/.zarray
 delete mode 100644 tests/data/gsp/test.zarr/116/.zattrs
 delete mode 100644 tests/data/gsp/test.zarr/116/0
 rename tests/data/gsp/test.zarr/{1 => generation_mw}/.zarray (78%)
 create mode 100644 tests/data/gsp/test.zarr/generation_mw/.zattrs
 create mode 100644 tests/data/gsp/test.zarr/generation_mw/0.0
 rename tests/data/gsp/test.zarr/{101 => gsp_id}/.zarray (77%)
 rename tests/data/gsp/test.zarr/{10 => gsp_id}/.zattrs (59%)
 create mode 100644 tests/data/gsp/test.zarr/gsp_id/0
 rename tests/data/gsp/test.zarr/{100 => installedcapacity_mwp}/.zarray (78%)
 create mode 100644 tests/data/gsp/test.zarr/installedcapacity_mwp/.zattrs
 create mode 100644 tests/data/gsp/test.zarr/installedcapacity_mwp/0.0

diff --git a/nowcasting_dataset/dataset/split/__init__.py b/nowcasting_dataset/dataset/split/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/data/gsp/test.zarr/.zmetadata b/tests/data/gsp/test.zarr/.zmetadata
index a6cf4ed1..ab697b80 100644
--- a/tests/data/gsp/test.zarr/.zmetadata
+++ b/tests/data/gsp/test.zarr/.zmetadata
@@ -4,384 +4,9 @@
         ".zgroup": {
             "zarr_format": 2
         },
-        "1/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "1/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "10/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "10/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "100/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "100/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "101/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "101/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "102/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "102/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "103/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "103/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "104/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "104/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "105/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "105/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "106/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "106/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "107/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "107/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "108/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "108/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "109/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "109/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "11/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "11/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "110/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "110/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "111/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "111/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
-        },
-        "112/.zarray": {
+        "datetime_gmt/.zarray": {
             "chunks": [
-                145
+                49
             ],
             "compressor": {
                 "blocksize": 0,
@@ -390,53 +15,31 @@
                 "id": "blosc",
                 "shuffle": 1
             },
-            "dtype": "<f8",
-            "fill_value": "NaN",
+            "dtype": "<i8",
+            "fill_value": null,
             "filters": null,
             "order": "C",
             "shape": [
-                145
+                49
             ],
             "zarr_format": 2
         },
-        "112/.zattrs": {
+        "datetime_gmt/.zattrs": {
             "_ARRAY_DIMENSIONS": [
                 "datetime_gmt"
-            ]
-        },
-        "113/.zarray": {
-            "chunks": [
-                145
             ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<f8",
-            "fill_value": "NaN",
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "113/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ]
+            "calendar": "proleptic_gregorian",
+            "units": "minutes since 2019-01-01"
         },
-        "114/.zarray": {
+        "generation_mw/.zarray": {
             "chunks": [
-                145
+                49,
+                21
             ],
             "compressor": {
                 "blocksize": 0,
                 "clevel": 5,
-                "cname": "lz4",
+                "cname": "zstd",
                 "id": "blosc",
                 "shuffle": 1
             },
@@ -445,18 +48,20 @@
             "filters": null,
             "order": "C",
             "shape": [
-                145
+                49,
+                21
             ],
             "zarr_format": 2
         },
-        "114/.zattrs": {
+        "generation_mw/.zattrs": {
             "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
+                "datetime_gmt",
+                "gsp_id"
             ]
         },
-        "115/.zarray": {
+        "gsp_id/.zarray": {
             "chunks": [
-                145
+                21
             ],
             "compressor": {
                 "blocksize": 0,
@@ -465,28 +70,29 @@
                 "id": "blosc",
                 "shuffle": 1
             },
-            "dtype": "<f8",
-            "fill_value": "NaN",
+            "dtype": "<U2",
+            "fill_value": null,
             "filters": null,
             "order": "C",
             "shape": [
-                145
+                21
             ],
             "zarr_format": 2
         },
-        "115/.zattrs": {
+        "gsp_id/.zattrs": {
             "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
+                "gsp_id"
             ]
         },
-        "116/.zarray": {
+        "installedcapacity_mwp/.zarray": {
             "chunks": [
-                145
+                49,
+                21
             ],
             "compressor": {
                 "blocksize": 0,
                 "clevel": 5,
-                "cname": "lz4",
+                "cname": "zstd",
                 "id": "blosc",
                 "shuffle": 1
             },
@@ -495,42 +101,17 @@
             "filters": null,
             "order": "C",
             "shape": [
-                145
+                49,
+                21
             ],
             "zarr_format": 2
         },
-        "116/.zattrs": {
+        "installedcapacity_mwp/.zattrs": {
             "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
+                "datetime_gmt",
+                "gsp_id"
             ]
-        },
-        "datetime_gmt/.zarray": {
-            "chunks": [
-                145
-            ],
-            "compressor": {
-                "blocksize": 0,
-                "clevel": 5,
-                "cname": "lz4",
-                "id": "blosc",
-                "shuffle": 1
-            },
-            "dtype": "<i8",
-            "fill_value": null,
-            "filters": null,
-            "order": "C",
-            "shape": [
-                145
-            ],
-            "zarr_format": 2
-        },
-        "datetime_gmt/.zattrs": {
-            "_ARRAY_DIMENSIONS": [
-                "datetime_gmt"
-            ],
-            "calendar": "proleptic_gregorian",
-            "units": "minutes since 2019-01-01 00:00:00"
         }
     },
     "zarr_consolidated_format": 1
-}
\ No newline at end of file
+}
diff --git a/tests/data/gsp/test.zarr/1/.zattrs b/tests/data/gsp/test.zarr/1/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/1/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/1/0 b/tests/data/gsp/test.zarr/1/0
deleted file mode 100644
index 012ae98baeb9024c4476faf7917431fa7786ec70..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 688
zcmZQ#ROIMjVPF8^4NMFSA`A=+VL&Xyz{v22sdDW!kywV!YgeR3@m#fDCYz_oz|F+)
zo2mG2)Pww~2Ae&u*`~1Ex|!Rpz`(=A@Imm)bhZP$U3F(xFLKX+7hu1C5zr)LfH3*P
z{^X=-3^(KV7S~zu{16Z<f|~qh-H(*5W)<7sZgvP14SOJG4mDZgU~uBmANwZ!R{hW@
zXnn}@8?v!LHp1l2qK=c*41$i|1s}cMEIH-jYpBV!&w}@M#4E<_y&?D}Wy6gRg;0~<
z&3&5UDlc*8tyOw$$lW^*Q*oNSVb8*FbA}s--E$)OUwuD!iydmR$)(flCpu64W8b{3
zBI^0^nw3zKRZmWPoqpi{^j(jBUSH;Y!{s1OlOLRXvqp$P;=X0*V!zDv%dTorlNAb1
zA62^l?$xRHpVQ;7syPKgO`aOY{n1s>Ibf<_XXu}4)y9W#np`yD^ZMTmyUHtaH)d=*
zJ7f7KsL7j`pWN_o!yN&~8^?a%>9_c-3^jSKoXu62ziJ`^is=?IFS(Ag<1|@e(eX#G
zKfHeP^x>^5$M<e5RDzmpws`NEt2b}oeR%)Q)#F<yi2i|^EMLCy@R=(&Z{N9k{=}he
k%aXsNnXCllgHoN0JtG4T*eo7>25CF6Zhiv>jsUQF0PVNsN&o-=

diff --git a/tests/data/gsp/test.zarr/10/.zarray b/tests/data/gsp/test.zarr/10/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/10/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/10/0 b/tests/data/gsp/test.zarr/10/0
deleted file mode 100644
index 0381d53e9f13a1bdf410895d4c3a646b99e1cee5..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 699
zcmZQ#ROIMjVPF8^-AoJ&A`A=+VL&Xyz{v22NjOf7ll2QTg9!7c@1~QS7~&NfxS1G!
zGr8@5urTm4&zyO;6Cykw1gF1HVBle5_#h~Lgzehr-_x&1mewrUS?|W+05l01AWV*~
zaPg>j<6*E#@7%XkH;`cq)MS^Js`EC!-RaZx#XITO)D-RvsL6#+c8i}k$mw24I(&gm
z^#Fqxvavum!sNKMd>6KPm@*izXvjY=K8HaLYO;l8gSm}Ku0TlQulDV8Bqj8rCi5~Z
zJM~88Mp@Ky@h|D=S`1fknyjuDly}7FCBv8C-;3CNt}vKGO;-PxxF$JNxOlxm)mHvT
zU9TQPO}^5_-2W-=c=^f<i&yOz&M>^jX|h?~PM!K?7Z~hkW|sRW?O}+4ntWyMtOdS*
z=WOaaP#j|GtC83ZHM!w&>ci%RDgp~bB=35yb76Re(`22{Ovn09ObjjS1<RhfxiWM^
zP2LmI-MCM(&|Q75L87DkJY^lI$(e1YHL;t%9C;RHt~1fCf?+>SlZB@rxc%z=n-9+(
z+_`pg{bB}bsL25<4_~|c@WJhSH?Lngxoeu@SE$MI4)b;#J$vEe<qPMJ?OxN#@D0sm
sT_7Kn>ZI%$8FU!z!KQKPGuYeP+o>=xZ~)V$3IjV6gE)f$LjYJS0JNgY@Bjb+

diff --git a/tests/data/gsp/test.zarr/100/.zattrs b/tests/data/gsp/test.zarr/100/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/100/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/100/0 b/tests/data/gsp/test.zarr/100/0
deleted file mode 100644
index ede126e6581586a5b14562e904e225ec452c0a1b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 685
zcmZQ#ROIMjVPF8^wM+~QA`A=+p+GFdz{v22X`1TUbX&G*3}OY$tPHQt*aj#va5FLd
zVZ6v@5n*?-?6;P$(3j^=SQ-=<c$gSI2yVT6<{6jgwAd@#8y;pht>Oflg8~o+tK}Y*
znf1$yVbeAVp)UXMvpb;%n|<3@?kC1R?W0*_<@|fR$xwsO+?JYe?7sH6>M@1Hv-YgP
zD5e5A2!oT3cD*?+ufgzQYH_B>Y9;F=sKG9t`LW$P%U&yAjY_SME4c$T_<UjORA>Dq
zp`2B_LzyNppT}*mkg3sKH?9zd_T3XFxjok2x(I48pJCm+y>mYt@-qpzxx@U<DyYGC
zBtM%yywJSBYQeYVm%D6Sa2tH)1B>1amX!<<<}V}UZZDVK0X10m?b(3ehZ7fnxqknq
zsIP(?)Zn6+BPv$*x;q1=vn<@ITkMb9VEr3O?weeHF^F=mllEEX@S_WA@bcXkZTr+*
zcI3Ilyh^{RYzj4)$0^`})&uTp{YeoUJAAkn;x;&X+OESlpS}6;=JEB5r*|(1egiew
zvU}N%LuW2szjpP)p{+~W)Sw36s$90?@X2#$&zwAVXxFA0Vdw^{0R=$GP1>H3LB<Yj
Y7Pmf-C(ppk#2{^F$iS}15CGN$0QM2c?EnA(

diff --git a/tests/data/gsp/test.zarr/101/.zattrs b/tests/data/gsp/test.zarr/101/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/101/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/101/0 b/tests/data/gsp/test.zarr/101/0
deleted file mode 100644
index b5a8c9ab767dd4f1da26df4b2066ede7363f86fd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 685
zcmZQ#ROIMjVPF8^wM+~QA`A=+p+GFdz{v22Nq2g~*H^3_5e#7_+k5qQ&AFt=z|F+)
zhtdB3(&Y1Y&zRei&N4@|vaM5K;9+9;Aox2_NNDP0W{#NY+g+}ERh<Kxg8~o+zvjuz
zKX6w;lVOuVx~z0ZgdEi1s!o&W^Y^2;GdHPuefK%K2x_o*kM_i*9QWHc>@D(%k;l_f
zOa*cf1{cQ8j&%4II*Vb=wcevnPA!|E20!0fd1gcRnj@>q+ut;tGWh^CILm$APdh2o
z{k5(E2NaS#&2Ss6%ztjSZDQO3hTk5_d&FBaE1?E^pD{kCJ(1-?;Ry+Q@j^u<sKMtg
z4mV$V?xDZ)b%gY%aLY}&4bIHE_~R{ms1?KS#io;bbMtDT27j=Bn6JrWuFL9rQE20f
zdAv}A4}=AX8yfFz*vNC<-Dj=XBisgeJvHdKC;TLwp*%+6k%75JA=KbK)_-NB<fk#M
z_--~kVqrxI)Zj^v8Wb1EESi|6?Roar@9tRK2Ino=cI5Dx8?QgSxqs>6iLEU^pa#pd
zE!@28(6O@@&K}#hYuyAzsKJ*C=dIng_u$D>NA_)9ySU8(-C#AK04TXh+cPrA*n!RB
W)(7(B8F-l(r0onD*cBN9z?uNx5y&$D

diff --git a/tests/data/gsp/test.zarr/102/.zarray b/tests/data/gsp/test.zarr/102/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/102/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/102/.zattrs b/tests/data/gsp/test.zarr/102/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/102/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/102/0 b/tests/data/gsp/test.zarr/102/0
deleted file mode 100644
index 05736fe3003ff03cab1caf505429763469f4d258..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 686
zcmZQ#ROIMjVPF8^bxaHlA`A=+VL&Xyz{v22sddughp#N8pMC3d>p$68&2wFmft!ip
zH<O$D%#62BrR?T1nA+_OWYSYmVBle5_#n93<|ywy-!BaEpT8N;K3UFC4>SoGAWY^8
zl(stL{n~1Z%f_jGQ`CcwK~3&o_*C}D7vEbB3}<4^?5{HiLruQ$(Z+837NZYQ^XjH7
z-gk~+1G2F|Hp1ksOSr=}2A<{=$ey;*q#$<@7u4jKSJ`4-3lB0sVwmUYH%le=GSp;|
z<695cMxUyFsl7*)RkxGj7EY6&X!nHO*tY!em21x~@YOBlSPnJ$Q$S)#T*xBhe1=1I
z&tES+_z7zAaW<)Av)j#L&Trg(SaMwz10PP4Ew0-yYzX^R{dd9f%Be@#%;!Q)KJt0u
zKJE3Jzt=I$-N2F`-hKpX@&TUAQz288%tGoD9@v#WW|)H0WaWD{U2fOU{7@2EES8yU
zsdp1<@}t}8lHxB4zNRuPxvRsYa9I~>vb=`ambTyLt#ysp{fUZu%pic%<knr6E?vI<
z@Y#(=kIx?2zBJ|=)a2|ryDr>&^#0ka4>!-AI=C$3A=KoI$x9F1c=YJT;X6mJT%A+J
k@D|Nv1t1@k>ZI)%8D#9hhH>jN@Eb75iZd861b|fo01ZIc$N&HU

diff --git a/tests/data/gsp/test.zarr/103/.zarray b/tests/data/gsp/test.zarr/103/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/103/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/103/.zattrs b/tests/data/gsp/test.zarr/103/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/103/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/103/0 b/tests/data/gsp/test.zarr/103/0
deleted file mode 100644
index 99b5249ead17dc8350ea61bc635d0e0383d30f88..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 685
zcmZQ#ROIMjVPF8^wM+~QA`A=+p+GFdz{v22sq$fxDUbgnUk0AjVe6{mmS0n3;AUd@
z!&vw4vVHQSrS5F(g8a*41tuym@Gvoa5cE7bb;CYCKZdjJ3%Pwtxif&~pa6uy=Po6$
zST1>XF2j#omA_XBthI+4T)oPvda@bw>a@=`#qV~V&w(2JclYOsf%%E)V(n}G8rlp0
zM==%1K^VMN?Ad<FP1el}vs7hX{(4?>0BZ0Di>$=+lA>yhLU>r4)=Hm+8oVNNeb+ll
zq10{Pckq5ozEp_Y;F>3ymaj8Dv@@*Q@b~g3Vdfc7gO{%q*DWa9{`tuNv(BX^HEK|U
zpYuFB`_8|fYt4M#s9sIgaNGuW)@iL)3(|0BNSwuZaly)k`=AD2j1cUqjS67i@Zh9u
z%lx8+P=h6AJgI$PygTWP`)+BWV(VYH4Nmp+X0)%rH=99wU8{*s@TW+q!IF+zdm~rB
z46vG6v+~>Ds?|_~pBYM-JUk<|ImP1Tu@IqCR=5q$Te@cF`71YHe|U56+>u>N@;^fj
zHtk-s<G_KVr_P)@v}e=8P7A2P_cCX0*mLOcfitJ~ZC$shCkfqPHJ|_}xk=kIGRWA0
Y&EnPv^5hwKnHZ$)3>nxJ83MqX0N1b2ZU6uP

diff --git a/tests/data/gsp/test.zarr/104/.zarray b/tests/data/gsp/test.zarr/104/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/104/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/104/.zattrs b/tests/data/gsp/test.zarr/104/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/104/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/104/0 b/tests/data/gsp/test.zarr/104/0
deleted file mode 100644
index e9d530b5b99654a43abacdd18fe83ce3dea7e86f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 691
zcmZQ#ROIMjVPF8^%}fjoA`A=+p+GFdz{v22Njs#y_xeT^hNs^ypX~c)bo{&`12+@H
zZzkLSbuWUh9@Y%JdhzW02Jvsl6c~7!7(NI{X^6{y<K#YJW#af*^(ohEph3t0VX$?3
zt;9~7E{3Idmz)3mJS}<^)Zh=krx`tjb&9U}ecA9!?Wm?1)L=0aiFuU)&!%M;+*;l@
zzb*{fR3IB+u-~EbHCOqf7|MA>xK75nwW&Z2=8Qe5(RuQ=u+u-E1qbFX`fmp{c<)5!
zc}*$-cYOOfWKWl}%i%Qm$NH=KH{NP6EWCVC?-1uKIap9c$(Zx?g$lH|s?8{Vb}{E{
zIMm>IGbX;e9MSS|ee{Yn&4)iia2lMOV^BDmPoClL^xE$=YdR%ap)O|rZSW;3rrojl
z$)ml`zwc`*fEwH)KYRPP`;Xo(sI>XVb$I)3oCY_qxtP+jAf4gqG1fJ4?XFe{P=jOd
zo{pST@U?A4h26I9`%+AiP=ojD{BCArbF0|3bjMby=Qmn#8vK9h;d@V>KmG9N*5%XN
z)^wHUKn;#xeDv6*`*&Zwe(~z%(;J7ECqNCZSi0xvxr>*tJ$&=*;{FXCc4!9c0QsQg
lre$Z($RJ|}HjGP;!5#=y7#KL27#LL<*qIpEWEld$IspyE+^hfq

diff --git a/tests/data/gsp/test.zarr/105/.zarray b/tests/data/gsp/test.zarr/105/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/105/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/105/.zattrs b/tests/data/gsp/test.zarr/105/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/105/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/105/0 b/tests/data/gsp/test.zarr/105/0
deleted file mode 100644
index 1f615b4e032d7f3a49f8090cbaafe035591ed3fc..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 688
zcmZQ#ROIMjVPF8^4NMFSA`A=+p+GFdz{v22DWzU=Gb6+HiGpgHt+}0E`|A`LxS1G!
zGwGf4s9asDt#L=i^rpMxyQPr|47^MX9|Vrh^3#7mKT|8`x0Z>(`&;rrgOCBj;Ot9<
z^S(30uhNM4I6YumpiDB<;I{v(4lO(3^-8+_fo;Jwm;ZmE21_0PDO3Au>;DQ4pYmzn
zj_D$s3S=V;Ui!E}Re-^)==8ag6|3XSySG9OKK9YjNg-|v<AqpT>2=S3x-WtnY{z0W
zo2jF7&AvK*m(S}}F5ooSe`^Nc6$bHpQNI4^_h$S!b^>a!uH{>qtl5Tji&tsBe)J~r
zdLz`}!^y_^SLPcTvX*Vzxw^q+DNche<e6(0Gi+OW%YoT4U$j9`8ftKHbC#3p%X6La
zsdBu}ttu3Qp$0#@ksq_TI99gmcR;O+!}4=D4L)oBv+*c{OzgWgdMe`F_o_EQ4SsZU
zc>+(m>OAKY{)s6+9kv~Y8XS46FMNu%zsj6G<0*b0QcQ6gY&84St!E!zJ%9A@^11yR
z7Bwz}8l1H9(B&J?pS^u?`^t?Q7xt{MgBq;sFmv<X6X#E#zIFHX-i;G2&<xfD@<GW>
g+Mba?#tv*2w?2cmy**evg9-x&69bzfLjYJC0O&2zp#T5?

diff --git a/tests/data/gsp/test.zarr/106/.zarray b/tests/data/gsp/test.zarr/106/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/106/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/106/.zattrs b/tests/data/gsp/test.zarr/106/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/106/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/106/0 b/tests/data/gsp/test.zarr/106/0
deleted file mode 100644
index a52b587908c0b19eef884d74fe52cc6992342076..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 671
zcmZQ#ROIMjVPF8^`AiH9A`A=+AwVp{z{v2I@ik}H9<8dJt2=KVZaT&urm4ul&BXAV
z>A8qQnI?n5GdZ_a7gL^mWbsj8;ALX?AYkR7rMU9v=}<NE&u?VPqKXoM1|b22xjEUc
zg*;}dE7re0b!z^(i)~PIR~Ge})G)YkCKaqUb}e5V^%81s&-_^9>Qi<L_qok}UbC}q
z-dZF>flP$C0U=Ef4m|Kbey8-d{KIxNJ*c@Fx{~GZ7`~JnpLSDH?_8vQ4r*?3egKR4
zMu}_Fm6jJ;nf{zxfXm#*`yaLiO#5M*6sTLqru@MMYHq!wQCSbe%@n@s10kDSSY|dr
z&AlkT&ZO#KU;d`n`zDc7^-XJWnQK=w^W;Cnh2ObLWgDBnO?m@0w|YsjLj=Q_;G2g;
zf0#dJHSLC)yS#hVCLP9gmoD`A`gHP5-mx8*xz<zH+tgi*f3E&k)%r))wtA?!=a$?%
z9>QQ9XDIk>mko2wzc{G5!AH6Ujcj`R4dgQP=lb$I<i};MK>79O$F{EAvhC92vj;XV
zYl51awdTUhJ0EV`Ir!w|nf=@6nM*;<En2dA_n{5jFP*vc@apDSi{fvinkx_Fg3_6^
bJtG5;0s{{d1Ghc{zafLXs9g|409Y*ma52xf

diff --git a/tests/data/gsp/test.zarr/107/.zarray b/tests/data/gsp/test.zarr/107/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/107/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/107/.zattrs b/tests/data/gsp/test.zarr/107/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/107/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/107/0 b/tests/data/gsp/test.zarr/107/0
deleted file mode 100644
index 567aa02d56eb2bc08055cf1cc2447c9cb1b314cd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 685
zcmZQ#ROIMjVPF8^wM+~QA`A=+p+GFdz{v22=^azXk-hb`d`>^aGq0=3E&QO!z|F+)
zn@Oa!UsiFG%j7nOy$dFmUJce$VBlq9_#iNKUL!|+BCq{x>8%}i{C9Z(4MGM8gA@IJ
zy<L=J^I5X0G**?b`N|5Y!QDoeBrQIFTgb;?*M9KNxxNKZgB>Ic!k1k9@W4c2^S|4(
z*MCAb705;y%=_7B+J76>O%J+WvfD(ms-K1$ta@gj#gop}PkI?%X-J<-{r?<lu-@?l
zUS5{pLtk%|zE^3Y$cWQm-?!`?bGd6d-q(6mELhce=o-}EfYKk=5Bz`iRGJ~fZJl=0
z6m_V<%Advm<?O$(^`2C|!9m}>i*OqJNB-|C_lLi=7iI~*WPH)+rvx>)A-??6jU7*1
zycv|^{dR2mH4kcVq4=wm&HBZ&<a(EOv;UugJ1E4gKP`1ze`mt;<lPU~8O}cX2x{;q
z?*B%wAC<>XXDIttcI)(MMX15SH9ve5BLZw#?rqVz%YXAeP8Z)UTCw}+@l%&CoI1L1
z=gwtaVo-x!mTWuu;Kl8SPd_}o_2}%DS&>kK%jRs@b@1r9vlq{uJ8@vsv<NhV)qs3Z
ia?`f6XJp_3o5iiiz;D37C%~Y>z`?}87Qhey)&l_VK;0$)

diff --git a/tests/data/gsp/test.zarr/108/.zarray b/tests/data/gsp/test.zarr/108/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/108/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/108/.zattrs b/tests/data/gsp/test.zarr/108/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/108/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/108/0 b/tests/data/gsp/test.zarr/108/0
deleted file mode 100644
index 70aee3c1bcc47e5af626f09ff0af663e61d46721..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 683
zcmZQ#ROIMjVPF8^)l3WwA`A=+p+GFdz{v22=_7OUqprrM6Z2o3Y*U#VD6&V9ft!ip
zH<Nn8>!vCD92nN^o0gj*vRj!+fq|Ea;e&vrrGQUl->s}8dghxRYF(ZUGzb|W4BnE^
zXZ4{)?~bW*S`yRzI)}$lgC!JJWF&l;#ZYwUjpvM6>PkPM2CL}W$SsNe5Wnh7`k{A0
zt=!0_0@(<IneIDqZ25O#`|Tx>o9n)K-dzMW_-KTdWlZgIhWV-~d>!`7m35#7Z#ven
z|4BCAb)AAQ#v=FL9>!_#mYrNYngvQ*HvfK9dQ0MaZ$8xEWYdicVt9)gM6cX7vay}A
zRt;*f^vctxjr{oo8VuFNA8-2i4yVBn)#kQt-XNQ_gWsma+&A;=E2zQ8ivng|s;FQ{
zalQLwdD7eN0GPoucHg((pObXaWKMd(XAeG{2JhBMUvpT6X<PF;?p1Fz{jWAb4L(?Z
ze0xIsQij_P0t}M&K5yoN8tkQ0o3N9?#j!w_=Sb^a&h<DAHk@(r^usp~9z42x?ZT;p
z8z*o=4NhNk@!_NQ??1eLc=g=bBfA$!Lk%uiv+dBavllO3zI5T#fsON>(F|4r@<GW>
b#-5RZ2W%9NK7+J9STm0y1Dhg409XeAb}G{%

diff --git a/tests/data/gsp/test.zarr/109/.zarray b/tests/data/gsp/test.zarr/109/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/109/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/109/.zattrs b/tests/data/gsp/test.zarr/109/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/109/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/109/0 b/tests/data/gsp/test.zarr/109/0
deleted file mode 100644
index 241f3b94afc5e17663340a29fab4fa0937082761..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 674
zcmZQ#ROIMjVPF8^MNA9~A`A=+AwVp{z{v2I@x`$$cmFJo(v^MXrrS?fFHvORW@7lw
z)H_>4?<Rv}S?5aEdw%uVL2(KUyi5!q1jOP`mssm+EI!$Lg{`XC_eVa^AS8e=ccb^E
zq=H#qYj^Kqe($-#kRNJpw5d(rYX;E;K68FdIv8uJHWg~Fo{M(B!oSA2MuC&(uP>Ug
znGeZOAQNG(SBcPj;oGb)yz^w7`qwD;Ld{h_Qzx{EA@65teP`*KjOwNTpyvAAc(LV}
z%#t?oo7V1+wtf-bjLY2CKg_AG>ndVuJEWfpCtp*Cnp+an_11~u!uP+WzAxvdsCpMd
z&E?p|5SDPdxF)PuV%ejgas2JL%q=>jB;7dOEVRM7#W5_^F#&3>;`Ox~*D>5x*nh!K
zcG17CJz7w6Whdl5P;JVY=N=iza&p0%kE*!L<=-MWulC^c_tNp@-`8>T@<Gi#v}%4s
zGlSHE5^c$*pVPj(e1V1U#^Sr8$LtoLUTKjYP-VoxgKBO#kPAr<E{paYKYQcp%QsK&
zT{yaJkqg+||Lj?NZ$5tc;mxyqcOINOxqnRuFdYE_D+9*|f#NlLj$gd-;K`Gxk8WN(
oGT-z$SPD+a16d%WrR^CRcoZ0Tm>9VA8Tbtu<VEd*7y`h`08>cOI{*Lx

diff --git a/tests/data/gsp/test.zarr/11/.zarray b/tests/data/gsp/test.zarr/11/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/11/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/11/.zattrs b/tests/data/gsp/test.zarr/11/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/11/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/11/0 b/tests/data/gsp/test.zarr/11/0
deleted file mode 100644
index ba63bb0b045f944fafb71eef5c16c3b75bf09e35..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 694
zcmZQ#ROIMjVPF8^ZA=UdA`A=+VL&Xyz{v22X@>Kscc;RQCzUc>6Ww3-r09_%12+@H
zZ>Gopg_qsjJ;$#jbEA+9W76+;3Jg3<3?BsdW!7~6vf4d!I(LV@w7cW8iLZf1ApwNR
zcUYZ7k8YXw^#TLO>iP)+>A6so?RKqcD*Sri^=u-K{h|+%F|VN}U)c3`?SW3`g<^9L
zO9jguEsEZaWGs-0F!^|os-ZYj%hlBk{tR+kA{1^zO}_EefF*bBvS7uQi3j-3F{b{4
znryBbV{-8Ls_(9S!exwIn<ZXw;WD}1n)8O1-t{Y?44)@8wpwn#3pM$#!@lzZ(^BMw
za@)6r_dhS43N`t#md0yyTgNXdi_b33atk$gK8$K|D3A*Y=fkg!qeHby7hGrPNh-Ld
ze}xBX^7dU1a$Nk}zV0i%aqGA4l0zX147|W_7AOo|a<wSb=9<6)2PeB->r*;W4Mqe;
zva8i>k=a*f1T(CAdG6X3w_2#d;ZG}ze?E-KT(t4-Kk3)I891R%mf5g+*F}fb&Cewd
zGM}Ar<+OqzE|Wtyp1Jqz`P0|$KRh~fc>T1D7f_R<HlMij;Q8CPZ{EIq^zh2TwN_A*
zWt&#)J#zfq<y-f!UA}N+^Wylss3vPd14G)LkwL}|Y#6sbgSNdrSU-a@11A#$TOvaM
GSQh}|>f((6

diff --git a/tests/data/gsp/test.zarr/110/.zarray b/tests/data/gsp/test.zarr/110/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/110/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/110/.zattrs b/tests/data/gsp/test.zarr/110/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/110/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/110/0 b/tests/data/gsp/test.zarr/110/0
deleted file mode 100644
index 00e0d9706b4d661a5486daf8508290653c3ad1be..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 683
zcmZQ#ROIMjVPF8^)l3WwA`A=+p+GFdz{v22X~{&7J)J384C~Z5?>lQ-^E^^y;AUd@
z%`|83t)s^R{v2a)W{5R(UGP{yfq|Ea;e&vW<_vC;GtBoV8*^>TSU&X<&>&=hF!<g3
z9e-cGvsDZFJtcf=t909IsKKs3Ce~hQiF(AK^;G?YS!k#})L_G>%?EW}tQC|w+vc|7
z{@VS>rUKargY&IkZQt`M>h(+0_3c^9Raf#r4X%`Mo~&uO`7lG-3B`<`8y0VX8eFt;
zhlGi$|Bjuj9(`D{^W++w2J?5Z89%qmx>@?*mPKdV490IzgXi(TI*?g@;Wxu`>1_s=
zl22qn4Sw-GWB=p#tIsxEyy|jveXS8rgEJ#$&GneRFU4<ldr8J@CTm@&!EGm^EO_HL
z8Z(^iT(|kog)jf01|O^FkhV)&GEx0})19?lFCODGc=nGlwt};MPxy~}2yWNC@$nbb
zV2|%RvUddRy}*zXp>`_8uWTCB;0G0a3&LY9uGzGHnisx^V+u}#wfmPJx_JA><!cwu
zo;$pAW2*(!;FJyfF5iCg`pt*;FJC;qerRPB)ZqTLyAPhaeC^7ed-rdi-m|<A&0rNE
eAC%l=>=_v#(Z!?BAZ-uU%wx#FrpOQg)&T&zfZyl<

diff --git a/tests/data/gsp/test.zarr/111/.zarray b/tests/data/gsp/test.zarr/111/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/111/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/111/.zattrs b/tests/data/gsp/test.zarr/111/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/111/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/111/0 b/tests/data/gsp/test.zarr/111/0
deleted file mode 100644
index a4de7db549804131b21a3cd0b781602b6c789313..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 683
zcmZQ#ROIMjVPF8^)l3WwA`A=+p+GFdz{v22$#ijXn$VQl!K|80v$8(@U6P~7z|F+)
zn`uRa_1|x?F%0297w%sfvEw?c0s}7-!v_IP@2YK<TFMfJYn#($<~V^2LIwzfXCD0V
zotNQWSCPrQGq<K*Qfz@5%$B!A>npD`!w-#`pp!SlJI+E4=G`;jJKeNaZqnO^DYyN6
zS0I}TWFrjL*?mzeHl%x1n_`7A`+|0Tn89}wwJ+3f?P6%Qi@v0C`h5-y)Zn~HxypyD
z-_4)8!)~SUX?ffR2j#^Y9ogn0otnS7wNdo*5>cp&53Qdl`=)h2!#9u4@c(!Dnxmlx
zpL!q|H1)UOxAgOYf`<;3dEj*MTO~%GwV!|f_%z4TRD<<knJ&~|!;jz7j<2a@s5r7}
z_gyo|((h1%Zv^E(mr#>iI9))W@5Xw@^EeGoNV<A0@>j%6FQwCb*-bL9WuXQiFuR?l
z@uHf+Ab)wQ)pe8glc5IJIJA9S)4BJow#M~4jc*y`aT;tm|ID=~&mKLxd*{;0T^pBm
zszD7-UUl%|o#)R#ynA-<-knPaR|G;0E}XGp*TJJFPF}cj?bP12lYP((Rsr%s$xX(d
Zk%0$n6pucGv^`iek0Aq_B0~UJ2LL8-)7Jn1

diff --git a/tests/data/gsp/test.zarr/112/.zarray b/tests/data/gsp/test.zarr/112/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/112/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/112/.zattrs b/tests/data/gsp/test.zarr/112/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/112/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/112/0 b/tests/data/gsp/test.zarr/112/0
deleted file mode 100644
index 2a415efd5a5a1945d5bdb0b1335c2a23b0e15d01..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 140
xcmZQ#ROIMjVPF8^9tH*m5g_IUVtEEehKc|ngZkVbAd`Q90NA4cAntz<002t)6-NL7

diff --git a/tests/data/gsp/test.zarr/113/.zarray b/tests/data/gsp/test.zarr/113/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/113/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/113/.zattrs b/tests/data/gsp/test.zarr/113/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/113/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/113/0 b/tests/data/gsp/test.zarr/113/0
deleted file mode 100644
index d7df7b80e59b986181d152890e3ad94e4c637cfc..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 681
zcmZQ#ROIMjVPF8^l}ro_A`A=+p+GFdz{v22sY0rk>5Iz;>7~!CUQEg1sOwc^;AUd@
z&Gha0CaHFYl*wVNAxz(mBrZHrVBlq9_#iONK~LFhm$S2@_NT8t;#YuXq5y=!a~jqp
z=a<greqSiI{fYQlet)RJ*-O1y3>YqETHRctYie}qq72kv50|y){_;%o_6}GY@MGQt
zb`)cgU2OGd`n&n-jaCS5mDc~L5j8stYViN#1u^#-bU$+F7ruC{C;77+YH-Hk2cKsg
zdTZOa<lTwm4WXhq4R#g1xHLv=!utr#@YtAl-5%^vgC{Fxg)V2PdS?~8dv@pL!wV~*
z2Dc?GeYZl;{e+vN!FQQ6b*ph2oT8I{Z2O6A?tvd(yfc~>d^#R#aLSp;^b-tw?IS+q
zxXORMbU_ws@MVopPa4IyAFpp-l*eWCBp0W_@~L5K&VKrG{lc7O-iHFO74L)^+{3;g
z)SMx9#@-zPx7V$#JzNAe*z<SikJKH#TX|*`K8pJ<cps<1lD%8bUVnAt?!)Wn&mP~i
zs^>S<;LJ_eo;~^S?!|)#H!ok>ynMoKsKJHnH|;-q;o8l6*LEG)ziN^Un!!pyJ}9|K
c+cPrA*n!RB)@R^1VBq65WMB(m2mq@G0Ib*B0ssI2

diff --git a/tests/data/gsp/test.zarr/114/.zarray b/tests/data/gsp/test.zarr/114/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/114/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/114/.zattrs b/tests/data/gsp/test.zarr/114/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/114/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/114/0 b/tests/data/gsp/test.zarr/114/0
deleted file mode 100644
index 2a415efd5a5a1945d5bdb0b1335c2a23b0e15d01..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 140
xcmZQ#ROIMjVPF8^9tH*m5g_IUVtEEehKc|ngZkVbAd`Q90NA4cAntz<002t)6-NL7

diff --git a/tests/data/gsp/test.zarr/115/.zarray b/tests/data/gsp/test.zarr/115/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/115/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/115/.zattrs b/tests/data/gsp/test.zarr/115/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/115/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/115/0 b/tests/data/gsp/test.zarr/115/0
deleted file mode 100644
index da5e79f1ace51f31113c8a892b60d025dd1fa945..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 677
zcmZQ#ROIMjVPF8^rA!PAA`A=+AwVp{z{v22NqBy`!5PI(m-#&<XO*}a)o)N_;AUd@
z!?^#kfe^#d+ZUe7P22qK?`I|j23{tH4+1p{mu(NYsk8Rt<j;a;zFa_KPyoW*OGkL7
z25`r?e0y@hCicu)=Sxs?nZl2kF;pzHubg9d^6-Ae9H_aLIj?lW4j+8Mnc`T%9~FyY
zD3F6NH#;wnUAt_<VZZjvab{b81ZzOeJz~f4oI!BMK^6AeiU}+B8$iun)^^44yVqg%
zM_d0)$UdBf+uTP-gElNm{`sCe**sSA>FXmWpyrzHe7ui=IsIVWA&J<i|8pKg&DB`-
zypo5bbIIAyzU7JaYjB&}XX^a=<@UzsK3C<|tmonXVGlL;=p+$chHXCogs1ejtV!E{
z4QlQh_l#dkP0ysSrtAHg^tT?jxlxOw6JE`^9aHdwBcwa>xmFs~-1oDVwlkC!9}+py
z^XAl!$3LLvJ~I1i#(7lf)w9o&^Hi*DaGPt;x&P*)2R9$weRS`^h5hRjKS0fOU32Bx
zhj*`^K7MfT=B1-+%b@0_Puh3kG6-Beb$HYCD0FjGfdZg(CT-8iAY%tMid!GZlV{*#
PV&FDpU{_=a0P6t&xfj+=

diff --git a/tests/data/gsp/test.zarr/116/.zarray b/tests/data/gsp/test.zarr/116/.zarray
deleted file mode 100644
index 8fe50e27..00000000
--- a/tests/data/gsp/test.zarr/116/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        145
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        145
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/116/.zattrs b/tests/data/gsp/test.zarr/116/.zattrs
deleted file mode 100644
index e4c12287..00000000
--- a/tests/data/gsp/test.zarr/116/.zattrs
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
-    ]
-}
\ No newline at end of file
diff --git a/tests/data/gsp/test.zarr/116/0 b/tests/data/gsp/test.zarr/116/0
deleted file mode 100644
index e59842d8393ca3e7d9ce8d6f578b356e23007f2d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 684
zcmZQ#ROIMjVPF8^HB1Z)A`A=+p+GFlz{v2Ak<BWodW!Rb90oqE=ZoYHcq%b)GBNyQ
ziazbe&79`cwK?IR$g#<ryUgSm_?Q?z2<|#8bVS8nbH<)VnO~n3s|sU)1|b84!I#$T
zo4aT2A8UsALk`WmS1`g1ZrZ)o!146_xlNMpWtpGzu0RcDonK<RYSM1zw6zl?8=vgU
zkwrEY$VM1!bf-Ad`sB7V4DT0TEv{f!gc^Kz#~G&+?k5W0>N#nZacVAVgc>}X!Kgqb
z_Vf-(-D?3Ua$k53;57Kj?Mo7~^4~-<)XaJw^Wzvh)L@a5%}PP56_$5?*w(oFlxLD9
z)L@S}7nOF*U-u`Xi-|LRTfN0}oCeS6kml$z=32)PllObZ?)$f)2H&p;&Y$r5%8AKx
z_t?c(ecxUWHF$1fq5a>>&Yb+g`3-MVW^@VSG<YHV<RypPlDimwD2r_?Uy=niSis@*
zO20?%4_{(gx$@ikJ>kYsgO6x5TFu^ic>SC`YXY_%biR5Ar@=uJ7py#a=kfawFYaGE
zdtgy0)L^cK2QOT?^YqdEdk-Jnx^ie~Ak^TGaT8{*-MSqJw(Q)td2x#dn!&0-J}9|K
i+cProDl%{}F>vcK@Eb7j3D~JHa4<2j`7;E7wEzIE=+(*q

diff --git a/tests/data/gsp/test.zarr/datetime_gmt/.zarray b/tests/data/gsp/test.zarr/datetime_gmt/.zarray
index 7f5ce911..44c9a184 100644
--- a/tests/data/gsp/test.zarr/datetime_gmt/.zarray
+++ b/tests/data/gsp/test.zarr/datetime_gmt/.zarray
@@ -1,6 +1,6 @@
 {
     "chunks": [
-        145
+        49
     ],
     "compressor": {
         "blocksize": 0,
@@ -14,7 +14,7 @@
     "filters": null,
     "order": "C",
     "shape": [
-        145
+        49
     ],
     "zarr_format": 2
-}
\ No newline at end of file
+}
diff --git a/tests/data/gsp/test.zarr/datetime_gmt/.zattrs b/tests/data/gsp/test.zarr/datetime_gmt/.zattrs
index 9487f1c5..e9976b75 100644
--- a/tests/data/gsp/test.zarr/datetime_gmt/.zattrs
+++ b/tests/data/gsp/test.zarr/datetime_gmt/.zattrs
@@ -3,5 +3,5 @@
         "datetime_gmt"
     ],
     "calendar": "proleptic_gregorian",
-    "units": "minutes since 2019-01-01 00:00:00"
-}
\ No newline at end of file
+    "units": "minutes since 2019-01-01"
+}
diff --git a/tests/data/gsp/test.zarr/datetime_gmt/0 b/tests/data/gsp/test.zarr/datetime_gmt/0
index 7e859e54ee92ebd92537f8d16bf254415b0e7704..76b3c55515371e8758dbee65b015ce50d7d08a60 100644
GIT binary patch
delta 98
zcmcb>)Ws;p#AwLT!N|Y>!d(mu3?e{W1jJuMCQ2CE0~rv&2m?$|zzhQ{P{0ZX3`Pw6
LObq{71Hjq<4hIN9

literal 336
zcmZQ#ROIMjVPF8^07eD|5e5c^i9pOCXA@O1ZOf$(d^%nkZA%V4_$Ol(Q8s1Mg?Bt!
z9%(I$4&3`AWf4{~X~VfUTpDgEO$+wj`6Xc%QZ!-RnO7WYE=djZ_T2g*W)f7;x8~Ff
zHWjCYx;eXUd=oJW$m>~k;u(vQLtM?Q9oN1H8TjRNtvL3CNx?3rYR0xJp9J)LvO1O>
zd4yrVC(unI42%pSjEoGzOhB3$NV5QGRv^s=q}hQq2ax6j(p*5A8%Xm2X<i`B2c-EK
X85#tjfE#F-JkYR;0FYN1sK^BXF=J9^

diff --git a/tests/data/gsp/test.zarr/1/.zarray b/tests/data/gsp/test.zarr/generation_mw/.zarray
similarity index 78%
rename from tests/data/gsp/test.zarr/1/.zarray
rename to tests/data/gsp/test.zarr/generation_mw/.zarray
index 8fe50e27..e133606f 100644
--- a/tests/data/gsp/test.zarr/1/.zarray
+++ b/tests/data/gsp/test.zarr/generation_mw/.zarray
@@ -1,11 +1,12 @@
 {
     "chunks": [
-        145
+        49,
+        21
     ],
     "compressor": {
         "blocksize": 0,
         "clevel": 5,
-        "cname": "lz4",
+        "cname": "zstd",
         "id": "blosc",
         "shuffle": 1
     },
@@ -14,7 +15,8 @@
     "filters": null,
     "order": "C",
     "shape": [
-        145
+        49,
+        21
     ],
     "zarr_format": 2
-}
\ No newline at end of file
+}
diff --git a/tests/data/gsp/test.zarr/generation_mw/.zattrs b/tests/data/gsp/test.zarr/generation_mw/.zattrs
new file mode 100644
index 00000000..758489d4
--- /dev/null
+++ b/tests/data/gsp/test.zarr/generation_mw/.zattrs
@@ -0,0 +1,6 @@
+{
+    "_ARRAY_DIMENSIONS": [
+        "datetime_gmt",
+        "gsp_id"
+    ]
+}
diff --git a/tests/data/gsp/test.zarr/generation_mw/0.0 b/tests/data/gsp/test.zarr/generation_mw/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..28971bccf18470194aa05d34feeab3b0300dcef5
GIT binary patch
literal 2630
zcmWlac{~%0AIHC<2YV8t<tT)ZkYA`rM4`EAmgGv5vykM>lH464*WBhP#~5uh+H!50
zBXZ5==83}N>2j4t@$32H^ZvYEpU;1ve?B|-(uC9$0R9sS0U!wgTEAVb{^+l(YVwCJ
zz*+%-Y<vjEH$i3jekdB?Mj`}3u1KIH>Ro)1fRFgkJh`Y3>FviH!5hUtvxpq=5G<b9
zmHG8)<Dos5pTo-3JjX$l@PhQF+oh~Vz?iJVe-sMx2E|SoaL3p29b;FSmEERVuTbzf
z^NUDe>^D;JodJ2o^O#QAHQ7c`wj8p!`6)HW!_=RDY4w75^IyeraAB-^^rRk3Y?It)
zQ}WSY)#vncP3Ry_b6SGeB3~TYI;AIzHPJXTkO5N*>L&^3BM3!kWS=fE-d`YW(Htxb
zWzglOOZul>T3*D_|G;(A6a64)N8wddtndkwpOv2)drh%h1S@?Axs14!4^elbauj@p
zF{bG<h8H<-eftnQV;Azypz`E$XZA*Gvw<!KJ|Atq=tbycG%M^35o-!!$_6%*!M|IF
ztXkNgAUPmt3HI&+Cx<okq4%8Tz*G^OhGl>ZHz)_L`f*@d2~<yQm76v)vH+@GVGL+{
zXhQhEdkL17b@z7!?AB;HGA{<~1Sj`IAsXxb&Cis@fac)Yk_1KAg<H65O}#7}c$qpf
z8r6k!)#`iz8ahvMotLTBM+`cjyyhl{Uv@ch3G76RQ>~M>W#*m<cgYKB?@WuCk%#Ox
zX3gDREUw>irmZ1YRr1PAx;I$s<l1e<>PV)Z7i~qAnHWi2_jQEFDj&m8SH?vuB~Ph&
zB=zq&7^{#0A}01#zGxZM&^McDn)FoS542(l442OIT|5t4Ij*d|Mh3RupInpt3bQPd
zVTbp?cd?U`A?C+_#$XEiqM?M?;05c4>_s1@nkRh>95pW@lR$G}P7z<6K63CvwgDRJ
zo1i#lU<g@_+R3^_dqPj0yDPlF_pfvGPU8{iUGeqAXHycL>o&q*%<rbKZ}0*fv0En%
z&O#Yr0YvOT80J4V!&!pA>fv#-n?;$m09~Ds3}Fj#rSG$_w~<wOdpej$6&qC!{0Uxr
z_VbGoD>o)o^#l<F4{axuNhP#@`}aq5C-p<-sk_nWZi(x+@uVSOy8Yo(=!Fb9_Ndgu
zQ&lI7%*$*955Z4WXO*4xzjA%Two7yd_nU<x;P$<UsbfV3dlCM73xBchd2_4f*Wi4J
zyuc~!&Z}>Zdwxticz4)tPDvNaiX^hU_+&=Kkx{8d&F}eeH*RhM^5uN(_yMUvbgZw)
zLU=&-O0VK!Xm`r4)@S|j?7(wHaCZ|rnyWkbfZ(}0B%H_ji~Zx?PeU!;=h;g+OYlBB
zST0G^f%wwe(iSuG{l_Fx2?3-M!L5iEd(QEs<@#m+k2dYoehbh&uQO^OQG?GZT2!BC
z#rTR@V)sL`*Y{wmy8azSss8(*S9fvdcxxDFVKYOAlRya~0rK-97S3vK19|O>lf<<>
zmFrjB0Pb7|1k6qftfxpf;^jOCZA4d{PQS|!rGxTAilj<x0?Ntx<UX>}?Bp$7^G^`O
zxb!(jBqZFE)pqCH951@NUc(Hy<2za+zZE5Z>-L#+5Ln-zK8@i*#*2O*`b_R@`P^8L
zoCP`bVqT&*ROWdtNy*sui}L~nyF_;1(OMpD1)~!LC;QCYW^uf2M4f1?Q1kN`F$fqK
zHA6Yr>{Ux4n$&QEa$kQoc?phN7FiF`cvrtK7{n0`9WDEg15aTy;PjYvXtbONDsP}x
z^WV=C<nB_~?);qL_uABvW9_IvY-15c)=y4>)!O%4|2z=5_adyNaJ}~3IOXl*dQdxc
zu(-_$lc5oAwJ4P^qW3mKzz#Gv7LQ#iowk##vD!|l*OyPqQO1CpQG{V!9NWbtW()<R
zU(U!#sPuvZLh;afU04AhfOs-U!9%Vs<gr@D#ot58llD=7nGvu6+(bxw>?X3NM$9Kd
zPkT-)%OT7L5GJjTfij6Xse*h9b@FYl5|OwpK%%Qcah<UaVbpg|THT_p%g;U#2f-Vz
z*LC7_{kQ05`Hl<G?Z^0{ogtUtK4spx<mqaQl8UkVj+cG(Yzdg#?$fWNa|0b$cG@e9
zS8@A^zbc@o^;ewqjJnGAVD$$7lw7li{H5W5ByIODCaS7R(z9x73nqkA8-|a<55)f<
z_^XHL>R?Z##4LXrZz^2|ye(G`4-|gAqO=iPIwSCCe;l1C1%~26d8!^s^MQ$>Zl*c=
zqKlQW1HdslVm5v7xXLp<X4S8ab<fwttZGoUc*;R$cNNpG1`l6f{zq~$h$jrf?K~4T
zsu#yCZzy>t+T`++1EhI)a;|)Ju&Y7~&+)>5PB=&0j~xkmd<SeF+y^9}05BCW2$}`^
zfY0jTy}>L;zEu40LnQ5h0Z=W|2|g1-R3WqsPiQGExpK8X)WTuEi#2J^adh#f+37Ej
zE2MY#IwgWAo0DyB!l&HfLKxFK`wa2Rwp<xLB=)Jh3t<;r>K?~Z#VR-%gAw;(Wuh%|
zZL>ANGMDDzsCjN3)7%&byZIXhu(kq7rS}o+mn?+M?i=Hy@V8P*(7by4V@G2=>Y8NF
zlJ@ve0Z4xI=~J-6#_?}kMj?A_rKjYU3?bK-t)$VWEfyu%vUbj*O@Et;C&N4jM$e;P
zqZkwH2MldYNg1u$8jfYuWJ)LS(WLpR90Y{FZTP(tz6W$0b#J?VO1J4*jhOT7b&WGa
zhuq-ygVhta%uoN^{ZIrwryfAY;iXK0#&y4R<&TLD?5Qq8MnT*jzcx%9eEIRWm)EGR
zY~LeJ2vo^2%#KFl!MUX(@1J%kg_{6AgTLvv$ORs(Zhled+rNs`S~LcbmIG<tpC${T
z0mtK`amg8vZsmrB8OGfj=Uwx@o1f-ioLN&$B;cqiDdB}UEH62}xW3|PWeKI6*g)y1
zAr_M=t9j)m6b2DTBQ@1i$;^SaHVU(a#G}@|YOBN3$PF|awSBO!kNNg3g-35{V7#D^
zJ4oGhR$G66Z(sjlJ+FyHW#rOp-;jG4z3rT#p@E^%I^HWrE3Jq{>wDGR*55MB9p(&=
zz2v=YW>SlLyLw(SSX~VxL!4pG7@bF@w$v7Py?xWfXkt)$*lZ4asFhbwc~z0aV%629
zGf1`dy>Hp<ZVHe5qCUTxLcx(!o61XwOlD74LnV(;MouG^mIakX*X6`#*VAb9@-p7D
z98!FCT5>S%5k4v+Ix{mfCn}$p8k=qAcl83=?}m>AT(Gg4xg-<Jb2q>7ZJq%Pa0Xc6
z@P^+^ZyOmI85<k@-qzR{jJJ)pfAe<)@EiYiatcd3_Ie!ntJiE%$5XkBZ!+XGzl%u5
znG=DD?&6%v6S177T^rv=sA65(hBO%!sl%Ey5*lsK-02rYL1;N<WLyel<`ocQ{~Z3{
Mxz~lHlH*JIKjSCr9RL6T

literal 0
HcmV?d00001

diff --git a/tests/data/gsp/test.zarr/101/.zarray b/tests/data/gsp/test.zarr/gsp_id/.zarray
similarity index 77%
rename from tests/data/gsp/test.zarr/101/.zarray
rename to tests/data/gsp/test.zarr/gsp_id/.zarray
index 8fe50e27..7917cc83 100644
--- a/tests/data/gsp/test.zarr/101/.zarray
+++ b/tests/data/gsp/test.zarr/gsp_id/.zarray
@@ -1,6 +1,6 @@
 {
     "chunks": [
-        145
+        21
     ],
     "compressor": {
         "blocksize": 0,
@@ -9,12 +9,12 @@
         "id": "blosc",
         "shuffle": 1
     },
-    "dtype": "<f8",
-    "fill_value": "NaN",
+    "dtype": "<U2",
+    "fill_value": null,
     "filters": null,
     "order": "C",
     "shape": [
-        145
+        21
     ],
     "zarr_format": 2
-}
\ No newline at end of file
+}
diff --git a/tests/data/gsp/test.zarr/10/.zattrs b/tests/data/gsp/test.zarr/gsp_id/.zattrs
similarity index 59%
rename from tests/data/gsp/test.zarr/10/.zattrs
rename to tests/data/gsp/test.zarr/gsp_id/.zattrs
index e4c12287..6de1b9d6 100644
--- a/tests/data/gsp/test.zarr/10/.zattrs
+++ b/tests/data/gsp/test.zarr/gsp_id/.zattrs
@@ -1,5 +1,5 @@
 {
     "_ARRAY_DIMENSIONS": [
-        "datetime_gmt"
+        "gsp_id"
     ]
-}
\ No newline at end of file
+}
diff --git a/tests/data/gsp/test.zarr/gsp_id/0 b/tests/data/gsp/test.zarr/gsp_id/0
new file mode 100644
index 0000000000000000000000000000000000000000..042a70a9e49a305c2805449f8bb232f63384398c
GIT binary patch
literal 71
zcmZQ#G~`&pz`y{)?m#R8#QH$|pWD#L*u>P#+``fj4vdU|G7Jo?3<jbGaSZwfhJFm{
H0bn5jU%Cl-

literal 0
HcmV?d00001

diff --git a/tests/data/gsp/test.zarr/100/.zarray b/tests/data/gsp/test.zarr/installedcapacity_mwp/.zarray
similarity index 78%
rename from tests/data/gsp/test.zarr/100/.zarray
rename to tests/data/gsp/test.zarr/installedcapacity_mwp/.zarray
index 8fe50e27..e133606f 100644
--- a/tests/data/gsp/test.zarr/100/.zarray
+++ b/tests/data/gsp/test.zarr/installedcapacity_mwp/.zarray
@@ -1,11 +1,12 @@
 {
     "chunks": [
-        145
+        49,
+        21
     ],
     "compressor": {
         "blocksize": 0,
         "clevel": 5,
-        "cname": "lz4",
+        "cname": "zstd",
         "id": "blosc",
         "shuffle": 1
     },
@@ -14,7 +15,8 @@
     "filters": null,
     "order": "C",
     "shape": [
-        145
+        49,
+        21
     ],
     "zarr_format": 2
-}
\ No newline at end of file
+}
diff --git a/tests/data/gsp/test.zarr/installedcapacity_mwp/.zattrs b/tests/data/gsp/test.zarr/installedcapacity_mwp/.zattrs
new file mode 100644
index 00000000..758489d4
--- /dev/null
+++ b/tests/data/gsp/test.zarr/installedcapacity_mwp/.zattrs
@@ -0,0 +1,6 @@
+{
+    "_ARRAY_DIMENSIONS": [
+        "datetime_gmt",
+        "gsp_id"
+    ]
+}
diff --git a/tests/data/gsp/test.zarr/installedcapacity_mwp/0.0 b/tests/data/gsp/test.zarr/installedcapacity_mwp/0.0
new file mode 100644
index 0000000000000000000000000000000000000000..73480d16d72e482b908d606aa828d839f1133622
GIT binary patch
literal 218
zcmZQ#oXDY}z`y{)w-^{0M1c4Z5NmAJ|C^v8f0UIWg)_@kyCmc8-3ZT$6J6%VXZ=6J
zApPr|?^dDeb?;q5TIX=i_TRxVX(oeDL)GUTVb99}x<@uH66Tb8BpvdWLCztwZ|#+2
z50SGkTb12x_#e$%#>KEIz3a06hb`VO)~YD%-25zhMcj;9hL%eVo;R`^^|xrGtkt%Z
znqT93{se>4viGG4;rWqi&Y=axLE;i#iIx%!5~d;!3=Uih6PzF4(cZw~c;nlu-L<9L
S!|%Rb4Wjc)w;#{C&j<k4K3L}f

literal 0
HcmV?d00001

diff --git a/tests/data_sources/get_test_data.py b/tests/data_sources/get_test_data.py
index c038bfe4..73708b7e 100644
--- a/tests/data_sources/get_test_data.py
+++ b/tests/data_sources/get_test_data.py
@@ -6,6 +6,7 @@
 from pathlib import Path
 import pandas as pd
 import os
+import numcodecs
 import nowcasting_dataset
 from nowcasting_dataset.data_sources.nwp_data_source import open_nwp, NWP_VARIABLE_NAMES
 from nowcasting_dataset.data_sources.gsp.gsp_data_source import GSPDataSource
@@ -79,27 +80,19 @@
 nwp_data.to_zarr(f"{local_path}/tests/data/nwp_data/test.zarr")
 
 # ### GSP data
+filename = "gs://solar-pv-nowcasting-data/PV/GSP/v1/pv_gsp.zarr"
 
-gsp = GSPDataSource(
-    filename="gs://solar-pv-nowcasting-data/PV/GSP/v1/pv_gsp.zarr",
-    start_dt=start_dt,
-    end_dt=end_dt,
-    history_minutes=30,
-    forecast_minutes=60,
-    convert_to_numpy=True,
-    image_size_pixels=64,
-    meters_per_pixel=2000,
-)
-
+gsp_power = xr.open_dataset(filename, engine="zarr")
+gsp_power = gsp_power.sel(datetime_gmt=slice(start_dt, end_dt))
+gsp_power = gsp_power.sel(gsp_id=slice(gsp_power.gsp_id[0], gsp_power.gsp_id[20]))
 
-gsp.gsp_power.columns = [str(col) for col in gsp.gsp_power.columns]
+gsp_power["gsp_id"] = gsp_power.gsp_id.astype("str")
 
-# select limited data
-data = gsp.gsp_power
-data = data[data.columns[0:20]]
+encoding = {
+    var: {"compressor": numcodecs.Blosc(cname="zstd", clevel=5)} for var in gsp_power.data_vars
+}
 
-data_xarray = data.to_xarray()
-data_xarray.to_zarr(f"{local_path}/tests/data/gsp/test.zarr", mode="w")
+gsp_power.to_zarr(f"{local_path}/tests/data/gsp/test.zarr", mode="w", encoding=encoding)
 
 
 # ### satellite

From 021636ca4964d4bffc163605856e5d6e4cffffac Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Thu, 30 Sep 2021 14:55:22 +0100
Subject: [PATCH 7/8] use concurrent methods to get quicker data from pvlive
 api

---
 nowcasting_dataset/data_sources/gsp/pvlive.py | 71 +++++++++++--------
 1 file changed, 40 insertions(+), 31 deletions(-)

diff --git a/nowcasting_dataset/data_sources/gsp/pvlive.py b/nowcasting_dataset/data_sources/gsp/pvlive.py
index a7ffd6c8..c04f843c 100644
--- a/nowcasting_dataset/data_sources/gsp/pvlive.py
+++ b/nowcasting_dataset/data_sources/gsp/pvlive.py
@@ -4,6 +4,8 @@
 from pvlive_api import PVLive
 from typing import Optional
 import pytz
+from tqdm import tqdm
+from concurrent import futures
 
 from nowcasting_dataset.data_sources.gsp.eso import get_list_of_gsp_ids
 
@@ -40,21 +42,22 @@ def load_pv_gsp_raw_data_from_pvlive(
     gsp_data_df = []
     logger.debug(f"Will be getting data for {len(gsp_ids)} gsp ids")
     # loop over gsp ids
-    for gsp_id in gsp_ids:
-
-        one_gsp_data_df = []
-
-        # set the first chunk start and end times
-        start_chunk = first_start_chunk
-        end_chunk = first_end_chunk
-
-        # loop over 30 days chunks (nice to see progress instead of waiting a long time for one command - this might
-        # not be the fastest)
-        while start_chunk <= end:
-            logger.debug(f"Getting data for gsp id {gsp_id} from {start_chunk} to {end_chunk}")
-
-            one_gsp_data_df.append(
-                pvl.between(
+    # limit the total number of concurrent tasks to be 4, so that we don't hit the pvlive api too much
+    future_tasks = []
+    with futures.ThreadPoolExecutor(max_workers=4) as executor:
+        for gsp_id in gsp_ids:
+
+            # set the first chunk start and end times
+            start_chunk = first_start_chunk
+            end_chunk = first_end_chunk
+
+            # loop over 30 days chunks (nice to see progress instead of waiting a long time for one command - this might
+            # not be the fastest)
+            while start_chunk <= end:
+                logger.debug(f"Getting data for gsp id {gsp_id} from {start_chunk} to {end_chunk}")
+
+                task = executor.submit(
+                    pvl.between,
                     start=start_chunk,
                     end=end_chunk,
                     entity_type="gsp",
@@ -62,30 +65,36 @@ def load_pv_gsp_raw_data_from_pvlive(
                     extra_fields="installedcapacity_mwp",
                     dataframe=True,
                 )
-            )
 
-            # add 30 days to the chunk, to get the next chunk
-            start_chunk = start_chunk + CHUNK_DURATION
-            end_chunk = end_chunk + CHUNK_DURATION
+                future_tasks.append(task)
+
+                # add 30 days to the chunk, to get the next chunk
+                start_chunk = start_chunk + CHUNK_DURATION
+                end_chunk = end_chunk + CHUNK_DURATION
 
-            if end_chunk > end:
-                end_chunk = end
+                if end_chunk > end:
+                    end_chunk = end
 
-        # join together one gsp data, and sort
-        one_gsp_data_df = pd.concat(one_gsp_data_df)
-        one_gsp_data_df = one_gsp_data_df.sort_values(by=["gsp_id", "datetime_gmt"])
+        logger.debug(f"Getting results")
+        # Collect results from each thread.
+        for task in tqdm(future_tasks):
+            one_chunk_one_gsp_gsp_data_df = task.result()
 
-        # normalize
-        if normalize_data:
-            one_gsp_data_df["generation_mw"] = (
-                one_gsp_data_df["generation_mw"] / one_gsp_data_df["installedcapacity_mwp"]
-            )
+            if normalize_data:
+                one_chunk_one_gsp_gsp_data_df["generation_mw"] = (
+                    one_chunk_one_gsp_gsp_data_df["generation_mw"]
+                    / one_chunk_one_gsp_gsp_data_df["installedcapacity_mwp"]
+                )
 
-        # append to longer list
-        gsp_data_df.append(one_gsp_data_df)
+            # append to longer list
+            gsp_data_df.append(one_chunk_one_gsp_gsp_data_df)
 
+    # join together gsp data
     gsp_data_df = pd.concat(gsp_data_df)
 
+    # sort
+    gsp_data_df = gsp_data_df.sort_values(by=["gsp_id", "datetime_gmt"])
+
     # remove any extra data loaded
     gsp_data_df = gsp_data_df[gsp_data_df["datetime_gmt"] <= end]
 

From 9e2c94feec3dcafa6f43a06a2ee534611fae0591 Mon Sep 17 00:00:00 2001
From: peterdudfield <peter.dudfield@hotmail.com>
Date: Thu, 30 Sep 2021 16:37:46 +0100
Subject: [PATCH 8/8] pylint

---
 nowcasting_dataset/data_sources/gsp/pvlive.py | 1 -
 nowcasting_dataset/dataset/split/__init__.py  | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/nowcasting_dataset/data_sources/gsp/pvlive.py b/nowcasting_dataset/data_sources/gsp/pvlive.py
index e0b4e225..d68ba519 100644
--- a/nowcasting_dataset/data_sources/gsp/pvlive.py
+++ b/nowcasting_dataset/data_sources/gsp/pvlive.py
@@ -125,7 +125,6 @@ def get_installed_capacity(
     Returns: pd.Series of installed capacity indexed by gsp_id
 
     """
-
     logger.debug(f"Getting all installed capacity at {start}")
 
     # get a lit of gsp ids
diff --git a/nowcasting_dataset/dataset/split/__init__.py b/nowcasting_dataset/dataset/split/__init__.py
index e69de29b..c7eb89d1 100644
--- a/nowcasting_dataset/dataset/split/__init__.py
+++ b/nowcasting_dataset/dataset/split/__init__.py
@@ -0,0 +1 @@
+""" split functions """