Don't expect consolidated metadata when fetching (#342)

We're no longer writing consolidated metadata when we update the Zarr
array. However, we do still have a consolidated metadata object in the
store. For now, tell Zarr to ignore that consolidated metadata.

We also needed to update our exception handling as Xarray returns
different exceptions depending on if the data is consolidated or not.

CONTRIBUTING.md

@@ -39,6 +39,21 @@ poetry run mypy src/
 poetry run pytest
 ```
 
+**Note**: You will need a postgres database and a `.env` file with the required
+testing env variables set. You can do so with the following:
+
+```
+# Run a postgres container in the background ("-d")
+$ docker run --name postgres15 -d -p 5432:5432 -e POSTGRES_PASSWORD=<password> -e POSTGRES_USER=postgres postgres:15
+
+# Add the required variables to a .env file for testing
+$ cat << EOF >> .env
+TEST_DB_USER=postgres
+TEST_DB_PASS=<password>
+TEST_DB_HOST=localhost:5432
+EOF
+```
+
 **Note**: There aren’t any type definitions for `pytest`, so running `mypy` on
 the `tests/` directory results in errors. As such, it’s best to run `mypy` just
 for our `src/` directory.

src/unified_graphics/diag.py

@@ -207,7 +207,7 @@ def open_diagnostic(
         f"/{model}/{system}/{domain}/{background}/{frequency}"
         f"/{variable.value}/{initialization_time}/{loop.value}"
     )
-    return xr.open_zarr(store, group=group)
+    return xr.open_zarr(store, group=group, consolidated=False)
 
 
 def parse_filter_value(value):
@@ -512,7 +512,7 @@ def summary(
         ]
     )
 
-    ds = xr.open_zarr(store, group=path)
+    ds = xr.open_zarr(store, group=path, consolidated=False)
     ds = apply_filters(ds, filters)
     return DiagSummary.from_dataset(ds) if len(ds["nobs"]) > 0 else None
 

src/unified_graphics/routes.py

@@ -8,13 +8,22 @@
     stream_template,
     url_for,
 )
+from zarr.errors import FSPathExistNotDir, GroupNotFoundError  # type: ignore
 
 from unified_graphics import diag
 from unified_graphics.models import db
 
 bp = Blueprint("api", __name__)
 
 
+@bp.errorhandler(GroupNotFoundError)
+def handle_diag_group_not_found(e):
+    if isinstance(e, FSPathExistNotDir):
+        return jsonify(msg="Unable to read diagnostic file group"), 500
+
+    return jsonify(msg="Diagnostic file group not found"), 404
+
+
 # TODO: Don't hardcode the error message
 # Instead of hard-coding the error message, we should read e.args[0] to allow different
 # types of 404s, such as a missing variable, init time, or zarr

tests/conftest.py

@@ -342,7 +342,7 @@ def factory(
                 f"/{data.model}/{data.system}/{data.domain}/{data.background}"
                 f"/{data.frequency}/{data.attrs['name']}/{initialization_time}/{loop}"
             )
-            data.to_zarr(store, group=group)
+            data.to_zarr(store, group=group, consolidated=False)
             return zarr_file
 
         for variable in variables:
@@ -364,7 +364,7 @@ def factory(
                     f"/{ds.model}/{ds.system}/{ds.domain}/{ds.background}"
                     f"/{ds.frequency}/{variable}/{initialization_time}/{loop}"
                 )
-                ds.to_zarr(store, group=group)
+                ds.to_zarr(store, group=group, consolidated=False)
             except Exception as e:
                 raise e
 

tests/etl/test_save.py

@@ -74,6 +74,7 @@ def test_save_new(ps_anl, analysis, zarr_file, session):
                 f"{model}/{system}/{domain}/{background}/{frequency}/"
                 f"ps/{init_time}/anl"
             ),
+            consolidated=False,
         ),
         ps_anl,
     )
@@ -116,6 +117,7 @@ def test_add_loop(ps_anl, ps_ges, analysis, zarr_file, session):
                 f"{model}/{system}/{domain}/{background}/{frequency}/"
                 f"ps/{init_time}/anl"
             ),
+            consolidated=False,
         ),
         ps_anl,
     )
@@ -127,6 +129,7 @@ def test_add_loop(ps_anl, ps_ges, analysis, zarr_file, session):
                 f"{model}/{system}/{domain}/{background}/{frequency}/"
                 f"ps/{init_time}/ges"
             ),
+            consolidated=False,
         ),
         ps_ges,
     )
@@ -147,6 +150,7 @@ def test_add_variable(ps_anl, t_anl, analysis, zarr_file, session):
                 f"{model}/{system}/{domain}/{background}/{frequency}/"
                 f"ps/{init_time}/anl"
             ),
+            consolidated=False,
         ),
         ps_anl,
     )
@@ -158,6 +162,7 @@ def test_add_variable(ps_anl, t_anl, analysis, zarr_file, session):
                 f"{model}/{system}/{domain}/{background}/{frequency}/"
                 f"t/{init_time}/anl"
             ),
+            consolidated=False,
         ),
         t_anl,
     )

tests/test_diag.py

@@ -6,6 +6,7 @@
 import pytest
 import xarray as xr
 from werkzeug.datastructures import MultiDict
+from zarr.errors import GroupNotFoundError  # type: ignore
 
 from unified_graphics import diag
 from unified_graphics.models import Analysis, WeatherModel
@@ -138,9 +139,9 @@ def test_open_diagnostic_local_does_not_exist(diag_zarr_file):
     background = "HRRR"
     frequency = "REALTIME"
     init_time = "2022-05-16T04:00"
-    expected = r"No such file or directory: '.*test_diag.zarr'$"
+    expected = r"group not found at path .*$"
 
-    with pytest.raises(FileNotFoundError, match=expected):
+    with pytest.raises(GroupNotFoundError, match=expected):
         diag.open_diagnostic(
             diag_zarr_file,
             model,

tests/test_unified_graphics.py

@@ -648,7 +648,7 @@ def test_diag_not_found(variable, client):
     )
 
     assert response.status_code == 404
-    assert response.json == {"msg": "Diagnostic file not found"}
+    assert response.json == {"msg": "Diagnostic file group not found"}
 
 
 @pytest.mark.parametrize(
@@ -663,7 +663,7 @@ def test_diag_read_error(variable, app, client):
     )
 
     assert response.status_code == 500
-    assert response.json == {"msg": "Unable to read diagnostic file"}
+    assert response.json == {"msg": "Unable to read diagnostic file group"}
 
 
 @pytest.mark.parametrize(