Patch CMIP5/CMIP6

CHANGELOG.rst

@@ -71,19 +71,26 @@ Features
 - Revert back to using official DRS attributes when building CMIP5 and CMIP6 collections.
   (:pr:`126`) `Anderson Banihirwe`_
 
+- Add ``.df`` property for interfacing with the built collection via dataframe
+  To maintain backwards compatiblity. (:pr:`127`) `Anderson Banihirwe`_
+
 
 .. _NA-CORDEX: https://na-cordex.org/
 .. _intake-esm-datastore: https://github.com/NCAR/intake-esm-datastore
 
+
 Bug Fixes
 ----------
 
+-  For CMIP6, extract ``grid_label`` from directory path instead of file name. (:pr:`127`) `Anderson Banihirwe`_
+
 
 Internal Changes
 ----------------
 
 
 
+
 Intake-esm v2019.8.5 (2019-08-05)
 ==================================
 

intake_esm/cesm.py

@@ -33,6 +33,7 @@ def _get_file_attrs(self, filepath):
         fileparts = {key: None for key in keys}
         fileparts['file_basename'] = file_basename
         fileparts['file_fullpath'] = filepath
+        fileparts['file_dirname'] = os.path.dirname(filepath) + '/'
 
         date_str_regex = r'\d{4}\-\d{4}|\d{6}\-\d{6}|\d{8}\-\d{8}|\d{10}Z\-\d{10}Z|\d{12}Z\-\d{12}Z'
         datestr = _extract_attr_with_regex(file_basename, regex=date_str_regex)

intake_esm/cesm_aws.py

@@ -4,7 +4,7 @@
 
 import pandas as pd
 import xarray as xr
-from tqdm.autonotebook import tqdm
+from tqdm.auto import tqdm
 
 from . import aggregate, config
 from .bld_collection_utils import get_subset

intake_esm/cmip.py

@@ -56,10 +56,11 @@ def _get_file_attrs(self, filepath):
         fileparts.update(f)
 
         parent = os.path.dirname(filepath).strip('/')
-        parent_split = parent.split(fileparts['model'])
+        parent_split = parent.split(f"/{fileparts['model']}/")
         part_1 = parent_split[0].strip('/').split('/')
         part_2 = parent_split[1].strip('/').split('/')
 
+        fileparts['file_dirname'] = parent + '/'
         fileparts['institute'] = part_1[-1]
         fileparts['frequency'] = part_2[1]
         fileparts['modeling_realm'] = part_2[2]
@@ -150,10 +151,13 @@ def _get_file_attrs(self, filepath):
         fileparts.update(f)
 
         parent = os.path.dirname(filepath).strip('/')
-        parent_split = parent.split(fileparts['source_id'])
+        parent_split = parent.split(f"/{fileparts['source_id']}/")
         part_1 = parent_split[0].strip('/').split('/')
-        # part_2 = parent_split[1].strip('/').split('/')
 
+        grid_label = parent.split(f"/{fileparts['variable_id']}/")[1].strip('/').split('/')[0]
+
+        fileparts['file_dirname'] = parent + '/'
+        fileparts['grid_label'] = grid_label
         fileparts['activity_id'] = part_1[-2]
         fileparts['institution_id'] = part_1[-1]
 

intake_esm/collection.py

@@ -4,7 +4,7 @@
 import docrep
 import pandas as pd
 import xarray as xr
-from tqdm.autonotebook import tqdm
+from tqdm.auto import tqdm
 
 from . import config
 from .bld_collection_utils import make_attrs

intake_esm/collection_defs/cmip.yaml

@@ -9,6 +9,7 @@ collections:
     - experiment
     - file_basename
     - file_fullpath
+    - file_dirname
     - frequency
     - institute
     - mip_table
@@ -40,6 +41,7 @@ collections:
     - experiment_id
     - file_basename
     - file_fullpath
+    - file_dirname
     - grid_label
     - institution_id
     - member_id

intake_esm/collection_defs/cordex.yaml

@@ -13,7 +13,9 @@ collections:
     - bias_corrected_or_raw
     - file_fullpath
     - file_basename
+    - file_dirname
     order_by_columns:
-      - file_fullpath
+      - variable
     required_columns:
+      - variable
       - file_fullpath

intake_esm/collection_defs/gmet.yaml

@@ -11,6 +11,7 @@ collections:
       - resolution
       - file_fullpath
       - file_basename
+      - file_dirname
     order_by_columns:
       - member_id
       - time_range
@@ -23,6 +24,7 @@ collections:
       - direct_access
       - file_basename
       - file_fullpath
+      - file_dirname
       - forecast_initial_date
       - forecast_initial_hour
       - grid

intake_esm/collection_defs/mpige.yaml

@@ -12,6 +12,7 @@ collections:
     - ensemble
     - file_fullpath
     - file_basename
+    - file_dirname
     - ctrl_branch_year
     - year_offset
     - sequence_order

intake_esm/cordex.py

@@ -1,7 +1,7 @@
 """ Implementation for The ECMWF ERA5 Reanalyses data holdings """
 import os
 
-from tqdm.autonotebook import tqdm
+from tqdm.auto import tqdm
 
 from . import aggregate, config
 from .bld_collection_utils import _ensure_file_access, _reverse_filename_format, get_subset
@@ -25,6 +25,7 @@ def _get_file_attrs(self, filepath):
         fileparts = {key: None for key in keys}
         fileparts['file_basename'] = file_basename
         fileparts['file_fullpath'] = filepath
+        fileparts['file_dirname'] = os.path.dirname(filepath) + '/'
         filename_template = '{variable}.{experiment}.{global_climate_model}.{regional_climate_model}.{frequency}.{grid}.{bias_corrected_or_raw}.nc'
 
         f = _reverse_filename_format(file_basename, filename_template)

intake_esm/core.py

@@ -5,6 +5,7 @@
 
 import numpy as np
 import s3fs
+from cached_property import cached_property
 from intake.catalog import Catalog
 from intake.catalog.local import LocalCatalogEntry
 from intake.utils import yaml_load
@@ -123,6 +124,10 @@ def __init__(
 
         self._entries = {}
 
+    @cached_property
+    def df(self):
+        return self.ds.to_dataframe()
+
     def describe(self, variables=()):
         """
         TODO: Generate summary for variables in the .ds attribute

intake_esm/era5.py

@@ -2,7 +2,7 @@
 import os
 
 import xarray as xr
-from tqdm.autonotebook import tqdm
+from tqdm.auto import tqdm
 
 from . import aggregate, config
 from .bld_collection_utils import get_subset
@@ -28,6 +28,7 @@ def _get_file_attrs(self, filepath):
         fileparts = {key: None for key in keys}
         fileparts['file_basename'] = file_basename
         fileparts['file_fullpath'] = filepath
+        fileparts['file_dirname'] = os.path.dirname(filepath) + '/'
 
         fileparts['stream'] = fs[1]
         if fs[2] == 'an':

intake_esm/gmet.py

@@ -1,7 +1,7 @@
 """ Implementation for The Gridded Meteorological Ensemble Tool (GMET) data holdings """
 import os
 
-from tqdm.autonotebook import tqdm
+from tqdm.auto import tqdm
 
 from . import aggregate, config
 from .bld_collection_utils import _extract_attr_with_regex, get_subset
@@ -24,6 +24,7 @@ def _get_file_attrs(self, filepath):
         fileparts = {key: None for key in keys}
         fileparts['file_basename'] = file_basename
         fileparts['file_fullpath'] = filepath
+        fileparts['file_dirname'] = os.path.dirname(filepath) + '/'
 
         date_str_regex = r'\d{8}\_\d{8}'
         datestr = _extract_attr_with_regex(filepath, regex=date_str_regex)

intake_esm/mpige.py

@@ -6,7 +6,7 @@
 
 import pandas as pd
 import xarray as xr
-from tqdm.autonotebook import tqdm
+from tqdm.auto import tqdm
 
 from . import aggregate, config
 from .bld_collection_utils import _extract_attr_with_regex, get_subset
@@ -36,6 +36,7 @@ def _get_file_attrs(self, filepath):
         fileparts = {key: None for key in keys}
         fileparts['file_basename'] = file_basename
         fileparts['file_fullpath'] = filepath
+        fileparts['file_dirname'] = os.path.dirname(filepath) + '/'
 
         date_str_regex = r'\d{4}\_\d{4}|\d{6}\_\d{6}|\d{8}\_\d{8}|\d{10}\_\d{10}|\d{12}\_\d{12}'
         datestr = _extract_attr_with_regex(file_basename, regex=date_str_regex)

intake_esm/source.py

@@ -1,6 +1,7 @@
 import intake_xarray
 import xarray as xr
-from tqdm.autonotebook import tqdm
+from cached_property import cached_property
+from tqdm.auto import tqdm
 
 from . import aggregate, config
 from .bld_collection_utils import _ensure_file_access, get_subset
@@ -40,6 +41,10 @@ def get_results(self):
         ds = get_subset(self.collection_name, self.query)
         return ds
 
+    @cached_property
+    def df(self):
+        return self.ds.to_dataframe()
+
     def _validate_kwargs(self, kwargs):
 
         _kwargs = kwargs.copy()

intake_esm/storage.py

@@ -7,7 +7,7 @@
 from time import sleep
 from warnings import warn
 
-from tqdm.autonotebook import tqdm
+from tqdm.auto import tqdm
 
 from . import config
 

requirements.txt

@@ -4,3 +4,4 @@ pyyaml
 tqdm
 fsspec
 s3fs
+cached-property

setup.cfg

@@ -7,7 +7,7 @@ select = B,C,E,F,W,T4,B9
 
 [isort]
 known_first_party=intake_esm
-known_third_party=click,dask,docrep,fsspec,intake,intake_xarray,numpy,pandas,pkg_resources,pytest,s3fs,setuptools,tqdm,xarray,yaml
+known_third_party=cached_property,click,dask,docrep,fsspec,intake,intake_xarray,numpy,pandas,pkg_resources,pytest,s3fs,setuptools,tqdm,xarray,yaml
 multi_line_output=3
 include_trailing_comma=True
 force_grid_wrap=0

tests/cmip6/test_cmip6.py

@@ -30,6 +30,22 @@ def test_build_collection_file():
         )
         col = intake.open_esm_metadatastore(collection_name='cmip6_test_collection')
         assert isinstance(col.ds, xr.Dataset)
+        assert isinstance(col.df, pd.DataFrame)
+        assert set(col.df.grid_label.unique()) == set(['gr', 'gn'])
+        assert set(col.df.variable_id.unique()) == set(
+            [
+                'prsn',
+                'prra',
+                'tasmax',
+                'evspsblveg',
+                'landCoverFrac',
+                'mrso',
+                'co3',
+                'gpp',
+                'residualFrac',
+                'mrfso',
+            ]
+        )
 
 
 def test_search():
@@ -39,6 +55,8 @@ def test_search():
         assert isinstance(cat.ds, xr.Dataset)
         assert len(cat.ds.index) > 0
 
+        assert isinstance(cat.df, pd.DataFrame)
+
 
 @pytest.mark.parametrize(
     'chunks, expected_chunks',