Merge pull request #100 from brian-rose/datacache

Implement a download-and-cache strategy for data files

.gitignore

@@ -4,6 +4,11 @@
 *.so
 *.so.dSYM
 
+# Data files (no longer kept under version control)
+*.nc
+*.ASC
+*.dat
+
 *.DS_Store
 
 # Setuptools distribution and binary build folders.
@@ -22,4 +27,4 @@
 
 *.pyd
 *-f2pywrappers2.f90
-_*module.c
+_*module.c

MANIFEST.in

@@ -11,7 +11,6 @@ include .f2py_f2cmap
 include climlab/radiation/cam3/.f2py_f2cmap
 include climlab/radiation/rrtm/_rrtmg_lw/.f2py_f2cmap
 include climlab/radiation/rrtm/_rrtmg_sw/.f2py_f2cmap
-recursive-include climlab *.nc
 recursive-include climlab *.pyf
 recursive-include climlab *.sh
 recursive-include climlab *.f90

ci/requirements-py36-windows.yml

@@ -4,7 +4,7 @@ channels:
 dependencies:
   - python=3.6
   - pip
-  - netcdf4
+  - pydap
   - xarray
   - pytest
   - numpy

ci/requirements-py37-windows.yml

@@ -4,7 +4,7 @@ channels:
 dependencies:
   - python=3.7
   - pip
-  - netcdf4
+  - pydap
   - xarray
   - pytest
   - numpy

climlab/__init__.py

@@ -7,7 +7,7 @@
 has been documented for a comprehensive understanding and traceability.
 '''
 
-__version__ = '0.7.2.dev4'
+__version__ = '0.7.2.dev5'
 
 # this should ensure that we can still import constants.py as climlab.constants
 from .utils import constants, thermo, legendre

climlab/radiation/cam3/cam3.py

@@ -37,27 +37,33 @@
 from climlab import constants as const
 from climlab.utils.thermo import vmr_to_mmr
 from climlab.radiation.radiation import _Radiation_SW, _Radiation_LW
+from climlab.utils.data_source import load_data_source
 import os, warnings
 import xarray as xr
 
 
 def init_cam3(mod):
     # Initialise absorptivity / emissivity data
+    filename = 'abs_ems_factors_fastvx.c030508.nc'
     here = os.path.dirname(__file__)
     datadir = os.path.join(here, 'data')
-    AbsEmsDataFile = os.path.join(datadir, 'abs_ems_factors_fastvx.c030508.nc')
-    #  Open the absorption data file
-    data = xr.open_dataset(AbsEmsDataFile)
+    local_path = os.path.join(datadir, filename)
+    remotepath_http = 'http://thredds.atmos.albany.edu:8080/thredds/fileServer/CLIMLAB/absorptivity/' + filename
+    remotepath_opendap = 'http://thredds.atmos.albany.edu:8080/thredds/dodsC/CLIMLAB/absorptivity/' + filename
+    data, path = load_data_source(local_path=local_path,
+                            remote_source_list=[remotepath_http, remotepath_opendap],
+                            open_method=xr.open_dataset,
+                            remote_kwargs={'engine':'pydap'},
+                            verbose=False,)
     #  Populate storage arrays with values from netcdf file
     for field in ['ah2onw', 'eh2onw', 'ah2ow', 'ln_ah2ow', 'cn_ah2ow', 'ln_eh2ow', 'cn_eh2ow']:
         setattr(mod, field, data[field].transpose())
-    data.close()
 
 #  Wrapping these imports in try/except to avoid failures during documentation building on readthedocs
 try:
     from . import _cam3
     init_cam3(_cam3.absems)
-except:
+except Exception:
     warnings.warn('Cannot import and initialize compiled Fortran extension, CAM3 module will not be functional.')
 
 

climlab/radiation/cam3/data/README

@@ -0,0 +1,2 @@
+This is just a placeholder in the source directory
+where the CAM3 absorptivity data file will be cached locally.

climlab/radiation/cam3/setup.py

@@ -27,7 +27,7 @@ def configuration(parent_package='', top_path=None):
         build = False
 
     config = Configuration(package_name='cam3', parent_name=parent_package, top_path=top_path)
-    config.add_data_files(join('data', 'abs_ems_factors_fastvx.c030508.nc'))
+    config.add_data_files(join('data', 'README'))
     if build:
         config.add_extension(name='_cam3',
                          sources=[cam3_gen_source],

climlab/radiation/data/ozone/README

@@ -0,0 +1,2 @@
+This is just a placeholder in the source directory
+where the ozone data file (or files) will be cached locally.

climlab/radiation/radiation.py

@@ -76,15 +76,13 @@
 import numpy as np
 from climlab.process import EnergyBudget
 from climlab.radiation import ManabeWaterVapor
-import os
 from climlab import constants as const
 from climlab.domain.field import Field
+from climlab.utils.data_source import load_data_source
 import warnings
-try:
-    import xarray as xr
-except:
-    warnings.warn('Cannot import `xarray`. Will not be able to initialize ozone from data file.',
-                FutureWarning, stacklevel=2)
+import os
+import xarray as xr
+
 
 def default_specific_humidity(Tatm):
     '''Initialize a specific humidity distribution
@@ -99,8 +97,7 @@ def default_specific_humidity(Tatm):
 
 def default_absorbers(Tatm,
                 ozone_file = 'apeozone_cam3_5_54.nc',
-                verbose = True,
-                      ):
+                verbose = True,):
     '''Initialize a dictionary of well-mixed radiatively active gases
     All values are volumetric mixing ratios.
 
@@ -135,12 +132,14 @@ def default_absorbers(Tatm,
     xTatm = Tatm.to_xarray()
     O3 = 0. * xTatm
     if ozone_file is not None:
-        datadir = os.path.join(os.path.dirname(__file__), 'data', 'ozone')
-        ozonefilepath = os.path.join(datadir, ozone_file)
-        #  Open the ozone data file
-        if verbose:
-            print('Getting ozone data from', ozonefilepath)
-        ozonedata = xr.open_dataset(ozonefilepath)
+        ozonefilepath = os.path.join(os.path.dirname(__file__), 'data', 'ozone', ozone_file)
+        remotepath_http = 'http://thredds.atmos.albany.edu:8080/thredds/fileServer/CLIMLAB/ozone/' + ozone_file
+        remotepath_opendap = 'http://thredds.atmos.albany.edu:8080/thredds/dodsC/CLIMLAB/ozone/' + ozone_file
+        ozonedata, path = load_data_source(local_path=ozonefilepath,
+                                     remote_source_list=[remotepath_http, remotepath_opendap],
+                                     open_method=xr.open_dataset,
+                                     remote_kwargs={'engine':'pydap'},
+                                     verbose=verbose,)
         ##  zonal and time average
         ozone_zon = ozonedata.OZONE.mean(dim=('time','lon')).transpose('lat','lev')
         if ('lat' in xTatm.dims):

climlab/radiation/setup.py

@@ -8,8 +8,7 @@ def configuration(parent_package='',top_path=None):
     config = Configuration('radiation', parent_package, top_path)
     config.add_subpackage('rrtm')
     config.add_subpackage('cam3')
-    config.add_data_files(os.path.join('data','ozone','apeozone_cam3_5_54.nc'))
-    #config.make_config_py() # installs __config__.py
+    config.add_data_files(os.path.join('data','ozone','README'))
     return config
 
 if __name__ == '__main__':

climlab/solar/orbital/data/README

@@ -0,0 +1,2 @@
+This is just a placeholder in the source directory
+where the orbital data files will be cached locally.

climlab/solar/orbital/long.py

@@ -1,22 +1,34 @@
 from __future__ import division, print_function
 import numpy as np
+import os
 import pandas as pd
 import xarray as xr
+from climlab.utils.data_source import load_data_source
 
-def _get_Laskar_data():
-    base_url = 'http://vo.imcce.fr/insola/earth/online/earth/La2004/'
-    past_file = 'INSOLN.LA2004.BTL.ASC'
-    future_file = 'INSOLP.LA2004.BTL.ASC'
-    print('Accessing Laskar et al. (2004) orbital data from ' + base_url)
+
+base_url = 'http://vo.imcce.fr/insola/earth/online/earth/La2004/'
+filenames = {'past': 'INSOLN.LA2004.BTL.ASC',
+             'future': 'INSOLP.LA2004.BTL.ASC'}
+
+def _get_Laskar_data(verbose=True):
     longorbit = {}
+    sources = {}
+    pandas_kwargs = {'delim_whitespace':True,
+                     'header':None,
+                     'index_col':0,
+                     'names':['kyear','ecc','obliquity','long_peri'],}
+    for time in filenames:
+        local_path = os.path.join(os.path.dirname(__file__), "data", filenames[time])
+        remote_path = base_url + filenames[time]
+        if time is 'future':
+            pandas_kwargs['skiprows'] = 1 # first row is kyear=0, redundant
+        longorbit[time], path = load_data_source(local_path=local_path,
+                remote_source_list=[remote_path],
+                open_method = pd.read_csv,
+                open_method_kwargs=pandas_kwargs,
+                verbose=verbose)
+        sources[time] = path
     xlongorbit = {}
-    longorbit['past'] = pd.read_table(base_url + past_file,
-                            delim_whitespace=True, header=None, index_col=0,
-                             names=['kyear','ecc','obliquity','long_peri'])
-    longorbit['future'] = pd.read_table(base_url + future_file,
-                            delim_whitespace=True, header=None, index_col=0,
-                            skiprows=1, # first row is kyear=0, redundant
-                            names=['kyear','ecc','obliquity','long_peri'])
     for time in ['past', 'future']:
         # Cannot convert to float until we replace the D notation with E for floating point numbers
         longorbit[time].replace(to_replace='D', value='E', regex=True, inplace=True)
@@ -28,6 +40,10 @@ def _get_Laskar_data():
     # add 180 degrees to long_peri (see lambda definition, Berger 1978 Appendix)
     longorbit['long_peri'] += 180.
     longorbit['precession'] = longorbit.ecc*np.sin(np.deg2rad(longorbit.long_peri))
+    longorbit.attrs['Description'] = 'The Laskar et al. (2004) orbital data table'
+    longorbit.attrs['Citation'] = 'https://doi.org/10.1051/0004-6361:20041335'
+    longorbit.attrs['Source'] = [sources[time] for time in sources]
+    longorbit.attrs['Note'] = 'Longitude of perihelion is defined to be 0 degrees at Northern Vernal Equinox. This differs by 180 degrees from the source files.'
     return longorbit
 
 OrbitalTable = _get_Laskar_data()