From e4aabb1de9fc81eb37851a86efed58a7ac10875f Mon Sep 17 00:00:00 2001 From: Pingu Carsti Date: Wed, 7 Aug 2019 14:56:15 +0200 Subject: [PATCH] added demo notebooks --- docs/source/_static/README.rst | 4 + docs/source/examples.rst | 14 ++- notebooks/demo/subset-cmip5.ipynb | 180 +++++++++++++++++++++++++++++ notebooks/demo/subset-cmip6.ipynb | 185 ++++++++++++++++++++++++++++++ notebooks/environment.yml | 5 + pyesgf/search/context.py | 7 +- pyesgf/search/results.py | 4 +- 7 files changed, 392 insertions(+), 7 deletions(-) create mode 100644 docs/source/_static/README.rst create mode 100644 notebooks/demo/subset-cmip5.ipynb create mode 100644 notebooks/demo/subset-cmip6.ipynb diff --git a/docs/source/_static/README.rst b/docs/source/_static/README.rst new file mode 100644 index 0000000..4008b03 --- /dev/null +++ b/docs/source/_static/README.rst @@ -0,0 +1,4 @@ +Readme +====== + +placeholder for _static folder. diff --git a/docs/source/examples.rst b/docs/source/examples.rst index f5a455f..f5303d8 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -4,7 +4,7 @@ Examples ######## -You can try this notebook examples online using Binder, or view the notebooks on NBViewer. +You can try these notebook online using Binder, or view the notebooks on NBViewer. .. image:: https://mybinder.org/badge_logo.svg :target: https://mybinder.org/v2/gh/ESGF/esgf-pyclient.git/master?filepath=notebooks @@ -16,9 +16,21 @@ You can try this notebook examples online using Binder, or view the notebooks on :alt: NBViewer :height: 20 +Basic Usage +=========== + .. toctree:: :maxdepth: 1 notebooks/examples/search notebooks/examples/logon notebooks/examples/download + +Demo +==== + +.. toctree:: + :maxdepth: 1 + + notebooks/demo/subset-cmip5 + notebooks/demo/subset-cmip6 diff --git a/notebooks/demo/subset-cmip5.ipynb b/notebooks/demo/subset-cmip5.ipynb new file mode 100644 index 0000000..6af7164 --- /dev/null +++ b/notebooks/demo/subset-cmip5.ipynb @@ -0,0 +1,180 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Subset CMIP5 Datasets with xarray\n", + "\n", + "xarray: http://xarray.pydata.org/en/stable/index.html" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search CMIP5 Dataset with ESGF pyclient\n", + "\n", + "using: https://esgf-pyclient.readthedocs.io/en/latest/index.html" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pyesgf.search import SearchConnection\n", + "conn = SearchConnection('https://esgf-data.dkrz.de/esg-search', distrib=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ctx = conn.new_context(\n", + " project='CMIP5', \n", + " experiment='rcp45',\n", + " model='HadCM3',\n", + " ensemble='r1i1p1',\n", + " time_frequency='mon',\n", + " realm='atmos',\n", + " data_node='esgf-data1.ceda.ac.uk',\n", + " )\n", + "ctx.hit_count" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = ctx.search()[0]\n", + "result.dataset_id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "files = result.file_context().search()\n", + "for file in files:\n", + " if 'tasmax' in file.opendap_url:\n", + " tasmax_url = file.opendap_url\n", + " print(tasmax_url)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ESGF Logon" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pyesgf.logon import LogonManager\n", + "lm = LogonManager()\n", + "lm.logoff()\n", + "lm.is_logged_on()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lm.logon(hostname='esgf-data.dkrz.de', interactive=True, bootstrap=True)\n", + "lm.is_logged_on()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Subset single dataset with xarray\n", + "\n", + "Using OpenDAP: http://xarray.pydata.org/en/stable/io.html?highlight=opendap#opendap" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import xarray as xr\n", + "ds = xr.open_dataset(tasmax_url, chunks={'time': 120})\n", + "print(ds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "da = ds['tasmax']\n", + "da = da.isel(time=slice(0, 1))\n", + "da = da.sel(lat=slice(-50, 50), lon=slice(0, 50))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "da.plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download to NetCDF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "da.to_netcdf('tasmax.nc')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/demo/subset-cmip6.ipynb b/notebooks/demo/subset-cmip6.ipynb new file mode 100644 index 0000000..9855c70 --- /dev/null +++ b/notebooks/demo/subset-cmip6.ipynb @@ -0,0 +1,185 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Subset CMIP6 Datasets with xarray\n", + "\n", + "xarray: http://xarray.pydata.org/en/stable/index.html" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search CMIP6 Dataset with ESGF pyclient\n", + "\n", + "using: https://esgf-pyclient.readthedocs.io/en/latest/index.html" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pyesgf.search import SearchConnection\n", + "conn = SearchConnection('https://esgf-data.dkrz.de/esg-search', distrib=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ctx = conn.new_context(\n", + " project='CMIP6', \n", + " source_id='UKESM1-0-LL', \n", + " experiment_id='historical', \n", + " variable='tas', \n", + " frequency='mon', \n", + " variant_label='r1i1p1f2',\n", + " data_node='esgf-data3.ceda.ac.uk')\n", + "ctx.hit_count" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = ctx.search()[0]\n", + "result.dataset_id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "files = result.file_context().search()\n", + "for file in files:\n", + " print(file.opendap_url)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Subset single dataset with xarray\n", + "\n", + "Using OpenDAP: http://xarray.pydata.org/en/stable/io.html?highlight=opendap#opendap" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import xarray as xr\n", + "ds = xr.open_dataset(files[0].opendap_url, chunks={'time': 120})\n", + "print(ds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "da = ds['tas']\n", + "da = da.isel(time=slice(0, 1))\n", + "da = da.sel(lat=slice(-50, 50), lon=slice(0, 50))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "da.plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Subset over multiple datasets\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds_agg = xr.open_mfdataset([files[0].opendap_url, files[1].opendap_url], chunks={'time': 120}, combine='nested', concat_dim='time')\n", + "print(ds_agg)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "da = ds_agg['tas']\n", + "da = da.isel(time=slice(1200, 1201))\n", + "da = da.sel(lat=slice(-50, 50), lon=slice(0, 50))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "da.plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "da.to_netcdf('tas_africa_19500116.nc')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/environment.yml b/notebooks/environment.yml index e5243f4..423e112 100644 --- a/notebooks/environment.yml +++ b/notebooks/environment.yml @@ -4,3 +4,8 @@ channels: dependencies: - pip - jupyterlab + # demo + - xarray + - dask + - netcdf4 + - matplotlib diff --git a/pyesgf/search/context.py b/pyesgf/search/context.py index 23dd373..e11a606 100644 --- a/pyesgf/search/context.py +++ b/pyesgf/search/context.py @@ -114,7 +114,7 @@ def search(self, batch_size=DEFAULT_BATCH_SIZE, ignore_facet_check=False, :batch_size: The number of results to get per HTTP request. :param constraints: Further constraints for this query. Equivilent - to calling self.constrain(**constraints).search() + to calling ``self.constrain(**constraints).search()`` :return: A ResultSet for this query """ @@ -140,10 +140,9 @@ def get_download_script(self, **constraints): """ Download a script for downloading all files in the set of results. - :param constraints: Further constraints for this query. Equivilent - to calling self.constrain(**constraints).get_download_script() + :param constraints: Further constraints for this query. Equivilent + to calling ``self.constrain(**constraints).get_download_script()`` :return: A string containing the script - """ if constraints: sc = self.constrain(**constraints) diff --git a/pyesgf/search/results.py b/pyesgf/search/results.py index 474f236..7ad5970 100644 --- a/pyesgf/search/results.py +++ b/pyesgf/search/results.py @@ -92,7 +92,7 @@ class BaseResult(object): :ivar json: The oroginial json representation of the result. :ivar context: The SearchContext which generated this result. :property urls: a dictionary of the form - {service: [(url, mime_type), ...], ...} + ``{service: [(url, mime_type), ...], ...}`` :property opendap_url: The url of an OPeNDAP endpoint for this result if available :property las_url: The url of an LAS endpoint for this result if available @@ -101,7 +101,7 @@ class BaseResult(object): :property gridftp_url: The url for downloading the result by Globus if available :property index_node: The index node from where the metadata is stored. - Calls to *_context() will optimise queries to only address this node. + Calls to ``*_context()`` will optimise queries to only address this node. """ def __init__(self, json, context):