Skip to content
This repository has been archived by the owner. It is now read-only.
Browse files
CLIMATE-316 Add ESGF Download Script to repository
  • Loading branch information
MichaelArthurAnderson committed Feb 25, 2018
1 parent 48a18fc commit 848cdb692774cba27fe435e2730b08118cb9daf3
Showing 1 changed file with 33 additions and 33 deletions.
@@ -16,9 +16,26 @@
# specific language governing permissions and limitations
# under the License.
A set of functions to wrap downloading ESGF datasets into an OCW dataset object.
*** Note *** The ESGF data source requires that the user have certain credentials downloaded from
the ESG. The current version of the module should download these automatically. Older versions of
the library will not download them. The solution is to use the WGET script from the EGS to download
a test dataset to get the credentials. The data source should work as expected then.
import os
import sys

import requests
from bs4 import BeautifulSoup

import ocw.data_source.local as local
from ocw.esgf.constants import DEFAULT_ESGF_SEARCH
from import download
from ocw.esgf.logon import logon

if sys.version_info[0] >= 3:
from urllib.error import HTTPError
@@ -27,15 +44,6 @@
# might be around one day
from urllib2 import HTTPError

from ocw.esgf.constants import DEFAULT_ESGF_SEARCH
from import download
from ocw.esgf.logon import logon
from import SearchClient
import ocw.data_source.local as local

from bs4 import BeautifulSoup
import requests

def load_dataset(dataset_id,
@@ -44,9 +52,8 @@ def load_dataset(dataset_id,
''' Load an ESGF dataset.
""" Load an ESGF dataset.
:param dataset_id: The ESGF ID of the dataset to load.
:type dataset_id: :mod:`string`
@@ -74,52 +81,45 @@ def load_dataset(dataset_id,
:param save_path: (Optional) Path to where downloaded files should be saved.
:type save_path: :mod:`string`
:param additional_constraints: (Optional) Additional key,value pairs to
pass as constraints to the search wrapper. These can be anything found
on the ESGF metadata page for a dataset.
:returns: A :class:`list` of :class:`dataset.Dataset` contained the
requested dataset. If the dataset is stored in multiple files each will
be loaded into a separate :class:`dataset.Dataset`.
:raises ValueError: If no dataset can be found for the supplied ID and
variable, or if the requested dataset is a multi-file dataset.
download_data = _get_file_download_data(url=search_url,
download_data = \
_get_file_download_data(url=search_url, dataset_id=dataset_id, variable=variable_name)

datasets = []

for url, var in download_data:
_download_files([url], esgf_username, esgf_password, download_directory=save_path)

file_save_path = os.path.join(save_path, url.split('/')[-1])

datasets.append(local.load_file(file_save_path, var, name=name,

origin = {
'source': 'esgf',
'dataset_id': dataset_id,
'variable': variable_name
for ds in datasets:
ds.origin = origin

for dataset in datasets:
dataset.origin = origin

return datasets

def _get_file_download_data(dataset_id, variable, url=DEFAULT_ESGF_SEARCH):
url += '?type=File&dataset_id={}&variable={}'
url = url.format(dataset_id, variable)

r = requests.get(url)
xml = BeautifulSoup(r.content, "html.parser")
raw_data = requests.get(url)
xml = BeautifulSoup(raw_data.content, "html.parser")

dont_have_results = not bool(xml.response.result['numfound'])

@@ -141,7 +141,7 @@ def _get_file_download_data(dataset_id, variable, url=DEFAULT_ESGF_SEARCH):

def _download_files(file_urls, username, password, download_directory='/tmp'):
logon(username, password)
except HTTPError:

0 comments on commit 848cdb6

Please sign in to comment.