Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
Merge branch 'CLIMATE-966'
  • Loading branch information
huikyole committed Aug 9, 2019
2 parents 8ba9330 + d943a72 commit 81c65012db25a0696da286a6b2daebbf9cb5fdbc
Showing 1 changed file with 166 additions and 0 deletions.
@@ -0,0 +1,166 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"import subprocess\n",
"import jinja2\n",
"import requests\n",
"from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor\n",
"from tqdm import tqdm_notebook as tqdm\n",
"from glob import glob\n",
"from IPython.display import Markdown, Image, FileLink"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"domain should be one of these three: 'AFR-44', 'EUR-11', 'NAM-44'\n",
"* AFR-44: CORDEX Africa RCMs at 44 km resolution\n",
"* EUR-11: CORDEX Europe RCMs at 11 km resolution\n",
"* NAM-44: CORDEX North America RCMs at 44 km resolution"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"domain = 'NAM-44'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# The output directory\n",
"cwd = os.getcwd()\n",
"workdir = cwd +'/evaluation_result'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Location of obs4Mips files\n",
"obs_dir = '/mnt/efs/obs4mips'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Location of CORDEX files\n",
"models_dir = '/mnt/efs/'+domain+'/*'.format(domain=domain)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Extract metadata from model and obs files, pairing up files with the same\n",
"# variables for separate evaluations\n",
"obs_extractor = obs4MIPSMetadataExtractor(obs_dir)\n",
"models_extractor = CORDEXMetadataExtractor(models_dir)\n",
"groups = obs_extractor.group(models_extractor, 'variable')\n",
"\n",
"# Configuration file template, to be rendered repeatedly for each evaluation\n",
"# run\n",
"env = jinja2.Environment(loader=jinja2.FileSystemLoader('./templates'),\n",
" trim_blocks=True, lstrip_blocks=True)\n",
"t = env.get_template('CORDEX.yaml.template')\n",
"\n",
"# Each group represents a single evaluation. Repeat the evaluation for\n",
"# three seasons: Summer, Winter, and Annual.\n",
"seasons = ['annual', 'winter', 'summer']\n",
"errored = []\n",
"for group in tqdm(groups, desc='variable loop'):\n",
" obs_info, models_info = group\n",
" instrument = obs_info['instrument']\n",
" variable = obs_info['variable']\n",
" for season in tqdm(seasons, desc='season loop'):\n",
" configfile_basename = '_'.join([domain, instrument, variable, season]) + '.yaml'\n",
" configfile_path = os.path.join(workdir, domain, instrument,\n",
" variable, season)\n",
" if not os.path.exists(configfile_path):\n",
" os.makedirs(configfile_path)\n",
" configfile_path = os.path.join(configfile_path, configfile_basename)\n",
" with open(configfile_path, 'w') as configfile:\n",
" configfile.write(t.render(obs_info=obs_info, models_info=models_info,\n",
" season=season, output_dir=workdir))\n",
"\n",
" # TODO: Do this in parallel. Will change this once this approach\n",
" # is well tested.\n",
" code = subprocess.call([sys.executable, '../run_RCMES.py', configfile_path])\n",
" if code:\n",
" errored.append(configfile_path)\n",
"print(\"All runs done. The following ended with an error: {}\".format(errored))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Check the evaluation result or download the processed obs4mips and model output."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"display(Markdown('Evaluation results'))\n",
"ip_address = (requests.get('http://169.254.169.254/latest/meta-data/public-ipv4').content).decode('utf-8')\n",
"for obs_info in obs_extractor.data:\n",
" inst_name = obs_info['instrument']\n",
" var_name = obs_info['variable']\n",
" display(Markdown('Instrument: '+inst_name+'& Variable: '+var_name))\n",
" for season in seasons:\n",
" savedir = os.path.join('evaluation_result', domain, inst_name, var_name, season)\n",
" png_files = glob(os.path.join(savedir, '*.png'))\n",
" for png_file in png_files: \n",
" display(Image(png_file))\n",
" nc_file = glob(os.path.join(savedir, '*.nc'))[0]\n",
" display(FileLink(nc_file))\n",
"os.chdir(cwd) \n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

0 comments on commit 81c6501

Please sign in to comment.