Merge branch 'CLIMATE-966'

RCMES/CORDEX/CORDEX.ipynb

@@ -0,0 +1,166 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "import subprocess\n",
+    "import jinja2\n",
+    "import requests\n",
+    "from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor\n",
+    "from tqdm import tqdm_notebook as tqdm\n",
+    "from glob import glob\n",
+    "from IPython.display import Markdown, Image, FileLink"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "domain should be one of these three: 'AFR-44', 'EUR-11', 'NAM-44'\n",
+    "* AFR-44: CORDEX Africa RCMs at 44 km resolution\n",
+    "* EUR-11: CORDEX Europe RCMs at 11 km resolution\n",
+    "* NAM-44: CORDEX North America RCMs at 44 km resolution"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "domain = 'NAM-44'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# The output directory\n",
+    "cwd = os.getcwd()\n",
+    "workdir = cwd +'/evaluation_result'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Location of obs4Mips files\n",
+    "obs_dir = '/mnt/efs/obs4mips'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Location of CORDEX files\n",
+    "models_dir = '/mnt/efs/'+domain+'/*'.format(domain=domain)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Extract metadata from model and obs files, pairing up files with the same\n",
+    "# variables for separate evaluations\n",
+    "obs_extractor = obs4MIPSMetadataExtractor(obs_dir)\n",
+    "models_extractor = CORDEXMetadataExtractor(models_dir)\n",
+    "groups = obs_extractor.group(models_extractor, 'variable')\n",
+    "\n",
+    "# Configuration file template, to be rendered repeatedly for each evaluation\n",
+    "# run\n",
+    "env =  jinja2.Environment(loader=jinja2.FileSystemLoader('./templates'),\n",
+    "                          trim_blocks=True, lstrip_blocks=True)\n",
+    "t = env.get_template('CORDEX.yaml.template')\n",
+    "\n",
+    "# Each group represents a single evaluation. Repeat the evaluation for\n",
+    "# three seasons: Summer, Winter, and Annual.\n",
+    "seasons = ['annual', 'winter', 'summer']\n",
+    "errored = []\n",
+    "for group in tqdm(groups, desc='variable loop'):\n",
+    "    obs_info, models_info = group\n",
+    "    instrument = obs_info['instrument']\n",
+    "    variable = obs_info['variable']\n",
+    "    for season in tqdm(seasons, desc='season loop'):\n",
+    "        configfile_basename = '_'.join([domain, instrument, variable, season]) + '.yaml'\n",
+    "        configfile_path = os.path.join(workdir, domain, instrument,\n",
+    "                                       variable, season)\n",
+    "        if not os.path.exists(configfile_path):\n",
+    "            os.makedirs(configfile_path)\n",
+    "        configfile_path = os.path.join(configfile_path, configfile_basename)\n",
+    "        with open(configfile_path, 'w') as configfile:\n",
+    "            configfile.write(t.render(obs_info=obs_info, models_info=models_info,\n",
+    "                                      season=season, output_dir=workdir))\n",
+    "\n",
+    "        # TODO: Do this in parallel. Will change this once this approach\n",
+    "        # is well tested.\n",
+    "        code = subprocess.call([sys.executable, '../run_RCMES.py', configfile_path])\n",
+    "        if code:\n",
+    "            errored.append(configfile_path)\n",
+    "print(\"All runs done. The following ended with an error: {}\".format(errored))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Check the evaluation result or download the processed obs4mips and model output."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "display(Markdown('Evaluation results'))\n",
+    "ip_address = (requests.get('http://169.254.169.254/latest/meta-data/public-ipv4').content).decode('utf-8')\n",
+    "for obs_info in obs_extractor.data:\n",
+    "    inst_name = obs_info['instrument']\n",
+    "    var_name = obs_info['variable']\n",
+    "    display(Markdown('Instrument: '+inst_name+'& Variable: '+var_name))\n",
+    "    for season in seasons:\n",
+    "        savedir = os.path.join('evaluation_result', domain, inst_name, var_name, season)\n",
+    "        png_files = glob(os.path.join(savedir, '*.png'))\n",
+    "        for png_file in png_files:    \n",
+    "            display(Image(png_file))\n",
+    "        nc_file = glob(os.path.join(savedir, '*.nc'))[0]\n",
+    "        display(FileLink(nc_file))\n",
+    "os.chdir(cwd)              \n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}