PythonNotebooks/In_Situ_data_download_by_platform_category.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<H2>MADATORY PYTHON LIBRARIES</H2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/protllan/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:2: DeprecationWarning: Python 2 suport will be removed in ftputil 4.0.0\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import ftputil\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> PLATFORM CATEGORY SELECTION CRITERIA</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you had a look to the notebook dedicated to the In Situ netCDFs selection criteria (here), selecting by platform category is available for files in monthly and history directories:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Selection criteria</th>\n",
       "      <th>index_history.txt</th>\n",
       "      <th>index_latest.txt</th>\n",
       "      <th>index_monthly.txt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>platform category</td>\n",
       "      <td>✔</td>\n",
       "      <td>-</td>\n",
       "      <td>✔</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Selection criteria index_history.txt index_latest.txt index_monthly.txt\n",
       "0  platform category                 ✔               -                  ✔"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = { 'Selection criteria': ['platform category'], 'index_history.txt': [' ✔'], 'index_monthly.txt': [' ✔'],'index_latest.txt': [' - ']}\n",
    "pd.DataFrame(data=data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<H2>SET OUTPUT DIRECTORY</H2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output_directory = os.getcwd() #default to current working directory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> SET YOUR CREDENTIALS</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "user = '' #type CMEMS user name\n",
    "password = '' #type CMEMS password"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> TARGET A PRODUCT, HOST AND INDEX FILE</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "product_name = 'INSITU_MED_NRT_OBSERVATIONS_013_035' #type aimed In Situ product \n",
    "host = 'nrt.cmems-du.eu' #type aimed host (nrt.cmems-du.eu or my.cmems-du)\n",
    "index_file = 'index_monthly.txt' #type aimed index file "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> SPECIFY PLATFORM CATEGORY </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aimed_platform_category = 'drifter'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> DOWNLOAD THE FILES MATCHING THE ABOVE PLATFORM CATEGORY</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/protllan/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:2: DeprecationWarning: `use_list_a_option` will default to `False` in ftputil 4.x.x\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "#connect to CMEMS FTP\n",
    "with ftputil.FTPHost(host, user, password) as ftp_host: \n",
    "    \n",
    "    #open the index file to read\n",
    "    with ftp_host.open(\"Core\"+'/'+product_name+'/'+index_file, \"r\") as indexfile:\n",
    "        \n",
    "        #read the index file as a comma-separate-value file\n",
    "        index = np.genfromtxt(indexfile, skip_header=6, unpack=False, delimiter=',', dtype=None, names=['catalog_id', 'file_name','geospatial_lat_min', 'geospatial_lat_max', 'geospatial_lon_min','geospatial_lon_max','time_coverage_start', 'time_coverage_end', 'provider', 'date_update', 'data_mode', 'parameters'])\n",
    "        \n",
    "        #loop over the lines/netCDFs and download the most sutable ones for you\n",
    "        for netCDF in index:\n",
    "            \n",
    "            #getting ftplink, filepath and filename\n",
    "            ftplink = netCDF['file_name'].decode('utf-8')\n",
    "            filepath = '/'.join(ftplink.split('/')[3:len(ftplink.split('/'))])\n",
    "            ncdf_file_name = ftplink[ftplink.rfind('/')+1:]\n",
    "            \n",
    "            #download netCDF if meeting selection criteria\n",
    "            platform_category = filepath.split('/')[3]\n",
    "            if aimed_platform_category in platform_category: \n",
    "                if ftp_host.path.isfile(filepath):\n",
    "                    cwd = os.getcwd()\n",
    "                    os.chdir(output_directory)\n",
    "                    ftp_host.download(filepath, ncdf_file_name)  # remote, local\n",
    "                    os.chdir(cwd)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [anaconda2]",
   "language": "python",
   "name": "Python [anaconda2]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}