gdal2netcdf.ipynb

{
 "metadata": {
  "name": "gdal2netcdf"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "'''\n",
      "Convert a bunch of GDAL readable grids to a NetCDF Time Series.\n",
      "Here we read a bunch of files that have names like:\n",
      "/usgs/data0/prism/1890-1899/us_tmin_1895.01\n",
      "/usgs/data0/prism/1890-1899/us_tmin_1895.02\n",
      "...\n",
      "/usgs/data0/prism/1890-1899/us_tmin_1895.12\n",
      "'''"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "import datetime as dt\n",
      "import os\n",
      "import gdal\n",
      "import netCDF4\n",
      "import re"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ds = gdal.Open('/usgs/data0/prism/1890-1899/us_tmin_1895.01')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "a = ds.ReadAsArray()\n",
      "nlat,nlon = np.shape(a)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 37
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "b = ds.GetGeoTransform() #bbox, interval\n",
      "lon = np.arange(nlon)*b[1]+b[0]\n",
      "lat = np.arange(nlat)*b[5]+b[3]\n",
      "\n",
      "basedate = dt.datetime(1858,11,17,0,0,0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 39
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cd /usgs/data2/notebook"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "/usgs/data2/notebook\n"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# create NetCDF file\n",
      "nco = netCDF4.Dataset('time_series.nc','w',clobber=True)\n",
      "\n",
      "# chunking is optional, but can improve access a lot: \n",
      "# (see: http://www.unidata.ucar.edu/blogs/developer/entry/chunking_data_choosing_shapes)\n",
      "chunk_lon=16\n",
      "chunk_lat=16\n",
      "chunk_time=12\n",
      "\n",
      "# create dimensions, variables and attributes:\n",
      "nco.createDimension('lon',nlon)\n",
      "nco.createDimension('lat',nlat)\n",
      "nco.createDimension('time',None)\n",
      "timeo = nco.createVariable('time','f4',('time'))\n",
      "timeo.units = 'days since 1858-11-17 00:00:00'\n",
      "timeo.standard_name = 'time'\n",
      "\n",
      "lono = nco.createVariable('lon','f4',('lon'))\n",
      "lono.units = 'degrees_east'\n",
      "lono.standard_name = 'longitude'\n",
      "\n",
      "lato = nco.createVariable('lat','f4',('lat'))\n",
      "lato.units = 'degrees_north'\n",
      "lato.standard_name = 'latitude'\n",
      "\n",
      "# create container variable for CRS: lon/lat WGS84 datum\n",
      "crso = nco.createVariable('crs','i4')\n",
      "csro.long_name = 'Lon/Lat Coords in WGS84'\n",
      "crso.grid_mapping_name='latitude_longitude'\n",
      "crso.longitude_of_prime_meridian = 0.0\n",
      "crso.semi_major_axis = 6378137.0\n",
      "crso.inverse_flattening = 298.257223563\n",
      "\n",
      "# create short integer variable for temperature data, with chunking\n",
      "tmno = nco.createVariable('tmn', 'i2',  ('time', 'lat', 'lon'), \n",
      "   zlib=True,chunksizes=[chunk_time,chunk_lat,chunk_lon],fill_value=-9999)\n",
      "tmno.units = 'degC'\n",
      "tmno.scale_factor = 0.01\n",
      "tmno.add_offset = 0.00\n",
      "tmno.long_name = 'minimum monthly temperature'\n",
      "tmno.standard_name = 'air_temperature'\n",
      "tmno.grid_mapping = 'crs'\n",
      "tmno.set_auto_maskandscale(False)\n",
      "\n",
      "nco.Conventions='CF-1.6'\n",
      "\n",
      "#write lon,lat\n",
      "lono[:]=lon\n",
      "lato[:]=lat\n",
      "\n",
      "pat = re.compile('us_tmin_[0-9]{4}\\.[0-9]{2}')\n",
      "itime=0\n",
      "\n",
      "#step through data, writing time and data to NetCDF\n",
      "for root, dirs, files in os.walk('/usgs/data0/prism/1890-1899/'):\n",
      "    dirs.sort()\n",
      "    files.sort()\n",
      "    for f in files:\n",
      "        if re.match(pat,f):\n",
      "            # read the time values by parsing the filename\n",
      "            year=int(f[8:12])\n",
      "            mon=int(f[13:15])\n",
      "            date=dt.datetime(year,mon,1,0,0,0)\n",
      "            print(date)\n",
      "            dtime=(date-basedate).total_seconds()/86400.\n",
      "            timeo[itime]=dtime\n",
      "           # min temp\n",
      "            tmn_path = os.path.join(root,f)\n",
      "            print(tmn_path)\n",
      "            tmn=gdal.Open(tmn_path)\n",
      "            a=tmn.ReadAsArray()  #data\n",
      "            tmno[itime,:,:]=a\n",
      "            itime=itime+1\n",
      "\n",
      "nco.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1895-01-01 00:00:00\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.01\n",
        "1895-02-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.02\n",
        "1895-03-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.03\n",
        "1895-04-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.04\n",
        "1895-05-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.05\n",
        "1895-06-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.06\n",
        "1895-07-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.07\n",
        "1895-08-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.08\n",
        "1895-09-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.09\n",
        "1895-10-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.10\n",
        "1895-11-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.11\n",
        "1895-12-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1895.12\n",
        "1896-01-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.01\n",
        "1896-02-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.02\n",
        "1896-03-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.03\n",
        "1896-04-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.04\n",
        "1896-05-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.05\n",
        "1896-06-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.06\n",
        "1896-07-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.07\n",
        "1896-08-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.08\n",
        "1896-09-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.09\n",
        "1896-10-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.10\n",
        "1896-11-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.11\n",
        "1896-12-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1896.12\n",
        "1897-01-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.01\n",
        "1897-02-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.02\n",
        "1897-03-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.03\n",
        "1897-04-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.04\n",
        "1897-05-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.05\n",
        "1897-06-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.06\n",
        "1897-07-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.07\n",
        "1897-08-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.08\n",
        "1897-09-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.09\n",
        "1897-10-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.10\n",
        "1897-11-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.11\n",
        "1897-12-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1897.12\n",
        "1898-01-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.01\n",
        "1898-02-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.02\n",
        "1898-03-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.03\n",
        "1898-04-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.04\n",
        "1898-05-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.05\n",
        "1898-06-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.06\n",
        "1898-07-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.07\n",
        "1898-08-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.08\n",
        "1898-09-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.09\n",
        "1898-10-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.10\n",
        "1898-11-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.11\n",
        "1898-12-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1898.12\n",
        "1899-01-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.01\n",
        "1899-02-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.02\n",
        "1899-03-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.03\n",
        "1899-04-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.04\n",
        "1899-05-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.05\n",
        "1899-06-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.06\n",
        "1899-07-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.07\n",
        "1899-08-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.08\n",
        "1899-09-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.09\n",
        "1899-10-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.10\n",
        "1899-11-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.11\n",
        "1899-12-01 00:00:00"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "/usgs/data0/prism/1890-1899/us_tmin_1899.12\n"
       ]
      }
     ],
     "prompt_number": 46
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}