Odo Lightening Talk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Loading data with odo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>type_</th>\n",
       "      <th>user</th>\n",
       "      <th>repo</th>\n",
       "      <th>commits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2015-01-01 00:00:00+00:00</td>\n",
       "      <td>PushEvent</td>\n",
       "      <td>davidjhulse</td>\n",
       "      <td>davidjhulse/davesbingrewardsbot</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2015-01-01 00:00:00+00:00</td>\n",
       "      <td>PushEvent</td>\n",
       "      <td>jmoon018</td>\n",
       "      <td>jmoon018/rshell-unit-tester</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2015-01-01 00:00:01+00:00</td>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>christoferpeterson</td>\n",
       "      <td>christoferpeterson/Vadek</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2015-01-01 00:00:01+00:00</td>\n",
       "      <td>PushEvent</td>\n",
       "      <td>JakeWharton</td>\n",
       "      <td>square/okhttp</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015-01-01 00:00:03+00:00</td>\n",
       "      <td>PushEvent</td>\n",
       "      <td>git4ruby</td>\n",
       "      <td>git4ruby/movie_review1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  created_at        type_                user  \\\n",
       "0  2015-01-01 00:00:00+00:00    PushEvent         davidjhulse   \n",
       "1  2015-01-01 00:00:00+00:00    PushEvent            jmoon018   \n",
       "2  2015-01-01 00:00:01+00:00  CreateEvent  christoferpeterson   \n",
       "3  2015-01-01 00:00:01+00:00    PushEvent         JakeWharton   \n",
       "4  2015-01-01 00:00:03+00:00    PushEvent            git4ruby   \n",
       "\n",
       "                              repo  commits  \n",
       "0  davidjhulse/davesbingrewardsbot        1  \n",
       "1      jmoon018/rshell-unit-tester        1  \n",
       "2         christoferpeterson/Vadek      NaN  \n",
       "3                    square/okhttp        8  \n",
       "4           git4ruby/movie_review1        1  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv('data/github_archive_sample.csv')\n",
    "df.head()\n",
    "# df.to_sql(...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Table('github_archive', MetaData(bind=Engine(sqlite:///data/db.sqlite)), Column('created_at', DATETIME(), table=<github_archive>), Column('type_', TEXT(), table=<github_archive>), Column('user', TEXT(), table=<github_archive>), Column('repo', TEXT(), table=<github_archive>), Column('commits', FLOAT(), table=<github_archive>), schema=None)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from odo import odo\n",
    "odo('data/github_archive_sample.csv', 'sqlite:///data/db.sqlite::github_archive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<odo.backends.json.JSONLines at 0x11b2bed0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "odo('data/github_archive_sample.csv.gz', 'jsonlines://data/github_archive_sample.json.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "var * {\n",
      "  created_at: ?datetime,\n",
      "  type_: ?string,\n",
      "  user: ?string,\n",
      "  repo: ?string,\n",
      "  commits: ?float64\n",
      "  }\n"
     ]
    }
   ],
   "source": [
    "from odo import resource, discover\n",
    "csv = resource('data/github_archive_sample.csv.gz')\n",
    "print discover(csv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src='img/chinookschema.png'>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dshape(\"var * {ArtistId: int32, Name: ?string[120]}\")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "discover(resource('sqlite:///data/Chinook_Sqlite.sqlite::Artist'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<odo.backends.csv.CSV at 0x11afedb0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "odo('sqlite:///data/Chinook_Sqlite.sqlite::Artist', 'data/chinook_artist.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<img src='img/conversions.png'>"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}