Pandas.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook includes some quick notes about using pandas for data analysis and manipulation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Creating, Reading and Writing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Creating DataFrames and Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "fruits = pd.DataFrame(data=[[30,21]], columns=['Apples', 'Bananas'])\n",
    "fruits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "fruit_sales = pd.DataFrame(data=[[35,21],[41, 34]], columns=['Apples', 'Bananas'],\n",
    "                           index=['2017 Sales','2018 Sales'])\n",
    "fruit_sales"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "quantities = ['4 cups', '1 cup', '2 large', '1 can']\n",
    "items = ['Flour', 'Milk', 'Eggs', 'Spam']\n",
    "recipe = pd.Series(quantities, index=items, name='Dinner')\n",
    "recipe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Reading CSVs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "reviews = pd.read_csv('data/wine_reviews/winemag-data-130k-v2.csv', index_col=0)\n",
    "reviews.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Writing CSVs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "animals = pd.DataFrame({'Cows': [12, 20], 'Goats': [22, 19]}, index=['Year 1', 'Year 2'])\n",
    "animals.to_csv('cows_and_goats.csv')\n",
    "animals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Indexing, Selecting and Assigning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "reviews = pd.read_csv('data/wine_reviews/winemag-data-130k-v2.csv', index_col=0)\n",
    "reviews.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Selecting a column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "desc = reviews[\"description\"]\n",
    "points = reviews.points"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Selecting Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "first_description = reviews.description.iloc[0]\n",
    "first_description"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Selecting a row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "first_row = reviews.iloc[0]\n",
    "first_row"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Selecting a range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "first_descriptions = reviews.description[:10]\n",
    "first_descriptions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Selecting indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "indices = [1, 2, 3, 5, 8]\n",
    "reviews.iloc[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "cols = ['country', 'province', 'region_1', 'region_2']\n",
    "indices = [0, 1, 10, 100]\n",
    "df = reviews.loc[indices, cols]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "cols = ['country', 'variety']\n",
    "df1 = reviews.loc[:99, cols] # loc is inclusive\n",
    "\n",
    "# alternative\n",
    "\n",
    "cols_idx = [0, 11]\n",
    "df2 = reviews.iloc[:100, cols_idx] # iloc is exclusive\n",
    "\n",
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "italian_wines = reviews[reviews.country == 'Italy']\n",
    "italian_wines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "top_oceania_wines = reviews.loc[\n",
    "    (reviews.country.isin(['Australia', 'New Zealand']))\n",
    "    & (reviews.points >= 95)\n",
    "]\n",
    "\n",
    "top_oceania_wines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Summary functions and maps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Aggregator functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "median_points = reviews.points.median()\n",
    "median_points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "centered_price = reviews.price - reviews.price.mean()\n",
    "centered_price"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Unique values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "countries = reviews.country.unique()\n",
    "countries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "reviews_per_country = reviews.country.value_counts()\n",
    "reviews_per_country"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Indices Operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "bargain_idx = (reviews.points / reviews.price).idxmax()\n",
    "bargain_wine = reviews.loc[bargain_idx, 'title']\n",
    "bargain_wine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Mapping operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "n_trop = reviews.description.map(lambda desc: \"tropical\" in desc).sum()\n",
    "n_fruity = reviews.description.map(lambda desc: \"fruity\" in desc).sum()\n",
    "descriptor_counts = pd.Series([n_trop, n_fruity], index=['tropical', 'fruity'])\n",
    "descriptor_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def stars(row):\n",
    "    if row.country == 'Canada':\n",
    "        return 3\n",
    "    elif row.points >= 95:\n",
    "        return 3\n",
    "    elif row.points >= 85:\n",
    "        return 2\n",
    "    else:\n",
    "        return 1\n",
    "\n",
    "star_ratings = reviews.apply(stars, axis='columns')\n",
    "star_ratings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Grouping and Sorting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Counting occurrences of group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "reviews_written = reviews.groupby('taster_twitter_handle').size()\n",
    "reviews_written"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Aggregation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "best_rating_per_price = reviews.groupby('price')['points'].max().sort_index()\n",
    "best_rating_per_price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "price_extremes = reviews.groupby('variety').price.agg([min, max])\n",
    "price_extremes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "reviewer_mean_ratings = reviews.groupby('taster_name').points.mean()\n",
    "reviewer_mean_ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "reviewer_mean_ratings.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Sorting values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "sorted_varieties = price_extremes.sort_values(by=['min', 'max'], ascending=False)\n",
    "sorted_varieties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "country_variety_counts = reviews.groupby(['country', 'variety']).size().sort_values(ascending=False)\n",
    "country_variety_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Data types and missing data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Type of a column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "reviews.points.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Type conversions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "reviews.points.astype(str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Counting missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "n_missing_prices = reviews.price.isnull().sum()\n",
    "n_missing_prices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Filling Na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "reviews_per_region = reviews.region_1.fillna('Unknown').value_counts().sort_values(ascending=False)\n",
    "reviews_per_region"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Renaming and combining"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Renaming columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "renamed = reviews.rename(columns=dict(region_1='region', region_2='locale'))\n",
    "renamed.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Renaming axis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "reindexed = reviews.rename_axis('wines', axis='rows')\n",
    "reindexed.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Concatenating datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "italy_wines = reviews[reviews.country == 'Italy']\n",
    "portugal_wines = reviews[reviews.country == 'Portugal']\n",
    "\n",
    "print(f\"Size of Italy dataset:{len(italy_wines)}\")\n",
    "print(f\"Size of Portugal dataset:{len(portugal_wines)}\")\n",
    "\n",
    "combined = pd.concat([italy_wines, portugal_wines])\n",
    "print(f\"Size of combined dataset:{len(combined)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}