notebooks/information_theory/review-conditional_entropy.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conditional Entropy\n",
    "\n",
    "### References\n",
    "\n",
    "- [DataScience - StackExchange: Conditional entropy calculation in python, H(Y|X)](https://datascience.stackexchange.com/questions/58565/conditional-entropy-calculation-in-python-hyx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Entropy\n",
    "def entropy(Y):\n",
    "    \"\"\"\n",
    "    Also known as Shanon Entropy\n",
    "    Reference: https://en.wikipedia.org/wiki/Entropy_(information_theory)\n",
    "    \"\"\"\n",
    "    unique, count = np.unique(Y, return_counts=True, axis=0)\n",
    "    prob = count/len(Y)\n",
    "    en = np.sum((-1)*prob*np.log2(prob))\n",
    "    return en\n",
    "\n",
    "\n",
    "#Joint Entropy\n",
    "def jEntropy(Y,X):\n",
    "    \"\"\"\n",
    "    H(Y;X)\n",
    "    Reference: https://en.wikipedia.org/wiki/Joint_entropy\n",
    "    \"\"\"\n",
    "    YX = np.c_[Y,X]\n",
    "    return entropy(YX)\n",
    "\n",
    "#Conditional Entropy\n",
    "def cEntropy(Y, X):\n",
    "    \"\"\"\n",
    "    conditional entropy = Joint Entropy - Entropy of X\n",
    "    H(Y|X) = H(Y;X) - H(X)\n",
    "    Reference: https://en.wikipedia.org/wiki/Conditional_entropy\n",
    "    \"\"\"\n",
    "    return jEntropy(Y, X) - entropy(X)\n",
    "\n",
    "\n",
    "#Information Gain\n",
    "def gain(Y, X):\n",
    "    \"\"\"\n",
    "    Information Gain, I(Y;X) = H(Y) - H(Y|X)\n",
    "    Reference: https://en.wikipedia.org/wiki/Information_gain_in_decision_trees#Formal_definition\n",
    "    \"\"\"\n",
    "    return entropy(Y) - cEntropy(Y,X)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### dataset 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## data creation\n",
    "attrNms = [\"x1\", \"x2\", \"x3\", \"x4\", \"x5\"]\n",
    "data = pd.DataFrame(\n",
    "    [\n",
    "        [1, 0, 1, 1, 1],\n",
    "        [1, 1, 0, 0, 1],\n",
    "        [0, 1, 1, 1, 1],\n",
    "        [1, 0, 1, 0, 1],\n",
    "        [1, 0, 0, 1, 1],\n",
    "        [0, 0, 1, 1, 1],\n",
    "        [1, 1, 1, 1, 1],\n",
    "        [1, 0, 0, 1, 1],\n",
    "        [0, 1, 0, 0, 1],\n",
    "        [0, 0, 0, 1, 1],\n",
    "    ],\n",
    "    columns=attrNms,\n",
    ")\n",
    "data[\"y\"] = [1, 1, 1, 0, 0, 1, 1, 1, 0, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9709505944546686, 0.20000000000000018, 0.7709505944546684)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get values\n",
    "y = data[\"y\"].values\n",
    "X = data[attrNms].values\n",
    "# estimation and display\n",
    "entropy(y), cEntropy(y,X), gain(y, X)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### dataset 2: Iris"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## load dataset\n",
    "from sklearn.datasets import load_iris\n",
    "# load dataset\n",
    "dataset = load_iris()\n",
    "dataset.keys()\n",
    "# dataset to df\n",
    "data = pd.DataFrame(dataset.data, columns = dataset.feature_names)\n",
    "data['class'] = dataset.target\n",
    "dclass = dict()\n",
    "#for i, ic in enumerate(dataset.target_names):\n",
    "#    dclass[i] = ic\n",
    "#data['class'] = data['class'].map(dclass)\n",
    "# columns\n",
    "col_y = \"class\"\n",
    "cols_x = list(filter(lambda x: x != col_y, data.columns.tolist()))\n",
    "## data pareparation\n",
    "num_discrete_values = 5\n",
    "for col in cols_x:\n",
    "    data[col] = pd.cut(data[col], bins=num_discrete_values, labels=np.arange(num_discrete_values), right=False)\n",
    "    data[col] = data[col].astype(int)\n",
    "# add random feature\n",
    "data[\"randint5\"] = np.random.randint(5, size=len(data))\n",
    "cols_x = list(filter(lambda x: x != col_y, data.columns.tolist()))\n",
    "## data collection\n",
    "X = data[cols_x].values\n",
    "y = data[col_y].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.584962500721156, 0.026666666666666394, 1.5582958340544897)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(y), cEntropy(y,X), gain(y, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 ['sepal length (cm)'] 0.9447200734403691\n",
      "2 ['sepal length (cm)', 'sepal width (cm)'] 0.5539641975174541\n",
      "3 ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)'] 0.2289531340851374\n",
      "4 ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)'] 0.11972104013355711\n",
      "5 ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)', 'randint5'] 0.026666666666666394\n"
     ]
    }
   ],
   "source": [
    "for i in np.arange(1,6,1):\n",
    "    print(i,cols_x[:i], cEntropy(y,X[:,:i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.17713406521696218"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cEntropy(y,data[['petal length (cm)', 'petal width (cm)']].values)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> NOTA: El aumento del numero de features aumenta la predicibilidad."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 ['sepal length (cm)'] 0.9447200734403691\n",
      "1 ['sepal width (cm)'] 1.1934869489363455\n",
      "2 ['petal length (cm)'] 0.31870954563264053\n",
      "3 ['petal width (cm)'] 0.26043148725915666\n",
      "4 ['randint5'] 1.5312594817831102\n"
     ]
    }
   ],
   "source": [
    "for i in np.arange(1,6,1):\n",
    "    print(i-1, [cols_x[i-1]], cEntropy(y,X[:,i-1]))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> NOTA: Es una buena manera para seleccionar variables. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5037561830274229,\n",
       " 0.05507329123352483,\n",
       " ['sepal length (cm)', 'sepal width (cm)'],\n",
       " 0.5539641975174541)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## entropy between from sub-samples (average) vs from the whole dataset\n",
    "ncols = 2\n",
    "cols_sel = cols_x[:ncols]\n",
    "list_results = list()\n",
    "for i in np.arange(50):\n",
    "    temp = data.sample(int(len(data)/2))\n",
    "    Xt = temp[cols_x].values\n",
    "    yt = temp[col_y].values    \n",
    "    list_results.append(cEntropy(yt,Xt[:, :ncols]))\n",
    "np.mean(list_results), np.std(list_results), cols_sel, cEntropy(y,X[:, :ncols])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> NOTA: Usando las mismas columnas, **la predibilidad no cambia demasiado entre todo el dataset y random samples**. Por tanto, pueden ser validados sub-samples de datasets para ver su repercusion en la predicbilidad. No obstante, un poco cambia debido al **efecto \"size\"** pero supongo que esto solo ocurrira ante datasets / sub-samples pequeños."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "analysis",
   "language": "python",
   "name": "analysis"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}