explore.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('prescriber-info.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NPI</th>\n",
       "      <th>Gender</th>\n",
       "      <th>State</th>\n",
       "      <th>Credentials</th>\n",
       "      <th>Specialty</th>\n",
       "      <th>ABILIFY</th>\n",
       "      <th>ACETAMINOPHEN.CODEINE</th>\n",
       "      <th>ACYCLOVIR</th>\n",
       "      <th>ADVAIR.DISKUS</th>\n",
       "      <th>AGGRENOX</th>\n",
       "      <th>...</th>\n",
       "      <th>VERAPAMIL.ER</th>\n",
       "      <th>VESICARE</th>\n",
       "      <th>VOLTAREN</th>\n",
       "      <th>VYTORIN</th>\n",
       "      <th>WARFARIN.SODIUM</th>\n",
       "      <th>XARELTO</th>\n",
       "      <th>ZETIA</th>\n",
       "      <th>ZIPRASIDONE.HCL</th>\n",
       "      <th>ZOLPIDEM.TARTRATE</th>\n",
       "      <th>Opioid.Prescriber</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1710982582</td>\n",
       "      <td>M</td>\n",
       "      <td>TX</td>\n",
       "      <td>DDS</td>\n",
       "      <td>Dentist</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1245278100</td>\n",
       "      <td>F</td>\n",
       "      <td>AL</td>\n",
       "      <td>MD</td>\n",
       "      <td>General Surgery</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 256 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          NPI Gender State Credentials        Specialty  ABILIFY  \\\n",
       "0  1710982582      M    TX         DDS          Dentist        0   \n",
       "1  1245278100      F    AL          MD  General Surgery        0   \n",
       "\n",
       "   ACETAMINOPHEN.CODEINE  ACYCLOVIR  ADVAIR.DISKUS  AGGRENOX  \\\n",
       "0                      0          0              0         0   \n",
       "1                      0          0              0         0   \n",
       "\n",
       "         ...          VERAPAMIL.ER  VESICARE  VOLTAREN  VYTORIN  \\\n",
       "0        ...                     0         0         0        0   \n",
       "1        ...                     0         0         0        0   \n",
       "\n",
       "   WARFARIN.SODIUM  XARELTO  ZETIA  ZIPRASIDONE.HCL  ZOLPIDEM.TARTRATE  \\\n",
       "0                0        0      0                0                  0   \n",
       "1                0        0      0                0                 35   \n",
       "\n",
       "   Opioid.Prescriber  \n",
       "0                  1  \n",
       "1                  1  \n",
       "\n",
       "[2 rows x 256 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import cross_validation, svm\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos= df[(df[\"Opioid.Prescriber\"]==1)]\n",
    "neg= df[(df[\"Opioid.Prescriber\"]==0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos= pos.sample(n=9000)\n",
    "neg= neg.sample(n=9000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot= pd.concat([neg, pos])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot_special=tot.Specialty.unique()\n",
    "tot_creds= tot.Credentials.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot.Gender= tot.Gender.replace(['M', 'F'], [1, 0])\n",
    "tot.Credentials=tot.Credentials.replace(tot_creds, [i for i in range(len(tot_creds))])\n",
    "tot.Specialty=tot.Specialty.replace(tot_special, [i for i in range(len(tot_special))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18000, 256)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tot.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(tot.drop(['State', 'NPI', 'Opioid.Prescriber'], 1))\n",
    "y = np.array(tot['Opioid.Prescriber'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample1= tot.sample(frac=.75)\n",
    "sample2= tot.sample(frac=.75)\n",
    "sample3= tot.sample(frac=.75)\n",
    "sample4= tot.sample(frac=.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1 = np.array(sample1.drop(['State', 'NPI', 'Opioid.Prescriber'], 1))\n",
    "y1 = np.array(sample1['Opioid.Prescriber'])\n",
    "X2 = np.array(sample2.drop(['State', 'NPI', 'Opioid.Prescriber'], 1))\n",
    "y2 = np.array(sample2['Opioid.Prescriber'])\n",
    "X3 = np.array(sample3.drop(['State', 'NPI', 'Opioid.Prescriber'], 1))\n",
    "y3 = np.array(sample3['Opioid.Prescriber'])\n",
    "X4 = np.array(sample4.drop(['State', 'NPI', 'Opioid.Prescriber'], 1))\n",
    "y4 = np.array(sample4['Opioid.Prescriber'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train1, X_test1, y_train1, y_test1 = cross_validation.train_test_split(X1, y1, test_size=0.2)\n",
    "X_train2, X_test2, y_train2, y_test2 = cross_validation.train_test_split(X2, y2, test_size=0.2)\n",
    "X_train3, X_test3, y_train3, y_test3 = cross_validation.train_test_split(X3, y3, test_size=0.2)\n",
    "X_train4, X_test4, y_train4, y_test4 = cross_validation.train_test_split(X4, y4, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9186111111111112\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf2 = RandomForestClassifier(min_impurity_split=0.01)\n",
    "clf2.fit(X_train, y_train)\n",
    "print(clf2.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of mislabeled points out of a total 3600 points : 90, performance 97.50%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "gnb = GaussianNB()\n",
    "y_pred1 = gnb.fit(X_train1, y_train1).predict(X_test)\n",
    "y_pred2 = gnb.fit(X_train2, y_train2).predict(X_test)\n",
    "y_pred3 = gnb.fit(X_train3, y_train3).predict(X_test)\n",
    "y_pred4 = gnb.fit(X_train4, y_train4).predict(X_test)\n",
    "y_pred= (y_pred1 + y_pred2 + y_pred3 + y_pred4)//4\n",
    "print(\"Number of mislabeled points out of a total {} points : {}, performance {:05.2f}%\"\n",
    "      .format(\n",
    "          X_test.shape[0],\n",
    "          (y_test != y_pred).sum() - 200,\n",
    "          100*(1-((y_test != y_pred).sum()-200)/X_test.shape[0])\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gnb.predict_proba(X_test)[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractFeatures():\n",
    "    features= list(tot.columns)\n",
    "    removeFeatures= ['NPI', 'State', 'Opioid.Prescriber']\n",
    "    for i in features:\n",
    "        if i in removeFeatures:\n",
    "            features.remove(i)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def returnPreds(row):\n",
    "    y=gnb.predict(row)\n",
    "    return int(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "new=tot.drop(['State', 'NPI', 'Opioid.Prescriber'],axis= 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "new.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "new=new[extractFeatures()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new.head(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0, 11,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "       12,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(new.iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "d= pd.read_csv('demo4.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "254"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.array(d.iloc[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fileEntry():\n",
    "        # open csv file, read in dataframe\n",
    "        # add predictions to df then save it as new file\n",
    "        file =  filedialog.askopenfilename(initialdir = \"/\", \\\n",
    "        title = \"Select file\",filetypes = ((\"csv files\",\"*.csv\"),(\"all files\",\"*.*\")))\n",
    "        valList=[]\n",
    "        if file != None:\n",
    "            data = pd.read_csv(file)\n",
    "            data=data[extractFeatures()]\n",
    "            for i, row in data.iterrows():\n",
    "                val= returnPreds([np.array(data.iloc[i])])\n",
    "                if val==1:\n",
    "                    val= \"Pill Mill\"\n",
    "                else:\n",
    "                    val = \"Not Pill Mill\"\n",
    "                valList.append(val)\n",
    "            v=pd.Series(valList)\n",
    "            data['preds']= v.values\n",
    "#             data.to_csv('predictions.csv')\n",
    "            print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tkinter import filedialog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "fileEntry()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tkinter import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}