01-1-exporting-to-elastic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clinical NLP & Analysis with Elasticsearch & Kibana\n",
    "In this notebook, I show how to leverage Elasticsearch APIs to perform scalable and rapid text analystics using a collection of online mental disorder, self-diagnosed users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import helpers.pickle_helpers as ph\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "bipolar = ph.load_from_pickle(\"data/users/bipolar_group_clean.p\")\n",
    "bpd = ph.load_from_pickle(\"data/users/bpd_group_clean.p\")\n",
    "regular = ph.load_from_pickle(\"data/users/regular_users.p\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "278"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bipolar.group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>polarity</th>\n",
       "      <th>emotion</th>\n",
       "      <th>emotion_2</th>\n",
       "      <th>ambiguous</th>\n",
       "      <th>dt</th>\n",
       "      <th>date</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2014-09-30 23:26:47</th>\n",
       "      <td>@DerekActual hehe Yeah it's definitely 1 that ...</td>\n",
       "      <td>1</td>\n",
       "      <td>joy</td>\n",
       "      <td>anger</td>\n",
       "      <td>True</td>\n",
       "      <td>1.533333</td>\n",
       "      <td>2014-09-30 23:26:47</td>\n",
       "      <td>1</td>\n",
       "      <td>bipolar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-09-30 23:28:19</th>\n",
       "      <td>@DestinyTheGame Omg plz bring it out for pc.</td>\n",
       "      <td>0</td>\n",
       "      <td>anticipation</td>\n",
       "      <td>joy</td>\n",
       "      <td>False</td>\n",
       "      <td>4.050000</td>\n",
       "      <td>2014-09-30 23:28:19</td>\n",
       "      <td>1</td>\n",
       "      <td>bipolar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-09-30 23:32:22</th>\n",
       "      <td>@Redtippertruck with great pleasure. Xxx</td>\n",
       "      <td>1</td>\n",
       "      <td>joy</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>1.316667</td>\n",
       "      <td>2014-09-30 23:32:22</td>\n",
       "      <td>1</td>\n",
       "      <td>bipolar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-09-30 23:33:41</th>\n",
       "      <td>@TherapyAfterCSA every day. Xxx</td>\n",
       "      <td>0</td>\n",
       "      <td>joy</td>\n",
       "      <td>trust</td>\n",
       "      <td>False</td>\n",
       "      <td>1.650000</td>\n",
       "      <td>2014-09-30 23:33:41</td>\n",
       "      <td>1</td>\n",
       "      <td>bipolar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-09-30 23:35:20</th>\n",
       "      <td>@Redtippertruck Hehe I signed it lol. Also ask...</td>\n",
       "      <td>1</td>\n",
       "      <td>sadness</td>\n",
       "      <td>joy</td>\n",
       "      <td>True</td>\n",
       "      <td>7.033333</td>\n",
       "      <td>2014-09-30 23:35:20</td>\n",
       "      <td>1</td>\n",
       "      <td>bipolar</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                  text  \\\n",
       "2014-09-30 23:26:47  @DerekActual hehe Yeah it's definitely 1 that ...   \n",
       "2014-09-30 23:28:19       @DestinyTheGame Omg plz bring it out for pc.   \n",
       "2014-09-30 23:32:22           @Redtippertruck with great pleasure. Xxx   \n",
       "2014-09-30 23:33:41                    @TherapyAfterCSA every day. Xxx   \n",
       "2014-09-30 23:35:20  @Redtippertruck Hehe I signed it lol. Also ask...   \n",
       "\n",
       "                     polarity       emotion emotion_2  ambiguous        dt  \\\n",
       "2014-09-30 23:26:47         1           joy     anger       True  1.533333   \n",
       "2014-09-30 23:28:19         0  anticipation       joy      False  4.050000   \n",
       "2014-09-30 23:32:22         1           joy         0      False  1.316667   \n",
       "2014-09-30 23:33:41         0           joy     trust      False  1.650000   \n",
       "2014-09-30 23:35:20         1       sadness       joy       True  7.033333   \n",
       "\n",
       "                                   date  user_id user_type  \n",
       "2014-09-30 23:26:47 2014-09-30 23:26:47        1   bipolar  \n",
       "2014-09-30 23:28:19 2014-09-30 23:28:19        1   bipolar  \n",
       "2014-09-30 23:32:22 2014-09-30 23:32:22        1   bipolar  \n",
       "2014-09-30 23:33:41 2014-09-30 23:33:41        1   bipolar  \n",
       "2014-09-30 23:35:20 2014-09-30 23:35:20        1   bipolar  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bipolar.group[1].head(n=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert and Index the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text': \"@DerekActual hehe Yeah it's definitely 1 that defies logic and explanation. Stranger things exist in heaven &amp; earth..\",\n",
       "  'polarity': 1,\n",
       "  'emotion': 'joy',\n",
       "  'emotion_2': 'anger',\n",
       "  'ambiguous': True,\n",
       "  'dt': 1.5333333333,\n",
       "  'date': '2014-09-30T23:26:47.000Z',\n",
       "  'user_id': 1,\n",
       "  'user_type': 'bipolar'},\n",
       " {'text': '@DestinyTheGame Omg plz bring it out for pc.',\n",
       "  'polarity': 0,\n",
       "  'emotion': 'anticipation',\n",
       "  'emotion_2': 'joy',\n",
       "  'ambiguous': False,\n",
       "  'dt': 4.05,\n",
       "  'date': '2014-09-30T23:28:19.000Z',\n",
       "  'user_id': 1,\n",
       "  'user_type': 'bipolar'}]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# example code of how to convert a user into json\n",
    "bipolar.group[1][\"date\"] = bipolar.group[1].index\n",
    "bipolar.group[1][\"user_id\"] = 1\n",
    "bipolar.group[1][\"user_type\"] = \"bipolar\"\n",
    "bipolar.group[1].to_json(orient=\"records\", date_format=\"iso\", path_or_buf=\"data/user_json/user.json\", index=True)\n",
    "converted = json.load(open(\"data/user_json/user.json\"))\n",
    "converted[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch import helpers\n",
    "\n",
    "# Elasticsearch configurations\n",
    "ELASTICSEARCH = dict(\n",
    "    hostname = \"127.0.0.1:9200\",\n",
    "    index = \"twitter\",\n",
    "    type = \"_doc\"\n",
    ")\n",
    "\n",
    "es =  Elasticsearch(ELASTICSEARCH['hostname'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to format data\n",
    "def convert_to_es_format(docs):\n",
    "    return [\n",
    "        {\n",
    "            \"_index\": ELASTICSEARCH['index'],\n",
    "            \"_type\": ELASTICSEARCH['type'],\n",
    "            \"_source\": doc\n",
    "        }\n",
    "        for doc in docs\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, [])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# index to elasticsearch\n",
    "helpers.bulk(es, convert_to_es_format(converted[0:2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Elasticsearch Bulk Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# insert the entire collection of users to Elasticsearch\n",
    "def insert_users(users, utype):\n",
    "    print(\"Indexing users...\")\n",
    "    total = len(users.group)\n",
    "    for i in range(total):\n",
    "        users.group[i][\"date\"] = users.group[i].index\n",
    "        users.group[i][\"user_id\"] = i\n",
    "        users.group[i][\"user_type\"] = utype\n",
    "        users.group[i].to_json(orient=\"records\", date_format=\"iso\", path_or_buf=\"data/user_json/user.json\", index=True)\n",
    "        converted = json.load(open(\"data/user_json/user.json\"))\n",
    "        helpers.bulk(es, convert_to_es_format(converted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 µs, sys: 1 µs, total: 5 µs\n",
      "Wall time: 9.06 µs\n",
      "Indexing users...\n",
      "Indexing users...\n",
      "Indexing users...\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "# insert the bipolar users\n",
    "insert_users(bipolar, \"bipolar\")\n",
    "# insert the normal users\n",
    "insert_users(regular, \"normal\")\n",
    "# insert the bpd users\n",
    "insert_users(bpd, \"bpd\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}