notebooks/watson_engineering_insights.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Engineering Insights\n",
    "\n",
    "\n",
    "# 1. Setup\n",
    "\n",
    "To prepare your environment, you need to install some packages and enter credentials for the Watson services.\n",
    "\n",
    "# 1.1 Install the necessary packages\n",
    "\n",
    "You need the latest versions of these packages:<br>\n",
    "** Watson Developer Cloud:** a client library for Watson services.<br>\n",
    "** NLTK: **leading platform for building Python programs to work with human language data.<br>\n",
    "** stop_words: **List of common stop words.<br>\n",
    "** python-swiftclient:** is a python client for the Swift API.<br>\n",
    "** websocket-client: ** is a python client for the Websockets.<br>\n",
    "** pyorient: ** is a python client for the Orient DB.<br><br>\n",
    "\n",
    "** Install the Watson Developer Cloud package: **\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting watson-developer-cloud\n",
      "  Downloading https://files.pythonhosted.org/packages/57/fc/1a76bd8d60a6912db4c3382d288b10e033548abaa6a55a1292dedfe63e35/watson-developer-cloud-1.5.0.tar.gz (215kB)\n",
      "\u001b[K    100% |████████████████████████████████| 225kB 3.9MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement not upgraded as not directly required: requests<3.0,>=2.0 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from watson-developer-cloud)\n",
      "Requirement not upgraded as not directly required: python_dateutil>=2.5.3 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from watson-developer-cloud)\n",
      "Collecting autobahn>=0.10.9 (from watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/c9/7a/140264ec2c162bb22f91be76a11554f8ab0eda9bb2c775b6bc0dbbef0d4a/autobahn-18.6.1-py2.py3-none-any.whl (294kB)\n",
      "\u001b[K    100% |████████████████████████████████| 296kB 3.1MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting Twisted>=13.2.0 (from watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/90/50/4c315ce5d119f67189d1819629cae7908ca0b0a6c572980df5cc6942bc22/Twisted-18.7.0.tar.bz2 (3.1MB)\n",
      "\u001b[K    100% |████████████████████████████████| 3.1MB 316kB/s eta 0:00:01\n",
      "\u001b[?25hRequirement not upgraded as not directly required: pyOpenSSL>=16.2.0 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from watson-developer-cloud)\n",
      "Collecting service-identity>=17.0.0 (from watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/29/fa/995e364220979e577e7ca232440961db0bf996b6edaf586a7d1bd14d81f1/service_identity-17.0.0-py2.py3-none-any.whl\n",
      "Requirement not upgraded as not directly required: chardet<3.1.0,>=3.0.2 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from requests<3.0,>=2.0->watson-developer-cloud)\n",
      "Requirement not upgraded as not directly required: idna<2.7,>=2.5 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from requests<3.0,>=2.0->watson-developer-cloud)\n",
      "Requirement not upgraded as not directly required: urllib3<1.23,>=1.21.1 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from requests<3.0,>=2.0->watson-developer-cloud)\n",
      "Requirement not upgraded as not directly required: certifi>=2017.4.17 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from requests<3.0,>=2.0->watson-developer-cloud)\n",
      "Requirement not upgraded as not directly required: six>=1.5 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from python_dateutil>=2.5.3->watson-developer-cloud)\n",
      "Collecting txaio>=2.10.0 (from autobahn>=0.10.9->watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/dc/2e/c8a877b0a5c2798fa93ebcc1465a72a68c089e5f8b0a852ca335751dcc5a/txaio-2.10.0-py2.py3-none-any.whl\n",
      "Collecting zope.interface>=4.4.2 (from Twisted>=13.2.0->watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/ac/8a/657532df378c2cd2a1fe6b12be3b4097521570769d4852ec02c24bd3594e/zope.interface-4.5.0.tar.gz (151kB)\n",
      "\u001b[K    100% |████████████████████████████████| 153kB 6.1MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting constantly>=15.1 (from Twisted>=13.2.0->watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/b9/65/48c1909d0c0aeae6c10213340ce682db01b48ea900a7d9fce7a7910ff318/constantly-15.1.0-py2.py3-none-any.whl\n",
      "Collecting incremental>=16.10.1 (from Twisted>=13.2.0->watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/f5/1d/c98a587dc06e107115cf4a58b49de20b19222c83d75335a192052af4c4b7/incremental-17.5.0-py2.py3-none-any.whl\n",
      "Collecting Automat>=0.3.0 (from Twisted>=13.2.0->watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/a3/86/14c16bb98a5a3542ed8fed5d74fb064a902de3bdd98d6584b34553353c45/Automat-0.7.0-py2.py3-none-any.whl\n",
      "Collecting hyperlink>=17.1.1 (from Twisted>=13.2.0->watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/a7/b6/84d0c863ff81e8e7de87cff3bd8fd8f1054c227ce09af1b679a8b17a9274/hyperlink-18.0.0-py2.py3-none-any.whl\n",
      "Collecting PyHamcrest>=1.9.0 (from Twisted>=13.2.0->watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/9a/d5/d37fd731b7d0e91afcc84577edeccf4638b4f9b82f5ffe2f8b62e2ddc609/PyHamcrest-1.9.0-py2.py3-none-any.whl (52kB)\n",
      "\u001b[K    100% |████████████████████████████████| 61kB 10.0MB/s ta 0:00:01\n",
      "\u001b[?25hCollecting attrs>=17.4.0 (from Twisted>=13.2.0->watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/41/59/cedf87e91ed541be7957c501a92102f9cc6363c623a7666d69d51c78ac5b/attrs-18.1.0-py2.py3-none-any.whl\n",
      "Requirement not upgraded as not directly required: cryptography>=2.2.1 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from pyOpenSSL>=16.2.0->watson-developer-cloud)\n",
      "Collecting pyasn1 (from service-identity>=17.0.0->watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/a0/70/2c27740f08e477499ce19eefe05dbcae6f19fdc49e9e82ce4768be0643b9/pyasn1-0.4.3-py2.py3-none-any.whl (72kB)\n",
      "\u001b[K    100% |████████████████████████████████| 81kB 9.7MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pyasn1-modules (from service-identity>=17.0.0->watson-developer-cloud)\n",
      "  Downloading https://files.pythonhosted.org/packages/19/02/fa63f7ba30a0d7b925ca29d034510fc1ffde53264b71b4155022ddf3ab5d/pyasn1_modules-0.2.2-py2.py3-none-any.whl (62kB)\n",
      "\u001b[K    100% |████████████████████████████████| 71kB 10.0MB/s ta 0:00:01\n",
      "\u001b[?25hRequirement not upgraded as not directly required: setuptools in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from zope.interface>=4.4.2->Twisted>=13.2.0->watson-developer-cloud)\n",
      "Requirement not upgraded as not directly required: asn1crypto>=0.21.0 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from cryptography>=2.2.1->pyOpenSSL>=16.2.0->watson-developer-cloud)\n",
      "Requirement not upgraded as not directly required: cffi>=1.7 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from cryptography>=2.2.1->pyOpenSSL>=16.2.0->watson-developer-cloud)\n",
      "Requirement not upgraded as not directly required: pycparser in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from cffi>=1.7->cryptography>=2.2.1->pyOpenSSL>=16.2.0->watson-developer-cloud)\n",
      "Building wheels for collected packages: watson-developer-cloud, Twisted, zope.interface\n",
      "  Running setup.py bdist_wheel for watson-developer-cloud ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/dsxuser/.cache/pip/wheels/81/cd/de/4e0916f623c2d125502e493394fd333ed693960264d4b7e524\n",
      "  Running setup.py bdist_wheel for Twisted ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/dsxuser/.cache/pip/wheels/a9/85/24/fc82998fb686cb31e65a26c027a20120fd1219c9f1e925913a\n",
      "  Running setup.py bdist_wheel for zope.interface ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/dsxuser/.cache/pip/wheels/c6/b2/d2/be6785a207eaa58d76debc10c9d5c66196b40a88abb61d6af7\n",
      "Successfully built watson-developer-cloud Twisted zope.interface\n",
      "Installing collected packages: txaio, autobahn, zope.interface, constantly, incremental, attrs, Automat, hyperlink, PyHamcrest, Twisted, pyasn1, pyasn1-modules, service-identity, watson-developer-cloud\n",
      "Successfully installed Automat-0.7.0 PyHamcrest-1.9.0 Twisted-18.7.0 attrs-18.1.0 autobahn-18.6.1 constantly-15.1.0 hyperlink-18.0.0 incremental-17.5.0 pyasn1-0.4.3 pyasn1-modules-0.2.2 service-identity-17.0.0 txaio-2.10.0 watson-developer-cloud-1.5.0 zope.interface-4.5.0\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade watson-developer-cloud"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Install NLTK: **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting nltk\n",
      "  Downloading https://files.pythonhosted.org/packages/50/09/3b1755d528ad9156ee7243d52aa5cd2b809ef053a0f31b53d92853dd653a/nltk-3.3.0.zip (1.4MB)\n",
      "\u001b[K    100% |████████████████████████████████| 1.4MB 738kB/s eta 0:00:01\n",
      "\u001b[?25hRequirement not upgraded as not directly required: six in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from nltk)\n",
      "Building wheels for collected packages: nltk\n",
      "  Running setup.py bdist_wheel for nltk ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/dsxuser/.cache/pip/wheels/d1/ab/40/3bceea46922767e42986aef7606a600538ca80de6062dc266c\n",
      "Successfully built nltk\n",
      "Installing collected packages: nltk\n",
      "  Found existing installation: nltk 3.2.4\n",
      "    Uninstalling nltk-3.2.4:\n",
      "      Successfully uninstalled nltk-3.2.4\n",
      "Successfully installed nltk-3.3\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade nltk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Install IBM Object Storage Client **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement not upgraded as not directly required: ibm-cos-sdk in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages\r\n",
      "Requirement not upgraded as not directly required: ibm-cos-sdk-core==2.*,>=2.0.0 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from ibm-cos-sdk)\r\n",
      "Requirement not upgraded as not directly required: ibm-cos-sdk-s3transfer==2.*,>=2.0.0 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from ibm-cos-sdk)\r\n",
      "Requirement not upgraded as not directly required: jmespath<1.0.0,>=0.7.1 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from ibm-cos-sdk-core==2.*,>=2.0.0->ibm-cos-sdk)\r\n",
      "Requirement not upgraded as not directly required: python-dateutil<3.0.0,>=2.1 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from ibm-cos-sdk-core==2.*,>=2.0.0->ibm-cos-sdk)\r\n",
      "Requirement not upgraded as not directly required: docutils>=0.10 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from ibm-cos-sdk-core==2.*,>=2.0.0->ibm-cos-sdk)\r\n",
      "Requirement not upgraded as not directly required: six>=1.5 in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from python-dateutil<3.0.0,>=2.1->ibm-cos-sdk-core==2.*,>=2.0.0->ibm-cos-sdk)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install ibm-cos-sdk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Install stop_words **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting stop-words\n",
      "  Downloading https://files.pythonhosted.org/packages/60/7f/8dee73924e2cb06a223523dd661856e9b32b17202bf9c20c77de2c154d72/stop-words-2015.2.23.1.tar.gz\n",
      "Building wheels for collected packages: stop-words\n",
      "  Running setup.py bdist_wheel for stop-words ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/dsxuser/.cache/pip/wheels/d3/60/7e/bc83ee0496b334c47a572d498f029dc7be23a96a7166187885\n",
      "Successfully built stop-words\n",
      "Installing collected packages: stop-words\n",
      "Successfully installed stop-words-2015.2.23.1\n"
     ]
    }
   ],
   "source": [
    "!pip install stop-words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Install websocket client: **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting websocket-client\n",
      "  Downloading https://files.pythonhosted.org/packages/8a/a1/72ef9aa26cfe1a75cee09fc1957e4723add9de098c15719416a1ee89386b/websocket_client-0.48.0-py2.py3-none-any.whl (198kB)\n",
      "\u001b[K    100% |████████████████████████████████| 204kB 4.4MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement not upgraded as not directly required: six in /opt/conda/envs/DSX-Python35/lib/python3.5/site-packages (from websocket-client)\n",
      "Installing collected packages: websocket-client\n",
      "Successfully installed websocket-client-0.48.0\n"
     ]
    }
   ],
   "source": [
    "!pip install websocket-client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Install pyorient: **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pyorient\n",
      "  Downloading https://files.pythonhosted.org/packages/c4/13/cfab515139c58f86e7c42c64b1b740d28a13d60f8eb2a427effbfcba17b4/pyorient-1.5.5.tar.gz (68kB)\n",
      "\u001b[K    100% |████████████████████████████████| 71kB 6.7MB/s eta 0:00:01\n",
      "\u001b[?25hBuilding wheels for collected packages: pyorient\n",
      "  Running setup.py bdist_wheel for pyorient ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/dsxuser/.cache/pip/wheels/e4/e3/fa/7767bf3473c9a04d04cb655d1a5560b7183b1fb2df015cd861\n",
      "Successfully built pyorient\n",
      "Installing collected packages: pyorient\n",
      "Successfully installed pyorient-1.5.5\n"
     ]
    }
   ],
   "source": [
    "! pip install pyorient --user"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.2 Import packages and libraries \n",
    "\n",
    "Import the packages and libraries that you'll use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/dsxuser/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/dsxuser/nltk_data...\n",
      "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import watson_developer_cloud\n",
    "from watson_developer_cloud import NaturalLanguageUnderstandingV1\n",
    "from watson_developer_cloud.natural_language_understanding_v1 \\\n",
    "  import Features, EntitiesOptions, KeywordsOptions\n",
    "    \n",
    "import ibm_boto3\n",
    "from botocore.client import Config\n",
    "import re\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "from nltk.cluster.util import cosine_distance\n",
    "from stop_words import get_stop_words\n",
    "import numpy\n",
    "\n",
    "import websocket\n",
    "import _thread\n",
    "import time\n",
    "\n",
    "from io import BytesIO\n",
    "import pandas as pd\n",
    "import pyorient, json\n",
    "import sys\n",
    "import types"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Configuration\n",
    "\n",
    "Add configurable items of the notebook below\n",
    "## 2.1 Add your service credentials from Bluemix for the Watson services\n",
    "\n",
    "You must create a Watson Natural Language Understanding service on Bluemix. Create a service for Natural Language Understanding (NLU). Insert the username and password values for your NLU in the following cell. Do not change the values of the version fields.\n",
    "\n",
    "Run the cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @hidden_cell\n",
    "natural_language_understanding = NaturalLanguageUnderstandingV1(\n",
    "    version='',\n",
    "    username=\"\",\n",
    "    password=\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Add your service credentials for Object Storage\n",
    "\n",
    "You must create Object Storage service on Bluemix. To access data in a file in Object Storage, you need the Object Storage authentication credentials. Insert the Object Storage authentication credentials as credentials_1 in the following cell after removing the current contents in the cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @hidden_cell\n",
    "# The following code contains the credentials for a file in your IBM Cloud Object Storage.\n",
    "# You might want to remove those credentials before you share your notebook.\n",
    "credentials_1 = {\n",
    "    'IBM_API_KEY_ID': '',\n",
    "    'IAM_SERVICE_ID': '',\n",
    "    'ENDPOINT': '',\n",
    "    'IBM_AUTH_ENDPOINT': '',\n",
    "    'BUCKET': '',\n",
    "    'FILE': ''\n",
    "}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.  Watson Text Classification\n",
    "\n",
    "Write the classification related utility functions in a modularalized form.\n",
    "\n",
    "## 3.1 Watson NLU Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze_using_NLU(analysistext):\n",
    "    \"\"\" Call Watson Natural Language Understanding service to obtain analysis results.\n",
    "    \"\"\"\n",
    "    response = natural_language_understanding.analyze( \n",
    "        text=analysistext,\n",
    "        features=Features(entities=EntitiesOptions(), \n",
    "                          keywords=KeywordsOptions()))\n",
    "    return response"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Augumented Classification\n",
    "\n",
    "Custom classification utlity fucntions for augumenting the results of Watson NLU API call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_sentences(text):\n",
    "    \"\"\" Split text into sentences.\n",
    "    \"\"\"\n",
    "    sentence_delimiters = re.compile(u'[\\\\[\\\\]\\n.!?]')\n",
    "    sentences = sentence_delimiters.split(text)\n",
    "    return sentences\n",
    "\n",
    "def split_into_tokens(text):\n",
    "    \"\"\" Split text into tokens.\n",
    "    \"\"\"\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "    return tokens\n",
    "    \n",
    "def POS_tagging(text):\n",
    "    \"\"\" Generate Part of speech tagging of the text.\n",
    "    \"\"\"\n",
    "    POSofText = nltk.tag.pos_tag(text)\n",
    "    return POSofText\n",
    "\n",
    "def keyword_tagging(tag,tagtext,text):\n",
    "    \"\"\" Tag the text matching keywords.\n",
    "    \"\"\"\n",
    "    if (text.lower().find(tagtext.lower()) != -1):\n",
    "        return text[text.lower().find(tagtext.lower()):text.lower().find(tagtext.lower())+len(tagtext)]\n",
    "    else:\n",
    "        return 'UNKNOWN'\n",
    "    \n",
    "def regex_tagging(tag,regex,text):\n",
    "    \"\"\" Tag the text matching REGEX.\n",
    "    \"\"\"    \n",
    "    p = re.compile(regex, re.IGNORECASE)\n",
    "    matchtext = p.findall(text)\n",
    "    regex_list=[]    \n",
    "    if (len(matchtext)>0):\n",
    "        for regword in matchtext:\n",
    "            regex_list.append(regword)\n",
    "    return regex_list\n",
    "\n",
    "def chunk_tagging(tag,chunk,text):\n",
    "    \"\"\" Tag the text using chunking.\n",
    "    \"\"\"\n",
    "    parsed_cp = nltk.RegexpParser(chunk)\n",
    "    pos_cp = parsed_cp.parse(text)\n",
    "    chunk_list=[]\n",
    "    for root in pos_cp:\n",
    "        if isinstance(root, nltk.tree.Tree):               \n",
    "            if root.label() == tag:\n",
    "                chunk_word = ''\n",
    "                for child_root in root:\n",
    "                    chunk_word = chunk_word +' '+ child_root[0]\n",
    "                chunk_list.append(chunk_word)\n",
    "    return chunk_list\n",
    "    \n",
    "def augument_NLUResponse(responsejson,updateType,text,tag):\n",
    "    \"\"\" Update the NLU response JSON with augumented classifications.\n",
    "    \"\"\"\n",
    "    if(updateType == 'keyword'):\n",
    "        if not any(d.get('text', None) == text for d in responsejson['keywords']):\n",
    "            responsejson['keywords'].append({\"text\":text,\"relevance\":0.5})\n",
    "    else:\n",
    "        if not any(d.get('text', None) == text for d in responsejson['entities']):\n",
    "            responsejson['entities'].append({\"type\":tag,\"text\":text,\"relevance\":0.5,\"count\":1})        \n",
    "    \n",
    "\n",
    "def classify_text(text, config):\n",
    "    \"\"\" Perform augumented classification of the text.\n",
    "    \"\"\"\n",
    "    \n",
    "    response = analyze_using_NLU(text)\n",
    "    responsejson = response\n",
    "    \n",
    "    sentenceList = split_sentences(text)\n",
    "    \n",
    "    tokens = split_into_tokens(text)\n",
    "    \n",
    "    postags = POS_tagging(tokens)\n",
    "    \n",
    "    configjson = json.loads(config)\n",
    "    \n",
    "    for stages in configjson['configuration']['classification']['stages']:\n",
    "        # print('Stage - Performing ' + stages['name']+':')\n",
    "        for steps in stages['steps']:\n",
    "            # print('    Step - ' + steps['type']+':')\n",
    "            if (steps['type'] == 'keywords'):\n",
    "                for keyword in steps['keywords']:\n",
    "                    for word in sentenceList:\n",
    "                        wordtag = keyword_tagging(keyword['tag'],keyword['text'],word)\n",
    "                        if(wordtag != 'UNKNOWN'):\n",
    "                            # print('      '+keyword['tag']+':'+wordtag)\n",
    "                            augument_NLUResponse(responsejson,'entities',wordtag,keyword['tag'])\n",
    "            elif(steps['type'] == 'd_regex'):\n",
    "                for regex in steps['d_regex']:\n",
    "                    for word in sentenceList:\n",
    "                        regextags = regex_tagging(regex['tag'],regex['pattern'],word)\n",
    "                        if (len(regextags)>0):\n",
    "                            for words in regextags:\n",
    "                                # print('      '+regex['tag']+':'+words)\n",
    "                                augument_NLUResponse(responsejson,'entities',words,regex['tag'])\n",
    "            elif(steps['type'] == 'chunking'):\n",
    "                for chunk in steps['chunk']:\n",
    "                    chunktags = chunk_tagging(chunk['tag'],chunk['pattern'],postags)\n",
    "                    if (len(chunktags)>0):\n",
    "                        for words in chunktags:\n",
    "                            # print('      '+chunk['tag']+':'+words)\n",
    "                            augument_NLUResponse(responsejson,'entities',words,chunk['tag'])\n",
    "            else:\n",
    "                print('UNKNOWN STEP')\n",
    "    \n",
    "    return responsejson\n",
    "\n",
    "def replace_unicode_strings(response):\n",
    "    \"\"\" Convert dict with unicode strings to strings.\n",
    "    \"\"\"\n",
    "    if isinstance(response, dict):\n",
    "        return {replace_unicode_strings(key): replace_unicode_strings(value) for key, value in response.iteritems()}\n",
    "    elif isinstance(response, list):\n",
    "        return [replace_unicode_strings(element) for element in response]\n",
    "    elif isinstance(response, unicode):\n",
    "        return response.encode('utf-8')\n",
    "    else:\n",
    "        return response"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Correlate text content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopWords = get_stop_words('english')\n",
    "# List of words to be ignored for text similarity\n",
    "stopWords.extend([\"The\",\"This\",\"That\",\".\",\"!\",\"?\"])\n",
    "\n",
    "def compute_text_similarity(text1, text2, text1tags, text2tags):\n",
    "    \"\"\" Compute text similarity using cosine\n",
    "    \"\"\"\n",
    "    stemmer = nltk.stem.porter.PorterStemmer()\n",
    "    sentences_text1 = split_sentences(text1)\n",
    "    sentences_text2 = split_sentences(text2)\n",
    "    tokens_text1 = []\n",
    "    tokens_text2 = []\n",
    "    \n",
    "    for sentence in sentences_text1:\n",
    "        tokenstemp = split_into_tokens(sentence.lower())\n",
    "        tokens_text1.extend(tokenstemp)\n",
    "    \n",
    "    for sentence in sentences_text2:\n",
    "        tokenstemp = split_into_tokens(sentence.lower())\n",
    "        tokens_text2.extend(tokenstemp)\n",
    "    if (len(text1tags) > 0):  \n",
    "        tokens_text1.extend(text1tags)\n",
    "    if (len(text2tags) > 0):    \n",
    "        tokens_text2.extend(text2tags)\n",
    "    \n",
    "    tokens1Filtered = [stemmer.stem(x) for x in tokens_text1 if x not in stopWords]\n",
    "    \n",
    "    tokens2Filtered = [stemmer.stem(x) for x in tokens_text2 if x not in stopWords]\n",
    "    \n",
    "    #  remove duplicate tokens\n",
    "    tokens1Filtered = set(tokens1Filtered)\n",
    "    tokens2Filtered = set(tokens2Filtered)\n",
    "   \n",
    "    tokensList=[]\n",
    "\n",
    "    text1vector = []\n",
    "    text2vector = []\n",
    "    \n",
    "    if len(tokens1Filtered) < len(tokens2Filtered):\n",
    "        tokensList = tokens1Filtered\n",
    "    else:\n",
    "        tokensList = tokens2Filtered\n",
    "\n",
    "    for token in tokensList:\n",
    "        if token in tokens1Filtered:\n",
    "            text1vector.append(1)\n",
    "        else:\n",
    "            text1vector.append(0)\n",
    "        if token in tokens2Filtered:\n",
    "            text2vector.append(1)\n",
    "        else:\n",
    "            text2vector.append(0)  \n",
    "\n",
    "    cosine_similarity = 1-cosine_distance(text1vector,text2vector)\n",
    "    if numpy.isnan(cosine_similarity):\n",
    "        cosine_similarity = 0\n",
    "    \n",
    "    return cosine_similarity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Persistence and Storage\n",
    "## 5.1 Configure Object Storage Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "cos = ibm_boto3.client('s3',\n",
    "                    ibm_api_key_id=credentials_1['IBM_API_KEY_ID'],\n",
    "                    ibm_service_instance_id=credentials_1['IAM_SERVICE_ID'],\n",
    "                    ibm_auth_endpoint=credentials_1['IBM_AUTH_ENDPOINT'],\n",
    "                    config=Config(signature_version='oauth'),\n",
    "                    endpoint_url=credentials_1['ENDPOINT'])\n",
    "\n",
    "def get_file(filename):\n",
    "    '''Retrieve file from Cloud Object Storage'''\n",
    "    fileobject = cos.get_object(Bucket=credentials_1['BUCKET'], Key=filename)['Body']\n",
    "    return fileobject\n",
    "\n",
    "def load_string(fileobject):\n",
    "    '''Load the file contents into a Python string'''\n",
    "    text = fileobject.read()\n",
    "    return text.decode('utf-8')\n",
    "\n",
    "def load_df(fileobject,sheetname):\n",
    "    '''Load file contents into a Pandas dataframe'''\n",
    "    excelFile = pd.ExcelFile(fileobject)\n",
    "    df = excelFile.parse(sheetname)\n",
    "    return df\n",
    "\n",
    "def put_file(filename, filecontents):\n",
    "    '''Write file to Cloud Object Storage'''\n",
    "    resp = cos.put_object(Bucket=credentials_1['BUCKET'], Key=filename, Body=filecontents)\n",
    "    return resp\n",
    "\n",
    "def __iter__(self): return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.2 OrientDB client - functions to connect, store and retrieve data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Connect to OrientDB **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = pyorient.OrientDB(host=\"\", port=)\n",
    "user = \"\"\n",
    "passw = \"\"\n",
    "session_id = client.connect(user, passw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** OrientDB Core functions **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_database(dbname, username, password):\n",
    "    \"\"\" Create a database\n",
    "    \"\"\"\n",
    "    client.db_create( dbname, pyorient.DB_TYPE_GRAPH, pyorient.STORAGE_TYPE_MEMORY )\n",
    "    print(dbname  + \" created and opened successfully\")\n",
    "        \n",
    "def drop_database(dbname):\n",
    "    \"\"\" Drop a database\n",
    "    \"\"\"\n",
    "    if client.db_exists( dbname, pyorient.STORAGE_TYPE_MEMORY ):\n",
    "        client.db_drop(dbname)\n",
    "    \n",
    "def create_class(classname):\n",
    "    \"\"\" Create a class\n",
    "    \"\"\"\n",
    "    command = \"create class \"+classname + \" extends V\"\n",
    "    client.command(command)\n",
    "    \n",
    "def create_record(classname, entityname, attributes):\n",
    "    \"\"\" Create a record\n",
    "    \"\"\"\n",
    "    command = \"insert into \" + classname + \" set \" \n",
    "    attrstring = \"\"\n",
    "    for index,key in enumerate(attributes):\n",
    "        attrstring = attrstring + key + \" = '\"+ attributes[key] + \"'\"\n",
    "        if index != len(attributes) -1:\n",
    "            attrstring = attrstring +\",\"\n",
    "    command = command + attrstring\n",
    "    client.command(command)\n",
    "    \n",
    "def create_defect_testcase_edge(defectid, testcaseid, attributes):\n",
    "    \"\"\" Create an edge between a defect and a testcase\n",
    "    \"\"\"\n",
    "    command = \"create edge linkedtestcases from (select from Defect where ID = \" + \"'\" + defectid + \"') to (select from Testcase where ID = \" + \"'\" + testcaseid + \"')\" \n",
    "    if len(attributes) > 0:\n",
    "        command = command + \" set \"\n",
    "    attrstring = \"\"\n",
    "    for index,key in enumerate(attributes):\n",
    "        val = attributes[key]\n",
    "        if not isinstance(val, str):\n",
    "            val = str(val)\n",
    "        attrstring = attrstring + key + \" = '\"+ val + \"'\"\n",
    "        if index != len(attributes) -1:\n",
    "            attrstring = attrstring +\",\"\n",
    "    command = command + attrstring\n",
    "    client.command(command)    \n",
    "    \n",
    "def create_testcase_requirement_edge(testcaseid, reqid, attributes):\n",
    "    \"\"\" Create an edge between a testcase and a requirement\n",
    "    \"\"\"\n",
    "    command = \"create edge linkedrequirements from (select from Testcase where ID = \"+ \"'\" + testcaseid+\"') to (select from Requirement where ID = \"+\"'\"+reqid+\"')\" \n",
    "    if len(attributes) > 0:\n",
    "        command = command + \" set \"\n",
    "    attrstring = \"\"\n",
    "    for index,key in enumerate(attributes):\n",
    "        val = attributes[key]\n",
    "        if not isinstance(val, str):\n",
    "            val = str(val)\n",
    "        attrstring = attrstring + key + \" = '\"+ val + \"'\"\n",
    "        if index != len(attributes) -1:\n",
    "            attrstring = attrstring +\",\"\n",
    "    command = command + attrstring\n",
    "    client.command(command)  \n",
    "\n",
    "    \n",
    "def create_requirement_defect_edge(reqid, defectid, attributes):\n",
    "    \"\"\" Create an edge between a requirement and a defect\n",
    "    \"\"\"\n",
    "    command = \"create edge linkeddefects from (select from Requirement where ID = \"+ \"'\" + reqid+\"') to (select from Defect where ID = \"+\"'\"+defectid+\"')\" \n",
    "    \n",
    "    if len(attributes) > 0:\n",
    "         command = command + \" set \"\n",
    "    attrstring = \"\"\n",
    "    for index,key in enumerate(attributes):\n",
    "        val = attributes[key]\n",
    "        if not isinstance(val, str):\n",
    "            val = str(val)\n",
    "        attrstring = attrstring + key + \" = '\"+ val + \"'\"\n",
    "        if index != len(attributes) -1:\n",
    "            attrstring = attrstring +\",\"\n",
    "    command = command + attrstring\n",
    "    client.command(command) \n",
    "    \n",
    "def execute_query(query):\n",
    "    \"\"\" Execute a query\n",
    "    \"\"\"\n",
    "    return client.query(query)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** OrientDB Insights **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_related_testcases(defectid):\n",
    "    \"\"\" Get the related testcases for a defect\n",
    "    \"\"\"\n",
    "    testcasesQuery = \"select * from ( select expand( out('linkedtestcases')) from Defect where ID = '\" + defectid +\"' )\"\n",
    "    testcases = execute_query(testcasesQuery)\n",
    "    scoresQuery = \"select expand(out_linkedtestcases) from Defect where ID = '\"+defectid+\"'\"\n",
    "    scores = execute_query(scoresQuery)\n",
    "    testcaseList =[]\n",
    "    scoresList= []\n",
    "    for testcase in testcases:\n",
    "        testcaseList.append(testcase.ID)\n",
    "    for score in scores:\n",
    "        scoresList.append(score.score)\n",
    "    result = {}\n",
    "    length = len(testcaseList)\n",
    "    for i in range(0, length):\n",
    "        result[testcaseList[i]] = scoresList[i]\n",
    "    return result\n",
    "\n",
    "def get_related_requirements(testcaseid):\n",
    "    \"\"\" Get the related requirements for a testcase\n",
    "    \"\"\"\n",
    "    requirementsQuery = \"select * from ( select expand( out('linkedrequirements') ) from Testcase where ID = '\" + testcaseid +\"' )\"\n",
    "    requirements = execute_query(requirementsQuery)\n",
    "    print(requirements)\n",
    "    scoresQuery = \"select expand(out_linkedrequirements) from Testcase where ID = '\"+testcaseid+\"'\"\n",
    "    scores = execute_query(scoresQuery)\n",
    "    requirementsList =[]\n",
    "    scoresList= []\n",
    "    for requirement in requirements:\n",
    "        requirementsList.append(requirement.ID)\n",
    "    for score in scores:\n",
    "        scoresList.append(score.score)\n",
    "    result = {}\n",
    "    length = len(requirementsList)\n",
    "    print(requirementsList, scoresList)\n",
    "    for i in range(0, length):\n",
    "        result[requirementsList[i]] = scoresList[i]\n",
    "    return result\n",
    "\n",
    "def get_related_defects(reqid):\n",
    "    \"\"\" Get the related defects for a requirement\n",
    "    \"\"\"\n",
    "    defectsQuery = \"select * from ( select expand( out('linkeddefects')) from Requirement where ID = '\" + reqid +\"' )\"\n",
    "    defects = execute_query(defectsQuery)\n",
    "    scoresQuery = \"select expand(out_linkeddefects) from Requirement where ID = '\"+reqid+\"'\"\n",
    "    scores = execute_query(scoresQuery)\n",
    "    defectsList =[]\n",
    "    scoresList= []\n",
    "    for defect in defects:\n",
    "        defectsList.append(defect.ID)\n",
    "    for score in scores:\n",
    "        scoresList.append(score.score)\n",
    "    result = {}\n",
    "    length = len(defectsList)\n",
    "    for i in range(0, length):\n",
    "        result[defectsList[i]] = scoresList[i]\n",
    "    return result\n",
    "\n",
    "def build_format_defects_list(defectsResult):\n",
    "    \"\"\" Build and format the OrientDB query results for defects\n",
    "    \"\"\"\n",
    "    defects = []\n",
    "    for defect in defectsResult:\n",
    "        detail = {}\n",
    "        detail['ID'] = defect.ID\n",
    "        detail['Severity'] = defect.Severity\n",
    "        detail['Description'] = defect.Description\n",
    "        defects.append(detail)\n",
    "    return defects\n",
    "\n",
    "def build_format_testcases_list(testcasesResult):\n",
    "    \"\"\" Build and format the OrientDB query results for testcases\n",
    "    \"\"\"\n",
    "    testcases = []\n",
    "    for testcase in testcasesResult:\n",
    "        detail = {}\n",
    "        detail['ID'] = testcase.ID\n",
    "        detail['Category'] = testcase.Category\n",
    "        detail['Description'] = testcase.Description\n",
    "        testcases.append(detail)\n",
    "    return testcases  \n",
    "\n",
    "def build_format_requirements_list(requirementsResult):\n",
    "    \"\"\" Build and format the OrientDB query results for requirements\n",
    "    \"\"\"\n",
    "    requirements = []\n",
    "    for requirement in requirementsResult:\n",
    "        detail = {}\n",
    "        detail['ID'] =requirement.ID\n",
    "        detail['Description'] = requirement.Description\n",
    "        detail['Priority'] = requirement.Priority\n",
    "        requirements.append(detail)\n",
    "    return requirements  \n",
    "\n",
    "def get_defects():\n",
    "    \"\"\" Get all defects\n",
    "    \"\"\"\n",
    "    defectsQuery = \"select * from Defect\"\n",
    "    defectsResult = execute_query(defectsQuery)\n",
    "    defects = build_format_defects_list(defectsResult)\n",
    "    return defects\n",
    "\n",
    "def get_testcases():\n",
    "    \"\"\" Get all testcases\n",
    "    \"\"\"\n",
    "    testcasesQuery = \"select * from Testcase\"\n",
    "    testcasesResult = execute_query(testcasesQuery)\n",
    "    testcases = build_format_testcases_list(testcasesResult)\n",
    "    return testcases\n",
    "\n",
    "def get_requirements():\n",
    "    \"\"\" Get all requirements\n",
    "    \"\"\"\n",
    "    requirementsQuery = \"select * from Requirement\"\n",
    "    requirementsResult =  execute_query(requirementsQuery)\n",
    "    requirements = build_format_requirements_list(requirementsResult)\n",
    "    return requirements  \n",
    "\n",
    "def get_defects_severity(severity):\n",
    "    \"\"\" Get defects of a given severity\n",
    "    \"\"\"\n",
    "    query = \"select * from Defect where Severity = \" + str(severity)\n",
    "    queryResult =  execute_query(query)\n",
    "    defects = build_format_defects_list(queryResult)    \n",
    "    return defects\n",
    "\n",
    "def get_testcases_category(category):\n",
    "    \"\"\" Get testcases of a given category\n",
    "    \"\"\"\n",
    "    testcasesQuery = \"select * from Testcase where Category = '\"+str(category)+\"'\"\n",
    "    testcasesResult = execute_query(testcasesQuery)\n",
    "    testcases = build_format_testcases_list(testcasesResult)\n",
    "    return testcases\n",
    "\n",
    "def get_testcases_zero_defects():\n",
    "    \"\"\" Get testcases that did not generate any defects\n",
    "    \"\"\"\n",
    "    testcasesQuery = \"Select * from Testcase where in('linkedtestcases').size() = 0\"\n",
    "    testcasesResult = execute_query(testcasesQuery)\n",
    "    testcases = build_format_testcases_list(testcasesResult)\n",
    "    print(testcases)\n",
    "    return testcases\n",
    "\n",
    "def get_defects_zero_testcases():\n",
    "    \"\"\" Get defects that have no associated testcases\n",
    "    \"\"\"\n",
    "    query = \"Select * from Defect where out('linkedtestcases').size() = 0\"\n",
    "    queryResult =  execute_query(query)\n",
    "    defects = build_format_defects_list(queryResult)   \n",
    "    print(defects)\n",
    "    return defects\n",
    "\n",
    "def get_requirements_zero_defect():\n",
    "    \"\"\" Get requirements that have no defects\n",
    "    \"\"\"\n",
    "    query = \"Select * from Requirement where out('linkeddefects').size() = 0\"\n",
    "    requirementsResult =  execute_query(query)\n",
    "    requirements = build_format_requirements_list(requirementsResult)\n",
    "    return requirements  \n",
    "\n",
    "def get_requirements_zero_testcases():\n",
    "    \"\"\" Get requirements that have no associated testcases\n",
    "    \"\"\"\n",
    "    query = \"Select * from Requirement where in('linkedrequirements').size() = 0\"\n",
    "    requirementsResult =  execute_query(query)\n",
    "    requirements = build_format_requirements_list(requirementsResult)\n",
    "    return requirements  \n",
    "\n",
    "def get_requirement_defects(numdefects):\n",
    "    \"\"\" Get requirements that have more than a given number of defects\n",
    "    \"\"\"\n",
    "    query = \"select ID,Description,Priority from Requirement where out('linkeddefects').size() >= \" + str(numdefects)\n",
    "    requirementsResult =  execute_query(query)\n",
    "    requirements = build_format_requirements_list(requirementsResult)\n",
    "    for requirement in requirements:\n",
    "        num = len(get_related_defects(requirement['ID']))\n",
    "        requirement['defectcount'] = num\n",
    "    return requirements  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.1 Global variables and functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Name of the excel file with data in Object Storage\n",
    "dataFileName = \"sample_data.xlsx\"\n",
    "\n",
    "# Name of the config file in Object Storage\n",
    "configFileName = \"sample_config.txt\"\n",
    "\n",
    "# Config contents\n",
    "config = None;\n",
    "\n",
    "# Data file\n",
    "datafile = None\n",
    "\n",
    "# Requirements dataframe\n",
    "requirements_sheet_name = \"Requirements\"\n",
    "requirements_df = None\n",
    "\n",
    "# Defects dataframe\n",
    "defects_sheet_name = \"Defects\"\n",
    "defects_df = None\n",
    "\n",
    "# Testcases dataframe\n",
    "testcases_sheet_name =\"TestCases\"\n",
    "testcases_df = None\n",
    "\n",
    "def load_artifacts():\n",
    "    \"\"\" Load the artifacts into a pandas dataframe\n",
    "    \"\"\"\n",
    "    global requirements_df \n",
    "    global defects_df \n",
    "    global testcases_df \n",
    "    global config\n",
    "    global datafile\n",
    "    config = load_string(get_file(configFileName))\n",
    "    datafile = get_file(dataFileName)\n",
    "    # add missing __iter__ method, so pandas accepts body as file-like object\n",
    "    if not hasattr(datafile, \"__iter__\"): datafile.__iter__ = types.MethodType( __iter__, datafile )\n",
    "    excel_file = pd.ExcelFile(datafile)\n",
    "    requirements_df = excel_file.parse(requirements_sheet_name)\n",
    "    defects_df = excel_file.parse(defects_sheet_name)\n",
    "    testcases_df = excel_file.parse(testcases_sheet_name)\n",
    "    \n",
    "def prepare_artifact_dataframes():\n",
    "    \"\"\" Prepare artifact dataframes by creating necessary output columns\n",
    "    \"\"\"\n",
    "    global requirements_df \n",
    "    global defects_df \n",
    "    global testcases_df \n",
    "    req_cols_len = len(requirements_df.columns)\n",
    "    def_cols_len = len(defects_df.columns)\n",
    "    tcs_cols_len = len(testcases_df.columns)\n",
    "    requirements_df.insert(req_cols_len, \"ClassifiedText\",\"\")\n",
    "    requirements_df.insert(req_cols_len+1, \"Keywords\",\"\")\n",
    "    requirements_df.insert(req_cols_len+2, \"DefectsMatchScore\",\"\")\n",
    "\n",
    "    defects_df.insert(def_cols_len, \"ClassifiedText\",\"\")\n",
    "    defects_df.insert(def_cols_len+1, \"Keywords\",\"\")\n",
    "    defects_df.insert(def_cols_len+2, \"TestCasesMatchScore\",\"\")\n",
    "\n",
    "    testcases_df.insert(tcs_cols_len, \"ClassifiedText\",\"\")\n",
    "    testcases_df.insert(tcs_cols_len+1, \"Keywords\",\"\")\n",
    "    testcases_df.insert(tcs_cols_len+2, \"RequirementsMatchScore\",\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.2 Utility functions for Engineering Insights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def add_text_classifier_output(artifact_df, config, output_column_name):\n",
    "    \"\"\" Add Watson text classifier output to the artifact dataframe\n",
    "    \"\"\"\n",
    "    for index, row in artifact_df.iterrows():\n",
    "        summary = row[\"Description\"]\n",
    "        classifier_journey_output = classify_text(summary, config)\n",
    "        artifact_df.set_value(index, output_column_name, classifier_journey_output)\n",
    "    return artifact_df \n",
    "           \n",
    "def add_keywords_entities(artifact_df, classify_text_column_name, output_column_name):\n",
    "    \"\"\" Add keywords and entities to the artifact dataframe\"\"\"\n",
    "    for index, artifact in artifact_df.iterrows():\n",
    "        keywords_array = []\n",
    "        for row in artifact[classify_text_column_name]['keywords']:\n",
    "            if not row['text'] in keywords_array:\n",
    "                keywords_array.append(row['text'])\n",
    "                \n",
    "        for entities in artifact[classify_text_column_name]['entities']:\n",
    "            if not entities['text'] in keywords_array:\n",
    "                keywords_array.append(entities['text'])\n",
    "            if not entities['type'] in keywords_array:\n",
    "                keywords_array.append(entities['type'])\n",
    "        artifact_df.set_value(index, output_column_name, keywords_array)\n",
    "    return artifact_df \n",
    "\n",
    "def populate_text_similarity_score(artifact_df1, artifact_df2, keywords_column_name, output_column_name):\n",
    "    \"\"\" Populate text similarity score to the artifact dataframes\n",
    "    \"\"\"\n",
    "    for index1, artifact1 in artifact_df1.iterrows():\n",
    "        matches = []\n",
    "        top_matches = []\n",
    "        for index2, artifact2 in artifact_df2.iterrows():\n",
    "            matches.append({'ID': artifact2['ID'], \n",
    "                            'cosine_score': 0, \n",
    "                            'SubjectID':artifact1['ID']})\n",
    "            cosine_score = compute_text_similarity(\n",
    "                artifact1['Description'], \n",
    "                artifact2['Description'], \n",
    "                artifact1['Keywords'], \n",
    "                artifact2['Keywords'])\n",
    "            matches[index2][\"cosine_score\"] = cosine_score\n",
    "       \n",
    "        sorted_obj = sorted(matches, key=lambda x : x['cosine_score'], reverse=True)\n",
    "      \n",
    "        for obj in sorted_obj:\n",
    "            if obj['cosine_score'] > 0.4:\n",
    "                top_matches.append(obj)\n",
    "               \n",
    "        artifact_df1.set_value(index1, output_column_name, top_matches)\n",
    "    return artifact_df1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.3 Process flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Prepare data **\n",
    "* Load artifacts from object storage and create pandas dataframes\n",
    "* Prepare the pandas dataframes. Add additional columns required for further processing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "load_artifacts()\n",
    "prepare_artifact_dataframes()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Run Watson Text Classifier on data **\n",
    "* Add the text classification output to the artifact dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/ipykernel/__main__.py:7: FutureWarning: set_value is deprecated and will be removed in a future release. Please use .at[] or .iat[] accessors instead\n"
     ]
    }
   ],
   "source": [
    "output_column_name = \"ClassifiedText\"\n",
    "defects_df = add_text_classifier_output(defects_df,config, output_column_name)\n",
    "testcases_df = add_text_classifier_output(testcases_df,config, output_column_name)\n",
    "requirements_df = add_text_classifier_output(requirements_df,config, output_column_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Populate keywords and entities **\n",
    "* Add the keywords and entities extracted from the unstructured text to the artifact dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/ipykernel/__main__.py:23: FutureWarning: set_value is deprecated and will be removed in a future release. Please use .at[] or .iat[] accessors instead\n"
     ]
    }
   ],
   "source": [
    "classify_text_column_name = \"ClassifiedText\"\n",
    "output_column_name = \"Keywords\"\n",
    "defects_df = add_keywords_entities(defects_df, classify_text_column_name, output_column_name)\n",
    "testcases_df = add_keywords_entities(testcases_df, classify_text_column_name, output_column_name)\n",
    "requirements_df = add_keywords_entities(requirements_df, classify_text_column_name, output_column_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Correlate keywords between artifacts **\n",
    "* Add the text similarity score of associated artifacts to the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/nltk/cluster/util.py:133: RuntimeWarning: invalid value encountered in true_divide\n",
      "  sqrt(numpy.dot(u, u)) * sqrt(numpy.dot(v, v))))\n",
      "/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/ipykernel/__main__.py:49: FutureWarning: set_value is deprecated and will be removed in a future release. Please use .at[] or .iat[] accessors instead\n"
     ]
    }
   ],
   "source": [
    "keywords_column_name = \"Keywords\"\n",
    "output_column_name = \"TestCasesMatchScore\"\n",
    "defects_df = populate_text_similarity_score(defects_df, testcases_df, keywords_column_name, output_column_name)\n",
    "\n",
    "output_column_name = \"RequirementsMatchScore\"\n",
    "testcases_df = populate_text_similarity_score(testcases_df, requirements_df, keywords_column_name, output_column_name)\n",
    "\n",
    "output_column_name = \"DefectsMatchScore\"\n",
    "requirements_df = populate_text_similarity_score(requirements_df, defects_df, keywords_column_name, output_column_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Utility functions to store entities and relations in Orient DB **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def store_requirements(requirements_df):\n",
    "    \"\"\" Store requirements into the database\n",
    "    \"\"\"\n",
    "    for index, row in requirements_df.iterrows():\n",
    "        attrs = {}\n",
    "        reqid = row[\"ID\"]\n",
    "        attrs[\"Description\"] = row[\"Description\"].replace('\\n', ' ').replace('\\r', '')\n",
    "        attrs[\"ID\"] = reqid\n",
    "        attrs[\"Priority\"]= str(row[\"Priority\"])\n",
    "        create_record(requirement_classname, reqid, attrs)    \n",
    "        \n",
    "def store_testcases(testcases_df):  \n",
    "    \"\"\" Store testcases into the database\n",
    "    \"\"\"\n",
    "    for index, row in testcases_df.iterrows():\n",
    "        attrs = {}\n",
    "        tcaseid = row[\"ID\"]\n",
    "        attrs[\"Description\"] = row[\"Description\"].replace('\\n', ' ').replace('\\r', '')\n",
    "        attrs[\"ID\"] = tcaseid\n",
    "        attrs[\"Category\"] = row[\"Category\"]\n",
    "        create_record(testcase_classname, tcaseid, attrs)\n",
    "        \n",
    "def store_defects(defects_df):\n",
    "    \"\"\" Store defects into the database\n",
    "    \"\"\"\n",
    "    for index, row in defects_df.iterrows():\n",
    "        attrs = {}\n",
    "        defid = row[\"ID\"]\n",
    "        attrs[\"Description\"] = row[\"Description\"].replace('\\n', ' ').replace('\\r', '')\n",
    "        attrs[\"ID\"] = defid\n",
    "        attrs[\"Severity\"] = str(row[\"Severity\"])\n",
    "        create_record(defect_classname, defid, attrs)\n",
    "        \n",
    "def store_testcases_requirement_mapping(testcases_df):\n",
    "    \"\"\" Store the related requirements for testcases into the database\n",
    "    \"\"\"\n",
    "    for index, row in testcases_df.iterrows():\n",
    "        tcaseid = row[\"ID\"]\n",
    "        requirements = row[\"RequirementsMatchScore\"]\n",
    "        for requirement in requirements:\n",
    "            reqid = requirement[\"ID\"]\n",
    "            attributes = {}\n",
    "            attributes['score'] = requirement['cosine_score']\n",
    "            create_testcase_requirement_edge(tcaseid,reqid, attributes)\n",
    "            \n",
    "def store_defect_testcase_mapping(defects_df):\n",
    "    \"\"\" Store the related testcases for the defects into the database\n",
    "    \"\"\"\n",
    "    for index, row in defects_df.iterrows():\n",
    "        defid = row[\"ID\"]\n",
    "        testcases = row[\"TestCasesMatchScore\"]\n",
    "        for testcase in testcases:\n",
    "            testcaseid = testcase[\"ID\"]\n",
    "            attributes = {}\n",
    "            attributes['score'] = testcase[\"cosine_score\"]\n",
    "            create_defect_testcase_edge(defid,testcaseid, attributes)\n",
    "            \n",
    "def store_requirement_defect_mapping(requirements_df):\n",
    "    \"\"\" Store the related defects for the requirements in the database\n",
    "    \"\"\"\n",
    "    for index, row in requirements_df.iterrows():\n",
    "        reqid = row[\"ID\"]\n",
    "        defects = row[\"DefectsMatchScore\"]\n",
    "        for defect in defects:\n",
    "            defectid = defect[\"ID\"]\n",
    "            cosine_score =  defect[\"cosine_score\"]\n",
    "            attributes = {}\n",
    "            attributes['score'] = cosine_score\n",
    "            create_requirement_defect_edge(reqid,defectid, attributes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Store artifacts data and relations into OrientDB **\n",
    "* Drop and create a database\n",
    "* Create classes for each category of artifact\n",
    "* Store artifact data\n",
    "* Store artifact relations data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EInsights created and opened successfully\n"
     ]
    }
   ],
   "source": [
    "drop_database(\"EInsights\")\n",
    "create_database(\"EInsights\", \"admin\", \"admin\")\n",
    "\n",
    "requirement_classname = \"Requirement\"\n",
    "defect_classname = \"Defect\"\n",
    "testcase_classname = \"Testcase\"\n",
    "\n",
    "create_class(requirement_classname)\n",
    "create_class(defect_classname)\n",
    "create_class(testcase_classname)\n",
    "\n",
    "store_requirements(requirements_df)\n",
    "store_defects(defects_df)\n",
    "store_testcases(testcases_df)\n",
    "\n",
    "store_testcases_requirement_mapping(testcases_df)\n",
    "store_defect_testcase_mapping(defects_df)\n",
    "store_requirement_defect_mapping(requirements_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. Transform results for Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_artifacts_mapping_d3_tree(defectId):\n",
    "    \"\"\" Create an artifacts mapping json for display by d3js tree widget\n",
    "    \"\"\"\n",
    "    depTree = {}\n",
    "    depTree['ID'] = defectId\n",
    "    testcases = get_related_testcases(defectId)\n",
    "    \n",
    "    depTree['children'] = []\n",
    "    i=1\n",
    "    for key in testcases:\n",
    "        print (key,testcases[key])\n",
    "        testcaseChildren = {}\n",
    "        testcaseChildren['ID'] = key\n",
    "        testcaseChildren['Score'] = testcases[key]\n",
    "        testcaseChildren['children'] = []\n",
    "        depTree['children'].append(testcaseChildren)\n",
    "        requirements = get_related_requirements(key)\n",
    "        \n",
    "        for key in requirements:\n",
    "            requirementChildren = {}\n",
    "            requirementChildren['ID']=key\n",
    "            requirementChildren['Score']=requirements[key]\n",
    "            testcaseChildren['children'].append(requirementChildren)\n",
    "    return depTree \n",
    "\n",
    "def get_artifacts_mapping_d3_network(defectid):\n",
    "    \"\"\" Create an artifacts mapping json for display by d3js network widget\n",
    "    \"\"\"\n",
    "    nodes =[]\n",
    "    links =[] \n",
    "    defect = {}\n",
    "    defect['id'] = defectid\n",
    "    defect['group'] = 1\n",
    "    nodes.append(defect)\n",
    "    \n",
    "    testcases = get_related_testcases(defectid)\n",
    "    \n",
    "    for key in testcases:\n",
    "        testcase ={}\n",
    "        testcaseid = key\n",
    "        testcase['id'] = testcaseid\n",
    "        testcase['group'] = 2\n",
    "        if testcase not in nodes:\n",
    "            nodes.append(testcase)\n",
    "        \n",
    "        link = {}\n",
    "        link['source'] = defectid\n",
    "        link['target']=testcaseid\n",
    "        link['value']=testcases[testcaseid]\n",
    "        links.append(link)\n",
    "        \n",
    "        requirements = get_related_requirements(key)\n",
    "        for key in requirements:\n",
    "            requirement ={}\n",
    "            requirement['id'] = key\n",
    "            requirement['group'] = 3\n",
    "            if requirement not in nodes:\n",
    "                nodes.append(requirement)\n",
    "            \n",
    "            link = {}\n",
    "            link['source'] = testcaseid\n",
    "            link['target'] = key\n",
    "            link['value'] = requirements[key]\n",
    "            links.append(link)\n",
    "    result ={}\n",
    "    result[\"nodes\"] = nodes\n",
    "    result[\"links\"] = links\n",
    "    return result\n",
    "\n",
    "def get_tc_req_mapping_d3_network(testcaseid):\n",
    "    \"\"\" Create a testcases to requirement mapping json for display by d3js network widget\n",
    "    \"\"\"\n",
    "    nodes =[]\n",
    "    links =[] \n",
    "    testcase = {}\n",
    "    testcase['id'] = testcaseid\n",
    "    testcase['group'] = 2\n",
    "    nodes.append(testcase)\n",
    "    requirements = get_related_requirements(testcaseid)\n",
    "    for key in requirements:            \n",
    "        requirement ={}\n",
    "        requirement['id'] = key\n",
    "        requirement['group'] = 3\n",
    "        nodes.append(requirement)\n",
    "            \n",
    "        link = {}\n",
    "        link['source'] = testcaseid\n",
    "        link['target'] = key\n",
    "        link['value'] = requirements[key]\n",
    "        links.append(link)\n",
    "    result ={}\n",
    "    result[\"nodes\"] = nodes\n",
    "    result[\"links\"] = links\n",
    "    return result\n",
    "\n",
    "def transform_defects_d3_bubble(defects):\n",
    "    \"\"\" Transform the defects list output to a json for display by d3js bubble chart\"\"\"\n",
    "    defectsList = {}\n",
    "    defectsList['name'] = \"defect\"\n",
    "    children = []\n",
    "    for defect in defects:\n",
    "        detail = {}\n",
    "        sizeList = [400,230,130]\n",
    "        detail[\"ID\"] = defect['ID']\n",
    "        severity = int(defect['Severity'])\n",
    "        detail[\"group\"] = str(severity)\n",
    "        detail[\"size\"] = sizeList[severity-1]\n",
    "        children.append(detail)\n",
    "    defectsList['children'] = children \n",
    "    return defectsList\n",
    "\n",
    "def transform_testcases_d3_bubble(testcases):\n",
    "    \"\"\" Transform the testcases list output to a json for display by d3js bubble chart\"\"\"\n",
    "    testcasesList = {}\n",
    "    testcasesList['name'] = \"test\"\n",
    "    sizeList = {}\n",
    "    sizeList[\"FVT\"]=200\n",
    "    sizeList[\"TVT\"]=110\n",
    "    sizeList[\"SVT\"]=400\n",
    "    children = []\n",
    "    for testcase in testcases:\n",
    "        detail = {}\n",
    "        detail[\"ID\"] = testcase['ID']\n",
    "        detail[\"group\"] = testcase['Category']\n",
    "        detail[\"size\"]= sizeList[testcase['Category']]\n",
    "        children.append(detail)\n",
    "    testcasesList['children'] = children \n",
    "    return testcasesList\n",
    "\n",
    "def transform_requirements_d3_bubble(requirements):\n",
    "    \"\"\" Transform the requirements list output to a json for display by d3js bubble chart\"\"\"\n",
    "    requirementsList = {}\n",
    "    requirementsList['name'] = \"requirement\"\n",
    "    sizeList = {}\n",
    "    sizeList[1]=400\n",
    "    sizeList[2]=200\n",
    "    sizeList[3]=110\n",
    "    children = []\n",
    "    for requirement in requirements:\n",
    "        detail = {}\n",
    "        detail[\"ID\"] = requirement['ID']\n",
    "        detail[\"group\"] = requirement['Priority']\n",
    "        detail[\"size\"]= sizeList[int(requirement['Priority'])]\n",
    "        if 'defectcount' in requirement:\n",
    "            detail['defectcount'] = requirement['defectcount']\n",
    "        children.append(detail)\n",
    "    requirementsList['children'] = children \n",
    "    return requirementsList\n",
    "\n",
    "def merge_apply_filters_d3_bubble(mainList, filterList):\n",
    "    \"\"\" Add a filter attribute to the list elements for processing on UI\n",
    "    \"\"\"\n",
    "    mainListChildren = mainList['children']\n",
    "    filterListChildren = filterList['children']\n",
    "    for child in mainListChildren:\n",
    "        child['filter'] = 0\n",
    "        for child1 in filterListChildren:\n",
    "            if ( child['ID'] == child1['ID']):\n",
    "                child['filter'] = 1\n",
    "    return mainList  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8. Expose integration point with a websocket client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def on_message(ws, message):\n",
    "    print(message)\n",
    "    msg = json.loads(message)\n",
    "    print(\"message\",msg)\n",
    "    cmd = msg['cmd']\n",
    "    \n",
    "    print(\"Command :\", cmd)\n",
    "\n",
    "    if cmd == 'DefectList':\n",
    "        wsresponse = {}\n",
    "        wsresponse[\"forCmd\"] = \"DefectList\" \n",
    "        defects = get_defects()\n",
    "        wsresponse[\"response\"] = transform_defects_d3_bubble(defects)\n",
    "        ws.send(json.dumps(wsresponse))\n",
    "\n",
    "    if cmd == 'TestcaseList':\n",
    "        wsresponse = {}\n",
    "        wsresponse[\"forCmd\"] = \"TestcaseList\"\n",
    "        testcases = get_testcases()\n",
    "        wsresponse[\"response\"] = transform_testcases_d3_bubble(testcases)\n",
    "        ws.send(json.dumps(wsresponse))\n",
    "\n",
    "    if cmd == 'ReqsList':\n",
    "        wsresponse = {}\n",
    "        wsresponse[\"forCmd\"] = \"ReqsList\"\n",
    "        requirements = get_requirements()\n",
    "        wsresponse[\"response\"] = transform_requirements_d3_bubble(requirements)\n",
    "        ws.send(json.dumps(wsresponse))\n",
    "\n",
    "    if cmd == 'DefectRelation':\n",
    "        defect_id = msg['ID']\n",
    "        wsresponse = {}\n",
    "        wsresponse[\"forCmd\"] = \"DefectRelation\" \n",
    "        wsresponse[\"response\"] = get_artifacts_mapping_d3_network(defect_id)\n",
    "        ws.send(json.dumps(wsresponse))\n",
    "\n",
    "    if cmd == 'TestcaseRelation':\n",
    "        testcase_id = msg['ID']\n",
    "        wsresponse = {}\n",
    "        wsresponse[\"forCmd\"] = \"TestcaseRelation\" \n",
    "        wsresponse[\"response\"] = get_tc_req_mapping_d3_network(testcase_id)\n",
    "        ws.send(json.dumps(wsresponse))\n",
    "\n",
    "    if cmd == 'DefectInsight':\n",
    "        insight_id = msg['ID']\n",
    "        defects = get_defects()\n",
    "        defects = transform_defects_d3_bubble(defects)\n",
    "        if (insight_id.find('Insight1') != -1):\n",
    "            defectsSev1 = get_defects_severity(1)\n",
    "            defectsSev1 = transform_defects_d3_bubble(defectsSev1)\n",
    "            response = merge_apply_filters_d3_bubble(defects, defectsSev1)\n",
    "        if (insight_id.find('Insight2') != -1):\n",
    "            defectsSev2 = get_defects_severity(2)\n",
    "            defectsSev2 = transform_defects_d3_bubble(defectsSev2)\n",
    "            response = merge_apply_filters_d3_bubble(defects, defectsSev2)\n",
    "        if (insight_id.find('Insight3') != -1):\n",
    "            defectsSev3 = get_defects_severity(3)\n",
    "            defectsSev3 = transform_defects_d3_bubble(defectsSev3)\n",
    "            response = merge_apply_filters_d3_bubble(defects, defectsSev3)\n",
    "        if (insight_id.find('Insight4') != -1):\n",
    "            defects_zero_tc = get_defects_zero_testcases()\n",
    "            defects_zero_tc = transform_defects_d3_bubble(defects_zero_tc)\n",
    "            response = merge_apply_filters_d3_bubble(defects, defects_zero_tc)\n",
    "        wsresponse = {}\n",
    "        wsresponse[\"forCmd\"] = \"Insight\" \n",
    "        wsresponse[\"response\"] = response\n",
    "        ws.send(json.dumps(wsresponse))\n",
    "\n",
    "    if cmd == 'TestInsight':\n",
    "        insight_id = msg['ID']\n",
    "        testcases = get_testcases()\n",
    "        testcases = transform_testcases_d3_bubble(testcases)\n",
    "        if (insight_id.find('Insight1') != -1):\n",
    "            fvtTests = get_testcases_category('FVT')\n",
    "            fvtTests = transform_testcases_d3_bubble(fvtTests)\n",
    "            response = merge_apply_filters_d3_bubble(testcases, fvtTests)\n",
    "        if (insight_id.find('Insight2') != -1):\n",
    "            svtTests = get_testcases_category('SVT')\n",
    "            svtTests = transform_testcases_d3_bubble(svtTests)\n",
    "            response = merge_apply_filters_d3_bubble(testcases, svtTests)\n",
    "        if (insight_id.find('Insight3') != -1):\n",
    "            tvtTests = get_testcases_category('TVT')\n",
    "            tvtTests = transform_testcases_d3_bubble(tvtTests)\n",
    "            response = merge_apply_filters_d3_bubble(testcases, tvtTests)\n",
    "        if (insight_id.find('Insight4') != -1):\n",
    "            testcase_zero_defect = get_testcases_zero_defects()\n",
    "            testcase_zero_defect = transform_testcases_d3_bubble(testcase_zero_defect)\n",
    "            response = merge_apply_filters_d3_bubble(testcases, testcase_zero_defect)\n",
    "        wsresponse = {}\n",
    "        wsresponse[\"forCmd\"] = \"Insight\" \n",
    "        wsresponse[\"response\"] = response\n",
    "        ws.send(json.dumps(wsresponse))\n",
    "\n",
    "    if cmd == 'ReqInsight':\n",
    "        insight_id = msg['ID']\n",
    "        requirements = get_requirements()\n",
    "        requirements = transform_requirements_d3_bubble(requirements)\n",
    "        if (insight_id.find('Insight1') != -1):\n",
    "            req = get_requirements_zero_defect()\n",
    "            req = transform_requirements_d3_bubble(req)\n",
    "            response = merge_apply_filters_d3_bubble(requirements, req)\n",
    "        if (insight_id.find('Insight2') != -1):\n",
    "            req = get_requirements_zero_testcases()\n",
    "            req = transform_requirements_d3_bubble(req)\n",
    "            response = merge_apply_filters_d3_bubble(requirements, req)\n",
    "        if (insight_id.find('Insight3') != -1):\n",
    "            req = get_requirement_defects(5)\n",
    "            req = transform_requirements_d3_bubble(req)\n",
    "            response = merge_apply_filters_d3_bubble(requirements, req)\n",
    "        wsresponse = {}\n",
    "        wsresponse[\"forCmd\"] = \"Insight\" \n",
    "        wsresponse[\"response\"] = response\n",
    "        ws.send(json.dumps(wsresponse)) \n",
    "\n",
    "def on_error(ws, error):\n",
    "    print(error)\n",
    "\n",
    "def on_close(ws):\n",
    "    print (\"DSX Listen End\")\n",
    "    start_websocket_listener()\n",
    "    ws.send(\"DSX Listen End\")\n",
    "\n",
    "def on_open(ws):\n",
    "    def run(*args):\n",
    "        for i in range(10000):\n",
    "            hbeat = '{\"cmd\":\"EI DSX HeartBeat\"}'\n",
    "            ws.send(hbeat)\n",
    "            time.sleep(100)\n",
    "            \n",
    "    thread.start_new_thread(run, ())\n",
    "\n",
    "\n",
    "def start_websocket_listener():\n",
    "    websocket.enableTrace(True)\n",
    "    ws = websocket.WebSocketApp(\"ws://<NODERED_BASE_URL>/ws/orchestrate\",\n",
    "                              on_message = on_message,\n",
    "                              on_error = on_error,\n",
    "                              on_close = on_close)\n",
    "    ws.on_open = on_open\n",
    "    ws.run_forever()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8.1 Start websocket client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/websocket/_app.py\", line 320, in _callback\n",
      "    callback(self, *args)\n",
      "  File \"<ipython-input-58-5b9a22af86c1>\", line 129, in on_open\n",
      "    thread.start_new_thread(run, ())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"cmd\":\"Client connected\"}\n",
      "message {'cmd': 'Client connected'}\n",
      "Command : Client connected\n",
      "{\"cmd\":\"DefectList\"}\n",
      "message {'cmd': 'DefectList'}\n",
      "Command : DefectList\n",
      "{\"forCmd\": \"DefectList\", \"response\": {\"name\": \"defect\", \"children\": [{\"ID\": \"D10\", \"group\": \"2\", \"size\": 230}, {\"ID\": \"D20\", \"group\": \"1\", \"size\": 400}, {\"ID\": \"D16\", \"group\": \"2\", \"size\": 230}, {\"ID\": \"D90\", \"group\": \"3\", \"size\": 130}, {\"ID\": \"D41\", \"group\": \"3\", \"size\": 130}, {\"ID\": \"D17\", \"group\": \"1\", \"size\": 400}, {\"ID\": \"D47\", \"group\": \"1\", \"size\": 400}, {\"ID\": \"D38\", \"group\": \"2\", \"size\": 230}, {\"ID\": \"D48\", \"group\": \"2\", \"size\": 230}, {\"ID\": \"D75\", \"group\": \"2\", \"size\": 230}, {\"ID\": \"D11\", \"group\": \"2\", \"size\": 230}, {\"ID\": \"D13\", \"group\": \"3\", \"size\": 130}, {\"ID\": \"D21\", \"group\": \"3\", \"size\": 130}, {\"ID\": \"D31\", \"group\": \"3\", \"size\": 130}, {\"ID\": \"D18\", \"group\": \"1\", \"size\": 400}, {\"ID\": \"D56\", \"group\": \"1\", \"size\": 400}, {\"ID\": \"D76\", \"group\": \"1\", \"size\": 400}, {\"ID\": \"D26\", \"group\": \"2\", \"size\": 230}, {\"ID\": \"D59\", \"group\": \"3\", \"size\": 130}, {\"ID\": \"D92\", \"group\": \"2\", \"size\": 230}]}}\n",
      "message {'forCmd': 'DefectList', 'response': {'name': 'defect', 'children': [{'ID': 'D10', 'group': '2', 'size': 230}, {'ID': 'D20', 'group': '1', 'size': 400}, {'ID': 'D16', 'group': '2', 'size': 230}, {'ID': 'D90', 'group': '3', 'size': 130}, {'ID': 'D41', 'group': '3', 'size': 130}, {'ID': 'D17', 'group': '1', 'size': 400}, {'ID': 'D47', 'group': '1', 'size': 400}, {'ID': 'D38', 'group': '2', 'size': 230}, {'ID': 'D48', 'group': '2', 'size': 230}, {'ID': 'D75', 'group': '2', 'size': 230}, {'ID': 'D11', 'group': '2', 'size': 230}, {'ID': 'D13', 'group': '3', 'size': 130}, {'ID': 'D21', 'group': '3', 'size': 130}, {'ID': 'D31', 'group': '3', 'size': 130}, {'ID': 'D18', 'group': '1', 'size': 400}, {'ID': 'D56', 'group': '1', 'size': 400}, {'ID': 'D76', 'group': '1', 'size': 400}, {'ID': 'D26', 'group': '2', 'size': 230}, {'ID': 'D59', 'group': '3', 'size': 130}, {'ID': 'D92', 'group': '2', 'size': 230}]}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/websocket/_app.py\", line 320, in _callback\n",
      "    callback(self, *args)\n",
      "  File \"<ipython-input-58-5b9a22af86c1>\", line 5, in on_message\n",
      "    cmd = msg['cmd']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"cmd\":\"DefectRelation\",\"ID\":\"D18\"}\n",
      "message {'ID': 'D18', 'cmd': 'DefectRelation'}\n",
      "Command : DefectRelation\n",
      "[<pyorient.otypes.OrientRecord object at 0x7fd7d1122128>]\n",
      "['R279'] ['0.547722557505']\n",
      "[<pyorient.otypes.OrientRecord object at 0x7fd7d1122b38>]\n",
      "['R279'] ['0.606976978667']\n",
      "[<pyorient.otypes.OrientRecord object at 0x7fd7d1122a20>]\n",
      "['R279'] ['0.816496580928']\n",
      "[<pyorient.otypes.OrientRecord object at 0x7fd7d11226d8>, <pyorient.otypes.OrientRecord object at 0x7fd7cb5cb320>]\n",
      "['R220', 'R279'] ['0.5', '0.4472135955']\n",
      "{\"forCmd\": \"DefectRelation\", \"response\": {\"nodes\": [{\"group\": 1, \"id\": \"D18\"}, {\"group\": 2, \"id\": \"TC075\"}, {\"group\": 3, \"id\": \"R279\"}, {\"group\": 2, \"id\": \"TC072\"}, {\"group\": 2, \"id\": \"TC078\"}, {\"group\": 2, \"id\": \"TC120\"}, {\"group\": 3, \"id\": \"R220\"}], \"links\": [{\"source\": \"D18\", \"target\": \"TC075\", \"value\": \"0.866025403784\"}, {\"source\": \"TC075\", \"target\": \"R279\", \"value\": \"0.547722557505\"}, {\"source\": \"D18\", \"target\": \"TC072\", \"value\": \"0.790569415042\"}, {\"source\": \"TC072\", \"target\": \"R279\", \"value\": \"0.606976978667\"}, {\"source\": \"D18\", \"target\": \"TC078\", \"value\": \"0.790569415042\"}, {\"source\": \"TC078\", \"target\": \"R279\", \"value\": \"0.816496580928\"}, {\"source\": \"D18\", \"target\": \"TC120\", \"value\": \"0.790569415042\"}, {\"source\": \"TC120\", \"target\": \"R220\", \"value\": \"0.5\"}, {\"source\": \"TC120\", \"target\": \"R279\", \"value\": \"0.4472135955\"}]}}\n",
      "message {'forCmd': 'DefectRelation', 'response': {'nodes': [{'group': 1, 'id': 'D18'}, {'group': 2, 'id': 'TC075'}, {'group': 3, 'id': 'R279'}, {'group': 2, 'id': 'TC072'}, {'group': 2, 'id': 'TC078'}, {'group': 2, 'id': 'TC120'}, {'group': 3, 'id': 'R220'}], 'links': [{'source': 'D18', 'target': 'TC075', 'value': '0.866025403784'}, {'source': 'TC075', 'target': 'R279', 'value': '0.547722557505'}, {'source': 'D18', 'target': 'TC072', 'value': '0.790569415042'}, {'source': 'TC072', 'target': 'R279', 'value': '0.606976978667'}, {'source': 'D18', 'target': 'TC078', 'value': '0.790569415042'}, {'source': 'TC078', 'target': 'R279', 'value': '0.816496580928'}, {'source': 'D18', 'target': 'TC120', 'value': '0.790569415042'}, {'source': 'TC120', 'target': 'R220', 'value': '0.5'}, {'source': 'TC120', 'target': 'R279', 'value': '0.4472135955'}]}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/websocket/_app.py\", line 320, in _callback\n",
      "    callback(self, *args)\n",
      "  File \"<ipython-input-58-5b9a22af86c1>\", line 5, in on_message\n",
      "    cmd = msg['cmd']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "DSX Listen End\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/websocket/_app.py\", line 320, in _callback\n",
      "    callback(self, *args)\n",
      "  File \"<ipython-input-58-5b9a22af86c1>\", line 120, in on_close\n",
      "    ws.send(\"DSX Listen End\")\n",
      "  File \"/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/websocket/_app.py\", line 151, in send\n",
      "    if not self.sock or self.sock.send(data, opcode) == 0:\n",
      "  File \"/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/websocket/_core.py\", line 240, in send\n",
      "    return self.send_frame(frame)\n",
      "  File \"/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/websocket/_core.py\", line 265, in send_frame\n",
      "    l = self._send(data)\n",
      "  File \"/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/websocket/_core.py\", line 430, in _send\n",
      "    return send(self.sock, data)\n",
      "  File \"/opt/conda/envs/DSX-Python35/lib/python3.5/site-packages/websocket/_socket.py\", line 114, in send\n",
      "    raise WebSocketConnectionClosedException(\"socket is already closed.\")\n"
     ]
    }
   ],
   "source": [
    "start_websocket_listener()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}