Edge_Investigation_vast_bu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "displaythis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import struct, socket\n",
    "import shutil\n",
    "import numpy as np\n",
    "import linecache, bisect\n",
    "import csv\n",
    "import operator\n",
    "from IPython.html import widgets\n",
    "from IPython.html.widgets import interact, interactive, fixed\n",
    "from IPython.display import display\n",
    "\n",
    "spath = '/home/hadoop/ipython/vast/user/vast/'\n",
    "sconnect = spath + 'lda_scores.csv'\n",
    "sconnectbu = spath + 'lda_scores_bu.csv'\n",
    "tmpconnect = sconnect+'.tmp'\n",
    "stemp = sconnect + '.new'\n",
    "coff = 150;\n",
    "contxtpath = '/home/hadoop/ipython/vast/iploc/'\n",
    "nwloc = contxtpath + 'networkcontext.txt'\n",
    "#load LDA scores#update lda doc with scores - either one edge,# or ALL that contain x IP and y port\n",
    "def displaythis() :\n",
    "    searches = widgets.Box(width=550,height=200)\n",
    "    searchp = widgets.HBox(width=550,height=150)\n",
    "    actions = widgets.Box(width=50,height=150)\n",
    "    srcselect = widgets.Select(options=srclist,height=min(len(srclist)*10,150),width=120)\n",
    "    dstselect = widgets.Select(options=dstlist,height=min(len(dstlist)*10,150),width=120)\n",
    "    sportselect = widgets.Select(options=sportlist,height=min(len(sportlist)*10,150),width=75)\n",
    "    dportselect = widgets.Select(options=dportlist,height=min(len(dportlist)*10,150),width=75)\n",
    "    tableHTML = (\"<table width='100%'><tr><td>\" +\n",
    "    \"<td>Source IP:</td><td>Dest IP:</td><td>Src Port</td><td>Dst Port:</td></tr></table>\")\n",
    "    toplabel = widgets.HTML(value=tableHTML,width=550)\n",
    "    ratingbut = widgets.RadioButtons(description='Rating:',options=['1', '2', '3'],width=50)\n",
    "    assignbut = widgets.Button(description='Assign')\n",
    "    updatebut = widgets.Button(description='Update')\n",
    "    actions.children = [ratingbut,assignbut,updatebut]\n",
    "    searchp.children = [srcselect,dstselect,sportselect,dportselect,actions]\n",
    "    searches.children = [toplabel,searchp]\n",
    "    display(searches)\n",
    "    \n",
    "    def update_sconnects(b):\n",
    "        #define logic based on combo of input\n",
    "        src_ip,dst_ip = srcselect.value,dstselect.value\n",
    "        sport,dport = sportselect.value,dportselect.value\n",
    "        risk = ratingbut.value\n",
    "        srchstr = '';\n",
    "        if len(src_ip) > 0:\n",
    "            srchstr += 'sip'\n",
    "        if len(dst_ip) > 0:\n",
    "            srchstr += 'dip'\n",
    "        if len(sport) > 0:\n",
    "            srchstr += 'sport'\n",
    "        if len(dport) > 0:\n",
    "            srchstr += 'dport'\n",
    "        with open(tmpconnect,'w') as g:\n",
    "            riter = csv.writer(g,delimiter=',')\n",
    "            riter.writerow([\"sev\"]+[\"score\"]+[\"Date\"]+[\"srcIP\"]+[\"dstIP\"]+[\"SPort\"]+\n",
    "                           [\"DPort\"]+[\"Sfirst\"]+[\"Dfirst\"]+[\"Stotal\"]+[\"Dtotal\"]+[\"Spkts\"]+[\"Dpkts\"])\n",
    "            with open(sconnect, 'r') as f:\n",
    "                reader = csv.reader(f,delimiter=',')\n",
    "                reader.next();\n",
    "                rowct = 1\n",
    "                #sev,score,date,srcIP,dstIP,sport,dport,proto,ipkt,ibyt,lda_score\n",
    "                for row in reader:\n",
    "                    if srchstr == 'dipdport':\n",
    "                        if row[4] == dst_ip and row[6] == dport:\n",
    "                           row[0] = risk; \n",
    "                    elif srchstr == 'dip':\n",
    "                        if row[4] == dst_ip:\n",
    "                            row[0] = risk; \n",
    "                    elif srchstr == 'sipdip':\n",
    "                        if row[3] == src_ip and row[4] == dst_ip:\n",
    "                           row[0] = risk; \n",
    "                    elif srchstr == 'sipdipdport':\n",
    "                        if row[3] == src_ip and row[4] == dst_ip and row[6] == dport:\n",
    "                            row[0] = risk;\n",
    "                    riter.writerow(row) \n",
    "                    rowct += 1\n",
    "        shutil.copyfile(tmpconnect,sconnect)\n",
    "            \n",
    "    def savesort(b):\n",
    "        with open(stemp,'w') as g:\n",
    "            riter = csv.writer(g,delimiter=',')\n",
    "            riter.writerow([\"sev\"]+[\"score\"]+[\"Date\"]+[\"srcIP\"]+[\"dstIP\"]+[\"SPort\"]+[\"DPort\"]+[\"Sfirst\"]+[\"Dfirst\"]+[\"Stotal\"]+[\"Dtotal\"]+[\"Spkts\"]+[\"Dpkts\"])\n",
    "            reader = csv.reader(open(sconnect), delimiter=\",\")\n",
    "            reader.next()\n",
    "            srtlist = sorted(reader, key=operator.itemgetter(0,1))\n",
    "            riter.writerows(srtlist)\n",
    "        shutil.copyfile(stemp,sconnect)\n",
    "\n",
    "    assignbut.on_click(update_sconnects)\n",
    "    updatebut.on_click(savesort)\n",
    "\n",
    "# build dict of IP addresses\n",
    "#sev,score, tstart,srcIP,dstIP,sport,dport,proto,ipkt,ibyt\n",
    "srcdict,srclist = {},[]\n",
    "dstdict,dstlist = {},[]\n",
    "sportdict,sportlist = {},[]\n",
    "dportdict,dportlist = {},[]\n",
    "srclist.append('')\n",
    "dstlist.append('')\n",
    "sportlist.append('')\n",
    "dportlist.append('')\n",
    "with open(sconnect, 'r') as f:\n",
    "    reader = csv.reader(f,delimiter=',') \n",
    "    reader.next();\n",
    "    rowct = 1\n",
    "    for row in reader:\n",
    "        if row[3] not in srcdict:\n",
    "            srclist.append(row[3])\n",
    "            srcdict[row[3]] = struct.unpack(\"!L\", socket.inet_aton(row[3]))[0]\n",
    "        if row[4] not in dstdict:\n",
    "            dstlist.append(row[4])\n",
    "            dstdict[row[4]] = struct.unpack(\"!L\", socket.inet_aton(row[4]))[0]\n",
    "        if row[5] not in sportdict:\n",
    "            sportlist.append(row[5])\n",
    "            sportdict[row[5]] = row[5]\n",
    "        if row[6] not in dportdict:\n",
    "            dportlist.append(row[6])\n",
    "            dportdict[row[6]] = row[6]\n",
    "        if rowct == coff:\n",
    "            break;\n",
    "        rowct += 1\n",
    "iplist = np.loadtxt(nwloc,dtype=np.uint32,delimiter=',',usecols={0},\n",
    "                    converters={0: lambda s: np.uint32(s.replace('\"',''))})\n",
    "#bisect - linecache\n",
    "# iterate through IP addresses and attach ip location info\n",
    "for srcip in srcdict:\n",
    "        srcdict[srcip] = linecache.getline(nwloc, bisect.bisect(iplist,srcdict[srcip])).replace('\\n','')\n",
    "for dstip in dstdict:\n",
    "        dstdict[dstip] = linecache.getline(nwloc, bisect.bisect(iplist,dstdict[dstip])).replace('\\n','')\n",
    "\n",
    "# print IP addresses\n",
    "with open(sconnect, 'r') as f:\n",
    "    reader = csv.reader(f,delimiter=',') \n",
    "    reader.next();\n",
    "    rowct = 1\n",
    "    for row in reader:\n",
    "        #print \"line\",rowct,row,\":\"\n",
    "        if srcdict[row[3]] == '':\n",
    "            srcip = 'internet'\n",
    "        else:\n",
    "            srcip = srcdict[row[3]]\n",
    "        if dstdict[row[4]] == '':\n",
    "            dstip = 'internet'\n",
    "        else:\n",
    "            dstip = dstdict[row[4]]\n",
    "            \n",
    "        #print 'line ',rowct,': ',srcip,\"-->\",dstip\n",
    "        if rowct == 5:\n",
    "            break\n",
    "        rowct += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Occasionally, when you wish to update the suspicious connect list based on your review, run the cell below."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>End Workflow</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sev,score,Date,srcIP,dstIP,SPort,DPort,Sfirst,Dfirst,Stotal,Dtotal,Spkts,Dpkts\r",
      "\r\n",
      "0,1.50E-07,4/14/2013 15:10,10.70.68.127,172.30.0.4,6395,80,0,0,278,116,5,2\r",
      "\r\n",
      "0,1.50E-07,4/14/2013 15:09,10.70.68.127,172.30.0.4,55759,80,19,1006,297,1114,5,2\r",
      "\r\n",
      "0,1.51E-07,4/10/2013 17:16,10.13.77.49,172.10.0.40,61783,80,0,0,148,108,2,2\r",
      "\r\n",
      "0,1.51E-07,4/14/2013 15:16,10.13.77.49,172.10.0.2,0,0,0,0,354,354,2,2\r",
      "\r\n",
      "0,1.51E-07,4/14/2013 17:16,10.13.77.49,172.10.0.40,47131,80,0,0,206,162,3,3\r",
      "\r\n",
      "0,1.51E-07,4/14/2013 14:56,10.13.77.49,172.10.0.3,35579,25,0,0,112,58,2,1\r",
      "\r\n",
      "0,1.51E-07,4/14/2013 15:16,10.13.77.49,172.20.0.3,46032,25,0,0,496,280,8,4\r",
      "\r\n",
      "0,1.72E-07,4/11/2013 12:57,10.200.20.2,172.30.0.4,3247,80,0,0,286,62,5,1\r",
      "\r\n",
      "0,1.75E-07,4/14/2013 16:21,10.138.235.111,172.10.0.40,61471,80,0,0,148,108,2,2\r",
      "\r\n"
     ]
    }
   ],
   "source": [
    "#only needed to reset demo\n",
    "!cp $sconnectbu $sconnect\n",
    "!head -n 10 $sconnect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#!cp $tmpconnect $nwloc\n",
    "#!head -n 10 $nwloc"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}