Untitled1.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:34a5d4311170b1cf226379d63c4a8cd0f4f5489b2f4c9e47a6524ba69c035748"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "text = open(\"/home/wenlei/new_untitled.ipynb\").read().splitlines()\n",
      "\n",
      "new_text = []\n",
      "start = 0\n",
      "end = 0\n",
      "\n",
      "for i in range(len(text)):\n",
      "    if \"\\\"input\\\": [\" in text[i]:\n",
      "        start = i\n",
      "    if \"     ],\" in text[i] and 'language' in text[i+1] and start != 0:\n",
      "        end = i\n",
      "        new_text.append(text[start+1:end])\n",
      "for ii in new_text:\n",
      "    for j in ii:\n",
      "        #print j\n",
      "        print j.strip('\\\\n\\\",')\n",
      "    print '======================================'\n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "    \n",
      "\n",
      "#new_file = open(\"/home/wenlei/new_untitled.ipynb\",'w')\n",
      "#new_file.write('\\n'.join(new_text))\n",
      "        \n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os,sys\n",
      "from Bio import SeqIO\n",
      "'''\n",
      "Extract proteins from FGENESV predictions.\n",
      "If one predicted protein has no hits aginst all the cmv proteins or poor hits qualities, it will be extracted.\n",
      "argv[1] is the folder which contains blast tabular results.\n",
      "argv[2] is the folder which contains the predicted proteins.\n",
      "argv[3] is the output folder of extraction.\n",
      "'''\n",
      "      \n",
      "def tabular_parse(filename):\n",
      "    '''\n",
      "    parse a blast tabular file and return the sequences without a hit or in poor identity.\n",
      "    '''\n",
      "    \n",
      "    tabulars = open(filename,'r').read()\n",
      "      \n",
      "    hits = tabulars.split(\"# BLASTP 2.2.31+\\\\n\")[1:-1]\n",
      "      \n",
      "    no_hit_count = 0\n",
      "    no_hits = []\n",
      "    for hit in hits:\n",
      "        items = hit.split(\"\\\\n\")\n",
      "        mquery = items[0].split(\":\")[1].strip()\n",
      "        mhits = items[3].split(\" \")\n",
      "        hit_num = 0\n",
      "        if mhits[0] == '#':\n",
      "            hit_num = int(mhits[1])\n",
      "        if hit_num == 0:\n",
      "            #print mquery\n",
      "            no_hits.append(mquery)\n",
      "            no_hit_count+=1\n",
      "        else:\n",
      "            hit_details =[i for i in items[-(hit_num+1):-1] if float(i.split('\\\\t')[-1])>50 and float(i.split('\\\\t')[-4])<0.0000001]\n",
      "            if not hit_details:\n",
      "                #print mquery\n",
      "                no_hits.append(mquery)\n",
      "                no_hit_count += 1\n",
      "    print no_hit_count\n",
      "      \n",
      "    return no_hits    \n",
      "      \n",
      "      \n",
      "mfiles = os.listdir(sys.argv[1])\n",
      "      \n",
      "for mfile in mfiles:\n",
      "    no_hits = tabular_parse(sys.argv[1]+mfile)\n",
      "    print no_hits\n",
      "    mfastas = SeqIO.parse(sys.argv[2]+mfile,'fasta')\n",
      "    no_hits_record = []\n",
      "    for mfasta in mfastas:\n",
      "        print mfasta.descriptio\n",
      "        if mfasta.description in no_hits:\n",
      "            print True\n",
      "            no_hits_record.append(mfasta)\n",
      "    SeqIO.write(no_hits_record,sys.argv[3]+mfile,'fasta')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os,sys\n",
      "from Bio import SeqIO\n",
      "'''\n",
      "Extract proteins from FGENESV predictions.\n",
      "If one predicted protein has no hits aginst all the cmv proteins or poor hits qualities, it will be extracted.\n",
      "argv[1] is the folder which contains blast tabular results.\n",
      "argv[2] is the folder which contains the predicted proteins.\n",
      "argv[3] is the output folder of extraction.\n",
      "'''\n",
      "      \n",
      "def tabular_parse(filename):\n",
      "    '''\n",
      "    parse a blast tabular file and return the sequences without a hit or in poor identity.\n",
      "    '''\n",
      "    \n",
      "    tabulars = open(filename,'r').read()\n",
      "      \n",
      "    hits = tabulars.split(\"# BLASTP 2.2.31+\\n\")[1:-1]\n",
      "      \n",
      "    no_hit_count = 0\n",
      "    no_hits = []\n",
      "    my_hits = []\n",
      "    for hit in hits:\n",
      "        items = hit.split(\"\\n\")\n",
      "        mquery = items[0].split(\":\")[1].strip()\n",
      "        mhits = items[3].split(\" \")\n",
      "        hit_num = 0\n",
      "        if mhits[0] == '#':\n",
      "            hit_num = int(mhits[1])\n",
      "        if hit_num == 0:\n",
      "            #print mquery\n",
      "            no_hits.append(mquery)\n",
      "            no_hit_count+=1\n",
      "        else:\n",
      "            hit_details =[i for i in items[-(hit_num+1):-1] if float(i.split('\\t')[-1])>10]\n",
      "            if not hit_details:\n",
      "                #print mquery\n",
      "                no_hits.append(mquery)\n",
      "                no_hit_count += 1\n",
      "            else:\n",
      "                #print hit_details\n",
      "                if len(hit_details)>20:\n",
      "                    my_hits.extend(hit_details[0:20])\n",
      "                else:\n",
      "                    my_hits.extend(hit_details)\n",
      "                \n",
      "    return my_hits    \n",
      "      \n",
      "mdir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/unhits_against_nr/\n",
      "out_dir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output/\n",
      "      \n",
      "mfiles = os.listdir(mdir)\n",
      "      \n",
      "accessions = []\n",
      "      \n",
      "for mfile in mfiles:\n",
      "    mhits = tabular_parse(mdir+mfile)\n",
      "    out_file = open(out_dir+mfile,'w')\n",
      "    print mfile\n",
      "    print mhits\n",
      "    out_file.write('\\n'.join(mhits))\n",
      "    out_file.close()\n",
      "      "
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import re\n",
      "\n",
      "items = open(\\\"/home/wenlei/LUO/Bat\\\",'r').readlines()\n",
      "\n",
      "pattern = re.compile('\\\\d+')\n",
      "\n",
      "all = []\n",
      "for i in items:\n",
      "if ('+' in i or '-' in i) and 'CDS' in i:\n",
      "m = pattern.findall(i)\n",
      "mstr = m[1]+'-'+m[2]\n",
      "#print mstr\n",
      "all.append(mstr)\n",
      "\n",
      "print len(set(all))\n",
      "tmp = []\n",
      "for ii in set(all):\n",
      "if all.count(ii)>2:\n",
      "tmp.append(ii)\n",
      "print len(tmp)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "SyntaxError",
       "evalue": "unexpected character after line continuation character (<ipython-input-2-102347e53d5f>, line 3)",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-2-102347e53d5f>\"\u001b[1;36m, line \u001b[1;32m3\u001b[0m\n\u001b[1;33m    items = open(\\\"/home/wenlei/LUO/Bat\\\",'r').readlines()\u001b[0m\n\u001b[1;37m                                                          ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m unexpected character after line continuation character\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#coding:utf-8\n",
      "# This is an example that uses pygame.draw.rect:\n",
      "import os, sys\n",
      "import random\n",
      "import pygame\n",
      "import xlrd\n",
      "from pygame.locals import *\n",
      "      \n",
      "pygame.init()\n",
      "APPLICATION_x_size = 1500\n",
      "APPLICATION_y_size = 2000\n",
      "screen = pygame.display.set_mode((APPLICATION_x_size, APPLICATION_y_size))\n",
      "pygame.display.set_caption('Fun Boring Example comes with Source Code too!!')\n",
      "      \n",
      "mSurface = pygame.Surface(screen.get_size())\n",
      "mSurface.fill((255, 255, 255))\n",
      "screen.blit(mSurface, (0, 0))\n",
      "      \n",
      "      \n",
      "        \n",
      "        \n",
      "myfont = pygame.font.SysFont(\"monospace\", 12)  \n",
      "      \n",
      "names = open(\"/home/wenlei/LUO/Simon/results/overall_count\").read().splitlines() \n",
      "names_2 = open(\"/home/wenlei/LUO/Simon/results/mutual_hits_results\").read().splitlines() \n",
      "name_3 = open(\"/home/wenlei/LUO/Simon/results/fgenev_hit_nr_count\").read().splitlines()\n",
      "name_4 = [name_3[i] for i in range(1,len(name_3),2)]\n",
      "\n",
      "name_5 = open(\"/home/wenlei/newly_uncovered_genes\").read().splitlines()\n",
      "\n",
      "name_5 = [name_5[i] for i in range(1,len(name_5),2)]\n",
      "print name_5\n",
      "\n",
      "content_11 = open(\"/home/wenlei/LUO/Simon/results/gene_density\").read().splitlines()\n",
      "\n",
      "      \n",
      "Y = 100\n",
      "purple = (129,93,141)\n",
      "red = (191,69,61)\n",
      "yellow = (248,161,63)\n",
      "green = (50,205,50)\n",
      "white = (255,255,255)\n",
      "blue = (30,144,255)\n",
      "      \n",
      "max_width = 0\n",
      "max_length = 0\n",
      "for name in names:\n",
      "    label = myfont.render(name.split('-----')[0],1,(0,0,0,))\n",
      "    if label.get_width() > max_width:\n",
      "        max_width = label.get_width()\n",
      "    if int(name.split('-----')[1])>max_length:\n",
      "        max_length = int(name.split('-----')[1])\n",
      "print max_length\n",
      "print \"-------------\"\n",
      "    \n",
      "for i in range(len(names)):\n",
      "    name = names[i]\n",
      "    name_2 = names_2[i]\n",
      "    \n",
      "    label = myfont.render(name.split('-----')[0],1,(0,0,0,))\n",
      "    length = name_5[i]\n",
      "    label_1 = myfont.render(length,1,(0,0,0))\n",
      "    length_1 = name.split('-----')[1]\n",
      "    #length_fgene = name.split('-----')[2]\n",
      "    #length_no_hit = name.split('-----')[3]\n",
      "    #length_mutual_ori = name_2.split('--')[1]\n",
      "    #length_mutual_fgene = name_2.split('--')[2]\n",
      "    #length_fgene_hit_nr = name_4[i]\n",
      "    \n",
      "    #minus = abs(int(length)-(int(length_fgene)-int(length_no_hit)))\n",
      "   \n",
      "    my_X = max_width-label.get_width()\n",
      "    screen.blit(label,(my_X,Y-1))\n",
      "    #screen.blit(label_1,(max_width+int(length)*4+5,Y-1))\n",
      "    pygame.draw.rect(screen , blue , (max_width+3, Y,   int(length_1)*2,   10))\n",
      "    if int(length) != 0:\n",
      "        pygame.draw.rect(screen , yellow , (max_width+3+int(length_1)*2, Y,   int(length)*2,   10))\n",
      "    else:\n",
      "        pygame.draw.rect(screen , white , (max_width+3+int(length_1)*2, Y,   int(length)*2,   10))\n",
      "    print name.split('-----')[0]\n",
      "    print int(length_1)+int(length)\n",
      "\n",
      "\n",
      "    #pygame.draw.rect(screen , red , (max_width+3, Y,   int(length_mutual_ori)*2,   7))\n",
      "    \n",
      "    #pygame.draw.rect(screen , yellow , (max_width+3, Y,   int(length)*2,   7))\n",
      "    #pygame.draw.rect(screen , red , (max_width+3, Y,   int(length_mutual_fgene)*2,   7))\n",
      "    \n",
      "    #pygame.draw.rect(screen , green , (max_width+3+int(length_mutual_fgene)*2, Y,   int(length_fgene_hit_nr)*2,   7))\n",
      "    \n",
      "    Y = Y+12\n",
      "    \n",
      "pygame.draw.line(screen,(0,0,0),(max_width+3,95),(max_width+350*2,95),2)#\\u753b\\u76f4\\u7ebf\n",
      "      \n",
      "for i in range(0,351,50):\n",
      "    x = max_width+3+i*2\n",
      "    print x\n",
      "    pygame.draw.rect(screen , (0,0,0) , (x, 91,   1,   6))#\\u5750\\u6807\\u8f74\\u523b\\u5ea6\n",
      "    label = myfont.render(str(i), 1, (0,0,0))\n",
      "    screen.blit(label, (x-6, 79))#\\u5750\\u6807\\u6570\\u5b57\\u5fae\\u8c03\n",
      "    \n",
      "    \n",
      "'''\n",
      "Y = 1\n",
      "for i in responders:\n",
      "    myDraw(80,Y,i)\n",
      "    name = i[0]\n",
      "    label = myfont.render(name, 1, (0,0,0))\n",
      "    screen.blit(label, (0, Y))#\\u753b\\u6587\\u5b57\n",
      "    Y = Y+20\n",
      "    \n",
      "for i in partial_res:\n",
      "    myDraw(80,Y,i)\n",
      "    name = i[0]\n",
      "    label = myfont.render(name, 1, (0,0,0))\n",
      "    screen.blit(label, (0, Y))#\\u753b\\u6587\\u5b57\n",
      "    Y = Y+20\n",
      "    \n",
      "for i in vbk:\n",
      "    myDraw(80,Y,i)\n",
      "    name = i[0]\n",
      "    label = myfont.render(name, 1, (0,0,0))\n",
      "    screen.blit(label, (0, Y))#\\u753b\\u6587\\u5b57\n",
      "    Y = Y+20\n",
      "    \n",
      "    \n",
      "pygame.draw.line(screen,(0,0,0),(80,530),(1300,530))#\\u753b\\u76f4\\u7ebf\n",
      "      \n",
      "for i in range(0,380,50):\n",
      "    x = 80+i*3\n",
      "    pygame.draw.rect(screen , (0,0,0) , (x, 525,   1,   6))#\\u5750\\u6807\\u8f74\\u523b\\u5ea6\n",
      "    label = myfont.render(str(i), 1, (0,0,0))\n",
      "    screen.blit(label, (x-6, 535))#\\u5750\\u6807\\u6570\\u5b57\\u5fae\\u8c03\n",
      "      \n",
      "first_color = (255,211,199)\n",
      "second_color = (207,138,138)\n",
      "third_color = (157,78,78)\n",
      "forth_color = (106,30,30)\n",
      "fivth_color = (63,8,8)    \n",
      "      \n",
      "pygame.draw.rect(screen , first_color , (1300, 423,   20,   25))#\\u5750\\u6807\\u8f74\\u523b\\u5ea6\n",
      "pygame.draw.rect(screen , second_color , (1300, 450,   20,   25))#\\u5750\\u6807\\u8f74\\u523b\\u5ea6\n",
      "pygame.draw.rect(screen , third_color , (1300, 477,   20,   25))#\\u5750\\u6807\\u8f74\\u523b\\u5ea6\n",
      "pygame.draw.rect(screen , forth_color , (1300, 504,   20,   25))#\\u5750\\u6807\\u8f74\\u523b\\u5ea6\n",
      "pygame.draw.rect(screen , fivth_color , (1300, 531,   20,   25))#\\u5750\\u6807\\u8f74\\u523b\\u5ea6\n",
      "'''\n",
      "pygame.display.flip()\n",
      "#pygame.image.save(screen,'temp.png')\n",
      "      \n",
      "while True:\n",
      "    for event in pygame.event.get():\n",
      "        if event.type == QUIT:\n",
      "            pygame.quit()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "SyntaxError",
       "evalue": "invalid syntax (<ipython-input-6-86be93b64a42>, line 3)",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-6-86be93b64a42>\"\u001b[1;36m, line \u001b[1;32m3\u001b[0m\n\u001b[1;33m    iimport os, sys\u001b[0m\n\u001b[1;37m             ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os, sys\n",
      "import random\n",
      "import pygame\n",
      "from pygame.locals import *\n",
      "      \n",
      "pygame.init()\n",
      "APPLICATION_x_size = 1500\n",
      "APPLICATION_y_size = 2000\n",
      "screen = pygame.display.set_mode((APPLICATION_x_size, APPLICATION_y_size))\n",
      "pygame.display.set_caption('My Plotting Surface')\n",
      "      \n",
      "mSurface = pygame.Surface(screen.get_size())\n",
      "mSurface.fill((255, 255, 255))\n",
      "screen.blit(mSurface, (0, 0))\n",
      "                \n",
      "myfont = pygame.font.SysFont(\"monospace\", 12)  \n",
      "      \n",
      "init_y = 100\n",
      "init_x = 50\n",
      "purple = (129,93,141)\n",
      "red = (191,69,61)\n",
      "yellow = (248,161,63)\n",
      "green = (50,205,50)\n",
      "white = (255,255,255)\n",
      "blue = (30,144,255)\n",
      "\n",
      "\n",
      "content_11 = open(\"/home/wenlei/LUO/Simon/results/gene_density\").read().splitlines()\n",
      "labels = [content_11[i] for i in range(0,len(content_11),4)]\n",
      "gene_densitys = [float(content_11[i]) for i in range(3,len(content_11),4)]\n",
      "\n",
      "base = 0\n",
      "for label in labels:\n",
      "    label_length = myfont.render(label,1,(0,0,0,)).get_width()\n",
      "    if label_length>base:\n",
      "        base = label_length\n",
      "\n",
      "\n",
      "\n",
      "for i in range(len(labels)):\n",
      "    rendering_label = myfont.render(labels[i],1,(0,0,0))\n",
      "    rendering_label_x = init_x+base-rendering_label.get_width()\n",
      "    screen.blit(rendering_label,(rendering_label_x,init_y))\n",
      "    pygame.draw.rect(screen , blue , (init_x+base+3, init_y, int(gene_densitys[i]*300*1000), 10))\n",
      "    init_y+=12\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "pygame.draw.line(screen,(0,0,0),(init_x+base+3,95),(base+355*2,95),2)\n",
      "\n",
      "tick = 0.0\n",
      "myfont = pygame.font.SysFont(\"monospace\", 10)  \n",
      "for i in range(18):\n",
      "    x = init_x+base+3+i*30\n",
      "    pygame.draw.rect(screen , (0,0,0) , (x, 91,   1,   6))\n",
      "    tick += 0.1\n",
      "    tick_label = myfont.render(str(tick), 1, (0,0,0))\n",
      "    screen.blit(tick_label, (x-7, 81))\n",
      "    \n",
      "\n",
      "\n",
      "\n",
      "pygame.display.flip()\n",
      "pygame.image.save(screen,'temp.png')\n",
      "      \n",
      "while True:\n",
      "    for event in pygame.event.get():\n",
      "        if event.type == QUIT:\n",
      "            pygame.quit()\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "error",
       "evalue": "video system not initialized",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31merror\u001b[0m                                     Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-1-29edcc846dbe>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     67\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m \u001b[1;32mwhile\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 69\u001b[1;33m     \u001b[1;32mfor\u001b[0m \u001b[0mevent\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mpygame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     70\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mevent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mQUIT\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     71\u001b[0m             \u001b[0mpygame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mquit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31merror\u001b[0m: video system not initialized"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from Bio import SeqIO\n",
      "import os\n",
      "\n",
      "mdir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output/\"\n",
      "mdir_1 = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_1/\"\n",
      "\n",
      "\n",
      "seqs = SeqIO.parse(\"/home/wenlei/accession_out_file\",'fasta')\n",
      "\n",
      "mdeslist = []\n",
      "for seq in seqs:\n",
      "    mdes = seq.description.split('>gi')[0]\n",
      "    mdeslist.append(mdes)\n",
      "#print mdeslist\n",
      "    \n",
      "    \n",
      "mfiles = os.listdir(mdir)\n",
      "for mfile in mfiles:\n",
      "    contents = open(mdir+mfile).read().splitlines()\n",
      "    new_contents = []\n",
      "    for content in contents:\n",
      "        acc = content.split('\\t')[1]\n",
      "        pre_gene = content.split('\\t')[0]\n",
      "        for i in mdeslist:\n",
      "            if acc in i:\n",
      "                new_content = content.split('\\t')\n",
      "                new_content[1] = i.split('|')[-1]\n",
      "                new_contents.append('\\t'.join(new_content))\n",
      "    print mfile\n",
      "    #out_file = open(mdir_1+mfile,'w')\n",
      "    #out_file.write('\\n'.join(new_contents))\n",
      "    #out_file.close()\n",
      "        \n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Muromegalovirus G4\n",
        "Caviid herpesvirus 2 strain 21222\n",
        "Murid herpesvirus 1"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cercopithecine herpesvirus 5 strain Colburn\n",
        "Cercopithecine herpesvirus 5 isolate OCOM4-52-duplicate"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Murid herpesvirus 8 isolate Berlin\n",
        "Panine herpesvirus 2 strain Heberling\n",
        "Panine herpesvirus 2 strain Heberling-duplicate-5\n",
        "Cercopithecine herpesvirus 5 strain 2715"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cercopithecine herpesvirus 5 isolate OCOM4-52"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cercopithecine herpesvirus 8 isolate CMV 180.92\n",
        "Murid herpesvirus 1 strain NO7"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Murid herpesvirus 1 strain C4B"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Caviid herpesvirus 2 strain 21222-duplicate-71\n",
        "Porcine cytomegalovirus strain BJ09-duplicate-79\n",
        "Saimiriine herpesvirus 4 strain SqSHV\n",
        "Murine cytomegalovirus strain K181"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Muromegalovirus C4A\n",
        "Rhesus cytomegalovirus strain 68-1"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Caviid herpesvirus 2 DNA\n",
        "Cercopithecine herpesvirus 5 strain 2715-duplicate-45"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Muromegalovirus WP15B\n",
        "Aotine herpesvirus 1 strain S34E-duplicate-46\n",
        "Drill monkey cytomegalovirus isolate OCOM6-2"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cynomolgus macaque cytomegalovirus strain Ottawa\n",
        "Murid herpesvirus 8 isolate England\n",
        "Rat cytomegalovirus ALL-03 isolate Malaysian"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Macacine herpesvirus 3"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Tupaiid herpesvirus 1\n",
        "Saimiriine herpesvirus 4 strain SqSHV-duplicate-47\n",
        "Ostreid herpesvirus 1-duplicate-78\n",
        "Murid herpesvirus 1 strain N1"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Murid herpesvirus 1 strain Smith"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Aotine herpesvirus 1 strain S34E\n",
        "Rat cytomegalovirus Maastricht\n",
        "Porcine cytomegalovirus strain BJ09\n",
        "Macacine herpesvirus 3 strain 68-1 BAC"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Ostreid herpesvirus 1\n",
        "Tupaia herpesvirus strain 2\n",
        "Murid herpesvirus 1 strain AA18d"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Murid herpesvirus 1 strain C4D\n",
        "Murid herpesvirus 8"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Bat betaherpesvirus B7D8\n",
        "Murid herpesvirus 2\n",
        "Caviid herpesvirus 2 str. CIDMTR complete genome.\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "a = [1,2,3]\n",
      "b = a\n",
      "b[0] = 2\n",
      "print b\n",
      "print a\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[2, 2, 3]\n",
        "[2, 2, 3]\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from Bio.Blast import NCBIXML\n",
      "      \n",
      "'''\n",
      "blast xml results merger script.\n",
      "It merges all overlapped HSPs in to one big HSP\n",
      "'''\n",
      "def mergeTuples(l):    \n",
      "    tmp = []\n",
      "    for i in l:\n",
      "        tmp.extend(list(i))\n",
      "    tmp.sort()\n",
      "    return (tmp[0],tmp[3])\n",
      "      \n",
      "def isOverlap(t1,t2):\n",
      "    tmp = [t1,t2]\n",
      "    tmp.sort()\n",
      "    if tmp[1][0]<=tmp[0][1]:\n",
      "        return True\n",
      "        \n",
      "    elif tmp[1][0]-tmp[0][1]<19:\n",
      "        return True\n",
      "    else:\n",
      "        return False\n",
      "    \n",
      "def cmpToList(mTuple,mList):\n",
      "    for i in range(len(mList)):           \n",
      "        mList.sort()\n",
      "        print mList\n",
      "        tmp_list = [mList[0]]\n",
      "        \n",
      "        for i in mList:\n",
      "            mark = cmpToList(i,tmp_list)\n",
      "            if mark != None:\n",
      "                tmp_list[mark] = mergeTuples([i,tmp_list[mark]])\n",
      "            else:\n",
      "                tmp_list.append(i)\n",
      "        print tmp_list"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from Bio import SeqIO\n",
      "import os\n",
      "\n",
      "\n",
      "def get_positions(filename):\n",
      "    mdict = {}\n",
      "    seqs = SeqIO.parse(mdir_fgene+filename,'fasta')\n",
      "    for seq in seqs:\n",
      "        header_items = seq.description.split(\"_\")\n",
      "        name = '_'.join(header_items[0:2])\n",
      "        strand_symbol = header_items[-1]\n",
      "        if strand_symbol == '+':\n",
      "            strand = 1\n",
      "        else:\n",
      "            strand = -1\n",
      "        mdict[name] = (int(header_items[2])-1,int(header_items[4]),strand)\n",
      "        \n",
      "    return mdict\n",
      "\n",
      "def isOverlap(t1,t2):\n",
      "    tmp = [t1,t2]\n",
      "    tmp.sort()\n",
      "    if tmp[1][0]<tmp[0][1]:\n",
      "        return True\n",
      "    else:    \n",
      "        return False\n",
      "    \n",
      "def overlapCount(t1,t2):\n",
      "    tmp = [t1,t2]\n",
      "    tmp.sort()\n",
      "    if tmp[1][1]<tmp[0][1]:\n",
      "        count = tmp[1][1]-tmp[1][0]\n",
      "    else:\n",
      "        count = tmp[0][1]-tmp[1][0]\n",
      "    return count\n",
      "    \n",
      "\n",
      "def cmp_positions(fgene,origins):\n",
      "    \n",
      "    \n",
      "    total_length = 0\n",
      "    for origin in origins:\n",
      "        pos_origin = (origin.start.position,origin.end.position,origin.strand)\n",
      "        if fgene[2] == pos_origin[2] and isOverlap(fgene[0:2],pos_origin[0:2]):\n",
      "            overlap_length = overlapCount(fgene[0:2],pos_origin[0:2])\n",
      "            total_length += overlap_length\n",
      "            \n",
      "    return total_length\n",
      "        \n",
      "\n",
      "                \n",
      "    \n",
      "\n",
      "mdir = \"/home/wenlei/LUO/Simon/non-humans_gb/\"\n",
      "mdir_fgene = \"/home/wenlei/LUO/Simon/2/\"\n",
      "mout_dir = \"/home/wenlei/LUO/Simon/overlap_results/\"\n",
      "\n",
      "\n",
      "\n",
      "for mfile in os.listdir(mdir):\n",
      "    mdict = get_positions(mfile)\n",
      "    cds_locs = []\n",
      "    mdict_2 = {}\n",
      "\n",
      "    record = SeqIO.read(mdir+mfile,'genbank')\n",
      "    print mfile\n",
      "    for feature in record.features:\n",
      "        mlocation = feature.location\n",
      "        if feature.type == 'CDS':\n",
      "            cds_loc = feature.location.parts\n",
      "            cds_locs.append(cds_loc)\n",
      "            \n",
      "    for (k,v) in mdict.items():\n",
      "        last_list = [v]\n",
      "        for item in cds_locs:\n",
      "            moverlap_length = cmp_positions(v,item)\n",
      "            if  moverlap_length > 0:\n",
      "                tmplist = [(origin.start.position,origin.end.position,origin.strand) for origin in item]\n",
      "                tmplist.append(moverlap_length)\n",
      "                last_list.append(tmplist)\n",
      "\n",
      "        if len(last_list) != 1:\n",
      "            mdict_2[k] = last_list\n",
      "    #print mdict_2\n",
      "    \n",
      "    key_list = []\n",
      "    writing_list = []\n",
      "    \n",
      "    for (k,v) in mdict_2.items():\n",
      "        if len(v) == 2 and v[0] in [ kk[0:3] for kk in v[1][0:-1]]:\n",
      "            #print \"right\"\n",
      "            pass\n",
      "        else:\n",
      "            key_list.append(int(k.split('_')[-1]))\n",
      "            \n",
      "    key_list.sort()\n",
      "    \n",
      "    for mkey in key_list:\n",
      "        mkey = \"GENE_\"+str(mkey)\n",
      "        writing_list.append(mkey)\n",
      "        mvalue = mdict_2[mkey]\n",
      "        writing_list.append(str(mvalue[0]))\n",
      "        for jjj in mvalue[1::]:\n",
      "            writing_list.append('-'.join([str(kkk) for kkk in jjj]))\n",
      "            \n",
      "            \n",
      "    out_content = '\\n'.join(writing_list)\n",
      "    mwriting_handle = open(mout_dir+mfile,'w')\n",
      "    mwriting_handle.write(out_content)\n",
      "    mwriting_handle.close()\n",
      "        \n",
      "    \n",
      "\n",
      "\n",
      "                \n",
      "            \n",
      "        \n",
      "\n",
      "\n",
      "    \n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "SyntaxError",
       "evalue": "invalid syntax (<ipython-input-5-a569a375a2c4>, line 1)",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-5-a569a375a2c4>\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    ifrom Bio import SeqIO\u001b[0m\n\u001b[1;37m            ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 89
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import vcf\n",
      "\n",
      "\n",
      "def indel_replace(genotype):\n",
      "    \n",
      "\n",
      "\n",
      "vcf_reader = vcf.Reader(open(\"/home/wenlei/filtered_1.207320000-207500000.ALL.chr1.phase3_shapeit2_mvncall_integrated_v5a.20130502.genotypes.vcf\",'r'))\n",
      "samples = vcf_reader.samples\n",
      "\n",
      "dicts = []\n",
      "for record in vcf_reader:    \n",
      "    if record.ID == 'rs2564978' or record.ID == 'rs150046210' :\n",
      "        print record.ID\n",
      "        mdict = {}\n",
      "        for sample in record.samples:\n",
      "            sample_name = sample.sample\n",
      "            sample_gt = sample.gt_bases.replace('|',' ')\n",
      "            mdict[sample_name] = sample_gt\n",
      "        dicts.append(mdict)\n",
      "count = 1        \n",
      "for sample in samples:\n",
      "    output_list = []\n",
      "    output_list.append(sample)\n",
      "    for i in dicts:\n",
      "        output_list.append(i[sample])\n",
      "    \n",
      "        \n",
      "\n",
      "       \n",
      "    print \"CHB_\"+str(count)+\"\\t\"+sample+\"\\t\"+\"\\t\".join(['0','0','0','0'])+'\\t'+'\\t'.join(output_list[1:3])\n",
      "    count+=1\n",
      "        \n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "IndentationError",
       "evalue": "expected an indented block (<ipython-input-2-90b8b33fbf78>, line 8)",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-2-90b8b33fbf78>\"\u001b[1;36m, line \u001b[1;32m8\u001b[0m\n\u001b[1;33m    vcf_reader = vcf.Reader(open(\"/home/wenlei/filtered_1.207320000-207500000.ALL.chr1.phase3_shapeit2_mvncall_integrated_v5a.20130502.genotypes.vcf\",'r'))\u001b[0m\n\u001b[1;37m             ^\u001b[0m\n\u001b[1;31mIndentationError\u001b[0m\u001b[1;31m:\u001b[0m expected an indented block\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lds = open(\"/home/wenlei/tttttttttttttt\").read().splitlines()\n",
      "\n",
      "sites = []\n",
      "for ld in lds:\n",
      "    items = ld.split('\\t')\n",
      "    ld_value = items[4]\n",
      "    pos = items[1:3]\n",
      "    pos.remove('207494416')\n",
      "    if ld_value:\n",
      "        sites.append(int(pos[0]))\n",
      "#print sites\n",
      "\n",
      "indels = open(\"/home/wenlei/indel_only.recode.vcf\").read().splitlines()\n",
      "indel_sites = []\n",
      "for i in indels:\n",
      "    if not i.startswith(\"#\"):\n",
      "        position = int(i.split('\\t')[1])\n",
      "        indel_sites.append(position)\n",
      "#print indel_sites\n",
      "\n",
      "mset = list(set(sites)&set(indel_sites))\n",
      "mset.sort()\n",
      "print mset"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[207371468, 207373891, 207375642, 207386752, 207408911, 207414375, 207415871, 207419267, 207434839, 207451139, 207457875, 207463746, 207474357, 207474395, 207478250, 207484091, 207489003, 207494221, 207494484, 207498679, 207499365]\n"
       ]
      }
     ],
     "prompt_number": 37
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os\n",
      "\n",
      "basedir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_with_homogrous_protein/\"\n",
      "outputdir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_annotated/\"\n",
      "mfiles = os.listdir(basedir)\n",
      "\n",
      "for mfile in mfiles:\n",
      "    gene_set = set()\n",
      "    gene_dict = {}\n",
      "    content = open(basedir+mfile).read().splitlines()\n",
      "    for line in content:\n",
      "        gene = line.split('\\t')[0]\n",
      "        gene_set.add(gene)\n",
      "    for i in gene_set:\n",
      "        mkey = int(i.split('_')[1])\n",
      "        gene_dict[mkey] = i\n",
      "    m_dict = sorted(gene_dict.iteritems(),key = lambda d:d[0])\n",
      "    output_content = [i[1] for i in m_dict]\n",
      "    output_file = open(outputdir+mfile,'w')\n",
      "    output_file.write('\\n'.join(output_content))\n",
      "    output_file.close()\n",
      "\n",
      "        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os\n",
      "\n",
      "overlap_dir = \"/home/wenlei/LUO/Simon/overlap_results/\"\n",
      "genes_dir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_annotated/\"\n",
      "filter_dir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/out_put_filter/\"\n",
      "filenames = os.listdir(genes_dir)\n",
      "for filename in filenames:\n",
      "    mdict = {}\n",
      "    writting_content = []\n",
      "    print filename\n",
      "    writting_content.append(filename)\n",
      "    genes = open(genes_dir+filename).read().splitlines()\n",
      "    genes = [i.split(\"_\")[1] for i in genes]\n",
      "    \n",
      "    overlaps = open(overlap_dir+filename).read().split(\"GENE_\")[1::]\n",
      "    #print overlaps\n",
      "    for overlap in overlaps:\n",
      "        num = overlap.split('\\n')[0]\n",
      "        mdict[num] = overlap\n",
      "    for gene in genes:\n",
      "        if gene in mdict.keys():\n",
      "            pass\n",
      "            print mdict[gene]\n",
      "            writting_content.append(mdict[gene])\n",
      "        else:\n",
      "            pass\n",
      "            #print gene\n",
      "    print '='*25\n",
      "    \n",
      "    writting_handle = open(filter_dir+filename,'w')\n",
      "    writting_handle.write('\\n'.join(writting_content))\n",
      "    writting_handle.close()\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "ImportError",
       "evalue": "No module named ios",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-1-78685da34587>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mios\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0moverlap_dir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"/home/wenlei/LUO/Simon/overlap_results/\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mgenes_dir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_annotated/\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mfilter_dir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"/home/wenlei/LUO/Simon/hits_dir/fgenev/out_put_filter/\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mImportError\u001b[0m: No module named ios"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os\n",
      "\n",
      "overlap_dir = \"/home/wenlei/LUO/Simon/overlap_results/\"\n",
      "genes_dir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_annotated/\"\n",
      "filter_dir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/out_put_filter/\"\n",
      "output_dir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_tmp/\"\n",
      "\n",
      "filenames = os.listdir(filter_dir)\n",
      "\n",
      "for filename in filenames:\n",
      "    genes_read_handle = open(genes_dir+filename)\n",
      "    genes_write_handle = open(output_dir+filename,'w')\n",
      "    genes = genes_read_handle.read().splitlines()\n",
      "    genes_dict = {}\n",
      "    for gene in genes:\n",
      "        gene_num = gene.split('_')[1]\n",
      "        genes_dict[int(gene_num)] = gene\n",
      "    print filename\n",
      "    read_handle = open(filter_dir+filename)\n",
      "    content = read_handle.read().splitlines()\n",
      "    if content[0][0] in [str(i) for i in range(0,10)]:\n",
      "        for line in content:\n",
      "            del genes_dict[int(line)]\n",
      "    genes_dict = sorted(genes_dict.iteritems(),key = lambda d:d[0])\n",
      "    print len([i[1] for i in genes_dict])\n",
      "    genes_write_handle.write('\\n'.join([i[1] for i in genes_dict]))\n",
      "    genes_write_handle.close()\n",
      "    genes_read_handle.close()\n",
      "    read_handle.close()\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "IOError",
       "evalue": "[Errno 2] No such file or directory: '/home/wenlei/LUO/Simon/hits_dir/fgenev/output_tmp/Muromegalovirus G4'",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mIOError\u001b[0m                                   Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-2-d8d2bea99274>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfilename\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mfilenames\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[0mgenes_read_handle\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgenes_dir\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[0mgenes_write_handle\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput_dir\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'w'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m     \u001b[0mgenes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgenes_read_handle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplitlines\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m     \u001b[0mgenes_dict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mIOError\u001b[0m: [Errno 2] No such file or directory: '/home/wenlei/LUO/Simon/hits_dir/fgenev/output_tmp/Muromegalovirus G4'"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os\n",
      "\n",
      "genes_dir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_overlap_removed/\"\n",
      "\n",
      "protein_dir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_with_homogrous_protein/\"\n",
      "\n",
      "output_dir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_removed_with_protein/\"\n",
      "\n",
      "filenames = os.listdir(protein_dir)\n",
      "\n",
      "for filename in filenames:\n",
      "    print filename\n",
      "    mdict = {}\n",
      "    keys = open(genes_dir+filename).read().splitlines()\n",
      "    for key in keys:\n",
      "        mdict[key] = []\n",
      "    \n",
      "    \n",
      "    lines = open(protein_dir+filename).read().splitlines()\n",
      "    for line in lines:\n",
      "        items = line.split('\\t')\n",
      "        gene_name = items[0]\n",
      "        gene_product = items[1]\n",
      "        if gene_name in mdict.keys():\n",
      "            mdict[gene_name].append(gene_product)\n",
      "            \n",
      "    dict_for_sorting = {}\n",
      "    for key in keys:\n",
      "        new_key = key.split('_')[1]\n",
      "        dict_for_sorting[int(new_key)] = key\n",
      "        \n",
      "    dict_for_sorting = sorted(dict_for_sorting.iteritems(),key=lambda d:d[0])\n",
      "    \n",
      "    writting_list = []\n",
      "    for i in dict_for_sorting:\n",
      "        key = i[1]\n",
      "        product_list = mdict[key]\n",
      "        product_list.sort()\n",
      "        for i in range(len(product_list)):\n",
      "            if i == i:\n",
      "                writting_list.append(key+'\\t'+product_list[i])\n",
      "            else:\n",
      "                writting_list.append(' '*len(key)+'\\t'+product_list[i])\n",
      "        writting_list.append('\\n')\n",
      "    for i in writting_list:\n",
      "        print i\n",
      "    \n",
      "    writting_handle = open(output_dir+filename,'w')\n",
      "    writting_handle.write('\\n'.join(writting_list))\n",
      "    writting_handle.close()\n",
      "        \n",
      "            \n",
      "\n",
      "        \n",
      "\n",
      "\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "ImportError",
       "evalue": "No module named ios",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-2-ef693c63fb9f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mios\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mgenes_dir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_overlap_removed/\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mprotein_dir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_with_homogrous_protein/\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mImportError\u001b[0m: No module named ios"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os,time\n",
      "from Bio import SeqIO\n",
      "from Bio.SeqRecord import SeqRecord\n",
      "\n",
      "ori_dir = \"/home/wenlei/LUO/Simon/cmv_cdss/\"\n",
      "new_gene_dir = \"/home/wenlei/LUO/Simon/hits_dir/fgenev/output_removed_with_protein_fullname/\"\n",
      "raw_fasta_dir = \"/home/wenlei/LUO/Simon/raw_fasta/\"\n",
      "\n",
      "filenames_new = os.listdir(new_gene_dir)\n",
      "\n",
      "st_dict = {'-':'-1','+':'1'}\n",
      "\n",
      "for filename in filenames_new:\n",
      "    print filename\n",
      "    total_seqs = []\n",
      "    \n",
      "    content_1 = SeqIO.parse(ori_dir+filename,'fasta')#original annotation\n",
      "    for i in content_1:\n",
      "        seq_description = i.description.split('|')\n",
      "        total_seqs.append(i)\n",
      "    gn = seq_description[11].strip()#gn\n",
      "    tx = seq_description[-2].strip()#taxonomy\n",
      "\n",
      "    \n",
      "    protein_dict = {}\n",
      "    content_2 = SeqIO.parse(raw_fasta_dir+filename,'fasta')#raw fasta for protein seq\n",
      "    for j in content_2:\n",
      "        raw_description = j.description\n",
      "        protein = j.seq\n",
      "        protein_dict[raw_description] = protein\n",
      "\n",
      "    \n",
      "    content = open(new_gene_dir+filename).read().splitlines()\n",
      "    content = [i for i in content if i.strip()]\n",
      "    time_base = int(str(time.time())[3:10])\n",
      "    \n",
      "    for ii in content:\n",
      "        gene_name = ii.split('\\t')[0]\n",
      "        product = ii.split('\\t')[1]\n",
      "        lo = (int(gene_name.split('_')[2]),int(gene_name.split('_')[4]))#location\n",
      "    \n",
      "    \n",
      "    \n",
      "    for ii in content:\n",
      "        gene_name = ii.split('\\t')[0]\n",
      "        product = ii.split('\\t')[1]+'_fgenesv'\n",
      "        lo = gene_name.split('_')[2]+'-'+gene_name.split('_')[4]#location\n",
      "        st = st_dict[gene_name.split('_')[-1]]#strand\n",
      "        product_seq = protein_dict[gene_name]\n",
      "        time_base += 1\n",
      "        gi = '000'+str(time_base)\n",
      "        gb = \"IOV\"+str(time_base)\n",
      "        new_seq_description = 'gi|'+gi+'|gb|'+gb+'|or|1'+'|st|'+st+'|lo|'+lo+'|gn|'+gn+'|tx|'+tx+'|'+product\n",
      "        mseqrecord = SeqRecord(product_seq,id=new_seq_description,description='')\n",
      "        total_seqs.append(mseqrecord)\n",
      "    print total_seqs\n",
      "    SeqIO.write(total_seqs,'/home/wenlei/tmp/'+filename,'fasta')\n",
      "    \n",
      "    \n",
      "    \n",
      "        \n",
      "        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "ImportError",
       "evalue": "No module named ois",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-3-5f12675adfae>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mois\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mBio\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mSeqIO\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mBio\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSeqRecord\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mSeqRecord\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mori_dir\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"/home/wenlei/LUO/Simon/cmv_cdss/\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mImportError\u001b[0m: No module named ois"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from Bio import SeqIO\n",
      "import os,re\n",
      "\n",
      "dir_1 = \"/home/wenlei/tmp/\"\n",
      "dir_2 = \"/home/wenlei/tmp2/\"\n",
      "\n",
      "filenames = os.listdir(dir_1)\n",
      "for filename in filenames:\n",
      "    print filename\n",
      "    seq_dict = {}\n",
      "    seqs = SeqIO.parse(dir_1+filename,'fasta')\n",
      "    for seq in seqs:\n",
      "        description = seq.description\n",
      "        lo = description.split('|')[9]\n",
      "        lo = (int(lo.split('-')[0]),int(lo.split('-')[1]))\n",
      "        seq_dict[lo] = seq\n",
      "    lo_list = seq_dict.keys()\n",
      "    lo_list.sort()\n",
      "    for i in range(len(lo_list)):\n",
      "        mkey = lo_list[i]\n",
      "        replacement = seq_dict[mkey].description\n",
      "        replacement = re.sub('or\\|\\d{1,3}','or|'+str(i+1),replacement)\n",
      "        seq_dict[mkey].id = replacement\n",
      "        seq_dict[mkey].description = ''\n",
      "        \n",
      "    output_list = []\n",
      "    \n",
      "    for i in lo_list:\n",
      "        output_list.append(seq_dict[i])\n",
      "\n",
      "    SeqIO.write(output_list,dir_2+filename,'fasta')\n",
      "    \n",
      "        \n",
      "\n",
      "        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Muromegalovirus G4\n",
        "Caviid herpesvirus 2 strain 21222\n",
        "Murid herpesvirus 1\n",
        "Cercopithecine herpesvirus 5 strain Colburn\n",
        "Murid herpesvirus 8 isolate Berlin\n",
        "Panine herpesvirus 2 strain Heberling\n",
        "Cercopithecine herpesvirus 5 strain 2715\n",
        "Cercopithecine herpesvirus 5 isolate OCOM4-52\n",
        "Cercopithecine herpesvirus 8 isolate CMV 180.92\n",
        "Murid herpesvirus 1 strain NO7"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Murid herpesvirus 1 strain C4B\n",
        "Saimiriine herpesvirus 4 strain SqSHV\n",
        "Murine cytomegalovirus strain K181\n",
        "Muromegalovirus C4A\n",
        "Rhesus cytomegalovirus strain 68-1\n",
        "Caviid herpesvirus 2 DNA\n",
        "Muromegalovirus WP15B\n",
        "Drill monkey cytomegalovirus isolate OCOM6-2\n",
        "Cynomolgus macaque cytomegalovirus strain Ottawa"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Murid herpesvirus 8 isolate England\n",
        "Rat cytomegalovirus ALL-03 isolate Malaysian\n",
        "Macacine herpesvirus 3\n",
        "Tupaiid herpesvirus 1\n",
        "Murid herpesvirus 1 strain N1\n",
        "Murid herpesvirus 1 strain Smith\n",
        "Aotine herpesvirus 1 strain S34E\n",
        "Rat cytomegalovirus Maastricht\n",
        "Porcine cytomegalovirus strain BJ09"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Macacine herpesvirus 3 strain 68-1 BAC\n",
        "Ostreid herpesvirus 1\n",
        "Tupaia herpesvirus strain 2\n",
        "Murid herpesvirus 1 strain AA18d\n",
        "Murid herpesvirus 1 strain C4D\n",
        "Murid herpesvirus 8\n",
        "Bat betaherpesvirus B7D8\n",
        "Murid herpesvirus 2\n",
        "Caviid herpesvirus 2 str. CIDMTR complete genome.\n"
       ]
      }
     ],
     "prompt_number": 145
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os\n",
      "from Bio import SeqIO\n",
      "\n",
      "dir_1 = \"/home/wenlei/tmp2/\"\n",
      "sub_dict = {}\n",
      "\n",
      "filenames = os.listdir(dir_1)\n",
      "for filename in filenames:\n",
      "    records = SeqIO.parse(dir_1+filename,'fasta')\n",
      "    records = list(records)\n",
      "    for record in records:\n",
      "        if 'fgenesv' not in record.description:\n",
      "            organism = '['+ record.description.split('[')[1]\n",
      "            break\n",
      "    for record in records:\n",
      "        if 'fgenesv' in record.description:\n",
      "            replacement = record.description.replace('fgenesv','FGENESV')+' '+organism\n",
      "            #print record.description\n",
      "            #print replacement\n",
      "            sub_dict[record.description] = replacement\n",
      "            \n",
      "#print sub_dict \n",
      "keys = sub_dict.keys()\n",
      "\n",
      "read_dir = \"/home/wenlei/LUO/Simon/blast_results/\"\n",
      "write_dir = \"/home/wenlei/LUO/Simon/new_blast_results/\"\n",
      "\n",
      "for filename in os.listdir(read_dir):\n",
      "    writting_handle = open(write_dir+filename,'a')\n",
      "    count = 0\n",
      "    writting_list = []\n",
      "    print filename\n",
      "    \n",
      "    with open(read_dir+filename) as xml_lines:\n",
      "        while True:\n",
      "            line = xml_lines.readline()\n",
      "            if line:\n",
      "                count += 1\n",
      "                if 'fgenesv' in line:\n",
      "                    for key in keys:\n",
      "                        if key in line:\n",
      "                            line = line.replace(key,sub_dict[key])\n",
      "                            writting_list.append(line)\n",
      "                else:\n",
      "                    writting_list.append(line)\n",
      "                if count == 5000:\n",
      "                    writting_handle.write(''.join(writting_list))\n",
      "                    count = 0\n",
      "                    writting_list = []\n",
      "\n",
      "            else:\n",
      "                if writting_list:\n",
      "                    print len(writting_list)\n",
      "                    writting_handle.write(''.join(writting_list))\n",
      "                    count = 0\n",
      "                    writting_list = []\n",
      "                break\n",
      "    writting_handle.close()\n",
      "    break\n",
      "\n",
      "                \n",
      "            "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "ImportError",
       "evalue": "No module named iios",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-4-6d57fa6e9a87>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0miios\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mBio\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mSeqIO\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mdir_1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"/home/wenlei/tmp2/\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0msub_dict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mImportError\u001b[0m: No module named iios"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "csv = open(\"/home/wenlei/gene_content_ultimate.csv\").read().splitlines()\n",
      "csv = [i.split(',') for i in csv]\n",
      "\n",
      "for i in range(918):\n",
      "    tmp_list = []\n",
      "    core = True\n",
      "    \n",
      "    for j in csv:\n",
      "        tmp_list.append(int(j[i]))\n",
      "    for k in tmp_list:\n",
      "        if k:\n",
      "            pass\n",
      "        else:\n",
      "            core = False\n",
      "    if core:\n",
      "        print str(i+1)+' '+str(tmp_list)\n",
      "        #print i+1    \n",
      "        \n",
      "\n",
      "\n",
      "\n",
      "    \n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "5 [294, 4, 2, 2, 6, 6, 2, 6, 14, 2, 2, 2, 2, 4, 2, 2, 2, 2, 2, 2, 2, 2, 2]\n",
        "6 [294, 4, 2, 1, 4, 3, 1, 6, 7, 1, 1, 1, 2, 2, 2, 2, 1, 1, 2, 2, 2, 1, 1]\n",
        "14 [147, 2, 1, 1, 6, 3, 1, 6, 7, 1, 2, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1]\n",
        "15 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 3, 1, 1]\n",
        "16 [150, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "17 [147, 2, 1, 1, 3, 3, 1, 4, 5, 1, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1]\n",
        "19 [146, 2, 1, 1, 3, 3, 1, 3, 9, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "21 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "22 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "23 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "24 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "25 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "27 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "28 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "29 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "30 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "31 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "33 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "34 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "35 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "36 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "37 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "38 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "39 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "41 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "42 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "43 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "44 [147, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "45 [146, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "49 [147, 2, 1, 1, 3, 3, 1, 4, 5, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "51 [146, 2, 1, 1, 3, 3, 1, 3, 7, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "57 [147, 2, 1, 1, 3, 3, 1, 3, 5, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "58 [147, 2, 1, 1, 3, 3, 1, 3, 5, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
        "59 [147, 2, 1, 1, 3, 3, 1, 3, 5, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
       ]
      },
      {
       "ename": "IndexError",
       "evalue": "list index out of range",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-20-b556c446eccd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcsv\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m         \u001b[0mtmp_list\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtmp_list\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mIndexError\u001b[0m: list index out of range"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os\n",
      "\n",
      "#mdir = \"/home/wenlei/LUO/Simon/core_gene_subgraphs/\"\n",
      "mdir = \"/home/wenlei/tmp/\"\n",
      "\n",
      "filenames = os.listdir(mdir)\n",
      "filenames = [int(i) for i in filenames]\n",
      "filenames.sort()\n",
      "\n",
      "for filename in filenames:\n",
      "    print filename\n",
      "    \n",
      "    lines = open(mdir+str(filename)).read().splitlines()\n",
      "    mdict = {}\n",
      "\n",
      "    for line in lines:\n",
      "        organism = line.split('\\t')[-2].strip()\n",
      "        product = line.split('\\t')[-1].strip()\n",
      "        if organism not in mdict.keys() and \"Ostreid\" not in organism:\n",
      "            mdict[organism] = set()\n",
      "            mdict[organism].add(product)\n",
      "        else:\n",
      "            mdict[organism].add(product)\n",
      "    organisms = mdict.keys()\n",
      "    organisms.sort()\n",
      "    print len(organisms)\n",
      "    for i in organisms:\n",
      "        print '/'.join(mdict[i])\n",
      "    print \n",
      "        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "9\n",
        "9\n",
        "membrane protein UL14\n",
        "UL141/UL14/membrane glycoprotein UL141/membrane protein UL14\n",
        "membrane glycoprotein UL141/putative membrane glycoprotein UL14\n",
        "UL14_FGENESV\n",
        "membrane glycoprotein UL141/membrane protein UL14/HCMVUL14\n",
        "rh33/Rh164/Rh33/rhUL14/rh164\n",
        "membrane glycoprotein UL141/membrane protein UL14\n",
        "rh33/rh164\n",
        "membrane protein UL14\n",
        "\n",
        "74\n",
        "9\n",
        "protein UL145\n",
        "protein UL145/protein UL145_FGENESV\n",
        "protein UL145\n",
        "protein UL145\n",
        "protein UL145\n",
        "Rh162/rh162\n",
        "protein UL145\n",
        "rh162\n",
        "protein UL145\n",
        "\n",
        "77\n",
        "9\n",
        "protein RL1\n",
        "protein RL1/RL1 protein_FGENESV\n",
        "protein TRL1\n",
        "RL1 protein_FGENESV\n",
        "HCMVIRL1 = TRL1/protein RL1/RL1/RL1 protein/HCMVTRL1 = IRL1\n",
        "rhRL1/rh01/Rh01\n",
        "protein RL1\n",
        "rh01\n",
        "protein RL1\n",
        "\n",
        "78\n",
        "9\n",
        "protein UL116\n",
        "UL116/protein UL116\n",
        "putative membrane glycoprotein UL116\n",
        "protein UL116\n",
        "protein UL116/HCMVUL116\n",
        "Rh148/rhUL116/rh148\n",
        "protein UL116\n",
        "rh148\n",
        "protein UL116\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 51
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "gene_content_lines = open(\"/home/wenlei/gene_content_directly\").read().splitlines()\n",
      "\n",
      "\n",
      "lines = [i.split('\\t') for i in gene_content_lines[0:-1]]\n",
      "cmv_index = [0,2,4,7,10,15,17,18,19]\n",
      "probos_index = [1,12,13,14]\n",
      "muro_index = [3,5,6,8,9,11,16,20,21]\n",
      "\n",
      "for i in range(919):\n",
      "    tmp_list = []\n",
      "    core = True\n",
      "    \n",
      "    for index in muro_index:\n",
      "        if int(lines[index][i]):\n",
      "            tmp_list.append(int(lines[index][i]))\n",
      "        else:\n",
      "            core = False\n",
      "            break\n",
      "    if core:\n",
      "        print str(i+1)+' '+str(tmp_list)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1 [5, 15, 5, 30, 5, 5, 5, 5, 5]\n",
        "4 [3, 6, 2, 13, 3, 2, 2, 2, 2]\n",
        "5 [2, 6, 2, 14, 2, 2, 2, 2, 2]\n",
        "6 [1, 3, 1, 7, 1, 1, 1, 2, 1]\n",
        "13 [2, 6, 2, 11, 2, 2, 2, 2, 2]\n",
        "14 [1, 3, 1, 7, 1, 1, 1, 3, 1]\n",
        "15 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "17 [1, 3, 1, 9, 1, 1, 1, 1, 1]\n",
        "18 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "19 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "20 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "21 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "22 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "23 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "24 [1, 3, 1, 8, 1, 1, 1, 1, 1]\n",
        "25 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "26 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "27 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "28 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "29 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "30 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "31 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "32 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "33 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "34 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "35 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "36 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "37 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "38 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "39 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "40 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "41 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "42 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "43 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "44 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "45 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "46 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "48 [1, 3, 1, 5, 1, 1, 1, 1, 1]\n",
        "49 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "50 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "51 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "52 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "53 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "54 [1, 3, 1, 5, 1, 1, 1, 1, 1]\n",
        "55 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "56 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "57 [1, 3, 1, 5, 1, 1, 1, 1, 1]\n",
        "58 [1, 3, 1, 5, 1, 1, 1, 1, 1]\n",
        "59 [1, 3, 1, 5, 1, 1, 1, 1, 1]\n",
        "60 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "61 [1, 3, 1, 5, 1, 1, 1, 1, 1]\n",
        "65 [1, 3, 1, 6, 1, 1, 1, 1, 1]\n",
        "151 [1, 15, 2, 14, 1, 2, 2, 5, 2]\n",
        "156 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "158 [1, 3, 2, 11, 1, 2, 2, 1, 2]\n",
        "159 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "168 [3, 3, 1, 5, 3, 1, 1, 1, 1]\n",
        "169 [1, 3, 1, 9, 1, 1, 1, 1, 1]\n",
        "170 [1, 3, 1, 8, 1, 1, 1, 1, 2]\n",
        "172 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "174 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "175 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "176 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "177 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "178 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "179 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "180 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "181 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "182 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "183 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "185 [1, 3, 1, 7, 1, 1, 1, 1, 1]\n",
        "187 [1, 3, 1, 6, 1, 1, 1, 1, 1]\n",
        "195 [1, 3, 1, 4, 1, 1, 1, 1, 1]\n"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "'''\n",
      "detect classfication specific common genes\n",
      "'''\n",
      "\n",
      "\n",
      "gene_content_lines = open(\"/home/wenlei/gene_content_directly\").read().splitlines()\n",
      "\n",
      "\n",
      "lines = [i.split('\\t') for i in gene_content_lines]\n",
      "cmv_index = [0,2,4,7,10,15,17,18,19]\n",
      "probos_index = [1,12,13,14]\n",
      "muro_index = [3,5,6,8,9,11,16,20,21]\n",
      "ostreid_index = [23]\n",
      "suid_index = [22]\n",
      "hcmv_index = [0]\n",
      "\n",
      "for i in range(919):\n",
      "    tmp_list = []\n",
      "    core_specific = True\n",
      "    \n",
      "    for j in range(len(lines)):\n",
      "        tmp_list.append(int(lines[j][i]))\n",
      "        if j in hcmv_index:\n",
      "            if int(lines[j][i]) == 0:\n",
      "                core_specific = False\n",
      "        else:\n",
      "            if int(lines[j][i]) != 0:\n",
      "                core_specific = False\n",
      "\n",
      "    if core_specific:\n",
      "        print str(i+1)+' '+str(tmp_list)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "98 [150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "118 [147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "120 [146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "130 [144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "136 [143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "137 [143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "143 [139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "148 [129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "149 [101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "150 [46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "443 [2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "507 [2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "508 [2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "509 [2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "510 [2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "537 [2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "540 [2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "626 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "627 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "628 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "629 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "631 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "632 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "645 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "646 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "659 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "680 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "681 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "682 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "755 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "765 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "776 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "777 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "778 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "779 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "783 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "785 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "798 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "799 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "800 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "801 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "802 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "803 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "804 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "805 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "806 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "807 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "808 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "836 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "837 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "838 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "846 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "854 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "856 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "860 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "880 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "881 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "882 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "886 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
        "888 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"
       ]
      }
     ],
     "prompt_number": 50
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pnas = open(\"/home/wenlei/core_genes_from_pnas\").read().splitlines()\n",
      "pnas_list = []\n",
      "for i in pnas:\n",
      "    pnas_list.append(i.split(' ')[0].strip(\"UL\"))\n",
      "print pnas_list\n",
      "mine = open(\"/home/wenlei/core_genes_from_mine\").read().splitlines()\n",
      "print mine\n",
      "\n",
      "common_set = set(pnas_list)&set(mine)\n",
      "\n",
      "print set(mine)-common_set\n",
      "print set(pnas_list)-common_set"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "['44', '46', '48', '48.5', '49', '50', '51', '52', '53', '54', '55', '56', '57', '70', '71', '73', '75', '76', '77', '79', '80', '85', '86', '87', '89.1', '93', '94', '95', '98', '99', '100', '102', '104', '105', '115', '69', '47', '72', '97', '103', '114', '89.2']\n",
        "['25', '35', '80', '80a', '87', '32', '31', '98', '102', '88', '85', '73', '32', '52', '114', '69', '105', '93', '17', '38', '16', '94', '46', '104', '97', '49', '56', '100', '70', '55', '53', '14', '96', '48', '24', '76', '33', '86', '57', '37', '47']\n",
        "set(['24', '25', '38', '14', '17', '16', '33', '32', '80a', '37', '35', '88', '31', '96'])\n",
        "set(['89.2', '89.1', '77', '54', '44', '51', '50', '115', '99', '75', '72', '71', '48.5', '103', '95', '79'])\n"
       ]
      }
     ],
     "prompt_number": 61
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import urllib2\n",
      "from bs4 import BeautifulSoup\n",
      "\n",
      "\n",
      "def lott_num_to_file(file_handle,index):\n",
      "    output_file = open(file_handle,'a')\n",
      "    html_content = urllib2.urlopen('http://kaijiang.zhcw.com/zhcw/inc/3d/3d_wqhg.jsp?pageNum=1').read()\n",
      "    soup = BeautifulSoup(html_content,'html.parser')\n",
      "    tr_list = soup.find_all('tr')\n",
      "    tr_list = tr_list[2:-1]\n",
      "    for tr in tr_list[::-1]:\n",
      "        tmp_list = []\n",
      "        td_list = tr.find_all('td')\n",
      "        for td in td_list:\n",
      "            mtext = td.get_text()\n",
      "            if mtext:\n",
      "                tmp_list.append(mtext)\n",
      "        if int(tmp_list[1])>index:\n",
      "            new_index = int(tmp_list[1])\n",
      "            output_file.write('-'*10+'\\n')\n",
      "            output_file.write('\\n'.join(tmp_list))\n",
      "            output_file.write('\\n')\n",
      "    output_file.close()\n",
      "    return new_index\n",
      "            \n",
      "file_handle = '/home/wenlei/tttttttttttttttest'\n",
      "print lott_num_to_file(file_handle,330)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2015334\n"
       ]
      }
     ],
     "prompt_number": 282
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import urllib2,sys\n",
      "from bs4 import BeautifulSoup\n",
      "\n",
      "\n",
      "def lott_num_to_file(file_handle,index):\n",
      "    new_index = 0\n",
      "    output_file = open(file_handle,'a')\n",
      "    html_content = urllib2.urlopen('http://kaijiang.zhcw.com/zhcw/inc/3d/3d_wqhg.jsp?pageNum=1').read()\n",
      "    soup = BeautifulSoup(html_content,'html.parser')\n",
      "    tr_list = soup.find_all('tr')\n",
      "    tr_list = tr_list[2:-1]\n",
      "    for tr in tr_list[::-1]:\n",
      "        tmp_list = []\n",
      "        td_list = tr.find_all('td')\n",
      "        for td in td_list:\n",
      "            mtext = td.get_text()\n",
      "            if mtext:\n",
      "                tmp_list.append(mtext)\n",
      "        if int(tmp_list[1])>index:\n",
      "            new_index = int(tmp_list[1])\n",
      "            output_file.write('-'*10+'\\n')\n",
      "            output_file.write('\\n'.join(tmp_list))\n",
      "            output_file.write('\\n')\n",
      "    output_file.close()\n",
      "    return new_index\n",
      "\n",
      "def get_index():\n",
      "    index_file = open('/home/wenlei/lott_index')\n",
      "    index_num = index_file.read()\n",
      "    index_file.close()\n",
      "    return int(index_num)\n",
      "\n",
      "def write_index(index_num):\n",
      "    index_file = open('/home/wenlei/lott_index','w')\n",
      "    index_file.write(str(index_num))\n",
      "    index_file.close()\n",
      "    \n",
      "    \n",
      "def update_work():\n",
      "    file_handle = '/home/wenlei/lott'\n",
      "    new_index = lott_num_to_file(file_handle,get_index())\n",
      "    if new_index != 0:\n",
      "        write_index(new_index)\n",
      "update_work()\n",
      "\n",
      "lott_lines = open('/home/wenlei/lott').read().split('----------')\n",
      "ones = []\n",
      "twos = []\n",
      "threes = []\n",
      "sums = []\n",
      "\n",
      "\n",
      "for line in lott_lines[-3000:-100]:\n",
      "    items = line.splitlines()\n",
      "    items = [i for i in items if i != '']\n",
      "    date = items[0]\n",
      "    index = items[1]\n",
      "    one = items[2]\n",
      "    two = items[3]\n",
      "    three = items[4]\n",
      "    ones.append(one)\n",
      "    twos.append(two)\n",
      "    threes.append(three)\n",
      "    sums.append(str(int(one)+int(two)+int(three)))\n",
      "    #print date\n",
      "    \n",
      "def get_probability(l):\n",
      "    mdict = {}\n",
      "    for j in range(10):\n",
      "        mdict[j] = []\n",
      "        for i in range(len(l)-1):\n",
      "            if l[i] == str(j):\n",
      "                mdict[j].append(l[i+1])\n",
      "    out_list = []\n",
      "    for k in range(10):\n",
      "        tmp_list = []\n",
      "        for i in range(10):\n",
      "            probability = mdict[k].count(str(i))/float(len(mdict[k]))\n",
      "            tmp_list.append(probability)\n",
      "        out_list.append(tmp_list) \n",
      "    return out_list\n",
      "\n",
      "def get_sum_probability(l):\n",
      "    mdict = {}\n",
      "    for j in range(28):\n",
      "        mdict[j] = []\n",
      "        for i in range(len(l)-1):\n",
      "            if l[i] == str(j):\n",
      "                mdict[j].append(l[i+1])\n",
      "    out_list = []\n",
      "    for k in range(28):\n",
      "        tmp_list = []\n",
      "        for i in range(28):\n",
      "            probability = mdict[k].count(str(i))/float(len(mdict[k]))\n",
      "            tmp_list.append(probability)\n",
      "        out_list.append(tmp_list) \n",
      "    return out_list\n",
      "\n",
      "def perm(l,n):\n",
      "    output_list = []\n",
      "    if n <= 1:\n",
      "        for i in l:\n",
      "            output_list.append([i])\n",
      "        return output_list\n",
      "    else:\n",
      "        for k in l:\n",
      "            for j in perm(l,n-1):\n",
      "                j.append(k)\n",
      "                output_list.append(j)\n",
      "        return output_list\n",
      "\n",
      "def sum_probability_model(l):\n",
      "    output_dict = {}\n",
      "    for i in range(28):\n",
      "        output_dict[i] = 0\n",
      "        for j in l:\n",
      "            if int(j[0])+int(j[1])+int(j[2]) == i:\n",
      "                output_dict[i] += 1\n",
      "        output_dict[i] = output_dict[i]/1000.0\n",
      "    return output_dict\n",
      "                \n",
      "numbers = perm([0,1,2,3,4,5,6,7,8,9],3)\n",
      "        \n",
      "sum_model = sum_probability_model(numbers)\n",
      "one_prob_matrix = get_probability(ones)\n",
      "two_prob_matrix = get_probability(twos)\n",
      "three_prob_matrix = get_probability(threes)\n",
      "\n",
      "   \n",
      "stat = [] \n",
      "\n",
      "ones = []\n",
      "twos = []\n",
      "threes = []\n",
      "\n",
      "for line in lott_lines[-100::]:\n",
      "    items = line.splitlines()\n",
      "    items = [i for i in items if i != '']\n",
      "    date = items[0]\n",
      "    index = items[1]\n",
      "    one = items[2]\n",
      "    two = items[3]\n",
      "    three = items[4]\n",
      "    ones.append(one)\n",
      "    twos.append(two)\n",
      "    threes.append(three)\n",
      "    \n",
      "for jj in range(len(ones)-1): \n",
      "    #print ones[jj]+twos[jj]+threes[jj]\n",
      "    dict_2 = {}\n",
      "    for i in numbers:\n",
      "        first = one_prob_matrix[int(ones[jj])][i[0]]\n",
      "        second = two_prob_matrix[int(twos[jj])][i[1]]\n",
      "        third = three_prob_matrix[int(threes[jj])][i[2]]\n",
      "        mod_weight = (i[0]+i[1]+i[2])%15\n",
      "        dict_2[str(i[0])+str(i[1])+str(i[2])] = first*second*third#*sum_model[i[0]+i[1]+i[2]]*mod_weight\n",
      "\n",
      "    values = dict_2.values()\n",
      "    values.sort()\n",
      "    stat.append(values.index(dict_2[ones[jj+1]+twos[jj+1]+threes[jj+1]]))\n",
      "\n",
      "\n",
      "def print_stat():\n",
      "    c1 = 0\n",
      "    c2 = 0\n",
      "    c3 = 0\n",
      "    c4 = 0\n",
      "    c5 = 0\n",
      "    c6 = 0\n",
      "    c7 = 0\n",
      "    c8 = 0\n",
      "    c9 = 0\n",
      "    c10 = 0\n",
      "\n",
      "    for i in stat:\n",
      "        if i<100:\n",
      "            c1 += 1\n",
      "        elif i>100 and i<200:\n",
      "            c2 += 1\n",
      "        elif i>200 and i<300:\n",
      "            c3 += 1\n",
      "        elif i>300 and i<400:\n",
      "            c4 += 1\n",
      "        elif i>400 and i<500:\n",
      "            c5 += 1\n",
      "        elif i>500 and i<600:\n",
      "            c6 += 1\n",
      "        elif i>600 and i<700:\n",
      "            c7 += 1\n",
      "        elif i>700 and i<800:\n",
      "            c8 += 1\n",
      "        elif i>800 and i<900:\n",
      "            c9 += 1\n",
      "        elif i>900 and i<1000:\n",
      "            c10 += 1\n",
      "    print c1\n",
      "    print c2\n",
      "    print c3\n",
      "    print c4\n",
      "    print c5\n",
      "    print c6\n",
      "    print c7\n",
      "    print c8\n",
      "    print c9\n",
      "    print c10\n",
      "    \n",
      "#print_stat()\n",
      "def tmp():\n",
      "    count = 0\n",
      "    for i in stat:\n",
      "        if i>800:\n",
      "            print count\n",
      "            count = 0\n",
      "        else:\n",
      "            count += 1\n",
      "#tmp()\n",
      "print stat"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[177, 316, 53, 367, 989, 235, 509, 670, 963, 666, 460, 277, 161, 577, 360, 959, 983, 266, 770, 34, 216, 150, 570, 879, 218, 109, 782, 428, 805, 125, 724, 115, 265, 895, 359, 860, 611, 669, 938, 948, 399, 616, 733, 182, 318, 74, 681, 335, 398, 735, 621, 970, 652, 179, 842, 189, 188, 21, 293, 662, 734, 818, 216, 125, 525, 683, 850, 737, 493, 998, 684, 260, 537, 197, 577, 319, 271, 110, 947, 958, 971, 986, 6, 136, 283, 978, 221, 835, 254, 303, 177, 911, 816, 168, 604, 984, 961, 225, 476]\n"
       ]
      }
     ],
     "prompt_number": 377
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "l = [1,2,3,4,5,6]\n",
      "\n",
      "print l[-3::]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[4, 5, 6]\n"
       ]
      }
     ],
     "prompt_number": 308
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print (23+1)%15"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "9\n"
       ]
      }
     ],
     "prompt_number": 362
    }
   ],
   "metadata": {}
  }
 ]
}