Analysis(Non-Strict).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nAnalysis on how the ownership of a token holder change\\n'"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Analysis on how the ownership of a token holder change\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import urllib3\n",
    "#disable the annoying security warnings\n",
    "urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n",
    "import re\n",
    "import pandas\n",
    "import numpy\n",
    "from time import sleep\n",
    "import time\n",
    "from multiprocessing.dummy import Pool as ThreadPool \n",
    "import itertools\n",
    "#avoid counting duplicate transactions\n",
    "from sets import Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting 3 threads...\n",
      "<iframe frameborder=\"0\" id=\"tokentxnsiframe\" onload=\"javascript:resizeIframe(this, 0);\" scrolling=\"yes\" src=\"\" width=\"100%\"></iframe>\n",
      "processing page 0 of owner: 0x000ea6df3b680cb96f66a42041235d1ed776ef4f\n",
      "1.93633699417 second for each request (Owner: 0x000ea6df3b680cb96f66a42041235d1ed776ef4f )\n",
      "processing page 1 of owner: 0x000ea6df3b680cb96f66a42041235d1ed776ef4f\n",
      "0.130918979645 second for each request (Owner: 0x000ea6df3b680cb96f66a42041235d1ed776ef4f )\n"
     ]
    }
   ],
   "source": [
    "#the base url for etherscan\n",
    "baseUrl='https://etherscan.io/'\n",
    "#the connection pool, making 3 connections\n",
    "pool=urllib3.PoolManager(3)\n",
    "#number of threads\n",
    "numthread=3\n",
    "def html_convert_top100(tokenid, classname):\n",
    "    '''\n",
    "    Get the top 100 owners of the coin\n",
    "    '''\n",
    "    global baseUrl\n",
    "    global pool\n",
    "    funUrl_chart='token/tokenholderchart/'\n",
    "    r=pool.request('GET',baseUrl+funUrl_chart+tokenid)\n",
    "    html=r.data\n",
    "    soup = BeautifulSoup(html)\n",
    "    table = soup.find(\"table\", attrs=classname)\n",
    "    headings = [th.get_text() for th in table.find(\"tr\").find_all(\"th\")]\n",
    "    for idx in range(len(headings)):\n",
    "        headings[idx]=str(headings[idx])\n",
    "    datasets = []\n",
    "    for row in table.find_all(\"tr\")[1:]:\n",
    "        dataset = (td.get_text() for td in row.find_all(\"td\"))\n",
    "        ls=list(dataset)\n",
    "        datasets.append(ls)\n",
    "\n",
    "    '''\n",
    "    Clean the data\n",
    "    '''\n",
    "    for idx in range(len(datasets)):\n",
    "        for idxx in range(len(datasets[idx])):\n",
    "            tmp=re.sub(r'\\([^)]*\\)', '', datasets[idx][idxx])\n",
    "            tmp=tmp.strip()\n",
    "            tmp=str(tmp)\n",
    "            datasets[idx][idxx]=tmp\n",
    "\n",
    "    '''\n",
    "    Create pandas dataframe and convert it to float\n",
    "    '''\n",
    "    df=pandas.DataFrame(datasets, columns=headings)\n",
    "    df['Quantity (Token)']=df['Quantity (Token)'].str.replace(',','')\n",
    "    df['Quantity (Token)']=df['Quantity (Token)'].astype(numpy.float64)\n",
    "    df['Percentage']=df['Percentage'].str.strip('%')\n",
    "    df['Percentage']=df['Percentage'].astype(numpy.float64)/100\n",
    "    return df\n",
    "\n",
    "def owners_tr(ownerid, tokenname, classname):\n",
    "    global baseUrl\n",
    "    global pool\n",
    "    #the token balance\n",
    "    balance=0  \n",
    "    #first of all, get a snapshot of the earliest block and the corresponding balance\n",
    "    earlUrl='token/'+tokenname.upper()+'?a='+ownerid\n",
    "    pre=pool.request('GET',baseUrl+earlUrl)\n",
    "    preHtml=pre.data\n",
    "    preSoup=BeautifulSoup(preHtml)\n",
    "    summ_div = preSoup.find('div' , {'id': 'ContentPlaceHolder1_divSummary'})\n",
    "    summ_table = summ_div.find('table')\n",
    "    #find the current token balance\n",
    "    for row in summ_table.find_all(\"tr\")[0:]:\n",
    "        dataset = (td.get_text() for td in row.find_all(\"td\"))\n",
    "        ls=list(dataset)\n",
    "        for n,i in enumerate(ls):\n",
    "            ls[n]=i.encode('utf-8').strip()\n",
    "        if len(ls)==2 and ls[0]=='Token Balance:':\n",
    "            ls[1]=ls[1].replace(\"EOS\",\"\")\n",
    "            ls[1]=ls[1].replace(\",\",\"\")\n",
    "            ls[1].strip()\n",
    "            balance = float(ls[1])\n",
    "            break\n",
    "        else:\n",
    "            continue\n",
    "    #if no balance is found\n",
    "    if balance==0:\n",
    "        print \"No balance found for owner \"+ownerid+\" , abandoning job\"\n",
    "        voiddic=[]\n",
    "        return voiddic\n",
    "    #now find the earliest transaction\n",
    "    iframe = preSoup.find('iframe',{'id':'tokentxnsiframe'})\n",
    "    print iframe\n",
    "    \n",
    "    \n",
    "    transUrl='tokentxns?a='\n",
    "    nextlinks=[]\n",
    "    nextlinks.append(transUrl+ownerid)\n",
    "    '''\n",
    "    trans_dic=[\n",
    "        tx:{\n",
    "            Value:'', \n",
    "            Block:''\n",
    "        }\n",
    "    ]\n",
    "    '''\n",
    "    trans_dic=[]\n",
    "    #avoid counting duplicate transactions\n",
    "    trans_set=Set()\n",
    "    #the earliest block (only block after this will be considered)\n",
    "    earliest_block=0\n",
    "    i=0\n",
    "    while len(nextlinks)>0:\n",
    "        starttime=time.time()\n",
    "        print \"processing page \"+str(i)+\" of owner: \"+ownerid\n",
    "        link=nextlinks.pop()\n",
    "        r=pool.request('GET',baseUrl+link)\n",
    "        html=r.data\n",
    "        soup = BeautifulSoup(html)\n",
    "        '''\n",
    "        Get the next link\n",
    "        '''\n",
    "        a_tag=soup.find_all('a', id=\"ContentPlaceHolder1_HyperLinkNext\",href=True)\n",
    "        if a_tag[0]['href']!='#':\n",
    "            next_link=a_tag[0]['href']\n",
    "            nextlinks.append(next_link)\n",
    "        table = soup.find(\"table\", attrs=classname)\n",
    "        headings = [th.get_text() for th in table.find(\"tr\").find_all(\"th\")]\n",
    "        for idx in range(len(headings)):\n",
    "            headings[idx]=str(headings[idx])\n",
    "            if headings[idx]=='':\n",
    "                headings[idx]='direction'\n",
    "        datasets = []\n",
    "        for row in table.find_all(\"tr\")[1:]:\n",
    "            dataset = (td.get_text() for td in row.find_all(\"td\"))\n",
    "            ls=list(dataset)\n",
    "            datasets.append(ls)\n",
    "        '''\n",
    "        Clean the data\n",
    "        '''\n",
    "        for idx in range(len(datasets)):\n",
    "            for idxx in range(len(datasets[idx])):\n",
    "                tmp=re.sub(r'\\([^)]*\\)', '', datasets[idx][idxx])\n",
    "                tmp=tmp.strip()\n",
    "                tmp=str(tmp)\n",
    "                datasets[idx][idxx]=tmp\n",
    "        '''\n",
    "        Create pandas dataframe and convert it to float\n",
    "        '''\n",
    "        df=pandas.DataFrame(datasets, columns=headings)\n",
    "        df['Token']=df['Token'].str.upper()\n",
    "        df=df.loc[df['Token'] == tokenname]\n",
    "        df['Value']=df['Value'].str.replace(',','')\n",
    "        df['Value']=df['Value'].astype(numpy.float64)\n",
    "        j=0\n",
    "        for index, row in df.iterrows():\n",
    "            if row['TxHash']!='' and row['TxHash'] not in trans_set:\n",
    "                trans_set.add(row['TxHash'])\n",
    "                tmp_dic={}\n",
    "                if row['Value']!=0:\n",
    "                    if row['direction'] == 'OUT':\n",
    "                        val=-row['Value']\n",
    "                    else:\n",
    "                        val=+row['Value']\n",
    "                    tx=row['TxHash']\n",
    "                    tmp_dic[tx]={}\n",
    "                    tmp_dic[tx]['Value']=val\n",
    "                    '''\n",
    "                    Now, get the block number\n",
    "                    '''\n",
    "                    txurl='tx/'\n",
    "                    block=''\n",
    "                    req=pool.request('GET',baseUrl+txurl+tx)\n",
    "                    html_tx=req.data\n",
    "                    tx_soup=BeautifulSoup(html_tx)\n",
    "                    tx_a_tag=tx_soup.find_all('a',href=True)\n",
    "                    m=0\n",
    "                    for tag in tx_a_tag:\n",
    "                        if '/block/' in tag['href']:\n",
    "                            block=str(tag.getText())\n",
    "                            if block=='':\n",
    "                                if m==len(tx_a_tag)-1:\n",
    "                                    block=0\n",
    "                                else:\n",
    "                                    j=-1\n",
    "                                    continue\n",
    "                            if i==0 and j==0:\n",
    "                                earliest_block=long(block)\n",
    "                            tmp_dic[tx]['Block']=block\n",
    "                        m=m+1\n",
    "                    if 'Block' in tmp_dic[tx] and long(tmp_dic[tx]['Block']) <= earliest_block:\n",
    "                        trans_dic.append(tmp_dic)\n",
    "                else:\n",
    "                    j=-1\n",
    "            j=j+1\n",
    "        i=i+1\n",
    "        elapsed=time.time()-starttime\n",
    "        print str(elapsed)+\" second for each request (Owner: \"+ownerid+\" )\"\n",
    "    return trans_dic\n",
    "\n",
    "def tr_wrapper(args):\n",
    "    tokenname=args[0]\n",
    "    owners=args[1]\n",
    "    '''\n",
    "    Construct the ownership transaction table\n",
    "    '''\n",
    "    trans_history={}\n",
    "    for owner in owners:\n",
    "        if owners != '':\n",
    "            trans_history[owner]=owners_tr(owner,tokenname, 'table table-hover ')\n",
    "        else: \n",
    "            pass\n",
    "    '''\n",
    "    Backout the transaction history\n",
    "    '''\n",
    "    headtable=['Block Height', 'Owner', 'TransactionID', 'TOKEN', 'BALANCE']\n",
    "    for owner in owners:\n",
    "        content=[]\n",
    "        balance=owners[owner]\n",
    "        trans=trans_history[owner]\n",
    "        i=0\n",
    "        for l in trans:\n",
    "            #The first record\n",
    "            if i==0:\n",
    "                entry=[]\n",
    "                TID='ENDING BALANCE'\n",
    "                Block='N/A'\n",
    "                entry.append(Block)\n",
    "                entry.append(owner)\n",
    "                entry.append(TID)\n",
    "                entry.append(tokenname)\n",
    "                entry.append(balance)\n",
    "                content.append(entry)\n",
    "                i=i+1\n",
    "            for t in l:\n",
    "                entry=[]\n",
    "                #transaction ID\n",
    "                TID=t\n",
    "                #Block Height\n",
    "                Block=l[t]['Block']\n",
    "                entry.append(Block)\n",
    "                entry.append(owner)\n",
    "                entry.append(TID)\n",
    "                entry.append(tokenname)\n",
    "                entry.append(balance)\n",
    "                content.append(entry)\n",
    "                balance=balance-l[t]['Value']\n",
    "                #The last record\n",
    "                if i==len(trans):\n",
    "                    entry=[]\n",
    "                    TID='BEGINNING BALANCE'\n",
    "                    Block='N/A'\n",
    "                    entry.append(Block)\n",
    "                    entry.append(owner)\n",
    "                    entry.append(TID)\n",
    "                    entry.append(tokenname)\n",
    "                    entry.append(balance)\n",
    "                    content.append(entry)\n",
    "            i=i+1\n",
    "        dataframe=pandas.DataFrame(content, columns=headtable)\n",
    "        dataframe.to_csv('./csv/'+owner+'top100.csv')\n",
    "     \n",
    "\n",
    "def ICO_TOKEN(tokenid, tokenname):\n",
    "    global numthread\n",
    "    df=html_convert_top100(tokenid, \"table table-hover \")\n",
    "    '''\n",
    "    Construct the ownership table\n",
    "    '''\n",
    "    owners={}\n",
    "    for index, row in df.iterrows():\n",
    "        owners[row['Address']]=row['Quantity (Token)']\n",
    "    owner_list=[] \n",
    "    for key in owners:\n",
    "        tmp_dict={}\n",
    "        tmp_dict[key]=owners[key]\n",
    "        owner_list.append(tmp_dict)\n",
    "    owner_list=[{'0x000ea6df3b680cb96f66a42041235d1ed776ef4f':38012}]\n",
    "    # make the Pool of workers\n",
    "    print \"starting \"+str(numthread)+\" threads...\" \n",
    "    tpool = ThreadPool(numthread)\n",
    "    tpool.map(tr_wrapper, itertools.izip(itertools.repeat(tokenname), owner_list))\n",
    "    tpool.close()\n",
    "    tpool.join()\n",
    "    \n",
    "ICO_TOKEN('0x86Fa049857E0209aa7D9e616F7eb3b3B78ECfdb0','EOS')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}