jupyter_notebooks/analyze_dim_name.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Share/home/chenxupeng/anaconda2/lib/python2.7/site-packages/IPython/core/magics/pylab.py:161: UserWarning: pylab import has clobbered these variables: ['f']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n",
      "  \"\\n`%matplotlib` prevents importing * from pylab and numpy\"\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Share/home/chenxupeng/projects/economics\n"
     ]
    }
   ],
   "source": [
    "cd /Share/home/chenxupeng/projects/economics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexnames = ['L3_1_DVA_FIN','L3_4_RDV','L3_6_FVA_INT','L3_2_DVA_INT',\n",
    "              'L3_5_FVA_FIN','L3_7_DDC','L3_3_DVA_INTrex','L3_5_MVA','L3_8_FDC']\n",
    "years = np.arange(1995,2012).astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenamerdata= [''.join((x,'.RData')) for x in indexnames]\n",
    "filenamecsv =  [''.join((x,'.csv')) for x in indexnames]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['1995', '1996', '1997', '1998', '1999', '2000', '2001', '2002',\n",
       "       '2003', '2004', '2005', '2006', '2007', '2008', '2009', '2010',\n",
       "       '2011'], \n",
       "      dtype='|S21')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "years"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "rowstd =np.loadtxt('1995/1995_L3_1_DVA_FIN_row.csv',dtype='str',delimiter=',')[1:]\n",
    "colstd = np.loadtxt('1995/1995_L3_1_DVA_FIN_col.csv',dtype='str',delimiter=',')[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "58835"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colstd.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1405"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.loadtxt('1996/1996_L3_4_RDV_col.csv',dtype='str',delimiter=',')[1:].shape[0]/35"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "检查1995-2000年的行和列名"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1995_L3_1_DVA_FIN: row right\n",
      "1995_L3_1_DVA_FIN: col right\n",
      "1995_L3_4_RDV: row right\n",
      "1995_L3_4_RDV: col right\n",
      "1995_L3_6_FVA_INT: row right\n",
      "1995_L3_6_FVA_INT: col right\n",
      "1995_L3_2_DVA_INT: row right\n",
      "1995_L3_2_DVA_INT: col right\n",
      "1995_L3_5_FVA_FIN: row right\n",
      "1995_L3_5_FVA_FIN: col right\n",
      "1995_L3_7_DDC: row right\n",
      "1995_L3_7_DDC: col right\n",
      "1995_L3_3_DVA_INTrex: row right\n",
      "1995_L3_3_DVA_INTrex: col right\n",
      "1995_L3_5_MVA: row right\n",
      "1995_L3_5_MVA: col right\n",
      "1995_L3_8_FDC: row right\n",
      "1995_L3_8_FDC: col right\n",
      "1996_L3_1_DVA_FIN: row right\n",
      "1996_L3_1_DVA_FIN: col right\n",
      "1996_L3_4_RDV: row right\n",
      "1996_L3_4_RDV: col wrong shape mismatch, shape is :49209\n",
      "1996_L3_6_FVA_INT: row right\n",
      "1996_L3_6_FVA_INT: col right\n",
      "1996_L3_2_DVA_INT: row right\n",
      "1996_L3_2_DVA_INT: col right\n",
      "1996_L3_5_FVA_FIN: row right\n",
      "1996_L3_5_FVA_FIN: col wrong shape mismatch, shape is :18864\n",
      "1996_L3_7_DDC: row right\n",
      "1996_L3_7_DDC: col right\n",
      "1996_L3_3_DVA_INTrex: row right\n",
      "1996_L3_3_DVA_INTrex: col right\n",
      "1996_L3_5_MVA: row right\n",
      "1996_L3_5_MVA: col right\n",
      "1996_L3_8_FDC: row right\n",
      "1996_L3_8_FDC: col right\n",
      "1997_L3_1_DVA_FIN: row right\n",
      "1997_L3_1_DVA_FIN: col right\n",
      "1997_L3_4_RDV: row right\n",
      "1997_L3_4_RDV: col right\n",
      "1997_L3_6_FVA_INT: row right\n",
      "1997_L3_6_FVA_INT: col right\n",
      "1997_L3_2_DVA_INT: row right\n",
      "1997_L3_2_DVA_INT: col right\n",
      "1997_L3_5_FVA_FIN: row right\n",
      "1997_L3_5_FVA_FIN: col right\n",
      "1997_L3_7_DDC: row right\n",
      "1997_L3_7_DDC: col right\n",
      "1997_L3_3_DVA_INTrex: row right\n",
      "1997_L3_3_DVA_INTrex: col right\n",
      "1997_L3_5_MVA: row right\n",
      "1997_L3_5_MVA: col right\n",
      "1997_L3_8_FDC: row right\n",
      "1997_L3_8_FDC: col right\n",
      "1998_L3_1_DVA_FIN: row right\n",
      "1998_L3_1_DVA_FIN: col right\n",
      "1998_L3_4_RDV: row right\n",
      "1998_L3_4_RDV: col right\n",
      "1998_L3_6_FVA_INT: row right\n",
      "1998_L3_6_FVA_INT: col right\n",
      "1998_L3_2_DVA_INT: row right\n",
      "1998_L3_2_DVA_INT: col right\n",
      "1998_L3_5_FVA_FIN: row right\n",
      "1998_L3_5_FVA_FIN: col right\n",
      "1998_L3_7_DDC: row right\n",
      "1998_L3_7_DDC: col right\n",
      "1998_L3_3_DVA_INTrex: row right\n",
      "1998_L3_3_DVA_INTrex: col right\n",
      "1998_L3_5_MVA: row right\n",
      "1998_L3_5_MVA: col right\n",
      "1998_L3_8_FDC: row right\n",
      "1998_L3_8_FDC: col right\n",
      "1999_L3_1_DVA_FIN: row right\n",
      "1999_L3_1_DVA_FIN: col wrong shape mismatch, shape is :717\n",
      "1999_L3_4_RDV: row right\n",
      "1999_L3_4_RDV: col right\n",
      "1999_L3_6_FVA_INT: row right\n",
      "1999_L3_6_FVA_INT: col wrong shape mismatch, shape is :4449\n",
      "1999_L3_2_DVA_INT: row right\n",
      "1999_L3_2_DVA_INT: col right\n",
      "1999_L3_5_FVA_FIN: row right\n",
      "1999_L3_5_FVA_FIN: col wrong shape mismatch, shape is :44659\n",
      "1999_L3_7_DDC: row right\n",
      "1999_L3_7_DDC: col right\n",
      "1999_L3_3_DVA_INTrex: row right\n",
      "1999_L3_3_DVA_INTrex: col right\n",
      "1999_L3_5_MVA: row right\n",
      "1999_L3_5_MVA: col right\n",
      "1999_L3_8_FDC: row right\n",
      "1999_L3_8_FDC: col wrong shape mismatch, shape is :46600\n",
      "2000_L3_1_DVA_FIN: row right\n",
      "2000_L3_1_DVA_FIN: col wrong shape mismatch, shape is :45846\n",
      "2000_L3_4_RDV: row right\n",
      "2000_L3_4_RDV: col wrong shape mismatch, shape is :27802\n",
      "2000_L3_6_FVA_INT: row right\n",
      "2000_L3_6_FVA_INT: col right\n",
      "2000_L3_2_DVA_INT: row right\n",
      "2000_L3_2_DVA_INT: col wrong shape mismatch, shape is :29300\n",
      "2000_L3_5_FVA_FIN: row right\n",
      "2000_L3_5_FVA_FIN: col wrong shape mismatch, shape is :7719\n",
      "2000_L3_7_DDC: row right\n",
      "2000_L3_7_DDC: col right\n",
      "2000_L3_3_DVA_INTrex: row right\n",
      "2000_L3_3_DVA_INTrex: col wrong shape mismatch, shape is :20975\n",
      "2000_L3_5_MVA: row right\n",
      "2000_L3_5_MVA: col right\n",
      "2000_L3_8_FDC: row right\n",
      "2000_L3_8_FDC: col right\n",
      "There are 11 Mistakes!\n"
     ]
    }
   ],
   "source": [
    "countflag = 0\n",
    "for i in range(6):\n",
    "    for j in range(9):\n",
    "        rowshape = np.loadtxt(years[i]+'/'+years[i]+'_'+indexnames[j]+'_row.csv',dtype='str',delimiter=',')[1:].shape[0]\n",
    "        if rowshape!=rowstd.shape[0]:\n",
    "            print years[i]+'_'+indexnames[j]+': row wrong shape mismatch, shape is :' +str(rowshape)\n",
    "            countflag+=1\n",
    "        elif not (np.loadtxt(years[i]+'/'+years[i]+'_'+indexnames[j]+'_row.csv',dtype='str',delimiter=',')[1:]==rowstd).all():\n",
    "            print years[i]+'_'+indexnames[j]+': row wrong' \n",
    "            countflag+=1\n",
    "        else:\n",
    "            print years[i]+'_'+indexnames[j]+': row right'\n",
    "        colshape =np.loadtxt(years[i]+'/'+years[i]+'_'+indexnames[j]+'_col.csv',dtype='str',delimiter=',')[1:].shape[0]\n",
    "        if colshape!=colstd.shape[0]:\n",
    "            print years[i]+'_'+indexnames[j]+': col wrong shape mismatch, shape is :' +str(colshape)\n",
    "            countflag+=1\n",
    "        elif not (np.loadtxt(years[i]+'/'+years[i]+'_'+indexnames[j]+'_col.csv',dtype='str',delimiter=',')[1:]==colstd).all():\n",
    "            print years[i]+'_'+indexnames[j]+': col wrong'\n",
    "            countflag+=1\n",
    "        else:\n",
    "            print years[i]+'_'+indexnames[j]+': col right'\n",
    "if countflag >0:\n",
    "    print \"There are \" +str(countflag)+\" Mistakes!\"\n",
    "else:\n",
    "    print \"All is Right!\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wrong_file =['1996_L3_4_RDV','1996_L3_5_FVA_FIN','1999_L3_1_DVA_FIN','1999_L3_6_FVA_INT','1999_L3_5_FVA_FIN',\n",
    "'1999_L3_8_FDC','2000_L3_1_DVA_FIN','2000_L3_4_RDV','2000_L3_2_DVA_INT','2000_L3_5_FVA_FIN',\n",
    "'2000_L3_3_DVA_INTrex']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2001以后的，用R脚本转出来csv，同样的方式读取并判断"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "rowstd_1 =np.loadtxt('2001/2001_L3_5_MVA_row.csv',dtype='str',delimiter=',')[1:,1:].ravel()\n",
    "colstd_1 = np.loadtxt('2001/2001_L3_5_MVA_col.csv',dtype='str',delimiter=',')[1:,1:].ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2001', '2002', '2003', '2004', '2005', '2006', '2007', '2008',\n",
       "       '2009', '2010', '2011'], \n",
       "      dtype='|S21')"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "years[np.arange(6,17)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2001_L3_1_DVA_FIN: row right\n",
      "2001_L3_1_DVA_FIN: col right\n",
      "2001_L3_4_RDV: row right\n",
      "2001_L3_4_RDV: col right\n",
      "2001_L3_6_FVA_INT: row right\n",
      "2001_L3_6_FVA_INT: col right\n",
      "2001_L3_2_DVA_INT: row right\n",
      "2001_L3_2_DVA_INT: col right\n",
      "2001_L3_5_FVA_FIN: row right\n",
      "2001_L3_5_FVA_FIN: col right\n",
      "2001_L3_7_DDC: row right\n",
      "2001_L3_7_DDC: col right\n",
      "2001_L3_3_DVA_INTrex: row right\n",
      "2001_L3_3_DVA_INTrex: col right\n",
      "2001_L3_5_MVA: row right\n",
      "2001_L3_5_MVA: col right\n",
      "2001_L3_8_FDC: row right\n",
      "2001_L3_8_FDC: col right\n",
      "2002_L3_1_DVA_FIN: row right\n",
      "2002_L3_1_DVA_FIN: col right\n",
      "2002_L3_4_RDV: row right\n",
      "2002_L3_4_RDV: col right\n",
      "2002_L3_6_FVA_INT: row right\n",
      "2002_L3_6_FVA_INT: col right\n",
      "2002_L3_2_DVA_INT: row right\n",
      "2002_L3_2_DVA_INT: col right\n",
      "2002_L3_5_FVA_FIN: row right\n",
      "2002_L3_5_FVA_FIN: col right\n",
      "2002_L3_7_DDC: row right\n",
      "2002_L3_7_DDC: col right\n",
      "2002_L3_3_DVA_INTrex: row right\n",
      "2002_L3_3_DVA_INTrex: col right\n",
      "2002_L3_5_MVA: row right\n",
      "2002_L3_5_MVA: col right\n",
      "2002_L3_8_FDC: row right\n",
      "2002_L3_8_FDC: col right\n",
      "2003_L3_1_DVA_FIN: row right\n",
      "2003_L3_1_DVA_FIN: col right\n",
      "2003_L3_4_RDV: row right\n",
      "2003_L3_4_RDV: col right\n",
      "2003_L3_6_FVA_INT: row right\n",
      "2003_L3_6_FVA_INT: col right\n",
      "2003_L3_2_DVA_INT: row right\n",
      "2003_L3_2_DVA_INT: col right\n",
      "2003_L3_5_FVA_FIN: row right\n",
      "2003_L3_5_FVA_FIN: col right\n",
      "2003_L3_7_DDC: row right\n",
      "2003_L3_7_DDC: col right\n",
      "2003_L3_3_DVA_INTrex: row right\n",
      "2003_L3_3_DVA_INTrex: col right\n",
      "2003_L3_5_MVA: row right\n",
      "2003_L3_5_MVA: col right\n",
      "2003_L3_8_FDC: row right\n",
      "2003_L3_8_FDC: col right\n",
      "2004_L3_1_DVA_FIN: row right\n",
      "2004_L3_1_DVA_FIN: col right\n",
      "2004_L3_4_RDV: row right\n",
      "2004_L3_4_RDV: col right\n",
      "2004_L3_6_FVA_INT: row right\n",
      "2004_L3_6_FVA_INT: col right\n",
      "2004_L3_2_DVA_INT: row right\n",
      "2004_L3_2_DVA_INT: col right\n",
      "2004_L3_5_FVA_FIN: row right\n",
      "2004_L3_5_FVA_FIN: col right\n",
      "2004_L3_7_DDC: row right\n",
      "2004_L3_7_DDC: col right\n",
      "2004_L3_3_DVA_INTrex: row right\n",
      "2004_L3_3_DVA_INTrex: col right\n",
      "2004_L3_5_MVA: row right\n",
      "2004_L3_5_MVA: col right\n",
      "2004_L3_8_FDC: row right\n",
      "2004_L3_8_FDC: col right\n",
      "2005_L3_1_DVA_FIN: row right\n",
      "2005_L3_1_DVA_FIN: col right\n",
      "2005_L3_4_RDV: row right\n",
      "2005_L3_4_RDV: col right\n",
      "2005_L3_6_FVA_INT: row right\n",
      "2005_L3_6_FVA_INT: col right\n",
      "2005_L3_2_DVA_INT: row right\n",
      "2005_L3_2_DVA_INT: col right\n",
      "2005_L3_5_FVA_FIN: row right\n",
      "2005_L3_5_FVA_FIN: col right\n",
      "2005_L3_7_DDC: row right\n",
      "2005_L3_7_DDC: col right\n",
      "2005_L3_3_DVA_INTrex: row right\n",
      "2005_L3_3_DVA_INTrex: col right\n",
      "2005_L3_5_MVA: row right\n",
      "2005_L3_5_MVA: col right\n",
      "2005_L3_8_FDC: row right\n",
      "2005_L3_8_FDC: col right\n",
      "2006_L3_1_DVA_FIN: row right\n",
      "2006_L3_1_DVA_FIN: col right\n",
      "2006_L3_4_RDV: row right\n",
      "2006_L3_4_RDV: col right\n",
      "2006_L3_6_FVA_INT: row right\n",
      "2006_L3_6_FVA_INT: col right\n",
      "2006_L3_2_DVA_INT: row right\n",
      "2006_L3_2_DVA_INT: col right\n",
      "2006_L3_5_FVA_FIN: row right\n",
      "2006_L3_5_FVA_FIN: col right\n",
      "2006_L3_7_DDC: row right\n",
      "2006_L3_7_DDC: col right\n",
      "2006_L3_3_DVA_INTrex: read file wrong:\n",
      "2006_L3_5_MVA: row right\n",
      "2006_L3_5_MVA: col right\n",
      "2006_L3_8_FDC: row right\n",
      "2006_L3_8_FDC: col right\n",
      "2007_L3_1_DVA_FIN: row right\n",
      "2007_L3_1_DVA_FIN: col right\n",
      "2007_L3_4_RDV: row right\n",
      "2007_L3_4_RDV: col right\n",
      "2007_L3_6_FVA_INT: row right\n",
      "2007_L3_6_FVA_INT: col right\n",
      "2007_L3_2_DVA_INT: row right\n",
      "2007_L3_2_DVA_INT: col right\n",
      "2007_L3_5_FVA_FIN: row right\n",
      "2007_L3_5_FVA_FIN: col right\n",
      "2007_L3_7_DDC: row right\n",
      "2007_L3_7_DDC: col right\n",
      "2007_L3_3_DVA_INTrex: row right\n",
      "2007_L3_3_DVA_INTrex: col right\n",
      "2007_L3_5_MVA: row right\n",
      "2007_L3_5_MVA: col right\n",
      "2007_L3_8_FDC: row right\n",
      "2007_L3_8_FDC: col right\n",
      "2008_L3_1_DVA_FIN: row right\n",
      "2008_L3_1_DVA_FIN: col right\n",
      "2008_L3_4_RDV: row right\n",
      "2008_L3_4_RDV: col right\n",
      "2008_L3_6_FVA_INT: row right\n",
      "2008_L3_6_FVA_INT: col right\n",
      "2008_L3_2_DVA_INT: row right\n",
      "2008_L3_2_DVA_INT: col right\n",
      "2008_L3_5_FVA_FIN: row right\n",
      "2008_L3_5_FVA_FIN: col right\n",
      "2008_L3_7_DDC: row right\n",
      "2008_L3_7_DDC: col right\n",
      "2008_L3_3_DVA_INTrex: row right\n",
      "2008_L3_3_DVA_INTrex: col right\n",
      "2008_L3_5_MVA: row right\n",
      "2008_L3_5_MVA: col right\n",
      "2008_L3_8_FDC: row right\n",
      "2008_L3_8_FDC: col right\n",
      "2009_L3_1_DVA_FIN: row right\n",
      "2009_L3_1_DVA_FIN: col right\n",
      "2009_L3_4_RDV: row right\n",
      "2009_L3_4_RDV: col right\n",
      "2009_L3_6_FVA_INT: row right\n",
      "2009_L3_6_FVA_INT: col right\n",
      "2009_L3_2_DVA_INT: row right\n",
      "2009_L3_2_DVA_INT: col right\n",
      "2009_L3_5_FVA_FIN: row right\n",
      "2009_L3_5_FVA_FIN: col right\n",
      "2009_L3_7_DDC: row right\n",
      "2009_L3_7_DDC: col right\n",
      "2009_L3_3_DVA_INTrex: row right\n",
      "2009_L3_3_DVA_INTrex: col right\n",
      "2009_L3_5_MVA: row right\n",
      "2009_L3_5_MVA: col right\n",
      "2009_L3_8_FDC: row right\n",
      "2009_L3_8_FDC: col right\n",
      "2010_L3_1_DVA_FIN: row right\n",
      "2010_L3_1_DVA_FIN: col right\n",
      "2010_L3_4_RDV: row right\n",
      "2010_L3_4_RDV: col right\n",
      "2010_L3_6_FVA_INT: row right\n",
      "2010_L3_6_FVA_INT: col right\n",
      "2010_L3_2_DVA_INT: row right\n",
      "2010_L3_2_DVA_INT: col right\n",
      "2010_L3_5_FVA_FIN: row right\n",
      "2010_L3_5_FVA_FIN: col right\n",
      "2010_L3_7_DDC: row right\n",
      "2010_L3_7_DDC: col right\n",
      "2010_L3_3_DVA_INTrex: row right\n",
      "2010_L3_3_DVA_INTrex: col right\n",
      "2010_L3_5_MVA: row right\n",
      "2010_L3_5_MVA: col right\n",
      "2010_L3_8_FDC: row right\n",
      "2010_L3_8_FDC: col right\n",
      "2011_L3_1_DVA_FIN: row right\n",
      "2011_L3_1_DVA_FIN: col right\n",
      "2011_L3_4_RDV: row right\n",
      "2011_L3_4_RDV: col right\n",
      "2011_L3_6_FVA_INT: row right\n",
      "2011_L3_6_FVA_INT: col right\n",
      "2011_L3_2_DVA_INT: row right\n",
      "2011_L3_2_DVA_INT: col right\n",
      "2011_L3_5_FVA_FIN: row right\n",
      "2011_L3_5_FVA_FIN: col right\n",
      "2011_L3_7_DDC: row right\n",
      "2011_L3_7_DDC: col right\n",
      "2011_L3_3_DVA_INTrex: row right\n",
      "2011_L3_3_DVA_INTrex: col right\n",
      "2011_L3_5_MVA: row right\n",
      "2011_L3_5_MVA: col right\n",
      "2011_L3_8_FDC: row right\n",
      "2011_L3_8_FDC: col right\n",
      "All is Right!\n"
     ]
    }
   ],
   "source": [
    "countflag = 0\n",
    "for i in np.arange(6,17):\n",
    "    for j in range(9):\n",
    "        try:\n",
    "            rowshape = np.loadtxt(years[i]+'/'+years[i]+'_'+indexnames[j]+'_row.csv',dtype='str',delimiter=',')[1:,1:].ravel().shape[0]\n",
    "            if rowshape!=rowstd_1.shape[0]:\n",
    "                print years[i]+'_'+indexnames[j]+': row wrong shape mismatch, shape is :' +str(rowshape)\n",
    "                countflag+=1\n",
    "            elif not (np.loadtxt(years[i]+'/'+years[i]+'_'+indexnames[j]+'_row.csv',dtype='str',delimiter=',')[1:,1:].ravel()==rowstd_1).all():\n",
    "                print years[i]+'_'+indexnames[j]+': row wrong' \n",
    "                countflag+=1\n",
    "            else:\n",
    "                print years[i]+'_'+indexnames[j]+': row right'\n",
    "            colshape =np.loadtxt(years[i]+'/'+years[i]+'_'+indexnames[j]+'_col.csv',dtype='str',delimiter=',')[1:,1:].ravel().shape[0]\n",
    "            if colshape!=colstd_1.shape[0]:\n",
    "                print years[i]+'_'+indexnames[j]+': col wrong shape mismatch, shape is :' +str(colshape)\n",
    "                countflag+=1\n",
    "            elif not (np.loadtxt(years[i]+'/'+years[i]+'_'+indexnames[j]+'_col.csv',dtype='str',delimiter=',')[1:,1:].ravel()==colstd_1).all():\n",
    "                print years[i]+'_'+indexnames[j]+': col wrong'\n",
    "                countflag+=1\n",
    "            else:\n",
    "                print years[i]+'_'+indexnames[j]+': col right'\n",
    "        except:\n",
    "            print years[i]+'_'+indexnames[j]+': read file wrong:'\n",
    "if countflag >0:\n",
    "    print \"There are \" +str(countflag)+\" Mistakes!\"\n",
    "else:\n",
    "    print \"All is Right!\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "wrong file:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "wrong_file =['1996_L3_4_RDV','1996_L3_5_FVA_FIN','1999_L3_1_DVA_FIN','1999_L3_6_FVA_INT','1999_L3_5_FVA_FIN',\n",
    "'1999_L3_8_FDC','2000_L3_1_DVA_FIN','2000_L3_4_RDV','2000_L3_2_DVA_INT','2000_L3_5_FVA_FIN',\n",
    "'2000_L3_3_DVA_INTrex']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(wrong_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "dat = np.loadtxt('1996/1996_L3_1_DVA_FIN.csv',dtype='str',delimiter=',')[1:,1:].astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(58835, 1435)"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "datt = dat[1435*6:1435*7,:]\n",
    "tochnarr = np.ndarray([35,35*41])\n",
    "for i in range(41):\n",
    "    tochnarr[:,35*i:35*(i+1)] = datt[35*i:35*(i+1),35*i:35*(i+1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "chntoarr=np.ndarray([35,35*41])\n",
    "for i in range(41):\n",
    "    chntoarr[:,35*i:35*(i+1)] =dat[1435*i:1435*(i+1),:][35*6:35*(6+1),35*6:35*(6+1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractfromcsv(year,indexname):\n",
    "    '''\n",
    "    this is for extract CHN from years before 2001\n",
    "    the data contains other countrys to CHN and CHN to other countrys\n",
    "    '''\n",
    "    dat = np.loadtxt(year+'/'+year+'_'+indexname+'.csv',dtype='str',delimiter=',')[1:,1:].astype('float')\n",
    "    datt = dat[1435*6:1435*7,:]\n",
    "    tochnarr = np.ndarray([35,35*41])\n",
    "    for i in range(41):\n",
    "        tochnarr[:,35*i:35*(i+1)] = datt[35*i:35*(i+1),35*i:35*(i+1)]\n",
    "    chntoarr=np.ndarray([35,35*41])\n",
    "    for i in range(41):\n",
    "        chntoarr[:,35*i:35*(i+1)] =dat[1435*i:1435*(i+1),:][35*6:35*(6+1),35*6:35*(6+1)]\n",
    "    dataframe1 = pd.DataFrame(tochnarr,index=np.arange(1,36),columns=np.repeat(country,35).reshape(-1,35).ravel())\n",
    "    dataframe2 = pd.DataFrame(chntoarr,index=np.arange(1,36),columns=np.repeat(country,35).reshape(-1,35).ravel())\n",
    "    dataframe1.to_csv('out/'+year+'/'+year+'_'+indexname+'tochn.csv')\n",
    "    dataframe2.to_csv('out/'+year+'/'+year+'_'+indexname+'chnto.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "extractfromcsv(years[0],indexnames[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1996_L3_4_RDV',\n",
       " '1996_L3_5_FVA_FIN',\n",
       " '1999_L3_1_DVA_FIN',\n",
       " '1999_L3_6_FVA_INT',\n",
       " '1999_L3_5_FVA_FIN',\n",
       " '1999_L3_8_FDC',\n",
       " '2000_L3_1_DVA_FIN',\n",
       " '2000_L3_4_RDV',\n",
       " '2000_L3_2_DVA_INT',\n",
       " '2000_L3_5_FVA_FIN',\n",
       " '2000_L3_3_DVA_INTrex']"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wrong_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['L3_1_DVA_FIN',\n",
       " 'L3_4_RDV',\n",
       " 'L3_6_FVA_INT',\n",
       " 'L3_2_DVA_INT',\n",
       " 'L3_5_FVA_FIN',\n",
       " 'L3_7_DDC',\n",
       " 'L3_3_DVA_INTrex',\n",
       " 'L3_5_MVA',\n",
       " 'L3_8_FDC']"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indexnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "years[1]+'_'+indexnames[1] in wrong_file ==0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "  0%|          | 0/9 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\n",
      "\u001b[A\n",
      "\n",
      "\n",
      " 11%|█         | 1/9 [00:35<04:47, 35.95s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 22%|██▏       | 2/9 [01:10<04:08, 35.55s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 33%|███▎      | 3/9 [02:03<04:05, 40.86s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\u001b[A\u001b[A"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-196-b03ebd5b50bb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m9\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m         \u001b[0mextractfromcsv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myears\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mindexnames\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-195-fbe2cc4b7b26>\u001b[0m in \u001b[0;36mextractfromcsv\u001b[0;34m(year, indexname)\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mthe\u001b[0m \u001b[0mdata\u001b[0m \u001b[0mcontains\u001b[0m \u001b[0mother\u001b[0m \u001b[0mcountrys\u001b[0m \u001b[0mto\u001b[0m \u001b[0mCHN\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mCHN\u001b[0m \u001b[0mto\u001b[0m \u001b[0mother\u001b[0m \u001b[0mcountrys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     '''\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mdat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloadtxt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myear\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'/'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0myear\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'_'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mindexname\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'str'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdelimiter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m','\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'float'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0mdatt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1435\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1435\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mtochnarr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m35\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m35\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m41\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for i in tqdm(range(6)):\n",
    "    for j in tqdm(range(9)):\n",
    "        if years[i]+'_'+indexnames[j] in wrong_file:\n",
    "            print \"wrong file! skip\"\n",
    "        else:\n",
    "            extractfromcsv(years[i],indexnames[j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "years[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "country = np.array(['AUS', 'AUT', 'BEL', 'BGR', 'BRA', 'CAN', 'CHN', 'CYP', 'CZE',\n",
    "       'DEU', 'DNK', 'ESP','EST', 'FIN', 'FRA', 'GBR', 'GRC', 'HUN', 'IDN',\n",
    "       'IND', 'IRL', 'ITA', 'JPN', 'KOR', 'LTU', 'LUX', 'LVA', 'MEX',\n",
    "       'MLT', 'NLD', 'POL', 'PRT', 'ROM', 'RUS', 'SVK', 'SVN', 'SWE',\n",
    "       'TUR', 'TWN', 'USA', 'RoW'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['AUS', 'AUT', 'BEL', 'BGR', 'BRA', 'CAN', 'CHN', 'CYP', 'CZE',\n",
       "       'DEU', 'DNK', 'ESP', 'EST', 'FIN', 'FRA', 'GBR', 'GRC', 'HUN',\n",
       "       'IDN', 'IND', 'IRL', 'ITA', 'JPN', 'KOR', 'LTU', 'LUX', 'LVA',\n",
       "       'MEX', 'MLT', 'NLD', 'POL', 'PRT', 'ROM', 'RUS', 'SVK', 'SVN',\n",
       "       'SWE', 'TUR', 'TWN', 'USA', 'RoW'], \n",
       "      dtype='|S3')"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "country"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "np.array([rowstd_1[i] for i in np.arange(0,41*35,35)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1435"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "41*35"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('countrynamesasrow.txt',np.repeat(country,35).reshape(-1,35).ravel(),fmt='%s',delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1435,)"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.repeat(country,35).reshape(-1,35).ravel().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}