1_downlaod_gwas_neale.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import subprocess \n",
    "import glob\n",
    "\n",
    "from basic_tools import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "http://www.nealelab.is/uk-biobank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ukbb_table=pd.read_csv(ukbb_table_path,sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Phenotype Code</th>\n",
       "      <th>check</th>\n",
       "      <th>Phenotype Description</th>\n",
       "      <th>UK Biobank Data Showcase Link</th>\n",
       "      <th>Sex</th>\n",
       "      <th>File</th>\n",
       "      <th>wget command</th>\n",
       "      <th>Dropbox File</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>README</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>README</td>\n",
       "      <td>wget https://www.dropbox.com/s/wro30igqkmit5ig...</td>\n",
       "      <td>https://www.dropbox.com/s/wro30igqkmit5ig/READ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>List of European samples</td>\n",
       "      <td>NaN</td>\n",
       "      <td>both_sexes</td>\n",
       "      <td>european_samples.tsv.bgz</td>\n",
       "      <td>wget https://www.dropbox.com/s/bvnd723tl67lh8v...</td>\n",
       "      <td>https://www.dropbox.com/s/bvnd723tl67lh8v/euro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>List of samples used in GWAS - both_sexes</td>\n",
       "      <td>NaN</td>\n",
       "      <td>both_sexes</td>\n",
       "      <td>samples.both_sexes.tsv.bgz</td>\n",
       "      <td>wget https://www.dropbox.com/s/ypcmzukh2vwhtkh...</td>\n",
       "      <td>https://www.dropbox.com/s/ypcmzukh2vwhtkh/samp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>List of samples used in GWAS - female</td>\n",
       "      <td>NaN</td>\n",
       "      <td>female</td>\n",
       "      <td>samples.female.tsv.bgz</td>\n",
       "      <td>wget https://www.dropbox.com/s/nnn2zc3z5uqmlra...</td>\n",
       "      <td>https://www.dropbox.com/s/nnn2zc3z5uqmlra/samp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>List of samples used in GWAS - male</td>\n",
       "      <td>NaN</td>\n",
       "      <td>male</td>\n",
       "      <td>samples.male.tsv.bgz</td>\n",
       "      <td>wget https://www.dropbox.com/s/r2o91gr3r4l9kz9...</td>\n",
       "      <td>https://www.dropbox.com/s/r2o91gr3r4l9kz9/samp...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Phenotype Code  check                      Phenotype Description  \\\n",
       "0            NaN      1                                     README   \n",
       "1            NaN      1                   List of European samples   \n",
       "2            NaN      1  List of samples used in GWAS - both_sexes   \n",
       "3            NaN      1      List of samples used in GWAS - female   \n",
       "4            NaN      1        List of samples used in GWAS - male   \n",
       "\n",
       "  UK Biobank Data Showcase Link         Sex                        File  \\\n",
       "0                           NaN         NaN                      README   \n",
       "1                           NaN  both_sexes    european_samples.tsv.bgz   \n",
       "2                           NaN  both_sexes  samples.both_sexes.tsv.bgz   \n",
       "3                           NaN      female      samples.female.tsv.bgz   \n",
       "4                           NaN        male        samples.male.tsv.bgz   \n",
       "\n",
       "                                        wget command  \\\n",
       "0  wget https://www.dropbox.com/s/wro30igqkmit5ig...   \n",
       "1  wget https://www.dropbox.com/s/bvnd723tl67lh8v...   \n",
       "2  wget https://www.dropbox.com/s/ypcmzukh2vwhtkh...   \n",
       "3  wget https://www.dropbox.com/s/nnn2zc3z5uqmlra...   \n",
       "4  wget https://www.dropbox.com/s/r2o91gr3r4l9kz9...   \n",
       "\n",
       "                                        Dropbox File  \n",
       "0  https://www.dropbox.com/s/wro30igqkmit5ig/READ...  \n",
       "1  https://www.dropbox.com/s/bvnd723tl67lh8v/euro...  \n",
       "2  https://www.dropbox.com/s/ypcmzukh2vwhtkh/samp...  \n",
       "3  https://www.dropbox.com/s/nnn2zc3z5uqmlra/samp...  \n",
       "4  https://www.dropbox.com/s/r2o91gr3r4l9kz9/samp...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ukbb_table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "phenotypes_both_sexes_v2=pd.read_csv(phenotypes_both_sexes_v2_file_path,sep='\\t',compression='gzip')\n",
    "phenotypes_male_v2=pd.read_csv(phenotypes_male_v2_file_path,sep='\\t',compression='gzip')\n",
    "phenotypes_female_v2=pd.read_csv(phenotypes_female_v2_file_path,sep='\\t',compression='gzip')\n",
    "phenotypes_both_sexes_v2['Sex']='both_sexes';phenotypes_male_v2['Sex']='male';phenotypes_female_v2['Sex']='female';\n",
    "phenotypes=pd.concat([phenotypes_both_sexes_v2,phenotypes_male_v2,phenotypes_female_v2],sort='False')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "phenotypes_select_list=[]\n",
    "for idx,phenotype in enumerate(phenotypes['phenotype'].unique()):\n",
    "    ukbb_table_select=ukbb_table[ukbb_table['Phenotype Code']==phenotype]\n",
    "    if (ukbb_table_select['check']==0).sum()>0:\n",
    "        continue\n",
    "    phenotypes_select=phenotypes[phenotypes['phenotype']==phenotype]\n",
    "    if phenotypes_select[phenotypes_select['Sex']=='female'].shape[0]==0:\n",
    "        phenotypes_select_list.append(phenotypes_select[phenotypes_select['Sex']=='male'])\n",
    "        print(phenotypes_select[phenotypes_select['Sex']=='male'][['Sex','description']])\n",
    "    elif phenotypes_select[phenotypes_select['Sex']=='male'].shape[0]==0:\n",
    "        phenotypes_select_list.append(phenotypes_select[phenotypes_select['Sex']=='female'])\n",
    "        print(phenotypes_select[phenotypes_select['Sex']=='female'][['Sex','description']])\n",
    "    else:\n",
    "        phenotypes_select_list.append(phenotypes_select[phenotypes_select['Sex']=='both_sexes'])\n",
    "phenotypes_filtered=pd.concat(phenotypes_select_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'phenotypes_filtered' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-d43ed59cedcd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mphenotypes_filtered\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'phenotypes_filtered' is not defined"
     ]
    }
   ],
   "source": [
    "phenotypes_filtered.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3486"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ukbb_table['Phenotype Code'][ukbb_table['check']!=0].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ukbb_table_file_list=[]\n",
    "for idx,row in phenotypes_filtered.iterrows():\n",
    "    ukbb_table_select=ukbb_table[ukbb_table['Phenotype Code']==row['phenotype']]\n",
    "    ukbb_table_select=ukbb_table_select[ukbb_table_select['Sex']==row['Sex']]\n",
    "    \n",
    "    if ukbb_table_select['File'].str.contains('v2').sum()>0:\n",
    "        ukbb_table_select=ukbb_table_select[ukbb_table_select['File'].str.contains('v2')]\n",
    "    #& (ukbb_table['Sex']==row['Sex'])]\n",
    "    #print(ukbb_table_select)\n",
    "    if ukbb_table_select.shape[0]==1:\n",
    "        ukbb_table_file_list.append(ukbb_table_select.iloc[0][['Phenotype Code','UK Biobank Data Showcase Link','File','wget command','Dropbox File']])\n",
    "    else:\n",
    "        print(ukbb_table_select)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "phenotypes_filtered=phenotypes_filtered.merge(right=pd.DataFrame(ukbb_table_file_list),left_on='phenotype',right_on='Phenotype Code')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "phenotypes_filtered=phenotypes_filtered.set_index('phenotype')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "phenotypes_filtered.to_csv(phenotypes_filtered_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(gwas_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx,row in phenotypes_filtered.iterrows():\n",
    "    if not os.path.exists(row['File']):\n",
    "        run_command(row['wget command'],quiet=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}