Map PEGASUS out to NCBI.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import mygene\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mg = mygene.MyGeneInfo()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Asthma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chr</th>\n",
       "      <th>Gene</th>\n",
       "      <th>nSNPs</th>\n",
       "      <th>Start</th>\n",
       "      <th>Stop</th>\n",
       "      <th>Test-Stat</th>\n",
       "      <th>Pvalue</th>\n",
       "      <th>Error</th>\n",
       "      <th>Best-SNP</th>\n",
       "      <th>SNP-pvalue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A3GALT2</td>\n",
       "      <td>68</td>\n",
       "      <td>33772366.0</td>\n",
       "      <td>33786699.0</td>\n",
       "      <td>44.811425</td>\n",
       "      <td>0.574424</td>\n",
       "      <td>1.292712e-14</td>\n",
       "      <td>rs10914692</td>\n",
       "      <td>0.009231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL3</td>\n",
       "      <td>94</td>\n",
       "      <td>12776117.0</td>\n",
       "      <td>12788726.0</td>\n",
       "      <td>83.344493</td>\n",
       "      <td>0.451577</td>\n",
       "      <td>7.943619e-15</td>\n",
       "      <td>rs3000905</td>\n",
       "      <td>0.052147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL4</td>\n",
       "      <td>54</td>\n",
       "      <td>12704565.0</td>\n",
       "      <td>12727097.0</td>\n",
       "      <td>39.493893</td>\n",
       "      <td>0.656680</td>\n",
       "      <td>9.349860e-15</td>\n",
       "      <td>rs11121955</td>\n",
       "      <td>0.012345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCA4</td>\n",
       "      <td>261</td>\n",
       "      <td>94458393.0</td>\n",
       "      <td>94586705.0</td>\n",
       "      <td>210.388204</td>\n",
       "      <td>0.688397</td>\n",
       "      <td>3.012240e-15</td>\n",
       "      <td>rs11165076</td>\n",
       "      <td>0.025043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCB10</td>\n",
       "      <td>60</td>\n",
       "      <td>229652328.0</td>\n",
       "      <td>229694442.0</td>\n",
       "      <td>41.712991</td>\n",
       "      <td>0.483655</td>\n",
       "      <td>8.694164e-15</td>\n",
       "      <td>rs12734075</td>\n",
       "      <td>0.089851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17656</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF280A</td>\n",
       "      <td>51</td>\n",
       "      <td>22868060.0</td>\n",
       "      <td>22874613.0</td>\n",
       "      <td>51.315894</td>\n",
       "      <td>0.381351</td>\n",
       "      <td>1.428313e-14</td>\n",
       "      <td>rs2330191</td>\n",
       "      <td>0.065178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17657</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF280B</td>\n",
       "      <td>51</td>\n",
       "      <td>22838771.0</td>\n",
       "      <td>22863505.0</td>\n",
       "      <td>44.383404</td>\n",
       "      <td>0.457183</td>\n",
       "      <td>9.159330e-15</td>\n",
       "      <td>rs2266984</td>\n",
       "      <td>0.067977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17658</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF70</td>\n",
       "      <td>58</td>\n",
       "      <td>24083771.0</td>\n",
       "      <td>24093279.0</td>\n",
       "      <td>56.767506</td>\n",
       "      <td>0.415667</td>\n",
       "      <td>6.594791e-15</td>\n",
       "      <td>rs738791</td>\n",
       "      <td>0.070605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17659</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF74</td>\n",
       "      <td>54</td>\n",
       "      <td>20748479.0</td>\n",
       "      <td>20762752.0</td>\n",
       "      <td>198.180693</td>\n",
       "      <td>0.001879</td>\n",
       "      <td>6.312862e-14</td>\n",
       "      <td>rs1153427</td>\n",
       "      <td>0.000161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17660</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNRF3</td>\n",
       "      <td>168</td>\n",
       "      <td>29279889.0</td>\n",
       "      <td>29453475.0</td>\n",
       "      <td>165.679176</td>\n",
       "      <td>0.417596</td>\n",
       "      <td>4.939124e-15</td>\n",
       "      <td>rs16986864</td>\n",
       "      <td>0.033690</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17661 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Chr     Gene  nSNPs        Start         Stop   Test-Stat    Pvalue  \\\n",
       "0        1  A3GALT2     68   33772366.0   33786699.0   44.811425  0.574424   \n",
       "1        1  AADACL3     94   12776117.0   12788726.0   83.344493  0.451577   \n",
       "2        1  AADACL4     54   12704565.0   12727097.0   39.493893  0.656680   \n",
       "3        1    ABCA4    261   94458393.0   94586705.0  210.388204  0.688397   \n",
       "4        1   ABCB10     60  229652328.0  229694442.0   41.712991  0.483655   \n",
       "...    ...      ...    ...          ...          ...         ...       ...   \n",
       "17656   22  ZNF280A     51   22868060.0   22874613.0   51.315894  0.381351   \n",
       "17657   22  ZNF280B     51   22838771.0   22863505.0   44.383404  0.457183   \n",
       "17658   22    ZNF70     58   24083771.0   24093279.0   56.767506  0.415667   \n",
       "17659   22    ZNF74     54   20748479.0   20762752.0  198.180693  0.001879   \n",
       "17660   22    ZNRF3    168   29279889.0   29453475.0  165.679176  0.417596   \n",
       "\n",
       "              Error    Best-SNP  SNP-pvalue  \n",
       "0      1.292712e-14  rs10914692    0.009231  \n",
       "1      7.943619e-15   rs3000905    0.052147  \n",
       "2      9.349860e-15  rs11121955    0.012345  \n",
       "3      3.012240e-15  rs11165076    0.025043  \n",
       "4      8.694164e-15  rs12734075    0.089851  \n",
       "...             ...         ...         ...  \n",
       "17656  1.428313e-14   rs2330191    0.065178  \n",
       "17657  9.159330e-15   rs2266984    0.067977  \n",
       "17658  6.594791e-15    rs738791    0.070605  \n",
       "17659  6.312862e-14   rs1153427    0.000161  \n",
       "17660  4.939124e-15  rs16986864    0.033690  \n",
       "\n",
       "[17661 rows x 10 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asthma_data = pd.read_csv(\"processed_data/gwas_gene_pvals/asthma/PEGASUS_asthma_gwas_data.csv\")\n",
    "asthma_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "querying 1-1000...done.\n",
      "querying 1001-2000...done.\n",
      "querying 2001-3000...done.\n",
      "querying 3001-4000...done.\n",
      "querying 4001-5000...done.\n",
      "querying 5001-6000...done.\n",
      "querying 6001-7000...done.\n",
      "querying 7001-8000...done.\n",
      "querying 8001-9000...done.\n",
      "querying 9001-10000...done.\n",
      "querying 10001-11000...done.\n",
      "querying 11001-12000...done.\n",
      "querying 12001-13000...done.\n",
      "querying 13001-14000...done.\n",
      "querying 14001-15000...done.\n",
      "querying 15001-16000...done.\n",
      "querying 16001-17000...done.\n",
      "querying 17001-17661...done.\n",
      "Finished.\n",
      "1001 input query terms found dup hits:\n",
      "\t[('ADC', 2), ('AGT', 2), ('AIDA', 2), ('AK3L1', 2), ('APITD1', 2), ('APOA2', 2), ('ARF1', 2), ('ARNT\n",
      "508 input query terms found no hit:\n",
      "\t['AMY1A-1', 'AMY1A-3', 'AMY1B-1', 'AMY1B-3', 'AMY1C-1', 'AMY1C-3', 'FLJ13137', 'FLJ45717', 'GPR89C-2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'query': 'A3GALT2',\n",
       "  '_id': '127550',\n",
       "  '_score': 18.73485,\n",
       "  'entrezgene': '127550',\n",
       "  'name': 'alpha 1,3-galactosyltransferase 2',\n",
       "  'symbol': 'A3GALT2',\n",
       "  'taxid': 9606},\n",
       " {'query': 'AADACL3',\n",
       "  '_id': '126767',\n",
       "  '_score': 19.469177,\n",
       "  'entrezgene': '126767',\n",
       "  'name': 'arylacetamide deacetylase like 3',\n",
       "  'symbol': 'AADACL3',\n",
       "  'taxid': 9606},\n",
       " {'query': 'AADACL4',\n",
       "  '_id': '343066',\n",
       "  '_score': 18.745165,\n",
       "  'entrezgene': '343066',\n",
       "  'name': 'arylacetamide deacetylase like 4',\n",
       "  'symbol': 'AADACL4',\n",
       "  'taxid': 9606},\n",
       " {'query': 'ABCA4',\n",
       "  '_id': '24',\n",
       "  '_score': 18.067741,\n",
       "  'entrezgene': '24',\n",
       "  'name': 'ATP binding cassette subfamily A member 4',\n",
       "  'symbol': 'ABCA4',\n",
       "  'taxid': 9606},\n",
       " {'query': 'ABCB10',\n",
       "  '_id': '23456',\n",
       "  '_score': 17.55544,\n",
       "  'entrezgene': '23456',\n",
       "  'name': 'ATP binding cassette subfamily B member 10',\n",
       "  'symbol': 'ABCB10',\n",
       "  'taxid': 9606},\n",
       " {'query': 'ABCD3',\n",
       "  '_id': '5825',\n",
       "  '_score': 18.064056,\n",
       "  'entrezgene': '5825',\n",
       "  'name': 'ATP binding cassette subfamily D member 3',\n",
       "  'symbol': 'ABCD3',\n",
       "  'taxid': 9606},\n",
       " {'query': 'ABHD7',\n",
       "  '_id': '253152',\n",
       "  '_score': 21.504927,\n",
       "  'entrezgene': '253152',\n",
       "  'name': 'epoxide hydrolase 4',\n",
       "  'symbol': 'EPHX4',\n",
       "  'taxid': 9606},\n",
       " {'query': 'ABL2',\n",
       "  '_id': '27',\n",
       "  '_score': 17.573725,\n",
       "  'entrezgene': '27',\n",
       "  'name': 'ABL proto-oncogene 2, non-receptor tyrosine kinase',\n",
       "  'symbol': 'ABL2',\n",
       "  'taxid': 9606},\n",
       " {'query': 'ACADM',\n",
       "  '_id': '34',\n",
       "  '_score': 17.633652,\n",
       "  'entrezgene': '34',\n",
       "  'name': 'acyl-CoA dehydrogenase medium chain',\n",
       "  'symbol': 'ACADM',\n",
       "  'taxid': 9606},\n",
       " {'query': 'ACBD3',\n",
       "  '_id': '64746',\n",
       "  '_score': 17.549555,\n",
       "  'entrezgene': '64746',\n",
       "  'name': 'acyl-CoA binding domain containing 3',\n",
       "  'symbol': 'ACBD3',\n",
       "  'taxid': 9606}]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gids = list(asthma_data[\"Gene\"])\n",
    "gene_mapping = mg.querymany(gids, scopes=[\"symbol\", \"alias\"], species=\"human\", returnall=True)\n",
    "gene_mapping[\"out\"][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "querying 1-508...done.\n",
      "Finished.\n",
      "1 input query terms found dup hits:\n",
      "\t[('PC3', 5)]\n",
      "501 input query terms found no hit:\n",
      "\t['AMY1A1', 'AMY1A3', 'AMY1B1', 'AMY1B3', 'AMY1C1', 'AMY1C3', 'FLJ13137', 'FLJ45717', 'GPR89C2', 'HIS\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'query': 'AMY1A1', 'notfound': True},\n",
       " {'query': 'AMY1A3', 'notfound': True},\n",
       " {'query': 'AMY1B1', 'notfound': True},\n",
       " {'query': 'AMY1B3', 'notfound': True}]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Try again removing symbols from missing genes\n",
    "refined_query = [g.replace(\"-\" , \"\").replace(\".\", \"\") for g in gene_mapping[\"missing\"]]\n",
    "secondary_gene_mapping = mg.querymany(refined_query, scopes=[\"symbol\", \"alias\"], species=\"human\", returnall=True)\n",
    "secondary_gene_mapping[\"out\"][:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8be162e5f524abcb7131feda717e0b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17661 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None values  509\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "17661"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ncbi_ids = []\n",
    "for gn in tqdm(gids):\n",
    "    res = [e for e in gene_mapping[\"out\"] if e[\"query\"]==gn]\n",
    "    res = [r for r in res if \"_score\" in r.keys() and \"entrezgene\" in r.keys()]\n",
    "    if len(res)<1:\n",
    "        ncbi_ids.append(None)\n",
    "        continue\n",
    "        \n",
    "    res = sorted(res, key=lambda x: x[\"_score\"], reverse=True)\n",
    "    ncbi_ids.append(res[0][\"entrezgene\"])\n",
    "print(\"None values \", len([i for i in ncbi_ids if i is None]))\n",
    "len(ncbi_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chr</th>\n",
       "      <th>Gene</th>\n",
       "      <th>nSNPs</th>\n",
       "      <th>Start</th>\n",
       "      <th>Stop</th>\n",
       "      <th>Test-Stat</th>\n",
       "      <th>Pvalue</th>\n",
       "      <th>Error</th>\n",
       "      <th>Best-SNP</th>\n",
       "      <th>SNP-pvalue</th>\n",
       "      <th>NCBI_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A3GALT2</td>\n",
       "      <td>68</td>\n",
       "      <td>33772366.0</td>\n",
       "      <td>33786699.0</td>\n",
       "      <td>44.811425</td>\n",
       "      <td>0.574424</td>\n",
       "      <td>1.292712e-14</td>\n",
       "      <td>rs10914692</td>\n",
       "      <td>0.009231</td>\n",
       "      <td>127550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL3</td>\n",
       "      <td>94</td>\n",
       "      <td>12776117.0</td>\n",
       "      <td>12788726.0</td>\n",
       "      <td>83.344493</td>\n",
       "      <td>0.451577</td>\n",
       "      <td>7.943619e-15</td>\n",
       "      <td>rs3000905</td>\n",
       "      <td>0.052147</td>\n",
       "      <td>126767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL4</td>\n",
       "      <td>54</td>\n",
       "      <td>12704565.0</td>\n",
       "      <td>12727097.0</td>\n",
       "      <td>39.493893</td>\n",
       "      <td>0.656680</td>\n",
       "      <td>9.349860e-15</td>\n",
       "      <td>rs11121955</td>\n",
       "      <td>0.012345</td>\n",
       "      <td>343066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCA4</td>\n",
       "      <td>261</td>\n",
       "      <td>94458393.0</td>\n",
       "      <td>94586705.0</td>\n",
       "      <td>210.388204</td>\n",
       "      <td>0.688397</td>\n",
       "      <td>3.012240e-15</td>\n",
       "      <td>rs11165076</td>\n",
       "      <td>0.025043</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCB10</td>\n",
       "      <td>60</td>\n",
       "      <td>229652328.0</td>\n",
       "      <td>229694442.0</td>\n",
       "      <td>41.712991</td>\n",
       "      <td>0.483655</td>\n",
       "      <td>8.694164e-15</td>\n",
       "      <td>rs12734075</td>\n",
       "      <td>0.089851</td>\n",
       "      <td>23456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17656</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF280A</td>\n",
       "      <td>51</td>\n",
       "      <td>22868060.0</td>\n",
       "      <td>22874613.0</td>\n",
       "      <td>51.315894</td>\n",
       "      <td>0.381351</td>\n",
       "      <td>1.428313e-14</td>\n",
       "      <td>rs2330191</td>\n",
       "      <td>0.065178</td>\n",
       "      <td>129025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17657</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF280B</td>\n",
       "      <td>51</td>\n",
       "      <td>22838771.0</td>\n",
       "      <td>22863505.0</td>\n",
       "      <td>44.383404</td>\n",
       "      <td>0.457183</td>\n",
       "      <td>9.159330e-15</td>\n",
       "      <td>rs2266984</td>\n",
       "      <td>0.067977</td>\n",
       "      <td>140883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17658</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF70</td>\n",
       "      <td>58</td>\n",
       "      <td>24083771.0</td>\n",
       "      <td>24093279.0</td>\n",
       "      <td>56.767506</td>\n",
       "      <td>0.415667</td>\n",
       "      <td>6.594791e-15</td>\n",
       "      <td>rs738791</td>\n",
       "      <td>0.070605</td>\n",
       "      <td>7621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17659</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF74</td>\n",
       "      <td>54</td>\n",
       "      <td>20748479.0</td>\n",
       "      <td>20762752.0</td>\n",
       "      <td>198.180693</td>\n",
       "      <td>0.001879</td>\n",
       "      <td>6.312862e-14</td>\n",
       "      <td>rs1153427</td>\n",
       "      <td>0.000161</td>\n",
       "      <td>7625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17660</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNRF3</td>\n",
       "      <td>168</td>\n",
       "      <td>29279889.0</td>\n",
       "      <td>29453475.0</td>\n",
       "      <td>165.679176</td>\n",
       "      <td>0.417596</td>\n",
       "      <td>4.939124e-15</td>\n",
       "      <td>rs16986864</td>\n",
       "      <td>0.033690</td>\n",
       "      <td>84133</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17152 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Chr     Gene  nSNPs        Start         Stop   Test-Stat    Pvalue  \\\n",
       "0        1  A3GALT2     68   33772366.0   33786699.0   44.811425  0.574424   \n",
       "1        1  AADACL3     94   12776117.0   12788726.0   83.344493  0.451577   \n",
       "2        1  AADACL4     54   12704565.0   12727097.0   39.493893  0.656680   \n",
       "3        1    ABCA4    261   94458393.0   94586705.0  210.388204  0.688397   \n",
       "4        1   ABCB10     60  229652328.0  229694442.0   41.712991  0.483655   \n",
       "...    ...      ...    ...          ...          ...         ...       ...   \n",
       "17656   22  ZNF280A     51   22868060.0   22874613.0   51.315894  0.381351   \n",
       "17657   22  ZNF280B     51   22838771.0   22863505.0   44.383404  0.457183   \n",
       "17658   22    ZNF70     58   24083771.0   24093279.0   56.767506  0.415667   \n",
       "17659   22    ZNF74     54   20748479.0   20762752.0  198.180693  0.001879   \n",
       "17660   22    ZNRF3    168   29279889.0   29453475.0  165.679176  0.417596   \n",
       "\n",
       "              Error    Best-SNP  SNP-pvalue NCBI_id  \n",
       "0      1.292712e-14  rs10914692    0.009231  127550  \n",
       "1      7.943619e-15   rs3000905    0.052147  126767  \n",
       "2      9.349860e-15  rs11121955    0.012345  343066  \n",
       "3      3.012240e-15  rs11165076    0.025043      24  \n",
       "4      8.694164e-15  rs12734075    0.089851   23456  \n",
       "...             ...         ...         ...     ...  \n",
       "17656  1.428313e-14   rs2330191    0.065178  129025  \n",
       "17657  9.159330e-15   rs2266984    0.067977  140883  \n",
       "17658  6.594791e-15    rs738791    0.070605    7621  \n",
       "17659  6.312862e-14   rs1153427    0.000161    7625  \n",
       "17660  4.939124e-15  rs16986864    0.033690   84133  \n",
       "\n",
       "[17152 rows x 11 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asthma_data[\"NCBI_id\"] = ncbi_ids\n",
    "asthma_data = asthma_data[~asthma_data[\"NCBI_id\"].isna()]\n",
    "asthma_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "asthma_data.to_csv(\"processed_data/gwas_gene_pvals/asthma/filtered_ncbi_PEGASUS_asthma_gwas_data.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Autism"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chr</th>\n",
       "      <th>Gene</th>\n",
       "      <th>nSNPs</th>\n",
       "      <th>Start</th>\n",
       "      <th>Stop</th>\n",
       "      <th>Test-Stat</th>\n",
       "      <th>Pvalue</th>\n",
       "      <th>Error</th>\n",
       "      <th>Best-SNP</th>\n",
       "      <th>SNP-pvalue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A3GALT2</td>\n",
       "      <td>89</td>\n",
       "      <td>33772366.0</td>\n",
       "      <td>33786699.0</td>\n",
       "      <td>137.053070</td>\n",
       "      <td>0.185235</td>\n",
       "      <td>4.569034e-14</td>\n",
       "      <td>rs4653044</td>\n",
       "      <td>0.001152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL3</td>\n",
       "      <td>132</td>\n",
       "      <td>12776117.0</td>\n",
       "      <td>12788726.0</td>\n",
       "      <td>32.016457</td>\n",
       "      <td>0.970413</td>\n",
       "      <td>1.839305e-14</td>\n",
       "      <td>rs3010873</td>\n",
       "      <td>0.265100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL4</td>\n",
       "      <td>110</td>\n",
       "      <td>12704565.0</td>\n",
       "      <td>12727097.0</td>\n",
       "      <td>84.549660</td>\n",
       "      <td>0.588825</td>\n",
       "      <td>6.345503e-15</td>\n",
       "      <td>rs6699993</td>\n",
       "      <td>0.006778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCA4</td>\n",
       "      <td>342</td>\n",
       "      <td>94458393.0</td>\n",
       "      <td>94586705.0</td>\n",
       "      <td>370.418540</td>\n",
       "      <td>0.341536</td>\n",
       "      <td>5.743797e-15</td>\n",
       "      <td>rs3120133</td>\n",
       "      <td>0.025470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCB10</td>\n",
       "      <td>86</td>\n",
       "      <td>229652328.0</td>\n",
       "      <td>229694442.0</td>\n",
       "      <td>142.749985</td>\n",
       "      <td>0.153780</td>\n",
       "      <td>3.450117e-14</td>\n",
       "      <td>rs10916485</td>\n",
       "      <td>0.036440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17768</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF280A</td>\n",
       "      <td>14</td>\n",
       "      <td>22868060.0</td>\n",
       "      <td>22874613.0</td>\n",
       "      <td>13.958949</td>\n",
       "      <td>0.318221</td>\n",
       "      <td>1.988166e-11</td>\n",
       "      <td>rs3747098</td>\n",
       "      <td>0.195700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17769</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF280B</td>\n",
       "      <td>21</td>\n",
       "      <td>22838771.0</td>\n",
       "      <td>22863505.0</td>\n",
       "      <td>17.537348</td>\n",
       "      <td>0.431712</td>\n",
       "      <td>2.477566e-14</td>\n",
       "      <td>rs3747098</td>\n",
       "      <td>0.195700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17770</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF70</td>\n",
       "      <td>75</td>\n",
       "      <td>24083771.0</td>\n",
       "      <td>24093279.0</td>\n",
       "      <td>86.686562</td>\n",
       "      <td>0.304061</td>\n",
       "      <td>1.291541e-14</td>\n",
       "      <td>rs6003875</td>\n",
       "      <td>0.018310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17771</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF74</td>\n",
       "      <td>82</td>\n",
       "      <td>20748479.0</td>\n",
       "      <td>20762752.0</td>\n",
       "      <td>53.933570</td>\n",
       "      <td>0.688966</td>\n",
       "      <td>1.232943e-14</td>\n",
       "      <td>rs8748</td>\n",
       "      <td>0.065030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17772</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNRF3</td>\n",
       "      <td>235</td>\n",
       "      <td>29279889.0</td>\n",
       "      <td>29453475.0</td>\n",
       "      <td>207.898057</td>\n",
       "      <td>0.513777</td>\n",
       "      <td>3.283712e-15</td>\n",
       "      <td>rs11703010</td>\n",
       "      <td>0.004727</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17773 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Chr     Gene  nSNPs        Start         Stop   Test-Stat    Pvalue  \\\n",
       "0        1  A3GALT2     89   33772366.0   33786699.0  137.053070  0.185235   \n",
       "1        1  AADACL3    132   12776117.0   12788726.0   32.016457  0.970413   \n",
       "2        1  AADACL4    110   12704565.0   12727097.0   84.549660  0.588825   \n",
       "3        1    ABCA4    342   94458393.0   94586705.0  370.418540  0.341536   \n",
       "4        1   ABCB10     86  229652328.0  229694442.0  142.749985  0.153780   \n",
       "...    ...      ...    ...          ...          ...         ...       ...   \n",
       "17768   22  ZNF280A     14   22868060.0   22874613.0   13.958949  0.318221   \n",
       "17769   22  ZNF280B     21   22838771.0   22863505.0   17.537348  0.431712   \n",
       "17770   22    ZNF70     75   24083771.0   24093279.0   86.686562  0.304061   \n",
       "17771   22    ZNF74     82   20748479.0   20762752.0   53.933570  0.688966   \n",
       "17772   22    ZNRF3    235   29279889.0   29453475.0  207.898057  0.513777   \n",
       "\n",
       "              Error    Best-SNP  SNP-pvalue  \n",
       "0      4.569034e-14   rs4653044    0.001152  \n",
       "1      1.839305e-14   rs3010873    0.265100  \n",
       "2      6.345503e-15   rs6699993    0.006778  \n",
       "3      5.743797e-15   rs3120133    0.025470  \n",
       "4      3.450117e-14  rs10916485    0.036440  \n",
       "...             ...         ...         ...  \n",
       "17768  1.988166e-11   rs3747098    0.195700  \n",
       "17769  2.477566e-14   rs3747098    0.195700  \n",
       "17770  1.291541e-14   rs6003875    0.018310  \n",
       "17771  1.232943e-14      rs8748    0.065030  \n",
       "17772  3.283712e-15  rs11703010    0.004727  \n",
       "\n",
       "[17773 rows x 10 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "autism_data = pd.read_csv(\"processed_data/gwas_gene_pvals/autism/PEGASUS_autism_gwas_data.csv\")\n",
    "autism_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "querying 1-1000...done.\n",
      "querying 1001-2000...done.\n",
      "querying 2001-3000...done.\n",
      "querying 3001-4000...done.\n",
      "querying 4001-5000...done.\n",
      "querying 5001-6000...done.\n",
      "querying 6001-7000...done.\n",
      "querying 7001-8000...done.\n",
      "querying 8001-9000...done.\n",
      "querying 9001-10000...done.\n",
      "querying 10001-11000...done.\n",
      "querying 11001-12000...done.\n",
      "querying 12001-13000...done.\n",
      "querying 13001-14000...done.\n",
      "querying 14001-15000...done.\n",
      "querying 15001-16000...done.\n",
      "querying 16001-17000...done.\n",
      "querying 17001-17773...done.\n",
      "Finished.\n",
      "998 input query terms found dup hits:\n",
      "\t[('ADC', 2), ('AGT', 2), ('AIDA', 2), ('AK3L1', 2), ('APITD1', 2), ('APOA2', 2), ('ARF1', 2), ('ARNT\n",
      "562 input query terms found no hit:\n",
      "\t['AMY1A-1', 'AMY1A-3', 'AMY1B-1', 'AMY1B-3', 'AMY1C-1', 'AMY1C-3', 'FLJ13137', 'FLJ45717', 'GPR89C-2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'query': 'A3GALT2',\n",
       "  '_id': '127550',\n",
       "  '_score': 18.802341,\n",
       "  'entrezgene': '127550',\n",
       "  'name': 'alpha 1,3-galactosyltransferase 2',\n",
       "  'symbol': 'A3GALT2',\n",
       "  'taxid': 9606},\n",
       " {'query': 'AADACL3',\n",
       "  '_id': '126767',\n",
       "  '_score': 19.529512,\n",
       "  'entrezgene': '126767',\n",
       "  'name': 'arylacetamide deacetylase like 3',\n",
       "  'symbol': 'AADACL3',\n",
       "  'taxid': 9606},\n",
       " {'query': 'AADACL4',\n",
       "  '_id': '343066',\n",
       "  '_score': 18.81738,\n",
       "  'entrezgene': '343066',\n",
       "  'name': 'arylacetamide deacetylase like 4',\n",
       "  'symbol': 'AADACL4',\n",
       "  'taxid': 9606},\n",
       " {'query': 'ABCA4',\n",
       "  '_id': '24',\n",
       "  '_score': 18.21474,\n",
       "  'entrezgene': '24',\n",
       "  'name': 'ATP binding cassette subfamily A member 4',\n",
       "  'symbol': 'ABCA4',\n",
       "  'taxid': 9606}]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gids = list(autism_data[\"Gene\"])\n",
    "gene_mapping = mg.querymany(gids, scopes=[\"symbol\", \"alias\"], species=\"human\", returnall=True)\n",
    "gene_mapping[\"out\"][:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6de81c1203e430db78760a12e1b0e64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17773 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None values  563\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "17773"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ncbi_ids = []\n",
    "for gn in tqdm(gids):\n",
    "    res = [e for e in gene_mapping[\"out\"] if e[\"query\"]==gn]\n",
    "    res = [r for r in res if \"_score\" in r.keys() and \"entrezgene\" in r.keys()]\n",
    "    if len(res)<1:\n",
    "        ncbi_ids.append(None)\n",
    "        continue\n",
    "#     elif len(res)>1:\n",
    "#         break\n",
    "        \n",
    "    res = sorted(res, key=lambda x: x[\"_score\"], reverse=True)\n",
    "    ncbi_ids.append(res[0][\"entrezgene\"])\n",
    "print(\"None values \", len([i for i in ncbi_ids if i is None]))\n",
    "len(ncbi_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chr</th>\n",
       "      <th>Gene</th>\n",
       "      <th>nSNPs</th>\n",
       "      <th>Start</th>\n",
       "      <th>Stop</th>\n",
       "      <th>Test-Stat</th>\n",
       "      <th>Pvalue</th>\n",
       "      <th>Error</th>\n",
       "      <th>Best-SNP</th>\n",
       "      <th>SNP-pvalue</th>\n",
       "      <th>NCBI_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A3GALT2</td>\n",
       "      <td>89</td>\n",
       "      <td>33772366.0</td>\n",
       "      <td>33786699.0</td>\n",
       "      <td>137.053070</td>\n",
       "      <td>0.185235</td>\n",
       "      <td>4.569034e-14</td>\n",
       "      <td>rs4653044</td>\n",
       "      <td>0.001152</td>\n",
       "      <td>127550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL3</td>\n",
       "      <td>132</td>\n",
       "      <td>12776117.0</td>\n",
       "      <td>12788726.0</td>\n",
       "      <td>32.016457</td>\n",
       "      <td>0.970413</td>\n",
       "      <td>1.839305e-14</td>\n",
       "      <td>rs3010873</td>\n",
       "      <td>0.265100</td>\n",
       "      <td>126767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL4</td>\n",
       "      <td>110</td>\n",
       "      <td>12704565.0</td>\n",
       "      <td>12727097.0</td>\n",
       "      <td>84.549660</td>\n",
       "      <td>0.588825</td>\n",
       "      <td>6.345503e-15</td>\n",
       "      <td>rs6699993</td>\n",
       "      <td>0.006778</td>\n",
       "      <td>343066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCA4</td>\n",
       "      <td>342</td>\n",
       "      <td>94458393.0</td>\n",
       "      <td>94586705.0</td>\n",
       "      <td>370.418540</td>\n",
       "      <td>0.341536</td>\n",
       "      <td>5.743797e-15</td>\n",
       "      <td>rs3120133</td>\n",
       "      <td>0.025470</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCB10</td>\n",
       "      <td>86</td>\n",
       "      <td>229652328.0</td>\n",
       "      <td>229694442.0</td>\n",
       "      <td>142.749985</td>\n",
       "      <td>0.153780</td>\n",
       "      <td>3.450117e-14</td>\n",
       "      <td>rs10916485</td>\n",
       "      <td>0.036440</td>\n",
       "      <td>23456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17768</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF280A</td>\n",
       "      <td>14</td>\n",
       "      <td>22868060.0</td>\n",
       "      <td>22874613.0</td>\n",
       "      <td>13.958949</td>\n",
       "      <td>0.318221</td>\n",
       "      <td>1.988166e-11</td>\n",
       "      <td>rs3747098</td>\n",
       "      <td>0.195700</td>\n",
       "      <td>129025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17769</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF280B</td>\n",
       "      <td>21</td>\n",
       "      <td>22838771.0</td>\n",
       "      <td>22863505.0</td>\n",
       "      <td>17.537348</td>\n",
       "      <td>0.431712</td>\n",
       "      <td>2.477566e-14</td>\n",
       "      <td>rs3747098</td>\n",
       "      <td>0.195700</td>\n",
       "      <td>140883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17770</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF70</td>\n",
       "      <td>75</td>\n",
       "      <td>24083771.0</td>\n",
       "      <td>24093279.0</td>\n",
       "      <td>86.686562</td>\n",
       "      <td>0.304061</td>\n",
       "      <td>1.291541e-14</td>\n",
       "      <td>rs6003875</td>\n",
       "      <td>0.018310</td>\n",
       "      <td>7621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17771</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF74</td>\n",
       "      <td>82</td>\n",
       "      <td>20748479.0</td>\n",
       "      <td>20762752.0</td>\n",
       "      <td>53.933570</td>\n",
       "      <td>0.688966</td>\n",
       "      <td>1.232943e-14</td>\n",
       "      <td>rs8748</td>\n",
       "      <td>0.065030</td>\n",
       "      <td>7625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17772</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNRF3</td>\n",
       "      <td>235</td>\n",
       "      <td>29279889.0</td>\n",
       "      <td>29453475.0</td>\n",
       "      <td>207.898057</td>\n",
       "      <td>0.513777</td>\n",
       "      <td>3.283712e-15</td>\n",
       "      <td>rs11703010</td>\n",
       "      <td>0.004727</td>\n",
       "      <td>84133</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17210 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Chr     Gene  nSNPs        Start         Stop   Test-Stat    Pvalue  \\\n",
       "0        1  A3GALT2     89   33772366.0   33786699.0  137.053070  0.185235   \n",
       "1        1  AADACL3    132   12776117.0   12788726.0   32.016457  0.970413   \n",
       "2        1  AADACL4    110   12704565.0   12727097.0   84.549660  0.588825   \n",
       "3        1    ABCA4    342   94458393.0   94586705.0  370.418540  0.341536   \n",
       "4        1   ABCB10     86  229652328.0  229694442.0  142.749985  0.153780   \n",
       "...    ...      ...    ...          ...          ...         ...       ...   \n",
       "17768   22  ZNF280A     14   22868060.0   22874613.0   13.958949  0.318221   \n",
       "17769   22  ZNF280B     21   22838771.0   22863505.0   17.537348  0.431712   \n",
       "17770   22    ZNF70     75   24083771.0   24093279.0   86.686562  0.304061   \n",
       "17771   22    ZNF74     82   20748479.0   20762752.0   53.933570  0.688966   \n",
       "17772   22    ZNRF3    235   29279889.0   29453475.0  207.898057  0.513777   \n",
       "\n",
       "              Error    Best-SNP  SNP-pvalue NCBI_id  \n",
       "0      4.569034e-14   rs4653044    0.001152  127550  \n",
       "1      1.839305e-14   rs3010873    0.265100  126767  \n",
       "2      6.345503e-15   rs6699993    0.006778  343066  \n",
       "3      5.743797e-15   rs3120133    0.025470      24  \n",
       "4      3.450117e-14  rs10916485    0.036440   23456  \n",
       "...             ...         ...         ...     ...  \n",
       "17768  1.988166e-11   rs3747098    0.195700  129025  \n",
       "17769  2.477566e-14   rs3747098    0.195700  140883  \n",
       "17770  1.291541e-14   rs6003875    0.018310    7621  \n",
       "17771  1.232943e-14      rs8748    0.065030    7625  \n",
       "17772  3.283712e-15  rs11703010    0.004727   84133  \n",
       "\n",
       "[17210 rows x 11 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "autism_data[\"NCBI_id\"] = ncbi_ids\n",
    "autism_data = autism_data[~autism_data[\"NCBI_id\"].isna()]\n",
    "autism_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "autism_data.to_csv(\"processed_data/gwas_gene_pvals/autism/filtered_ncbi_PEGASUS_autism_gwas_data.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Schizophrenia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chr</th>\n",
       "      <th>Gene</th>\n",
       "      <th>nSNPs</th>\n",
       "      <th>Start</th>\n",
       "      <th>Stop</th>\n",
       "      <th>Test-Stat</th>\n",
       "      <th>Pvalue</th>\n",
       "      <th>Error</th>\n",
       "      <th>Best-SNP</th>\n",
       "      <th>SNP-pvalue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A3GALT2</td>\n",
       "      <td>89</td>\n",
       "      <td>33772366.0</td>\n",
       "      <td>33786699.0</td>\n",
       "      <td>242.644950</td>\n",
       "      <td>0.039346</td>\n",
       "      <td>1.017755e-13</td>\n",
       "      <td>rs12034407</td>\n",
       "      <td>1.400000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL3</td>\n",
       "      <td>133</td>\n",
       "      <td>12776117.0</td>\n",
       "      <td>12788726.0</td>\n",
       "      <td>181.875486</td>\n",
       "      <td>0.213243</td>\n",
       "      <td>2.296784e-14</td>\n",
       "      <td>rs3000870</td>\n",
       "      <td>6.200000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL4</td>\n",
       "      <td>115</td>\n",
       "      <td>12704565.0</td>\n",
       "      <td>12727097.0</td>\n",
       "      <td>108.047041</td>\n",
       "      <td>0.431211</td>\n",
       "      <td>4.167978e-15</td>\n",
       "      <td>rs3000870</td>\n",
       "      <td>6.200000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCA4</td>\n",
       "      <td>346</td>\n",
       "      <td>94458393.0</td>\n",
       "      <td>94586705.0</td>\n",
       "      <td>449.555425</td>\n",
       "      <td>0.162139</td>\n",
       "      <td>1.278153e-14</td>\n",
       "      <td>rs581244</td>\n",
       "      <td>2.000000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCB10</td>\n",
       "      <td>85</td>\n",
       "      <td>229652328.0</td>\n",
       "      <td>229694442.0</td>\n",
       "      <td>29.481566</td>\n",
       "      <td>0.843688</td>\n",
       "      <td>1.592276e-14</td>\n",
       "      <td>rs348326</td>\n",
       "      <td>8.830000e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17484</th>\n",
       "      <td>22</td>\n",
       "      <td>ZDHHC8</td>\n",
       "      <td>123</td>\n",
       "      <td>20119364.0</td>\n",
       "      <td>20133974.0</td>\n",
       "      <td>859.926651</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>1.008769e-13</td>\n",
       "      <td>rs8137258</td>\n",
       "      <td>9.870000e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17485</th>\n",
       "      <td>22</td>\n",
       "      <td>ZMAT5</td>\n",
       "      <td>146</td>\n",
       "      <td>30126944.0</td>\n",
       "      <td>30162969.0</td>\n",
       "      <td>620.945084</td>\n",
       "      <td>0.003213</td>\n",
       "      <td>7.477507e-14</td>\n",
       "      <td>rs737932</td>\n",
       "      <td>1.000000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17486</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF70</td>\n",
       "      <td>76</td>\n",
       "      <td>24083771.0</td>\n",
       "      <td>24093279.0</td>\n",
       "      <td>133.184031</td>\n",
       "      <td>0.095392</td>\n",
       "      <td>2.946103e-14</td>\n",
       "      <td>rs17629631</td>\n",
       "      <td>6.700000e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17487</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF74</td>\n",
       "      <td>82</td>\n",
       "      <td>20748479.0</td>\n",
       "      <td>20762752.0</td>\n",
       "      <td>64.069484</td>\n",
       "      <td>0.576769</td>\n",
       "      <td>6.663133e-15</td>\n",
       "      <td>rs5749316</td>\n",
       "      <td>2.820000e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17488</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNRF3</td>\n",
       "      <td>234</td>\n",
       "      <td>29279889.0</td>\n",
       "      <td>29453475.0</td>\n",
       "      <td>1077.888639</td>\n",
       "      <td>0.000094</td>\n",
       "      <td>3.782169e-14</td>\n",
       "      <td>rs6519766</td>\n",
       "      <td>5.880000e-05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17489 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Chr     Gene  nSNPs        Start         Stop    Test-Stat    Pvalue  \\\n",
       "0        1  A3GALT2     89   33772366.0   33786699.0   242.644950  0.039346   \n",
       "1        1  AADACL3    133   12776117.0   12788726.0   181.875486  0.213243   \n",
       "2        1  AADACL4    115   12704565.0   12727097.0   108.047041  0.431211   \n",
       "3        1    ABCA4    346   94458393.0   94586705.0   449.555425  0.162139   \n",
       "4        1   ABCB10     85  229652328.0  229694442.0    29.481566  0.843688   \n",
       "...    ...      ...    ...          ...          ...          ...       ...   \n",
       "17484   22   ZDHHC8    123   20119364.0   20133974.0   859.926651  0.000004   \n",
       "17485   22    ZMAT5    146   30126944.0   30162969.0   620.945084  0.003213   \n",
       "17486   22    ZNF70     76   24083771.0   24093279.0   133.184031  0.095392   \n",
       "17487   22    ZNF74     82   20748479.0   20762752.0    64.069484  0.576769   \n",
       "17488   22    ZNRF3    234   29279889.0   29453475.0  1077.888639  0.000094   \n",
       "\n",
       "              Error    Best-SNP    SNP-pvalue  \n",
       "0      1.017755e-13  rs12034407  1.400000e-03  \n",
       "1      2.296784e-14   rs3000870  6.200000e-03  \n",
       "2      4.167978e-15   rs3000870  6.200000e-03  \n",
       "3      1.278153e-14    rs581244  2.000000e-03  \n",
       "4      1.592276e-14    rs348326  8.830000e-02  \n",
       "...             ...         ...           ...  \n",
       "17484  1.008769e-13   rs8137258  9.870000e-07  \n",
       "17485  7.477507e-14    rs737932  1.000000e-03  \n",
       "17486  2.946103e-14  rs17629631  6.700000e-03  \n",
       "17487  6.663133e-15   rs5749316  2.820000e-02  \n",
       "17488  3.782169e-14   rs6519766  5.880000e-05  \n",
       "\n",
       "[17489 rows x 10 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sch_data = pd.read_csv(\"processed_data/gwas_gene_pvals/schizophrenia/PEGASUS_schizophrenia_gwas_data.csv\")\n",
    "sch_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "querying 1-1000...done.\n",
      "querying 1001-2000...done.\n",
      "querying 2001-3000...done.\n",
      "querying 3001-4000...done.\n",
      "querying 4001-5000...done.\n",
      "querying 5001-6000...done.\n",
      "querying 6001-7000...done.\n",
      "querying 7001-8000...done.\n",
      "querying 8001-9000...done.\n",
      "querying 9001-10000...done.\n",
      "querying 10001-11000...done.\n",
      "querying 11001-12000...done.\n",
      "querying 12001-13000...done.\n",
      "querying 13001-14000...done.\n",
      "querying 14001-15000...done.\n",
      "querying 15001-16000...done.\n",
      "querying 16001-17000...done.\n",
      "querying 17001-17489...done.\n",
      "Finished.\n",
      "977 input query terms found dup hits:\n",
      "\t[('ADC', 2), ('AGT', 2), ('AIDA', 2), ('AK3L1', 2), ('APITD1', 2), ('APOA2', 2), ('ARF1', 2), ('ARNT\n",
      "524 input query terms found no hit:\n",
      "\t['AMY1A-1', 'AMY1A-3', 'AMY1B-1', 'AMY1B-3', 'AMY1C-1', 'AMY1C-3', 'FLJ13137', 'FLJ45717', 'GPR89C-2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[{'query': 'A3GALT2',\n",
       "  '_id': '127550',\n",
       "  '_score': 92.45881,\n",
       "  'entrezgene': '127550',\n",
       "  'name': 'alpha 1,3-galactosyltransferase 2',\n",
       "  'symbol': 'A3GALT2',\n",
       "  'taxid': 9606},\n",
       " {'query': 'AADACL3',\n",
       "  '_id': '126767',\n",
       "  '_score': 96.26664,\n",
       "  'entrezgene': '126767',\n",
       "  'name': 'arylacetamide deacetylase like 3',\n",
       "  'symbol': 'AADACL3',\n",
       "  'taxid': 9606},\n",
       " {'query': 'AADACL4',\n",
       "  '_id': '343066',\n",
       "  '_score': 93.129395,\n",
       "  'entrezgene': '343066',\n",
       "  'name': 'arylacetamide deacetylase like 4',\n",
       "  'symbol': 'AADACL4',\n",
       "  'taxid': 9606},\n",
       " {'query': 'ABCA4',\n",
       "  '_id': '24',\n",
       "  '_score': 89.94049,\n",
       "  'entrezgene': '24',\n",
       "  'name': 'ATP binding cassette subfamily A member 4',\n",
       "  'symbol': 'ABCA4',\n",
       "  'taxid': 9606}]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gids = list(sch_data[\"Gene\"])\n",
    "gene_mapping = mg.querymany(gids, scopes=[\"symbol\", \"alias\"], species=\"human\", returnall=True)\n",
    "gene_mapping[\"out\"][:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f036879a43f4801bcf230a68b3d45aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17489 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None values  525\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "17489"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ncbi_ids = []\n",
    "for gn in tqdm(gids):\n",
    "    res = [e for e in gene_mapping[\"out\"] if e[\"query\"]==gn]\n",
    "    res = [r for r in res if \"_score\" in r.keys() and \"entrezgene\" in r.keys()]\n",
    "    if len(res)<1:\n",
    "        ncbi_ids.append(None)\n",
    "        continue\n",
    "    res = sorted(res, key=lambda x: x[\"_score\"], reverse=True)\n",
    "    ncbi_ids.append(res[0][\"entrezgene\"])\n",
    "print(\"None values \", len([i for i in ncbi_ids if i is None]))\n",
    "len(ncbi_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chr</th>\n",
       "      <th>Gene</th>\n",
       "      <th>nSNPs</th>\n",
       "      <th>Start</th>\n",
       "      <th>Stop</th>\n",
       "      <th>Test-Stat</th>\n",
       "      <th>Pvalue</th>\n",
       "      <th>Error</th>\n",
       "      <th>Best-SNP</th>\n",
       "      <th>SNP-pvalue</th>\n",
       "      <th>NCBI_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A3GALT2</td>\n",
       "      <td>89</td>\n",
       "      <td>33772366.0</td>\n",
       "      <td>33786699.0</td>\n",
       "      <td>242.644950</td>\n",
       "      <td>0.039346</td>\n",
       "      <td>1.017755e-13</td>\n",
       "      <td>rs12034407</td>\n",
       "      <td>1.400000e-03</td>\n",
       "      <td>127550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL3</td>\n",
       "      <td>133</td>\n",
       "      <td>12776117.0</td>\n",
       "      <td>12788726.0</td>\n",
       "      <td>181.875486</td>\n",
       "      <td>0.213243</td>\n",
       "      <td>2.296784e-14</td>\n",
       "      <td>rs3000870</td>\n",
       "      <td>6.200000e-03</td>\n",
       "      <td>126767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>AADACL4</td>\n",
       "      <td>115</td>\n",
       "      <td>12704565.0</td>\n",
       "      <td>12727097.0</td>\n",
       "      <td>108.047041</td>\n",
       "      <td>0.431211</td>\n",
       "      <td>4.167978e-15</td>\n",
       "      <td>rs3000870</td>\n",
       "      <td>6.200000e-03</td>\n",
       "      <td>343066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCA4</td>\n",
       "      <td>346</td>\n",
       "      <td>94458393.0</td>\n",
       "      <td>94586705.0</td>\n",
       "      <td>449.555425</td>\n",
       "      <td>0.162139</td>\n",
       "      <td>1.278153e-14</td>\n",
       "      <td>rs581244</td>\n",
       "      <td>2.000000e-03</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>ABCB10</td>\n",
       "      <td>85</td>\n",
       "      <td>229652328.0</td>\n",
       "      <td>229694442.0</td>\n",
       "      <td>29.481566</td>\n",
       "      <td>0.843688</td>\n",
       "      <td>1.592276e-14</td>\n",
       "      <td>rs348326</td>\n",
       "      <td>8.830000e-02</td>\n",
       "      <td>23456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17484</th>\n",
       "      <td>22</td>\n",
       "      <td>ZDHHC8</td>\n",
       "      <td>123</td>\n",
       "      <td>20119364.0</td>\n",
       "      <td>20133974.0</td>\n",
       "      <td>859.926651</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>1.008769e-13</td>\n",
       "      <td>rs8137258</td>\n",
       "      <td>9.870000e-07</td>\n",
       "      <td>29801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17485</th>\n",
       "      <td>22</td>\n",
       "      <td>ZMAT5</td>\n",
       "      <td>146</td>\n",
       "      <td>30126944.0</td>\n",
       "      <td>30162969.0</td>\n",
       "      <td>620.945084</td>\n",
       "      <td>0.003213</td>\n",
       "      <td>7.477507e-14</td>\n",
       "      <td>rs737932</td>\n",
       "      <td>1.000000e-03</td>\n",
       "      <td>55954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17486</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF70</td>\n",
       "      <td>76</td>\n",
       "      <td>24083771.0</td>\n",
       "      <td>24093279.0</td>\n",
       "      <td>133.184031</td>\n",
       "      <td>0.095392</td>\n",
       "      <td>2.946103e-14</td>\n",
       "      <td>rs17629631</td>\n",
       "      <td>6.700000e-03</td>\n",
       "      <td>7621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17487</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNF74</td>\n",
       "      <td>82</td>\n",
       "      <td>20748479.0</td>\n",
       "      <td>20762752.0</td>\n",
       "      <td>64.069484</td>\n",
       "      <td>0.576769</td>\n",
       "      <td>6.663133e-15</td>\n",
       "      <td>rs5749316</td>\n",
       "      <td>2.820000e-02</td>\n",
       "      <td>7625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17488</th>\n",
       "      <td>22</td>\n",
       "      <td>ZNRF3</td>\n",
       "      <td>234</td>\n",
       "      <td>29279889.0</td>\n",
       "      <td>29453475.0</td>\n",
       "      <td>1077.888639</td>\n",
       "      <td>0.000094</td>\n",
       "      <td>3.782169e-14</td>\n",
       "      <td>rs6519766</td>\n",
       "      <td>5.880000e-05</td>\n",
       "      <td>84133</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16964 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Chr     Gene  nSNPs        Start         Stop    Test-Stat    Pvalue  \\\n",
       "0        1  A3GALT2     89   33772366.0   33786699.0   242.644950  0.039346   \n",
       "1        1  AADACL3    133   12776117.0   12788726.0   181.875486  0.213243   \n",
       "2        1  AADACL4    115   12704565.0   12727097.0   108.047041  0.431211   \n",
       "3        1    ABCA4    346   94458393.0   94586705.0   449.555425  0.162139   \n",
       "4        1   ABCB10     85  229652328.0  229694442.0    29.481566  0.843688   \n",
       "...    ...      ...    ...          ...          ...          ...       ...   \n",
       "17484   22   ZDHHC8    123   20119364.0   20133974.0   859.926651  0.000004   \n",
       "17485   22    ZMAT5    146   30126944.0   30162969.0   620.945084  0.003213   \n",
       "17486   22    ZNF70     76   24083771.0   24093279.0   133.184031  0.095392   \n",
       "17487   22    ZNF74     82   20748479.0   20762752.0    64.069484  0.576769   \n",
       "17488   22    ZNRF3    234   29279889.0   29453475.0  1077.888639  0.000094   \n",
       "\n",
       "              Error    Best-SNP    SNP-pvalue NCBI_id  \n",
       "0      1.017755e-13  rs12034407  1.400000e-03  127550  \n",
       "1      2.296784e-14   rs3000870  6.200000e-03  126767  \n",
       "2      4.167978e-15   rs3000870  6.200000e-03  343066  \n",
       "3      1.278153e-14    rs581244  2.000000e-03      24  \n",
       "4      1.592276e-14    rs348326  8.830000e-02   23456  \n",
       "...             ...         ...           ...     ...  \n",
       "17484  1.008769e-13   rs8137258  9.870000e-07   29801  \n",
       "17485  7.477507e-14    rs737932  1.000000e-03   55954  \n",
       "17486  2.946103e-14  rs17629631  6.700000e-03    7621  \n",
       "17487  6.663133e-15   rs5749316  2.820000e-02    7625  \n",
       "17488  3.782169e-14   rs6519766  5.880000e-05   84133  \n",
       "\n",
       "[16964 rows x 11 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sch_data[\"NCBI_id\"] = ncbi_ids\n",
    "sch_data = sch_data[~sch_data[\"NCBI_id\"].isna()]\n",
    "sch_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "sch_data.to_csv(\"processed_data/gwas_gene_pvals/schizophrenia/filtered_ncbi_PEGASUS_schizophrenia_gwas_data.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}