Analysis/.ipynb_checkpoints/merge_variant_aligning_statistics-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "running in terminal is "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "bowtie2RegexL=[    \"(\\d+) reads; of these:\",\n",
    "    '(\\d+) \\([0-9.]*%\\) aligned exactly 1 time',\n",
    "    '(\\d+) \\([0-9.]*%\\) aligned >1 times','(\\d+) mates make up the pairs']\n",
    "fastqcRegexL=[    'Total reads processed:\\s+([0-9,]+)',\n",
    "    'Reads with adapters:\\s+([0-9,]+)',\n",
    "    'Adapter sequence:(.*)',\n",
    "    'Total basepairs processed:\\s+([0-9,]+)',\n",
    "    'Total written \\(filtered\\):\\s+([0-9,]+)',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#%matplotlib inline\n",
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "tmpFDir='/cellar/users/btsui/Data/SRA/DUMP/merged_variant_aligning_statistics.tsv'\n",
    "logInDir='/nrnb/users/btsui/Data/all_seq/log_snp/'\n",
    "inFnames=os.listdir(logInDir)\n",
    "snp_files=os.listdir('/nrnb/users/btsui/Data/all_seq/snp/')\n",
    "inFnameS=pd.Series(inFnames)\n",
    "\n",
    "rerun=True\n",
    "\n",
    "\n",
    "RegexL=bamRegexL+fastqcRegexL\n",
    "spacer='asdfasdf'\n",
    "\n",
    "\n",
    "if rerun:\n",
    "    with open(tmpFDir ,'w')as wf:\n",
    "        for i,inFname in enumerate(tqdm(inFnameS)):\n",
    "            with open(logInDir+inFname)as f:\n",
    "                lineS=pd.Series(f.readlines())\n",
    "                for regex in RegexL:\n",
    "                    linesWithRegex=lineS.str.extract(regex,expand=False).dropna()\n",
    "                    if len(linesWithRegex)>0:\n",
    "                        wf.write(\"{inFname}{spacer}{regex}{spacer}{linesWithRegex}\\n\".format(\n",
    "                            inFname=inFname,regex=regex,linesWithRegex=linesWithRegex.iloc[0],\n",
    "                        spacer=spacer))\n",
    "            #if i>0:\n",
    "            #    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lineS.str.extract('Total reads processed:\\s+([0-9,]+)',expand=False).dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merge alignment statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!ls -lah /cellar/users/btsui/Data/SRA/DUMP/merged_variant_aligning_statistics.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmpFDir='/cellar/users/btsui/Data/SRA/DUMP/merged_variant_aligning_statistics.tsv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/cellar/users/btsui/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "merged_variant_aligning_statistics=pd.read_csv(tmpFDir,sep=spacer,names=np.arange(3),error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_variant_aligning_statistics.columns=['inFname','regex','linesWithRegex']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_fastqc=merged_variant_aligning_statistics.regex.isin(fastqcRegexL)\n",
    "\n",
    "merged_variant_aligning_statistics.loc[m_fastqc,'group']='fastqc'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_bowtie=merged_variant_aligning_statistics.regex.isin(bowtie2RegexL)\n",
    "\n",
    "merged_variant_aligning_statistics.loc[m_bowtie,'group']='bowtie2'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merge alignment table with existing SRA metadata dump"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sra_dump_pickle_dir='/cellar/users/btsui/Data/SRA/DUMP/sra_dump.pickle'\n",
    "sra_dump_df=pd.read_pickle(sra_dump_pickle_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sra_dump_pickle_dir_annotated_dir=sra_dump_pickle_dir.replace('.pickle','.fastqc.bowtie_algn.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g=['group','regex','inFname']\n",
    "mergedAlignmentStat=merged_variant_aligning_statistics.groupby(g).last()['linesWithRegex'].unstack().T#.set_index(['group','inFname'])['linesWithRegex'].unstack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mergedAlignmentStat.index=mergedAlignmentStat.index.str.replace('.log','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sra_dump_dfMultI=pd.concat([sra_dump_df],axis=1,keys=['SRAmeta'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mergedDf=pd.concat([sra_dump_dfMultI,mergedAlignmentStat],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mergedDf=mergedDf.sort_values([('fastqc','Total basepairs processed:\\s+([0-9,]+)')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mergedDf.to_pickle(sra_dump_pickle_dir_annotated_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#valid_m=mergedDf[('fastqc','Total basepairs processed:\\s+([0-9,]+)')].notnull()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mergedDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#valid_m.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}