analyses/20200403_feature_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#downloaded feature_tracks from OSF\n",
    "#downloaded intergenic and intron bed files from Gannet\n",
    "\n",
    "```\n",
    "Shellys-MacBook-Pro:feature_tracks Shelly$ wget https://gannet.fish.washington.edu/Atumefaciens/20191105_swoose_pgen_v074_renaming/Panopea-generosa-v1.0.a4.intergenic.bed\n",
    "--2020-04-03 08:01:35--  https://gannet.fish.washington.edu/Atumefaciens/20191105_swoose_pgen_v074_renaming/Panopea-generosa-v1.0.a4.intergenic.bed\n",
    "Resolving gannet.fish.washington.edu (gannet.fish.washington.edu)... 128.95.149.52\n",
    "Connecting to gannet.fish.washington.edu (gannet.fish.washington.edu)|128.95.149.52|:443... connected.\n",
    "HTTP request sent, awaiting response... 200 OK\n",
    "Length: 1019566 (996K) [application/vnd.realvnc.bed]\n",
    "Saving to: ‘Panopea-generosa-v1.0.a4.intergenic.bed’\n",
    "\n",
    "Panopea-generosa-v1 100%[===================>] 995.67K   249KB/s    in 4.0s    \n",
    "\n",
    "2020-04-03 08:01:39 (249 KB/s) - ‘Panopea-generosa-v1.0.a4.intergenic.bed’ saved [1019566/1019566]\n",
    "\n",
    "Shellys-MacBook-Pro:feature_tracks Shelly$ wget https://gannet.fish.washington.edu/Atumefaciens/20191105_swoose_pgen_v074_renaming/Panopea-generosa-v1.0.a4.introns.bed\n",
    "--2020-04-03 08:01:54--  https://gannet.fish.washington.edu/Atumefaciens/20191105_swoose_pgen_v074_renaming/Panopea-generosa-v1.0.a4.introns.bed\n",
    "Resolving gannet.fish.washington.edu (gannet.fish.washington.edu)... 128.95.149.52\n",
    "Connecting to gannet.fish.washington.edu (gannet.fish.washington.edu)|128.95.149.52|:443... connected.\n",
    "HTTP request sent, awaiting response... 200 OK\n",
    "Length: 4579012 (4.4M) [application/vnd.realvnc.bed]\n",
    "Saving to: ‘Panopea-generosa-v1.0.a4.introns.bed’\n",
    "\n",
    "Panopea-generosa-v1 100%[===================>]   4.37M   797KB/s    in 6.8s    \n",
    "\n",
    "2020-04-03 08:02:02 (660 KB/s) - ‘Panopea-generosa-v1.0.a4.introns.bed’ saved [4579012/4579012]\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Panopea-generosa-v1.0.CpG.gff\r\n",
      "Panopea-generosa-v1.0.a4.CDS.gff3\r\n",
      "Panopea-generosa-v1.0.a4.exon.gff3\r\n",
      "Panopea-generosa-v1.0.a4.gene.gff3\r\n",
      "Panopea-generosa-v1.0.a4.intergenic.bed\r\n",
      "Panopea-generosa-v1.0.a4.introns.bed\r\n",
      "Panopea-generosa-v1.0.a4.mRNA.gff3\r\n",
      "Panopea-generosa-v1.0.a4.rRNA.gff3\r\n",
      "Panopea-generosa-v1.0.a4.repeat_region.gff3\r\n",
      "Panopea-generosa-v1.0.a4.repeats.DNA.gff3\r\n",
      "Panopea-generosa-v1.0.a4.repeats.LINE.gff3\r\n",
      "Panopea-generosa-v1.0.a4.repeats.LTR.gff3\r\n",
      "Panopea-generosa-v1.0.a4.repeats.RC.gff3\r\n",
      "Panopea-generosa-v1.0.a4.repeats.SINE.gff3\r\n",
      "Panopea-generosa-v1.0.a4.repeats.Simple_repeat.gff3\r\n",
      "Panopea-generosa-v1.0.a4.repeats.Unknown.gff3\r\n",
      "Panopea-generosa-v1.0.a4.tRNA.gff3\r\n"
     ]
    }
   ],
   "source": [
    "!ls /Users/Shelly/Downloads/feature_tracks/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "##gff-version 3\r\n",
      "##Generated using GenSAS, Tuesday 26th of November 2019 07:12:25 PM\r\n",
      "##Project Name : Pgenerosa_v074\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\texon\t2\t125\t.\t+\t.\tID=PGEN_.00g000010.m01.exon01;Name=PGEN_.00g000010.m01.exon01;Parent=PGEN_.00g000010.m01;original_ID=21510-PGEN_.00g234140.m01.exon1;Alias=21510-PGEN_.00g234140.m01.exon1\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\texon\t1995\t2095\t.\t+\t.\tID=PGEN_.00g000010.m01.exon02;Name=PGEN_.00g000010.m01.exon02;Parent=PGEN_.00g000010.m01;original_ID=21510-PGEN_.00g234140.m01.exon2;Alias=21510-PGEN_.00g234140.m01.exon2\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\texon\t3325\t3495\t.\t+\t.\tID=PGEN_.00g000010.m01.exon03;Name=PGEN_.00g000010.m01.exon03;Parent=PGEN_.00g000010.m01;original_ID=21510-PGEN_.00g234140.m01.exon3;Alias=21510-PGEN_.00g234140.m01.exon3\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\texon\t4651\t4719\t.\t+\t.\tID=PGEN_.00g000010.m01.exon04;Name=PGEN_.00g000010.m01.exon04;Parent=PGEN_.00g000010.m01;original_ID=21510-PGEN_.00g234140.m01.exon4;Alias=21510-PGEN_.00g234140.m01.exon4\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\texon\t19808\t19943\t.\t-\t.\tID=PGEN_.00g000020.m01.exon01;Name=PGEN_.00g000020.m01.exon01;Parent=PGEN_.00g000020.m01;original_ID=21510-PGEN_.00g234150.m01.exon10;Alias=21510-PGEN_.00g234150.m01.exon10\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\texon\t21133\t21362\t.\t-\t.\tID=PGEN_.00g000020.m01.exon02;Name=PGEN_.00g000020.m01.exon02;Parent=PGEN_.00g000020.m01;original_ID=21510-PGEN_.00g234150.m01.exon9;Alias=21510-PGEN_.00g234150.m01.exon9\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\texon\t22487\t22613\t.\t-\t.\tID=PGEN_.00g000020.m01.exon03;Name=PGEN_.00g000020.m01.exon03;Parent=PGEN_.00g000020.m01;original_ID=21510-PGEN_.00g234150.m01.exon8;Alias=21510-PGEN_.00g234150.m01.exon8\r\n"
     ]
    }
   ],
   "source": [
    "!head /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.exon.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!rm /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.CpG.gff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#generate intron gff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/Shelly/Downloads/feature_tracks\n"
     ]
    }
   ],
   "source": [
    "cd /Users/Shelly/Downloads/feature_tracks/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_10\t0\t1\r\n",
      "Scaffold_10\t1145\t19539\r\n",
      "Scaffold_10\t36845\t36983\r\n",
      "Scaffold_10\t37535\t41156\r\n",
      "Scaffold_10\t41960\t61913\r\n",
      "Scaffold_10\t82099\t90960\r\n",
      "Scaffold_10\t109412\t135014\r\n",
      "Scaffold_10\t215664\t218086\r\n",
      "Scaffold_10\t219013\t225102\r\n",
      "Scaffold_10\t230053\t232785\r\n"
     ]
    }
   ],
   "source": [
    "!head Panopea-generosa-v1.0.a4.intergenic.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_10\t19684\t20882\r\n",
      "Scaffold_10\t21135\t21899\r\n",
      "Scaffold_10\t21979\t23162\r\n",
      "Scaffold_10\t23252\t24165\r\n",
      "Scaffold_10\t24256\t26334\r\n",
      "Scaffold_10\t26461\t28467\r\n",
      "Scaffold_10\t35936\t36093\r\n",
      "Scaffold_10\t28624\t35328\r\n",
      "Scaffold_10\t61983\t65309\r\n",
      "Scaffold_10\t65860\t81002\r\n"
     ]
    }
   ],
   "source": [
    "!head Panopea-generosa-v1.0.a4.introns.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#convert .bed files to .gff format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!cat Panopea-generosa-v1.0.a4.intergenic.bed | \\\n",
    "awk -F\"\\t\" '{print $1\"\\t\"\"NA\"\"\\t\"\"intergenic\"\"\\t\"$2\"\\t\"$3\"\\t\"\"NA\"\"\\t\"\"NA\"\"\\t\"\"NA\"\"\\t\"\"NA\"}' \\\n",
    "> Panopea-generosa-v1.0.a4.intergenic.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_10\tNA\tintergenic\t0\t1\tNA\tNA\tNA\tNA\r\n",
      "Scaffold_10\tNA\tintergenic\t1145\t19539\tNA\tNA\tNA\tNA\r\n",
      "Scaffold_10\tNA\tintergenic\t36845\t36983\tNA\tNA\tNA\tNA\r\n",
      "Scaffold_10\tNA\tintergenic\t37535\t41156\tNA\tNA\tNA\tNA\r\n",
      "Scaffold_10\tNA\tintergenic\t41960\t61913\tNA\tNA\tNA\tNA\r\n",
      "Scaffold_10\tNA\tintergenic\t82099\t90960\tNA\tNA\tNA\tNA\r\n",
      "Scaffold_10\tNA\tintergenic\t109412\t135014\tNA\tNA\tNA\tNA\r\n",
      "Scaffold_10\tNA\tintergenic\t215664\t218086\tNA\tNA\tNA\tNA\r\n",
      "Scaffold_10\tNA\tintergenic\t219013\t225102\tNA\tNA\tNA\tNA\r\n",
      "Scaffold_10\tNA\tintergenic\t230053\t232785\tNA\tNA\tNA\tNA\r\n"
     ]
    }
   ],
   "source": [
    "!head Panopea-generosa-v1.0.a4.intergenic.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!cat Panopea-generosa-v1.0.a4.introns.bed | \\\n",
    "awk -F\"\\t\" '{print $1\"\\t\"\"NA\"\"\\t\"\"intron\"\"\\t\"$2\"\\t\"$3\"\\t\"\"NA\"\"\\t\"\"NA\"\"\\t\"\"NA\"\"\\t\"\"NA\"}' \\\n",
    "> Panopea-generosa-v1.0.a4.intron.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#concatenate all gffs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat /Users/Shelly/Downloads/feature_tracks/*.gff3 \\\n",
    "> /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error: malformed GFF entry at line 508877. Coordinate detected that is < 1. Exiting.\r\n"
     ]
    }
   ],
   "source": [
    "!/Users/Shelly/anaconda3/bin/bedtools sort -i Panopea-generosa-v1.0.a4.gff3 > Panopea-generosa-v1.0.a4.sorted.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14\r\n"
     ]
    }
   ],
   "source": [
    "!grep -c gff-version Panopea-generosa-v1.0.a4.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 4106483 Panopea-generosa-v1.0.a4.gff3\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l Panopea-generosa-v1.0.a4.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!awk -F\"\\t\" '{if(($1 !~/gff-version/) && ($1 !~/Generated/)&&($1!~/Project/))print $0}' \\\n",
    "Panopea-generosa-v1.0.a4.gff3 \\\n",
    "> Panopea-generosa-v1.0.a4.noheader.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 4106441 Panopea-generosa-v1.0.a4.noheader.gff3\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l Panopea-generosa-v1.0.a4.noheader.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error: malformed GFF entry at line 508868. Coordinate detected that is < 1. Exiting.\r\n"
     ]
    }
   ],
   "source": [
    "!/Users/Shelly/anaconda3/bin/bedtools sort -i \\\n",
    "Panopea-generosa-v1.0.a4.noheader.gff3 \\\n",
    "> Panopea-generosa-v1.0.a4.sorted.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_18\tGenSAS_5d9637f372b5d-publish\tgene\t27538258\t27547464\t.\t+\t.\tID=PGEN_.00g349390;Name=PGEN_.00g349390;original_ID=21510-PGEN_.00g167540;Alias=21510-PGEN_.00g167540;original_name=21510-PGEN_.00g167540;Notes=IPR006212 [InterProScan 5.29-68.0]\r\n",
      "Scaffold_18\tGenSAS_5d9637f372b5d-publish\tgene\t27560960\t27568028\t.\t+\t.\tID=PGEN_.00g349400;Name=PGEN_.00g349400;original_ID=21510-PGEN_.00g167550;Alias=21510-PGEN_.00g167550;original_name=21510-PGEN_.00g167550;Notes=sp|P62870|ELOB_RAT [BLAST protein vs protein (blastp) 2.7.1],sp|P62869|ELOB_MOUSE [DIAMOND Functional 0.9.22],IPR000626 [InterProScan 5.29-68.0],PF11976.3 [Pfam 1.6]\r\n",
      "Scaffold_18\tGenSAS_5d9637f372b5d-publish\tgene\t27596434\t27597132\t.\t+\t.\tID=PGEN_.00g349410;Name=PGEN_.00g349410;original_ID=21510-PGEN_.00g167560;Alias=21510-PGEN_.00g167560;original_name=21510-PGEN_.00g167560;Notes=PF07423.6 [Pfam 1.6]\r\n",
      "Scaffold_18\tGenSAS_5d9637f372b5d-publish\tgene\t27616208\t27641361\t.\t-\t.\tID=PGEN_.00g349420;Name=PGEN_.00g349420;original_ID=21510-PGEN_.00g167570;Alias=21510-PGEN_.00g167570;original_name=21510-PGEN_.00g167570\r\n",
      "Scaffold_18\tGenSAS_5d9637f372b5d-publish\tgene\t27644055\t27647245\t.\t-\t.\tID=PGEN_.00g349430;Name=PGEN_.00g349430;original_ID=21510-PGEN_.00g167580;Alias=21510-PGEN_.00g167580;original_name=21510-PGEN_.00g167580;Notes=PF06794.7 [Pfam 1.6]\r\n",
      "Scaffold_18\tGenSAS_5d9637f372b5d-publish\tgene\t27686936\t27710873\t.\t+\t.\tID=PGEN_.00g349440;Name=PGEN_.00g349440;original_ID=21510-PGEN_.00g167590;Alias=21510-PGEN_.00g167590;original_name=21510-PGEN_.00g167590\r\n",
      "Scaffold_18\tGenSAS_5d9637f372b5d-publish\tgene\t27711059\t27711466\t.\t-\t.\tID=PGEN_.00g349450;Name=PGEN_.00g349450;original_ID=21510-PGEN_.00g167600;Alias=21510-PGEN_.00g167600;original_name=21510-PGEN_.00g167600;Notes=IPR016197 [InterProScan 5.29-68.0]\r\n",
      "Scaffold_18\tGenSAS_5d9637f372b5d-publish\tgene\t27713091\t27719397\t.\t-\t.\tID=PGEN_.00g349460;Name=PGEN_.00g349460;original_ID=21510-PGEN_.00g167610;Alias=21510-PGEN_.00g167610;original_name=21510-PGEN_.00g167610\r\n",
      "Scaffold_18\tGenSAS_5d9637f372b5d-publish\tgene\t27719756\t27720187\t.\t-\t.\tID=PGEN_.00g349470;Name=PGEN_.00g349470;original_ID=21510-PGEN_.00g167620;Alias=21510-PGEN_.00g167620;original_name=21510-PGEN_.00g167620\r\n",
      "Scaffold_10\tNA\tintergenic\t0\t1\tNA\tNA\tNA\tNA\r\n"
     ]
    }
   ],
   "source": [
    "!head -n 508868 Panopea-generosa-v1.0.a4.noheader.gff3 | tail"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#just make a big bed file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!awk -F\"\\t\" '{print $1\"\\t\"$4\"\\t\"$5\"\\t\"$3}' Panopea-generosa-v1.0.a4.noheader.gff3 \\\n",
    "> Panopea-generosa-v1.0.a4.noheader.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!/Users/Shelly/anaconda3/bin/bedtools sort -i \\\n",
    "Panopea-generosa-v1.0.a4.noheader.bed \\\n",
    "> Panopea-generosa-v1.0.a4.sorted.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Panopea-generosa-v1.0.a4.sorted.bed\n",
      "   176421900 100%  151.68MB/s    0:00:01 (xfer#1, to-check=0/1)\n",
      "rsync: mkstemp \"/Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/.Panopea-generosa-v1.0.a4.sorted.bed.hMGzMH\" failed: Permission denied (13)\n",
      "\n",
      "sent 176443548 bytes  received 42 bytes  70577436.00 bytes/sec\n",
      "total size is 176421900  speedup is 1.00\n",
      "rsync error: some files could not be transferred (code 23) at /BuildRoot/Library/Caches/com.apple.xbs/Sources/rsync/rsync-52/rsync/main.c(996) [sender=2.6.9]\n"
     ]
    }
   ],
   "source": [
    "#copied sorted bed file to Gannet:\n",
    "!rsync --verbose --progress Panopea-generosa-v1.0.a4.sorted.bed /Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Shellys-MacBook-Pro:feature_tracks Shelly$ rsync --verbose --progress Panopea-generosa-v1.0.a4.sorted.bed /Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/\n",
    "Panopea-generosa-v1.0.a4.sorted.bed\n",
    "   176421900 100%  130.81MB/s    0:00:01 (xfer#1, to-check=0/1)\n",
    "rsync: mkstemp \"/Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/.Panopea-generosa-v1.0.a4.sorted.bed.ZoKXrD\" failed: Permission denied (13)\n",
    "\n",
    "sent 176443548 bytes  received 42 bytes  117629060.00 bytes/sec\n",
    "total size is 176421900  speedup is 1.00\n",
    "rsync error: some files could not be transferred (code 23) at /BuildRoot/Library/Caches/com.apple.xbs/Sources/rsync/rsync-52/rsync/main.c(996) [sender=2.6.9]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t0\t1\tintergenic\r\n",
      "Scaffold_01\t2\t4719\tgene\r\n",
      "Scaffold_01\t2\t125\texon\r\n",
      "Scaffold_01\t2\t4719\tmRNA\r\n",
      "Scaffold_01\t2\t125\tCDS\r\n",
      "Scaffold_01\t126\t1994\tintron\r\n",
      "Scaffold_01\t265\t338\trepeat_region\r\n",
      "Scaffold_01\t265\t338\trepeat_region\r\n",
      "Scaffold_01\t855\t1050\trepeat_region\r\n",
      "Scaffold_01\t855\t1050\trepeat_region\r\n"
     ]
    }
   ],
   "source": [
    "!head /Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/Panopea-generosa-v1.0.a4.sorted.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t0\t1\tintergenic\r\n",
      "Scaffold_01\t2\t4719\tgene\r\n",
      "Scaffold_01\t2\t125\texon\r\n",
      "Scaffold_01\t2\t4719\tmRNA\r\n",
      "Scaffold_01\t2\t125\tCDS\r\n",
      "Scaffold_01\t126\t1994\tintron\r\n",
      "Scaffold_01\t265\t338\trepeat_region\r\n",
      "Scaffold_01\t855\t1050\trepeat_region\r\n",
      "Scaffold_01\t1040\t1183\trepeat_region\r\n",
      "Scaffold_01\t1275\t1466\trepeat_region\r\n"
     ]
    }
   ],
   "source": [
    "!cat /Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/Panopea-generosa-v1.0.a4.sorted.bed |\\\n",
    "uniq | head\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "cat /Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/Panopea-generosa-v1.0.a4.sorted.bed | \\\n",
    "uniq > /Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/Panopea-generosa-v1.0.a4.uniq.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#DMR files to match with features are here:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t26811206\t26811326\td10\r\n",
      "Scaffold_02\t47739891\t47740441\td10\r\n",
      "Scaffold_02\t58122038\t58122086\td10\r\n",
      "Scaffold_02\t61737304\t61737425\td10\r\n",
      "Scaffold_03\t1735906\t1736047\td10\r\n",
      "Scaffold_03\t4272163\t4272822\td10\r\n",
      "Scaffold_03\t4273751\t4274584\td10\r\n",
      "Scaffold_03\t7176961\t7176994\td10\r\n",
      "Scaffold_03\t13433176\t13433235\td10\r\n",
      "Scaffold_03\t14697169\t14697606\td10\r\n"
     ]
    }
   ],
   "source": [
    "!head /Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses/20200320_anno/aov_0.05pH_d10DMR.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      41 /Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses/20200320_anno/aov_0.05pH_d10DMR.bed\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l /Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses/20200320_anno/aov_0.05pH_d10DMR.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# match DMRs with features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "FILES=/Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses/20200320_anno/aov_0.05*DMR.bed\n",
    "\n",
    "for f in $FILES\n",
    "do\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/intersectBed \\\n",
    "-a ${f} \\\n",
    "-b /Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks/Panopea-generosa-v1.0.a4.uniq.bed \\\n",
    "-wb \\\n",
    "> /Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses/20200320_anno/$(basename ${f%DMR.bed})_0403.txt\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t3532891\t3533128\tamb\tScaffold_01\t3530730\t3534814\tintron\r\n",
      "Scaffold_01\t3532891\t3533128\tamb\tScaffold_01\t3532687\t3533445\trepeat_region\r\n",
      "Scaffold_01\t3532891\t3533128\tamb\tScaffold_01\t3530653\t3562178\tmRNA\r\n",
      "Scaffold_01\t3532891\t3533128\tamb\tScaffold_01\t3530653\t3562178\tgene\r\n",
      "Scaffold_01\t22426167\t22426204\tamb\tScaffold_01\t22426110\t22426204\trepeat_region\r\n",
      "Scaffold_01\t22426183\t22426265\tamb\tScaffold_01\t22426183\t22426277\trepeat_region\r\n",
      "Scaffold_01\t22426167\t22426265\tamb\tScaffold_01\t22419088\t22454118\tintergenic\r\n",
      "Scaffold_01\t46681763\t46681990\tamb\tScaffold_01\t46678058\t46691928\tintergenic\r\n",
      "Scaffold_01\t52430867\t52430898\tamb\tScaffold_01\t52399513\t52442421\tintergenic\r\n",
      "Scaffold_01\t88434573\t88434644\tamb\tScaffold_01\t88433511\t88434919\tmRNA\r\n",
      "Scaffold_01\t88434573\t88434644\tamb\tScaffold_01\t88433511\t88434919\tgene\r\n",
      "Scaffold_01\t88434573\t88434644\tamb\tScaffold_01\t88434050\t88434919\texon\r\n",
      "Scaffold_01\t88434573\t88434644\tamb\tScaffold_01\t88434050\t88434919\tCDS\r\n",
      "Scaffold_02\t59898836\t59898874\tamb\tScaffold_02\t59897854\t59899328\trepeat_region\r\n",
      "Scaffold_02\t59898836\t59898874\tamb\tScaffold_02\t59857300\t59904751\tintergenic\r\n",
      "Scaffold_03\t4272247\t4272284\tamb\tScaffold_03\t4272247\t4272284\trepeat_region\r\n",
      "Scaffold_03\t4272751\t4272797\tamb\tScaffold_03\t4272751\t4272797\trepeat_region\r\n",
      "Scaffold_03\t4272115\t4272863\tamb\tScaffold_03\t4255486\t4274021\tintergenic\r\n",
      "Scaffold_03\t4274179\t4274270\tamb\tScaffold_03\t4274022\t4274270\tmRNA\r\n",
      "Scaffold_03\t4274179\t4274270\tamb\tScaffold_03\t4274022\t4274270\tgene\r\n",
      "Scaffold_03\t4274179\t4274270\tamb\tScaffold_03\t4274022\t4274270\tCDS\r\n",
      "Scaffold_03\t4274179\t4274270\tamb\tScaffold_03\t4274022\t4274270\texon\r\n",
      "Scaffold_03\t4274270\t4274332\tamb\tScaffold_03\t4274270\t4274344\tintergenic\r\n",
      "Scaffold_03\t6261197\t6261340\tamb\tScaffold_03\t6261189\t6261495\trepeat_region\r\n",
      "Scaffold_03\t6261197\t6261340\tamb\tScaffold_03\t6208421\t6351133\tintergenic\r\n",
      "Scaffold_03\t10781017\t10781087\tamb\tScaffold_03\t10780695\t10781112\tCDS\r\n",
      "Scaffold_03\t10781017\t10781087\tamb\tScaffold_03\t10780695\t10781112\texon\r\n",
      "Scaffold_03\t10781017\t10781087\tamb\tScaffold_03\t10777571\t10781112\tgene\r\n",
      "Scaffold_03\t10781017\t10781087\tamb\tScaffold_03\t10777571\t10781112\tmRNA\r\n",
      "Scaffold_03\t40805936\t40806118\tamb\tScaffold_03\t40793203\t40807304\tintergenic\r\n",
      "Scaffold_04\t716827\t716907\tamb\tScaffold_04\t680954\t761593\tintergenic\r\n",
      "Scaffold_04\t4220453\t4220617\tamb\tScaffold_04\t4214862\t4220716\tgene\r\n",
      "Scaffold_04\t4220453\t4220617\tamb\tScaffold_04\t4214862\t4220716\tmRNA\r\n",
      "Scaffold_04\t4220453\t4220590\tamb\tScaffold_04\t4220019\t4220590\tintron\r\n",
      "Scaffold_04\t4220455\t4220564\tamb\tScaffold_04\t4220455\t4220564\trepeat_region\r\n",
      "Scaffold_04\t4220591\t4220617\tamb\tScaffold_04\t4220591\t4220716\texon\r\n",
      "Scaffold_04\t4220591\t4220617\tamb\tScaffold_04\t4220591\t4220716\tCDS\r\n",
      "Scaffold_04\t7485185\t7485481\tamb\tScaffold_04\t7485185\t7485481\tCDS\r\n",
      "Scaffold_04\t7485185\t7485481\tamb\tScaffold_04\t7485185\t7485481\tgene\r\n",
      "Scaffold_04\t7485185\t7485481\tamb\tScaffold_04\t7485185\t7485481\tmRNA\r\n",
      "Scaffold_04\t7485185\t7485481\tamb\tScaffold_04\t7485185\t7485481\texon\r\n",
      "Scaffold_04\t7485481\t7485697\tamb\tScaffold_04\t7485481\t7486070\tintergenic\r\n",
      "Scaffold_04\t7484907\t7485184\tamb\tScaffold_04\t7467079\t7485184\tintergenic\r\n",
      "Scaffold_04\t10439183\t10439416\tamb\tScaffold_04\t10438458\t10440402\tintron\r\n",
      "Scaffold_04\t10439183\t10439416\tamb\tScaffold_04\t10434223\t10451849\tmRNA\r\n",
      "Scaffold_04\t10439183\t10439416\tamb\tScaffold_04\t10434223\t10451849\tgene\r\n",
      "Scaffold_04\t10742822\t10742936\tamb\tScaffold_04\t10727860\t10743952\tintergenic\r\n",
      "Scaffold_04\t12562450\t12563263\tamb\tScaffold_04\t12560752\t12564727\tmRNA\r\n",
      "Scaffold_04\t12562450\t12563263\tamb\tScaffold_04\t12560752\t12564727\tgene\r\n",
      "Scaffold_04\t12562450\t12563263\tamb\tScaffold_04\t12562239\t12563391\texon\r\n"
     ]
    }
   ],
   "source": [
    "!head -50 /Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses/20200320_anno/aov_0.05pH_amb_0403.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!mkdir 20200403_anno"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!mv /Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses/20200320_anno/aov_0.05*_0403.txt \\\n",
    "20200403_anno/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses/20200403_anno\n"
     ]
    }
   ],
   "source": [
    "cd 20200403_anno/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "aov_0.05pH_amb_0403.txt  aov_0.05pH_d135_0403.txt\r\n",
      "aov_0.05pH_d10_0403.txt  aov_0.05pH_d145_0403.txt\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Match features to background regions\n",
    "## need to create background:\n",
    "## 1. find all CpGs covered by at least 3/4 samples per group (use merge.tab files)\n",
    "## 2. find CpGs common to all groups in comparison (e.g. all ambient samples over time, all day 10 samples, etc.)\n",
    "## 3. bin features in master gff with bedtools make windows (DMRs range from ~6bp to ~2kb, so bin features 2kb)\n",
    "## 4. determine binned features that covered CpGs overlap with (bedtools intersect)\n",
    "## 5. filter features for those with at least 3 CpGs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## 1. find all CpGs covered by at least 3/4 samples per group (use merge.tab files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t208\t210\t0.000000\t0\t5\r\n",
      "Scaffold_01\t212\t214\t0.000000\t0\t5\r\n",
      "Scaffold_01\t220\t222\t0.000000\t0\t6\r\n",
      "Scaffold_01\t237\t239\t0.000000\t0\t8\r\n",
      "Scaffold_01\t564\t566\t0.000000\t0\t6\r\n",
      "Scaffold_01\t576\t578\t0.000000\t0\t6\r\n",
      "Scaffold_01\t582\t584\t0.000000\t0\t5\r\n",
      "Scaffold_01\t2508\t2510\t0.000000\t0\t7\r\n",
      "Scaffold_01\t2553\t2555\t0.000000\t0\t6\r\n",
      "Scaffold_01\t3001\t3003\t0.000000\t0\t5\r\n"
     ]
    }
   ],
   "source": [
    "!head /Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# to combine CpG data for all samples for each treatment group,\n",
    "# I created a script to that I could run it without interruption through a screen session\n",
    "# https://gannet.fish.washington.edu/metacarcinus/Pgenerosa/analyses/20200406/0.75Xcov_CpG.sh\n",
    "# I also pasted the script below\n",
    "\n",
    "```\n",
    "%%bash\n",
    "\n",
    "set -ex\n",
    "\n",
    "#first create variables for each file grouping\n",
    "declare -a amb_day0=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-41_S38_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-42_S39_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-43_S40_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-44_S41_L005_5x.tab\"\n",
    "\n",
    "declare -a amb_day10=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-119_S31_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-120_S32_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-135_S35_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-136_S36_L005_5x.tab\"\n",
    "\n",
    "declare -a amb_day135=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-151_S2_L002_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-152_S3_L002_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-153_S4_L002_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-154_S5_L002_5x.tab\"\n",
    "\n",
    "declare -a amb_day145=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-181_S16_L003_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-182_S17_L003_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-184_S18_L003_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-185_S19_L003_5x.tab\"\n",
    "\n",
    "declare -a low_day10=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-104_S28_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-127_S33_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-128_S34_L005_5x.tab\"\n",
    "\n",
    "declare -a low_day135=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-160_S7_L002_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-161_S8_L002_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-162_S9_L002_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-159_S6_L002_5x.tab\"\n",
    "\n",
    "declare -a slow_day10=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-111_S29_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-113_S30_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-143_S37_L005_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-145_S38_L005_5x.tab\"\n",
    "\n",
    "declare -a slow_day135=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-167_S10_L002_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-168_S11_L002_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-169_S12_L002_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-170_S13_L002_5x.tab\"\n",
    "\n",
    "declare -a lamb_day145=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-175_S14_L003_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-176_S15_L003_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-193_S22_L003_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-194_S23_L003_5x.tab\"\n",
    "\n",
    "declare -a slamb_day145=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-187_S20_L003_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-188_S21_L003_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-199_S24_L003_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-200_S25_L003_5x.tab\"\n",
    "\n",
    "declare -a ambl_day145=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-205_S26_L004_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-206_S27_L004_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-226_S34_L004_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-227_S35_L004_5x.tab\"\n",
    "\n",
    "declare -a lowlow_day145=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-208_S28_L004_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-209_S29_L004_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-229_S36_L004_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-230_S37_L004_5x.tab\"\n",
    "\n",
    "declare -a sllow_day145=\"/Users/strigg/Desktop/20200327/merged_tab/EPI-214_S30_L004_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-215_S31_L004_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-220_S32_L004_5x.tab \n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-221_S33_L004_5x.tab\"\n",
    "\n",
    "#next create another list of variable names\n",
    "\n",
    "groups=(amb_day0 amb_day10 amb_day135 amb_day145 low_day10 low_day135 slow_day10 slow_day135 lamb_day145 slamb_day145 ambl_day145 lowlow_day145 sllow_day145)\n",
    "\n",
    "#next loop through variable names to combine the files within each experimental group\n",
    "#cat all group files together\n",
    "#if the context is CG print the chromosome and position\n",
    "#sort data\n",
    "#unique and count the frequency of positions\n",
    "#if the position is overlapping in 3/4 samples keep\n",
    "\n",
    "\n",
    "for f in \"${groups[@]}\"\n",
    "do\n",
    "x=\"${f}\" \n",
    "cat ${!x} | \\\n",
    "awk '{print $1,$2,$3}' | \\\n",
    "sort | \\\n",
    "uniq -c | \\\n",
    "awk '{if($1>2)print $2\"\\t\"$3\"\\t\"$4}' \\\n",
    "> ${f}.3xCpG.bed\n",
    "done\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# the log file is here: https://gannet.fish.washington.edu/metacarcinus/Pgenerosa/analyses/20200406/screenlog.0\n",
    "# the script output is also here: https://gannet.fish.washington.edu/metacarcinus/Pgenerosa/analyses/20200406"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# next combine all CpG covered by at least 3 individuals per group\n",
    "#ran code in "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m0.75Xcov_CpG.sh\u001b[m\u001b[m         ambl_day145.3xCpG.bed   screenlog.0\r\n",
      "amb_day0.3xCpG.bed      lamb_day145.3xCpG.bed   slamb_day145.3xCpG.bed\r\n",
      "amb_day10.3xCpG.bed     low_day10.3xCpG.bed     sllow_day145.3xCpG.bed\r\n",
      "amb_day135.3xCpG.bed    low_day135.3xCpG.bed    slow_day10.3xCpG.bed\r\n",
      "amb_day145.3xCpG.bed    lowlow_day145.3xCpG.bed slow_day135.3xCpG.bed\r\n"
     ]
    }
   ],
   "source": [
    "!ls /Users/strigg/Desktop/20200406"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## 2. find CpGs common to all groups in comparison (e.g. all ambient samples over time, all day 10 samples, etc.)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Day 10 samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 strigg  staff    36M Apr  6 11:45 /Users/strigg/Desktop/20200406/amb_day10.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    30M Apr  6 11:52 /Users/strigg/Desktop/20200406/low_day10.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    31M Apr  6 11:57 /Users/strigg/Desktop/20200406/slow_day10.3xCpG.bed\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lh /Users/strigg/Desktop/20200406/*day10*.3xCpG.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!cat /Users/strigg/Desktop/20200406/*day10*.3xCpG.bed |\\\n",
    "sort | \\\n",
    "uniq -c | \\\n",
    "awk '{if($1==3)print $2\"\\t\"$3\"\\t\"$4}' \\\n",
    "> /Users/strigg/Desktop/20200406/day10.3xCpG.allgrps.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  650976 /Users/strigg/Desktop/20200406/day10.3xCpG.allgrps.bed\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l /Users/strigg/Desktop/20200406/day10.3xCpG.allgrps.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Day 135 samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 strigg  staff    17M Apr  6 11:47 /Users/strigg/Desktop/20200406/amb_day135.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    11M Apr  6 11:54 /Users/strigg/Desktop/20200406/low_day135.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    15M Apr  6 11:59 /Users/strigg/Desktop/20200406/slow_day135.3xCpG.bed\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lh /Users/strigg/Desktop/20200406/*day135*.3xCpG.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# day 135\n",
    "\n",
    "!cat /Users/strigg/Desktop/20200406/*day135*.3xCpG.bed |\\\n",
    "sort | \\\n",
    "uniq -c | \\\n",
    "awk '{if($1==3)print $2\"\\t\"$3\"\\t\"$4}' \\\n",
    "> /Users/strigg/Desktop/20200406/day135.3xCpG.allgrps.bed\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  251164 /Users/strigg/Desktop/20200406/day135.3xCpG.allgrps.bed\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l /Users/strigg/Desktop/20200406/day135.3xCpG.allgrps.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Day 145 samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 strigg  staff    13M Apr  6 11:49 /Users/strigg/Desktop/20200406/amb_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff   6.7M Apr  6 12:03 /Users/strigg/Desktop/20200406/ambl_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    24M Apr  6 12:01 /Users/strigg/Desktop/20200406/lamb_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    17M Apr  6 12:05 /Users/strigg/Desktop/20200406/lowlow_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    12M Apr  6 12:03 /Users/strigg/Desktop/20200406/slamb_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    10M Apr  6 12:07 /Users/strigg/Desktop/20200406/sllow_day145.3xCpG.bed\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lh /Users/strigg/Desktop/20200406/*day145*.3xCpG.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# day 145\n",
    "\n",
    "!cat /Users/strigg/Desktop/20200406/*day145*.3xCpG.bed |\\\n",
    "sort | \\\n",
    "uniq -c | \\\n",
    "awk '{if($1==6)print $2\"\\t\"$3\"\\t\"$4}' \\\n",
    "> /Users/strigg/Desktop/20200406/day145.3xCpG.allgrps.bed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  142346 /Users/strigg/Desktop/20200406/day145.3xCpG.allgrps.bed\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l /Users/strigg/Desktop/20200406/day145.3xCpG.allgrps.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# All ambient samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r--  1 strigg  staff    13M Apr  6 11:41 /Users/strigg/Desktop/20200406/amb_day0.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    36M Apr  6 11:45 /Users/strigg/Desktop/20200406/amb_day10.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    17M Apr  6 11:47 /Users/strigg/Desktop/20200406/amb_day135.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    13M Apr  6 11:49 /Users/strigg/Desktop/20200406/amb_day145.3xCpG.bed\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lh /Users/strigg/Desktop/20200406/amb_*.3xCpG.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ambient samples\n",
    "!cat /Users/strigg/Desktop/20200406/amb_*.3xCpG.bed |\\\n",
    "sort | \\\n",
    "uniq -c | \\\n",
    "awk '{if($1==4)print $2\"\\t\"$3\"\\t\"$4}' \\\n",
    "> /Users/strigg/Desktop/20200406/amb.3xCpG.allgrps.bed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  227081 /Users/strigg/Desktop/20200406/amb.3xCpG.allgrps.bed\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l /Users/strigg/Desktop/20200406/amb.3xCpG.allgrps.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# STEP 3: overlap CpGs with coverage in all groups and at least 3/4 individuals per group\n",
    "# with genomic features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t10017891\t10017893\tScaffold_01\t10015950\t10025898\tintergenic\n",
      "Scaffold_01\t10017891\t10017893\tScaffold_01\t10017821\t10018112\trepeat_region\n",
      "Scaffold_01\t10018551\t10018553\tScaffold_01\t10015950\t10025898\tintergenic\n",
      "Scaffold_01\t10018551\t10018553\tScaffold_01\t10018516\t10018602\trepeat_region\n",
      "Scaffold_01\t10018563\t10018565\tScaffold_01\t10015950\t10025898\tintergenic\n",
      "Scaffold_01\t10018563\t10018565\tScaffold_01\t10018516\t10018602\trepeat_region\n",
      "Scaffold_01\t10018572\t10018574\tScaffold_01\t10015950\t10025898\tintergenic\n",
      "Scaffold_01\t10018572\t10018574\tScaffold_01\t10018516\t10018602\trepeat_region\n",
      "Scaffold_01\t10018624\t10018626\tScaffold_01\t10015950\t10025898\tintergenic\n",
      "Scaffold_01\t10018624\t10018626\tScaffold_01\t10018589\t10018683\trepeat_region\n"
     ]
    }
   ],
   "source": [
    "%%bash \n",
    "\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/intersectBed \\\n",
    "-a /Users/strigg/Desktop/20200406/amb.3xCpG.allgrps.bed \\\n",
    "-b /Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/Panopea-generosa-v1.0.a4.uniq.bed \\\n",
    "-wb |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   4 Scaffold_01\t10017891\t10017893\r\n",
      "   4 Scaffold_01\t10018551\t10018553\r\n",
      "   4 Scaffold_01\t10018563\t10018565\r\n",
      "   4 Scaffold_01\t10018572\t10018574\r\n",
      "   4 Scaffold_01\t10018624\t10018626\r\n",
      "   4 Scaffold_01\t10018645\t10018647\r\n",
      "   4 Scaffold_01\t10018697\t10018699\r\n",
      "   4 Scaffold_01\t10018718\t10018720\r\n",
      "   4 Scaffold_01\t10018770\t10018772\r\n",
      "   4 Scaffold_01\t10018791\t10018793\r\n"
     ]
    }
   ],
   "source": [
    "!head /Users/strigg/Desktop/20200406/amb.3xCpG.allgrps.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "FILES=/Users/strigg/Desktop/20200406/*.3xCpG.allgrps.bed\n",
    "\n",
    "for f in $FILES\n",
    "do\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/intersectBed \\\n",
    "-a ${f} \\\n",
    "-b /Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/Panopea-generosa-v1.0.a4.uniq.bed \\\n",
    "-wb \\\n",
    "> /Users/strigg/Desktop/20200406/$(basename ${f%.3xCpG.allgrps.bed})_features.txt\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 1027488\r\n",
      "-rwxr-xr-x  1 strigg  staff   4.4K Apr  6 11:38 \u001b[31m0.75Xcov_CpG.sh\u001b[m\u001b[m\r\n",
      "-rw-r--r--  1 strigg  staff   6.4M Apr  6 12:32 amb.3xCpG.allgrps.bed\r\n",
      "-rw-r--r--  1 strigg  staff    13M Apr  6 11:41 amb_day0.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    36M Apr  6 11:45 amb_day10.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    17M Apr  6 11:47 amb_day135.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    13M Apr  6 11:49 amb_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    42M Apr  6 12:37 amb_features.txt\r\n",
      "-rw-r--r--  1 strigg  staff   6.7M Apr  6 12:03 ambl_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    18M Apr  6 12:36 day10.3xCpG.allgrps.bed\r\n",
      "-rw-r--r--  1 strigg  staff   118M Apr  6 12:37 day10_features.txt\r\n",
      "-rw-r--r--  1 strigg  staff   7.1M Apr  6 12:37 day135.3xCpG.allgrps.bed\r\n",
      "-rw-r--r--  1 strigg  staff    45M Apr  6 12:38 day135_features.txt\r\n",
      "-rw-r--r--  1 strigg  staff   4.0M Apr  6 12:33 day145.3xCpG.allgrps.bed\r\n",
      "-rw-r--r--  1 strigg  staff    25M Apr  6 12:38 day145_features.txt\r\n",
      "-rw-r--r--  1 strigg  staff    24M Apr  6 12:01 lamb_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    30M Apr  6 11:52 low_day10.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    11M Apr  6 11:54 low_day135.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    17M Apr  6 12:05 lowlow_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff   9.3K Apr  6 12:07 screenlog.0\r\n",
      "-rw-r--r--  1 strigg  staff    12M Apr  6 12:03 slamb_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    10M Apr  6 12:07 sllow_day145.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    31M Apr  6 11:57 slow_day10.3xCpG.bed\r\n",
      "-rw-r--r--  1 strigg  staff    15M Apr  6 11:59 slow_day135.3xCpG.bed\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lh /Users/strigg/Desktop/20200406/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  652535 /Users/strigg/Desktop/20200406/amb_features.txt\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l /Users/strigg/Desktop/20200406/amb_features.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t10017891\t10017893\tScaffold_01\t10015950\t10025898\tintergenic\r\n",
      "Scaffold_01\t10017891\t10017893\tScaffold_01\t10017821\t10018112\trepeat_region\r\n",
      "Scaffold_01\t10018551\t10018553\tScaffold_01\t10015950\t10025898\tintergenic\r\n",
      "Scaffold_01\t10018551\t10018553\tScaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10018563\t10018565\tScaffold_01\t10015950\t10025898\tintergenic\r\n",
      "Scaffold_01\t10018563\t10018565\tScaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10018572\t10018574\tScaffold_01\t10015950\t10025898\tintergenic\r\n",
      "Scaffold_01\t10018572\t10018574\tScaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10018624\t10018626\tScaffold_01\t10015950\t10025898\tintergenic\r\n",
      "Scaffold_01\t10018624\t10018626\tScaffold_01\t10018589\t10018683\trepeat_region\r\n"
     ]
    }
   ],
   "source": [
    "!head /Users/strigg/Desktop/20200406/amb_features.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# STEP 4: filter features for those with at least 3 CpGs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t10015950\t10025898\tintergenic\r\n",
      "Scaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10019296\t10019390\trepeat_region\r\n",
      "Scaffold_01\t10019581\t10019667\trepeat_region\r\n",
      "Scaffold_01\t10019946\t10020040\trepeat_region\r\n",
      "Scaffold_01\t10022552\t10023073\trepeat_region\r\n",
      "Scaffold_01\t10025899\t10027125\tCDS\r\n",
      "Scaffold_01\t10025899\t10027125\texon\r\n",
      "Scaffold_01\t10025899\t10027125\tgene\r\n",
      "Scaffold_01\t10025899\t10027125\tmRNA\r\n",
      "awk: write error on /dev/stdout\r\n",
      " input record number 1416, file \r\n",
      " source line number 1\r\n"
     ]
    }
   ],
   "source": [
    "!cat /Users/strigg/Desktop/20200406/day10_features.txt |\\\n",
    "awk -F\"\\t\" '{print $4,$5,$6,$7}' |\\\n",
    "sort |\\\n",
    "uniq -c |\\\n",
    "awk '{if($1>2)print $2\"\\t\"$3\"\\t\"$4\"\\t\"$5}' |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#seems like i need to bin the feature file because the features are too big\n",
    "# e.g. a DMR would never be 10kb long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!/Users/Shared/bioinformatics/bedtools2/bin/bedtools makewindows \\\n",
    "-b /Volumes/web/metacarcinus/Pgenerosa/analyses/20200403/Panopea-generosa-v1.0.a4.uniq.bed \\\n",
    "-w 2000 \\\n",
    "-i src \\\n",
    "> /Users/strigg/Desktop/20200406/Panopea-generosa-v1.0.a4.2Kbin.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#determine overlap with binned features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "FILES=/Users/strigg/Desktop/20200406/*.3xCpG.allgrps.bed\n",
    "\n",
    "for f in $FILES\n",
    "do\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/intersectBed \\\n",
    "-a ${f} \\\n",
    "-b /Users/strigg/Desktop/20200406/Panopea-generosa-v1.0.a4.2Kbin.bed \\\n",
    "-wb \\\n",
    "> /Users/strigg/Desktop/20200406/$(basename ${f%.3xCpG.allgrps.bed})_features.txt\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t10018551\t10018553\tScaffold_01\t10017950\t10019950\tintergenic\r\n",
      "Scaffold_01\t10018551\t10018553\tScaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10018563\t10018565\tScaffold_01\t10017950\t10019950\tintergenic\r\n",
      "Scaffold_01\t10018563\t10018565\tScaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10018572\t10018574\tScaffold_01\t10017950\t10019950\tintergenic\r\n",
      "Scaffold_01\t10018572\t10018574\tScaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10018624\t10018626\tScaffold_01\t10017950\t10019950\tintergenic\r\n",
      "Scaffold_01\t10018624\t10018626\tScaffold_01\t10018589\t10018683\trepeat_region\r\n",
      "Scaffold_01\t10018645\t10018647\tScaffold_01\t10017950\t10019950\tintergenic\r\n",
      "Scaffold_01\t10018645\t10018647\tScaffold_01\t10018589\t10018683\trepeat_region\r\n"
     ]
    }
   ],
   "source": [
    "!head /Users/strigg/Desktop/20200406/day10_features.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#find features with at least 3 CpGs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t10017950\t10019950\tintergenic\n",
      "Scaffold_01\t10018516\t10018602\trepeat_region\n",
      "Scaffold_01\t10019296\t10019390\trepeat_region\n",
      "Scaffold_01\t10019581\t10019667\trepeat_region\n",
      "Scaffold_01\t10019946\t10020040\trepeat_region\n",
      "Scaffold_01\t10019950\t10021950\tintergenic\n",
      "Scaffold_01\t10021950\t10023950\tintergenic\n",
      "Scaffold_01\t10022552\t10023073\trepeat_region\n",
      "Scaffold_01\t10025899\t10027125\tCDS\n",
      "Scaffold_01\t10025899\t10027125\texon\n",
      "awk: write error on /dev/stdout\n",
      " input record number 1545, file \n",
      " source line number 1\n"
     ]
    }
   ],
   "source": [
    "!cat /Users/strigg/Desktop/20200406/day10_features.txt |\\\n",
    "awk -F\"\\t\" '{print $4,$5,$6,$7}' |\\\n",
    "sort |\\\n",
    "uniq -c |\\\n",
    "awk '{if($1>2)print $2\"\\t\"$3\"\\t\"$4\"\\t\"$5}' |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "FILES=/Users/strigg/Desktop/20200406/*_features.txt\n",
    "\n",
    "for f in $FILES\n",
    "do\n",
    "cat ${f} |\\\n",
    "awk -F\"\\t\" '{print $4,$5,$6,$7}' |\\\n",
    "sort |\\\n",
    "uniq -c |\\\n",
    "awk '{if($1>2)print $2\"\\t\"$3\"\\t\"$4\"\\t\"$5}'\\\n",
    "> /Users/strigg/Desktop/20200406/$(basename ${f%_features.txt})_features.3CpG.txt\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   78649 /Users/strigg/Desktop/20200406/amb_features.3CpG.txt\r\n",
      "  220190 /Users/strigg/Desktop/20200406/day10_features.3CpG.txt\r\n",
      "   84922 /Users/strigg/Desktop/20200406/day135_features.3CpG.txt\r\n",
      "   47393 /Users/strigg/Desktop/20200406/day145_features.3CpG.txt\r\n",
      "  431154 total\r\n"
     ]
    }
   ],
   "source": [
    "#count the number of features containing at least 3 CpGs covered by 3/4 samples per group and by all groups in comparison\n",
    "!wc -l /Users/strigg/Desktop/20200406/*_features.3CpG.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t10017950\t10019950\tintergenic\r\n",
      "Scaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10019223\t10019317\trepeat_region\r\n",
      "Scaffold_01\t10019296\t10019390\trepeat_region\r\n",
      "Scaffold_01\t10019581\t10019667\trepeat_region\r\n",
      "Scaffold_01\t10019946\t10020040\trepeat_region\r\n",
      "Scaffold_01\t10019950\t10021950\tintergenic\r\n",
      "Scaffold_01\t10021950\t10023950\tintergenic\r\n",
      "Scaffold_01\t10022552\t10023073\trepeat_region\r\n",
      "Scaffold_01\t10025899\t10027125\tCDS\r\n"
     ]
    }
   ],
   "source": [
    "!head /Users/strigg/Desktop/20200406/amb_features.3CpG.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#copy data to Gannet\n",
    "(base) ostrich:~ strigg$ cd ~/Desktop/20200406/\n",
    "(base) ostrich:20200406 strigg$ ls\n",
    "0.75Xcov_CpG.sh\t\t\t\tday135_features.3CpG.txt\n",
    "Panopea-generosa-v1.0.a4.2Kbin.bed\tday135_features.txt\n",
    "amb.3xCpG.allgrps.bed\t\t\tday145.3xCpG.allgrps.bed\n",
    "amb_day0.3xCpG.bed\t\t\tday145_features.3CpG.txt\n",
    "amb_day10.3xCpG.bed\t\t\tday145_features.txt\n",
    "amb_day135.3xCpG.bed\t\t\tlamb_day145.3xCpG.bed\n",
    "amb_day145.3xCpG.bed\t\t\tlow_day10.3xCpG.bed\n",
    "amb_features.3CpG.txt\t\t\tlow_day135.3xCpG.bed\n",
    "amb_features.txt\t\t\tlowlow_day145.3xCpG.bed\n",
    "ambl_day145.3xCpG.bed\t\t\tscreenlog.0\n",
    "day10.3xCpG.allgrps.bed\t\t\tslamb_day145.3xCpG.bed\n",
    "day10_features.3CpG.txt\t\t\tsllow_day145.3xCpG.bed\n",
    "day10_features.txt\t\t\tslow_day10.3xCpG.bed\n",
    "day135.3xCpG.allgrps.bed\t\tslow_day135.3xCpG.bed\n",
    "(base) ostrich:20200406 strigg$ cd ..\n",
    "(base) ostrich:Desktop strigg$ rsync --verbose --archive --progress 20200406 /Volumes/web/metacarcinus/Pgenerosa/analyses/\n",
    "building file list ... \n",
    "29 files to consider\n",
    "20200406/\n",
    "20200406/0.75Xcov_CpG.sh\n",
    "        4500 100%    0.00kB/s    0:00:00 (xfer#1, to-check=27/29)\n",
    "20200406/Panopea-generosa-v1.0.a4.2Kbin.bed\n",
    "   123558028 100%   42.60MB/s    0:00:02 (xfer#2, to-check=26/29)\n",
    "20200406/amb.3xCpG.allgrps.bed\n",
    "     6721935 100%    6.64MB/s    0:00:00 (xfer#3, to-check=25/29)\n",
    "20200406/amb_day0.3xCpG.bed\n",
    "    13299555 100%    9.43MB/s    0:00:01 (xfer#4, to-check=24/29)\n",
    "20200406/amb_day10.3xCpG.bed\n",
    "    37920475 100%   28.79MB/s    0:00:01 (xfer#5, to-check=23/29)\n",
    "20200406/amb_day135.3xCpG.bed\n",
    "    18210084 100%   26.31MB/s    0:00:00 (xfer#6, to-check=22/29)\n",
    "20200406/amb_day145.3xCpG.bed\n",
    "    13208291 100%   11.61MB/s    0:00:01 (xfer#7, to-check=21/29)\n",
    "20200406/amb_features.3CpG.txt\n",
    "     2936952 100%   17.73MB/s    0:00:00 (xfer#8, to-check=20/29)\n",
    "20200406/amb_features.txt\n",
    "    43633708 100%   36.22MB/s    0:00:01 (xfer#9, to-check=19/29)\n",
    "20200406/ambl_day145.3xCpG.bed\n",
    "     6989662 100%   18.88MB/s    0:00:00 (xfer#10, to-check=18/29)\n",
    "20200406/day10.3xCpG.allgrps.bed\n",
    "    19272607 100%   21.27MB/s    0:00:00 (xfer#11, to-check=17/29)\n",
    "20200406/day10_features.3CpG.txt\n",
    "     8265802 100%    7.49MB/s    0:00:01 (xfer#12, to-check=16/29)\n",
    "20200406/day10_features.txt\n",
    "   123506566 100%   42.45MB/s    0:00:02 (xfer#13, to-check=15/29)\n",
    "20200406/day135.3xCpG.allgrps.bed\n",
    "     7433676 100%    7.10MB/s    0:00:00 (xfer#14, to-check=14/29)\n",
    "20200406/day135_features.3CpG.txt\n",
    "     3172232 100%    2.78MB/s    0:00:01 (xfer#15, to-check=13/29)\n",
    "20200406/day135_features.txt\n",
    "    47614737 100%   40.29MB/s    0:00:01 (xfer#16, to-check=12/29)\n",
    "20200406/day145.3xCpG.allgrps.bed\n",
    "     4213035 100%   17.47MB/s    0:00:00 (xfer#17, to-check=11/29)\n",
    "20200406/day145_features.3CpG.txt\n",
    "     1768386 100%    6.13MB/s    0:00:00 (xfer#18, to-check=10/29)\n",
    "20200406/day145_features.txt\n",
    "    26554556 100%   28.39MB/s    0:00:00 (xfer#19, to-check=9/29)\n",
    "20200406/lamb_day145.3xCpG.bed\n",
    "    25450730 100%   16.30MB/s    0:00:01 (xfer#20, to-check=8/29)\n",
    "20200406/low_day10.3xCpG.bed\n",
    "    31777745 100%   24.92MB/s    0:00:01 (xfer#21, to-check=7/29)\n",
    "20200406/low_day135.3xCpG.bed\n",
    "    11337506 100%   21.58MB/s    0:00:00 (xfer#22, to-check=6/29)\n",
    "20200406/lowlow_day145.3xCpG.bed\n",
    "    17549704 100%   16.27MB/s    0:00:01 (xfer#23, to-check=5/29)\n",
    "20200406/screenlog.0\n",
    "        9527 100%  281.93kB/s    0:00:00 (xfer#24, to-check=4/29)\n",
    "20200406/slamb_day145.3xCpG.bed\n",
    "    12305784 100%   31.63MB/s    0:00:00 (xfer#25, to-check=3/29)\n",
    "20200406/sllow_day145.3xCpG.bed\n",
    "    10933324 100%   13.35MB/s    0:00:00 (xfer#26, to-check=2/29)\n",
    "20200406/slow_day10.3xCpG.bed\n",
    "    32833889 100%   20.27MB/s    0:00:01 (xfer#27, to-check=1/29)\n",
    "20200406/slow_day135.3xCpG.bed\n",
    "    15313600 100%   14.59MB/s    0:00:01 (xfer#28, to-check=0/29)\n",
    "\n",
    "sent 665879945 bytes  received 642 bytes  40356399.21 bytes/sec\n",
    "total size is 665796596  speedup is 1.00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Validate feature coverage in IGV\n",
    "#1. need to create bw files or just load bams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# My initial thought was to bin the genome evenly to generate background features\n",
    "# instead of binning the genome evenly, it seems best to just match up CpGs with features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#binning genome by 250bp is not the best way because DMRs in fact range from 7 - 2000 bp\n",
    "# this is because the max dist parameter is actually the distance that two DMS can be \n",
    "### for them to be placed into the same block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t0\t250\n",
      "Scaffold_01\t250\t500\n",
      "Scaffold_01\t500\t750\n",
      "Scaffold_01\t750\t1000\n",
      "Scaffold_01\t1000\t1250\n",
      "Scaffold_01\t1250\t1500\n",
      "Scaffold_01\t1500\t1750\n",
      "Scaffold_01\t1750\t2000\n",
      "Scaffold_01\t2000\t2250\n",
      "Scaffold_01\t2250\t2500\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/bedtools makewindows \\\n",
    "-g /Volumes/web/metacarcinus/Pgenerosa/GENOMES/Panopea-generosa-v1.0.fa.fai \\\n",
    "-w 250 | head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/bedtools makewindows \\\n",
    "-g /Volumes/web/metacarcinus/Pgenerosa/GENOMES/Panopea-generosa-v1.0.fa.fai \\\n",
    "-w 250 \\\n",
    "> Panopea-generosa-v1.0.250bp.bin.bed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#now overlap with groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t0\t250\t/Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab\tScaffold_01\t208\t210\t0.000000\t0\t5\n",
      "Scaffold_01\t0\t250\t/Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab\tScaffold_01\t212\t214\t0.000000\t0\t5\n",
      "Scaffold_01\t0\t250\t/Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab\tScaffold_01\t220\t222\t0.000000\t0\t6\n",
      "Scaffold_01\t0\t250\t/Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab\tScaffold_01\t237\t239\t0.000000\t0\t8\n",
      "Scaffold_01\t0\t250\t/Users/strigg/Desktop/20200327/merged_tab/EPI-104_S28_L005_5x.tab\tScaffold_01\t118\t120\t0.000000\t0\t5\n",
      "Scaffold_01\t0\t250\t/Users/strigg/Desktop/20200327/merged_tab/EPI-104_S28_L005_5x.tab\tScaffold_01\t237\t239\t0.000000\t0\t5\n",
      "Scaffold_01\t500\t750\t/Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab\tScaffold_01\t564\t566\t0.000000\t0\t6\n",
      "Scaffold_01\t500\t750\t/Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab\tScaffold_01\t576\t578\t0.000000\t0\t6\n",
      "Scaffold_01\t500\t750\t/Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab\tScaffold_01\t582\t584\t0.000000\t0\t5\n",
      "Scaffold_01\t2000\t2250\t/Users/strigg/Desktop/20200327/merged_tab/EPI-128_S34_L005_5x.tab\tScaffold_01\t2082\t2084\t0.000000\t0\t6\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/bedtools intersect -wa -wb \\\n",
    "-a Panopea-generosa-v1.0.250bp.bin.bed \\\n",
    "-b /Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab \\\n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-104_S28_L005_5x.tab \\\n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-127_S33_L005_5x.tab \\\n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-128_S34_L005_5x.tab \\\n",
    "-sorted \\\n",
    "-filenames | \\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/bedtools intersect -wa -wb \\\n",
    "-a Panopea-generosa-v1.0.250bp.bin.bed \\\n",
    "-b /Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab \\\n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-104_S28_L005_5x.tab \\\n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-127_S33_L005_5x.tab \\\n",
    "/Users/strigg/Desktop/20200327/merged_tab/EPI-128_S34_L005_5x.tab \\\n",
    "-sorted \\\n",
    "-filenames \\\n",
    "> D10_low_5xCpG_250bin.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#count lines in file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 7015852 D10_low_5xCpG_250bin.txt\r\n"
     ]
    }
   ],
   "source": [
    "!wc -l D10_low_5xCpG_250bin.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#check for uniq lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "uniq -c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   3 Scaffold_01 10000412 10000414\n",
      "   3 Scaffold_01 10001661 10001663\n",
      "   3 Scaffold_01 10001757 10001759\n",
      "   3 Scaffold_01 10001777 10001779\n",
      "   4 Scaffold_01 10016762 10016764\n",
      "   3 Scaffold_01 10016979 10016981\n",
      "   4 Scaffold_01 10017904 10017906\n",
      "   4 Scaffold_01 10017920 10017922\n",
      "   4 Scaffold_01 10017943 10017945\n",
      "   3 Scaffold_01 10017987 10017989\n",
      "awk: write error on /dev/stdout\n",
      " input record number 3587, file \n",
      " source line number 1\n"
     ]
    }
   ],
   "source": [
    "! cat D10_low_5xCpG_250bin.txt | \\\n",
    "awk -F\"\\t\" '{print $5, $6, $7}' |\\\n",
    "sort |\\\n",
    "uniq -c |\\\n",
    "awk '{if($1>2)print $0}' |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   3 Scaffold_01 65750 66000 Scaffold_01 65824 65826\n",
      "   4 Scaffold_01 66250 66500 Scaffold_01 66423 66425\n",
      "   3 Scaffold_01 67500 67750 Scaffold_01 67627 67629\n",
      "   3 Scaffold_01 75250 75500 Scaffold_01 75436 75438\n",
      "   4 Scaffold_01 75500 75750 Scaffold_01 75504 75506\n",
      "   3 Scaffold_01 76000 76250 Scaffold_01 76061 76063\n",
      "   4 Scaffold_01 80250 80500 Scaffold_01 80435 80437\n",
      "   3 Scaffold_01 81250 81500 Scaffold_01 81490 81492\n",
      "   3 Scaffold_01 130250 130500 Scaffold_01 130493 130495\n",
      "   3 Scaffold_01 130500 130750 Scaffold_01 130556 130558\n",
      "awk: write error on /dev/stdout\n",
      " input record number 66928, file \n",
      " source line number 1\n"
     ]
    }
   ],
   "source": [
    "! cat D10_low_5xCpG_250bin.txt | \\\n",
    "awk -F\"\\t\" '{print $1,$2,$3, $5, $6, $7}' |\\\n",
    "uniq -c |\\\n",
    "awk '{if($1>2)print $0}' |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t65750\t66000\t/Users/strigg/Desktop/20200327/merged_tab/EPI-104_S28_L005_5x.tab\tScaffold_01\t65824\t65826\t0.000000\t0\t7\n",
      "Scaffold_01\t65750\t66000\t/Users/strigg/Desktop/20200327/merged_tab/EPI-127_S33_L005_5x.tab\tScaffold_01\t65824\t65826\t0.000000\t0\t8\n",
      "Scaffold_01\t65750\t66000\t/Users/strigg/Desktop/20200327/merged_tab/EPI-128_S34_L005_5x.tab\tScaffold_01\t65824\t65826\t0.000000\t0\t9\n",
      "Scaffold_01\t66250\t66500\t/Users/strigg/Desktop/20200327/merged_tab/EPI-103_S27_L005_5x.tab\tScaffold_01\t66423\t66425\t0.000000\t0\t5\n",
      "Scaffold_01\t66250\t66500\t/Users/strigg/Desktop/20200327/merged_tab/EPI-104_S28_L005_5x.tab\tScaffold_01\t66423\t66425\t0.000000\t0\t22\n",
      "Scaffold_01\t66250\t66500\t/Users/strigg/Desktop/20200327/merged_tab/EPI-127_S33_L005_5x.tab\tScaffold_01\t66423\t66425\t0.000000\t0\t10\n",
      "Scaffold_01\t66250\t66500\t/Users/strigg/Desktop/20200327/merged_tab/EPI-128_S34_L005_5x.tab\tScaffold_01\t66423\t66425\t0.000000\t0\t20\n",
      "Scaffold_01\t67500\t67750\t/Users/strigg/Desktop/20200327/merged_tab/EPI-104_S28_L005_5x.tab\tScaffold_01\t67627\t67629\t0.000000\t0\t5\n",
      "Scaffold_01\t67500\t67750\t/Users/strigg/Desktop/20200327/merged_tab/EPI-127_S33_L005_5x.tab\tScaffold_01\t67627\t67629\t0.000000\t0\t7\n",
      "Scaffold_01\t67500\t67750\t/Users/strigg/Desktop/20200327/merged_tab/EPI-128_S34_L005_5x.tab\tScaffold_01\t67627\t67629\t40.000000\t4\t6\n",
      "awk: write error on /dev/stdout\n",
      " input record number 7194, file D10_low_5xCpG_250bin.txt\n",
      " source line number 1\n"
     ]
    }
   ],
   "source": [
    "!cat D10_low_5xCpG_250bin.txt | \\\n",
    "awk -F\"\\t\" '{print $1,$2,$3,$5, $6, $7}' |\\\n",
    "uniq -c |\\\n",
    "awk '{if($1>2)print $2\"\\t\"$3\"\\t\"$4\"\\t\"$5\"\\t\"$6\"\\t\"$7}' |\\\n",
    "awk -F\"\\t\" 'NR==FNR{a[$1\"\\t\"$2\"\\t\"$3\"\\t\"$4\"\\t\"$5\"\\t\"$6]=$0;next}(($1\"\\t\"$2\"\\t\"$3\"\\t\"$5\"\\t\"$6\"\\t\"$7)in a){print $0}' \\\n",
    "- D10_low_5xCpG_250bin.txt |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!cat D10_low_5xCpG_250bin.txt | \\\n",
    "awk -F\"\\t\" '{print $1,$2,$3,$5, $6, $7}' |\\\n",
    "uniq -c |\\\n",
    "awk '{if($1>2)print $2\"\\t\"$3\"\\t\"$4\"\\t\"$5\"\\t\"$6\"\\t\"$7}' |\\\n",
    "awk -F\"\\t\" 'NR==FNR{a[$1\"\\t\"$2\"\\t\"$3\"\\t\"$4\"\\t\"$5\"\\t\"$6]=$0;next}(($1\"\\t\"$2\"\\t\"$3\"\\t\"$5\"\\t\"$6\"\\t\"$7)in a){print $0}' \\\n",
    "- D10_low_5xCpG_250bin.txt |\\\n",
    "awk -F\"\\t\" '{print $1\"\\t\"$2\"\\t\"$3}' |\\\n",
    "uniq -c |\\\n",
    "awk '{if($1>8)print $2\"\\t\"$3\"\\t\"$4}' |\\\n",
    "awk -F\"\\t\" 'NR==FNR{a[$1\"\\t\"$2\"\\t\"$3]=$0;next}(($1\"\\t\"$2\"\\t\"$3)in a){print $0}' \\\n",
    "- D10_low_5xCpG_250bin.txt |\\\n",
    "head -30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## filter for CpGs covered by at least 3/4 samples \n",
    "#### awk prints the region and the CpG site\n",
    "#### uniq -c counts the coverage unique CpG sites per region\n",
    "#### awk filters for CpG sites with coverage in 3 or more samples\n",
    "#### awk prints regions with CpGs that are covered by 3 or more samples\n",
    "#### awk prints the region coordinates\n",
    "#### uniq -c counts the number of regions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}