analyses/20200406_Create_UTR_tracks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#check if regions in exon and cds gffs don't overlap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "path=/Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks\n",
    "\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/intersectBed \\\n",
    "-a ${path}/Panopea-generosa-v1.0.a4.CDS.gff3 \\\n",
    "-b ${path}/Panopea-generosa-v1.0.a4.exon.gff3 \\\n",
    "-v |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "path=/Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks\n",
    "\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/intersectBed \\\n",
    "-a ${path}/Panopea-generosa-v1.0.a4.gene.gff3 \\\n",
    "-b ${path}/Panopea-generosa-v1.0.a4.mRNA.gff3 \\\n",
    "-v |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "path=/Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks\n",
    "\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/subtractBed \\\n",
    "-a ${path}/Panopea-generosa-v1.0.a4.gene.gff3 \\\n",
    "-b ${path}/Panopea-generosa-v1.0.a4.mRNA.gff3 |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "path=/Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks\n",
    "\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/subtractBed \\\n",
    "-a ${path}/Panopea-generosa-v1.0.a4.CDS.gff3 \\\n",
    "-b ${path}/Panopea-generosa-v1.0.a4.exon.gff3 |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "path=/Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks\n",
    "\n",
    "\n",
    "cat \\\n",
    "${path}/Panopea-generosa-v1.0.a4.CDS.gff3 \\\n",
    "${path}/Panopea-generosa-v1.0.a4.intron.gff3 | \\\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/sortBed -i - |\\\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/mergeBed -i - |\\\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/subtractBed \\\n",
    "-a ${path}/Panopea-generosa-v1.0.a4.gene.gff3 \\\n",
    "-b - | \\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# so far from these tests, CDS and exon tracks are identical\n",
    "# and gene and mRNA tracks are identical\n",
    "# this is what it looks like in IGV too"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# i will exclude the following gff files:\n",
    "# rRNA.gff3\n",
    "# mRNA.gff3\n",
    "# gene.gff3\n",
    "# CDS.gff3\n",
    "\n",
    "# I will include \n",
    "# exon.gff3\n",
    "# intron.gff3\n",
    "# putPromoter.gff3\n",
    "# 3primeUTR.gff3\n",
    "# intergenic.gff3\n",
    "# all repeat regions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I need to make the putative promoter and 3primeUTR tracks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#try to make a promoter track"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tgene\t1\t1\t.\t+\t.\tID=PGEN_.00g000010;Name=PGEN_.00g000010;original_ID=21510-PGEN_.00g234140;Alias=21510-PGEN_.00g234140;original_name=21510-PGEN_.00g234140;Notes=sp|Q86IC9|CAMT1_DICDI [BLAST protein vs protein (blastp) 2.7.1],PF01596.12 [Pfam 1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tgene\t36740\t37739\t.\t-\t.\tID=PGEN_.00g000020;Name=PGEN_.00g000020;original_ID=21510-PGEN_.00g234150;Alias=21510-PGEN_.00g234150;original_name=21510-PGEN_.00g234150;Notes=sp|P04177|TY3H_RAT [BLAST protein vs protein (blastp) 2.7.1],sp|P04177|TY3H_RAT [DIAMOND Functional 0.9.22],IPR036951 [InterProScan 5.29-68.0],PF00351.16 [Pfam 1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tgene\t52579\t53578\t.\t-\t.\tID=PGEN_.00g000030;Name=PGEN_.00g000030;original_ID=21510-PGEN_.00g234160;Alias=21510-PGEN_.00g234160;original_name=21510-PGEN_.00g234160;Notes=PF08054.6 [Pfam 1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tgene\t54792\t55791\t.\t+\t.\tID=PGEN_.00g000040;Name=PGEN_.00g000040;original_ID=21510-PGEN_.00g234170;Alias=21510-PGEN_.00g234170;original_name=21510-PGEN_.00g234170\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tgene\t69114\t70113\t.\t-\t.\tID=PGEN_.00g000050;Name=PGEN_.00g000050;original_ID=21510-PGEN_.00g234180;Alias=21510-PGEN_.00g234180;original_name=21510-PGEN_.00g234180;Notes=sp|Q8L840|RQL4A_ARATH [BLAST protein vs protein (blastp) 2.7.1],sp|Q8L840|RQL4A_ARATH [DIAMOND Functional 0.9.22],PF00270.24 [Pfam 1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tgene\t69713\t70712\t.\t+\t.\tID=PGEN_.00g000060;Name=PGEN_.00g000060;original_ID=21510-PGEN_.00g234190;Alias=21510-PGEN_.00g234190;original_name=21510-PGEN_.00g234190;Notes=sp|Q61043|NIN_MOUSE [DIAMOND Functional 0.9.22],PF04443.7 [Pfam 1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tgene\t182686\t183685\t.\t+\t.\tID=PGEN_.00g000070;Name=PGEN_.00g000070;original_ID=21510-PGEN_.00g234200;Alias=21510-PGEN_.00g234200;original_name=21510-PGEN_.00g234200;Notes=PF15364.1 [Pfam 1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tgene\t186328\t187327\t.\t+\t.\tID=PGEN_.00g000080;Name=PGEN_.00g000080;original_ID=21510-PGEN_.00g234210;Alias=21510-PGEN_.00g234210;original_name=21510-PGEN_.00g234210;Notes=sp|A1E2V0|BIRC3_CANLF [BLAST protein vs protein (blastp) 2.7.1],sp|Q24307|DIAP2_DROME [DIAMOND Functional 0.9.22],IPR001370 [InterProScan 5.29-68.0],PF02229.11 [Pfam 1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tgene\t190461\t191460\t.\t-\t.\tID=PGEN_.00g000090;Name=PGEN_.00g000090;original_ID=21510-PGEN_.00g234220;Alias=21510-PGEN_.00g234220;original_name=21510-PGEN_.00g234220;Notes=sp|P34456|YMD2_CAEEL [BLAST protein vs protein (blastp) 2.7.1],PF10284.4 [Pfam 1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tgene\t191411\t192410\t.\t-\t.\tID=PGEN_.00g000100;Name=PGEN_.00g000100;original_ID=21510-PGEN_.00g234230;Alias=21510-PGEN_.00g234230;original_name=21510-PGEN_.00g234230;Notes=PF10228.4 [Pfam 1.6]\r\n"
     ]
    }
   ],
   "source": [
    "!/Users/Shared/bioinformatics/bedtools2/bin/flankBed \\\n",
    "-i /Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks/Panopea-generosa-v1.0.a4.gene.gff3 \\\n",
    "-g /Volumes/web/metacarcinus/Pgenerosa/GENOMES/Panopea-generosa-v1.0.fa.fai \\\n",
    "-l 1000 \\\n",
    "-r 0 \\\n",
    "-s |\\\n",
    "head\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generate putative promoter gff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tput_promoter\t36740\t37739\t.\t-\t.\tID=PGEN_.00g000020;Name=PGEN_.00g000020;original_ID=21510-PGEN_.00g234150;Alias=21510-PGEN_.00g234150;original_name=21510-PGEN_.00g234150;Notes=sp|P04177|TY3H_RAT\t[BLAST\tprotein\tvs\tprotein\t(blastp)\t2.7.1],sp|P04177|TY3H_RAT\t[DIAMOND\tFunctional\t0.9.22],IPR036951\t[InterProScan\t5.29-68.0],PF00351.16\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tput_promoter\t52579\t53578\t.\t-\t.\tID=PGEN_.00g000030;Name=PGEN_.00g000030;original_ID=21510-PGEN_.00g234160;Alias=21510-PGEN_.00g234160;original_name=21510-PGEN_.00g234160;Notes=PF08054.6\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tput_promoter\t54792\t55791\t.\t+\t.\tID=PGEN_.00g000040;Name=PGEN_.00g000040;original_ID=21510-PGEN_.00g234170;Alias=21510-PGEN_.00g234170;original_name=21510-PGEN_.00g234170\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tput_promoter\t69114\t70113\t.\t-\t.\tID=PGEN_.00g000050;Name=PGEN_.00g000050;original_ID=21510-PGEN_.00g234180;Alias=21510-PGEN_.00g234180;original_name=21510-PGEN_.00g234180;Notes=sp|Q8L840|RQL4A_ARATH\t[BLAST\tprotein\tvs\tprotein\t(blastp)\t2.7.1],sp|Q8L840|RQL4A_ARATH\t[DIAMOND\tFunctional\t0.9.22],PF00270.24\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tput_promoter\t69713\t70712\t.\t+\t.\tID=PGEN_.00g000060;Name=PGEN_.00g000060;original_ID=21510-PGEN_.00g234190;Alias=21510-PGEN_.00g234190;original_name=21510-PGEN_.00g234190;Notes=sp|Q61043|NIN_MOUSE\t[DIAMOND\tFunctional\t0.9.22],PF04443.7\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tput_promoter\t182686\t183685\t.\t+\t.\tID=PGEN_.00g000070;Name=PGEN_.00g000070;original_ID=21510-PGEN_.00g234200;Alias=21510-PGEN_.00g234200;original_name=21510-PGEN_.00g234200;Notes=PF15364.1\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tput_promoter\t186328\t187327\t.\t+\t.\tID=PGEN_.00g000080;Name=PGEN_.00g000080;original_ID=21510-PGEN_.00g234210;Alias=21510-PGEN_.00g234210;original_name=21510-PGEN_.00g234210;Notes=sp|A1E2V0|BIRC3_CANLF\t[BLAST\tprotein\tvs\tprotein\t(blastp)\t2.7.1],sp|Q24307|DIAP2_DROME\t[DIAMOND\tFunctional\t0.9.22],IPR001370\t[InterProScan\t5.29-68.0],PF02229.11\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tput_promoter\t190461\t191460\t.\t-\t.\tID=PGEN_.00g000090;Name=PGEN_.00g000090;original_ID=21510-PGEN_.00g234220;Alias=21510-PGEN_.00g234220;original_name=21510-PGEN_.00g234220;Notes=sp|P34456|YMD2_CAEEL\t[BLAST\tprotein\tvs\tprotein\t(blastp)\t2.7.1],PF10284.4\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tput_promoter\t191411\t192410\t.\t-\t.\tID=PGEN_.00g000100;Name=PGEN_.00g000100;original_ID=21510-PGEN_.00g234230;Alias=21510-PGEN_.00g234230;original_name=21510-PGEN_.00g234230;Notes=PF10228.4\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\tput_promoter\t192283\t193282\t.\t-\t.\tID=PGEN_.00g000110;Name=PGEN_.00g000110;original_ID=21510-PGEN_.00g234240;Alias=21510-PGEN_.00g234240;original_name=21510-PGEN_.00g234240;Notes=IPR009044\t[InterProScan\t5.29-68.0],PF02229.11\t[Pfam\t1.6]\r\n"
     ]
    }
   ],
   "source": [
    "!/Users/Shared/bioinformatics/bedtools2/bin/flankBed \\\n",
    "-i /Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks/Panopea-generosa-v1.0.a4.gene.gff3 \\\n",
    "-g /Volumes/web/metacarcinus/Pgenerosa/GENOMES/Panopea-generosa-v1.0.fa.fai \\\n",
    "-l 1000 \\\n",
    "-r 0 \\\n",
    "-s |\\\n",
    "awk '{ gsub(\"gene\",\"put_promoter\",$3); print $0 }'|\\\n",
    "awk '{if($5-$4 > 3)print $0}'|\\\n",
    "tr ' ' '\\t' |\\\n",
    "head\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "!/Users/Shared/bioinformatics/bedtools2/bin/flankBed \\\n",
    "-i /Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks/Panopea-generosa-v1.0.a4.gene.gff3 \\\n",
    "-g /Volumes/web/metacarcinus/Pgenerosa/GENOMES/Panopea-generosa-v1.0.fa.fai \\\n",
    "-l 1000 \\\n",
    "-r 0 \\\n",
    "-s |\\\n",
    "awk '{ gsub(\"gene\",\"put_promoter\",$3); print $0 }'|\\\n",
    "awk '{if($5-$4 > 3)print $0}'|\\\n",
    "tr ' ' '\\t' \\\n",
    "> /Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks/Panopea-generosa-v1.0.a4.putprom.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate 3' UTR track"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\t3prime_UTR\t4720\t6719\t.\t+\t.\tID=PGEN_.00g000010;Name=PGEN_.00g000010;original_ID=21510-PGEN_.00g234140;Alias=21510-PGEN_.00g234140;original_name=21510-PGEN_.00g234140;Notes=sp|Q86IC9|CAMT1_DICDI\t[BLAST\tprotein\tvs\tprotein\t(blastp)\t2.7.1],PF01596.12\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\t3prime_UTR\t17808\t19807\t.\t-\t.\tID=PGEN_.00g000020;Name=PGEN_.00g000020;original_ID=21510-PGEN_.00g234150;Alias=21510-PGEN_.00g234150;original_name=21510-PGEN_.00g234150;Notes=sp|P04177|TY3H_RAT\t[BLAST\tprotein\tvs\tprotein\t(blastp)\t2.7.1],sp|P04177|TY3H_RAT\t[DIAMOND\tFunctional\t0.9.22],IPR036951\t[InterProScan\t5.29-68.0],PF00351.16\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\t3prime_UTR\t47248\t49247\t.\t-\t.\tID=PGEN_.00g000030;Name=PGEN_.00g000030;original_ID=21510-PGEN_.00g234160;Alias=21510-PGEN_.00g234160;original_name=21510-PGEN_.00g234160;Notes=PF08054.6\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\t3prime_UTR\t67547\t69546\t.\t+\t.\tID=PGEN_.00g000040;Name=PGEN_.00g000040;original_ID=21510-PGEN_.00g234170;Alias=21510-PGEN_.00g234170;original_name=21510-PGEN_.00g234170\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\t3prime_UTR\t65586\t67585\t.\t-\t.\tID=PGEN_.00g000050;Name=PGEN_.00g000050;original_ID=21510-PGEN_.00g234180;Alias=21510-PGEN_.00g234180;original_name=21510-PGEN_.00g234180;Notes=sp|Q8L840|RQL4A_ARATH\t[BLAST\tprotein\tvs\tprotein\t(blastp)\t2.7.1],sp|Q8L840|RQL4A_ARATH\t[DIAMOND\tFunctional\t0.9.22],PF00270.24\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\t3prime_UTR\t81100\t83099\t.\t+\t.\tID=PGEN_.00g000060;Name=PGEN_.00g000060;original_ID=21510-PGEN_.00g234190;Alias=21510-PGEN_.00g234190;original_name=21510-PGEN_.00g234190;Notes=sp|Q61043|NIN_MOUSE\t[DIAMOND\tFunctional\t0.9.22],PF04443.7\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\t3prime_UTR\t186074\t188073\t.\t+\t.\tID=PGEN_.00g000070;Name=PGEN_.00g000070;original_ID=21510-PGEN_.00g234200;Alias=21510-PGEN_.00g234200;original_name=21510-PGEN_.00g234200;Notes=PF15364.1\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\t3prime_UTR\t188354\t190353\t.\t+\t.\tID=PGEN_.00g000080;Name=PGEN_.00g000080;original_ID=21510-PGEN_.00g234210;Alias=21510-PGEN_.00g234210;original_name=21510-PGEN_.00g234210;Notes=sp|A1E2V0|BIRC3_CANLF\t[BLAST\tprotein\tvs\tprotein\t(blastp)\t2.7.1],sp|Q24307|DIAP2_DROME\t[DIAMOND\tFunctional\t0.9.22],IPR001370\t[InterProScan\t5.29-68.0],PF02229.11\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\t3prime_UTR\t187849\t189848\t.\t-\t.\tID=PGEN_.00g000090;Name=PGEN_.00g000090;original_ID=21510-PGEN_.00g234220;Alias=21510-PGEN_.00g234220;original_name=21510-PGEN_.00g234220;Notes=sp|P34456|YMD2_CAEEL\t[BLAST\tprotein\tvs\tprotein\t(blastp)\t2.7.1],PF10284.4\t[Pfam\t1.6]\r\n",
      "Scaffold_01\tGenSAS_5d9637f372b5d-publish\t3prime_UTR\t189069\t191068\t.\t-\t.\tID=PGEN_.00g000100;Name=PGEN_.00g000100;original_ID=21510-PGEN_.00g234230;Alias=21510-PGEN_.00g234230;original_name=21510-PGEN_.00g234230;Notes=PF10228.4\t[Pfam\t1.6]\r\n"
     ]
    }
   ],
   "source": [
    "!/Users/Shared/bioinformatics/bedtools2/bin/flankBed \\\n",
    "-i /Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks/Panopea-generosa-v1.0.a4.gene.gff3 \\\n",
    "-g /Volumes/web/metacarcinus/Pgenerosa/GENOMES/Panopea-generosa-v1.0.fa.fai \\\n",
    "-l 0 \\\n",
    "-r 2000 \\\n",
    "-s |\\\n",
    "awk '{ gsub(\"gene\",\"3prime_UTR\",$3); print $0 }'|\\\n",
    "awk '{if($5-$4 > 3)print $0}'|\\\n",
    "tr ' ' '\\t' |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "!/Users/Shared/bioinformatics/bedtools2/bin/flankBed \\\n",
    "-i /Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks/Panopea-generosa-v1.0.a4.gene.gff3 \\\n",
    "-g /Volumes/web/metacarcinus/Pgenerosa/GENOMES/Panopea-generosa-v1.0.fa.fai \\\n",
    "-l 0 \\\n",
    "-r 2000 \\\n",
    "-s |\\\n",
    "awk '{ gsub(\"gene\",\"3prime_UTR\",$3); print $0 }'|\\\n",
    "awk '{if($5-$4 > 3)print $0}'|\\\n",
    "tr ' ' '\\t' \\\n",
    "> /Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks/Panopea-generosa-v1.0.a4.3primeUTR.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#visualize in IGV\n",
    "# Tracks look good\n",
    "# now make new concatenated file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mPanopea-generosa-v1.0.a4.3primeUTR.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.CDS.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.exon.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.gene.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.intergenic.bed\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.intergenic.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.intron.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.introns.bed\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.mRNA.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.noheader.bed\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.noheader.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.putprom.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.rRNA.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.repeat_region.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.repeats.DNA.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.repeats.LINE.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.repeats.LTR.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.repeats.RC.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.repeats.SINE.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.repeats.Simple_repeat.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.repeats.Unknown.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.sorted.bed\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.sorted.gff3\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.tRNA.gff3\u001b[m\u001b[m\r\n"
     ]
    }
   ],
   "source": [
    "!ls /Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "path=/Volumes/web/metacarcinus/Pgenerosa/GENOMES/feature_tracks\n",
    "\n",
    "cat \\\n",
    "${path}/Panopea-generosa-v1.0.a4.exon.gff3 \\\n",
    "${path}/Panopea-generosa-v1.0.a4.intron.gff3 \\\n",
    "${path}/Panopea-generosa-v1.0.a4.putprom.gff3 \\\n",
    "${path}/Panopea-generosa-v1.0.a4.3primeUTR.gff3 \\\n",
    "${path}/Panopea-generosa-v1.0.a4.intergenic.gff3 \\\n",
    "${path}/Panopea-generosa-v1.0.a4.repeat*.gff3 \\\n",
    "${path}/Panopea-generosa-v1.0.a4.tRNA.gff3 \\\n",
    "> /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t2\t125\texon\r\n",
      "Scaffold_01\t1995\t2095\texon\r\n",
      "Scaffold_01\t3325\t3495\texon\r\n",
      "Scaffold_01\t4651\t4719\texon\r\n",
      "Scaffold_01\t19808\t19943\texon\r\n",
      "Scaffold_01\t21133\t21362\texon\r\n",
      "Scaffold_01\t22487\t22613\texon\r\n",
      "Scaffold_01\t24824\t24959\texon\r\n",
      "Scaffold_01\t25981\t26126\texon\r\n",
      "Scaffold_01\t27969\t28019\texon\r\n",
      "awk: write error on /dev/stdout\r\n",
      " input record number 1012, file \r\n",
      " source line number 1\r\n",
      "awk: write error on /dev/stdout\r\n",
      " input record number 1237, file \r\n",
      " source line number 1\r\n",
      "cat: stdout: Broken pipe\r\n"
     ]
    }
   ],
   "source": [
    "!cat /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.gff3 |\\\n",
    "awk -F\"\\t\" '{if(($1 !~/gff-version/)&&($1 !~/Generated/)&&($1 !~/Project/))print $0}' |\\\n",
    "awk -F \"\\t\" '{print $1\"\\t\"$4\"\\t\"$5\"\\t\"$3}'|\\\n",
    "head\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "cat /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.gff3 |\\\n",
    "awk -F\"\\t\" '{if(($1 !~/gff-version/)&&($1 !~/Generated/)&&($1 !~/Project/))print $0}' |\\\n",
    "awk -F\"\\t\" '{print $1\"\\t\"$4\"\\t\"$5\"\\t\"$3}'|\\\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/sortBed \\\n",
    "-i - \\\n",
    "> /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.sort.bed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t0\t1\tintergenic\n",
      "Scaffold_01\t2\t125\texon\n",
      "Scaffold_01\t126\t1994\tintron\n",
      "Scaffold_01\t265\t338\trepeat_region\n",
      "Scaffold_01\t855\t1050\trepeat_region\n",
      "Scaffold_01\t1040\t1183\trepeat_region\n",
      "Scaffold_01\t1275\t1466\trepeat_region\n",
      "Scaffold_01\t1618\t1752\trepeat_region\n",
      "Scaffold_01\t1764\t1846\trepeat_region\n",
      "Scaffold_01\t1995\t2095\texon\n",
      "^C\n"
     ]
    }
   ],
   "source": [
    "!cat /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.sort.bed |uniq |head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t0\t1\tintergenic\n",
      "Scaffold_01\t2\t125\texon\n",
      "Scaffold_01\t126\t1994\tintron\n",
      "Scaffold_01\t265\t338\trepeat_region\n",
      "Scaffold_01\t855\t1050\trepeat_region\n",
      "Scaffold_01\t1040\t1183\trepeat_region\n",
      "Scaffold_01\t1275\t1466\trepeat_region\n",
      "Scaffold_01\t1618\t1752\trepeat_region\n",
      "Scaffold_01\t1764\t1846\trepeat_region\n",
      "Scaffold_01\t1995\t2095\texon\n",
      "^C\n"
     ]
    }
   ],
   "source": [
    "!cat /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.sort.bed |\\\n",
    "uniq |\\\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/windowMaker \\\n",
    "-b - \\\n",
    "-w 2000 \\\n",
    "-i src |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t0\t1\tintergenic\r\n",
      "Scaffold_01\t2\t125\texon\r\n",
      "Scaffold_01\t126\t1994\tintron\r\n",
      "Scaffold_01\t265\t338\trepeat_region\r\n",
      "Scaffold_01\t855\t1050\trepeat_region\r\n",
      "Scaffold_01\t1040\t1183\trepeat_region\r\n",
      "Scaffold_01\t1275\t1466\trepeat_region\r\n",
      "Scaffold_01\t1618\t1752\trepeat_region\r\n",
      "Scaffold_01\t1764\t1846\trepeat_region\r\n",
      "Scaffold_01\t1995\t2095\texon\r\n"
     ]
    }
   ],
   "source": [
    "!cat /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.sort.bed |\\\n",
    "uniq |\\\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/windowMaker \\\n",
    "-b - \\\n",
    "-w 2000 \\\n",
    "-i src |\\\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/sortBed \\\n",
    "-i - |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.sort.bed |\\\n",
    "uniq |\\\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/windowMaker \\\n",
    "-b - \\\n",
    "-w 2000 \\\n",
    "-i src |\\\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/sortBed \\\n",
    "-i - \\\n",
    "> /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.2Kbin.sort.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.2Kbin.sort.bed |uniq > /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.2Kbin.uniq.bed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#match DMRs to features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "FILES=/Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses/20200320_anno/aov_0.05*DMR.bed\n",
    "\n",
    "for f in $FILES\n",
    "do\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/intersectBed \\\n",
    "-a ${f} \\\n",
    "-b /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.2Kbin.uniq.bed \\\n",
    "-wb \\\n",
    "> /Users/strigg/Desktop/20200408/$(basename ${f%DMR.bed})_0408.txt\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t3532891\t3533128\tamb\tScaffold_01\t3532687\t3533445\trepeat_region\r\n",
      "Scaffold_01\t3532891\t3533128\tamb\tScaffold_01\t3532730\t3534730\tintron\r\n",
      "Scaffold_01\t22426167\t22426265\tamb\tScaffold_01\t22425088\t22427088\tintergenic\r\n",
      "Scaffold_01\t22426167\t22426204\tamb\tScaffold_01\t22426110\t22426204\trepeat_region\r\n",
      "Scaffold_01\t22426183\t22426265\tamb\tScaffold_01\t22426183\t22426277\trepeat_region\r\n",
      "Scaffold_01\t46681763\t46681990\tamb\tScaffold_01\t46680058\t46682058\tintergenic\r\n",
      "Scaffold_01\t52430867\t52430898\tamb\tScaffold_01\t52429513\t52431513\tintergenic\r\n",
      "Scaffold_01\t88434573\t88434644\tamb\tScaffold_01\t88434050\t88434919\texon\r\n",
      "Scaffold_02\t59898836\t59898874\tamb\tScaffold_02\t59897300\t59899300\tintergenic\r\n",
      "Scaffold_02\t59898836\t59898874\tamb\tScaffold_02\t59897854\t59899328\trepeat_region\r\n"
     ]
    }
   ],
   "source": [
    "!head /Users/strigg/Desktop/20200408/aov_0.05pH_amb_0408.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# STEP 3: overlap CpGs with coverage in all groups and at least 3/4 individuals per group\n",
    "# with binned genomic features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "FILES=/Volumes/web/metacarcinus/Pgenerosa/analyses/20200406/*.3xCpG.allgrps.bed\n",
    "\n",
    "for f in $FILES\n",
    "do\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/intersectBed \\\n",
    "-a ${f} \\\n",
    "-b /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.2Kbin.uniq.bed \\\n",
    "-wb \\\n",
    "> /Users/strigg/Desktop/20200408/$(basename ${f%.3xCpG.allgrps.bed})_features.txt\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t10017891\t10017893\tScaffold_01\t10015950\t10017950\tintergenic\r\n",
      "Scaffold_01\t10017891\t10017893\tScaffold_01\t10015951\t10017950\t3prime_UTR\r\n",
      "Scaffold_01\t10017891\t10017893\tScaffold_01\t10017821\t10018112\trepeat_region\r\n",
      "Scaffold_01\t10018551\t10018553\tScaffold_01\t10017950\t10019950\tintergenic\r\n",
      "Scaffold_01\t10018551\t10018553\tScaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10018563\t10018565\tScaffold_01\t10017950\t10019950\tintergenic\r\n",
      "Scaffold_01\t10018563\t10018565\tScaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10018572\t10018574\tScaffold_01\t10017950\t10019950\tintergenic\r\n",
      "Scaffold_01\t10018572\t10018574\tScaffold_01\t10018516\t10018602\trepeat_region\r\n",
      "Scaffold_01\t10018624\t10018626\tScaffold_01\t10017950\t10019950\tintergenic\r\n"
     ]
    }
   ],
   "source": [
    "!head /Users/strigg/Desktop/20200408/amb_features.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#find features with at least 3 CpGs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaffold_01\t10017950\t10019950\tintergenic\n",
      "Scaffold_01\t10018516\t10018602\trepeat_region\n",
      "Scaffold_01\t10019296\t10019390\trepeat_region\n",
      "Scaffold_01\t10019581\t10019667\trepeat_region\n",
      "Scaffold_01\t10019946\t10020040\trepeat_region\n",
      "Scaffold_01\t10019950\t10021950\tintergenic\n",
      "Scaffold_01\t10021950\t10023950\tintergenic\n",
      "Scaffold_01\t10022552\t10023073\trepeat_region\n",
      "Scaffold_01\t10025899\t10027125\texon\n",
      "Scaffold_01\t10029125\t10031125\tintergenic\n",
      "awk: write error on /dev/stdout\n",
      " input record number 1483, file \n",
      " source line number 1\n"
     ]
    }
   ],
   "source": [
    "!cat /Users/strigg/Desktop/20200408/day10_features.txt |\\\n",
    "awk -F\"\\t\" '{print $4,$5,$6,$7}' |\\\n",
    "sort |\\\n",
    "uniq -c |\\\n",
    "awk '{if($1>2)print $2\"\\t\"$3\"\\t\"$4\"\\t\"$5}' |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "FILES=/Users/strigg/Desktop/20200408/*_features.txt\n",
    "\n",
    "for f in $FILES\n",
    "do\n",
    "cat ${f} |\\\n",
    "awk -F\"\\t\" '{print $4,$5,$6,$7}' |\\\n",
    "sort |\\\n",
    "uniq -c |\\\n",
    "awk '{if($1>2)print $2\"\\t\"$3\"\\t\"$4\"\\t\"$5}'\\\n",
    "> /Users/strigg/Desktop/20200408/$(basename ${f%_features.txt})_features.3CpG.txt\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   46837 /Users/strigg/Desktop/20200408/amb_features.3CpG.txt\r\n",
      "  134881 /Users/strigg/Desktop/20200408/day10_features.3CpG.txt\r\n",
      "   50835 /Users/strigg/Desktop/20200408/day135_features.3CpG.txt\r\n",
      "   28540 /Users/strigg/Desktop/20200408/day145_features.3CpG.txt\r\n",
      "  261093 total\r\n"
     ]
    }
   ],
   "source": [
    "#count the number of features containing at least 3 CpGs covered by 3/4 samples per group and by all groups in comparison\n",
    "!wc -l /Users/strigg/Desktop/20200408/*_features.3CpG.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#copy data to Gannet\n",
    "(base) ostrich:Desktop strigg$ rsync --archive --verbose --progress ~/Desktop/20200408 /Volumes/web/metacarcinus/Pgenerosa/analyses\n",
    "building file list ... \n",
    "18 files to consider\n",
    "20200408/\n",
    "20200408/Panopea-generosa-v1.0.a4.2Kbin.sort.bed\n",
    "   102705643 100%   43.85MB/s    0:00:02 (xfer#1, to-check=16/18)\n",
    "20200408/Panopea-generosa-v1.0.a4.2Kbin.uniq.bed\n",
    "   102696417 100%   37.24MB/s    0:00:02 (xfer#2, to-check=15/18)\n",
    "20200408/Panopea-generosa-v1.0.a4.gff3\n",
    "   839807510 100%   41.90MB/s    0:00:19 (xfer#3, to-check=14/18)\n",
    "20200408/Panopea-generosa-v1.0.a4.sort.bed\n",
    "   164797888 100%   40.27MB/s    0:00:03 (xfer#4, to-check=13/18)\n",
    "20200408/Panopea-generosa-v1.0.a4.uniq.bed\n",
    "           0 100%    0.00kB/s    0:00:00 (xfer#5, to-check=12/18)\n",
    "20200408/amb_features.3CpG.txt\n",
    "     1873839 100%    1.85MB/s    0:00:00 (xfer#6, to-check=11/18)\n",
    "20200408/amb_features.txt\n",
    "    26960178 100%   15.19MB/s    0:00:01 (xfer#7, to-check=10/18)\n",
    "20200408/aov_0.05pH_amb_0408.txt\n",
    "       13419 100%   18.64kB/s    0:00:00 (xfer#8, to-check=9/18)\n",
    "20200408/aov_0.05pH_d10_0408.txt\n",
    "        7764 100%   10.74kB/s    0:00:00 (xfer#9, to-check=8/18)\n",
    "20200408/aov_0.05pH_d135_0408.txt\n",
    "        3676 100%    5.02kB/s    0:00:00 (xfer#10, to-check=7/18)\n",
    "20200408/aov_0.05pH_d145_0408.txt\n",
    "        6503 100%    8.76kB/s    0:00:00 (xfer#11, to-check=6/18)\n",
    "20200408/day10_features.3CpG.txt\n",
    "     5411202 100%    5.50MB/s    0:00:00 (xfer#12, to-check=5/18)\n",
    "20200408/day10_features.txt\n",
    "    77229256 100%   26.76MB/s    0:00:02 (xfer#13, to-check=4/18)\n",
    "20200408/day135_features.3CpG.txt\n",
    "     2034656 100%    2.29MB/s    0:00:00 (xfer#14, to-check=3/18)\n",
    "20200408/day135_features.txt\n",
    "    29603769 100%   17.62MB/s    0:00:01 (xfer#15, to-check=2/18)\n",
    "20200408/day145_features.3CpG.txt\n",
    "     1141591 100%    1.70MB/s    0:00:00 (xfer#16, to-check=1/18)\n",
    "20200408/day145_features.txt\n",
    "    16694720 100%   15.37MB/s    0:00:01 (xfer#17, to-check=0/18)\n",
    "\n",
    "sent 1371156638 bytes  received 400 bytes  42189447.32 bytes/sec\n",
    "total size is 1370988031  speedup is 1.00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# determine if it matters whether features are binned or not \n",
    "#when matching up DMRs\n",
    "\n",
    "\n",
    "#create an concatenated feature file to match "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat /Users/strigg/Desktop/20200408/Panopea-generosa-v1.0.a4.sort.bed |\\\n",
    "uniq > /Volumes/web/metacarcinus/Pgenerosa/analyses/20200408/Panopea-generosa-v1.0.a4.uniq.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "\n",
    "FILES=/Users/strigg/Documents/GitHub/Shelly_Pgenerosa/analyses/20200320_anno/aov_0.05*DMR.bed\n",
    "\n",
    "for f in $FILES\n",
    "do\n",
    "/Users/Shared/bioinformatics/bedtools2/bin/intersectBed \\\n",
    "-a ${f} \\\n",
    "-b /Volumes/web/metacarcinus/Pgenerosa/analyses/20200408/Panopea-generosa-v1.0.a4.uniq.bed \\\n",
    "-wb \\\n",
    "> /Users/strigg/Desktop/20200408/$(basename ${f%DMR.bed})_0408b.txt\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> /Users/strigg/Desktop/20200408/aov_0.05pH_amb_0408b.txt <==\r\n",
      "Scaffold_01\t3532891\t3533128\tamb\tScaffold_01\t3530730\t3534814\tintron\r\n",
      "Scaffold_01\t3532891\t3533128\tamb\tScaffold_01\t3532687\t3533445\trepeat_region\r\n",
      "Scaffold_01\t22426167\t22426204\tamb\tScaffold_01\t22426110\t22426204\trepeat_region\r\n",
      "Scaffold_01\t22426183\t22426265\tamb\tScaffold_01\t22426183\t22426277\trepeat_region\r\n",
      "Scaffold_01\t22426167\t22426265\tamb\tScaffold_01\t22419088\t22454118\tintergenic\r\n",
      "Scaffold_01\t46681763\t46681990\tamb\tScaffold_01\t46678058\t46691928\tintergenic\r\n",
      "Scaffold_01\t52430867\t52430898\tamb\tScaffold_01\t52399513\t52442421\tintergenic\r\n",
      "Scaffold_01\t88434573\t88434644\tamb\tScaffold_01\t88434050\t88434919\texon\r\n",
      "Scaffold_02\t59898836\t59898874\tamb\tScaffold_02\t59897854\t59899328\trepeat_region\r\n",
      "Scaffold_02\t59898836\t59898874\tamb\tScaffold_02\t59857300\t59904751\tintergenic\r\n",
      "\r\n",
      "==> /Users/strigg/Desktop/20200408/aov_0.05pH_d10_0408b.txt <==\r\n",
      "Scaffold_01\t26811206\t26811305\td10\tScaffold_01\t26811122\t26811305\texon\r\n",
      "Scaffold_01\t26811305\t26811326\td10\tScaffold_01\t26811305\t26818981\tintergenic\r\n",
      "Scaffold_01\t26811306\t26811326\td10\tScaffold_01\t26811306\t26813305\t3prime_UTR\r\n",
      "Scaffold_02\t47739891\t47740441\td10\tScaffold_02\t47739650\t47740450\trepeat_region\r\n",
      "Scaffold_02\t47739891\t47740441\td10\tScaffold_02\t47730974\t47744304\tintergenic\r\n",
      "Scaffold_02\t58122038\t58122086\td10\tScaffold_02\t58119032\t58122110\tintron\r\n",
      "Scaffold_02\t58122038\t58122063\td10\tScaffold_02\t58121899\t58122063\trepeat_region\r\n",
      "Scaffold_02\t61737412\t61737425\td10\tScaffold_02\t61737412\t61737515\trepeat_region\r\n",
      "Scaffold_02\t61737304\t61737425\td10\tScaffold_02\t61719313\t61742944\tintergenic\r\n",
      "Scaffold_03\t1735906\t1736047\td10\tScaffold_03\t1735854\t1736314\trepeat_region\r\n",
      "\r\n",
      "==> /Users/strigg/Desktop/20200408/aov_0.05pH_d135_0408b.txt <==\r\n",
      "Scaffold_01\t37487642\t37487694\td135\tScaffold_01\t37487616\t37487694\trepeat_region\r\n",
      "Scaffold_01\t37487715\t37487765\td135\tScaffold_01\t37487715\t37487786\trepeat_region\r\n",
      "Scaffold_01\t37487642\t37487765\td135\tScaffold_01\t37466097\t37499469\tintergenic\r\n",
      "Scaffold_01\t60237055\t60237175\td135\tScaffold_01\t60236838\t60238837\t3prime_UTR\r\n",
      "Scaffold_01\t60237055\t60237175\td135\tScaffold_01\t60213212\t60238837\tintergenic\r\n",
      "Scaffold_01\t88395172\t88395200\td135\tScaffold_01\t88390975\t88404593\tintergenic\r\n",
      "Scaffold_02\t41360718\t41360876\td135\tScaffold_02\t41359948\t41362642\tintron\r\n",
      "Scaffold_04\t12714271\t12714332\td135\tScaffold_04\t12706685\t12715484\tintron\r\n",
      "Scaffold_04\t22243623\t22243695\td135\tScaffold_04\t22239926\t22244632\tintron\r\n",
      "Scaffold_04\t22243623\t22243674\td135\tScaffold_04\t22243587\t22243674\trepeat_region\r\n",
      "\r\n",
      "==> /Users/strigg/Desktop/20200408/aov_0.05pH_d145_0408b.txt <==\r\n",
      "Scaffold_01\t8256540\t8256708\td145\tScaffold_01\t8255067\t8257066\t3prime_UTR\r\n",
      "Scaffold_01\t8256540\t8256708\td145\tScaffold_01\t8256420\t8256734\texon\r\n",
      "Scaffold_01\t8256721\t8257066\td145\tScaffold_01\t8255067\t8257066\t3prime_UTR\r\n",
      "Scaffold_01\t8256721\t8256734\td145\tScaffold_01\t8256420\t8256734\texon\r\n",
      "Scaffold_01\t8256734\t8257066\td145\tScaffold_01\t8256734\t8257066\tintergenic\r\n",
      "Scaffold_01\t8256735\t8257612\td145\tScaffold_01\t8256735\t8257734\tput_promoter\r\n",
      "Scaffold_01\t8257067\t8257612\td145\tScaffold_01\t8257067\t8258104\texon\r\n",
      "Scaffold_01\t8518380\t8518420\td145\tScaffold_01\t8518359\t8518420\texon\r\n",
      "Scaffold_01\t8518421\t8518655\td145\tScaffold_01\t8518421\t8519905\tintron\r\n",
      "Scaffold_01\t22426146\t22426204\td145\tScaffold_01\t22426110\t22426204\trepeat_region\r\n"
     ]
    }
   ],
   "source": [
    "!head /Users/strigg/Desktop/20200408/*_0408b.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp /Users/strigg/Desktop/20200408/*_0408b.txt \\\n",
    "/Volumes/web/metacarcinus/Pgenerosa/analyses/20200408/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mPanopea-generosa-v1.0.a4.2Kbin.sort.bed\u001b[m\u001b[m \u001b[31maov_0.05pH_d135_0408.txt\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.2Kbin.uniq.bed\u001b[m\u001b[m \u001b[31maov_0.05pH_d135_0408b.txt\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.gff3\u001b[m\u001b[m           \u001b[31maov_0.05pH_d145_0408.txt\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.sort.bed\u001b[m\u001b[m       \u001b[31maov_0.05pH_d145_0408b.txt\u001b[m\u001b[m\r\n",
      "\u001b[31mPanopea-generosa-v1.0.a4.uniq.bed\u001b[m\u001b[m       \u001b[31mday10_features.3CpG.txt\u001b[m\u001b[m\r\n",
      "\u001b[31mamb_features.3CpG.txt\u001b[m\u001b[m                   \u001b[31mday10_features.txt\u001b[m\u001b[m\r\n",
      "\u001b[31mamb_features.txt\u001b[m\u001b[m                        \u001b[31mday135_features.3CpG.txt\u001b[m\u001b[m\r\n",
      "\u001b[31maov_0.05pH_amb_0408.txt\u001b[m\u001b[m                 \u001b[31mday135_features.txt\u001b[m\u001b[m\r\n",
      "\u001b[31maov_0.05pH_amb_0408b.txt\u001b[m\u001b[m                \u001b[31mday145_features.3CpG.txt\u001b[m\u001b[m\r\n",
      "\u001b[31maov_0.05pH_d10_0408.txt\u001b[m\u001b[m                 \u001b[31mday145_features.txt\u001b[m\u001b[m\r\n",
      "\u001b[31maov_0.05pH_d10_0408b.txt\u001b[m\u001b[m\r\n"
     ]
    }
   ],
   "source": [
    "!ls /Volumes/web/metacarcinus/Pgenerosa/analyses/20200408/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generate first exon gff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.exon.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!grep \"exon1;\" /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.exon.gff3 |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!grep \"exon1;\" /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.exon.gff3 |\\\n",
    "awk '{ gsub(\"exon\",\"1stExon\",$3); print $0 }' |\\\n",
    "tr ' ' '\\t' |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!grep \"exon1;\" /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.exon.gff3 |\\\n",
    "awk '{ gsub(\"exon\",\"1stExon\",$3); print $0 }' |\\\n",
    "tr ' ' '\\t' |\\\n",
    "/Users/Shelly/anaconda3/bin/bedtools sort -i - \\\n",
    "> /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.1stexon.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generate first intron track"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.introns.bed \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!/Users/Shelly/anaconda3/bin/bedtools sort -i \\\n",
    "/Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.introns.bed |\\\n",
    "/Users/Shelly/anaconda3/bin/bedtools closest \\\n",
    "-a /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.1stexon.gff3 \\\n",
    "-b - \\\n",
    "-wb | \\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!/Users/Shelly/anaconda3/bin/bedtools sort -i \\\n",
    "/Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.introns.bed |\\\n",
    "/Users/Shelly/anaconda3/bin/bedtools closest \\\n",
    "-a /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.1stexon.gff3 \\\n",
    "-b - \\\n",
    "-wb | \\\n",
    "awk -F\"\\t\" '{if((($7==\"-\")&&($4-$12 < 5 )&&($4-$12 >=0 ))||(($7==\"+\")&&($11-$5 < 5)&&($11-$5 >=0)))print $0}' |\\\n",
    "head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!/Users/Shelly/anaconda3/bin/bedtools sort -i \\\n",
    "/Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.introns.bed |\\\n",
    "/Users/Shelly/anaconda3/bin/bedtools closest \\\n",
    "-a /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.1stexon.gff3 \\\n",
    "-b - \\\n",
    "-wb | \\\n",
    "awk -F\"\\t\" '{if((($7==\"-\")&&($4-$12 < 5 )&&($4-$12 >=0 ))||(($7==\"+\")&&($11-$5 < 5)&&($11-$5 >=0)))print $0}' |\\\n",
    "awk '{ gsub(\"1stExon\",\"1stIntron\",$3); print $1,$2,$3,$11,$12,$6,$7,$8,$9 }' |\\\n",
    "tr ' ' '\\t' \\\n",
    "> /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.1stintron.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head /Users/Shelly/Downloads/feature_tracks/Panopea-generosa-v1.0.a4.1stintron.gff3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}