Update miRBase download URLs (#413)

* Update mirbase.org URLs fixes bcbio/bcbio-nextgen#3721 fixes bcbio/bcbio-nextgen#3722 * Some miRBase downloads no longer require uncompressing. Fixes bcbio/bcbio-nextgen#3723 --------- Co-authored-by: Rob Chekaluk <rchekaluk@ppolitics.org>
chapmanb · Dec 11, 2023 · cd1b727 · cd1b727
1 parent acc5877
commit cd1b727
Show file tree

Hide file tree

Showing 6 changed files with 60 additions and 42 deletions.
diff --git a/ggd-recipes/BDGP6/mirbase.yaml b/ggd-recipes/BDGP6/mirbase.yaml
@@ -13,13 +13,16 @@ recipe:
         wget --random-wait --retry-connrefused -nv -c -O tmp.gtf.gz ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/GCF_000001215.4_Release_6_plus_ISO1_MT_genomic.gff.gz
         zgrep -v exon tmp.gtf.gz | grep -v region | sed 's/Name/name/g' | sed -e 's/NC_004353.4/chr4/g; s/NC_004354.4/chrX/g; s/NC_024511.2/chrM/g; s/NC_024512.1/chrY/g; s/NT_033777.3/chr3R/g; s/NT_033778.4/chr2R/g; s/NT_033779.5/chr2L/g; s/NT_037436.4/chr3L/g; s/NW_007931121.1/CP007120.1/g' | sed 's/=/ /g' > srna-transcripts.gtf
         # mirbase
-        wget --random-wait --retry-connrefused -nv -c -O hairpin.t.fa.gz ftp://mirbase.org/pub/mirbase/22/hairpin.fa.gz && gunzip -f hairpin.t.fa.gz
+        wget --random-wait --retry-connrefused -nv -c -O hairpin.t.fa https://mirbase.org/download/CURRENT/hairpin.fa
         cat hairpin.t.fa | awk '{if ($0~/>dme/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
-        wget --random-wait --retry-connrefused -nv -c -O mature.t.fa.gz ftp://mirbase.org/pub/mirbase/22/mature.fa.gz && gunzip -f mature.t.fa.gz
+        rm hairpin.t.fa
+        wget --random-wait --retry-connrefused -nv -c -O mature.t.fa https://mirbase.org/download/CURRENT/mature.fa
         cat mature.t.fa | awk '{if ($0~/>dme/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
-        wget --random-wait --retry-connrefused -nv -c -O miRNA.t.str.gz ftp://mirbase.org/pub/mirbase/22/miRNA.str.gz && gunzip -f miRNA.t.str.gz
+        rm mature.t.fa
+        wget --random-wait --retry-connrefused -nv -c -O miRNA.t.str https://mirbase.org/download/CURRENT/miRNA.dat
         cat miRNA.t.str | awk '{if ($0~/dme/)print $0}' > miRNA.str
-        wget --random-wait --retry-connrefused -nv -c -O mirbase.gff3 ftp://mirbase.org/pub/mirbase/22/genomes/dme.gff3
+        rm miRNA.t.str
+        wget --random-wait --retry-connrefused -nv -c -O mirbase.gff3 https://mirbase.org/download/dme.gff3
         wget --no-check-certificate -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip -f Rfam_for_miRDeep.fa.gz
         # targetscan analysis
         wget  --random-wait --retry-connrefused --no-check-certificate -nv -c -O Summary_Counts.txt.zip http://www.targetscan.org/fly_72/fly_72_data_download/Summary_Counts.all_predictions.txt.zip && unzip Summary_Counts.txt.zip

diff --git a/ggd-recipes/canFam3/mirbase.yaml b/ggd-recipes/canFam3/mirbase.yaml
@@ -10,20 +10,23 @@ recipe:
       - |
         mkdir -p srnaseq
         cd srnaseq
-        wget --random-wait --retry-connrefused -q -c -O cfa.gff3 ftp://mirbase.org/pub/mirbase/21/genomes/cfa.gff3
+        wget --random-wait --retry-connrefused -q -c -O cfa.gff3 https://mirbase.org/download/cfa.gff3
         awk '$3=="miRNA"' cfa.gff3 | sed 's/=/ /g' > srna-transcripts.gtf
         wget  --random-wait --retry-connrefused -q -c -O rmsk.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/canFam3/database/rmsk.txt.gz
         zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' >> srna-transcripts.gtf
         wget  --random-wait --retry-connrefused  -q -c -O refGene.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/canFam3/database/refGene.txt.gz
         zcat refGene.txt.gz | awk '{print $3"\t.\tgene\t"$5"\t"$6"\t.\t"$4"\t.\tname "$13";"}' >> srna-transcripts.gtf
         # mirbase
-        wget  --random-wait --retry-connrefused  -q -c -O hairpin.fa.gz ftp://mirbase.org/pub/mirbase/21/hairpin.fa.gz
+        wget  --random-wait --retry-connrefused  -q -c -O hairpin.fa.raw https://mirbase.org/download/CURRENT/hairpin.fa
-        zcat hairpin.fa.gz |  awk '{if ($0~/>cfa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
+        cat hairpin.fa.raw |  awk '{if ($0~/>cfa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
-        wget  --random-wait --retry-connrefused  -q -c -O mature.fa.gz ftp://mirbase.org/pub/mirbase/21/mature.fa.gz
+        rm hairpin.fa.raw
-        zcat mature.fa.gz |  awk '{if ($0~/>cfa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
+        wget  --random-wait --retry-connrefused  -q -c -O mature.fa.raw https://mirbase.org/download/CURRENT/mature.fa
-        wget  --random-wait --retry-connrefused  -q -c -O miRNA.str.gz ftp://mirbase.org/pub/mirbase/21/miRNA.str.gz
+        cat mature.fa.raw |  awk '{if ($0~/>cfa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
-        zcat miRNA.str.gz | awk '{if ($0~/cfa/)print $0}' > miRNA.str
+        rm mature.fa.raw
-        wget --random-wait --retry-connrefused -q -c -O mirbase.gff3 ftp://mirbase.org/pub/mirbase/21/genomes/cfa.gff3
+        wget  --random-wait --retry-connrefused  -q -c -O miRNA.dat.raw https://mirbase.org/download/CURRENT/miRNA.dat
+        cat miRNA.dat.raw | awk '{if ($0~/cfa/)print $0}' > miRNA.str
+        rm miRNA.dat.raw
+        wget --random-wait --retry-connrefused -q -c -O mirbase.gff3 https://mirbase.org/download/cfa.gff3
         # mintmap
         # mirdeep2
         wget  --random-wait --retry-connrefused --no-check-certificate -q -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip Rfam_for_miRDeep.fa.gz

diff --git a/ggd-recipes/hg19/mirbase.yaml b/ggd-recipes/hg19/mirbase.yaml
@@ -23,13 +23,16 @@ recipe:
         # wget --random-wait --retry-connrefused  -nv -c -O piR_hg19_v1.0.bed.gz http://www.regulatoryrna.org/database/piRNA/download/archive/v1.0/bed/piR_hg19_v1.0.bed.gz
         # zcat piR_hg19_v1.0.bed.gz | awk '{print $1"\t.\tpiRNA\t"$2"\t"$3"\t.\t"$6"\t.\tname "$4";"}' >> srna-transcripts.gtf
         # mirbase
-        wget  --random-wait --retry-connrefused  -nv -c -O hairpin.fa.gz https://www.mirbase.org/ftp/21/hairpin.fa.gz
+        wget  --random-wait --retry-connrefused  -nv -c -O hairpin.fa.raw https://mirbase.org/download/CURRENT/hairpin.fa
-        zcat hairpin.fa.gz |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
+        cat hairpin.fa.raw |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
-        wget  --random-wait --retry-connrefused  -nv -c -O mature.fa.gz https://www.mirbase.org/ftp/21/mature.fa.gz
+        rm hairpin.fa.raw
-        zcat mature.fa.gz |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
+        wget  --random-wait --retry-connrefused  -nv -c -O mature.fa.raw https://mirbase.org/download/CURRENT/mature.fa
-        wget  --random-wait --retry-connrefused  -nv -c -O miRNA.str.gz https://www.mirbase.org/ftp/21/miRNA.str.gz
+        cat mature.fa.raw |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
-        zcat miRNA.str.gz | awk '{if ($0~/hsa/)print $0}' > miRNA.str
+        rm mature.fa.raw
-        wget --random-wait --retry-connrefused -nv -c -O mirbase.gff3 https://www.mirbase.org/ftp/21/genomes/hsa.gff3
+        wget  --random-wait --retry-connrefused  -nv -c -O miRNA.dat.raw https://mirbase.org/download/CURRENT/miRNA.dat
+        cat miRNA.dat.raw | awk '{if ($0~/hsa/)print $0}' > miRNA.str
+        rm miRNA.dat.raw
+        wget --random-wait --retry-connrefused -nv -c -O mirbase.gff3 https://mirbase.org/download/hsa.gff3
         # tdrmapper
         wget   --random-wait --retry-connrefused  --no-check-certificate -nv -c -O trna_mature_pre.fa https://github.com/sararselitsky/tDRmapper/raw/master/hg19_mature_and_pre.fa
         # mintmap

diff --git a/ggd-recipes/hg38-noalt/mirbase.yaml b/ggd-recipes/hg38-noalt/mirbase.yaml
@@ -10,7 +10,7 @@ recipe:
       - |
         mkdir -p srnaseq
         cd srnaseq
-        wget --random-wait --retry-connrefused -q -c -O hsa.gff3 https://www.mirbase.org/ftp/21/genomes/hsa.gff3
+        wget --random-wait --retry-connrefused -q -c -O hsa.gff3 https://mirbase.org/download/hsa.gff3
         awk '$3=="miRNA"' hsa.gff3 | sed 's/=/ /g' > srna-transcripts.gtf
         wget  --random-wait --retry-connrefused -q -c -O wgRna.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/wgRna.txt.gz
         zgrep -v 'hsa-' wgRna.txt.gz | awk '{print $2"\t.\tncrna\t"$3"\t"$4"\t.\t"$7"\t.\tname "$5";"}' >> srna-transcripts.gtf
@@ -24,13 +24,16 @@ recipe:
         # wget http://www.regulatoryrna.org/database/piRNA/download/archive/v1.0/bed/piR_hg19_v1.0.bed.gz
         # zcat piR_hg19_v1.0.bed.gz | awk '{print $1"\t.\tpiRNA\t"$2"\t"$3"\t.\t"$6"\t.\tname "$4";"}' >> srna-transcripts.gtf
         # mirbase
-        wget  --random-wait --retry-connrefused  -q -c -O hairpin.fa.gz https://www.mirbase.org/ftp/21/hairpin.fa.gz
+        wget  --random-wait --retry-connrefused  -q -c -O hairpin.fa.raw https://mirbase.org/download/CURRENT/hairpin.fa
-        zcat hairpin.fa.gz |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
+        cat hairpin.fa.raw |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
-        wget  --random-wait --retry-connrefused  -q -c -O mature.fa.gz https://www.mirbase.org/ftp/21/mature.fa.gz
+        rm hairpin.fa.raw
-        zcat mature.fa.gz |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
+        wget  --random-wait --retry-connrefused  -q -c -O mature.fa.raw https://mirbase.org/download/CURRENT/mature.fa
-        wget  --random-wait --retry-connrefused  -q -c -O miRNA.str.gz https://www.mirbase.org/ftp/21/miRNA.str.gz
+        cat mature.fa.raw|  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
-        zcat miRNA.str.gz | awk '{if ($0~/hsa/)print $0}' > miRNA.str
+        rm mature.fa.raw
-        wget --random-wait --retry-connrefused -q -c -O mirbase.gff3 https://www.mirbase.org/ftp/21/genomes/hsa.gff3
+        wget  --random-wait --retry-connrefused  -q -c -O miRNA.dat.raw https://mirbase.org/download/CURRENT/miRNA.dat
+        cat miRNA.dat.raw | awk '{if ($0~/hsa/)print $0}' > miRNA.str
+        rm miRNA.dat.raw
+        wget --random-wait --retry-connrefused -q -c -O mirbase.gff3 https://mirbase.org/download/hsa.gff3
         #tdrmapper
         wget   --random-wait --retry-connrefused  --no-check-certificate -q -c -O trna_mature_pre.fa https://github.com/sararselitsky/tDRmapper/raw/master/hg19_mature_and_pre.fa
         # mintmap

diff --git a/ggd-recipes/hg38/mirbase.yaml b/ggd-recipes/hg38/mirbase.yaml
@@ -10,7 +10,7 @@ recipe:
       - |
         mkdir -p srnaseq
         cd srnaseq
-        wget --random-wait --retry-connrefused -q -c -O hsa.gff3 https://www.mirbase.org/ftp/21/genomes/hsa.gff3
+        wget --random-wait --retry-connrefused -q -c -O hsa.gff3 https://mirbase.org/download/hsa.gff3
         awk '$3=="miRNA"' hsa.gff3 | sed 's/=/ /g' > srna-transcripts.gtf
         wget  --random-wait --retry-connrefused -q -c -O wgRna.txt.gz http://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/wgRna.txt.gz
         zgrep -v 'hsa-' wgRna.txt.gz | awk '{print $2"\t.\tncrna\t"$3"\t"$4"\t.\t"$7"\t.\tname "$5";"}' >> srna-transcripts.gtf
@@ -24,13 +24,16 @@ recipe:
         # wget http://www.regulatoryrna.org/database/piRNA/download/archive/v1.0/bed/piR_hg19_v1.0.bed.gz
         # zcat piR_hg19_v1.0.bed.gz | awk '{print $1"\t.\tpiRNA\t"$2"\t"$3"\t.\t"$6"\t.\tname "$4";"}' >> srna-transcripts.gtf
         # mirbase
-        wget  --random-wait --retry-connrefused  -q -c -O hairpin.fa.gz https://www.mirbase.org/ftp/21/hairpin.fa.gz
+        wget  --random-wait --retry-connrefused  -q -c -O hairpin.fa.raw https://mirbase.org/download/CURRENT/hairpin.fa
-        zcat hairpin.fa.gz |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
+        cat hairpin.fa.raw |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
-        wget  --random-wait --retry-connrefused  -q -c -O mature.fa.gz https://www.mirbase.org/ftp/21/mature.fa.gz
+        rm hairpin.fa.raw
-        zcat mature.fa.gz |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
+        wget  --random-wait --retry-connrefused  -q -c -O mature.fa.raw https://mirbase.org/download/CURRENT/mature.fa
-        wget  --random-wait --retry-connrefused  -q -c -O miRNA.str.gz https://www.mirbase.org/ftp/21/miRNA.str.gz
+        cat mature.fa.raw |  awk '{if ($0~/>hsa/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
-        zcat miRNA.str.gz | awk '{if ($0~/hsa/)print $0}' > miRNA.str
+        rm mature.fa.raw
-        wget --random-wait --retry-connrefused -q -c -O mirbase.gff3 https://www.mirbase.org/ftp/21/genomes/hsa.gff3
+        wget  --random-wait --retry-connrefused  -q -c -O miRNA.dat.raw https://mirbase.org/download/CURRENT/miRNA.dat
+        cat miRNA.dat.raw | awk '{if ($0~/hsa/)print $0}' > miRNA.str
+        rm miRNA.dat.raw
+        wget --random-wait --retry-connrefused -q -c -O mirbase.gff3 https://mirbase.org/download/hsa.gff3
         #tdrmapper
         wget   --random-wait --retry-connrefused  --no-check-certificate -q -c -O trna_mature_pre.fa https://github.com/sararselitsky/tDRmapper/raw/master/hg19_mature_and_pre.fa
         # mintmap

diff --git a/ggd-recipes/mm10/mirbase.yaml b/ggd-recipes/mm10/mirbase.yaml
@@ -18,13 +18,16 @@ recipe:
         zcat rmsk.txt.gz | awk '{print $6"\t.\trepeat\t"$7+1"\t"$8+1"\t.\t"$10"\t.\tname "$12";"}' >> srna-transcripts.gtf
         wget  --random-wait --retry-connrefused -q -N -c https://www.mirbase.org/ftp/20/genomes/mmu.gff3
         awk '$3=="miRNA"' mmu.gff3 | sed 's/=/ /g' >> srna-transcripts.gtf
-        wget  --random-wait --retry-connrefused -q -c -O hairpin.fa.gz https://www.mirbase.org/ftp/21/hairpin.fa.gz
+        wget  --random-wait --retry-connrefused -q -c -O hairpin.fa.raw https://mirbase.org/download/CURRENT/hairpin.fa
-        zcat hairpin.fa.gz |  awk '{if ($0~/>mmu/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
+        cat hairpin.fa.raw |  awk '{if ($0~/>mmu/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > hairpin.fa
-        wget  --random-wait --retry-connrefused -q -c -O mature.fa.gz https://www.mirbase.org/ftp/21/mature.fa.gz
+        rm hairpin.fa.raw
-        zcat mature.fa.gz |  awk '{if ($0~/>mmu/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
+        wget  --random-wait --retry-connrefused -q -c -O mature.fa.raw https://mirbase.org/download/CURRENT/mature.fa
-        wget --random-wait --retry-connrefused  -q -c -O miRNA.str.gz https://www.mirbase.org/ftp/21/miRNA.str.gz
+        cat mature.fa.raw |  awk '{if ($0~/>mmu/){name=$0; print name} else if ($0~/^>/){name=0};if (name!=0 && $0!~/^>/){print $0;}}' | sed 's/U/T/g'  > mature.fa
-        zcat miRNA.str.gz | awk '{if ($0~/mmu/)print $0}' > miRNA.str
+        rm mature.fa.raw
-        wget --random-wait --retry-connrefused -nv -c -O mirbase.gff3 https://www.mirbase.org/ftp/21/genomes/mmu.gff3
+        wget --random-wait --retry-connrefused  -q -c -O miRNA.dat.raw https://mirbase.org/download/CURRENT/miRNA.dat
+        cat miRNA.dat.raw | awk '{if ($0~/mmu/)print $0}' > miRNA.str
+        rm miRNA.dat.raw
+        wget --random-wait --retry-connrefused -nv -c -O mirbase.gff3 https://mirbase.org/download/mmu.gff3
         wget --random-wait --retry-connrefused  --no-check-certificate -q -c -O Rfam_for_miRDeep.fa.gz https://github.com/lpantano/mirdeep2_core/raw/data/Rfam_for_miRDeep.fa.gz && gunzip Rfam_for_miRDeep.fa.gz
         # targetscan analysis
         # wget --no-check-certificate -q -c -O Summary_Counts.txt.zip http://www.targetscan.org/mmu_71/mmu_71_data_download/Summary_Counts.all_predictions.txt.zip && unzip Summary_Counts.txt.zip