Merge branch 'master' of github.com:McTavishLab/physcraper

.travis.yml

@@ -0,0 +1,14 @@
+language: python
+python:
+  - "2.7"
+# command to install dependencies
+install:
+  - pip install --quiet -r requirements.txt
+  - pip install --quiet . 
+# command to run tests
+script:
+  - wget 'https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz'
+  - gunzip  -cd taxdump.tar.gz | (tar xvf - names.dmp nodes.dmp)
+  - mv *.dmp tests/data/
+  - echo 'no' | python tests/testfilesetup.py
+  - py.test tests/ --setup-only

README.md

@@ -1,4 +1,7 @@
 # physcraper
+
+[![Build Status](https://travis-ci.org/McTavishLab/physcraper.svg?branch=master)](https://travis-ci.org/McTavishLab/physcraper)[![Documentation](https://readthedocs.org/projects/physcraper/badge/?version=latest&style=flat)](https://physcraper.readthedocs.io/en/latest/)
+
 Continual gene tree updating. 
 Uses a tree from Open tree of Life (or your own tree) and an alignment to search for and adds homologous sequences to phylogenetic inference. 
 

tests/concatsetup.py

@@ -4,77 +4,78 @@
 from physcraper import wrappers, OtuJsonDict, ConfigObj, IdDicts
 #
 
+def test():
 
-# tiny its
-seqaln = "tests/data/tiny_comb_its/tiny_comb_its.fasta"
-mattype = "fasta"
-trfn = "tests/data/tiny_comb_its/tiny_comb_its.tre"
-schema_trf = "newick"
-id_to_spn = r"tests/data/tiny_comb_its/nicespl.csv"
-workdir = "tests/data/tiny_comb_its"
-configfi = "tests/data/test.config"
-otu_jsonfi = "{}/otu_dict.json".format(workdir)
-threshold = 2
-selectby = "blast"
+    # tiny its
+    seqaln = "tests/data/tiny_comb_its/tiny_comb_its.fasta"
+    mattype = "fasta"
+    trfn = "tests/data/tiny_comb_its/tiny_comb_its.tre"
+    schema_trf = "newick"
+    id_to_spn = r"tests/data/tiny_comb_its/nicespl.csv"
+    workdir = "tests/data/tiny_comb_its"
+    configfi = "tests/data/test.config"
+    otu_jsonfi = "{}/otu_dict.json".format(workdir)
+    threshold = 2
+    selectby = "blast"
 
-if not os.path.exists("{}".format(workdir)):
-    os.makedirs("{}".format(workdir))
+    if not os.path.exists("{}".format(workdir)):
+        os.makedirs("{}".format(workdir))
 
-conf = ConfigObj(configfi)
-ids = IdDicts(conf, workdir=workdir)
+    conf = ConfigObj(configfi)
+    ids = IdDicts(conf, workdir=workdir)
 
 
-if os.path.exists(otu_jsonfi):
-    print("load json")
-    otu_json = json.load(open(otu_jsonfi))
-else:
-    otu_json = OtuJsonDict(id_to_spn, ids)
-    json.dump(otu_json, open(otu_jsonfi, "w"))
+    if os.path.exists(otu_jsonfi):
+        print("load json")
+        otu_json = json.load(open(otu_jsonfi))
+    else:
+        otu_json = OtuJsonDict(id_to_spn, ids)
+        json.dump(otu_json, open(otu_jsonfi, "w"))
 
 
-wrappers.filter_data_run(seqaln,
-                         mattype,
-                         trfn,
-                         schema_trf,
-                         workdir,
-                         threshold,
-                         otu_jsonfi,
-                         configfi,
-                         selectby=selectby)
+    wrappers.filter_data_run(seqaln,
+                             mattype,
+                             trfn,
+                             schema_trf,
+                             workdir,
+                             threshold,
+                             otu_jsonfi,
+                             configfi,
+                             selectby=selectby)
 
 
-# tiny ets
-seqaln = "tests/data/tiny_comb_ets/tiny_comb_ets.fasta"
-mattype = "fasta"
-trfn = "tests/data/tiny_comb_ets/tiny_comb_ets.tre"
-schema_trf = "newick"
-id_to_spn = r"tests/data/tiny_comb_ets/nicespl.csv"
+    # tiny ets
+    seqaln = "tests/data/tiny_comb_ets/tiny_comb_ets.fasta"
+    mattype = "fasta"
+    trfn = "tests/data/tiny_comb_ets/tiny_comb_ets.tre"
+    schema_trf = "newick"
+    id_to_spn = r"tests/data/tiny_comb_ets/nicespl.csv"
 
-workdir = "tests/data/tiny_comb_ets"
-configfi = "tests/data/test.config"
-otu_jsonfi = "{}/otu_dict.json".format(workdir)
-treshold = 2
-selectby = "blast"
+    workdir = "tests/data/tiny_comb_ets"
+    configfi = "tests/data/test.config"
+    otu_jsonfi = "{}/otu_dict.json".format(workdir)
+    treshold = 2
+    selectby = "blast"
 
-if not os.path.exists("{}".format(workdir)):
-        os.makedirs("{}".format(workdir))
+    if not os.path.exists("{}".format(workdir)):
+            os.makedirs("{}".format(workdir))
 
-conf = ConfigObj(configfi)
-ids = IdDicts(conf, workdir=workdir)
+    conf = ConfigObj(configfi)
+    ids = IdDicts(conf, workdir=workdir)
 
-if os.path.exists(otu_jsonfi):
-    print("load json")
-    otu_json = json.load(open(otu_jsonfi))
-else:
-    otu_json = OtuJsonDict(id_to_spn, ids)
-    json.dump(otu_json, open(otu_jsonfi, "w"))
+    if os.path.exists(otu_jsonfi):
+        print("load json")
+        otu_json = json.load(open(otu_jsonfi))
+    else:
+        otu_json = OtuJsonDict(id_to_spn, ids)
+        json.dump(otu_json, open(otu_jsonfi, "w"))
 
-wrappers.filter_data_run(seqaln,
-                         mattype,
-                         trfn,
-                         schema_trf,
-                         workdir,
-                         threshold,
-                         otu_jsonfi,
-                         configfi,
-                         selectby=selectby)
+    wrappers.filter_data_run(seqaln,
+                             mattype,
+                             trfn,
+                             schema_trf,
+                             workdir,
+                             threshold,
+                             otu_jsonfi,
+                             configfi,
+                             selectby=selectby)

tests/filter_OToL.py

@@ -2,24 +2,26 @@
 import json
 from physcraper import wrappers, OtuJsonDict
 
-# Use OpenTree phylesystem identifiers to get study and tree
-study_id = "pg_873"
-tree_id = "tree1679"
-seqaln = "tests/data/minitest.fas"
-mattype = "fasta"
-workdir = "tests/output/opentree"
-configfi = "example.config"
 
-threshold = 2
-selectby = "blast"
-downtorank = "species"
+def test():
+    # Use OpenTree phylesystem identifiers to get study and tree
+    study_id = "pg_873"
+    tree_id = "tree1679"
+    seqaln = "tests/data/minitest.fas"
+    mattype = "fasta"
+    workdir = "tests/output/opentree"
+    configfi = "example.config"
 
-# select a wrapper function, depending on what you want to do, see short tutorial:
-wrappers.filter_OTOL(study_id,
-                     tree_id,
-                     seqaln,
-                     mattype,
-                     configfi,
-                     threshold,
-                     selectby=selectby,
-                     downtorank=downtorank)
+    threshold = 2
+    selectby = "blast"
+    downtorank = "species"
+
+    # select a wrapper function, depending on what you want to do, see short tutorial:
+    wrappers.filter_OTOL(study_id,
+                         tree_id,
+                         seqaln,
+                         mattype,
+                         configfi,
+                         threshold,
+                         selectby=selectby,
+                         downtorank=downtorank)

tests/filter_blast.py

@@ -4,32 +4,33 @@
 from physcraper import wrappers
 #
 
-seqaln = "tests/data/tiny_test_example/test.fas"
-mattype = "fasta"
-trfn = "tests/data/tiny_test_example/test.tre"
-schema_trf = "newick"
-workdir = "test_filter"
-configfi = "tests/data/blubb_localblast.config"
-id_to_spn = r"tests/data/tiny_test_example/test_nicespl.csv"
-otu_jsonfi = "{}/otu_dict.json".format(workdir)
-threshold = 2
-selectby = "blast"
+def test():
+    seqaln = "tests/data/tiny_test_example/test.fas"
+    mattype = "fasta"
+    trfn = "tests/data/tiny_test_example/test.tre"
+    schema_trf = "newick"
+    workdir = "test_filter"
+    configfi = "tests/data/blubb_localblast.config"
+    id_to_spn = r"tests/data/tiny_test_example/test_nicespl.csv"
+    otu_jsonfi = "{}/otu_dict.json".format(workdir)
+    threshold = 2
+    selectby = "blast"
 
-if os.path.exists(otu_jsonfi):
-    otu_json = json.load(open(otu_jsonfi))
-else:
-    otu_json = wrappers.OtuJsonDict(id_to_spn, configfi)
-    if not os.path.exists(workdir):
-        os.mkdir(workdir)
-    json.dump(otu_json, open(otu_jsonfi, "w"))
+    if os.path.exists(otu_jsonfi):
+        otu_json = json.load(open(otu_jsonfi))
+    else:
+        otu_json = wrappers.OtuJsonDict(id_to_spn, configfi)
+        if not os.path.exists(workdir):
+            os.mkdir(workdir)
+        json.dump(otu_json, open(otu_jsonfi, "w"))
 
 
-wrappers.filter_data_run(seqaln,
-                         mattype,
-                         trfn,
-                         schema_trf,
-                         workdir,
-                         threshold,
-                         otu_jsonfi,
-                         configfi,
-                         selectby=selectby)
+    wrappers.filter_data_run(seqaln,
+                             mattype,
+                             trfn,
+                             schema_trf,
+                             workdir,
+                             threshold,
+                             otu_jsonfi,
+                             configfi,
+                             selectby=selectby)

tests/impl_concat.py

@@ -7,19 +7,21 @@
 from dendropy import DnaCharacterMatrix
 from copy import deepcopy
 
+def test():
 
-workdir_its = "./runs/tiny_comb_its"
-workdir_ets = "./runs/tiny_comb_ets"
-workdir_3 = "./runs/tiny_comb_ets"
-email = "martha.kandziora@yahoo.com"
-percentage = 0.4
 
-pickle_fn = "scrape_checkpoint.p"
+    workdir_its = "./runs/tiny_comb_its"
+    workdir_ets = "./runs/tiny_comb_ets"
+    workdir_3 = "./runs/tiny_comb_ets"
+    email = "martha.kandziora@yahoo.com"
+    percentage = 0.4
 
-workdir_comb = "./tests/output/impl_concat"
-genelist = {"its": {"workdir": workdir_its, "pickle": pickle_fn}, 
-            "ets": {"workdir": workdir_ets, "pickle": pickle_fn}
-            }
+    pickle_fn = "scrape_checkpoint.p"
 
-conc = wrappers.concat(genelistdict=genelist, workdir_comb=workdir_comb,
-                       email=email, percentage=percentage, user_concat_fn=None)
+    workdir_comb = "./tests/output/impl_concat"
+    genelist = {"its": {"workdir": workdir_its, "pickle": pickle_fn}, 
+                "ets": {"workdir": workdir_ets, "pickle": pickle_fn}
+                }
+
+    conc = wrappers.concat(genelistdict=genelist, workdir_comb=workdir_comb,
+                           email=email, percentage=percentage, user_concat_fn=None)

tests/own_data_scrape_local.py

@@ -3,33 +3,35 @@
 import json
 from physcraper import wrappers, OtuJsonDict, ConfigObj, IdDicts
 
-seqaln = "tests/data/tiny_test_example/test.fas"
-mattype = "fasta"
-trfn = "tests/data/tiny_test_example/test.tre"
-schema_trf = "newick"
-workdir = "tests/output/test_own_local"
-configfi = "tests/data/test.config"
-# configfi = "tests/data/aws.config"
-id_to_spn = r"tests/data/tiny_test_example/test_nicespl.csv"
-otu_jsonfi = "{}/otu_dict.json".format(workdir)
+def test():
 
+    seqaln = "tests/data/tiny_test_example/test.fas"
+    mattype = "fasta"
+    trfn = "tests/data/tiny_test_example/test.tre"
+    schema_trf = "newick"
+    workdir = "tests/output/test_own_local"
+    configfi = "tests/data/test.config"
+    # configfi = "tests/data/aws.config"
+    id_to_spn = r"tests/data/tiny_test_example/test_nicespl.csv"
+    otu_jsonfi = "{}/otu_dict.json".format(workdir)
 
-if not os.path.exists("{}".format(workdir)):
-    os.makedirs("{}".format(workdir))
 
-conf = ConfigObj(configfi)
-ids = IdDicts(conf, workdir=workdir)
+    if not os.path.exists("{}".format(workdir)):
+        os.makedirs("{}".format(workdir))
 
-if os.path.exists(otu_jsonfi):
-    print("load json")
-else:
-    otu_json = OtuJsonDict(id_to_spn, ids)
-    json.dump(otu_json, open(otu_jsonfi, "w"))
+    conf = ConfigObj(configfi)
+    ids = IdDicts(conf, workdir=workdir)
 
-wrappers.own_data_run(seqaln,
-                      mattype,
-                      trfn,
-                      schema_trf,
-                      workdir,
-                      otu_jsonfi,
-                      configfi)
+    if os.path.exists(otu_jsonfi):
+        print("load json")
+    else:
+        otu_json = OtuJsonDict(id_to_spn, ids)
+        json.dump(otu_json, open(otu_jsonfi, "w"))
+
+    wrappers.own_data_run(seqaln,
+                          mattype,
+                          trfn,
+                          schema_trf,
+                          workdir,
+                          otu_jsonfi,
+                          configfi)