Merge pull request #144 from squirrelo/add-demo-files-with-dowload-to…

…-the-database

Add demo files with dowload to the database

qiita_db/environment_manager.py

@@ -5,12 +5,23 @@
 #
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
-
+from tempfile import mkdtemp
+from tarfile import open as taropen
+from gzip import open as gzopen
+from os import remove
 from os.path import abspath, dirname, join
+from shutil import rmtree, move
 from functools import partial
+try:
+    from urllib import urlretrieve
+except ImportError:  # python3
+    from urllib.request import urlretrieve
+
 from psycopg2 import connect
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 
+from qiita_db.util import get_db_files_base_dir
+
 get_support_file = partial(join, join(dirname(abspath(__file__)),
                                       'support_files'))
 
@@ -165,6 +176,37 @@ def make_demo_environment(base_data_dir, base_work_dir, user, password, host):
     cur.close()
     conn.close()
 
+    # download files from thebeast
+    url = ("ftp://thebeast.colorado.edu/pub/QIIME_DB_Public_Studies/study_1001"
+           "_split_library_seqs_and_mapping.tgz")
+    outdir = mkdtemp()
+    basedir = join(outdir, "study_1001_split_library_seqs_and_mapping/")
+    try:
+        urlretrieve(url, join(outdir, "study_1001.tar.gz"))
+    except:
+        raise IOError("Error: DOWNLOAD FAILED")
+        rmtree(outdir)
+
+    # untar the files
+    with taropen(join(outdir, "study_1001.tar.gz")) as tar:
+        tar.extractall(outdir)
+    # un-gzip sequence file
+    with gzopen(join(basedir, "study_1001_split_library_seqs.fna.gz")) as gz:
+        with open(join(basedir, "seqs.fna"), 'w') as fout:
+            fout.write(gz.read())
+
+    # copy the preprocessed and procesed data to the study
+    dbbase = get_db_files_base_dir()
+    remove(join(dbbase, "processed_data/study_1001_closed_reference_otu"
+           "_table.biom"))
+    remove(join(dbbase, "preprocessed_data/seqs.fna"))
+    move(join(basedir, "study_1001_closed_reference_otu_table.biom"),
+         join(dbbase, "processed_data"))
+    move(join(basedir, "seqs.fna"), join(dbbase, "preprocessed_data"))
+
+    # clean up after ourselves
+    rmtree(outdir)
+
 
 def drop_demo_environment(user, password, host):
     r"""Drops the demo environment.
@@ -178,6 +220,7 @@ def drop_demo_environment(user, password, host):
     host : str
         The host where the postgres server is running
     """
+    base = get_db_files_base_dir()
     # Connect to the postgres server
     conn = connect(user=user, host=host, password=password)
     # Set the isolation level to AUTOCOMMIT so we can execute a
@@ -190,6 +233,13 @@ def drop_demo_environment(user, password, host):
     cur.close()
     conn.close()
 
+    # wipe the overwriiten test files so empty as on repo
+    with open(join(base, "preprocessed_data/seqs.fna"), 'w') as fout:
+        fout.write("\n")
+    with open(join(base, "processed_data/study_1001_closed_reference_otu_"
+              "table.biom"), 'w') as fout:
+        fout.write("\n")
+
 
 def make_production_environment():
     """TODO: Not implemented"""