add tests and installation in diagnostic

docs/source/tutorial/setup.rst

@@ -34,6 +34,14 @@ To recreate the conda environment in the `environment.yml`, use
 .. code-block:: bash
 
     conda env create -f environment.yml
+    
+We have also provided a utility to install required RNA-Seq tools via a single command:
+
+.. code-block:: bash
+    
+    pyrpipe_diagnostic build-tools
+    
+**Note** Users must verify the versions of the tools installed in the conda environment.
 
 
 Setting up NCBI SRA-Tools
@@ -46,6 +54,11 @@ To do this
 - Type `vdb-config -i` command in terminal to open the NCBI SRA-Tools configuration editor.
 - Under the TOOLS tab, set prefetch downloads option to **public user-repository**
 
+Users can easily test if SRA-Tools has been setup properly by invoking the following command
+
+.. code-block:: bash
+    
+    pyrpipe_diagnostic test
 
 
 

pyrpipe/__diagnostic__.py

@@ -11,7 +11,7 @@
 import argparse
 from pyrpipe import pyrpipe_utils as pu
 from pyrpipe import reports
-
+from pyrpipe import test_sratools,buildtools
 
 
 
@@ -228,10 +228,16 @@ def multiqc():
     else:
         reports.generate_multiqc(logFile,tempDir,outDir=outDir,coverage=args.c,verbose=args.v,cleanup=args.r)
 
-
+def testsra():
+    test_sratools.runtest()
+
+def installtools():
+    buildtools.build_tools()
+
+
 def main():
     ##Start parsing
-    subcommands=['report','shell','benchmark','multiqc','all']
+    subcommands=['report','shell','benchmark','multiqc','test','build-tools']
     parser = argparse.ArgumentParser(
 
                 description='pyrpipe diagnostic utility',
@@ -242,9 +248,11 @@ def main():
                         shell      Generate all commands to shell (bash) script
                         benchmark Generate bemchmarks
                         multiqc Generate HTML report using multiqc
+                        test Test if NCBI-SRA-Tools is working with pyrpipe
+                        build-tools Install the tools required by pyrpipe via  bioconda. pyrpipe must be installed under conda environment.
                         
                         ''')
-    parser.add_argument('command', help='Subcommand to run [report,bash,benchmark,all]')
+    parser.add_argument('command', help='Subcommand to run report, shell,benchmark,multiqc,test')
 
 
     #parse first and last argument as subcommand and logfile 
@@ -262,4 +270,9 @@ def main():
         benchmark()
     elif args.command == 'multiqc':
         multiqc()
+    elif args.command == 'test':
+        testsra()
+    elif args.command == 'build-tools':
+        installtools()
+
 

pyrpipe/__init__.py

@@ -29,6 +29,21 @@
 import uuid
 
 
+####################DEFAULTS#####################
+_dryrun=False
+_safe=False
+_threads='2'
+_mem='2'
+_params_dir='./params'
+_logs_dir='./pyrpipe_logs'
+_timestamp=str(datetime.now()).replace(" ","-").replace(":","_")
+_uuid=uuid.uuid4().hex[:5]
+_log_name=_timestamp+"_"+_uuid+"_pyrpipe"
+_logging=False
+_verbose=True
+_force=False
+_multiqc=False
+#########################################
 
 ###logger
 class LogFormatter():
@@ -260,7 +275,7 @@ def init_sys_args(self):
 """
 #if pyrpipe_diagnostic is invoked
 if sys.argv[0].split('/')[-1]=='pyrpipe_diagnostic':
-    #will go to __diagnostic__.main
+    #will go to __diagnostic__.main, this behaviour is specified in the setup
     pass
 elif sys.argv[0].split('/')[-1]=='pyrpipe':
     #will go to __main__.main

pyrpipe/__main__.py

@@ -87,27 +87,6 @@ def args_to_list(args):
 if __name__ == '__main__':
     main()
 
-def caller(threads,mem,dryrun,safemode,paramdir,logsdir,nologs,verbose,force,infile,unknownargs):
-    #write pyrpipe configuration
-    #everything saved as str
-    conf={}
-    conf['threads']=threads
-    conf['memory']=mem
-    conf['params_dir']=paramdir
-    conf['dry']=dryrun
-    conf['force']=force
-    conf['safe']=safemode
-    conf['logs_dir']=logsdir
-    conf['logs']= not nologs
-    conf['verbose']=verbose
-
-    #with open('pyrpipe.conf', 'w') as outfile:
-    #    json.dump(conf, outfile,indent=4)
-    #execute
-    cmd=['python',infile]+unknownargs
-    print('MAININFILE',infile,unknownargs)
-    print('CMD',' '.join(cmd))
-    os.system(' '.join(cmd))
-
+
 
 

pyrpipe/arg_parser.py

@@ -15,6 +15,8 @@
 pyrpipe [<pyrpipe options>] --in <script.py> [<script options>]
 OR
 python <script.py> [<pyrpipe options>] [<script options>]
+
+use pyrpipe_diagnostic command for reports, and tests and installation of RNA-Seq tools
 """)
 
 
@@ -27,10 +29,7 @@
 parser.add_argument("--no-logs", help="Disable pyrpipe logs\nDefault: False",default=False,dest='nologs', action='store_true')
 parser.add_argument("--param-dir", help="Directory containing parameter yaml files\nDefault: ./params",dest='paramdir',default='params')
 parser.add_argument("--logs-dir", help="Directory for saving log files\nDefault: ./pyrpipe_logs",dest='logsdir',default='pyrpipe_logs')
-
 parser.add_argument("--multiqc", help="Autorun MultiQC after execution\nDefault: False",default=False,dest='multiqc', action='store_true')
-parser.add_argument("--build-tools", help="Install the tools required by pyrpipe via  bioconda.",dest='build_conda',default=False,action='store_true')
-parser.add_argument("--test-sratools", help="Test if NCBI SRA-Tools is woking.",dest='test_sra',default=False,action='store_true')
 
 parser.add_argument("--version", help="Print version information and exit",default=False,dest='versioninfo', action='store_true')
 

pyrpipe/buildtools.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Jan 17 16:12:47 2021
+
+@author: usingh
+"""
+
+import os
+
+
+def build_tools():
+    cmd_ins_mamba="conda install -c conda-forge mamba"
+    #cmd_update_py='mamba install python=3.8'
+    cmd_bioconda='mamba install -c bioconda sra-tools=2.10.9 trim-galore=0.6.6  bbmap=38.87 hisat2=2.2.1  star=2.7.7a bowtie2=2.4.2 kallisto=0.46.2 salmon stringtie=2.1.4 orfipy=0.0.3 samtools=1.11'
+
+    print(cmd_ins_mamba)
+    os.system(cmd_ins_mamba)
+    print(cmd_bioconda)
+    os.system(cmd_bioconda)
+
+
+
+
+

pyrpipe/pyrpipe_engine.py

@@ -46,7 +46,8 @@ def skippable(func):
     """
     if not _safe:
         return func
-    return True
+
+    return lambda x: True
 
 #decorator function for dry runs
 def dryable(func):
@@ -91,7 +92,7 @@ def parse_cmd(cmd):
     return cmd
 
 @dryable
-def get_shell_output(cmd,verbose=None,shell=False):
+def get_shell_output(cmd,verbose=None):
     """Function to run a shell command and return returncode, stdout and stderr
     Currently (pyrpipe v 0.0.4) this function is called in 
     get_return_status(), get_program_version()
@@ -114,15 +115,18 @@ def get_shell_output(cmd,verbose=None,shell=False):
     #not logging these commands
     cmd=parse_cmd(cmd)
     log_message=cmd
-    #command_name=cmd.split()[0]
+    command_name=cmd.split()[0]
+    if _safe and command_name in ['rm']:
+        pu.print_warning('SAFE MODE: Skipping command {}'.format(cmd))
+        return True
 
 
     starttime_str=time.strftime("%y-%m-%d %H:%M:%S", time.localtime(time.time()))
     if verbose:
         pu.print_notification("Start:"+starttime_str)
         pu.print_blue("$ "+log_message)
     try:
-        result = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=shell)
+        result = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=True)
         stdout,stderr = result.communicate()
         if stdout:
             stdout=stdout.decode("utf-8")
@@ -213,6 +217,10 @@ def execute_command(cmd,verbose=None,logs=None,objectid=None,command_name=""):
         command_name=cmd[0]
     log_message=" ".join(cmd)
 
+    ###safe mode
+    if _safe and command_name in ['rm']:
+        pu.print_warning('SAFE MODE: Skipping command {}'.format(cmd))
+        return True
 
 
     pu.print_notification("Start:"+starttime_str)
@@ -423,8 +431,8 @@ def delete_file(file_path):
         Path to the file to be deleted
     Returns
     -------
-    TYPE
-        DESCRIPTION.
+    bool
+        True if file deleted.
 
     """
     #if none
@@ -433,7 +441,10 @@ def delete_file(file_path):
 
     #if pu.check_files_exist(file_path):
     rm_cmd=['rm',file_path]
-    rv= get_return_status(rm_cmd)
+
+    #rv= get_return_status(rm_cmd)
+    rv=execute_command(rm_cmd)
+
     return rv
 
     #if file doesn't exist return true

pyrpipe/pyrpipe_utils.py

@@ -382,7 +382,6 @@ def parse_unix_args(valid_args_list,passed_args):
         Unknown argument Attr2 XX. ignoring...
         ['-O', './test', 'IN1', 'IN2']
     """
-
     popen_args=[]
     """
     Define some special arguments.

pyrpipe/sra.py

@@ -403,6 +403,7 @@ def trim(self,qc_object,delete_original=False,**kwargs):
         #each qc_object has a function run() to execute their method
         qcStatus=qc_object.perform_qc(self,objectid=self.srr_accession,**kwargs)
 
+
         #if job failed
         if not qcStatus[0]:
             #print ("Error performing QC for "+self.srr_accession)
@@ -439,16 +440,18 @@ def delete_fastq(self):
         """Delte the fastq files from the disk.
         The files are referenced by self.fastq_path or self.fastq_path and self.fastq2_path
         """
-            
+
         if self.layout=='PAIRED':
             if pe.delete_files(self.fastq_path,self.fastq2_path):
                 del self.fastq_path
                 del self.fastq2_path
                 return True
         else:
+
             if pe.delete_file(self.fastq_path):
                 del self.fastq_path
                 return True
+
         return False
 
 
@@ -613,4 +616,13 @@ def download_sra(self,**kwargs):
             self.layout="SINGLE"
 
 
-        return True
+        return True
+
+
+    #def destroy(self):
+        """
+        Delete everything for this object from memory and disk
+        """
+      #  pass
+
+

pyrpipe/test_sratools.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Jan 17 15:24:42 2021
+
+@author: usingh
+"""
+
+from pyrpipe import sra
+from pyrpipe import pyrpipe_utils as pu
+import os
+
+
+def runtest():
+    failed=False
+    sraob=sra.SRA('ERR726985',directory='./pyrpipe_sratest')
+    if not sraob.fastq_exists(): pu.print_boldred('Test failed'); failed=True
+    pu.print_notification('Cleaning up...')
+    sraob.delete_fastq()
+    os.rmdir(sraob.directory)
+
+    if failed:
+        pu.print_boldred('Paired end test failed'); 
+        failed=False
+
+    sraob=sra.SRA('SRR2134545',directory='./pyrpipe_sratest')
+    if not sraob.fastq_exists(): pu.print_boldred('Test failed'); failed=True
+    pu.print_notification('Cleaning up...')
+    sraob.delete_fastq()
+    os.rmdir(sraob.directory)
+
+    if failed:
+        pu.print_boldred('Single end test failed'); 
+        failed=False
+
+    if not failed:
+        pu.print_green('\n#####################All Tests Passed#####################\n')
+        os.rmdir('./pyrpipe_sratest')