bug fixes, mr subpackage created and other bits refactored

simbad/command_line/__init__.py

@@ -12,9 +12,9 @@
 
 import simbad.constants
 import simbad.lattice.search
+import simbad.mr
 import simbad.rotsearch.amore_search
 import simbad.util.mtz_util
-import simbad.util.mr_util
 import simbad.version
 
 header = """
@@ -155,7 +155,7 @@ def _simbad_contaminant_search(args):
     os.makedirs(contaminant_log_dir)
     os.makedirs(contaminant_out_dir)
     rotation_search = simbad.rotsearch.amore_search.AmoreRotationSearch(
-        args.amore_exe, args.mtz, args.work_dir, args.max_to_keep
+        args.amore_exe, args.mtz, stem, args.max_to_keep
     )
     rotation_search.sortfun()
     rotation_search.amore_run(
@@ -167,10 +167,10 @@ def _simbad_contaminant_search(args):
         logger.debug("Contaminant search summary file: %s", rot_summary_f)
         rotation_search.summarize(rot_summary_f)
         # Create directories for the contaminant search MR
-        contaminant_dir = os.path.join(args.work_dir, 'mr_contaminant')
+        contaminant_dir = os.path.join(stem, 'mr_contaminant')
         # Run MR on results
-        molecular_replacement = simbad.util.mr_util.MrSubmit(
-            args.mtz, args.mr_program, args.refine_program, model_dir, output_dir, args.early_term, args.enan
+        molecular_replacement = simbad.mr.MrSubmit(
+            args.mtz, args.mr_program, args.refine_program, contaminant_out_dir, contaminant_dir, args.early_term, args.enan
         )
         molecular_replacement.multiprocessing(rotation_search.search_results, nproc=args.nproc)
         mr_summary_f = os.path.join(stem, 'cont_mr.csv')
@@ -250,7 +250,7 @@ def _simbad_lattice_search(args):
         else:
             lattice_search.download_results(lattice_in_mod)
         # Run MR on results
-        molecular_replacement = simbad.util.mr_util.MrSubmit(
+        molecular_replacement = simbad.mr.MrSubmit(
             args.mtz, args.mr_program, args.refine_program, lattice_in_mod, lattice_mr_dir, args.early_term, args.enan
         )
         molecular_replacement.multiprocessing(lattice_search.search_results, nproc=args.nproc)

simbad/util/mr_util.py → simbad/mr/__init__.py

@@ -1,26 +1,28 @@
 """Class to run MR on SIMBAD results using code from MrBump"""
 
+from __future__ import print_function
+
+__author__ = "Adam Simpkin"
+__date__ = "09 Mar 2017"
+__version__ = "0.2"
+
 import copy_reg
 import logging
 import multiprocessing
 import os
 import pandas
 import types
 
+from simbad.mr import anomalous_util
+from simbad.mr import molrep_mr
+from simbad.mr import phaser_mr
+from simbad.mr import refmac_refine
 from simbad.parsers import molrep_parser
 from simbad.parsers import phaser_parser
 from simbad.parsers import refmac_parser
-from simbad.util import anomalous_util
-from simbad.util import molrep_util
 from simbad.util import mtz_util
-from simbad.util import phaser_util
-from simbad.util import refmac_util
 from simbad.util import simbad_util
 
-__author__ = "Adam Simpkin"
-__date__ = "09 Mar 2017"
-__version__ = "0.2"
-
 logger = logging.getLogger(__name__)
 
 
@@ -71,7 +73,7 @@ def __repr__(self):
                                                                    self.peaks_over_12_rms_within_2A_of_model)
 
     def _as_dict(self):
-        """Convert the :obj:`_MrScore <simbad.util.mr_util._MrScore>`
+        """Convert the :obj:`_MrScore <simbad.mr._MrScore>`
         object to a dictionary"""
         dictionary = {}
         for k in self.__slots__:
@@ -107,7 +109,7 @@ class MrSubmit(object):
 
     Examples
     --------
-    >>> from simbad.util.mr_util import MrSubmit
+    >>> from simbad.mr import MrSubmit
     >>> MR = MrSubmit('<mtz>', '<mr_program>', '<refine_program>', '<model_dir>', '<output_dir>', '<early_term>',
     >>>               '<enam>')
     >>> MR.multiprocessing('<results>', '<time_out>', '<nproc>')
@@ -248,18 +250,15 @@ def output_dir(self, output_dir):
 
     def _run_job(self, model):
         """Function to run MR on each model"""
-        logger.info("Running MR and refinement on %s", model.pdb_code)
+        logger.debug("Running MR and refinement on %s", model.pdb_code)
 
         try:
             os.mkdir(self.output_dir)
         except OSError:
             pass
 
         # Make output directories
-        os.mkdir(os.path.join(self.output_dir, model.pdb_code))
-        os.mkdir(os.path.join(self.output_dir, model.pdb_code, 'mr'))
-        os.mkdir(os.path.join(self.output_dir, model.pdb_code, 'mr', self.mr_program))
-        os.mkdir(os.path.join(self.output_dir, model.pdb_code, 'mr', self.mr_program, 'refine'))
+        os.makedirs(os.path.join(self.output_dir, model.pdb_code, 'mr', self.mr_program, 'refine'))
 
         # Set up MR input paths
         mr_pdbin = os.path.join(self.model_dir, '{0}.pdb'.format(model.pdb_code))
@@ -276,27 +275,27 @@ def _run_job(self, model):
         # Run job
         if self.mr_program.upper() == 'MOLREP':
             # Set up class with MOLREP input arguments
-            molrep = molrep_util.Molrep(self.enant, self.mtz, mr_logfile, mr_pdbin, mr_pdbout, self.space_group,
-                                        mr_workdir)
+            molrep = molrep_mr.Molrep(self.enant, self.mtz, mr_logfile, mr_pdbin, mr_pdbout, self.space_group,
+                                      mr_workdir)
             # Run MOLREP
             molrep.run()
 
             # Set up class with REFMAC input arguments
-            refmac = refmac_util.Refmac(self.mtz, ref_hklout, ref_logfile, mr_pdbout, ref_pdbout, ref_workdir)
+            refmac = refmac_refine.Refmac(self.mtz, ref_hklout, ref_logfile, mr_pdbout, ref_pdbout, ref_workdir)
             # Run REFMAC
             refmac.run()
 
         elif self.mr_program.upper() == 'PHASER':
             hklout = os.path.join(mr_workdir, '{0}_mr_output.mtz'.format(model.pdb_code))
 
             # Set up class with PHASER input arguments
-            phaser = phaser_util.Phaser(self.enant, self.f, self.mtz, hklout, mr_logfile, mr_pdbin, mr_pdbout,
-                                        self.sigf, self.solvent, mr_workdir)
+            phaser = phaser_mr.Phaser(self.enant, self.f, self.mtz, hklout, mr_logfile, mr_pdbin, mr_pdbout,
+                                      self.sigf, self.solvent, mr_workdir)
             # Run PHASER
             phaser.run()
 
             # Set up class with REFMAC input arguments
-            refmac = refmac_util.Refmac(hklout, ref_hklout, ref_logfile, mr_pdbout, ref_pdbout, ref_workdir)
+            refmac = refmac_refine.Refmac(hklout, ref_hklout, ref_logfile, mr_pdbout, ref_pdbout, ref_workdir)
             # Run REFMAC
             refmac.run()
 
@@ -376,7 +375,7 @@ def run(job_queue):
                 terminate = self._run_job(model)
 
                 if terminate:
-                    print "MR with {0} was successful so removing remaining jobs from inqueue".format(model.pdb_code)
+                    logger.debug("MR with {0} was successful so removing remaining jobs from inqueue".format(model.pdb_code))
                     while not job_queue.empty():
                         job = job_queue.get()
                         logger.debug("Removed job [%s] from inqueue", job.pdb_code)
@@ -388,6 +387,7 @@ def run(job_queue):
         for result in results:
             job_queue.put(result)
 
+        logger.info("Running MR on %d AMORE rotation function results", job_queue.qsize())
         processes = []
         # Set up processes equal to the number of processors input
         for i in range(nproc):
@@ -565,7 +565,7 @@ def summarize(self, csv_file):
                            "peaks_over_12_rms_within_2A_of_model"]
             else:
                 columns = ["phaser_tfz", "phaser_llg", "phaser_rfz", "final_r_fact", "final_r_free"]
-
+        
         df = pandas.DataFrame(
             [r._as_dict() for r in search_results],
             index=[r.pdb_code for r in search_results],

simbad/util/anomalous_util.py → simbad/mr/anomalous_util.py

@@ -8,10 +8,11 @@
 
 import iotbx.pdb
 import os
-import simbad_util
-import mtz_util
 import numpy as np
 
+from simbad.util import mtz_util
+from simbad.util import simbad_util
+
 
 class _AnomScore(object):
     """An anomalous phased fourier scoring class"""
@@ -36,13 +37,14 @@ def __repr__(self):
                                                                   self.peaks_over_12_rms,
                                                                   self.peaks_over_12_rms_within_2A_of_model)
     def _as_dict(self):
-        """Convert the :obj:`_MrScore <simbad.util.mr_util._MrScore>`
+        """Convert the :obj:`_MrScore <simbad.mr._MrScore>`
         object to a dictionary"""
         dict = {}
         for k in self.__slots__:
             dict[k] = getattr(self, k)
         return dict
 
+
 class AnomSearch():
     """An anomalous phased fourier running class
 
@@ -57,8 +59,8 @@ class AnomSearch():
 
     Example
     -------
-    >>> from simbad.util import anomalous_util
-    >>> anomalous_search = anomalous_util.AnomSearch("<mtz>", "<output_dir>")
+    >>> from simbad.mr.anomalous_util import AnomSearch
+    >>> anomalous_search = AnomSearch("<mtz>", "<output_dir>")
     >>> anomalous_search.run("<model>")
     """
 

simbad/util/molrep_util.py → simbad/mr/molrep_mr.py

@@ -1,9 +1,10 @@
 """Module to run molrep on a model"""
 
 import os
-import simbad_util
 import shutil
 
+import simbad.util.simbad_util
+
 __author__ = "Adam Simpkin"
 __date__ = "02 May 2017"
 __version__ = "0.1"
@@ -31,7 +32,7 @@ class Molrep(object):
 
     Example
     -------
-    >>> from simbad.util.molrep_util import Molrep
+    >>> from simbad.mr.molrep_mr import Molrep
     >>> molrep = Molrep('<enant>', '<hklin>', '<logfile>', '<pdbin>', '<pdbout>', '<space_group>', '<work_dir>')
     >>> molrep.run()
 
@@ -289,5 +290,5 @@ def molrep(hklin, pdbin, key, logfile):
         cmd = ["molrep",
                "-f", hklin,
                "-m", pdbin]
-        simbad_util.run_job(cmd, logfile=logfile, stdin=key)
+        simbad.util.simbad_util.run_job(cmd, logfile=logfile, stdin=key)
 

simbad/util/phaser_util.py → simbad/mr/phaser_mr.py

@@ -1,13 +1,15 @@
 """Module to run phaser on a model"""
 
-import os
-import simbad_util
-import shutil
-
 __author__ = "Adam Simpkin"
 __date__ = "02 May 2017"
 __version__ = "0.1"
 
+import os
+import shutil
+
+import simbad.util.simbad_util
+
+
 class Phaser(object):
     """Class to run PHASER
 
@@ -36,7 +38,7 @@ class Phaser(object):
 
     Examples
     --------
-    >>> from simbad.util.phaser_util import Phaser
+    >>> from simbad.mr.phaser_mr import Phaser
     >>> phaser = Phaser('<enant>', '<f>', '<hklin>', '<hklout>', '<logfile>', '<pdbin>', '<pdbout>', '<sigf>',
     >>>                 '<solvent>', '<workdir>')
     >>> phaser.run()
@@ -248,5 +250,5 @@ def phaser(logfile, key):
             Output log file
         """
         cmd = "phaser"
-        simbad_util.run_job(cmd, logfile=logfile, stdin=key)
+        simbad.util.simbad_util.run_job(cmd, logfile=logfile, stdin=key)
 

simbad/util/refmac_util.py → simbad/mr/refmac_refine.py

@@ -1,12 +1,13 @@
 """Module to run refmac on a model"""
 
-import os
-import simbad_util
-
 __author__ = "Adam Simpkin"
 __date__ = "02 May 2017"
 __version__ = "0.1"
 
+import os
+
+import simbad.util.simbad_util
+
 
 class Refmac(object):
     """Class to run refmac
@@ -32,7 +33,7 @@ class Refmac(object):
 
     Examples
     --------
-    >>> from simbad.util.refmac_util import Refmac
+    >>> from simbad.mr.refmac_refine import Refmac
     >>> refmac = Refmac('<hklin>', '<hklout>', '<logfile>', '<pdbin>', '<pdbout>', '<work_dir>')
     >>> refmac.run('<ncyc>')
 
@@ -172,5 +173,5 @@ def refmac(hklin, hklout, pdbin, pdbout, logfile, key):
                'hklout', hklout,
                'xyzin', pdbin,
                'xyzout', pdbout]
-        simbad_util.run_job(cmd, logfile=logfile, stdin=key)
+        simbad.util.simbad_util.run_job(cmd, logfile=logfile, stdin=key)
 

simbad/rotsearch/amore_search.py

@@ -22,6 +22,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 def _pickle_method(m):
     if m.im_self is None:
         return getattr, (m.im_class, m.im_func.func_name)
@@ -30,6 +31,7 @@ def _pickle_method(m):
 
 copy_reg.pickle(types.MethodType, _pickle_method)
 
+
 class _AmoreRotationScore(object):
     """An amore rotation scoring class"""
 
@@ -103,7 +105,7 @@ class AmoreRotationSearch(object):
     >>> rotation_search = AmoreRotationSearch('<amore_exe>', '<mtz>', '<work_dir>', '<max_to_keep>')
     >>> rotation_search.sortfun()
     >>> rotation_search.amore_run('<models_dir>', '<logs_dir>', '<nproc>', '<shres>', '<pklim>', '<npic>', '<rotastep>',
-    >>>                           '<min_solvent_content>')
+    ...                           '<min_solvent_content>')
     >>> rotation_search.summarize()
     >>> search_results = rotation_search.search_results
 
@@ -287,12 +289,13 @@ def run(job_queue, timeout=60):
                 model = job_queue.get(timeout=timeout)
                 self._amore_run(model, logs_dir, shres, pklim, npic, rotastep, cell_parameters, space_group,
                                 min_solvent_content)
-
+        
         for e in os.walk(models_dir):
             for model in e[2]:
                 relpath = os.path.relpath(models_dir)
                 job_queue.put(os.path.join(relpath, model))
 
+        logger.info("Running AMORE rotation function on %d structures", job_queue.qsize())
         processes = []
         for i in range(nproc):
             process = multiprocessing.Process(target=run, args=(job_queue,))
@@ -355,14 +358,13 @@ def _amore_run(self, model, logs_dir, shres, pklim, npic, rotastep, cell_paramet
         """Function to run tabfun and rotfun sequentially"""
 
         self.name = os.path.splitext(os.path.basename(model)[0:6])[0]
-
         if self.matthews_coef(model, cell_parameters, space_group, min_solvent_content):
             # Make output directory if it doesn't exist
             output_dir = os.path.join(self.work_dir, 'output')
             if not os.path.exists(output_dir):
                 os.mkdir(output_dir)
 
-            logger.info("Running AMORE rotation function on {0}".format(self.name))
+            logger.debug("Running AMORE rotation function on {0}".format(self.name))
 
             # Set up variables for the run
             x, y, z, intrad = AmoreRotationSearch.calculate_integration_box(model)
@@ -376,8 +378,7 @@ def _amore_run(self, model, logs_dir, shres, pklim, npic, rotastep, cell_paramet
         else:
             msg = "Skipping {0}: solvent content is predicted to be less than {1}".format(self.name,
                                                                                           min_solvent_content)
-            logger.info(msg)
-            return
+            logger.debug(msg)
 
         return
 
@@ -418,7 +419,6 @@ def matthews_coef(self, model, cell_parameters, space_group, min_solvent_content
             Can the model fit in the unit cell with a solvent content higher than the min_solvent_content
 
         """
-
         # Get the molecular weight of the input model
         molecular_weight = simbad_util.molecular_weight(model)
 
@@ -592,7 +592,7 @@ def tabfun(self, model, x=200, y=200, z=200):
 
         cmd = [self.amore_exe,
                'xyzin1', model,
-               'xyzout1', os.path.join(self.work_dir, 'output', '{0}_rot.pdb'.format(self.name)),
+               'xyzout1', os.path.join(self.work_dir, 'output', '{0}.pdb'.format(self.name)),
                'table1', os.path.join(self.work_dir, 'output', '{0}_sfs.tab'.format(self.name))]
 
         key = """TITLE: Produce table for MODEL FRAGMENT