Merge pull request #146 from indralab/remove_multiproc

Remove multiprocessing when calling Preassembler

indra_db/preassembly/preassemble_db.py

@@ -61,17 +61,13 @@ class DbPreassembler:
 
     Parameters
     ----------
-    n_proc : int
-        Select the number of processes that will be used when performing
-        preassembly. Default is 1.
     batch_size : int
         Select the maximum number of statements you wish to be handled at a
         time. In general, a larger batch size will somewhat be faster, but
         require much more memory.
     """
-    def __init__(self, n_proc=None, batch_size=10000, s3_cache=None,
+    def __init__(self, batch_size=10000, s3_cache=None,
                  print_logs=False, stmt_type=None, yes_all=False):
-        self.n_proc = n_proc
         self.batch_size = batch_size
         if s3_cache is not None:
             # Make the cache specific to stmt type. This guards against
@@ -668,8 +664,7 @@ def _clean_statements(self, stmts):
     @clockit
     def _get_support_links(self, unique_stmts, split_idx=None):
         """Find the links of refinement/support between statements."""
-        id_maps = self.pa._generate_id_maps(unique_stmts, poolsize=self.n_proc,
-                                            split_idx=split_idx)
+        id_maps = self.pa._generate_id_maps(unique_stmts, split_idx=split_idx)
         ret = set()
         for ix_pair in id_maps:
             if ix_pair[0] == ix_pair[1]:
@@ -732,13 +727,6 @@ def _make_parser():
         action='store_true',
         help='Continue uploading or updating, picking up where you left off.'
     )
-    parser.add_argument(
-        '-n', '--num-procs',
-        type=int,
-        default=None,
-        help=('Select the number of processors to use during this operation. '
-              'Default is 1.')
-    )
     parser.add_argument(
         '-b', '--batch',
         type=int,
@@ -793,7 +781,7 @@ def _main():
     assert db is not None
     db.grab_session()
     s3_cache = S3Path.from_string(args.cache)
-    pa = DbPreassembler(args.num_procs, args.batch, s3_cache,
+    pa = DbPreassembler(args.batch, s3_cache,
                         stmt_type=args.stmt_type, yes_all=args.yes_all)
 
     desc = 'Continuing' if args.continuing else 'Beginning'

indra_db/tests/test_preassembly.py

@@ -358,7 +358,7 @@ def _check_statement_distillation(num_stmts):
 
 
 @needs_py3
-def _check_preassembly_with_database(num_stmts, batch_size, n_proc=1):
+def _check_preassembly_with_database(num_stmts, batch_size):
     db = get_pa_loaded_db(num_stmts)
 
     # Now test the set of preassembled (pa) statements from the database
@@ -372,7 +372,7 @@ def _check_preassembly_with_database(num_stmts, batch_size, n_proc=1):
 
     # Run the preassembly initialization.
     start = datetime.now()
-    pa_manager = pm.PreassemblyManager(batch_size=batch_size, n_proc=n_proc,
+    pa_manager = pm.PreassemblyManager(batch_size=batch_size,
                                        print_logs=True)
     pa_manager.create_corpus(db)
     end = datetime.now()
@@ -416,8 +416,8 @@ def _check_preassembly_with_database(num_stmts, batch_size, n_proc=1):
 
 
 @needs_py3
-def _check_db_pa_supplement(num_stmts, batch_size, split=0.8, n_proc=1):
-    pa_manager = pm.PreassemblyManager(batch_size=batch_size, n_proc=n_proc,
+def _check_db_pa_supplement(num_stmts, batch_size, split=0.8):
+    pa_manager = pm.PreassemblyManager(batch_size=batch_size,
                                        print_logs=True)
     db = get_pa_loaded_db(num_stmts, split=split, pam=pa_manager)
     opa_inp_stmts = _get_opa_input_stmts(db)
@@ -638,9 +638,9 @@ def test_db_incremental_preassembly_small():
 
 # @attr('nonpublic', 'slow')
 # def test_db_incremental_preassembly_very_large():
-#     _check_db_pa_supplement(100000, 20000, n_proc=2)
+#     _check_db_pa_supplement(100000, 20000)
 
 
 # @attr('nonpublic', 'slow')
 # def test_db_incremental_preassembly_1M():
-#     _check_db_pa_supplement(1000000, 200000, n_proc=6)
+#     _check_db_pa_supplement(1000000, 200000)