hack

SmokinCaterpillar · Feb 25, 2018 · 208e887 · 208e887
1 parent b913bbf
commit 208e887
Showing 1 changed file with 10 additions and 8 deletions.
diff --git a/trufflepig/main.py b/trufflepig/main.py
@@ -41,8 +41,13 @@ def configure_logging(directory, current_datetime):
                         handlers=handlers)
 
 
-def large_mp_preprocess(post_frame, directory, current_datetime):
+def large_mp_preprocess(directory, current_datetime, steem_kwargs, data_directory):
     configure_logging(directory, current_datetime)
+    post_frame = tpgd.load_or_scrape_training_data(steem_kwargs, data_directory,
+                                                       current_datetime=current_datetime,
+                                                       days=10,
+                                                       offset_days=8,
+                                                       ncores=16)
     return tppp.preprocess(post_frame, ncores=3)
 
 
@@ -73,17 +78,14 @@ def main():
     tppd.create_wallet(steem_kwargs, config.PASSWORD, config.POSTING_KEY)
 
     if not tpmo.model_exists(current_datetime, model_directoy):
-        post_frame = tpgd.load_or_scrape_training_data(steem_kwargs, data_directory,
-                                                       current_datetime=current_datetime,
-                                                       days=10,
-                                                       offset_days=8,
-                                                       ncores=16)
         # hack for better memory footprint,
         # see https://stackoverflow.com/questions/15455048/releasing-memory-in-python
         with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor:
             post_frame = executor.submit(large_mp_preprocess,
-                                         post_frame, log_directory,
-                                         current_datetime).result()
+                                         log_directory,
+                                         current_datetime,
+                                         steem_kwargs,
+                                         data_directory).result()
         logger.info('Garbage collecting')
         gc.collect()
     else: