also dropping sentences now!

SmokinCaterpillar · Feb 20, 2018 · 814a9d3 · 814a9d3
1 parent 615a1aa
commit 814a9d3
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/trufflepig/preprocessing.py b/trufflepig/preprocessing.py
@@ -221,6 +221,10 @@ def preprocess(post_df, ncores=4, chunksize=500,
                                                  post_df.filtered_sentences,
                                                  ncores=ncores,
                                                  chunksize=chunksize)
+    post_df.drop('filtered_sentences', axis=1, inplace=True)
+    logger.info('Intermediate garbage collection.')
+    gc.collect()
+
     to_drop = post_df.loc[post_df.grammar_errors_per_sentence > max_grammar_errors_per_sentence]
     post_df.drop(to_drop.index, inplace=True)
     logger.info('Filtered according to grammar mistake limit {} per sentence '