Merge pull request #94 from QData/fd-bug

Fix for file descriptor leak bug

textattack/constraints/semantics/word_embedding_distance.py

@@ -55,11 +55,13 @@ def __init__(self, embedding_type='paragramcf', include_unknown_words=True,
         # Precomputed distance matrices store distances at mat[x][y], where
         # x and y are word IDs and x < y.
         if self.max_mse_dist is not None and os.path.exists(mse_dist_file):
-            self.mse_dist_mat = pickle.load(open(mse_dist_file, 'rb'))
+            with open(mse_dist_file, 'rb') as f:
+                self.mse_dist_mat = pickle.load(f)
         else:
             self.mse_dist_mat = {}
         if self.min_cos_sim is not None and os.path.exists(cos_sim_file):
-            self.cos_sim_mat = pickle.load(open(cos_sim_file, 'rb'))
+            with open(cos_sim_file, 'rb') as f:
+                self.cos_sim_mat = pickle.load(f)
         else:
             self.cos_sim_mat = {}
 

textattack/datasets/dataset.py

@@ -38,7 +38,8 @@ def __next__(self):
     def _load_pickle_file(self, file_name, offset=0):
         self.i = 0
         file_path = utils.download_if_needed(file_name)
-        self.examples = pickle.load( open(file_path, "rb" ) )
+        with open(file_path, "rb") as f:
+            self.examples = pickle.load(f)
         self.examples = self.examples[offset:]
 
     def _load_classification_text_file(self, text_file_name, offset=0):

textattack/shared/scripts/mturk_prep/se_thresh/gather_csv.py

@@ -13,7 +13,8 @@ def main():
         if 'final.txt' not in run_files:
             continue
         args_path = os.path.join(run_path,'args.txt')
-        args_lines = open(args_path, 'r').readlines()
+        with open(args_path, 'r') as f:
+            args_lines = f.readlines()
         se_thresh_idx = args_lines[0].find('tf-adjusted') + len('tf-adjusted:')
         se_thresh = args_lines[0][se_thresh_idx:]
         se_thresh = se_thresh[:se_thresh.index(' ')]

textattack/shared/scripts/run_attack_parallel.py

@@ -11,6 +11,8 @@
 from .run_attack_args_helper import *
 
 def set_env_variables(gpu_id):
+    # Set sharing strategy to file_system to avoid file descriptor leaks
+    torch.multiprocessing.set_sharing_strategy('file_system')
     # Only use one GPU, if we have one.
     if 'CUDA_VISIBLE_DEVICES' not in os.environ:
         os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
@@ -28,7 +30,7 @@ def attack_from_queue(args, in_queue, out_queue):
     if gpu_id == 0:
         print(attack, '\n')
     while not in_queue.empty():
-        try: 
+        try:
             output, text = in_queue.get()
             results_gen = attack.attack_dataset([(output, text)], num_examples=1)
             result = next(results_gen)
@@ -107,6 +109,7 @@ def pytorch_multiprocessing_workaround():
     # This is a fix for a known bug
     try:
         torch.multiprocessing.set_start_method('spawn')
+        torch.multiprocessing.set_sharing_strategy('file_system')
     except RuntimeError:
         pass
 

textattack/shared/utils.py

@@ -264,5 +264,6 @@ def config(key):
 
 config_dict = {'CACHE_DIR': os.environ.get('TA_CACHE_DIR', os.path.expanduser('~/.cache/textattack'))}
 config_path = download_if_needed('config.yaml')
-config_dict.update(yaml.load(open(config_path, 'r'), Loader=yaml.FullLoader))
+with open(config_path, 'r') as f:
+    config_dict.update(yaml.load(f, Loader=yaml.FullLoader))
 _post_install_if_needed()

textattack/shared/word_embedding.py

@@ -39,11 +39,13 @@ def __init__(self, embedding_type='paragramcf'):
         # Precomputed distance matrices store distances at mat[x][y], where
         # x and y are word IDs and x < y.
         if os.path.exists(mse_dist_file):
-            self.mse_dist_mat = pickle.load(open(mse_dist_file, 'rb'))
+            with open(mse_dist_file, 'rb') as f:
+                self.mse_dist_mat = pickle.load(f)
         else:
             self.mse_dist_mat = {}
         if os.path.exists(cos_sim_file):
-            self.cos_sim_mat = pickle.load(open(cos_sim_file, 'rb'))
+            with open(cos_sim_file, 'rb') as f:
+                self.cos_sim_mat = pickle.load(f)
         else:
             self.cos_sim_mat = {}