Back to using normal processes

tools/preprocess_data.py

@@ -276,9 +276,9 @@ def main():
     readers, writers = list(zip(*[multiprocessing.Pipe(duplex=False) for _ in range(args.workers - 2)]))
     processes = [multiprocessing.Process(target=process_samples, args=(simple_queue, i, args, level, writer)) for i, writer in enumerate(writers)]
     log_thread = multiprocessing.Process(target=log, args=(list(readers), args.log_interval))
-    # fill_thread = multiprocessing.Process(target=fill_simple_queue, args=(args.input, simple_queue, chunk_size))
+    fill_thread = multiprocessing.Process(target=fill_simple_queue, args=(args.input, simple_queue, chunk_size))
 
-    # fill_thread.start()
+    fill_thread.start()
     log_thread.start()
     for i, process in enumerate(processes):
         process.start()
@@ -291,10 +291,10 @@ def main():
     for writer in writers:
         writer.close()
 
-    fill_simple_queue(args.input, simple_queue, chunk_size)
+    # fill_simple_queue(args.input, simple_queue, chunk_size)
 
-    # fill_thread.join()
-    # fill_thread.close()
+    fill_thread.join()
+    fill_thread.close()
     for process in processes:
         process.join()
         process.close()