Skip to content

Commit

Permalink
Using DataLoader in a subprocess. Fixed freezing.
Browse files Browse the repository at this point in the history
  • Loading branch information
Enolerobotti authored and swansonk14 committed Sep 10, 2020
1 parent 298527a commit d6b3209
Showing 1 changed file with 10 additions and 1 deletion.
11 changes: 10 additions & 1 deletion chemprop/data/data.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import threading
from collections import OrderedDict
from functools import partial
from random import Random
Expand Down Expand Up @@ -384,6 +385,12 @@ def __init__(self,
self._class_balance = class_balance
self._shuffle = shuffle
self._seed = seed
self._context = None
self._timeout = 0
is_main_thread = threading.current_thread() is threading.main_thread()
if not is_main_thread and self._num_workers > 0:
self._context = 'forkserver' # In order to prevent a hanging
self._timeout = 3600 # Just for sure that the DataLoader won't hang

self._sampler = MoleculeSampler(
dataset=self._dataset,
Expand All @@ -397,7 +404,9 @@ def __init__(self,
batch_size=self._batch_size,
sampler=self._sampler,
num_workers=self._num_workers,
collate_fn=partial(construct_molecule_batch, cache=self._cache)
collate_fn=partial(construct_molecule_batch, cache=self._cache),
multiprocessing_context=self._context,
timeout=self._timeout
)

@property
Expand Down

0 comments on commit d6b3209

Please sign in to comment.