Skip to content

Commit

Permalink
Add error message to train/dev split fn (#190)
Browse files Browse the repository at this point in the history
  • Loading branch information
brandenchan authored Jan 16, 2020
1 parent ad115c2 commit 6d7a7b8
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion farm/data_handler/data_silo.py
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,14 @@ def random_split_ConcatDataset(self, ds, lengths):
if sum(lengths) != len(ds):
raise ValueError("Sum of input lengths does not equal the length of the input dataset!")

idx_dataset = np.where(np.array(ds.cumulative_sizes) > lengths[0])[0][0]
try:
idx_dataset = np.where(np.array(ds.cumulative_sizes) > lengths[0])[0][0]
except IndexError:
raise Exception("All dataset chunks are being assigned to train set leaving no samples for dev set. "
"Either consider increasing dev_split or setting it to 0.0\n"
f"Cumulative chunk sizes: {ds.cumulative_sizes}\n"
f"train/dev split: {lengths}")

assert idx_dataset >= 1, "Dev_split ratio is too large, there is no data in train set. " \
f"Please lower dev_split = {self.processor.dev_split}"

Expand Down

0 comments on commit 6d7a7b8

Please sign in to comment.