From d00f7d3db239866b34019815c338f00456616eeb Mon Sep 17 00:00:00 2001 From: Mojtaba <11262163+mojtaba-komeili@users.noreply.github.com> Date: Wed, 15 Dec 2021 12:22:21 -0500 Subject: [PATCH] _load_data as a generator (#4253) --- parlai/tasks/wizard_of_internet/agents.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/parlai/tasks/wizard_of_internet/agents.py b/parlai/tasks/wizard_of_internet/agents.py index e3ea06b504c..3cf8c5e2aa5 100644 --- a/parlai/tasks/wizard_of_internet/agents.py +++ b/parlai/tasks/wizard_of_internet/agents.py @@ -183,11 +183,9 @@ def add_cmdline_args(cls, parser: ParlaiParser, partial_opt=None) -> ParlaiParse def _load_data(self, datafile): logging.info(f'Loading data from {datafile} ...') - dialogs = [] with jsonlines.open(datafile, 'r') as fin: for dialog_json in tqdm(fin): - dialogs.append(self._get_episode_examples(dialog_json)) - return dialogs + yield self._get_episode_examples(dialog_json) def _get_episode_examples(self, dialog_json): data = get_single_val_from_dict(dialog_json)