Merge pull request #196 from Billccx/main

About validation_file_dir
shibing624 · Sep 5, 2023 · 0f446e0 · 0f446e0
2 parents f1b8afd + bfd1f5b
commit 0f446e0
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/pretraining.py b/pretraining.py
@@ -458,8 +458,8 @@ def group_texts(examples):
             data_files["train"] = train_data_files
         if data_args.validation_file_dir is not None and os.path.exists(data_args.validation_file_dir):
             eval_data_files = glob(f'{data_args.validation_file_dir}/**/*.txt', recursive=True) + glob(
-                f'{data_args.train_file_dir}/**/*.json', recursive=True) + glob(
-                f'{data_args.train_file_dir}/**/*.jsonl', recursive=True)
+                f'{data_args.validation_file_dir}/**/*.json', recursive=True) + glob(
+                f'{data_args.validation_file_dir}/**/*.jsonl', recursive=True)
             logger.info(f"eval files: {eval_data_files}")
             data_files["validation"] = eval_data_files
             # Train data files must be same type, e.g. all txt or all jsonl