fix double remove issue. (#168)

Signed-off-by: Lv, Kaokao <kaokao.lv@intel.com>
intel · Aug 23, 2023 · 67ce6e8 · 67ce6e8
1 parent 81f96be
commit 67ce6e8
Showing 1 changed file with 1 addition and 4 deletions.
diff --git a/workflows/chatbot/fine_tuning/instruction_tuning_pipeline/finetune_clm.py b/workflows/chatbot/fine_tuning/instruction_tuning_pipeline/finetune_clm.py
@@ -575,11 +575,8 @@ def concatenate_data(dataset, max_seq_length):
                 concatenated_dataset[column] = reshaped_data
             return datasets.Dataset.from_dict(concatenated_dataset)
 
-        tokenized_datasets_ = tokenized_datasets["train"].remove_columns(
-            ["prompt_sources", "prompt_targets"]
-        )
         tokenized_datasets["train"] = concatenate_data(
-            tokenized_datasets_, data_args.max_seq_length
+            tokenized_datasets["train"], data_args.max_seq_length
         )
 
     if training_args.do_eval: