From 9e1e635c3d81655d5fbd4474971ef98272e5de51 Mon Sep 17 00:00:00 2001 From: nlpzhezhao Date: Mon, 2 Oct 2023 00:59:54 +0800 Subject: [PATCH] change corpora and datasets --- .github/workflows/github-actions.yml | 12 ++++++------ .../test_data/{douban_test => book_review}/dev.tsv | 0 .../test_data/{douban_test => book_review}/test.tsv | 0 .../{douban_test => book_review}/test_nolabel.tsv | 0 .../test_data/{douban_test => book_review}/train.tsv | 0 .../{chnsenticorp_test => chnsenticorp}/dev.tsv | 0 .../{chnsenticorp_test => chnsenticorp}/test.tsv | 0 .../test_nolabel.tsv | 0 .../{chnsenticorp_test => chnsenticorp}/train.tsv | 0 datasets/test_data/{cmrc_test => cmrc}/dev.json | 0 datasets/test_data/{cmrc_test => cmrc}/test.json | 0 datasets/test_data/{cmrc_test => cmrc}/train.json | 0 .../test_data/{msra_ner_test => msra_ner}/dev.tsv | 0 .../test_data/{msra_ner_test => msra_ner}/test.tsv | 0 .../{msra_ner_test => msra_ner}/test_nolabel.tsv | 0 .../test_data/{msra_ner_test => msra_ner}/train.tsv | 0 16 files changed, 6 insertions(+), 6 deletions(-) rename datasets/test_data/{douban_test => book_review}/dev.tsv (100%) rename datasets/test_data/{douban_test => book_review}/test.tsv (100%) rename datasets/test_data/{douban_test => book_review}/test_nolabel.tsv (100%) rename datasets/test_data/{douban_test => book_review}/train.tsv (100%) rename datasets/test_data/{chnsenticorp_test => chnsenticorp}/dev.tsv (100%) rename datasets/test_data/{chnsenticorp_test => chnsenticorp}/test.tsv (100%) rename datasets/test_data/{chnsenticorp_test => chnsenticorp}/test_nolabel.tsv (100%) rename datasets/test_data/{chnsenticorp_test => chnsenticorp}/train.tsv (100%) rename datasets/test_data/{cmrc_test => cmrc}/dev.json (100%) rename datasets/test_data/{cmrc_test => cmrc}/test.json (100%) rename datasets/test_data/{cmrc_test => cmrc}/train.json (100%) rename datasets/test_data/{msra_ner_test => msra_ner}/dev.tsv (100%) rename datasets/test_data/{msra_ner_test => msra_ner}/test.tsv (100%) rename datasets/test_data/{msra_ner_test => msra_ner}/test_nolabel.tsv (100%) rename datasets/test_data/{msra_ner_test => msra_ner}/train.tsv (100%) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index d3689c5f..ed05d92c 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -47,9 +47,9 @@ jobs: mv models/pegasus_model.bin-10 models/pegasus_model.bin python finetune/run_classifier.py --pretrained_model_path models/bert_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --output_model_path models/classifier_model.bin --train_path datasets/test_data/book_review/train.tsv --dev_path datasets/test_data/book_review/dev.tsv --epochs_num 3 --batch_size 2 python inference/run_classifier_infer.py --load_model_path models/classifier_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --test_path datasets/test_data/book_review/test_nolabel.tsv --prediction_path datasets/test_data/book_review/prediction.tsv --labels_num 2 - python finetune/run_classifier.py --pretrained_model_path models/albert_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/albert/base_config.json --output_model_path models/classifier_model.bin --train_path datasets/test_data/chnsenticorp_test/train.tsv --dev_path datasets/test_data/chnsenticorp_test/dev.tsv --learning_rate 4e-5 --epochs_num 3 --batch_size 2 - python finetune/run_classifier_mt.py --pretrained_model_path models/bert_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --dataset_path_list datasets/test_data/douban_test/ datasets/test_data/chnsenticorp_test/ --epochs_num 1 --batch_size 2 - python finetune/run_ner.py --pretrained_model_path models/bert_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --output_model_path models/ner_model.bin --train_path datasets/test_data/msra_ner_test/train.tsv --dev_path datasets/test_data/msra_ner_test/dev.tsv --label2id_path datasets/msra_ner/label2id.json --epochs_num 2 --batch_size 2 - python inference/run_ner_infer.py --load_model_path models/ner_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --test_path datasets/test_data/msra_ner_test/test_nolabel.tsv --prediction_path datasets/test_data/msra_ner_test/prediction.tsv --label2id_path datasets/msra_ner/label2id.json - python finetune/run_cmrc.py --pretrained_model_path models/bert_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --output_model_path models/cmrc_model.bin --train_path datasets/test_data/cmrc_test/train.json --dev_path datasets/test_data/cmrc_test/dev.json --epochs_num 2 --batch_size 2 --seq_length 128 - python inference/run_cmrc_infer.py --load_model_path models/cmrc_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --test_path datasets/test_data/cmrc_test/test.json --prediction_path datasets/test_data/cmrc_test/prediction.json --seq_length 128 + python finetune/run_classifier.py --pretrained_model_path models/albert_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/albert/base_config.json --output_model_path models/classifier_model.bin --train_path datasets/test_data/chnsenticorp/train.tsv --dev_path datasets/test_data/chnsenticorp/dev.tsv --learning_rate 4e-5 --epochs_num 3 --batch_size 2 + python finetune/run_classifier_mt.py --pretrained_model_path models/bert_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --dataset_path_list datasets/test_data/book_review/ datasets/test_data/chnsenticorp/ --epochs_num 1 --batch_size 2 + python finetune/run_ner.py --pretrained_model_path models/bert_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --output_model_path models/ner_model.bin --train_path datasets/test_data/msra_ner/train.tsv --dev_path datasets/test_data/msra_ner/dev.tsv --label2id_path datasets/msra_ner/label2id.json --epochs_num 2 --batch_size 2 + python inference/run_ner_infer.py --load_model_path models/ner_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --test_path datasets/test_data/msra_ner/test_nolabel.tsv --prediction_path datasets/test_data/msra_ner/prediction.tsv --label2id_path datasets/msra_ner/label2id.json + python finetune/run_cmrc.py --pretrained_model_path models/bert_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --output_model_path models/cmrc_model.bin --train_path datasets/test_data/cmrc/train.json --dev_path datasets/test_data/cmrc/dev.json --epochs_num 2 --batch_size 2 --seq_length 128 + python inference/run_cmrc_infer.py --load_model_path models/cmrc_model.bin --vocab_path models/google_zh_vocab.txt --config_path models/bert/mini_config.json --test_path datasets/test_data/cmrc/test.json --prediction_path datasets/test_data/cmrc/prediction.json --seq_length 128 diff --git a/datasets/test_data/douban_test/dev.tsv b/datasets/test_data/book_review/dev.tsv similarity index 100% rename from datasets/test_data/douban_test/dev.tsv rename to datasets/test_data/book_review/dev.tsv diff --git a/datasets/test_data/douban_test/test.tsv b/datasets/test_data/book_review/test.tsv similarity index 100% rename from datasets/test_data/douban_test/test.tsv rename to datasets/test_data/book_review/test.tsv diff --git a/datasets/test_data/douban_test/test_nolabel.tsv b/datasets/test_data/book_review/test_nolabel.tsv similarity index 100% rename from datasets/test_data/douban_test/test_nolabel.tsv rename to datasets/test_data/book_review/test_nolabel.tsv diff --git a/datasets/test_data/douban_test/train.tsv b/datasets/test_data/book_review/train.tsv similarity index 100% rename from datasets/test_data/douban_test/train.tsv rename to datasets/test_data/book_review/train.tsv diff --git a/datasets/test_data/chnsenticorp_test/dev.tsv b/datasets/test_data/chnsenticorp/dev.tsv similarity index 100% rename from datasets/test_data/chnsenticorp_test/dev.tsv rename to datasets/test_data/chnsenticorp/dev.tsv diff --git a/datasets/test_data/chnsenticorp_test/test.tsv b/datasets/test_data/chnsenticorp/test.tsv similarity index 100% rename from datasets/test_data/chnsenticorp_test/test.tsv rename to datasets/test_data/chnsenticorp/test.tsv diff --git a/datasets/test_data/chnsenticorp_test/test_nolabel.tsv b/datasets/test_data/chnsenticorp/test_nolabel.tsv similarity index 100% rename from datasets/test_data/chnsenticorp_test/test_nolabel.tsv rename to datasets/test_data/chnsenticorp/test_nolabel.tsv diff --git a/datasets/test_data/chnsenticorp_test/train.tsv b/datasets/test_data/chnsenticorp/train.tsv similarity index 100% rename from datasets/test_data/chnsenticorp_test/train.tsv rename to datasets/test_data/chnsenticorp/train.tsv diff --git a/datasets/test_data/cmrc_test/dev.json b/datasets/test_data/cmrc/dev.json similarity index 100% rename from datasets/test_data/cmrc_test/dev.json rename to datasets/test_data/cmrc/dev.json diff --git a/datasets/test_data/cmrc_test/test.json b/datasets/test_data/cmrc/test.json similarity index 100% rename from datasets/test_data/cmrc_test/test.json rename to datasets/test_data/cmrc/test.json diff --git a/datasets/test_data/cmrc_test/train.json b/datasets/test_data/cmrc/train.json similarity index 100% rename from datasets/test_data/cmrc_test/train.json rename to datasets/test_data/cmrc/train.json diff --git a/datasets/test_data/msra_ner_test/dev.tsv b/datasets/test_data/msra_ner/dev.tsv similarity index 100% rename from datasets/test_data/msra_ner_test/dev.tsv rename to datasets/test_data/msra_ner/dev.tsv diff --git a/datasets/test_data/msra_ner_test/test.tsv b/datasets/test_data/msra_ner/test.tsv similarity index 100% rename from datasets/test_data/msra_ner_test/test.tsv rename to datasets/test_data/msra_ner/test.tsv diff --git a/datasets/test_data/msra_ner_test/test_nolabel.tsv b/datasets/test_data/msra_ner/test_nolabel.tsv similarity index 100% rename from datasets/test_data/msra_ner_test/test_nolabel.tsv rename to datasets/test_data/msra_ner/test_nolabel.tsv diff --git a/datasets/test_data/msra_ner_test/train.tsv b/datasets/test_data/msra_ner/train.tsv similarity index 100% rename from datasets/test_data/msra_ner_test/train.tsv rename to datasets/test_data/msra_ner/train.tsv