Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[AutoNLP] Add english models for text classification #4704

Merged
merged 2 commits into from
Feb 9, 2023
Merged

[AutoNLP] Add english models for text classification #4704

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c0eed44
add english models
sijunhe Feb 8, 2023
41d483a
add tests
sijunhe Feb 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions paddlenlp/experimental/autonlp/auto_trainer_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,21 @@ def __init__(
self.train_dataset = train_dataset
self.eval_dataset = eval_dataset
self.greater_is_better = greater_is_better
if language not in self.supported_language:
raise ValueError(
sijunhe marked this conversation as resolved.
Show resolved Hide resolved
f"'{language}' is not supported. Please choose among the following: {self.supported_language}"
)

self.language = language
self.output_dir = output_dir

@property
@abstractmethod
def supported_language(self) -> List[str]:
"""
Override to store the supported languages for each auto trainer class
"""

@property
@abstractmethod
def _default_training_argument(self) -> TrainingArguments:
Expand Down
40 changes: 39 additions & 1 deletion paddlenlp/experimental/autonlp/text_classification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@ def __init__(
f"'{problem_type}' is not a supported problem_type. Please select among ['multi_label', 'multi_class']"
)

@property
def supported_language(self) -> List[str]:
return ["Chinese", "English"]

@property
def _default_training_argument(self) -> TrainingArguments:
return TrainingArguments(
Expand Down Expand Up @@ -129,19 +133,42 @@ def _model_candidates(self) -> List[Dict[str, Any]]:
"ernie-3.0-nano-zh", # 4-layer, 312-hidden, 12-heads, 18M parameters.
],
)
english_models = hp.choice(
Copy link
Contributor

@lugimzzz lugimzzz Feb 9, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

添加英文模型的单测

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

英文模型和中文模型逻辑一致，所以我没有添加新的测试。不过增加了test_model_language_filter来确保选英文能拿到英文模型的candidates

"models",
[
# add deberta-v3 when we have it
"roberta-large", # 24-layer, 1024-hidden, 16-heads, 334M parameters. Case-sensitive
"roberta-base", # 12-layer, 768-hidden, 12-heads, 110M parameters. Case-sensitive
"distilroberta-base", # 6-layer, 768-hidden, 12-heads, 66M parameters. Case-sensitive
"ernie-2.0-base-en", # 12-layer, 768-hidden, 12-heads, 103M parameters. Trained on lower-cased English text.
"ernie-2.0-large-en", # 24-layer, 1024-hidden, 16-heads, 336M parameters. Trained on lower-cased English text.
"distilbert-base-uncased", # 6-layer, 768-hidden, 12-heads, 66M parameters
],
)
return [
# fast learning: high LR, small early stop patience
{
"preset": "finetune",
"language": "Chinese",
"trainer_type": "Trainer",
"EarlyStoppingCallback.early_stopping_patience": 2,
"EarlyStoppingCallback.early_stopping_patience": 5,
"TrainingArguments.per_device_train_batch_size": train_batch_size,
"TrainingArguments.per_device_eval_batch_size": train_batch_size * 2,
"TrainingArguments.num_train_epochs": 100,
"TrainingArguments.model_name_or_path": chinese_models,
"TrainingArguments.learning_rate": 3e-5,
},
{
"preset": "finetune",
"language": "English",
"trainer_type": "Trainer",
"EarlyStoppingCallback.early_stopping_patience": 5,
"TrainingArguments.per_device_train_batch_size": train_batch_size,
"TrainingArguments.per_device_eval_batch_size": train_batch_size * 2,
"TrainingArguments.num_train_epochs": 100,
"TrainingArguments.model_name_or_path": english_models,
"TrainingArguments.learning_rate": 3e-5,
},
# slow learning: small LR, large early stop patience
{
"preset": "finetune",
Expand All @@ -154,6 +181,17 @@ def _model_candidates(self) -> List[Dict[str, Any]]:
"TrainingArguments.model_name_or_path": chinese_models,
"TrainingArguments.learning_rate": 5e-6,
},
{
"preset": "finetune",
"language": "English",
"trainer_type": "Trainer",
"EarlyStoppingCallback.early_stopping_patience": 5,
"TrainingArguments.per_device_train_batch_size": train_batch_size,
"TrainingArguments.per_device_eval_batch_size": train_batch_size * 2,
"TrainingArguments.num_train_epochs": 100,
"TrainingArguments.model_name_or_path": english_models,
"TrainingArguments.learning_rate": 5e-6,
},
# Note: prompt tuning candidates not included for now due to lack of inference capability
]

Expand Down