Skip to content
This repository has been archived by the owner on Nov 3, 2023. It is now read-only.

Warning on unrecommended ACUTE-Eval value #3536

Merged
merged 3 commits into from Mar 19, 2021
Merged

Warning on unrecommended ACUTE-Eval value #3536

Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aa71156
Check max units per worker
EricMichaelSmith Mar 18, 2021
09f673a
Change warning
EricMichaelSmith Mar 18, 2021
964194c
Reword
EricMichaelSmith Mar 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 13 additions & 2 deletions parlai/crowdsourcing/tasks/acute_eval/acute_eval_blueprint.py
View file
Open in desktop
Expand Up @@ -9,16 +9,17 @@
from dataclasses import dataclass, field
from typing import ClassVar, List, Type, Any, Dict, Iterable, TYPE_CHECKING

from mephisto.operations.registry import register_mephisto_abstraction
from mephisto.abstractions.blueprint import Blueprint, BlueprintArgs, SharedTaskState
from mephisto.data_model.assignment import InitializationData
from mephisto.operations.registry import register_mephisto_abstraction
from omegaconf import MISSING, DictConfig

from parlai.crowdsourcing.tasks.acute_eval.acute_eval_agent_state import (
AcuteEvalAgentState,
)
from parlai.crowdsourcing.tasks.acute_eval.acute_eval_builder import AcuteEvalBuilder
from parlai.crowdsourcing.tasks.acute_eval.acute_eval_runner import AcuteEvalRunner
from parlai.utils.misc import warn_once

if TYPE_CHECKING:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is super cool!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@klshuster

from mephisto.abstractions.blueprint import AgentState, TaskRunner, TaskBuilder
Expand Down Expand Up @@ -96,7 +97,10 @@ class AcuteEvalBlueprint(Blueprint):
@classmethod
def assert_task_args(cls, args: DictConfig, shared_state: SharedTaskState) -> None:
"""
Ensure that the data can be properly loaded.
ACUTE-Eval-specific argument checking.

Ensure that the data can be properly loaded and that recommended worker settings
are used.
"""
if args.blueprint.get("pairings_filepath", None) is not None:
pairings_filepath = os.path.expanduser(args.blueprint.pairings_filepath)
Expand All @@ -114,6 +118,13 @@ def assert_task_args(cls, args: DictConfig, shared_state: SharedTaskState) -> No
"Must provide `block_qualification` to use `block_on_onboarding_fail`"
)

if args.task.get('maximum_units_per_worker', None) != 1:
warn_once(
'It is *strongly* recommended to use a '
'args.task.maximum_units_per_worker value of 1, as was done in the '
'ACUTE-Eval paper! Go to GitHub Discussions for more information.'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A slight change suggestion:

'It is *strongly* recommended to use a '
'args.task.maximum_units_per_worker value of 1, as was done in the '
'ACUTE-Eval paper! Anything else will not provide consistent results as '
'quoted there. Go to ParlAI's GitHub Discussion page '
'if you'd like to discuss further.

I think the current wording implies we already have a discussion on it there.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, changed

)

def get_frontend_args(self) -> Dict[str, Any]:
"""
Specifies what options within a task_config should be forwarded to the client
Expand Down