From e686a02ab745388265ca653983321a90a58d68d4 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Tue, 4 Jan 2022 09:45:46 -0500 Subject: [PATCH] Pull out settings for model chat onboarding (#4274) * Layer changes * Documentation --- .../crowdsourcing/tasks/model_chat/README.md | 4 +- .../tasks/model_chat/constants.py | 2 - .../components/onboarding_components.jsx | 16 +++--- .../task_config/onboard_task_data.json | 5 +- .../crowdsourcing/tasks/model_chat/worlds.py | 57 +++---------------- 5 files changed, 24 insertions(+), 60 deletions(-) diff --git a/parlai/crowdsourcing/tasks/model_chat/README.md b/parlai/crowdsourcing/tasks/model_chat/README.md index 598884d42ea..75e31c035c0 100644 --- a/parlai/crowdsourcing/tasks/model_chat/README.md +++ b/parlai/crowdsourcing/tasks/model_chat/README.md @@ -24,7 +24,9 @@ The following flags can be passed in to specify filepaths for overriding the tex ## Onboarding -In `worlds.py`, modify `ModelChatOnboardWorld.check_onboarding_answers()` to change the worker selection criteria. +Set the `"min_correct"`, `"max_incorrect"`, and `"max_failures_allowed"` fields in the JSON file passed to `mephisto.blueprint.onboard_task_data_path` in order to specify how many onboarding questions workers can pass/fail on while still passing onboarding, as well as how many times they are allowed to re-take the onboarding before being soft-blocked. (See `task_config/onboard_task_data.json` for an example.) + +You can further modify the worker selection criteria in `handleOnboardingSubmit` in `frontend/components/onboarding_components.jsx`. ## Human+model image chat diff --git a/parlai/crowdsourcing/tasks/model_chat/constants.py b/parlai/crowdsourcing/tasks/model_chat/constants.py index eb5861e6dd0..86348a579ba 100644 --- a/parlai/crowdsourcing/tasks/model_chat/constants.py +++ b/parlai/crowdsourcing/tasks/model_chat/constants.py @@ -10,5 +10,3 @@ ONBOARD_FAIL = '[ONBOARD_FAIL]' ONBOARD_SUCCESS = '[ONBOARD_SUCCESS]' - -ONBOARD_CONFIG = {'min_correct': 4, 'max_incorrect': 3} diff --git a/parlai/crowdsourcing/tasks/model_chat/frontend/components/onboarding_components.jsx b/parlai/crowdsourcing/tasks/model_chat/frontend/components/onboarding_components.jsx index f197bcf33b5..ac2b0606f6e 100644 --- a/parlai/crowdsourcing/tasks/model_chat/frontend/components/onboarding_components.jsx +++ b/parlai/crowdsourcing/tasks/model_chat/frontend/components/onboarding_components.jsx @@ -9,9 +9,9 @@ import React from "react"; import { ErrorBoundary } from './error_boundary.jsx'; import { Checkboxes } from './checkboxes.jsx'; -const ONBOARDING_MIN_CORRECT = 4; -const ONBOARDING_MAX_INCORRECT = 3; -const ONBOARDING_MAX_FAILURES_ALLOWED = 1; +const DEFAULT_MIN_CORRECT = 4; +const DEFAULT_MAX_INCORRECT = 3; +const DEFAULT_MAX_FAILURES_ALLOWED = 1; var onboardingFailuresCount = 0; var renderOnboardingFail = function () { @@ -35,7 +35,6 @@ function arraysEqual(_arr1, _arr2) { } var handleOnboardingSubmit = function ({ onboardingData, currentTurnAnnotations, onSubmit }) { - // OVERRIDE: Re-implement this to change onboarding success criteria console.log('handleOnboardingSubmit'); var countCorrect = 0; var countIncorrect = 0; @@ -60,10 +59,13 @@ var handleOnboardingSubmit = function ({ onboardingData, currentTurnAnnotations, } } console.log('correct: ' + countCorrect + ', incorrect: ' + countIncorrect); - if (countCorrect >= ONBOARDING_MIN_CORRECT && countIncorrect <= ONBOARDING_MAX_INCORRECT) { + const min_correct = onboardingData.hasOwnProperty("min_correct") ? onboardingData.min_correct : DEFAULT_MIN_CORRECT; + const max_incorrect = onboardingData.hasOwnProperty("max_incorrect") ? onboardingData.max_incorrect : DEFAULT_MAX_INCORRECT; + const max_failures_allowed = onboardingData.hasOwnProperty("max_failures_allowed") ? onboardingData.max_failures_allowed : DEFAULT_MAX_FAILURES_ALLOWED; + if (countCorrect >= min_correct && countIncorrect <= max_incorrect) { onSubmit({ annotations: currentTurnAnnotations, success: true }); } else { - if (onboardingFailuresCount < ONBOARDING_MAX_FAILURES_ALLOWED) { + if (onboardingFailuresCount < max_failures_allowed) { onboardingFailuresCount += 1; alert('You did not label the sample conversation well enough. Please try one more time!'); } else { @@ -177,7 +179,7 @@ function OnboardingComponent({ onboardingData, annotationBuckets, annotationQues className="button is-link btn-lg" onClick={() => handleOnboardingSubmit({ onboardingData, - currentTurnAnnotations, + currentTurnAnnotations, onSubmit, })} > diff --git a/parlai/crowdsourcing/tasks/model_chat/task_config/onboard_task_data.json b/parlai/crowdsourcing/tasks/model_chat/task_config/onboard_task_data.json index 17056494cb4..538d72af899 100644 --- a/parlai/crowdsourcing/tasks/model_chat/task_config/onboard_task_data.json +++ b/parlai/crowdsourcing/tasks/model_chat/task_config/onboard_task_data.json @@ -85,5 +85,8 @@ ] } ] - ] + ], + "min_correct": 4, + "max_incorrect": 3, + "max_failures_allowed": 1 } \ No newline at end of file diff --git a/parlai/crowdsourcing/tasks/model_chat/worlds.py b/parlai/crowdsourcing/tasks/model_chat/worlds.py index 2f819d9409e..f0814abf17f 100644 --- a/parlai/crowdsourcing/tasks/model_chat/worlds.py +++ b/parlai/crowdsourcing/tasks/model_chat/worlds.py @@ -19,7 +19,6 @@ from parlai.crowdsourcing.utils.worlds import CrowdOnboardWorld, CrowdTaskWorld from parlai.crowdsourcing.tasks.model_chat.bot_agent import TurkLikeAgent from parlai.crowdsourcing.tasks.model_chat.constants import ( - ONBOARD_CONFIG, ONBOARD_FAIL, ONBOARD_SUCCESS, ) @@ -50,8 +49,6 @@ def __init__(self, opt, agent: "MephistoAgentWrapper"): self.skip_onboarding = opt['skip_onboarding'] - self.min_correct = ONBOARD_CONFIG['min_correct'] - self.max_incorrect = ONBOARD_CONFIG['max_incorrect'] self.onboard_task_data = opt['onboard_task_data'] self.status = 'DISCONNECT' self.onboard_statistics = opt['onboard_statistics'] @@ -59,49 +56,7 @@ def __init__(self, opt, agent: "MephistoAgentWrapper"): self.max_onboard_time = opt['max_onboard_time'] self.onboarding_qualification = opt['onboarding_qualification'] self.worker_id = get_mturk_id_from_mephisto_wrapper(self.agent) - - def has_same_answer(self, ans1, ans2): - if len(ans1) != len(ans2): - return False - - ans1_sort = sorted(ans1) - ans2_sort = sorted(ans2) - - for x in range(len(ans1_sort)): - if ans1_sort[x] != ans2_sort[x]: - return False - return True - - def check_onboarding_answers(self, worker_answers) -> bool: - """ - Calculate how many correct answers the user gave. - - `worker_answers` is a list of dicts containing mappings between an annotation - value and whether it was selected for each bucket. We return a boolean as to - whether the worker passed or failed the task. - """ - given_turns = self.onboard_task_data['dialog'] - correct_answers = [t[1]['answers'] for t in given_turns] - number_correct = 0 - number_incorrect = 0 - for worker_answer, correct_answer in zip(worker_answers, correct_answers): - worker_only_selected = [ - key for key, selected in worker_answer.items() if selected - ] - if self.has_same_answer(worker_only_selected, correct_answer): - number_correct += 1 - else: - number_incorrect += 1 - - print( - f'Worker {self.worker_id} got {number_correct} annotations correct and {number_incorrect} incorrect in onboarding.' - ) - if ( - number_correct >= self.min_correct - and number_incorrect <= self.max_incorrect - ): - return True - return False + self.annotations = None def parley(self): @@ -140,10 +95,11 @@ def _handle_act(self, act): print(f'{self.__class__.__name__}: {self.worker_id} had no data submitted') return ONBOARD_FAIL - worker_answers = act['task_data']['annotations'] + self.annotations = act['task_data'].get('annotations') + print('Onboarding annotation results: ', self.annotations) - if self.check_onboarding_answers(worker_answers): - print(f'Worker {self.worker_id} successfully passed the onboard task.') + if act['task_data']['success']: + print(f'Worker {self.worker_id} successfully passed the onboarding task.') # This will end the onboarding and send them directly to the HIT self.episodeDone = True @@ -164,6 +120,9 @@ def shutdown(self): self.onboard_statistics[self.status] = 0 self.onboard_statistics[self.status] += 1 + def get_custom_task_data(self): + return self.annotations + class BaseModelChatWorld(CrowdTaskWorld, ABC): def __init__(self, opt, agent, bot):