Modelchat worker qual (#4956)

* test cases for the allow list filter

* skip if not Mephisto

* added the allow list qualification to the model chat

* allow list filter

* cache worker qualification

* cache worker qualification

* nit logic bug

* used the correct usage of allow list

* removed the unnecessary test file

* seperated the mephistor import

* PR comments

parlai/crowdsourcing/tasks/chat_demo/run.py

@@ -8,7 +8,7 @@
 from dataclasses import dataclass, field
 
 import hydra
-from omegaconf import DictConfig
+from omegaconf import DictConfig, MISSING
 
 # From the Mephisto repo
 from examples.parlai_chat_task_demo.parlai_test_script import (

parlai/crowdsourcing/tasks/model_chat/README.md

@@ -54,6 +54,8 @@ Some options for running human+model image chat are as follows:
 - `mephisto.blueprint.stack_folder`: a folder in which to store a stack file that will keep track of which crowdsource workers have chatted with which models about which images. The stack will ensure that no worker chats about the same image more than once and that conversations about images are collected uniformly among all models.
 - `mephisto.blueprint.evals_per_image_model_combo`: the maximum number of conversations collected for each combination of image and model. For instance, if this is set to 3 and your 2 models are `model_1` and `model_2`, each image will have 6 conversations collected about it, 3 with `model_1` and 3 with `model_2`.
 - `mephisto.blueprint.world_file`: the path to the Python module containing the class definition for the chat World, used for setting the logic for each turn of the conversation, when to end the conversation, actions upon shutdown, etc. (The onboarding World, if it exists, will be defined in this module as well.) Modify this value if you would like to write your own World class without having to create a new Blueprint class.
+- `allowed_worker_qualification`: use this qualification to restrict the task to an exclusive list of workers. Note that you must first grant this qualification to the workers who are allowed to participate.
+Use `direct_assign_qual_mturk_workers` function from `mephisto.abstractions.providers.mturk.utils.script_utils` to mark workers who are allowed to participate.
 
 Note that onboarding is not currently supported with human+model image chat: use `ModelChatOnboardWorld` in `worlds.py` as a guide for how to set up onboarding for your specific task.
 

parlai/crowdsourcing/tasks/model_chat/impl.py

@@ -7,9 +7,11 @@
 import os
 import random
 
+from omegaconf import DictConfig, OmegaConf
 from mephisto.operations.operator import Operator
 from mephisto.tools.scripts import load_db_and_process_config
-from omegaconf import DictConfig, OmegaConf
+from mephisto.data_model.qualification import QUAL_EXISTS
+from mephisto.utils.qualifications import make_qualification_dict
 
 from parlai.crowdsourcing.utils.mturk import soft_block_mturk_workers
 from parlai.crowdsourcing.tasks.model_chat.model_chat_blueprint import (
@@ -46,8 +48,18 @@ def run_task(cfg: DictConfig, task_directory: str, world_module=None):
     # Default to a task-specific name to avoid soft-block collisions
     soft_block_mturk_workers(cfg=cfg, db=db, soft_block_qual_name=soft_block_qual_name)
 
-    # Init
-    shared_state = SharedModelChatTaskState(world_module=world_module)
+    # TODO: this maybe moved to the parent class: SharedModelChatTaskState 
+    if cfg.mephisto.blueprint.allowed_worker_qualification is not None:
+        use_qualifications = [
+            make_qualification_dict(
+                cfg.mephisto.blueprint.allowed_worker_qualification, QUAL_EXISTS, None
+            ),
+        ]
+        shared_state = SharedModelChatTaskState(
+            world_module=world_module, qualifications=use_qualifications
+        )
+    else:
+        shared_state = SharedModelChatTaskState(world_module=world_module)
 
     operator = Operator(db)
     operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=shared_state)

parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py

@@ -155,6 +155,10 @@ class BaseModelChatBlueprintArgs(ParlAIChatBlueprintArgs):
             "in order to override the parlai parser defaults."
         },
     )
+    allowed_worker_qualification: Optional[str] = field(
+        default=None,
+        metadata="The qualification name for the workers that are exclusively allowed to do the HITs from this task.",
+    )
 
 
 class BaseModelChatBlueprint(ParlAIChatBlueprint, ABC):