facebookresearch · meganung · Oct 25, 2021 · Oct 7, 2021 · Oct 7, 2021 · Oct 8, 2021
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -167,7 +167,7 @@ commands:
             command: |
               cd ..
               git clone git@github.com:facebookresearch/Mephisto.git Mephisto
-              cd Mephisto; git checkout v0.4.0 -b stable
+              cd Mephisto; git checkout v0.4.1 -b stable
               pip install -r requirements.txt
               python setup.py develop
               # `echo` so that ENTER will be pressed at the prompt

diff --git a/parlai/agents/rag/retrieve_api.py b/parlai/agents/rag/retrieve_api.py
@@ -75,7 +75,7 @@ class SearchEngineRetriever(RetrieverAPI):
     Queries a server (eg, search engine) for a set of documents.
 
     This module relies on a running HTTP server. For each retrieval it sends the query
-    to this server and receives a JSON; it parses the JSON to create the the response.
+    to this server and receives a JSON; it parses the JSON to create the response.
     """
 
     def __init__(self, opt: Opt):

diff --git a/parlai/agents/rag/retrievers.py b/parlai/agents/rag/retrievers.py
@@ -1128,7 +1128,7 @@ def pick_chunk(self, query, doc_text):
 
 class SearchQuerySearchEngineRetriever(SearchQueryRetriever):
     """
-    A retriever that uses a search engine server for rertrieving documents.
+    A retriever that uses a search engine server for retrieving documents.
 
     It instantiates a `SearchEngineRetriever` object that in turns send search queries
     to an external server for retrieving documents.

diff --git a/parlai/crowdsourcing/projects/multisession_chat/human_eval/hydra_configs/conf/example.yaml b/parlai/crowdsourcing/projects/multisession_chat/human_eval/hydra_configs/conf/example.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: model_chat_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     annotations_config_path: ${task_dir}/task_config/annotations_config.json

diff --git a/parlai/crowdsourcing/projects/multisession_chat/human_eval/run.py b/parlai/crowdsourcing/projects/multisession_chat/human_eval/run.py
@@ -18,15 +18,12 @@
 import json
 
 
+_ = BLUEPRINT_TYPE
+
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
 
-defaults = [
-    {'mephisto/blueprint': BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example"},
-]
+defaults = ["_self_", {"conf": "example"}]
 
 
 def check_override_opt(args):

diff --git a/parlai/crowdsourcing/projects/wizard_of_internet/hydra_configs/conf/dev.yaml b/parlai/crowdsourcing/projects/wizard_of_internet/hydra_configs/conf/dev.yaml
@@ -1,5 +1,9 @@
 
 #@package _global_
+defaults:
+  - /mephisto/blueprint: wizard_internet_parlaichat_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     onboarding_qualification: wizard-onboarding-dev

diff --git a/parlai/crowdsourcing/projects/wizard_of_internet/run.py b/parlai/crowdsourcing/projects/wizard_of_internet/run.py
@@ -28,14 +28,11 @@
 from mephisto.tools.scripts import load_db_and_process_config
 
 
+_ = WIZARD_INTERNET_PARLAICHAT_BLUEPRINT
+
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
-defaults = [
-    {'mephisto/blueprint': WIZARD_INTERNET_PARLAICHAT_BLUEPRINT},
-    {'mephisto/architect': 'local'},
-    {'mephisto/provider': 'mock'},
-    {'conf': 'dev'},
-]
+defaults = ['_self_', {"conf": "dev"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py b/parlai/crowdsourcing/tasks/acute_eval/fast_eval.py
@@ -43,6 +43,8 @@
 from parlai.utils.strings import normalize_reply
 from parlai.utils.testing import capture_output
 
+_ = FAST_ACUTE_BLUEPRINT_TYPE
+
 ########################
 # ACUTE EVAL CONSTANTS #
 ########################
@@ -547,14 +549,7 @@ def analyze_results(self, args: Optional[str] = None):
         self._print_progress(f'ACUTE results saved to {self.results_path}')
 
 
-defaults = [
-    '_self_',
-    {"mephisto/blueprint": FAST_ACUTE_BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    'conf/base_fast_acute',
-    {"conf": "example_fast_acute"},
-]
+defaults = ['_self_', {"conf": "example_fast_acute"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/base_fast_acute.yaml b/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/base_fast_acute.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: fast_acute
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   task:
     allowed_concurrent: 1

diff --git a/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example.yaml b/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: acute_eval
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     additional_task_description: ""

diff --git a/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example_fast_acute.yaml b/parlai/crowdsourcing/tasks/acute_eval/hydra_configs/conf/example_fast_acute.yaml
@@ -1,4 +1,6 @@
 #@package _global_
+defaults:
+  - base_fast_acute
 mephisto:
   blueprint:
     acute_eval_type: engaging

diff --git a/parlai/crowdsourcing/tasks/acute_eval/run.py b/parlai/crowdsourcing/tasks/acute_eval/run.py
@@ -30,12 +30,9 @@
 
 """
 
-defaults = [
-    {"mephisto/blueprint": BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example"},
-]
+_ = BLUEPRINT_TYPE
+
+defaults = ["_self_", {"conf": "example"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/chat_demo/run.py b/parlai/crowdsourcing/tasks/chat_demo/run.py
@@ -41,9 +41,10 @@ class ScriptConfig(MTurkRunScriptConfigMixin, TestScriptConfig):
 
 register_script_config(name="scriptconfig", module=ScriptConfig)
 relative_task_directory = os.path.relpath(TASK_DIRECTORY, os.path.dirname(__file__))
+config_path = os.path.join(relative_task_directory, 'hydra_configs')
 
 
-@hydra.main(config_path=relative_task_directory, config_name="scriptconfig")
+@hydra.main(config_path=config_path, config_name="scriptconfig")
 def main(cfg: DictConfig) -> None:
     db, cfg = load_db_and_process_config(cfg)
 

diff --git a/parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example.yaml b/parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: model_chat_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     annotations_config_path: ${task_dir}/task_config/annotations_config.json

diff --git a/parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example_image_chat.yaml b/parlai/crowdsourcing/tasks/model_chat/hydra_configs/conf/example_image_chat.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: model_image_chat_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     annotations_config_path: ""

diff --git a/parlai/crowdsourcing/tasks/model_chat/run.py b/parlai/crowdsourcing/tasks/model_chat/run.py
@@ -12,7 +12,6 @@
 from mephisto.operations.hydra_config import register_script_config
 from omegaconf import DictConfig
 
-from parlai.crowdsourcing.tasks.model_chat.model_chat_blueprint import BLUEPRINT_TYPE
 from parlai.crowdsourcing.tasks.model_chat.impl import run_task
 from parlai.crowdsourcing.utils.mturk import MTurkRunScriptConfig
 import parlai.crowdsourcing.tasks.model_chat.worlds as world_module
@@ -21,12 +20,7 @@
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
 
-defaults = [
-    {'mephisto/blueprint': BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example"},
-]
+defaults = ["_self_", {"conf": "example"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/model_chat/run_image_chat.py b/parlai/crowdsourcing/tasks/model_chat/run_image_chat.py
@@ -12,9 +12,6 @@
 from mephisto.operations.hydra_config import register_script_config
 from omegaconf import DictConfig
 
-from parlai.crowdsourcing.tasks.model_chat.model_chat_blueprint import (
-    IMAGE_CHAT_BLUEPRINT_TYPE,
-)
 from parlai.crowdsourcing.tasks.model_chat.impl import run_task
 from parlai.crowdsourcing.utils.mturk import MTurkRunScriptConfig
 import parlai.crowdsourcing.tasks.model_chat.worlds as world_module
@@ -23,12 +20,7 @@
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
 
-defaults = [
-    {'mephisto/blueprint': IMAGE_CHAT_BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example_image_chat"},
-]
+defaults = ["_self_", {"conf": "example_image_chat"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/qa_data_collection/hydra_configs/conf/example.yaml b/parlai/crowdsourcing/tasks/qa_data_collection/hydra_configs/conf/example.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: parlai_chat
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     world_file: ${task_dir}/worlds.py

diff --git a/parlai/crowdsourcing/tasks/qa_data_collection/run.py b/parlai/crowdsourcing/tasks/qa_data_collection/run.py
@@ -12,7 +12,6 @@
 import hydra
 from omegaconf import DictConfig
 from mephisto.abstractions.blueprints.parlai_chat.parlai_chat_blueprint import (
-    BLUEPRINT_TYPE,
     SharedParlAITaskState,
 )
 from mephisto.operations.hydra_config import register_script_config
@@ -26,12 +25,7 @@
 
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
-defaults = [
-    {"mephisto/blueprint": BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example"},
-]
+defaults = ["_self_", {"conf": "example"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example.yaml b/parlai/crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: turn_annotations_static_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     subtasks_per_unit: 6

diff --git a/.../crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example_in_flight_qa.yaml b/.../crowdsourcing/tasks/turn_annotations_static/hydra_configs/conf/example_in_flight_qa.yaml
@@ -1,4 +1,8 @@
 #@package _global_
+defaults:
+  - /mephisto/blueprint: turn_annotations_static_inflight_qa_blueprint
+  - /mephisto/architect: local
+  - /mephisto/provider: mock
 mephisto:
   blueprint:
     subtasks_per_unit: 6

diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/run.py b/parlai/crowdsourcing/tasks/turn_annotations_static/run.py
@@ -12,9 +12,6 @@
 from mephisto.operations.hydra_config import register_script_config
 from omegaconf import DictConfig
 
-from parlai.crowdsourcing.tasks.turn_annotations_static.turn_annotations_blueprint import (
-    STATIC_BLUEPRINT_TYPE,
-)
 from parlai.crowdsourcing.tasks.turn_annotations_static.util import run_static_task
 from parlai.crowdsourcing.utils.mturk import MTurkRunScriptConfig
 
@@ -26,12 +23,7 @@
 # check the README in parlai/crowdsourcing/
 # python turn_annotations_static/run.py conf=<conf name sans yaml> --config-dir <path to directory with a conf/ folder>
 
-defaults = [
-    {'mephisto/blueprint': STATIC_BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": 'example'},
-]
+defaults = ["_self_", {"conf": 'example'}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/run_in_flight_qa.py b/parlai/crowdsourcing/tasks/turn_annotations_static/run_in_flight_qa.py
@@ -12,9 +12,6 @@
 from mephisto.operations.hydra_config import register_script_config
 from omegaconf import DictConfig
 
-from parlai.crowdsourcing.tasks.turn_annotations_static.turn_annotations_blueprint import (
-    STATIC_IN_FLIGHT_QA_BLUEPRINT_TYPE,
-)
 from parlai.crowdsourcing.tasks.turn_annotations_static.util import run_static_task
 from parlai.crowdsourcing.utils.mturk import MTurkRunScriptConfig
 
@@ -26,12 +23,7 @@
 TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
 
 
-defaults = [
-    {'mephisto/blueprint': STATIC_IN_FLIGHT_QA_BLUEPRINT_TYPE},
-    {"mephisto/architect": "local"},
-    {"mephisto/provider": "mock"},
-    {"conf": "example_in_flight_qa"},
-]
+defaults = ["_self_", {"conf": "example_in_flight_qa"}]
 
 
 @dataclass

diff --git a/parlai/crowdsourcing/utils/tests.py b/parlai/crowdsourcing/utils/tests.py
@@ -89,10 +89,10 @@ def _set_up_config(
             self.config = compose(
                 config_name="example",
                 overrides=[
-                    f'+mephisto.blueprint._blueprint_type={blueprint_type}',
-                    f'+mephisto.blueprint.link_task_source=False',
-                    f'+mephisto/architect=mock',
-                    f'+mephisto/provider=mock',
+                    f'mephisto.blueprint._blueprint_type={blueprint_type}',
+                    f'++mephisto.blueprint.link_task_source=False',
+                    f'mephisto/architect=mock',
+                    f'mephisto/provider=mock',
                     f'+task_dir={task_directory}',
                     f'+current_time={int(time.time())}',
                 ]
@@ -132,9 +132,12 @@ def _get_channel_info(self):
         else:
             raise ValueError('No channel could be detected!')
 
-    def _register_mock_agents(self, num_agents: int = 1) -> List[str]:
+    def _register_mock_agents(
+        self, num_agents: int = 1, assume_onboarding: bool = False
+    ) -> List[str]:
         """
-        Register mock agents for testing, taking the place of crowdsourcing workers.
+        Register mock agents for testing and onboard them if needed, taking the place of
+        crowdsourcing workers.
 
         Specify the number of agents to register. Return the agents' IDs after creation.
         """
@@ -157,6 +160,14 @@ def _register_mock_agents(self, num_agents: int = 1) -> List[str]:
                     # Register the agent
                     mock_agent_details = f"FAKE_ASSIGNMENT_{idx:d}"
                     self.server.register_mock_agent(worker_id, mock_agent_details)
+
+                    if assume_onboarding:
+                        # Submit onboarding from the agent
+                        onboard_agents = self.db.find_onboarding_agents()
+                        onboard_data = {"onboarding_data": {"success": True}}
+                        self.server.register_mock_agent_after_onboarding(
+                            worker_id, onboard_agents[0].get_agent_id(), onboard_data
+                        )
                     _ = self.db.find_agents()[idx]
                     # Make sure the agent can be found, or else raise an IndexError
 
@@ -215,7 +226,12 @@ def _get_agent_state(self, task_data: Dict[str, Any]):
         """
 
         # Set up the mock human agent
-        agent_id = self._register_mock_agents(num_agents=1)[0]
+        if self.config.mephisto.blueprint.get("onboarding_qualification", None):
+            agent_id = self._register_mock_agents(num_agents=1, assume_onboarding=True)[
+                0
+            ]
+        else:
+            agent_id = self._register_mock_agents(num_agents=1)[0]
 
         # Set initial data
         self.server.request_init_data(agent_id)

diff --git a/projects/blenderbot2/agents/modules.py b/projects/blenderbot2/agents/modules.py
@@ -309,7 +309,7 @@ def retrieve_and_concat(
     ) -> Tuple[torch.LongTensor, List[List[Document]], torch.Tensor]:
         """
         Override RagModel.retrieve_and_concat to perform different retrieval, depending
-        on the.
+        on the RetrieverType.
         """
         start = time.time()
         logging.debug(f'Begin encoder: {time.time() - start:.2f}')