Add a randomize_conversations to ACUTE blueprint + use

parlai/crowdsourcing/tasks/acute_eval/conf/example.yaml

@@ -13,6 +13,7 @@ mephisto:
     s1_choice: "I would prefer to talk to <Speaker 1>"
     s2_choice: "I would prefer to talk to <Speaker 2>"
     subtasks_per_unit: 5
+    randomize_conversations: True
   task:
     allowed_concurrent: 1
     assignment_duration_in_seconds: 600

parlai/crowdsourcing/tasks/acute_eval/conf/example_fast_acute.yaml

@@ -7,3 +7,4 @@ mephisto:
     num_self_chats: 5
     root_dir: ${task_dir}/task_config
     task: blended_skill_talk
+    randomize_conversations: True

parlai/crowdsourcing/tasks/acute_eval/fast_acute_blueprint.py

@@ -81,6 +81,12 @@ class FastAcuteBlueprintArgs(AcuteEvalBlueprintArgs):
         default=False,
         metadata={'help': "Use any existing self-chat files without prompting"},
     )
+    randomize_conversations: bool = field(
+        default=True,
+        metadata={
+            'help': "Randomize conversations used for match-ups or take conversations in order"
+        },
+    )
 
 
 @register_mephisto_abstraction()

parlai/crowdsourcing/tasks/acute_eval/fast_eval.py

@@ -329,12 +329,15 @@ def _build_conversation_pairs(
             self.fast_acute_args.matchups_per_pair
             * self.fast_acute_args.sufficient_matchups_multiplier
         )
-        # Write random pairs of conversations
+        # Write pairs of conversations
         for model_pair in self.combos:
-            for _ in range(pairs_per_model):
-                conversation_indices = [
-                    random.choice(range(len(conversations[m]))) for m in model_pair
-                ]
+            for i in range(pairs_per_model):
+                if self.fast_acute_args.randomize_conversations:
+                    conversation_indices = [
+                        random.choice(range(len(conversations[m]))) for m in model_pair
+                    ]
+                else:
+                    conversation_indices = [i for _ in model_pair]
                 pair = []
                 pair_ids = []
                 for i, c_id in enumerate(conversation_indices):