Model chat: allow multiple final ratings (#4276)

* Layer changes * Docs and cleanup * Show original UI if only one rating * Fixes * Simplification * Dummy
facebookresearch · Jan 5, 2022 · 4f39183 · 4f39183
1 parent e45c8a4
commit 4f39183
Show file tree

Hide file tree

Showing 3 changed files with 127 additions and 64 deletions.
diff --git a/parlai/crowdsourcing/tasks/model_chat/README.md b/parlai/crowdsourcing/tasks/model_chat/README.md
@@ -12,6 +12,8 @@ Set `mephisto.blueprint.model_opt_path` to specify a path to a YAML file listing
 
 Set `mephisto.blueprint.chat_data_folder` to the root folder that you want all results of HITs to be saved in: all results will be saved to a date folder (of format `2021_01_15`) within that root folder.
 
+Set `mephisto.blueprint.final_rating_question` to specify the question to ask the worker at the end of the task, for which the worker will respond with a 1-to-5 Likert score. Separate multiple questions with a `|`.
+
 ## Passing in task config files
 
 The following flags can be passed in to specify filepaths for overriding the text shown to the workers and the settings of the annotation categories. If they are not specified, the defaults in the `task_config/` folder will be used.

diff --git a/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx b/parlai/crowdsourcing/tasks/model_chat/frontend/components/response_panes.jsx
@@ -8,13 +8,7 @@
 
 import React from "react";
 
-import { 
-  FormControl, 
-  Button,
-  Col,
-  FormGroup,
-  ControlLabel,
-} from "react-bootstrap";
+import { Button, Col, ControlLabel, Form, FormControl, FormGroup } from "react-bootstrap";
 
 
 function hasAnyAnnotations(annotations) {
@@ -28,26 +22,8 @@ function hasAnyAnnotations(annotations) {
   }
   return false;
 }
-
-function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes}) {
-  const [rating, setRating] = React.useState(0);
-  const [sending, setSending] = React.useState(false);
-
-  const tryMessageSend = React.useCallback(() => {
-    if (active && !sending) {
-      setSending(true);
-      onMessageSend({ 
-        text: "", 
-        task_data: {
-          problem_data_for_prior_message: currentCheckboxes,
-          final_rating: rating,
-        },
-      }).then(() => {
-        setSending(false);
-      });
-    }
-  }, [active, sending, rating, onMessageSend]);
 
+function RatingSelector({ active, ratings, sending, ratingQuestion, ratingIndex, setRatings }) {
   const ratingOptions = [<option key="empty_option" />].concat(
     ["1", "2", "3", "4", "5"].map((option_label, index) => {
       return (
@@ -56,55 +32,135 @@ function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes}) {
     })
   );
 
-  const ratingSelector = (
-    <FormGroup key={"final_survey"}>
+  function handleRatingSelection(val) {
+    const newRatings = ratings.map((item, index) => {
+      if (index === ratingIndex) {
+        return val;
+      } else {
+        return item;
+      }
+    });
+    setRatings(newRatings);
+  }
+
+  return (
+    <FormGroup key={"final_survey_" + ratingIndex.toString()}>
       <Col
         componentClass={ControlLabel}
         sm={6}
-        style={{ fontSize: "16px" }}
+        style={{ fontSize: "14px" }}
       >
-        {taskConfig.final_rating_question}
+        {ratingQuestion}
       </Col>
       <Col sm={5}>
         <FormControl
           componentClass="select"
-          style={{ fontSize: "16px" }}
-          value={rating}
-          onChange={(e) => {
-            var val = e.target.value;
-            setRating(val);
-          }}
+          style={{ fontSize: "14px" }}
+          value={ratings[ratingIndex]}
+          onChange={(e) => handleRatingSelection(e.target.value)}
           disabled={!active || sending}
         >
           {ratingOptions}
         </FormControl>
       </Col>
     </FormGroup>
   );
+}
 
-  return (
-    <div className="response-type-module">
-      <div>
-        You've completed the conversation. Please annotate the final turn, fill out
-        the following, and hit Done.
-      </div>
-      <br />
-      <div className="response-bar">
-        {ratingSelector}
-        <Button
-          className="btn btn-submit submit-response"
-          id="id_send_msg_button"
-          disabled={rating === 0 || !active || sending}
-          onClick={() => tryMessageSend()}
+function FinalSurvey({ taskConfig, onMessageSend, active, currentCheckboxes }) {
+  const [sending, setSending] = React.useState(false);
+
+  // Set up multiple response questions
+  let ratingQuestions = taskConfig.final_rating_question.split("|");
+  let initialRatings = [];
+  for (let _ of ratingQuestions) {
+    initialRatings.push("");
+  }
+  const [ratings, setRatings] = React.useState(initialRatings)
+
+  const tryMessageSend = React.useCallback(() => {
+
+    let all_ratings_filled = ratings.every((r) => r !== "");
+    let rating = ratings.join('|');
+
+    if (all_ratings_filled && active && !sending) {
+      setSending(true);
+      onMessageSend({
+        text: "",
+        task_data: {
+          problem_data_for_prior_message: currentCheckboxes,
+          final_rating: rating,
+        },
+      }).then(() => {
+        setSending(false);
+      });
+    }
+  }, [active, sending, ratings, onMessageSend]);
+
+  const listRatingSelectors = ratingQuestions.map((ratingQuestion, ratingIndex) => {
+    return (
+      <RatingSelector
+        active={active}
+        ratings={ratings}
+        sending={sending}
+        ratingQuestion={ratingQuestion}
+        ratingIndex={ratingIndex}
+        setRatings={setRatings}
+      >
+      </RatingSelector>
+    );
+  });
+
+  if (listRatingSelectors.length > 1) {
+    // Show ratings to the right of the questions
+    return (
+      <div className="response-type-module">
+        <div>
+          You've completed the conversation. Please annotate the final turn, fill out
+          the following, and hit Done.
+        </div>
+        <br />
+        <Form
+          horizontal
         >
-          Done
-        </Button>
+          {listRatingSelectors}
+          <Button
+            className="btn btn-submit submit-response"
+            id="id_send_msg_button"
+            disabled={!active || sending}
+            onClick={() => tryMessageSend()}
+          >
+            Done
+          </Button>
+        </Form>
       </div>
-    </div>
-  );
+    );
+  } else {
+    // Show the single rating below the single question
+    return (
+      <div className="response-type-module">
+        <div>
+          You've completed the conversation. Please annotate the final turn, fill out
+          the following, and hit Done.
+        </div>
+        <br />
+        <div className="response-bar">
+          {listRatingSelectors}
+          <Button
+            className="btn btn-submit submit-response"
+            id="id_send_msg_button"
+            disabled={!active || sending}
+            onClick={() => tryMessageSend()}
+          >
+            Done
+          </Button>
+        </div>
+      </div>
+    );
+  }
 }
 
-function CheckboxTextResponse({ onMessageSend, active, currentCheckboxes}) {
+function CheckboxTextResponse({ onMessageSend, active, currentCheckboxes }) {
   const [textValue, setTextValue] = React.useState("");
   const [sending, setSending] = React.useState(false);
 
@@ -119,9 +175,9 @@ function CheckboxTextResponse({ onMessageSend, active, currentCheckboxes}) {
   const tryMessageSend = React.useCallback(() => {
     if (textValue !== "" && active && !sending) {
       setSending(true);
-      onMessageSend({ 
-        text: textValue, 
-        task_data: {problem_data_for_prior_message: currentCheckboxes} 
+      onMessageSend({
+        text: textValue,
+        task_data: { problem_data_for_prior_message: currentCheckboxes }
       }).then(() => {
         setTextValue("");
         setSending(false);
@@ -169,15 +225,15 @@ function CheckboxTextResponse({ onMessageSend, active, currentCheckboxes}) {
 }
 
 function ResponseComponent({ taskConfig, appSettings, onMessageSend, active }) {
-  
+
   const lastMessageIdx = appSettings.numMessages - 1;
   const lastMessageAnnotations = appSettings.checkboxValues[lastMessageIdx];
-  
+
   const computedActive = (
-    taskConfig.annotation_buckets === null || 
+    taskConfig.annotation_buckets === null ||
     hasAnyAnnotations(lastMessageAnnotations) & active
   );
-  
+
   if (lastMessageIdx >= taskConfig.min_num_turns * 2) {
     return (
       <FinalSurvey
@@ -189,7 +245,7 @@ function ResponseComponent({ taskConfig, appSettings, onMessageSend, active }) {
     );
   } else {
     return (
-      <CheckboxTextResponse 
+      <CheckboxTextResponse
         onMessageSend={onMessageSend}
         active={computedActive}
         currentCheckboxes={lastMessageAnnotations}

diff --git a/parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py b/parlai/crowdsourcing/tasks/model_chat/model_chat_blueprint.py
@@ -137,7 +137,12 @@ class BaseModelChatBlueprintArgs(ParlAIChatBlueprintArgs):
     )
     final_rating_question: str = field(
         default='Please rate your partner on a scale of 1-5.',
-        metadata={"help": "Text to show when asking worker to make their final rating"},
+        metadata={
+            "help": (
+                "Text to show when asking worker to make their final rating. For "
+                "multiple final ratings, separate text strings with a '|'."
+            )
+        },
     )
     max_concurrent_responses: int = field(
         default=1,