add synthetic badge and all answers wrong checkbox to ranking task (#…

…1937)

![image](https://user-images.githubusercontent.com/33456881/222528454-380616d5-0536-432e-9022-6574f6538fec.png)

website/public/locales/en/message.json

@@ -23,5 +23,7 @@
   "submit_labels": "Submit",
   "tree_stopped": "Tree stopped {{id}}",
   "view_user": "View user",
-  "your_recent_messages": "Your Recent Messages"
+  "your_recent_messages": "Your Recent Messages",
+  "synthetic": "Synthetic",
+  "synthetic_explain": "This message is AI generated"
 }

website/public/locales/en/tasks.json

@@ -88,5 +88,6 @@
   "submitted_as": "This will be submitted as {{submit_lang}}",
   "tab_write": "Write",
   "tab_preview": "Preview",
-  "writing_wrong_langauge_a_b": "You appear to be writing in {{detected_lang}} but this will be submitted as {{submit_lang}}."
+  "writing_wrong_langauge_a_b": "You appear to be writing in {{detected_lang}} but this will be submitted as {{submit_lang}}.",
+  "not_rankable": "All answers are factually incorrect and cannot be ranked"
 }

website/src/components/Sortable/Sortable.tsx

@@ -25,30 +25,32 @@ import {
   sortableKeyboardCoordinates,
   verticalListSortingStrategy,
 } from "@dnd-kit/sortable";
-import { lazy, ReactNode, Suspense, useEffect, useState } from "react";
+import { lazy, Suspense, useCallback, useEffect, useState } from "react";
+import { Message } from "src/types/Conversation";
 
 import { CollapsableText } from "../CollapsableText";
 import { SortableItem } from "./SortableItem";
 
 const RenderedMarkdown = lazy(() => import("../Messages/RenderedMarkdown"));
 
 export interface SortableProps {
-  items: ReactNode[];
+  items: Message[];
   onChange?: (newSortedIndices: number[]) => void;
   isEditable: boolean;
   isDisabled?: boolean;
   className?: string;
+  revealSynthetic?: boolean;
 }
 
-interface SortableItems {
+interface SortableItem {
   id: number;
   originalIndex: number;
-  item: ReactNode;
+  item: Message;
 }
 
-export const Sortable = (props: SortableProps) => {
-  const [itemsWithIds, setItemsWithIds] = useState<SortableItems[]>([]);
-  const [modalText, setModalText] = useState(null);
+export const Sortable = ({ onChange, revealSynthetic, ...props }: SortableProps) => {
+  const [itemsWithIds, setItemsWithIds] = useState<SortableItem[]>([]);
+  const [modalText, setModalText] = useState<string | null>(null);
   useEffect(() => {
     setItemsWithIds(
       props.items.map((item, idx) => ({
@@ -71,6 +73,23 @@ export const Sortable = (props: SortableProps) => {
   const { isOpen, onOpen, onClose } = useDisclosure();
   const extraClasses = props.className || "";
 
+  const handleDragEnd = useCallback(
+    (event: DragEndEvent) => {
+      const { active, over } = event;
+      if (active.id === over?.id) {
+        return;
+      }
+      setItemsWithIds((items) => {
+        const oldIndex = items.findIndex((x) => x.id === active.id);
+        const newIndex = items.findIndex((x) => x.id === over?.id);
+        const newArray = arrayMove(items, oldIndex, newIndex);
+        onChange && onChange(newArray.map((item) => item.originalIndex));
+        return newArray;
+      });
+    },
+    [onChange]
+  );
+
   return (
     <>
       <DndContext
@@ -80,28 +99,29 @@ export const Sortable = (props: SortableProps) => {
         modifiers={[restrictToWindowEdges, restrictToVerticalAxis]}
       >
         <SortableContext items={itemsWithIds} strategy={verticalListSortingStrategy}>
-          <Flex direction="column" gap={2} className={extraClasses}>
+          <Flex direction="column" gap={4} className={extraClasses}>
             {itemsWithIds.map(({ id, item }, index) => (
               <SortableItem
                 OpenModal={() => {
-                  setModalText(item);
+                  setModalText(item.text);
                   onOpen();
                 }}
                 key={id}
                 id={id}
                 index={index}
                 isEditable={props.isEditable}
-                isDisabled={props.isDisabled}
+                isDisabled={!!props.isDisabled}
+                synthetic={item.synthetic && !!revealSynthetic}
               >
                 <button
                   className="w-full text-left"
                   aria-label="show full text"
                   onClick={() => {
-                    setModalText(item);
+                    setModalText(item.text);
                     onOpen();
                   }}
                 >
-                  <CollapsableText text={item} />
+                  <CollapsableText text={item.text} />
                 </button>
               </SortableItem>
             ))}
@@ -124,26 +144,12 @@ export const Sortable = (props: SortableProps) => {
             <ModalCloseButton />
             <ModalBody maxW="full">
               <Suspense fallback={modalText}>
-                <RenderedMarkdown markdown={modalText}></RenderedMarkdown>
+                <RenderedMarkdown markdown={modalText || ""}></RenderedMarkdown>
               </Suspense>
             </ModalBody>
           </ModalContent>
         </ModalOverlay>
       </Modal>
     </>
   );
-
-  function handleDragEnd(event: DragEndEvent) {
-    const { active, over } = event;
-    if (active.id === over.id) {
-      return;
-    }
-    setItemsWithIds((items) => {
-      const oldIndex = items.findIndex((x) => x.id === active.id);
-      const newIndex = items.findIndex((x) => x.id === over.id);
-      const newArray = arrayMove(items, oldIndex, newIndex);
-      props.onChange && props.onChange(newArray.map((item) => item.originalIndex));
-      return newArray;
-    });
-  }
 };

website/src/components/Sortable/SortableItem.tsx

@@ -1,8 +1,9 @@
-import { Box, useColorModeValue } from "@chakra-ui/react";
+import { Badge, Box, Flex, Tooltip, useColorModeValue } from "@chakra-ui/react";
 import { SyntheticListenerMap } from "@dnd-kit/core/dist/hooks/utilities";
 import { useSortable } from "@dnd-kit/sortable";
 import { CSS } from "@dnd-kit/utilities";
 import { GripVertical } from "lucide-react";
+import { useTranslation } from "next-i18next";
 import { PointerEventHandler, PropsWithChildren, useMemo } from "react";
 
 export const SortableItem = ({
@@ -12,12 +13,14 @@ export const SortableItem = ({
   isEditable,
   isDisabled,
   OpenModal,
+  synthetic,
 }: PropsWithChildren<{
   id: number;
   index: number;
   isEditable: boolean;
   isDisabled: boolean;
   OpenModal: () => void;
+  synthetic: boolean | null;
 }>) => {
   const backgroundColor = useColorModeValue("gray.700", "gray.500");
   const disabledBackgroundColor = useColorModeValue("gray.400", "gray.700");
@@ -36,6 +39,8 @@ export const SortableItem = ({
     [isEditable, activeBackgroundColor, backgroundColor]
   );
 
+  const { t } = useTranslation("message");
+
   return (
     <Box
       sx={sx}
@@ -48,11 +53,11 @@ export const SortableItem = ({
       p="4"
       whiteSpace="pre-wrap"
       color={textColor}
-      aria-roledescription="sortable"
       ref={setNodeRef}
       shadow="base"
       {...attributes}
       {...pcListeners}
+      aria-roledescription="sortable"
       className="relative"
     >
       <Box pr="4">{isEditable ? <GripVertical size="20px" /> : `${index + 1}.`}</Box>
@@ -62,6 +67,22 @@ export const SortableItem = ({
         onPointerDown={listeners?.onPointerDown as PointerEventHandler<HTMLDivElement>}
         className="w-[67%] lg:w-[80%]  h-full  absolute ltr:left-0 rtl:right-0 top-0 touch-none"
       ></div>
+      <Flex
+        position="absolute"
+        gap="2"
+        top="-2.5"
+        style={{
+          insetInlineEnd: "1.25rem",
+        }}
+      >
+        {synthetic && (
+          <Tooltip label={t("synthetic_explain")} placement="top" hasArrow>
+            <Badge size="sm" colorScheme="green" textTransform="capitalize">
+              {t("synthetic")}
+            </Badge>
+          </Tooltip>
+        )}
+      </Flex>
     </Box>
   );
 };

website/src/components/Tasks/EvaluateTask.tsx

@@ -1,10 +1,12 @@
-import { Box, useColorModeValue } from "@chakra-ui/react";
-import { useEffect, useState } from "react";
+import { Box, Checkbox, useColorModeValue } from "@chakra-ui/react";
+import { useTranslation } from "next-i18next";
+import { ChangeEvent, useCallback, useEffect, useState } from "react";
 import { MessageConversation } from "src/components/Messages/MessageConversation";
 import { Sortable } from "src/components/Sortable/Sortable";
 import { SurveyCard } from "src/components/Survey/SurveyCard";
 import { TaskSurveyProps } from "src/components/Tasks/Task";
 import { TaskHeader } from "src/components/Tasks/TaskHeader";
+import { Message } from "src/types/Conversation";
 import { TaskType } from "src/types/Task";
 import { EvaluateTaskReply } from "src/types/TaskResponses";
 import { RankTaskType } from "src/types/Tasks";
@@ -18,29 +20,40 @@ export const EvaluateTask = ({
   onValidityChanged,
 }: TaskSurveyProps<RankTaskType, EvaluateTaskReply>) => {
   const cardColor = useColorModeValue("gray.50", "gray.800");
-  const [ranking, setRanking] = useState<number[]>(null);
-
-  let messages = [];
+  const [ranking, setRanking] = useState<number[] | null>(null);
+  const [notRankable, setNotRankable] = useState(false);
+  let messages: Message[] = [];
   if (task.type !== TaskType.rank_initial_prompts) {
     messages = task.conversation.messages;
   }
 
   useEffect(() => {
     if (ranking === null) {
       if (task.type === TaskType.rank_initial_prompts) {
-        onReplyChanged({ ranking: task.prompts.map((_, idx) => idx) });
+        onReplyChanged({ ranking: task.prompts.map((_, idx) => idx), not_rankable: notRankable });
       } else {
-        onReplyChanged({ ranking: task.replies.map((_, idx) => idx) });
+        onReplyChanged({ ranking: task.replies.map((_, idx) => idx), not_rankable: notRankable });
+      }
+      if (!notRankable) {
+        onValidityChanged("DEFAULT");
       }
-      onValidityChanged("DEFAULT");
     } else {
-      onReplyChanged({ ranking });
+      onReplyChanged({ ranking, not_rankable: notRankable });
       onValidityChanged("VALID");
     }
-  }, [task, ranking, onReplyChanged, onValidityChanged]);
+    if (!notRankable) {
+      onValidityChanged("VALID");
+    }
+  }, [task, ranking, onReplyChanged, onValidityChanged, notRankable]);
 
-  const sortables = task.type === TaskType.rank_initial_prompts ? "prompts" : "replies";
+  const handleNotRankableChange = useCallback((e: ChangeEvent<HTMLInputElement>) => {
+    setNotRankable(e.target.checked);
+  }, []);
 
+  const { t } = useTranslation("tasks");
+  // @notmd: I haven't test `rank_initial_prompts` type yet
+  const sortableItems =
+    task.type === TaskType.rank_initial_prompts ? (task.prompts as unknown as Message[]) : task.reply_messages;
   return (
     <div data-cy="task" data-task-type="evaluate-task">
       <Box mb="4">
@@ -49,13 +62,18 @@ export const EvaluateTask = ({
           <Box mt="4" p="6" borderRadius="lg" bg={cardColor}>
             <MessageConversation messages={messages} highlightLastMessage />
           </Box>
-          <Sortable
-            items={task[sortables]}
-            isDisabled={isDisabled}
-            isEditable={isEditable}
-            onChange={setRanking}
-            className="my-8"
-          />
+          <Box mt="8">
+            <Sortable
+              items={sortableItems}
+              isDisabled={isDisabled}
+              isEditable={isEditable}
+              revealSynthetic={task.reveal_synthetic}
+              onChange={setRanking}
+            />
+            <Checkbox size="lg" mt="4" checked={notRankable} isDisabled={isDisabled} onChange={handleNotRankableChange}>
+              {t("not_rankable")}
+            </Checkbox>
+          </Box>
         </SurveyCard>
       </Box>
     </div>

website/src/types/TaskResponses.ts

@@ -4,6 +4,7 @@ export interface CreateTaskReply {
 
 export interface EvaluateTaskReply {
   ranking: number[];
+  not_rankable: boolean;
 }
 
 export interface LabelTaskReply {

website/src/types/Tasks.ts

@@ -21,18 +21,23 @@ export type CreateTaskType = CreateInitialPromptTask | CreateAssistantReplyTask
 export interface RankInitialPromptsTask extends BaseTask {
   type: TaskType.rank_initial_prompts;
   prompts: string[];
+  reveal_synthetic?: boolean;
 }
 
 export interface RankAssistantRepliesTask extends BaseTask {
   type: TaskType.rank_assistant_replies;
   conversation: Conversation;
   replies: string[];
+  reply_messages: Message[];
+  reveal_synthetic?: boolean;
 }
 
 export interface RankPrompterRepliesTask extends BaseTask {
   type: TaskType.rank_prompter_replies;
   conversation: Conversation;
   replies: string[];
+  reply_messages: Message[]; // not tested
+  reveal_synthetic?: boolean;
 }
 
 export type RankTaskType = RankInitialPromptsTask | RankAssistantRepliesTask | RankPrompterRepliesTask;