From b928c3bb4fd926e8f4eadbdd36ee48a1e2aed13d Mon Sep 17 00:00:00 2001 From: ciroye Date: Fri, 5 Apr 2024 18:35:56 -0500 Subject: [PATCH 01/39] feat: help me first version --- backend/app/api/endpoints/base/context.py | 5 + .../api/endpoints/base/rounduserexample.py | 2 +- .../domain/schemas/base/rounduserexample.py | 3 + backend/app/domain/services/base/context.py | 9 + backend/app/domain/services/base/model.py | 2 + .../services/base/rounduserexampleinfo.py | 6 +- backend/app/domain/services/base/task.py | 3 + backend/app/domain/services/utils/llm.py | 9 +- frontends/web/package-lock.json | 47 +++ frontends/web/package.json | 1 + .../CreateSamples/AnnotationButtonActions.tsx | 7 + .../Contexts/ChatWithInstructions.tsx | 273 +++++++++++++++--- .../AnnotationInterfaces/Contexts/Chatbot.tsx | 2 +- .../Contexts/DescribeImage.tsx | 203 ++++++++----- .../Goals/DoubleMultioptions.tsx | 6 +- .../BasicInputWithSliderSelection.tsx | 42 +++ .../SelectOptionWithSliderSelection.tsx | 2 + .../Inputs/BasicInputWithSlider.tsx | 103 +++++++ .../components/Inputs/DoubleDropDown.tsx | 77 ++--- .../Inputs/SelectOptionWithSlider.tsx | 36 ++- .../components/Modals/SignContractHelpMe.tsx | 205 +++++++++++++ .../pages/CreateSamples/CreateInterface.tsx | 19 +- .../src/new_front/pages/Login/LoginPage.tsx | 8 +- .../web/src/new_front/pages/Task/Test.tsx | 5 +- .../createSamples/annotationUserInputs.ts | 1 + .../utils/creation_interface_options.json | 1 + .../utils/helpers/functions/LoginFunctions.js | 27 +- 27 files changed, 927 insertions(+), 177 deletions(-) create mode 100644 frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/UserInput/BasicInputWithSliderSelection.tsx create mode 100644 frontends/web/src/new_front/components/Inputs/BasicInputWithSlider.tsx create mode 100644 frontends/web/src/new_front/components/Modals/SignContractHelpMe.tsx diff --git a/backend/app/api/endpoints/base/context.py b/backend/app/api/endpoints/base/context.py index 33510b7d..5b141316 100644 --- a/backend/app/api/endpoints/base/context.py +++ b/backend/app/api/endpoints/base/context.py @@ -75,3 +75,8 @@ async def event_generator(): def get_filter_context(model: GetFilterContext): context = ContextService().get_filter_context(model.real_round_id, model.filters) return context + + +@router.post("/get_contexts_from_s3") +def get_contexts_from_s3(artifacts: dict): + return ContextService().get_contexts_from_s3(artifacts) diff --git a/backend/app/api/endpoints/base/rounduserexample.py b/backend/app/api/endpoints/base/rounduserexample.py index 93517ba3..db071b4d 100644 --- a/backend/app/api/endpoints/base/rounduserexample.py +++ b/backend/app/api/endpoints/base/rounduserexample.py @@ -49,5 +49,5 @@ async def increment_counter_examples_submitted_today( @router.post("/redirect_to_third_party_provider", response_model={}) async def redirect_to_third_party_provider(model: RedirectThirdPartyProvider): return RoundUserExampleInfoService().redirect_to_third_party_provider( - model.task_id, model.user_id, model.round_id + model.task_id, model.user_id, model.round_id, model.url ) diff --git a/backend/app/domain/schemas/base/rounduserexample.py b/backend/app/domain/schemas/base/rounduserexample.py index 054b1647..9eafc4c6 100644 --- a/backend/app/domain/schemas/base/rounduserexample.py +++ b/backend/app/domain/schemas/base/rounduserexample.py @@ -1,6 +1,8 @@ # Copyright (c) MLCommons and its affiliates. # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +from typing import Optional + from pydantic import BaseModel @@ -23,3 +25,4 @@ class RedirectThirdPartyProvider(BaseModel): task_id: int user_id: int round_id: int + url: Optional[str] = None diff --git a/backend/app/domain/services/base/context.py b/backend/app/domain/services/base/context.py index 6fbc27c7..2e0e82d4 100644 --- a/backend/app/domain/services/base/context.py +++ b/backend/app/domain/services/base/context.py @@ -244,3 +244,12 @@ def get_filter_context(self, real_round_id: int, filters: dict) -> dict: if context.get(key).lower() == value.lower(): filter_contexts.append(context) return random.choice(filter_contexts) + + def get_contexts_from_s3(self, artifacts: dict): + artifacts = artifacts["artifacts"] + task_code = self.task_service.get_task_code_by_task_id(artifacts["task_id"])[0] + file_name = f"Top_{artifacts['country']}-{artifacts['language']}_Contexts.json" + key = f"{task_code}/{file_name}" + obj = self.s3.get_object(Bucket=self.dataperf_bucket, Key=key) + body = obj["Body"].read() + return json.loads(body) diff --git a/backend/app/domain/services/base/model.py b/backend/app/domain/services/base/model.py index 4320f032..100d0403 100644 --- a/backend/app/domain/services/base/model.py +++ b/backend/app/domain/services/base/model.py @@ -71,6 +71,7 @@ def __init__(self): "google": GoogleProvider(), "replicate": ReplicateProvider(), "huggingface_api": HuggingFaceAPIProvider(), + "openaihm": OpenAIProvider(task_id=56), } def get_model_in_the_loop(self, task_id: str) -> str: @@ -414,6 +415,7 @@ async def conversation_with_buffer_memory( prompt: str, num_answers: int, ): + print("Selected model was", model_name) print("Selected model was", model_name[provider]["model_name"]) responses = [] llm = self.providers[provider] diff --git a/backend/app/domain/services/base/rounduserexampleinfo.py b/backend/app/domain/services/base/rounduserexampleinfo.py index dcf91776..02186175 100644 --- a/backend/app/domain/services/base/rounduserexampleinfo.py +++ b/backend/app/domain/services/base/rounduserexampleinfo.py @@ -102,7 +102,7 @@ def number_of_examples_created(self, round_id: int, user_id: int): return number_of_examples_created[0] def redirect_to_third_party_provider( - self, task_id: int, user_id: int, round_id: int + self, task_id: int, user_id: int, round_id: int, url: str ): user_email = self.user_repository.get_user_email(user_id)[0] print(user_email.split("@")[1]) @@ -124,6 +124,8 @@ def redirect_to_third_party_provider( required_number_of_examples = task_configuration["external_validator"][ "required_number_of_examples" ] - redirect_url = task_configuration["external_validator"]["url"] + redirect_url = task_configuration.get("external_validator", {}).get("url") if number_of_examples_created == required_number_of_examples: + if url: + return url return redirect_url diff --git a/backend/app/domain/services/base/task.py b/backend/app/domain/services/base/task.py index 1827d18d..476b8ff5 100644 --- a/backend/app/domain/services/base/task.py +++ b/backend/app/domain/services/base/task.py @@ -35,6 +35,9 @@ def __init__(self): self.validation_repository = ValidationRepository() self.historical_task_repository = HistoricalDataRepository() + def get_task_code_by_task_id(self, task_id: int): + return self.task_repository.get_task_code_by_task_id(task_id) + def update_last_activity_date(self, task_id: int): self.task_repository.update_last_activity_date(task_id) diff --git a/backend/app/domain/services/utils/llm.py b/backend/app/domain/services/utils/llm.py index 64ad1314..e1b02eec 100644 --- a/backend/app/domain/services/utils/llm.py +++ b/backend/app/domain/services/utils/llm.py @@ -53,8 +53,11 @@ def provider_name(self): class OpenAIProvider(LLMProvider): - def __init__(self): - self.api_key = os.getenv("OPENAI") + def __init__(self, task_id: int = -1): + if task_id == 56: + self.api_key = os.getenv("OPENAI_HELPME") + else: + self.api_key = os.getenv("OPENAI") openai.api_key = self.api_key @async_timeout(30) @@ -128,7 +131,7 @@ async def conversational_generation( return "None" def provider_name(self): - return "openai" + return "openaihm" class HuggingFaceProvider(LLMProvider): diff --git a/frontends/web/package-lock.json b/frontends/web/package-lock.json index 827521bf..f99b9f0a 100644 --- a/frontends/web/package-lock.json +++ b/frontends/web/package-lock.json @@ -34,6 +34,7 @@ "moment": "^2.24.0", "prop-types": "^15.7.2", "qs": "^6.9.3", + "query-string": "^9.0.0", "react": "^18.2.0", "react-audio-player": "^0.17.0", "react-bootstrap": "^1.2.2", @@ -10878,6 +10879,17 @@ "node": ">=8" } }, + "node_modules/filter-obj": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-5.1.0.tgz", + "integrity": "sha512-qWeTREPoT7I0bifpPUXtxkZJ1XJzxWtfoWWkdVGqa+eCr3SHW/Ocp89o8vLvbUuQnadybJpjOKu4V+RwO6sGng==", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/finalhandler": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", @@ -20779,6 +20791,30 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/query-string": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-9.0.0.tgz", + "integrity": "sha512-4EWwcRGsO2H+yzq6ddHcVqkCQ2EFUSfDMEjF8ryp8ReymyZhIuaFRGLomeOQLkrzacMHoyky2HW0Qe30UbzkKw==", + "dependencies": { + "decode-uri-component": "^0.4.1", + "filter-obj": "^5.1.0", + "split-on-first": "^3.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/query-string/node_modules/decode-uri-component": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.4.1.tgz", + "integrity": "sha512-+8VxcR21HhTy8nOt6jf20w0c9CADrw1O8d+VZ/YzzCt4bJ3uBjw+D1q2osAB8RnpwwaeYBxy0HyKQxD5JBMuuQ==", + "engines": { + "node": ">=14.16" + } + }, "node_modules/querystringify": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", @@ -25030,6 +25066,17 @@ "specificity": "bin/specificity" } }, + "node_modules/split-on-first": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-3.0.0.tgz", + "integrity": "sha512-qxQJTx2ryR0Dw0ITYyekNQWpz6f8dGd7vffGNflQQ3Iqj9NJ6qiZ7ELpZsJ/QBhIVAiDfXdag3+Gp8RvWa62AA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", diff --git a/frontends/web/package.json b/frontends/web/package.json index 0dd34c25..de8a8c1b 100644 --- a/frontends/web/package.json +++ b/frontends/web/package.json @@ -29,6 +29,7 @@ "moment": "^2.24.0", "prop-types": "^15.7.2", "qs": "^6.9.3", + "query-string": "^9.0.0", "react": "^18.2.0", "react-audio-player": "^0.17.0", "react-bootstrap": "^1.2.2", diff --git a/frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationButtonActions.tsx b/frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationButtonActions.tsx index f1832312..2d81c769 100644 --- a/frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationButtonActions.tsx +++ b/frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationButtonActions.tsx @@ -76,6 +76,8 @@ const AnnotationButtonActions: FC = ({ model_evaluation_metric_info: modelEvaluationMetricInfo, model_metadata: metadataExample, }; + console.log("finaModelInputs", finaModelInputs); + return null; if (partialSampleId === 0) { const modelOutput = await post( `/model/single_model_prediction_submit`, @@ -138,6 +140,11 @@ const AnnotationButtonActions: FC = ({ } }; + useEffect(() => { + console.log("modelInputs", modelInputs); + console.log("metadataExample", metadataExample); + }, [modelInputs, metadataExample]); + return ( <> {!loading ? ( diff --git a/frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/Contexts/ChatWithInstructions.tsx b/frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/Contexts/ChatWithInstructions.tsx index 13b7784f..0fa44845 100644 --- a/frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/Contexts/ChatWithInstructions.tsx +++ b/frontends/web/src/new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/Contexts/ChatWithInstructions.tsx @@ -1,5 +1,5 @@ import React, { FC, useContext, useEffect, useState } from "react"; -import SignContract from "new_front/components/Modals/SignContract"; +import SignContractHelpMe from "new_front/components/Modals/SignContractHelpMe"; import Chatbot from "new_front/components/CreateSamples/CreateSamples/AnnotationInterfaces/Contexts/Chatbot"; import useFetch from "use-http"; import { ContextAnnotationFactoryType } from "new_front/types/createSamples/createSamples/annotationFactory"; @@ -10,39 +10,61 @@ import { ChatHistoryType } from "new_front/types/createSamples/createSamples/uti import { CreateInterfaceContext } from "new_front/context/CreateInterface/Context"; import BasicInstructions from "new_front/components/Inputs/BasicInstructions"; import GeneralButton from "new_front/components/Buttons/GeneralButton"; +import { useLocation } from "react-router-dom"; +import queryString from "query-string"; +import Swal from "sweetalert2"; + +enum TreatmentId { + Llama = "1", + GPT = "2", + Control = "3", +} const ChatWithInstructions: FC< ContextAnnotationFactoryType & ContextConfigType -> = ({ taskId, generative_context, setIsGenerativeContext, context }) => { +> = ({ + taskId, + generative_context, + setIsGenerativeContext, + context, + realRoundId, +}) => { const [signInConsent, setSignInConsent] = useState(false); - const { post, response } = useFetch(); const [chatHistory, setChatHistory] = useState({ user: [], bot: [], }); - const [modelName, setModelName] = useState({ - openai: { - model_name: "gpt-3.5-turbo", - frequency_penalty: 0, - presence_penalty: 0, - temperature: 1, - top_p: 1, - max_tokens: 256, - templates: { - header: - "You are a conversational assistant. Limit your answers to around 50 words. Do not refer to your word limit.", - footer: "", - }, - }, - }); - const [provider, setProvider] = useState("openai"); + const [modelName, setModelName] = useState({}); + const [provider, setProvider] = useState(""); const [artifactsInput, setArtifactsInput] = useState( generative_context.artifacts, ); + const [controlText, setControlText] = useState(""); const [finishConversation, setFinishConversation] = useState(false); const [readInstructions, setReadInstructions] = useState(false); const { updateModelInputs } = useContext(CreateInterfaceContext); + const [treatmentValue, setTreatmentValue] = useState(""); + const { post, response } = useFetch(); const { user } = useContext(UserContext); + const location = useLocation(); + const queryParams = queryString.parse(location.search); + let treatmentId = queryParams.treatmentId; + let prolificId = queryParams.assignmentId; + + function getTreatmentValue(treatmentId: TreatmentId): string { + switch (treatmentId) { + case TreatmentId.Llama: + setProvider("huggingface_api"); + return "huggingface_api"; + case TreatmentId.GPT: + setProvider("openaihm"); + return "openaihm"; + case TreatmentId.Control: + return "control"; + default: + throw new Error("Invalid treatment ID"); + } + } const checkIfUserIsSignedInConsent = async () => { const signConsent = await post("/task/check_signed_consent", { @@ -63,20 +85,154 @@ const ChatWithInstructions: FC< window.location.reload(); }; + const checkIfUserReachedNecessaryExamples = async () => { + const redirectUrl = await post( + "/rounduserexample/redirect_to_third_party_provider", + { + task_id: taskId, + user_id: user.id, + round_id: realRoundId, + url: `https://oii.qualtrics.com/jfe/form/SV_3rt2Z0hbvyocuMu?prolific_id=${prolificId}`, + }, + ); + if (response.ok) { + if (redirectUrl) { + Swal.fire({ + title: "You have reached the necessary examples", + text: "You will be redirected to the third party provider", + icon: "success", + confirmButtonText: "Ok", + }).then(() => { + window.location.href = redirectUrl; + }); + } + } + }; + + useEffect(() => { + console.log("treatmentId", treatmentId); + + if (treatmentId) { + setTreatmentValue(getTreatmentValue(treatmentId as TreatmentId)); + if (getTreatmentValue(treatmentId as TreatmentId) !== "control") { + const modelConfig = + // @ts-ignore + generative_context.artifacts.providers[ + getTreatmentValue(treatmentId as TreatmentId) + ][0]; + console.log("modelConfig", modelConfig); + + setModelName({ + [getTreatmentValue(treatmentId as TreatmentId)]: modelConfig, + }); + } + } + }, [treatmentId]); + useEffect(() => { checkIfUserIsSignedInConsent(); }, [signInConsent]); + useEffect(() => { + checkIfUserReachedNecessaryExamples(); + }, []); + return ( <> {signInConsent ? ( <> {!readInstructions ? (
- +
+
+

+ General Instructions for all participants +

+

+ In this study, you will be asked to complete two different + scenarios which simulate healthcare scenarios that a person + might encounter in everyday life... +

+ {treatmentValue !== "control" ? ( + <> +

+ Instructions for LLM treatments: +

+

+ The health scenario for this task is presented in the + right side panel. Please use the chatbot interface on + the left side to help you in deciding what to do in this + scenario... +

+

+ The questions you will be asked are: 1) “What should you + do next?” with the options Treat at Home, Call your GP, + Call 111, and Call 999; and 2) “What is the most likely + cause of the problems being reported?” which is free + response. +

+ + ) : ( + <> +

+ Instructions for control treatment: +

+

+ The health scenario for this task is presented in the + panel below. Please use any resource you would + ordinarily use to decide what to do in a health + scenario... +

+

+ The questions you will be asked are: 1) “What should you + do next?” with the options Treat at Home, Call your GP, + Call 111, and Call 999; and 2) “What is the most likely + cause of the problems being reported?” which is free + response. +

+ + )} +
+
+ {treatmentValue !== "control" && ( +
+

+ Brief Introduction to Large Language Models +

+

+ In this study, participants will interact with a large + language model to reach a decision about a healthcare + scenario. +

+

+ Large language models (LLMs) are a recent technology in the + field of artificial intelligence which can generate text. + They often take the form of a chatbot, which holds + “conversations” with users by responding to written prompts. + The most famous example is ChatGPT. +

+

+ The LLM used in this experiment also uses a chat interface. + You can type in sentences, such as questions about the + scenario, and the LLM will generate new text in response. + Your previous statements will be included as context, so + that each response reflects the whole conversation up to + that point. +

+
+ )} +
+ +
setReadInstructions(true)} className="border-0 font-weight-bold light-gray-bg task-action-btn" /> @@ -85,26 +241,63 @@ const ChatWithInstructions: FC< ) : (
- {}} - setFinishConversation={setFinishConversation} - updateModelInputs={updateModelInputs} - setIsGenerativeContext={setIsGenerativeContext} - /> + {treatmentValue !== "control" ? ( + {}} + setFinishConversation={setFinishConversation} + updateModelInputs={updateModelInputs} + setIsGenerativeContext={setIsGenerativeContext} + /> + ) : ( + <> +

+ Now use the any methods you would ordinarily use at home + to determine the best response to the scenario. The + scenario details are available for reference below. +

+