diff --git a/README.md b/README.md index 52220306..f050e978 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,6 @@ if st.session_state.logged_prompt: model=st.session_state.logged_prompt.config_model.model, prompt_id=st.session_state.logged_prompt.id, align="flex-start", - single_submit=False ) ``` diff --git a/docs/integrations/streamlit.md b/docs/integrations/streamlit.md index 78db704a..d429d5ec 100644 --- a/docs/integrations/streamlit.md +++ b/docs/integrations/streamlit.md @@ -90,7 +90,6 @@ if st.session_state.logged_prompt: model=st.session_state.logged_prompt.config_model.model, prompt_id=st.session_state.logged_prompt.id, align="flex-start", - single_submit=False ) ``` diff --git a/examples/streamlit/basic_app.py b/examples/streamlit/basic_app.py index 5f44492f..61f9bc26 100644 --- a/examples/streamlit/basic_app.py +++ b/examples/streamlit/basic_app.py @@ -23,5 +23,4 @@ model=st.session_state.logged_prompt.config_model.model, prompt_id=st.session_state.logged_prompt.id, align="flex-start", - single_submit=False, ) diff --git a/examples/streamlit/llm_app.py b/examples/streamlit/llm_app.py index 3ef380d2..8d79bdcb 100644 --- a/examples/streamlit/llm_app.py +++ b/examples/streamlit/llm_app.py @@ -6,6 +6,8 @@ if "response" not in st.session_state: st.session_state.response = "" +if "feedback_key" not in st.session_state: + st.session_state.feedback_key = 0 if "logged_prompt" not in st.session_state: st.session_state.logged_prompt = "" @@ -15,13 +17,17 @@ email, password = trubrics_config() if email and password: - collector = FeedbackCollector( - project="default", - email=email, - password=password, - ) + try: + collector = FeedbackCollector(email=email, password=password, project="default") + except Exception: + st.error(f"Error authenticating '{email}' with [Trubrics](https://trubrics.streamlit.app/). Please try again.") + st.stop() else: - st.warning("To save some feedback to Trubrics, add your account details in the sidebar.") + st.info( + "To ask a question to an LLM and save your feedback to Trubrics, add your email and password in the sidebar." + " Don't have an account yet? Create one for free [here](https://trubrics.streamlit.app/)!" + ) + st.stop() models = ("gpt-3.5-turbo",) model = st.selectbox( @@ -32,7 +38,8 @@ openai.api_key = st.secrets.get("OPENAI_API_KEY") if openai.api_key is None: - raise ValueError("OpenAI key is missing. Set OPENAI_API_KEY in st.secrets") + st.info("Please add your OpenAI API key to continue.") + st.stop() prompt = st.text_area(label="Prompt", label_visibility="collapsed", placeholder="What would you like to know?") button = st.button(f"Ask {model}") @@ -41,9 +48,10 @@ response = openai.ChatCompletion.create(model=model, messages=[{"role": "user", "content": prompt}]) response_text = response.choices[0].message["content"] st.session_state.logged_prompt = collector.log_prompt( - config_model={"model": model}, prompt=prompt, generation=response_text, tags=["llm_app.py"] + config_model={"model": model}, prompt=prompt, generation=response_text, tags=["llm_app.py"], user_id=email ) st.session_state.response = response_text + st.session_state.feedback_key += 1 if st.session_state.response: st.markdown(f"#### :violet[{st.session_state.response}]") @@ -55,8 +63,9 @@ prompt_id=st.session_state.logged_prompt.id, model=model, align="flex-start", - single_submit=False, tags=["llm_app.py"], + key=f"feedback_{st.session_state.feedback_key}", # overwrite with new key + user_id=email, ) if feedback: diff --git a/examples/streamlit/llm_chatbot.py b/examples/streamlit/llm_chatbot.py index 94ddd82e..fd87bc6e 100644 --- a/examples/streamlit/llm_chatbot.py +++ b/examples/streamlit/llm_chatbot.py @@ -6,20 +6,31 @@ from trubrics.integrations.streamlit import FeedbackCollector +st.title("💬 Trubrics - Chat with user feedback") + with st.sidebar: email, password = trubrics_config() +if not email or not password: + st.info( + "To chat with an LLM and save your feedback to Trubrics, add your email and password in the sidebar." + " Don't have an account yet? Create one for free [here](https://trubrics.streamlit.app/)!" + ) + st.stop() + @st.cache_data def init_trubrics(email, password): - collector = FeedbackCollector(email=email, password=password, project="default") - return collector + try: + collector = FeedbackCollector(email=email, password=password, project="default") + return collector + except Exception: + st.error(f"Error authenticating '{email}' with [Trubrics](https://trubrics.streamlit.app/). Please try again.") + st.stop() collector = init_trubrics(email, password) - -st.title("💬 Trubrics - Chat with user feedback") if "messages" not in st.session_state: st.session_state["messages"] = [{"role": "assistant", "content": "How can I help you?"}] if "prompt_ids" not in st.session_state: @@ -30,46 +41,66 @@ def init_trubrics(email, password): model = st.secrets.get("OPENAI_API_MODEL") or "gpt-3.5-turbo" openai_api_key = st.secrets.get("OPENAI_API_KEY") + +messages = st.session_state.messages +feedback_kwargs = { + "component": "default", + "feedback_type": "thumbs", + "open_feedback_label": "[Optional] Provide additional feedback", + "model": model, + "tags": ["llm_chatbot.py"], +} + +for n, msg in enumerate(messages): + st.chat_message(msg["role"]).write(msg["content"]) + + if msg["role"] == "assistant" and n > 1: + feedback_key = f"feedback_{int(n/2)}" + + if feedback_key not in st.session_state: + st.session_state[feedback_key] = None + + disable_with_score = st.session_state[feedback_key].get("score") if st.session_state[feedback_key] else None + feedback = collector.st_feedback( + **feedback_kwargs, + key=feedback_key, + disable_with_score=disable_with_score, + prompt_id=st.session_state.prompt_ids[int(n / 2) - 1], + user_id=email, + ) + if feedback: + trubrics_successful_feedback(feedback) + + if prompt := st.chat_input(): + messages.append({"role": "user", "content": prompt}) + st.chat_message("user").write(prompt) if not openai_api_key: st.info("Please add your OpenAI API key to continue.") st.stop() else: openai.api_key = openai_api_key - - st.session_state.messages.append({"role": "user", "content": prompt}) - response = openai.ChatCompletion.create(model=model, messages=st.session_state.messages) - msg = response.choices[0].message - logged_prompt = collector.log_prompt( - config_model={"model": model}, - prompt=prompt, - generation=msg["content"], - session_id=st.session_state.session_id, - tags=["llm_chatbot.py"], + response = openai.ChatCompletion.create(model=model, messages=messages) + generation = response.choices[0].message.content + + with st.chat_message("assistant"): + logged_prompt = collector.log_prompt( + config_model={"model": model}, + prompt=prompt, + generation=generation, + session_id=st.session_state.session_id, + tags=["llm_chatbot.py"], + user_id=email, + ) + st.session_state.prompt_ids.append(logged_prompt.id) + messages.append({"role": "assistant", "content": generation}) + st.write(generation) + + feedback = collector.st_feedback( + **feedback_kwargs, + key=f"feedback_{int(len(messages)/2)}", + prompt_id=st.session_state.prompt_ids[int(len(messages) / 2) - 1], + user_id=email, ) - st.session_state.prompt_ids.append(logged_prompt.id) - st.session_state.messages.append(msg) - -feedback = None -for n, msg in enumerate(st.session_state.messages): - st.chat_message(msg["role"]).write(msg["content"]) - - if msg["role"] == "assistant" and msg["content"] != "How can I help you?": - if email and password: - feedback = collector.st_feedback( - component="default", - feedback_type="thumbs", - model=model, - prompt_id=st.session_state.prompt_ids[int(n / 2) - 1], - open_feedback_label="[Optional] Provide additional feedback", - align="flex-end", - single_submit=True, - key=f"feedback_{int(n/2)}", - tags=["llm_chatbot.py"], - ) - - else: - st.warning("To save some feedback to Trubrics, add your account details in the sidebar.") - -if feedback: - trubrics_successful_feedback(feedback) + if feedback: + trubrics_successful_feedback(feedback) diff --git a/examples/streamlit/llm_chatbot_stream.py b/examples/streamlit/llm_chatbot_stream.py new file mode 100644 index 00000000..f7e5222b --- /dev/null +++ b/examples/streamlit/llm_chatbot_stream.py @@ -0,0 +1,112 @@ +import uuid + +import openai +import streamlit as st +from trubrics_utils import trubrics_config, trubrics_successful_feedback + +from trubrics.integrations.streamlit import FeedbackCollector + +st.title("💬 Trubrics - Streaming chat with user feedback") + +with st.sidebar: + email, password = trubrics_config() + +if not email or not password: + st.info( + "To chat with an LLM and save your feedback to Trubrics, add your email and password in the sidebar." + " Don't have an account yet? Create one for free [here](https://trubrics.streamlit.app/)!" + ) + st.stop() + + +@st.cache_data +def init_trubrics(email, password): + try: + collector = FeedbackCollector(email=email, password=password, project="default") + return collector + except Exception: + st.error(f"Error authenticating '{email}' with [Trubrics](https://trubrics.streamlit.app/). Please try again.") + st.stop() + + +collector = init_trubrics(email, password) + +if "messages" not in st.session_state: + st.session_state["messages"] = [{"role": "assistant", "content": "How can I help you?"}] +if "prompt_ids" not in st.session_state: + st.session_state["prompt_ids"] = [] +if "session_id" not in st.session_state: + st.session_state["session_id"] = str(uuid.uuid4()) + +model = st.secrets.get("OPENAI_API_MODEL") or "gpt-3.5-turbo" + +openai_api_key = st.secrets.get("OPENAI_API_KEY") + +messages = st.session_state.messages +feedback_kwargs = { + "component": "default", + "feedback_type": "thumbs", + "open_feedback_label": "[Optional] Provide additional feedback", + "model": model, + "tags": ["llm_chatbot.py"], +} + +for n, msg in enumerate(st.session_state.messages): + st.chat_message(msg["role"]).write(msg["content"]) + + if msg["role"] == "assistant" and n > 1: + feedback_key = f"feedback_{int(n/2)}" + + if feedback_key not in st.session_state: + st.session_state[feedback_key] = None + + disable_with_score = st.session_state[feedback_key].get("score") if st.session_state[feedback_key] else None + feedback = collector.st_feedback( + **feedback_kwargs, + key=feedback_key, + disable_with_score=disable_with_score, + prompt_id=st.session_state.prompt_ids[int(n / 2) - 1], + user_id=email, + ) + if feedback: + trubrics_successful_feedback(feedback) + +if prompt := st.chat_input("What is up?"): + st.session_state.messages.append({"role": "user", "content": prompt}) + with st.chat_message("user"): + st.markdown(prompt) + with st.chat_message("assistant"): + message_placeholder = st.empty() + full_response = "" + if not openai_api_key: + st.info("Please add your OpenAI API key to continue.") + st.stop() + else: + openai.api_key = openai_api_key + + for response in openai.ChatCompletion.create( + model=model, + messages=[{"role": m["role"], "content": m["content"]} for m in st.session_state.messages], + stream=True, + ): + full_response += response.choices[0].delta.get("content", "") + message_placeholder.markdown(full_response + "▌") + message_placeholder.markdown(full_response) + st.session_state.messages.append({"role": "assistant", "content": full_response}) + logged_prompt = collector.log_prompt( + config_model={"model": model}, + prompt=prompt, + generation=full_response, + session_id=st.session_state.session_id, + tags=["llm_chatbot.py"], + user_id=email, + ) + st.session_state.prompt_ids.append(logged_prompt.id) + feedback = collector.st_feedback( + **feedback_kwargs, + key=f"feedback_{int(len(messages)/2)}", + prompt_id=logged_prompt.id, + user_id=email, + ) + if feedback: + trubrics_successful_feedback(feedback) diff --git a/examples/streamlit/titanic_app.py b/examples/streamlit/titanic_app.py index 775b9471..000a9403 100644 --- a/examples/streamlit/titanic_app.py +++ b/examples/streamlit/titanic_app.py @@ -62,7 +62,6 @@ def init_trubrics(): open_feedback_label="[Optional] Provide additional feedback", metadata=metadata, align="flex-start", - single_submit=False, ) if feedback: trubrics_successful_feedback(feedback) diff --git a/examples/streamlit/trubrics_utils.py b/examples/streamlit/trubrics_utils.py index 15477864..3b00eac2 100644 --- a/examples/streamlit/trubrics_utils.py +++ b/examples/streamlit/trubrics_utils.py @@ -14,7 +14,6 @@ def trubrics_config(default_component: bool = True): type="password", value=st.secrets.get("TRUBRICS_PASSWORD", ""), ) - st.write("Don't have an account yet? Create one [here](https://trubrics.streamlit.app/)!") if default_component: return email, password diff --git a/mkdocs.yml b/mkdocs.yml index 3bffc4b1..82d270d0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ site_name: trubrics-sdk -site_description: Combine data science knowledge with business user feedback to validate machine learning +site_description: The first user analytics platform for AI. repo_url: https://github.com/trubrics/trubrics-sdk.git repo_name: trubrics/trubrics-sdk plugins: diff --git a/requirements-dev.txt b/requirements-dev.txt index 351618d4..fe073801 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -24,7 +24,7 @@ pytest-cov>=3.0.0 #extras (Note: in setup.cfg) streamlit>=1.21.0 -streamlit-feedback>=0.0.9 +streamlit-feedback==0.1.2 numpy>=1.21.6 pandas>=1.3.5 scikit-learn>=1.0.0,<1.1.0 diff --git a/setup.cfg b/setup.cfg index 935980f8..3bc222fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,4 +30,4 @@ profile = black # integrations [options.extras_require] -streamlit = streamlit>=1.20.0; streamlit-feedback>=0.0.9 +streamlit = streamlit>=1.20.0; streamlit-feedback==0.1.2 diff --git a/tests/test_feedback/test_feedback_streamlit.py b/tests/test_feedback/test_feedback_streamlit.py index c1729849..ad1feb50 100644 --- a/tests/test_feedback/test_feedback_streamlit.py +++ b/tests/test_feedback/test_feedback_streamlit.py @@ -7,10 +7,7 @@ "kwargs", [ ({"feedback_type": "random"}), - ({"feedback_type": "issue", "user_response": {"test": "test"}}), ({"feedback_type": "issue", "open_feedback_label": "test"}), - ({"feedback_type": "faces", "user_response": "desc"}), - ({"feedback_type": "thumbs", "user_response": "desc"}), ], ) def test_st_feedback_raises(kwargs): diff --git a/trubrics/integrations/streamlit/collect.py b/trubrics/integrations/streamlit/collect.py index 2e6756a3..dfb229e1 100644 --- a/trubrics/integrations/streamlit/collect.py +++ b/trubrics/integrations/streamlit/collect.py @@ -40,13 +40,12 @@ def st_feedback( prompt_id: Optional[str] = None, tags: list = [], metadata: dict = {}, - user_response: Optional[dict] = None, user_id: Optional[str] = None, key: Optional[str] = None, open_feedback_label: Optional[str] = None, save_to_trubrics: bool = True, align: str = "flex-end", - single_submit: bool = True, + disable_with_score: Optional[str] = None, success_fail_message: bool = True, ) -> Optional[dict]: """ @@ -63,61 +62,79 @@ def st_feedback( prompt_id: id of the prompt object tags: a list of tags for the feedback metadata: any data to save with the feedback - user_response: a dict of user response to save with the feedback for feedback_type='custom' user_id: an optional reference to a user, for example a username if there is a login, a cookie ID, etc - key: a key for the streamlit components (necessary if calling this method multiple times with the same type) + key: a key for the streamlit components (necessary if calling this method multiple times) open_feedback_label: label of optional text_input for "faces" or "thumbs" feedback_type save_to_trubrics: whether to save the feedback to Trubrics, or just to return the feedback object + disable_with_score: an optional score to disable the component. Must be a "thumbs" emoji or a "faces" emoji. + Can be used to pass state from one component to another. align: where to align the feedback component ["flex-end", "center", "flex-start"] - single_submit: disables re-submission. This prevents users re-submitting feedback for a given prediction - e.g. for a chatbot. - success_fail_message: whether to display a st.success / st.error message on feedback submission. + success_fail_message: whether to display an st.toast message on feedback submission. """ if key is None: key = feedback_type if feedback_type == "textbox": - if user_response: - raise ValueError( - "For feedback_type='textbox', user_response is set inside the component (must be None)." - ) text = self.st_textbox_ui(key, label=open_feedback_label) if text: user_response = {"type": feedback_type, "score": None, "text": text} + if save_to_trubrics: + feedback = self.log_feedback( + component=component, + user_response=user_response, + model=model, + prompt_id=prompt_id, + metadata=metadata, + tags=tags, + user_id=user_id, + ) + if feedback is None: + error_msg = "Error in pushing feedback issue to Trubrics." + if success_fail_message: + st.error(error_msg) + else: + if success_fail_message: + st.success("Feedback saved to Trubrics.") + return feedback.model_dump() + else: + user_response = Response(**user_response) + feedback = Feedback( + component=component, + model=model, + user_response=user_response, + prompt_id=prompt_id, + user_id=user_id, + tags=tags, + metadata=metadata, + ) + return feedback.model_dump() elif feedback_type in ("thumbs", "faces"): - if user_response: - raise ValueError( - f"For feedback_type='{feedback_type}', user_response is set inside the component (must be None)." - ) + + def _log_feedback_trubrics(user_response, **kwargs): + feedback = self.log_feedback(user_response=user_response, **kwargs) + if success_fail_message: + if feedback: + st.toast("Feedback saved to [Trubrics](https://trubrics.streamlit.app/).", icon="✅") + return feedback.model_dump() + else: + st.toast("Error in saving feedback to [Trubrics](https://trubrics.streamlit.app/).", icon="❌") + user_response = streamlit_feedback( feedback_type=feedback_type, optional_text_label=open_feedback_label, - single_submit=single_submit, + disable_with_score=disable_with_score, + on_submit=_log_feedback_trubrics if save_to_trubrics else None, + kwargs={ + "component": component, + "model": model, + "prompt_id": prompt_id, + "metadata": metadata, + "tags": tags, + "user_id": user_id, + }, align=align, key=key, ) - else: - raise ValueError("feedback_type must be one of ['textbox', 'faces', 'thumbs'].") - - if user_response: - if save_to_trubrics: - feedback = self.log_feedback( - component=component, - user_response=user_response, - model=model, - prompt_id=prompt_id, - metadata=metadata, - tags=tags, - user_id=user_id, - ) - if feedback is None: - error_msg = "Error in pushing feedback issue to Trubrics." - if success_fail_message: - st.error(error_msg) - else: - if success_fail_message: - st.success("Feedback saved to Trubrics.") - return feedback.model_dump() - else: + if save_to_trubrics is False and user_response: user_response = Response(**user_response) feedback = Feedback( component=component, @@ -129,6 +146,9 @@ def st_feedback( metadata=metadata, ) return feedback.model_dump() + return user_response + else: + raise ValueError("feedback_type must be one of ['textbox', 'faces', 'thumbs'].") return None @staticmethod