add LLMClient parent class

api/valor_api/backend/metrics/llm_call.py

@@ -10,9 +10,95 @@
 2. Assign a score for coherence on a scale of 1 to 5, where 1 is the lowest and 5 is the highest based on the Evaluation Criteria. Respond with just the number 1 to 5."""
 
 
-class OpenAIClient:
+class LLMClient:
     """
-    Wrapper for calls to OpenAI
+    Wrapper for calls to an LLM API.
+
+    Parameters
+    ----------
+    api_key : str, optional
+        The API key to use.
+    model_name : str
+        The model to use.
+    """
+
+    api_key: str | None = None
+    model_name: str
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+    ):
+        """
+        TODO should we use an __attrs_post_init__ instead?
+        """
+        self.api_key = api_key
+
+    def connect(
+        self,
+    ):
+        """
+        TODO This is separated for now because I want to mock connecting to the LLM API.
+        """
+        raise NotImplementedError
+
+    def process_messages(
+        self,
+        messages: list[dict[str, str]],
+    ) -> Any:
+        """
+        All messages should be formatted according to the standard set by OpenAI, and should be modified
+        as needed for other models. This function takes in messages in the OpenAI standard format and converts
+        them to the format required by the model.
+        """
+        raise NotImplementedError
+
+    def __call__(
+        self,
+        messages: list[dict[str, str]],
+    ) -> Any:
+        """
+        Call to the API.
+        """
+        raise NotImplementedError
+
+    def coherence(
+        self,
+        text: str,
+    ) -> int:
+        """
+        Computes coherence for a single piece of text.
+
+        Parameters
+        ----------
+        text : TODO
+
+        Returns
+        -------
+        int
+        """
+        messages = [
+            {"role": "system", "content": COHERENCE_INSTRUCTION},
+            {"role": "user", "content": text},
+        ]
+
+        response = self(messages)
+
+        try:
+            # Valid responses: "5", "\n5", "5\n", "5.", " 5", "5 {explanation}", etc.
+            ret = int(response.strip()[0])
+            assert ret in [1, 2, 3, 4, 5]
+        except Exception:
+            raise ValueError(
+                f"LLM response was not a valid coherence score: {response}"
+            )
+
+        return ret
+
+
+class OpenAIClient(LLMClient):
+    """
+    Wrapper for calls to OpenAI's API.
 
     Parameters
     ----------
@@ -25,7 +111,7 @@ class OpenAIClient:
     """
 
     # url: str
-    api_key: str | None = None  # If api_key is not specified, then OPENAI_API_KEY environment variable will be used.
+    api_key: str | None = None
     seed: int | None = None
     model_name: str = "gpt-3.5-turbo"  # gpt-3.5-turbo gpt-4-turbo
 
@@ -51,16 +137,32 @@ def connect(
         else:
             self.client = openai.OpenAI(api_key=self.api_key)
 
+    def process_messages(
+        self,
+        messages: list[dict[str, str]],
+    ) -> Any:
+        """
+        All messages should be formatted according to the standard set by OpenAI, and should be modified
+        as needed for other models. This function takes in messages in the OpenAI standard format and converts
+        them to the format required by the model.
+        """
+        return messages
+
     def __call__(
         self,
-        messages: list[dict],
+        messages: list[dict[str, str]],
     ) -> Any:
         """
+        Call to the API.
+
         TODO possibly change this to a call with the API. This would remove the openai python dependence.
         """
+        processed_messages = self.process_messages(messages)
         try:
             openai_response = self.client.chat.completions.create(
-                model=self.model_name, messages=messages, seed=self.seed
+                model=self.model_name,
+                messages=processed_messages,
+                seed=self.seed,
             )
         # TODO should we both catching this if we aren't going to do anything?
         except openai.BadRequestError as e:
@@ -80,36 +182,3 @@ def __call__(
             )
 
         return response
-
-    def coherence(
-        self,
-        text: str,
-    ) -> int:
-        """
-        Computes coherence for a single piece of text.
-
-        Parameters
-        ----------
-        text : TODO
-
-        Returns
-        -------
-        int
-        """
-        messages = [
-            {"role": "system", "content": COHERENCE_INSTRUCTION},
-            {"role": "user", "content": text},
-        ]
-
-        response = self(messages)
-        # TODO Should we take just the first word of the response, in case GPT outputs a number followed by an explanation?
-
-        try:
-            response = int(response)
-            assert response in [1, 2, 3, 4, 5]
-        except Exception:
-            raise ValueError(
-                f"OpenAI response was not a valid coherence score: {response}"
-            )
-
-        return response

examples/text-generation/text_generation.ipynb

@@ -431,7 +431,7 @@
     "    llm_api_params = {\n",
     "        \"api_url\":\"https://api.openai.com/v1/chat/completions\",\n",
     "        # TODO should be post request json payload for encryption\n",
-    "        # api_key:None, # If no key is specified, uses OPENAI_API_KEY environment variable for OpenAI API calls. Needs to be passed as bearer token?\n",
+    "        # api_key=os.environ[\"OPENAI_API_KEY\"], # If no key is specified, uses OPENAI_API_KEY environment variable for OpenAI API calls. Needs to be passed as bearer token?\n",
     "        \"data\":{\n",
     "            \"model\":\"gpt-4o\",\n",
     "        },\n",
@@ -911,7 +911,7 @@
     "    metrics=[\"SentenceBLEU\", \"ROUGE1\", \"LDistance\", \"Toxicity\", \"Summarization\"],\n",
     "    llm_api_params = {\n",
     "        \"api_url\":\"https://api.openai.com/v1/chat/completions\",\n",
-    "        # api_key:None, # If no key is specified, uses OPENAI_API_KEY environment variable for OpenAI API calls. Needs to be passed as bearer token?\n",
+    "        # api_key=os.environ[\"OPENAI_API_KEY\"], # If no key is specified, uses OPENAI_API_KEY environment variable for OpenAI API calls. Needs to be passed as bearer token?\n",
     "        \"data\":{\n",
     "            \"model\":\"gpt-4o\",\n",
     "        },\n",
@@ -1201,7 +1201,7 @@
     "    metrics=[\"Coherence\", \"Toxicity\", \"Bias\"],\n",
     "    llm_api_params = {\n",
     "        \"api_url\":\"https://api.openai.com/v1/chat/completions\",\n",
-    "        # api_key:None, # If no key is specified, uses OPENAI_API_KEY environment variable for OpenAI API calls. Needs to be passed as bearer token?\n",
+    "        # api_key=os.environ[\"OPENAI_API_KEY\"], # If no key is specified, uses OPENAI_API_KEY environment variable for OpenAI API calls. Needs to be passed as bearer token?\n",
     "        \"data\":{\n",
     "            \"model\":\"gpt-4o\",\n",
     "        },\n",