Merge branch 'main' into releases/rc-trulens-eval-0.26.0

trulens_eval/trulens_eval/feedback/feedback.py

@@ -727,7 +727,7 @@ def run(
         app: Optional[Union[AppDefinition, JSON]] = None,
         record: Optional[Record] = None,
         source_data: Optional[Dict] = None,
-        **kwargs: dict
+        **kwargs: Dict[str, Any]
     ) -> FeedbackResult:
         """
         Run the feedback function on the given `record`. The `app` that
@@ -786,7 +786,8 @@ def run(
             input_combinations = list(
                 self._extract_selection(
                     source_data=source_data,
-                    combinations=self.combinations
+                    combinations=self.combinations,
+                    **kwargs
                 )
             )
 
@@ -801,8 +802,6 @@ def run(
             multi_result = None
 
             for ins in input_combinations:
-                ins = dict(ins, **kwargs)
-
                 try:
                     result_and_meta, part_cost = Endpoint.track_all_costs_tally(
                         self.imp, **ins
@@ -973,19 +972,45 @@ def name(self) -> str:
     def _extract_selection(
         self,
         source_data: Dict,
-        combinations: FeedbackCombinations = FeedbackCombinations.PRODUCT
+        combinations: FeedbackCombinations = FeedbackCombinations.PRODUCT,
+        **kwargs: Dict[str, Any]
     ) -> Iterable[Dict[str, Any]]:
+        """
+        Create parameter assignments to self.imp from t he given data source or
+        optionally additional kwargs.
+
+        Args:
+            source_data: The data to select from.
+
+            combinations: How to combine assignments for various variables to
+                make an assignment to the while signature.
+
+            **kwargs: Additional keyword arguments to use instead of looking
+                them up from source data. Any parameters specified here will be
+                used as the assignment value and the selector for that paremeter
+                will be ignored.
+        
+        """
 
         arg_vals = {}
 
         for k, q in self.selectors.items():
             try:
-                arg_vals[k] = list(q.get(source_data))
+                if k in kwargs:
+                    arg_vals[k] = [kwargs[k]]
+                else:
+                    arg_vals[k] = list(q.get(source_data))
             except Exception as e:
                 raise RuntimeError(
                     f"Could not locate {q} in recorded data."
                 ) from e
 
+        # For anything specified in kwargs that did not have a selector, set the
+        # assignment here as the above loop will have missed it.
+        for k, v in kwargs.items():
+            if k not in self.selectors:
+                arg_vals[k] = [v]
+
         keys = arg_vals.keys()
         vals = arg_vals.values()
 

trulens_eval/trulens_eval/feedback/provider/litellm.py

@@ -100,27 +100,23 @@ def _create_chat_completion(
         **kwargs
     ) -> str:
 
+        completion_args = kwargs
+        completion_args['model'] = self.model_engine
+        completion_args.update(self.completion_args)
+
         if prompt is not None:
-            comp = completion(
-                model=self.model_engine,
-                messages=[{
+            completion_args['messages'] = [{
                     "role": "system",
                     "content": prompt
-                }],
-                **kwargs,
-                **self.completion_args
-            )
+                }]
         elif messages is not None:
-            comp = completion(
-                model=self.model_engine,
-                messages=messages,
-                **kwargs,
-                **self.completion_args
-            )
+            completion_args['messages']=messages
 
         else:
             raise ValueError("`prompt` or `messages` must be specified.")
 
+        comp = completion(**completion_args)
+
         assert isinstance(comp, object)
 
         return comp["choices"][0]["message"]["content"]

trulens_eval/trulens_eval/requirements.optional.txt

@@ -31,9 +31,7 @@ boto3    >= 1.33.6
 botocore >= 1.33.6
 
 # Models API
-litellm     >= 1.11.1, <=1.24.0
-# have to upper bound due to py3.8 incompatibility:
-# https://github.com/BerriAI/litellm/issues/2005
+litellm     >= 1.25.2
 
 # Local models
 transformers  >= 4.10.0