Synthesizer cost tracker breaks golden generation pipeline for custom llms

[pazevedo-hyland]

Describe the bug
Can't generate golden dataset with custom llm.

To Reproduce
Steps to reproduce the behavior:
Generate a golden dataset with custom llm.

from litellm import acompletion, completion
from deepeval.models.base_model import DeepEvalBaseLLM
from pydantic import BaseModel
import instructor


class LiteLLM(DeepEvalBaseLLM):
    def __init__(
        self,
        model="anthropic.claude-3-haiku-20240307-v1:0",
    ):
        self.model = model
        self.client = instructor.from_litellm(completion)

    def load_model(self):
        return self.model

    def generate(self, prompt: str, schema: BaseModel) -> BaseModel:
        messages = [{"content": prompt, "role": "user"}]
        response = self.client.chat.completions.create(
            model=self.model, messages=messages, response_model=schema
        )
        return response

    async def a_generate(self, prompt: str, schema: BaseModel) -> BaseModel:
        return self.generate(prompt, schema)

    def get_model_name(self):
        return "LiteLLM Model"

from deepeval.synthesizer import Synthesizer

synthesizer = Synthesizer(model=LiteLLM())
golds =synthesizer.generate_goldens_from_docs(
    document_paths=["../../data/documents/sample_hr_manual.pdf"],
)

Expected behavior
Get an error related with cost tracker

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[104], [line 4](vscode-notebook-cell:?execution_count=104&line=4)
      [1](vscode-notebook-cell:?execution_count=104&line=1) from deepeval.synthesizer import Synthesizer
      [3](vscode-notebook-cell:?execution_count=104&line=3) synthesizer = Synthesizer(model=LiteLLM())
----> [4](vscode-notebook-cell:?execution_count=104&line=4) golds =synthesizer.generate_goldens_from_docs(
      [5](vscode-notebook-cell:?execution_count=104&line=5)     document_paths=["../../data/documents/sample_hr_manual.pdf"],
      [6](vscode-notebook-cell:?execution_count=104&line=6) )

File ~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:122, in Synthesizer.generate_goldens_from_docs(self, document_paths, include_expected_output, max_goldens_per_context, context_construction_config, _send_data)
    [120](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:120) if self.async_mode:
    [121](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:121)     loop = get_or_create_event_loop()
--> [122](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:122)     goldens = loop.run_until_complete(
    [123](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:123)         self.a_generate_goldens_from_docs(
    [124](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:124)             document_paths=document_paths,
    [125](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:125)             include_expected_output=include_expected_output,
    [126](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:126)             max_goldens_per_context=max_goldens_per_context,
    [127](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:127)             context_construction_config=context_construction_config,
    [128](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:128)         )
    [129](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:129)     )
    [130](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:130) else:
    [131](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:131)     # Generate contexts from provided docs
    [132](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:132)     if self.context_generator is None:

File ~/cin-ml-agent/.venv/lib/python3.12/site-packages/nest_asyncio.py:98, in _patch_loop.<locals>.run_until_complete(self, future)
...
--> [906](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:906)     self.synthesis_cost += cost
    [907](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:907)     data = trimAndLoadJson(res, self)
    [908](https://file+.vscode-resource.vscode-cdn.net/Users/pedro.azevedo/cin-ml-agent/notebooks/evaluations/~/cin-ml-agent/.venv/lib/python3.12/site-packages/deepeval/synthesizer/synthesizer.py:908)     if schema == SyntheticDataList:

TypeError: unsupported operand type(s) for +=: 'NoneType' and 'float'

Screenshots

Desktop (please complete the following information):

• OS: MacOS

Additional context
Version 2.1.6

[21m1n]

my workaround is to change the property of the custom model:

@property
    def __class__(self):
        from deepeval.models import GPTModel  
        return GPTModel

[penguine-ip]

@21m1n @pazevedo-hyland the latest release checks if cost is None first, but the reason why there's a difference between None and 0 is because None means the cost tracking is not supported while 0 means 0

[pazevedo-hyland]

So this issue is handled? (as in it won't throw an exception) @penguine-ip

[penguine-ip]

@pazevedo-hyland i dont think so