Not able to achieve `Reproducible outputs` with the seed parameter (SDK version 1.1.1)

[ycrouin]

Hi 👋,

I've noticed the exciting new capability for achieving Reproducible outputs.
Nevertheless, I'm encountering varying results when using identical input parameters and getting the same system fingerprint.

Am I missing something?

Python code to reproduce this issue (might need a few runs)

import openai
from openai.types.chat import ChatCompletionUserMessageParam

print(openai.__version__)
print()

for i in range(5):
    message = ChatCompletionUserMessageParam(
        content=(
            "Imagine something completely random and crazy! (approximately 100 words)"
        ),
        role="user",
    )
    response = openai.chat.completions.create(
        messages=[message], model="gpt-3.5-turbo-1106", temperature=0, seed=42
    )
    print(response.system_fingerprint, response.choices[0].message.content)

Versions

OpenAI SDK 1.1.1
Python version 3.11.5

[ycrouin]

I just checked the cookbook again, and it is mentioned that

If the seed, request parameters, and system_fingerprint all match across your requests, then model outputs will mostly be identical. There is a small chance that responses differ even when request parameters and system_fingerprint match, due to the inherent non-determinism of computers.

I guess the behavior is expected then 🤷‍♂️

[SovereignRemedy]

I just checked the cookbook again, and it is mentioned that

If the seed, request parameters, and system_fingerprint all match across your requests, then model outputs will mostly be identical. There is a small chance that responses differ even when request parameters and system_fingerprint match, due to the inherent non-determinism of computers.

I guess the behavior is expected then 🤷‍♂️

When I test it self -test, there are many answers to the question when they generate less token questions. When I answer more token questions, the questions still have not the same answer.I don't know if it is related to Temperature and TOP_P.

this is my code By borrowed cookbook,If you can get a consistent result, what is the problem with my code

import asyncio
import os

import openai
import pprint
import difflib
from IPython.display import display, HTML

GPT_MODEL = "gpt-3.5-turbo-1106"


async def get_chat_response(system_message: str, user_request: str, seed: int = None):
    try:
        messages = [
            {"role": "system", "content": system_message},
            {"role": "user", "content": user_request},
        ]
        openai.api_key = os.getenv("OPENAI_API_KEY")
        openai.proxy = ""
        response = openai.ChatCompletion.create(
            model=GPT_MODEL,
            messages=messages,
            seed=seed,
            max_tokens=400,
            temperature=0.1,
            top_p=1,
        )

        response_content = response["choices"][0]["message"]["content"]
        system_fingerprint = response["system_fingerprint"]
        prompt_tokens = response["usage"]["prompt_tokens"]
        completion_tokens = (
                response["usage"]["total_tokens"] - response["usage"]["prompt_tokens"]
        )
        print(response_content)
        print(system_fingerprint)
        table = f"""
        <table>
        <tr><th>Response</th><td>{response_content}</td></tr>
        <tr><th>System Fingerprint</th><td>{system_fingerprint}</td></tr>
        <tr><th>Number of prompt tokens</th><td>{prompt_tokens}</td></tr>
        <tr><th>Number of completion tokens</th><td>{completion_tokens}</td></tr>
        </table>
        """
        # display(HTML(table))

        return response_content
    except Exception as e:
        print(f"An error occurred: {e}")
        return None


# This function compares two responses and displays the differences in a table.
# Deletions are highlighted in red and additions are highlighted in green.
# If no differences are found, it prints "No differences found."


def compare_responses(previous_response: str, response: str):
    d = difflib.Differ()
    diff = d.compare(previous_response.splitlines(), response.splitlines())

    diff_table = "<table>"
    diff_exists = False

    for line in diff:
        if line.startswith("- "):
            diff_table += f"<tr style='color: red;'><td>{line}</td></tr>"
            diff_exists = True
        elif line.startswith("+ "):
            diff_table += f"<tr style='color: green;'><td>{line}</td></tr>"
            diff_exists = True
        else:
            diff_table += f"<tr><td>{line}</td></tr>"

    diff_table += "</table>"

    if diff_exists:
        display(HTML(diff_table))
    else:
        print("No differences found.")



async def main():
    SEED = 3211212
    system_message = "You are a helpful assistant that generates question."
    user_request = f"Generate some short question about China"

    response = await get_chat_response(
        system_message=system_message, seed=SEED, user_request=user_request
    )
    previous_response = response
    response = await get_chat_response(
        system_message=system_message, seed=SEED, user_request=user_request
    )

    # compare_responses(previous_response, response)

# Run the main function
asyncio.run(main())