From 0b4d199cde3e2cea44f59bdd8bc250238d13015d Mon Sep 17 00:00:00 2001 From: Arnav Singhvi Date: Mon, 6 Nov 2023 13:20:41 -0800 Subject: [PATCH 1/4] included usage logging for openai calls --- dsp/modules/gpt3.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.py index 8f378bfe2c..9ffda9ac1c 100644 --- a/dsp/modules/gpt3.py +++ b/dsp/modules/gpt3.py @@ -1,3 +1,19 @@ +import logging +from logging.handlers import RotatingFileHandler + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(message)s', + handlers=[ + RotatingFileHandler( + 'openai_usage.log', # Filename + maxBytes=10000000, # Max size of a log file before being rotated (10MB) + backupCount=10 # Number of log files to keep + ) + ] +) + import functools import json from typing import Any, Literal, Optional, cast @@ -78,6 +94,13 @@ def __init__( def _openai_client(): return openai + def log_usage(self, response): + """Log the total tokens from the OpenAI API response.""" + usage_data = response.get('usage') + if usage_data: + total_tokens = usage_data.get('total_tokens') + logging.info(f'{total_tokens}') + def basic_request(self, prompt: str, **kwargs) -> OpenAIObject: raw_kwargs = kwargs @@ -150,6 +173,7 @@ def __call__( # kwargs = {**kwargs, "logprobs": 5} response = self.request(prompt, **kwargs) + self.log_usage(response) choices = response["choices"] completed_choices = [c for c in choices if c["finish_reason"] != "length"] From 0548650da39c24a3f930b35cc554f57c9bdd26a7 Mon Sep 17 00:00:00 2001 From: arnavsinghvi11 <54859892+arnavsinghvi11@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:27:19 -0800 Subject: [PATCH 2/4] Update logging to use FileHandler --- dsp/modules/gpt3.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.py index 9ffda9ac1c..55da51588f 100644 --- a/dsp/modules/gpt3.py +++ b/dsp/modules/gpt3.py @@ -6,11 +6,7 @@ level=logging.INFO, format='%(message)s', handlers=[ - RotatingFileHandler( - 'openai_usage.log', # Filename - maxBytes=10000000, # Max size of a log file before being rotated (10MB) - backupCount=10 # Number of log files to keep - ) + logging.FileHandler('openai_usage.log') ] ) From c06a4f38882f40a57a104857c59b100b28aefa55 Mon Sep 17 00:00:00 2001 From: Arnav Singhvi Date: Wed, 8 Nov 2023 00:42:33 -0800 Subject: [PATCH 3/4] added logging support only if enabled in settings --- dsp/modules/gpt3.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.py index 55da51588f..ade2417935 100644 --- a/dsp/modules/gpt3.py +++ b/dsp/modules/gpt3.py @@ -14,6 +14,7 @@ import json from typing import Any, Literal, Optional, cast +import dsp import backoff import openai import openai.error @@ -169,7 +170,8 @@ def __call__( # kwargs = {**kwargs, "logprobs": 5} response = self.request(prompt, **kwargs) - self.log_usage(response) + if dsp.settings.log_openai_usage: + self.log_usage(response) choices = response["choices"] completed_choices = [c for c in choices if c["finish_reason"] != "length"] From 65e74b0705e9fa9a99bfefc77bc70570a4a60d8d Mon Sep 17 00:00:00 2001 From: Arnav Singhvi Date: Wed, 8 Nov 2023 17:04:50 -0800 Subject: [PATCH 4/4] added openai_logging default flag to settings --- dsp/utils/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dsp/utils/settings.py b/dsp/utils/settings.py index b7f001e45d..44dda21d69 100644 --- a/dsp/utils/settings.py +++ b/dsp/utils/settings.py @@ -33,6 +33,7 @@ def __new__(cls): skip_logprobs=False, trace=None, release=0, + log_openai_usage=False, ) cls._instance.__append(config)