From ebf434e8d6e40237e42dff35bc0540bdef144f0e Mon Sep 17 00:00:00 2001 From: ramitsurana Date: Sat, 17 Apr 2021 01:41:20 +0530 Subject: [PATCH 1/4] add new api for counting tokens --- openai/api_resources/tokens.py | 16 ++++++++++++++++ openai/cli.py | 13 ++++++++++++- openai/common_constants.py | 5 +++++ openai/util.py | 8 +++++++- 4 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 openai/api_resources/tokens.py create mode 100644 openai/common_constants.py diff --git a/openai/api_resources/tokens.py b/openai/api_resources/tokens.py new file mode 100644 index 0000000000..f2c1991395 --- /dev/null +++ b/openai/api_resources/tokens.py @@ -0,0 +1,16 @@ +from openai.api_resources.abstract import ( + ListableAPIResource, + DeletableAPIResource, +) +from openai import util + + +class CountTokens(ListableAPIResource, DeletableAPIResource): + + @classmethod + def count_tokens( + cls, api_key=None, api_base=None, api_version=None, organization=None, **params + ): + return util.check_tokens( + params + ) diff --git a/openai/cli.py b/openai/cli.py index fb210c507e..ad7007a7f8 100644 --- a/openai/cli.py +++ b/openai/cli.py @@ -129,6 +129,13 @@ def list(cls, args): display(engines) +class Tokens: + @classmethod + def count(cls, args): + count = openai.Tokens.retrieve(id=args.text) + display(count) + + class Completion: @classmethod def create(cls, args): @@ -345,7 +352,7 @@ def help(args): sub.add_argument("-q", "--query", required=True, help="Search query") sub.set_defaults(func=Engine.search) - ## Completions + # Completions sub = subparsers.add_parser("completions.create") sub.add_argument("-e", "--engine", required=True, help="The engine to use") sub.add_argument( @@ -460,3 +467,7 @@ def help(args): sub = subparsers.add_parser("fine_tunes.cancel") sub.add_argument("-i", "--id", required=True, help="The id of the fine-tune job") sub.set_defaults(func=FineTuneCLI.cancel) + + sub = subparsers.add_parser("tokens.count_tokens") + sub.add_argument("-t", "--text", required=True) + sub.set_defaults(func=Tokens.count_tokens) diff --git a/openai/common_constants.py b/openai/common_constants.py new file mode 100644 index 0000000000..356c88cfda --- /dev/null +++ b/openai/common_constants.py @@ -0,0 +1,5 @@ +class Constants: + """Set common constants.""" + + # Constants + NUMBER_OF_CHARACTERS_PER_TOKEN = 4 diff --git a/openai/util.py b/openai/util.py index 7295710c44..ec5aa834c9 100644 --- a/openai/util.py +++ b/openai/util.py @@ -9,8 +9,8 @@ import re import openai +from common_constants import Constants from openai import six -from openai.six.moves.urllib.parse import parse_qsl OPENAI_LOG = os.environ.get("OPENAI_LOG") @@ -235,6 +235,12 @@ def merge_dicts(x, y): return z +def check_tokens(data): + number_of_characters = len(data) + tokens = number_of_characters / Constants.NUMBER_OF_CHARACTERS_PER_TOKEN + return tokens + + class class_method_variant(object): def __init__(self, class_method_name): self.class_method_name = class_method_name From 8c0396b1e1cdb00c065545cdd632ffeb5baa548f Mon Sep 17 00:00:00 2001 From: ramitsurana Date: Sat, 17 Apr 2021 01:42:27 +0530 Subject: [PATCH 2/4] add build and dist to git ignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a8b3cf4a28..2bc8efba56 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ __pycache__ /public/dist .idea -.python-version \ No newline at end of file +.python-version +build/ +dist/ \ No newline at end of file From 89c3a1c5108b286ec1037dcbfe6da692d917118b Mon Sep 17 00:00:00 2001 From: ramitsurana Date: Mon, 26 Apr 2021 22:13:34 +0530 Subject: [PATCH 3/4] add quick cleanup --- .gitignore | 4 +++- openai/__init__.py | 4 +--- openai/api_requestor.py | 25 +++++++++++-------------- openai/api_resources/engine.py | 2 +- openai/api_resources/fine_tune.py | 2 +- openai/api_resources/list_object.py | 4 +--- openai/api_resources/snapshot.py | 1 - openai/cli.py | 1 - openai/common_constants.py | 5 +++++ openai/multipart_data_generator.py | 23 ++++++++++++----------- openai/object_classes.py | 1 - openai/openai_object.py | 2 -- openai/openai_response.py | 1 - openai/upload_progress.py | 1 - openai/util.py | 11 ++++------- setup.py | 1 - 16 files changed, 39 insertions(+), 49 deletions(-) create mode 100644 openai/common_constants.py diff --git a/.gitignore b/.gitignore index a8b3cf4a28..2bc8efba56 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ __pycache__ /public/dist .idea -.python-version \ No newline at end of file +.python-version +build/ +dist/ \ No newline at end of file diff --git a/openai/__init__.py b/openai/__init__.py index 694db8ba3b..4905cebc98 100644 --- a/openai/__init__.py +++ b/openai/__init__.py @@ -27,6 +27,4 @@ log = None # API resources -from openai.api_resources import * # noqa - -from openai.error import OpenAIError, APIError, InvalidRequestError +from openai.api_resources import * # noqa \ No newline at end of file diff --git a/openai/api_requestor.py b/openai/api_requestor.py index 186bdd284a..79ef29d9d8 100644 --- a/openai/api_requestor.py +++ b/openai/api_requestor.py @@ -7,7 +7,6 @@ import time import uuid import warnings -import gzip from io import BytesIO from collections import OrderedDict @@ -18,13 +17,11 @@ from openai.openai_response import OpenAIResponse from openai.upload_progress import BufferReader - def _encode_datetime(dttime): if dttime.tzinfo and dttime.tzinfo.utcoffset(dttime) is not None: utc_timestamp = calendar.timegm(dttime.utctimetuple()) else: utc_timestamp = time.mktime(dttime.timetuple()) - return int(utc_timestamp) @@ -41,23 +38,23 @@ def _api_encode(data): if value is None: continue elif hasattr(value, "openai_id"): - yield (key, value.openai_id) + yield key, value.openai_id elif isinstance(value, list) or isinstance(value, tuple): for i, sv in enumerate(value): if isinstance(sv, dict): subdict = _encode_nested_dict("%s[%d]" % (key, i), sv) for k, v in _api_encode(subdict): - yield (k, v) + yield k, v else: - yield ("%s[%d]" % (key, i), util.utf8(sv)) + yield "%s[%d]" % (key, i), util.utf8(sv) elif isinstance(value, dict): subdict = _encode_nested_dict(key, value) for subkey, subvalue in _api_encode(subdict): - yield (subkey, subvalue) + yield subkey, subvalue elif isinstance(value, datetime.datetime): - yield (key, _encode_datetime(value)) + yield key, _encode_datetime(value) else: - yield (key, util.utf8(value)) + yield key, util.utf8(value) def _build_api_url(url, query): @@ -258,10 +255,10 @@ def request_raw( if my_api_key is None: raise error.AuthenticationError( "No API key provided. (HINT: set your API key using in code using " - '"openai.api_key = ", or you can set the environment variable OPENAI_API_KEY=). You can generate API keys ' + '"openai.api_key = ", or you can set the environment variable OPENAI_API_KEY=). ' + 'You can generate API keys ' "in the OpenAI web interface. See https://onboard.openai.com " - "for details, or email support@openai.com if you have any " - "questions." + "for details, or email support@openai.com if you have any questions" ) abs_url = "%s%s" % (self.api_base, url) @@ -313,9 +310,9 @@ def request_raw( post_data = GZIPCompressedStream(post_data, compression_level=9) else: raise error.APIConnectionError( - "Unrecognized HTTP method %r. This may indicate a bug in the " + "Unrecognized HTTP method {}. This may indicate a bug in the " "OpenAI bindings. Please contact support@openai.com for " - "assistance." % (method,) + "assistance.".format(method) ) headers = self.request_headers(my_api_key, method, headers) diff --git a/openai/api_resources/engine.py b/openai/api_resources/engine.py index 7b62c1c15b..5e7558a1aa 100644 --- a/openai/api_resources/engine.py +++ b/openai/api_resources/engine.py @@ -2,7 +2,7 @@ import time -from openai import api_requestor, util +from openai import util from openai.api_resources.abstract import ( ListableAPIResource, UpdateableAPIResource, diff --git a/openai/api_resources/fine_tune.py b/openai/api_resources/fine_tune.py index da888a17d8..0f34b45747 100644 --- a/openai/api_resources/fine_tune.py +++ b/openai/api_resources/fine_tune.py @@ -18,4 +18,4 @@ def cancel(cls, id, api_key=None, request_id=None, **params): url = "%s/%s/cancel" % (base, extn) instance = cls(id, api_key, **params) headers = util.populate_headers(request_id=request_id) - return instance.request("post", url, headers=headers) \ No newline at end of file + return instance.request("post", url, headers=headers) diff --git a/openai/api_resources/list_object.py b/openai/api_resources/list_object.py index 9696b67b40..29dc71de29 100644 --- a/openai/api_resources/list_object.py +++ b/openai/api_resources/list_object.py @@ -1,8 +1,6 @@ from __future__ import absolute_import, division, print_function - from openai import api_requestor, six, util from openai.openai_object import OpenAIObject - from openai.six.moves.urllib.parse import quote_plus @@ -67,7 +65,7 @@ def _request( openai_account = openai_account or self.openai_account requestor = api_requestor.APIRequestor( - api_key, api_version=openai_version, account=openai_account + api_key, api_version=openai_version ) headers = util.populate_headers(idempotency_key) response, api_key = requestor.request(method_, url_, params, headers) diff --git a/openai/api_resources/snapshot.py b/openai/api_resources/snapshot.py index a8cd46d92e..41ab51521f 100644 --- a/openai/api_resources/snapshot.py +++ b/openai/api_resources/snapshot.py @@ -1,4 +1,3 @@ -from openai.api_resources.abstract.engine_api_resource import EngineAPIResource from openai.api_resources.abstract import ( ListableAPIResource, DeletableAPIResource, diff --git a/openai/cli.py b/openai/cli.py index fb210c507e..fee572a4ec 100644 --- a/openai/cli.py +++ b/openai/cli.py @@ -1,7 +1,6 @@ import json import sys import warnings - import openai diff --git a/openai/common_constants.py b/openai/common_constants.py new file mode 100644 index 0000000000..356c88cfda --- /dev/null +++ b/openai/common_constants.py @@ -0,0 +1,5 @@ +class Constants: + """Set common constants.""" + + # Constants + NUMBER_OF_CHARACTERS_PER_TOKEN = 4 diff --git a/openai/multipart_data_generator.py b/openai/multipart_data_generator.py index 93a683ee7b..643a03e115 100644 --- a/openai/multipart_data_generator.py +++ b/openai/multipart_data_generator.py @@ -1,23 +1,27 @@ from __future__ import absolute_import, division, print_function - import random import io - import openai import re +from openai import six + + +def initialize_boundary(): + return random.randint(0, 2 ** 63) + + +def remove_array_element(input_string): + match = re.match(r"^(.*)\[.*\]$", input_string) + return match[1] if match else input_string class MultipartDataGenerator(object): def __init__(self, chunk_size=1028): self.data = io.BytesIO() self.line_break = "\r\n" - self.boundary = self._initialize_boundary() + self.boundary = initialize_boundary() self.chunk_size = chunk_size - def _remove_array_element(self, input_string): - match = re.match(r"^(.*)\[.*\]$", input_string) - return match[1] if match else input_string - def add_params(self, params): # Flatten parameters first params = dict(openai.api_requestor._api_encode(params)) @@ -25,7 +29,7 @@ def add_params(self, params): for key, value in openai.six.iteritems(params): # strip array elements if present from key - key = self._remove_array_element(key) + key = remove_array_element(key) if value is None: continue @@ -87,6 +91,3 @@ def _write_file(self, f): if not file_contents: break self._write(file_contents) - - def _initialize_boundary(self): - return random.randint(0, 2 ** 63) diff --git a/openai/object_classes.py b/openai/object_classes.py index c3a47577bb..a2da952eeb 100644 --- a/openai/object_classes.py +++ b/openai/object_classes.py @@ -1,5 +1,4 @@ from __future__ import absolute_import, division, print_function - from openai import api_resources from openai.api_resources.experimental.completion_config import CompletionConfig diff --git a/openai/openai_object.py b/openai/openai_object.py index 1f70915305..e234101a14 100644 --- a/openai/openai_object.py +++ b/openai/openai_object.py @@ -1,9 +1,7 @@ from __future__ import absolute_import, division, print_function - import datetime import json from copy import deepcopy - import openai from openai import api_requestor, util, six diff --git a/openai/openai_response.py b/openai/openai_response.py index 4550c22ec0..71d74a337c 100644 --- a/openai/openai_response.py +++ b/openai/openai_response.py @@ -1,5 +1,4 @@ from __future__ import absolute_import, division, print_function - import json diff --git a/openai/upload_progress.py b/openai/upload_progress.py index 4214202c63..deb484d3c7 100644 --- a/openai/upload_progress.py +++ b/openai/upload_progress.py @@ -1,4 +1,3 @@ -import requests import io diff --git a/openai/util.py b/openai/util.py index 7295710c44..47520b9af7 100644 --- a/openai/util.py +++ b/openai/util.py @@ -1,5 +1,4 @@ from __future__ import absolute_import, division, print_function - import functools import hmac import io @@ -7,11 +6,8 @@ import sys import os import re - import openai from openai import six -from openai.six.moves.urllib.parse import parse_qsl - OPENAI_LOG = os.environ.get("OPENAI_LOG") @@ -19,13 +15,13 @@ __all__ = [ "io", - "parse_qsl", "utf8", "log_info", "log_debug", "log_warn", "dashboard_link", "logfmt", + "merge_dicts" ] @@ -64,10 +60,11 @@ def log_info(message, **params): print(msg, file=sys.stderr) logger.info(msg) + def log_warn(message, **params): msg = logfmt(dict(message=message, **params)) print(msg, file=sys.stderr) - logger.warn(msg) + logger.warning(msg) def _test_or_live_environment(): @@ -87,7 +84,7 @@ def dashboard_link(request_id): def logfmt(props): def fmt(key, val): - # Handle case where val is a bytes or bytesarray + # Handle case where val is a bytes or bytes array if six.PY3 and hasattr(val, "decode"): val = val.decode("utf-8") # Check if val is already a string to avoid re-encoding into diff --git a/setup.py b/setup.py index 21d5f70f98..c3e169275c 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ import os - from setuptools import find_packages, setup version_contents = {} From a80642f7c409b1289a84876d169fe5f499228703 Mon Sep 17 00:00:00 2001 From: ramitsurana Date: Mon, 26 Apr 2021 22:23:30 +0530 Subject: [PATCH 4/4] remove unused items --- openai/common_constants.py | 5 ----- openai/util.py | 7 ------- 2 files changed, 12 deletions(-) delete mode 100644 openai/common_constants.py diff --git a/openai/common_constants.py b/openai/common_constants.py deleted file mode 100644 index 356c88cfda..0000000000 --- a/openai/common_constants.py +++ /dev/null @@ -1,5 +0,0 @@ -class Constants: - """Set common constants.""" - - # Constants - NUMBER_OF_CHARACTERS_PER_TOKEN = 4 diff --git a/openai/util.py b/openai/util.py index e191165906..4d5bd0368d 100644 --- a/openai/util.py +++ b/openai/util.py @@ -7,7 +7,6 @@ import os import re import openai -from common_constants import Constants from openai import six @@ -234,12 +233,6 @@ def merge_dicts(x, y): return z -def check_tokens(data): - number_of_characters = len(data) - tokens = number_of_characters / Constants.NUMBER_OF_CHARACTERS_PER_TOKEN - return tokens - - class class_method_variant(object): def __init__(self, class_method_name): self.class_method_name = class_method_name