From 0a4be6506bf26d2b9552ff3fd13a22c04b53ea18 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 18:11:15 +0000 Subject: [PATCH 1/2] chore(internal): move LineDecoder to a separate file (#1120) --- src/internal/decoders/line.ts | 114 ++++++++++++++++++++++++++++++++++ src/streaming.ts | 112 +-------------------------------- 2 files changed, 115 insertions(+), 111 deletions(-) create mode 100644 src/internal/decoders/line.ts diff --git a/src/internal/decoders/line.ts b/src/internal/decoders/line.ts new file mode 100644 index 000000000..1e0bbf390 --- /dev/null +++ b/src/internal/decoders/line.ts @@ -0,0 +1,114 @@ +import { OpenAIError } from '../../error'; + +type Bytes = string | ArrayBuffer | Uint8Array | Buffer | null | undefined; + +/** + * A re-implementation of httpx's `LineDecoder` in Python that handles incrementally + * reading lines from text. + * + * https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258 + */ +export class LineDecoder { + // prettier-ignore + static NEWLINE_CHARS = new Set(['\n', '\r']); + static NEWLINE_REGEXP = /\r\n|[\n\r]/g; + + buffer: string[]; + trailingCR: boolean; + textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either "dom" or "node" types. + + constructor() { + this.buffer = []; + this.trailingCR = false; + } + + decode(chunk: Bytes): string[] { + let text = this.decodeText(chunk); + + if (this.trailingCR) { + text = '\r' + text; + this.trailingCR = false; + } + if (text.endsWith('\r')) { + this.trailingCR = true; + text = text.slice(0, -1); + } + + if (!text) { + return []; + } + + const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || ''); + let lines = text.split(LineDecoder.NEWLINE_REGEXP); + + // if there is a trailing new line then the last entry will be an empty + // string which we don't care about + if (trailingNewline) { + lines.pop(); + } + + if (lines.length === 1 && !trailingNewline) { + this.buffer.push(lines[0]!); + return []; + } + + if (this.buffer.length > 0) { + lines = [this.buffer.join('') + lines[0], ...lines.slice(1)]; + this.buffer = []; + } + + if (!trailingNewline) { + this.buffer = [lines.pop() || '']; + } + + return lines; + } + + decodeText(bytes: Bytes): string { + if (bytes == null) return ''; + if (typeof bytes === 'string') return bytes; + + // Node: + if (typeof Buffer !== 'undefined') { + if (bytes instanceof Buffer) { + return bytes.toString(); + } + if (bytes instanceof Uint8Array) { + return Buffer.from(bytes).toString(); + } + + throw new OpenAIError( + `Unexpected: received non-Uint8Array (${bytes.constructor.name}) stream chunk in an environment with a global "Buffer" defined, which this library assumes to be Node. Please report this error.`, + ); + } + + // Browser + if (typeof TextDecoder !== 'undefined') { + if (bytes instanceof Uint8Array || bytes instanceof ArrayBuffer) { + this.textDecoder ??= new TextDecoder('utf8'); + return this.textDecoder.decode(bytes); + } + + throw new OpenAIError( + `Unexpected: received non-Uint8Array/ArrayBuffer (${ + (bytes as any).constructor.name + }) in a web platform. Please report this error.`, + ); + } + + throw new OpenAIError( + `Unexpected: neither Buffer nor TextDecoder are available as globals. Please report this error.`, + ); + } + + flush(): string[] { + if (!this.buffer.length && !this.trailingCR) { + return []; + } + + const lines = [this.buffer.join('')]; + this.buffer = []; + this.trailingCR = false; + return lines; + } +} diff --git a/src/streaming.ts b/src/streaming.ts index 722a8f69c..597ee89fa 100644 --- a/src/streaming.ts +++ b/src/streaming.ts @@ -1,5 +1,6 @@ import { ReadableStream, type Response } from './_shims/index'; import { OpenAIError } from './error'; +import { LineDecoder } from './internal/decoders/line'; import { APIError } from 'openai/error'; @@ -343,117 +344,6 @@ class SSEDecoder { } } -/** - * A re-implementation of httpx's `LineDecoder` in Python that handles incrementally - * reading lines from text. - * - * https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258 - */ -class LineDecoder { - // prettier-ignore - static NEWLINE_CHARS = new Set(['\n', '\r']); - static NEWLINE_REGEXP = /\r\n|[\n\r]/g; - - buffer: string[]; - trailingCR: boolean; - textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either "dom" or "node" types. - - constructor() { - this.buffer = []; - this.trailingCR = false; - } - - decode(chunk: Bytes): string[] { - let text = this.decodeText(chunk); - - if (this.trailingCR) { - text = '\r' + text; - this.trailingCR = false; - } - if (text.endsWith('\r')) { - this.trailingCR = true; - text = text.slice(0, -1); - } - - if (!text) { - return []; - } - - const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || ''); - let lines = text.split(LineDecoder.NEWLINE_REGEXP); - - // if there is a trailing new line then the last entry will be an empty - // string which we don't care about - if (trailingNewline) { - lines.pop(); - } - - if (lines.length === 1 && !trailingNewline) { - this.buffer.push(lines[0]!); - return []; - } - - if (this.buffer.length > 0) { - lines = [this.buffer.join('') + lines[0], ...lines.slice(1)]; - this.buffer = []; - } - - if (!trailingNewline) { - this.buffer = [lines.pop() || '']; - } - - return lines; - } - - decodeText(bytes: Bytes): string { - if (bytes == null) return ''; - if (typeof bytes === 'string') return bytes; - - // Node: - if (typeof Buffer !== 'undefined') { - if (bytes instanceof Buffer) { - return bytes.toString(); - } - if (bytes instanceof Uint8Array) { - return Buffer.from(bytes).toString(); - } - - throw new OpenAIError( - `Unexpected: received non-Uint8Array (${bytes.constructor.name}) stream chunk in an environment with a global "Buffer" defined, which this library assumes to be Node. Please report this error.`, - ); - } - - // Browser - if (typeof TextDecoder !== 'undefined') { - if (bytes instanceof Uint8Array || bytes instanceof ArrayBuffer) { - this.textDecoder ??= new TextDecoder('utf8'); - return this.textDecoder.decode(bytes); - } - - throw new OpenAIError( - `Unexpected: received non-Uint8Array/ArrayBuffer (${ - (bytes as any).constructor.name - }) in a web platform. Please report this error.`, - ); - } - - throw new OpenAIError( - `Unexpected: neither Buffer nor TextDecoder are available as globals. Please report this error.`, - ); - } - - flush(): string[] { - if (!this.buffer.length && !this.trailingCR) { - return []; - } - - const lines = [this.buffer.join('')]; - this.buffer = []; - this.trailingCR = false; - return lines; - } -} - /** This is an internal helper function that's just used for testing */ export function _decodeChunks(chunks: string[]): string[] { const decoder = new LineDecoder(); From 3c35890ee7eee1fb98f1562e46de61a2ef7b72fd Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 05:07:29 +0000 Subject: [PATCH 2/2] release: 4.67.2 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 8 ++++++++ README.md | 2 +- package.json | 2 +- scripts/build-deno | 2 +- src/version.ts | 2 +- 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 913154f97..f45113292 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "4.67.1" + ".": "4.67.2" } diff --git a/CHANGELOG.md b/CHANGELOG.md index b866aeff2..72e61a413 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 4.67.2 (2024-10-07) + +Full Changelog: [v4.67.1...v4.67.2](https://github.com/openai/openai-node/compare/v4.67.1...v4.67.2) + +### Chores + +* **internal:** move LineDecoder to a separate file ([#1120](https://github.com/openai/openai-node/issues/1120)) ([0a4be65](https://github.com/openai/openai-node/commit/0a4be6506bf26d2b9552ff3fd13a22c04b53ea18)) + ## 4.67.1 (2024-10-02) Full Changelog: [v4.67.0...v4.67.1](https://github.com/openai/openai-node/compare/v4.67.0...v4.67.1) diff --git a/README.md b/README.md index 058f412d5..11e1455e2 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ You can import in Deno via: ```ts -import OpenAI from 'https://deno.land/x/openai@v4.67.1/mod.ts'; +import OpenAI from 'https://deno.land/x/openai@v4.67.2/mod.ts'; ``` diff --git a/package.json b/package.json index b04cf8a6a..79b73d5cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openai", - "version": "4.67.1", + "version": "4.67.2", "description": "The official TypeScript library for the OpenAI API", "author": "OpenAI ", "types": "dist/index.d.ts", diff --git a/scripts/build-deno b/scripts/build-deno index a717ae498..46c3f3db3 100755 --- a/scripts/build-deno +++ b/scripts/build-deno @@ -16,7 +16,7 @@ This is a build produced from https://github.com/openai/openai-node – please g Usage: \`\`\`ts -import OpenAI from "https://deno.land/x/openai@v4.67.1/mod.ts"; +import OpenAI from "https://deno.land/x/openai@v4.67.2/mod.ts"; const client = new OpenAI(); \`\`\` diff --git a/src/version.ts b/src/version.ts index b60359315..fb7e251f7 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const VERSION = '4.67.1'; // x-release-please-version +export const VERSION = '4.67.2'; // x-release-please-version