diff --git a/io/_iotest.ts b/io/_iotest.ts deleted file mode 100644 index 547901bf5401..000000000000 --- a/io/_iotest.ts +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.// Ported to Deno from -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -import type { Reader } from "./types.d.ts"; - -/** OneByteReader returns a Reader that implements - * each non-empty Read by reading one byte from r. - */ -export class OneByteReader implements Reader { - constructor(readonly r: Reader) {} - - read(p: Uint8Array): Promise { - if (p.byteLength === 0) { - return Promise.resolve(0); - } - if (!(p instanceof Uint8Array)) { - throw Error("expected Uint8Array"); - } - return Promise.resolve(this.r.read(p.subarray(0, 1))); - } -} - -/** HalfReader returns a Reader that implements Read - * by reading half as many requested bytes from r. - */ -export class HalfReader implements Reader { - constructor(readonly r: Reader) {} - - read(p: Uint8Array): Promise { - if (!(p instanceof Uint8Array)) { - throw Error("expected Uint8Array"); - } - const half = Math.floor((p.byteLength + 1) / 2); - return Promise.resolve(this.r.read(p.subarray(0, half))); - } -} - -/** TimeoutReader returns `Deno.errors.TimedOut` on the second read - * with no data. Subsequent calls to read succeed. - */ -export class TimeoutReader implements Reader { - count = 0; - constructor(readonly r: Reader) {} - - read(p: Uint8Array): Promise { - this.count++; - if (this.count === 2) { - throw new Deno.errors.TimedOut(); - } - return Promise.resolve(this.r.read(p)); - } -} diff --git a/io/_test_common.ts b/io/_test_common.ts new file mode 100644 index 000000000000..fa74de9f88fd --- /dev/null +++ b/io/_test_common.ts @@ -0,0 +1,29 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import type { Reader } from "./types.d.ts"; + +export const MIN_READ_BUFFER_SIZE = 16; +export const bufsizes: number[] = [ + 0, + MIN_READ_BUFFER_SIZE, + 23, + 32, + 46, + 64, + 93, + 128, + 1024, + 4096, +]; + +export class BinaryReader implements Reader { + index = 0; + + constructor(private bytes: Uint8Array = new Uint8Array(0)) {} + + read(p: Uint8Array): Promise { + p.set(this.bytes.subarray(this.index, p.byteLength)); + this.index += p.byteLength; + return Promise.resolve(p.byteLength); + } +} diff --git a/io/buf_reader.ts b/io/buf_reader.ts new file mode 100644 index 000000000000..348bb2c80a8d --- /dev/null +++ b/io/buf_reader.ts @@ -0,0 +1,434 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { assert } from "../_util/asserts.ts"; +import { copy } from "../bytes/copy.ts"; +import type { Reader } from "./types.d.ts"; + +const DEFAULT_BUF_SIZE = 4096; +const MIN_BUF_SIZE = 16; +const MAX_CONSECUTIVE_EMPTY_READS = 100; +const CR = "\r".charCodeAt(0); +const LF = "\n".charCodeAt(0); + +export class BufferFullError extends Error { + override name = "BufferFullError"; + constructor(public partial: Uint8Array) { + super("Buffer full"); + } +} + +export class PartialReadError extends Error { + override name = "PartialReadError"; + partial?: Uint8Array; + constructor() { + super("Encountered UnexpectedEof, data only partially read"); + } +} + +/** Result type returned by of BufReader.readLine(). */ +export interface ReadLineResult { + line: Uint8Array; + more: boolean; +} + +export class BufReader implements Reader { + #buf!: Uint8Array; + #rd!: Reader; // Reader provided by caller. + #r = 0; // buf read position. + #w = 0; // buf write position. + #eof = false; + // private lastByte: number; + // private lastCharSize: number; + + /** return new BufReader unless r is BufReader */ + static create(r: Reader, size: number = DEFAULT_BUF_SIZE): BufReader { + return r instanceof BufReader ? r : new BufReader(r, size); + } + + constructor(rd: Reader, size: number = DEFAULT_BUF_SIZE) { + if (size < MIN_BUF_SIZE) { + size = MIN_BUF_SIZE; + } + this.#reset(new Uint8Array(size), rd); + } + + /** Returns the size of the underlying buffer in bytes. */ + size(): number { + return this.#buf.byteLength; + } + + buffered(): number { + return this.#w - this.#r; + } + + // Reads a new chunk into the buffer. + #fill = async () => { + // Slide existing data to beginning. + if (this.#r > 0) { + this.#buf.copyWithin(0, this.#r, this.#w); + this.#w -= this.#r; + this.#r = 0; + } + + if (this.#w >= this.#buf.byteLength) { + throw Error("bufio: tried to fill full buffer"); + } + + // Read new data: try a limited number of times. + for (let i = MAX_CONSECUTIVE_EMPTY_READS; i > 0; i--) { + const rr = await this.#rd.read(this.#buf.subarray(this.#w)); + if (rr === null) { + this.#eof = true; + return; + } + assert(rr >= 0, "negative read"); + this.#w += rr; + if (rr > 0) { + return; + } + } + + throw new Error( + `No progress after ${MAX_CONSECUTIVE_EMPTY_READS} read() calls`, + ); + }; + + /** Discards any buffered data, resets all state, and switches + * the buffered reader to read from r. + */ + reset(r: Reader) { + this.#reset(this.#buf, r); + } + + #reset = (buf: Uint8Array, rd: Reader) => { + this.#buf = buf; + this.#rd = rd; + this.#eof = false; + // this.lastByte = -1; + // this.lastCharSize = -1; + }; + + /** reads data into p. + * It returns the number of bytes read into p. + * The bytes are taken from at most one Read on the underlying Reader, + * hence n may be less than len(p). + * To read exactly len(p) bytes, use io.ReadFull(b, p). + */ + async read(p: Uint8Array): Promise { + let rr: number | null = p.byteLength; + if (p.byteLength === 0) return rr; + + if (this.#r === this.#w) { + if (p.byteLength >= this.#buf.byteLength) { + // Large read, empty buffer. + // Read directly into p to avoid copy. + const rr = await this.#rd.read(p); + const nread = rr ?? 0; + assert(nread >= 0, "negative read"); + // if (rr.nread > 0) { + // this.lastByte = p[rr.nread - 1]; + // this.lastCharSize = -1; + // } + return rr; + } + + // One read. + // Do not use this.fill, which will loop. + this.#r = 0; + this.#w = 0; + rr = await this.#rd.read(this.#buf); + if (rr === 0 || rr === null) return rr; + assert(rr >= 0, "negative read"); + this.#w += rr; + } + + // copy as much as we can + const copied = copy(this.#buf.subarray(this.#r, this.#w), p, 0); + this.#r += copied; + // this.lastByte = this.buf[this.r - 1]; + // this.lastCharSize = -1; + return copied; + } + + /** reads exactly `p.length` bytes into `p`. + * + * If successful, `p` is returned. + * + * If the end of the underlying stream has been reached, and there are no more + * bytes available in the buffer, `readFull()` returns `null` instead. + * + * An error is thrown if some bytes could be read, but not enough to fill `p` + * entirely before the underlying stream reported an error or EOF. Any error + * thrown will have a `partial` property that indicates the slice of the + * buffer that has been successfully filled with data. + * + * Ported from https://golang.org/pkg/io/#ReadFull + */ + async readFull(p: Uint8Array): Promise { + let bytesRead = 0; + while (bytesRead < p.length) { + try { + const rr = await this.read(p.subarray(bytesRead)); + if (rr === null) { + if (bytesRead === 0) { + return null; + } else { + throw new PartialReadError(); + } + } + bytesRead += rr; + } catch (err) { + if (err instanceof PartialReadError) { + err.partial = p.subarray(0, bytesRead); + } else if (err instanceof Error) { + const e = new PartialReadError(); + e.partial = p.subarray(0, bytesRead); + e.stack = err.stack; + e.message = err.message; + e.cause = err.cause; + throw err; + } + throw err; + } + } + return p; + } + + /** Returns the next byte [0, 255] or `null`. */ + async readByte(): Promise { + while (this.#r === this.#w) { + if (this.#eof) return null; + await this.#fill(); // buffer is empty. + } + const c = this.#buf[this.#r]; + this.#r++; + // this.lastByte = c; + return c; + } + + /** readString() reads until the first occurrence of delim in the input, + * returning a string containing the data up to and including the delimiter. + * If ReadString encounters an error before finding a delimiter, + * it returns the data read before the error and the error itself + * (often `null`). + * ReadString returns err != nil if and only if the returned data does not end + * in delim. + * For simple uses, a Scanner may be more convenient. + */ + async readString(delim: string): Promise { + if (delim.length !== 1) { + throw new Error("Delimiter should be a single character"); + } + const buffer = await this.readSlice(delim.charCodeAt(0)); + if (buffer === null) return null; + return new TextDecoder().decode(buffer); + } + + /** `readLine()` is a low-level line-reading primitive. Most callers should + * use `readString('\n')` instead or use a Scanner. + * + * `readLine()` tries to return a single line, not including the end-of-line + * bytes. If the line was too long for the buffer then `more` is set and the + * beginning of the line is returned. The rest of the line will be returned + * from future calls. `more` will be false when returning the last fragment + * of the line. The returned buffer is only valid until the next call to + * `readLine()`. + * + * The text returned from ReadLine does not include the line end ("\r\n" or + * "\n"). + * + * When the end of the underlying stream is reached, the final bytes in the + * stream are returned. No indication or error is given if the input ends + * without a final line end. When there are no more trailing bytes to read, + * `readLine()` returns `null`. + * + * Calling `unreadByte()` after `readLine()` will always unread the last byte + * read (possibly a character belonging to the line end) even if that byte is + * not part of the line returned by `readLine()`. + */ + async readLine(): Promise { + let line: Uint8Array | null = null; + + try { + line = await this.readSlice(LF); + } catch (err) { + let partial; + if (err instanceof PartialReadError) { + partial = err.partial; + assert( + partial instanceof Uint8Array, + "bufio: caught error from `readSlice()` without `partial` property", + ); + } + + // Don't throw if `readSlice()` failed with `BufferFullError`, instead we + // just return whatever is available and set the `more` flag. + if (!(err instanceof BufferFullError)) { + throw err; + } + + partial = err.partial; + + // Handle the case where "\r\n" straddles the buffer. + if ( + !this.#eof && partial && + partial.byteLength > 0 && + partial[partial.byteLength - 1] === CR + ) { + // Put the '\r' back on buf and drop it from line. + // Let the next call to ReadLine check for "\r\n". + assert(this.#r > 0, "bufio: tried to rewind past start of buffer"); + this.#r--; + partial = partial.subarray(0, partial.byteLength - 1); + } + + if (partial) { + return { line: partial, more: !this.#eof }; + } + } + + if (line === null) { + return null; + } + + if (line.byteLength === 0) { + return { line, more: false }; + } + + if (line[line.byteLength - 1] == LF) { + let drop = 1; + if (line.byteLength > 1 && line[line.byteLength - 2] === CR) { + drop = 2; + } + line = line.subarray(0, line.byteLength - drop); + } + return { line, more: false }; + } + + /** `readSlice()` reads until the first occurrence of `delim` in the input, + * returning a slice pointing at the bytes in the buffer. The bytes stop + * being valid at the next read. + * + * If `readSlice()` encounters an error before finding a delimiter, or the + * buffer fills without finding a delimiter, it throws an error with a + * `partial` property that contains the entire buffer. + * + * If `readSlice()` encounters the end of the underlying stream and there are + * any bytes left in the buffer, the rest of the buffer is returned. In other + * words, EOF is always treated as a delimiter. Once the buffer is empty, + * it returns `null`. + * + * Because the data returned from `readSlice()` will be overwritten by the + * next I/O operation, most clients should use `readString()` instead. + */ + async readSlice(delim: number): Promise { + let s = 0; // search start index + let slice: Uint8Array | undefined; + + while (true) { + // Search buffer. + let i = this.#buf.subarray(this.#r + s, this.#w).indexOf(delim); + if (i >= 0) { + i += s; + slice = this.#buf.subarray(this.#r, this.#r + i + 1); + this.#r += i + 1; + break; + } + + // EOF? + if (this.#eof) { + if (this.#r === this.#w) { + return null; + } + slice = this.#buf.subarray(this.#r, this.#w); + this.#r = this.#w; + break; + } + + // Buffer full? + if (this.buffered() >= this.#buf.byteLength) { + this.#r = this.#w; + // #4521 The internal buffer should not be reused across reads because it causes corruption of data. + const oldbuf = this.#buf; + const newbuf = this.#buf.slice(0); + this.#buf = newbuf; + throw new BufferFullError(oldbuf); + } + + s = this.#w - this.#r; // do not rescan area we scanned before + + // Buffer is not full. + try { + await this.#fill(); + } catch (err) { + if (err instanceof PartialReadError) { + err.partial = slice; + } else if (err instanceof Error) { + const e = new PartialReadError(); + e.partial = slice; + e.stack = err.stack; + e.message = err.message; + e.cause = err.cause; + throw err; + } + throw err; + } + } + + // Handle last byte, if any. + // const i = slice.byteLength - 1; + // if (i >= 0) { + // this.lastByte = slice[i]; + // this.lastCharSize = -1 + // } + + return slice; + } + + /** `peek()` returns the next `n` bytes without advancing the reader. The + * bytes stop being valid at the next read call. + * + * When the end of the underlying stream is reached, but there are unread + * bytes left in the buffer, those bytes are returned. If there are no bytes + * left in the buffer, it returns `null`. + * + * If an error is encountered before `n` bytes are available, `peek()` throws + * an error with the `partial` property set to a slice of the buffer that + * contains the bytes that were available before the error occurred. + */ + async peek(n: number): Promise { + if (n < 0) { + throw Error("negative count"); + } + + let avail = this.#w - this.#r; + while (avail < n && avail < this.#buf.byteLength && !this.#eof) { + try { + await this.#fill(); + } catch (err) { + if (err instanceof PartialReadError) { + err.partial = this.#buf.subarray(this.#r, this.#w); + } else if (err instanceof Error) { + const e = new PartialReadError(); + e.partial = this.#buf.subarray(this.#r, this.#w); + e.stack = err.stack; + e.message = err.message; + e.cause = err.cause; + throw err; + } + throw err; + } + avail = this.#w - this.#r; + } + + if (avail === 0 && this.#eof) { + return null; + } else if (avail < n && this.#eof) { + return this.#buf.subarray(this.#r, this.#r + avail); + } else if (avail < n) { + throw new BufferFullError(this.#buf.subarray(this.#r, this.#w)); + } + + return this.#buf.subarray(this.#r, this.#r + n); + } +} diff --git a/io/buf_reader_test.ts b/io/buf_reader_test.ts new file mode 100644 index 000000000000..114127b7069c --- /dev/null +++ b/io/buf_reader_test.ts @@ -0,0 +1,413 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +// This code has been ported almost directly from Go's src/bytes/buffer_test.go +// Copyright 2009 The Go Authors. All rights reserved. BSD license. +// https://github.com/golang/go/blob/master/LICENSE +import { + assert, + assertEquals, + assertRejects, + fail, +} from "../testing/asserts.ts"; +import { BufferFullError, BufReader, PartialReadError } from "./buf_reader.ts"; +import { StringReader } from "./string_reader.ts"; +import { bufsizes, MIN_READ_BUFFER_SIZE } from "./_test_common.ts"; +import { Buffer } from "./buffer.ts"; +import type { Reader } from "./types.d.ts"; +import { copy } from "../bytes/copy.ts"; + +/** OneByteReader returns a Reader that implements + * each non-empty Read by reading one byte from r. + */ +class OneByteReader implements Reader { + constructor(readonly r: Reader) {} + + read(p: Uint8Array): Promise { + if (p.byteLength === 0) { + return Promise.resolve(0); + } + if (!(p instanceof Uint8Array)) { + throw Error("expected Uint8Array"); + } + return Promise.resolve(this.r.read(p.subarray(0, 1))); + } +} + +/** HalfReader returns a Reader that implements Read + * by reading half as many requested bytes from r. + */ +class HalfReader implements Reader { + constructor(readonly r: Reader) {} + + read(p: Uint8Array): Promise { + if (!(p instanceof Uint8Array)) { + throw Error("expected Uint8Array"); + } + const half = Math.floor((p.byteLength + 1) / 2); + return Promise.resolve(this.r.read(p.subarray(0, half))); + } +} + +async function readBytes(buf: BufReader): Promise { + const b = new Uint8Array(1000); + let nb = 0; + while (true) { + const c = await buf.readByte(); + if (c === null) { + break; // EOF + } + b[nb] = c; + nb++; + } + const decoder = new TextDecoder(); + return decoder.decode(b.subarray(0, nb)); +} + +interface ReadMaker { + name: string; + fn: (r: Deno.Reader) => Deno.Reader; +} + +const readMakers: ReadMaker[] = [ + { name: "full", fn: (r): Deno.Reader => r }, + { + name: "byte", + fn: (r): OneByteReader => new OneByteReader(r), + }, + { name: "half", fn: (r): HalfReader => new HalfReader(r) }, + // TODO(bartlomieju): { name: "data+err", r => new DataErrReader(r) }, + // { name: "timeout", fn: r => new TimeoutReader(r) }, +]; + +// Call read to accumulate the text of a file +async function reads(buf: BufReader, m: number): Promise { + const b = new Uint8Array(1000); + let nb = 0; + while (true) { + const result = await buf.read(b.subarray(nb, nb + m)); + if (result === null) { + break; + } + nb += result; + } + const decoder = new TextDecoder(); + return decoder.decode(b.subarray(0, nb)); +} + +interface NamedBufReader { + name: string; + fn: (r: BufReader) => Promise; +} + +const bufreaders: NamedBufReader[] = [ + { name: "1", fn: (b: BufReader): Promise => reads(b, 1) }, + { name: "2", fn: (b: BufReader): Promise => reads(b, 2) }, + { name: "3", fn: (b: BufReader): Promise => reads(b, 3) }, + { name: "4", fn: (b: BufReader): Promise => reads(b, 4) }, + { name: "5", fn: (b: BufReader): Promise => reads(b, 5) }, + { name: "7", fn: (b: BufReader): Promise => reads(b, 7) }, + { name: "bytes", fn: readBytes }, + // { name: "lines", fn: readLines }, +]; + +Deno.test("bufioReaderSimple", async function () { + const data = "hello world"; + const b = new BufReader(new StringReader(data)); + const s = await readBytes(b); + assertEquals(s, data); +}); + +Deno.test("bufioBufReader", async function () { + const texts = new Array(31); + let str = ""; + let all = ""; + for (let i = 0; i < texts.length - 1; i++) { + texts[i] = str + "\n"; + all += texts[i]; + str += String.fromCharCode((i % 26) + 97); + } + texts[texts.length - 1] = all; + + for (const text of texts) { + for (const readmaker of readMakers) { + for (const bufreader of bufreaders) { + for (const bufsize of bufsizes) { + const read = readmaker.fn(new StringReader(text)); + const buf = new BufReader(read, bufsize); + const s = await bufreader.fn(buf); + const debugStr = `reader=${readmaker.name} ` + + `fn=${bufreader.name} bufsize=${bufsize} want=${text} got=${s}`; + assertEquals(s, text, debugStr); + } + } + } + } +}); + +Deno.test("bufioBufferFull", async function () { + const longString = + "And now, hello, world! It is the time for all good men to come to the" + + " aid of their party"; + const buf = new BufReader(new StringReader(longString), MIN_READ_BUFFER_SIZE); + const decoder = new TextDecoder(); + + try { + await buf.readSlice("!".charCodeAt(0)); + fail("readSlice should throw"); + } catch (err) { + assert(err instanceof BufferFullError); + assert(err.partial instanceof Uint8Array); + assertEquals(decoder.decode(err.partial), "And now, hello, "); + } + + const line = await buf.readSlice("!".charCodeAt(0)); + assert(line !== null); + const actual = decoder.decode(line); + assertEquals(actual, "world!"); +}); + +Deno.test("bufioReadString", async function () { + const string = "And now, hello world!"; + const buf = new BufReader(new StringReader(string), MIN_READ_BUFFER_SIZE); + + const line = await buf.readString(","); + assert(line !== null); + assertEquals(line, "And now,"); + assertEquals(line.length, 8); + + const line2 = await buf.readString(","); + assert(line2 !== null); + assertEquals(line2, " hello world!"); + + assertEquals(await buf.readString(","), null); + + try { + await buf.readString("deno"); + + fail("should throw"); + } catch (err) { + assert(err instanceof Error); + assert(err.message, "Delimiter should be a single character"); + } +}); + +Deno.test("bufReaderReadFull", async function () { + const enc = new TextEncoder(); + const dec = new TextDecoder(); + const text = "Hello World"; + const data = new Buffer(enc.encode(text)); + const bufr = new BufReader(data, 3); + { + const buf = new Uint8Array(6); + const r = await bufr.readFull(buf); + assert(r !== null); + assertEquals(r, buf); + assertEquals(dec.decode(buf), "Hello "); + } + { + const buf = new Uint8Array(6); + try { + await bufr.readFull(buf); + fail("readFull() should throw PartialReadError"); + } catch (err) { + assert(err instanceof PartialReadError); + assert(err.partial instanceof Uint8Array); + assertEquals(err.partial.length, 5); + assertEquals(dec.decode(buf.subarray(0, 5)), "World"); + } + } +}); + +Deno.test("bufioPeek", async function () { + const decoder = new TextDecoder(); + const p = new Uint8Array(10); + // string is 16 (minReadBufferSize) long. + const buf = new BufReader( + new StringReader("abcdefghijklmnop"), + MIN_READ_BUFFER_SIZE, + ); + + let actual = await buf.peek(1); + assert(actual !== null); + assertEquals(decoder.decode(actual), "a"); + + actual = await buf.peek(4); + assert(actual !== null); + assertEquals(decoder.decode(actual), "abcd"); + + try { + await buf.peek(32); + fail("peek() should throw"); + } catch (err) { + assert(err instanceof BufferFullError); + assert(err.partial instanceof Uint8Array); + assertEquals(decoder.decode(err.partial), "abcdefghijklmnop"); + } + + await buf.read(p.subarray(0, 3)); + assertEquals(decoder.decode(p.subarray(0, 3)), "abc"); + + actual = await buf.peek(1); + assert(actual !== null); + assertEquals(decoder.decode(actual), "d"); + + actual = await buf.peek(1); + assert(actual !== null); + assertEquals(decoder.decode(actual), "d"); + + actual = await buf.peek(1); + assert(actual !== null); + assertEquals(decoder.decode(actual), "d"); + + actual = await buf.peek(2); + assert(actual !== null); + assertEquals(decoder.decode(actual), "de"); + + const res = await buf.read(p.subarray(0, 3)); + assertEquals(decoder.decode(p.subarray(0, 3)), "def"); + assert(res !== null); + + actual = await buf.peek(4); + assert(actual !== null); + assertEquals(decoder.decode(actual), "ghij"); + + await buf.read(p); + assertEquals(decoder.decode(p), "ghijklmnop"); + + actual = await buf.peek(0); + assert(actual !== null); + assertEquals(decoder.decode(actual), ""); + + const r = await buf.peek(1); + assert(r === null); + /* TODO + Test for issue 3022, not exposing a reader's error on a successful Peek. + buf = NewReaderSize(dataAndEOFReader("abcd"), 32) + if s, err := buf.Peek(2); string(s) != "ab" || err != nil { + t.Errorf(`Peek(2) on "abcd", EOF = %q, %v; want "ab", nil`, string(s), err) + } + if s, err := buf.Peek(4); string(s) != "abcd" || err != nil { + t.Errorf( + `Peek(4) on "abcd", EOF = %q, %v; want "abcd", nil`, + string(s), + err + ) + } + if n, err := buf.Read(p[0:5]); string(p[0:n]) != "abcd" || err != nil { + t.Fatalf("Read after peek = %q, %v; want abcd, EOF", p[0:n], err) + } + if n, err := buf.Read(p[0:1]); string(p[0:n]) != "" || err != io.EOF { + t.Fatalf(`second Read after peek = %q, %v; want "", EOF`, p[0:n], err) + } + */ +}); + +const encoder = new TextEncoder(); + +const testInput = encoder.encode( + "012\n345\n678\n9ab\ncde\nfgh\nijk\nlmn\nopq\nrst\nuvw\nxy", +); +const testInputrn = encoder.encode( + "012\r\n345\r\n678\r\n9ab\r\ncde\r\nfgh\r\nijk\r\nlmn\r\nopq\r\nrst\r\n" + + "uvw\r\nxy\r\n\n\r\n", +); +const testOutput = encoder.encode("0123456789abcdefghijklmnopqrstuvwxy"); + +// TestReader wraps a Uint8Array and returns reads of a specific length. +class TestReader implements Deno.Reader { + constructor(private data: Uint8Array, private stride: number) {} + + read(buf: Uint8Array): Promise { + let nread = this.stride; + if (nread > this.data.byteLength) { + nread = this.data.byteLength; + } + if (nread > buf.byteLength) { + nread = buf.byteLength; + } + if (nread === 0) { + return Promise.resolve(null); + } + copy(this.data, buf as Uint8Array); + this.data = this.data.subarray(nread); + return Promise.resolve(nread); + } +} + +async function testReadLine(input: Uint8Array) { + for (let stride = 1; stride < 2; stride++) { + let done = 0; + const reader = new TestReader(input, stride); + const l = new BufReader(reader, input.byteLength + 1); + while (true) { + const r = await l.readLine(); + if (r === null) { + break; + } + const { line, more } = r; + assertEquals(more, false); + const want = testOutput.subarray(done, done + line.byteLength); + assertEquals( + line, + want, + `Bad line at stride ${stride}: want: ${want} got: ${line}`, + ); + done += line.byteLength; + } + assertEquals( + done, + testOutput.byteLength, + `readLine didn't return everything: got: ${done}, ` + + `want: ${testOutput} (stride: ${stride})`, + ); + } +} + +Deno.test("bufioReadLine", async function () { + await testReadLine(testInput); + await testReadLine(testInputrn); +}); + +Deno.test("bufioReadLineBadResource", async () => { + const file = await Deno.open("README.md"); + const bufReader = new BufReader(file); + file.close(); + await assertRejects(async () => { + await bufReader.readLine(); + }, Deno.errors.BadResource); +}); + +Deno.test("bufioReadLineBufferFullError", async () => { + const input = "@".repeat(5000) + "\n"; + const bufReader = new BufReader(new StringReader(input)); + const r = await bufReader.readLine(); + + assert(r !== null); + + const { line, more } = r; + assertEquals(more, true); + assertEquals(line, encoder.encode("@".repeat(4096))); +}); + +/* TODO(kt3k): Enable this test +Deno.test( + "bufReaderShouldNotShareArrayBufferAcrossReads", + async function () { + const decoder = new TextDecoder(); + const data = "abcdefghijklmnopqrstuvwxyz"; + const bufSize = 25; + const b = new BufReader(new StringReader(data), bufSize); + + const r1 = (await b.readLine()) as ReadLineResult; + assert(r1 !== null); + assertEquals(decoder.decode(r1.line), "abcdefghijklmnopqrstuvwxy"); + + const r2 = (await b.readLine()) as ReadLineResult; + assert(r2 !== null); + assertEquals(decoder.decode(r2.line), "z"); + assert( + r1.line.buffer !== r2.line.buffer, + "array buffer should not be shared across reads", + ); + }, +); +*/ diff --git a/io/buf_writer.ts b/io/buf_writer.ts new file mode 100644 index 000000000000..601b3a6cb3d6 --- /dev/null +++ b/io/buf_writer.ts @@ -0,0 +1,222 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { copy } from "../bytes/copy.ts"; +import type { Writer, WriterSync } from "./types.d.ts"; + +const DEFAULT_BUF_SIZE = 4096; + +abstract class AbstractBufBase { + buf: Uint8Array; + usedBufferBytes = 0; + err: Error | null = null; + + constructor(buf: Uint8Array) { + this.buf = buf; + } + + /** Size returns the size of the underlying buffer in bytes. */ + size(): number { + return this.buf.byteLength; + } + + /** Returns how many bytes are unused in the buffer. */ + available(): number { + return this.buf.byteLength - this.usedBufferBytes; + } + + /** buffered returns the number of bytes that have been written into the + * current buffer. + */ + buffered(): number { + return this.usedBufferBytes; + } +} + +/** BufWriter implements buffering for an deno.Writer object. + * If an error occurs writing to a Writer, no more data will be + * accepted and all subsequent writes, and flush(), will return the error. + * After all data has been written, the client should call the + * flush() method to guarantee all data has been forwarded to + * the underlying deno.Writer. + */ +export class BufWriter extends AbstractBufBase implements Writer { + #writer: Writer; + + /** return new BufWriter unless writer is BufWriter */ + static create(writer: Writer, size: number = DEFAULT_BUF_SIZE): BufWriter { + return writer instanceof BufWriter ? writer : new BufWriter(writer, size); + } + + constructor(writer: Writer, size: number = DEFAULT_BUF_SIZE) { + super(new Uint8Array(size <= 0 ? DEFAULT_BUF_SIZE : size)); + this.#writer = writer; + } + + /** Discards any unflushed buffered data, clears any error, and + * resets buffer to write its output to w. + */ + reset(w: Writer) { + this.err = null; + this.usedBufferBytes = 0; + this.#writer = w; + } + + /** Flush writes any buffered data to the underlying io.Writer. */ + async flush() { + if (this.err !== null) throw this.err; + if (this.usedBufferBytes === 0) return; + + try { + const p = this.buf.subarray(0, this.usedBufferBytes); + let nwritten = 0; + while (nwritten < p.length) { + nwritten += await this.#writer.write(p.subarray(nwritten)); + } + } catch (e) { + if (e instanceof Error) { + this.err = e; + } + throw e; + } + + this.buf = new Uint8Array(this.buf.length); + this.usedBufferBytes = 0; + } + + /** Writes the contents of `data` into the buffer. If the contents won't fully + * fit into the buffer, those bytes that can are copied into the buffer, the + * buffer is the flushed to the writer and the remaining bytes are copied into + * the now empty buffer. + * + * @return the number of bytes written to the buffer. + */ + async write(data: Uint8Array): Promise { + if (this.err !== null) throw this.err; + if (data.length === 0) return 0; + + let totalBytesWritten = 0; + let numBytesWritten = 0; + while (data.byteLength > this.available()) { + if (this.buffered() === 0) { + // Large write, empty buffer. + // Write directly from data to avoid copy. + try { + numBytesWritten = await this.#writer.write(data); + } catch (e) { + if (e instanceof Error) { + this.err = e; + } + throw e; + } + } else { + numBytesWritten = copy(data, this.buf, this.usedBufferBytes); + this.usedBufferBytes += numBytesWritten; + await this.flush(); + } + totalBytesWritten += numBytesWritten; + data = data.subarray(numBytesWritten); + } + + numBytesWritten = copy(data, this.buf, this.usedBufferBytes); + this.usedBufferBytes += numBytesWritten; + totalBytesWritten += numBytesWritten; + return totalBytesWritten; + } +} + +/** BufWriterSync implements buffering for a deno.WriterSync object. + * If an error occurs writing to a WriterSync, no more data will be + * accepted and all subsequent writes, and flush(), will return the error. + * After all data has been written, the client should call the + * flush() method to guarantee all data has been forwarded to + * the underlying deno.WriterSync. + */ +export class BufWriterSync extends AbstractBufBase implements WriterSync { + #writer: WriterSync; + + /** return new BufWriterSync unless writer is BufWriterSync */ + static create( + writer: WriterSync, + size: number = DEFAULT_BUF_SIZE, + ): BufWriterSync { + return writer instanceof BufWriterSync + ? writer + : new BufWriterSync(writer, size); + } + + constructor(writer: WriterSync, size: number = DEFAULT_BUF_SIZE) { + super(new Uint8Array(size <= 0 ? DEFAULT_BUF_SIZE : size)); + this.#writer = writer; + } + + /** Discards any unflushed buffered data, clears any error, and + * resets buffer to write its output to w. + */ + reset(w: WriterSync) { + this.err = null; + this.usedBufferBytes = 0; + this.#writer = w; + } + + /** Flush writes any buffered data to the underlying io.WriterSync. */ + flush() { + if (this.err !== null) throw this.err; + if (this.usedBufferBytes === 0) return; + + try { + const p = this.buf.subarray(0, this.usedBufferBytes); + let nwritten = 0; + while (nwritten < p.length) { + nwritten += this.#writer.writeSync(p.subarray(nwritten)); + } + } catch (e) { + if (e instanceof Error) { + this.err = e; + } + throw e; + } + + this.buf = new Uint8Array(this.buf.length); + this.usedBufferBytes = 0; + } + + /** Writes the contents of `data` into the buffer. If the contents won't fully + * fit into the buffer, those bytes that can are copied into the buffer, the + * buffer is the flushed to the writer and the remaining bytes are copied into + * the now empty buffer. + * + * @return the number of bytes written to the buffer. + */ + writeSync(data: Uint8Array): number { + if (this.err !== null) throw this.err; + if (data.length === 0) return 0; + + let totalBytesWritten = 0; + let numBytesWritten = 0; + while (data.byteLength > this.available()) { + if (this.buffered() === 0) { + // Large write, empty buffer. + // Write directly from data to avoid copy. + try { + numBytesWritten = this.#writer.writeSync(data); + } catch (e) { + if (e instanceof Error) { + this.err = e; + } + throw e; + } + } else { + numBytesWritten = copy(data, this.buf, this.usedBufferBytes); + this.usedBufferBytes += numBytesWritten; + this.flush(); + } + totalBytesWritten += numBytesWritten; + data = data.subarray(numBytesWritten); + } + + numBytesWritten = copy(data, this.buf, this.usedBufferBytes); + this.usedBufferBytes += numBytesWritten; + totalBytesWritten += numBytesWritten; + return totalBytesWritten; + } +} diff --git a/io/buf_writer_test.ts b/io/buf_writer_test.ts new file mode 100644 index 000000000000..0b56ffe315c9 --- /dev/null +++ b/io/buf_writer_test.ts @@ -0,0 +1,160 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +// This code has been ported almost directly from Go's src/bytes/buffer_test.go +// Copyright 2009 The Go Authors. All rights reserved. BSD license. +// https://github.com/golang/go/blob/master/LICENSE +import { assertEquals } from "../testing/asserts.ts"; +import { BufWriter, BufWriterSync } from "./buf_writer.ts"; +import { Buffer } from "./buffer.ts"; +import { StringWriter } from "./string_writer.ts"; +import { bufsizes } from "./_test_common.ts"; + +Deno.test("bufioWriter", async function () { + const data = new Uint8Array(8192); + + for (let i = 0; i < data.byteLength; i++) { + data[i] = " ".charCodeAt(0) + (i % ("~".charCodeAt(0) - " ".charCodeAt(0))); + } + + const w = new Buffer(); + for (const nwrite of bufsizes) { + for (const bs of bufsizes) { + // Write nwrite bytes using buffer size bs. + // Check that the right amount makes it out + // and that the data is correct. + + w.reset(); + const buf = new BufWriter(w, bs); + + const context = `nwrite=${nwrite} bufsize=${bs}`; + const n = await buf.write(data.subarray(0, nwrite)); + assertEquals(n, nwrite, context); + + await buf.flush(); + + const written = w.bytes(); + assertEquals(written.byteLength, nwrite); + + for (let l = 0; l < written.byteLength; l++) { + assertEquals(written[l], data[l]); + } + } + } +}); + +Deno.test("bufioWriterSync", function () { + const data = new Uint8Array(8192); + + for (let i = 0; i < data.byteLength; i++) { + data[i] = " ".charCodeAt(0) + (i % ("~".charCodeAt(0) - " ".charCodeAt(0))); + } + + const w = new Buffer(); + for (const nwrite of bufsizes) { + for (const bs of bufsizes) { + // Write nwrite bytes using buffer size bs. + // Check that the right amount makes it out + // and that the data is correct. + + w.reset(); + const buf = new BufWriterSync(w, bs); + + const context = `nwrite=${nwrite} bufsize=${bs}`; + const n = buf.writeSync(data.subarray(0, nwrite)); + assertEquals(n, nwrite, context); + + buf.flush(); + + const written = w.bytes(); + assertEquals(written.byteLength, nwrite); + + for (let l = 0; l < written.byteLength; l++) { + assertEquals(written[l], data[l]); + } + } + } +}); + +Deno.test({ + name: "Reset buffer after flush", + async fn() { + const stringWriter = new StringWriter(); + const bufWriter = new BufWriter(stringWriter); + const encoder = new TextEncoder(); + await bufWriter.write(encoder.encode("hello\nworld\nhow\nare\nyou?\n\n")); + await bufWriter.flush(); + await bufWriter.write(encoder.encode("foobar\n\n")); + await bufWriter.flush(); + const actual = stringWriter.toString(); + assertEquals(actual, "hello\nworld\nhow\nare\nyou?\n\nfoobar\n\n"); + }, +}); + +Deno.test({ + name: "Reset buffer after flush sync", + fn() { + const stringWriter = new StringWriter(); + const bufWriter = new BufWriterSync(stringWriter); + const encoder = new TextEncoder(); + bufWriter.writeSync(encoder.encode("hello\nworld\nhow\nare\nyou?\n\n")); + bufWriter.flush(); + bufWriter.writeSync(encoder.encode("foobar\n\n")); + bufWriter.flush(); + const actual = stringWriter.toString(); + assertEquals(actual, "hello\nworld\nhow\nare\nyou?\n\nfoobar\n\n"); + }, +}); + +Deno.test({ + name: "BufWriter.flush should write all bytes", + async fn() { + const bufSize = 16 * 1024; + const data = new Uint8Array(bufSize); + data.fill("a".charCodeAt(0)); + + const cache: Uint8Array[] = []; + const writer: Deno.Writer = { + write(p: Uint8Array): Promise { + cache.push(p.subarray(0, 1)); + + // Writer that only writes 1 byte at a time + return Promise.resolve(1); + }, + }; + + const bufWriter = new BufWriter(writer); + await bufWriter.write(data); + + await bufWriter.flush(); + const buf = new Uint8Array(cache.length); + for (let i = 0; i < cache.length; i++) buf.set(cache[i], i); + + assertEquals(data, buf); + }, +}); + +Deno.test({ + name: "BufWriterSync.flush should write all bytes", + fn() { + const bufSize = 16 * 1024; + const data = new Uint8Array(bufSize); + data.fill("a".charCodeAt(0)); + + const cache: Uint8Array[] = []; + const writer: Deno.WriterSync = { + writeSync(p: Uint8Array): number { + cache.push(p.subarray(0, 1)); + // Writer that only writes 1 byte at a time + return 1; + }, + }; + + const bufWriter = new BufWriterSync(writer); + bufWriter.writeSync(data); + + bufWriter.flush(); + const buf = new Uint8Array(cache.length); + for (let i = 0; i < cache.length; i++) buf.set(cache[i], i); + + assertEquals(data, buf); + }, +}); diff --git a/io/buffer.ts b/io/buffer.ts index 10bc3ab670cc..1708dc265521 100644 --- a/io/buffer.ts +++ b/io/buffer.ts @@ -1,9 +1,19 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. import { assert } from "../_util/asserts.ts"; -import { BytesList } from "../bytes/bytes_list.ts"; -import { concat } from "../bytes/concat.ts"; import { copy } from "../bytes/copy.ts"; -import type { Reader, ReaderSync, Writer, WriterSync } from "./types.d.ts"; +import type { Reader, ReaderSync } from "./types.d.ts"; +import { + BufferFullError as _BufferFullError, + BufReader as _BufReader, + PartialReadError as _PartialReadError, +} from "./buf_reader.ts"; +import { + BufWriter as _BufWriter, + BufWriterSync as _BufWriterSync, +} from "./buf_writer.ts"; +import { readDelim as _readDelim } from "./read_delim.ts"; +import { readStringDelim as _readStringDelim } from "./read_string_delim.ts"; +import { readLines as _readLines } from "./read_lines.ts"; // MIN_READ is the minimum ArrayBuffer size passed to a read call by // buffer.ReadFrom. As long as the Buffer has at least MIN_READ bytes beyond @@ -248,728 +258,62 @@ export class Buffer { } } -const DEFAULT_BUF_SIZE = 4096; -const MIN_BUF_SIZE = 16; -const MAX_CONSECUTIVE_EMPTY_READS = 100; -const CR = "\r".charCodeAt(0); -const LF = "\n".charCodeAt(0); +/** @deprecated (will be removed after 0.170.0) Import from `std/io/buf_reader.ts` instead */ +export const BufferFullError = _BufferFullError; -export class BufferFullError extends Error { - override name = "BufferFullError"; - constructor(public partial: Uint8Array) { - super("Buffer full"); - } -} +/** @deprecated (will be removed after 0.170.0) Import from `std/io/buf_reader.ts` instead */ +export const PartialReadError = _PartialReadError; -export class PartialReadError extends Error { - override name = "PartialReadError"; - partial?: Uint8Array; - constructor() { - super("Encountered UnexpectedEof, data only partially read"); - } -} - -/** Result type returned by of BufReader.readLine(). */ +/** + * @deprecated (will be removed after 0.170.0) Import from `std/io/buf_reader.ts` instead + * + * Result type returned by of BufReader.readLine(). + */ export interface ReadLineResult { line: Uint8Array; more: boolean; } -/** BufReader implements buffering for a Reader object. */ -export class BufReader implements Reader { - #buf!: Uint8Array; - #rd!: Reader; // Reader provided by caller. - #r = 0; // buf read position. - #w = 0; // buf write position. - #eof = false; - // private lastByte: number; - // private lastCharSize: number; - - /** return new BufReader unless r is BufReader */ - static create(r: Reader, size: number = DEFAULT_BUF_SIZE): BufReader { - return r instanceof BufReader ? r : new BufReader(r, size); - } - - constructor(rd: Reader, size: number = DEFAULT_BUF_SIZE) { - if (size < MIN_BUF_SIZE) { - size = MIN_BUF_SIZE; - } - this.#reset(new Uint8Array(size), rd); - } - - /** Returns the size of the underlying buffer in bytes. */ - size(): number { - return this.#buf.byteLength; - } - - buffered(): number { - return this.#w - this.#r; - } - - // Reads a new chunk into the buffer. - #fill = async () => { - // Slide existing data to beginning. - if (this.#r > 0) { - this.#buf.copyWithin(0, this.#r, this.#w); - this.#w -= this.#r; - this.#r = 0; - } - - if (this.#w >= this.#buf.byteLength) { - throw Error("bufio: tried to fill full buffer"); - } - - // Read new data: try a limited number of times. - for (let i = MAX_CONSECUTIVE_EMPTY_READS; i > 0; i--) { - const rr = await this.#rd.read(this.#buf.subarray(this.#w)); - if (rr === null) { - this.#eof = true; - return; - } - assert(rr >= 0, "negative read"); - this.#w += rr; - if (rr > 0) { - return; - } - } - - throw new Error( - `No progress after ${MAX_CONSECUTIVE_EMPTY_READS} read() calls`, - ); - }; - - /** Discards any buffered data, resets all state, and switches - * the buffered reader to read from r. - */ - reset(r: Reader) { - this.#reset(this.#buf, r); - } - - #reset = (buf: Uint8Array, rd: Reader) => { - this.#buf = buf; - this.#rd = rd; - this.#eof = false; - // this.lastByte = -1; - // this.lastCharSize = -1; - }; - - /** reads data into p. - * It returns the number of bytes read into p. - * The bytes are taken from at most one Read on the underlying Reader, - * hence n may be less than len(p). - * To read exactly len(p) bytes, use io.ReadFull(b, p). - */ - async read(p: Uint8Array): Promise { - let rr: number | null = p.byteLength; - if (p.byteLength === 0) return rr; - - if (this.#r === this.#w) { - if (p.byteLength >= this.#buf.byteLength) { - // Large read, empty buffer. - // Read directly into p to avoid copy. - const rr = await this.#rd.read(p); - const nread = rr ?? 0; - assert(nread >= 0, "negative read"); - // if (rr.nread > 0) { - // this.lastByte = p[rr.nread - 1]; - // this.lastCharSize = -1; - // } - return rr; - } - - // One read. - // Do not use this.fill, which will loop. - this.#r = 0; - this.#w = 0; - rr = await this.#rd.read(this.#buf); - if (rr === 0 || rr === null) return rr; - assert(rr >= 0, "negative read"); - this.#w += rr; - } - - // copy as much as we can - const copied = copy(this.#buf.subarray(this.#r, this.#w), p, 0); - this.#r += copied; - // this.lastByte = this.buf[this.r - 1]; - // this.lastCharSize = -1; - return copied; - } - - /** reads exactly `p.length` bytes into `p`. - * - * If successful, `p` is returned. - * - * If the end of the underlying stream has been reached, and there are no more - * bytes available in the buffer, `readFull()` returns `null` instead. - * - * An error is thrown if some bytes could be read, but not enough to fill `p` - * entirely before the underlying stream reported an error or EOF. Any error - * thrown will have a `partial` property that indicates the slice of the - * buffer that has been successfully filled with data. - * - * Ported from https://golang.org/pkg/io/#ReadFull - */ - async readFull(p: Uint8Array): Promise { - let bytesRead = 0; - while (bytesRead < p.length) { - try { - const rr = await this.read(p.subarray(bytesRead)); - if (rr === null) { - if (bytesRead === 0) { - return null; - } else { - throw new PartialReadError(); - } - } - bytesRead += rr; - } catch (err) { - if (err instanceof PartialReadError) { - err.partial = p.subarray(0, bytesRead); - } else if (err instanceof Error) { - const e = new PartialReadError(); - e.partial = p.subarray(0, bytesRead); - e.stack = err.stack; - e.message = err.message; - e.cause = err.cause; - throw err; - } - throw err; - } - } - return p; - } - - /** Returns the next byte [0, 255] or `null`. */ - async readByte(): Promise { - while (this.#r === this.#w) { - if (this.#eof) return null; - await this.#fill(); // buffer is empty. - } - const c = this.#buf[this.#r]; - this.#r++; - // this.lastByte = c; - return c; - } - - /** readString() reads until the first occurrence of delim in the input, - * returning a string containing the data up to and including the delimiter. - * If ReadString encounters an error before finding a delimiter, - * it returns the data read before the error and the error itself - * (often `null`). - * ReadString returns err != nil if and only if the returned data does not end - * in delim. - * For simple uses, a Scanner may be more convenient. - */ - async readString(delim: string): Promise { - if (delim.length !== 1) { - throw new Error("Delimiter should be a single character"); - } - const buffer = await this.readSlice(delim.charCodeAt(0)); - if (buffer === null) return null; - return new TextDecoder().decode(buffer); - } - - /** `readLine()` is a low-level line-reading primitive. Most callers should - * use `readString('\n')` instead or use a Scanner. - * - * `readLine()` tries to return a single line, not including the end-of-line - * bytes. If the line was too long for the buffer then `more` is set and the - * beginning of the line is returned. The rest of the line will be returned - * from future calls. `more` will be false when returning the last fragment - * of the line. The returned buffer is only valid until the next call to - * `readLine()`. - * - * The text returned from ReadLine does not include the line end ("\r\n" or - * "\n"). - * - * When the end of the underlying stream is reached, the final bytes in the - * stream are returned. No indication or error is given if the input ends - * without a final line end. When there are no more trailing bytes to read, - * `readLine()` returns `null`. - * - * Calling `unreadByte()` after `readLine()` will always unread the last byte - * read (possibly a character belonging to the line end) even if that byte is - * not part of the line returned by `readLine()`. - */ - async readLine(): Promise { - let line: Uint8Array | null = null; - - try { - line = await this.readSlice(LF); - } catch (err) { - let partial; - if (err instanceof PartialReadError) { - partial = err.partial; - assert( - partial instanceof Uint8Array, - "bufio: caught error from `readSlice()` without `partial` property", - ); - } - - // Don't throw if `readSlice()` failed with `BufferFullError`, instead we - // just return whatever is available and set the `more` flag. - if (!(err instanceof BufferFullError)) { - throw err; - } - - partial = err.partial; - - // Handle the case where "\r\n" straddles the buffer. - if ( - !this.#eof && partial && - partial.byteLength > 0 && - partial[partial.byteLength - 1] === CR - ) { - // Put the '\r' back on buf and drop it from line. - // Let the next call to ReadLine check for "\r\n". - assert(this.#r > 0, "bufio: tried to rewind past start of buffer"); - this.#r--; - partial = partial.subarray(0, partial.byteLength - 1); - } - - if (partial) { - return { line: partial, more: !this.#eof }; - } - } - - if (line === null) { - return null; - } - - if (line.byteLength === 0) { - return { line, more: false }; - } - - if (line[line.byteLength - 1] == LF) { - let drop = 1; - if (line.byteLength > 1 && line[line.byteLength - 2] === CR) { - drop = 2; - } - line = line.subarray(0, line.byteLength - drop); - } - return { line, more: false }; - } - - /** `readSlice()` reads until the first occurrence of `delim` in the input, - * returning a slice pointing at the bytes in the buffer. The bytes stop - * being valid at the next read. - * - * If `readSlice()` encounters an error before finding a delimiter, or the - * buffer fills without finding a delimiter, it throws an error with a - * `partial` property that contains the entire buffer. - * - * If `readSlice()` encounters the end of the underlying stream and there are - * any bytes left in the buffer, the rest of the buffer is returned. In other - * words, EOF is always treated as a delimiter. Once the buffer is empty, - * it returns `null`. - * - * Because the data returned from `readSlice()` will be overwritten by the - * next I/O operation, most clients should use `readString()` instead. - */ - async readSlice(delim: number): Promise { - let s = 0; // search start index - let slice: Uint8Array | undefined; - - while (true) { - // Search buffer. - let i = this.#buf.subarray(this.#r + s, this.#w).indexOf(delim); - if (i >= 0) { - i += s; - slice = this.#buf.subarray(this.#r, this.#r + i + 1); - this.#r += i + 1; - break; - } - - // EOF? - if (this.#eof) { - if (this.#r === this.#w) { - return null; - } - slice = this.#buf.subarray(this.#r, this.#w); - this.#r = this.#w; - break; - } - - // Buffer full? - if (this.buffered() >= this.#buf.byteLength) { - this.#r = this.#w; - // #4521 The internal buffer should not be reused across reads because it causes corruption of data. - const oldbuf = this.#buf; - const newbuf = this.#buf.slice(0); - this.#buf = newbuf; - throw new BufferFullError(oldbuf); - } - - s = this.#w - this.#r; // do not rescan area we scanned before - - // Buffer is not full. - try { - await this.#fill(); - } catch (err) { - if (err instanceof PartialReadError) { - err.partial = slice; - } else if (err instanceof Error) { - const e = new PartialReadError(); - e.partial = slice; - e.stack = err.stack; - e.message = err.message; - e.cause = err.cause; - throw err; - } - throw err; - } - } - - // Handle last byte, if any. - // const i = slice.byteLength - 1; - // if (i >= 0) { - // this.lastByte = slice[i]; - // this.lastCharSize = -1 - // } - - return slice; - } - - /** `peek()` returns the next `n` bytes without advancing the reader. The - * bytes stop being valid at the next read call. - * - * When the end of the underlying stream is reached, but there are unread - * bytes left in the buffer, those bytes are returned. If there are no bytes - * left in the buffer, it returns `null`. - * - * If an error is encountered before `n` bytes are available, `peek()` throws - * an error with the `partial` property set to a slice of the buffer that - * contains the bytes that were available before the error occurred. - */ - async peek(n: number): Promise { - if (n < 0) { - throw Error("negative count"); - } - - let avail = this.#w - this.#r; - while (avail < n && avail < this.#buf.byteLength && !this.#eof) { - try { - await this.#fill(); - } catch (err) { - if (err instanceof PartialReadError) { - err.partial = this.#buf.subarray(this.#r, this.#w); - } else if (err instanceof Error) { - const e = new PartialReadError(); - e.partial = this.#buf.subarray(this.#r, this.#w); - e.stack = err.stack; - e.message = err.message; - e.cause = err.cause; - throw err; - } - throw err; - } - avail = this.#w - this.#r; - } - - if (avail === 0 && this.#eof) { - return null; - } else if (avail < n && this.#eof) { - return this.#buf.subarray(this.#r, this.#r + avail); - } else if (avail < n) { - throw new BufferFullError(this.#buf.subarray(this.#r, this.#w)); - } - - return this.#buf.subarray(this.#r, this.#r + n); - } -} - -abstract class AbstractBufBase { - buf: Uint8Array; - usedBufferBytes = 0; - err: Error | null = null; - - constructor(buf: Uint8Array) { - this.buf = buf; - } - - /** Size returns the size of the underlying buffer in bytes. */ - size(): number { - return this.buf.byteLength; - } - - /** Returns how many bytes are unused in the buffer. */ - available(): number { - return this.buf.byteLength - this.usedBufferBytes; - } - - /** buffered returns the number of bytes that have been written into the - * current buffer. - */ - buffered(): number { - return this.usedBufferBytes; - } -} +/** + * @deprecated (will be removed after 0.170.0) Import from `std/io/buf_reader.ts` instead + * + * BufReader implements buffering for a Reader object. + */ +export const BufReader = _BufReader; -/** BufWriter implements buffering for an deno.Writer object. +/** + * @deprecated (will be removed after 0.170.0) Import from `std/io/buf_writer.ts` instead + * + * BufWriter implements buffering for an deno.Writer object. * If an error occurs writing to a Writer, no more data will be * accepted and all subsequent writes, and flush(), will return the error. * After all data has been written, the client should call the * flush() method to guarantee all data has been forwarded to * the underlying deno.Writer. */ -export class BufWriter extends AbstractBufBase implements Writer { - #writer: Writer; - - /** return new BufWriter unless writer is BufWriter */ - static create(writer: Writer, size: number = DEFAULT_BUF_SIZE): BufWriter { - return writer instanceof BufWriter ? writer : new BufWriter(writer, size); - } - - constructor(writer: Writer, size: number = DEFAULT_BUF_SIZE) { - super(new Uint8Array(size <= 0 ? DEFAULT_BUF_SIZE : size)); - this.#writer = writer; - } - - /** Discards any unflushed buffered data, clears any error, and - * resets buffer to write its output to w. - */ - reset(w: Writer) { - this.err = null; - this.usedBufferBytes = 0; - this.#writer = w; - } - - /** Flush writes any buffered data to the underlying io.Writer. */ - async flush() { - if (this.err !== null) throw this.err; - if (this.usedBufferBytes === 0) return; - - try { - const p = this.buf.subarray(0, this.usedBufferBytes); - let nwritten = 0; - while (nwritten < p.length) { - nwritten += await this.#writer.write(p.subarray(nwritten)); - } - } catch (e) { - if (e instanceof Error) { - this.err = e; - } - throw e; - } - - this.buf = new Uint8Array(this.buf.length); - this.usedBufferBytes = 0; - } - - /** Writes the contents of `data` into the buffer. If the contents won't fully - * fit into the buffer, those bytes that can are copied into the buffer, the - * buffer is the flushed to the writer and the remaining bytes are copied into - * the now empty buffer. - * - * @return the number of bytes written to the buffer. - */ - async write(data: Uint8Array): Promise { - if (this.err !== null) throw this.err; - if (data.length === 0) return 0; - - let totalBytesWritten = 0; - let numBytesWritten = 0; - while (data.byteLength > this.available()) { - if (this.buffered() === 0) { - // Large write, empty buffer. - // Write directly from data to avoid copy. - try { - numBytesWritten = await this.#writer.write(data); - } catch (e) { - if (e instanceof Error) { - this.err = e; - } - throw e; - } - } else { - numBytesWritten = copy(data, this.buf, this.usedBufferBytes); - this.usedBufferBytes += numBytesWritten; - await this.flush(); - } - totalBytesWritten += numBytesWritten; - data = data.subarray(numBytesWritten); - } - - numBytesWritten = copy(data, this.buf, this.usedBufferBytes); - this.usedBufferBytes += numBytesWritten; - totalBytesWritten += numBytesWritten; - return totalBytesWritten; - } -} +export const BufWriter = _BufWriter; -/** BufWriterSync implements buffering for a deno.WriterSync object. +/** + * @deprecated (will be removed after 0.170.0) Import from `std/io/buf_writer.ts` instead + * + * BufWriterSync implements buffering for a deno.WriterSync object. * If an error occurs writing to a WriterSync, no more data will be * accepted and all subsequent writes, and flush(), will return the error. * After all data has been written, the client should call the * flush() method to guarantee all data has been forwarded to * the underlying deno.WriterSync. */ -export class BufWriterSync extends AbstractBufBase implements WriterSync { - #writer: WriterSync; - - /** return new BufWriterSync unless writer is BufWriterSync */ - static create( - writer: WriterSync, - size: number = DEFAULT_BUF_SIZE, - ): BufWriterSync { - return writer instanceof BufWriterSync - ? writer - : new BufWriterSync(writer, size); - } - - constructor(writer: WriterSync, size: number = DEFAULT_BUF_SIZE) { - super(new Uint8Array(size <= 0 ? DEFAULT_BUF_SIZE : size)); - this.#writer = writer; - } - - /** Discards any unflushed buffered data, clears any error, and - * resets buffer to write its output to w. - */ - reset(w: WriterSync) { - this.err = null; - this.usedBufferBytes = 0; - this.#writer = w; - } - - /** Flush writes any buffered data to the underlying io.WriterSync. */ - flush() { - if (this.err !== null) throw this.err; - if (this.usedBufferBytes === 0) return; - - try { - const p = this.buf.subarray(0, this.usedBufferBytes); - let nwritten = 0; - while (nwritten < p.length) { - nwritten += this.#writer.writeSync(p.subarray(nwritten)); - } - } catch (e) { - if (e instanceof Error) { - this.err = e; - } - throw e; - } - - this.buf = new Uint8Array(this.buf.length); - this.usedBufferBytes = 0; - } - - /** Writes the contents of `data` into the buffer. If the contents won't fully - * fit into the buffer, those bytes that can are copied into the buffer, the - * buffer is the flushed to the writer and the remaining bytes are copied into - * the now empty buffer. - * - * @return the number of bytes written to the buffer. - */ - writeSync(data: Uint8Array): number { - if (this.err !== null) throw this.err; - if (data.length === 0) return 0; - - let totalBytesWritten = 0; - let numBytesWritten = 0; - while (data.byteLength > this.available()) { - if (this.buffered() === 0) { - // Large write, empty buffer. - // Write directly from data to avoid copy. - try { - numBytesWritten = this.#writer.writeSync(data); - } catch (e) { - if (e instanceof Error) { - this.err = e; - } - throw e; - } - } else { - numBytesWritten = copy(data, this.buf, this.usedBufferBytes); - this.usedBufferBytes += numBytesWritten; - this.flush(); - } - totalBytesWritten += numBytesWritten; - data = data.subarray(numBytesWritten); - } - - numBytesWritten = copy(data, this.buf, this.usedBufferBytes); - this.usedBufferBytes += numBytesWritten; - totalBytesWritten += numBytesWritten; - return totalBytesWritten; - } -} - -/** Generate longest proper prefix which is also suffix array. */ -function createLPS(pat: Uint8Array): Uint8Array { - const lps = new Uint8Array(pat.length); - lps[0] = 0; - let prefixEnd = 0; - let i = 1; - while (i < lps.length) { - if (pat[i] == pat[prefixEnd]) { - prefixEnd++; - lps[i] = prefixEnd; - i++; - } else if (prefixEnd === 0) { - lps[i] = 0; - i++; - } else { - prefixEnd = lps[prefixEnd - 1]; - } - } - return lps; -} - -/** Read delimited bytes from a Reader. */ -export async function* readDelim( - reader: Reader, - delim: Uint8Array, -): AsyncIterableIterator { - // Avoid unicode problems - const delimLen = delim.length; - const delimLPS = createLPS(delim); - const chunks = new BytesList(); - const bufSize = Math.max(1024, delimLen + 1); +export const BufWriterSync = _BufWriterSync; - // Modified KMP - let inspectIndex = 0; - let matchIndex = 0; - while (true) { - const inspectArr = new Uint8Array(bufSize); - const result = await reader.read(inspectArr); - if (result === null) { - // Yield last chunk. - yield chunks.concat(); - return; - } else if (result < 0) { - // Discard all remaining and silently fail. - return; - } - chunks.add(inspectArr, 0, result); - let localIndex = 0; - while (inspectIndex < chunks.size()) { - if (inspectArr[localIndex] === delim[matchIndex]) { - inspectIndex++; - localIndex++; - matchIndex++; - if (matchIndex === delimLen) { - // Full match - const matchEnd = inspectIndex - delimLen; - const readyBytes = chunks.slice(0, matchEnd); - yield readyBytes; - // Reset match, different from KMP. - chunks.shift(inspectIndex); - inspectIndex = 0; - matchIndex = 0; - } - } else { - if (matchIndex === 0) { - inspectIndex++; - localIndex++; - } else { - matchIndex = delimLPS[matchIndex - 1]; - } - } - } - } -} +/** + * @deprecated (will be removed after 0.170.0) Import from `std/io/read_delim.ts` instead + * + * Read delimited bytes from a Reader. */ +export const readDelim = _readDelim; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/read_string_delim.ts` instead + * * Read Reader chunk by chunk, splitting based on delimiter. * * @example @@ -985,23 +329,11 @@ export async function* readDelim( * } * ``` */ -export async function* readStringDelim( - reader: Reader, - delim: string, - decoderOpts?: { - encoding?: string; - fatal?: boolean; - ignoreBOM?: boolean; - }, -): AsyncIterableIterator { - const encoder = new TextEncoder(); - const decoder = new TextDecoder(decoderOpts?.encoding, decoderOpts); - for await (const chunk of readDelim(reader, encoder.encode(delim))) { - yield decoder.decode(chunk); - } -} +export const readStringDelim = _readStringDelim; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/read_lines.ts` instead + * * Read strings line-by-line from a Reader. * * @example @@ -1017,29 +349,4 @@ export async function* readStringDelim( * } * ``` */ -export async function* readLines( - reader: Reader, - decoderOpts?: { - encoding?: string; - fatal?: boolean; - ignoreBOM?: boolean; - }, -): AsyncIterableIterator { - const bufReader = new BufReader(reader); - let chunks: Uint8Array[] = []; - const decoder = new TextDecoder(decoderOpts?.encoding, decoderOpts); - while (true) { - const res = await bufReader.readLine(); - if (!res) { - if (chunks.length > 0) { - yield decoder.decode(concat(...chunks)); - } - break; - } - chunks.push(res.line); - if (!res.more) { - yield decoder.decode(concat(...chunks)); - chunks = []; - } - } -} +export const readLines = _readLines; diff --git a/io/buffer_test.ts b/io/buffer_test.ts index 7808e38cc8a6..e705b83a4a4e 100644 --- a/io/buffer_test.ts +++ b/io/buffer_test.ts @@ -3,28 +3,14 @@ // This code has been ported almost directly from Go's src/bytes/buffer_test.go // Copyright 2009 The Go Authors. All rights reserved. BSD license. // https://github.com/golang/go/blob/master/LICENSE -import { copy } from "../bytes/copy.ts"; import { assert, assertEquals, assertRejects, assertThrows, - fail, } from "../testing/asserts.ts"; -import { - Buffer, - BufferFullError, - BufReader, - BufWriter, - BufWriterSync, - PartialReadError, - readLines, - readStringDelim, -} from "./buffer.ts"; -import * as iotest from "./_iotest.ts"; -import { StringReader } from "./readers.ts"; +import { Buffer } from "./buffer.ts"; import { writeAllSync } from "../streams/write_all.ts"; -import { StringWriter } from "./writers.ts"; const MAX_SIZE = 2 ** 32 - 2; // N controls how many iterations of certain checks are performed. @@ -419,652 +405,3 @@ Deno.test("testBufferBytesCopyFalseGrowExactBytes", () => { assertEquals(actualBytes.byteLength, bufSize); assertEquals(actualBytes.buffer.byteLength, actualBytes.byteLength); }); - -async function readBytes(buf: BufReader): Promise { - const b = new Uint8Array(1000); - let nb = 0; - while (true) { - const c = await buf.readByte(); - if (c === null) { - break; // EOF - } - b[nb] = c; - nb++; - } - const decoder = new TextDecoder(); - return decoder.decode(b.subarray(0, nb)); -} - -Deno.test("bufioReaderSimple", async function () { - const data = "hello world"; - const b = new BufReader(new StringReader(data)); - const s = await readBytes(b); - assertEquals(s, data); -}); - -interface ReadMaker { - name: string; - fn: (r: Deno.Reader) => Deno.Reader; -} - -const readMakers: ReadMaker[] = [ - { name: "full", fn: (r): Deno.Reader => r }, - { - name: "byte", - fn: (r): iotest.OneByteReader => new iotest.OneByteReader(r), - }, - { name: "half", fn: (r): iotest.HalfReader => new iotest.HalfReader(r) }, - // TODO(bartlomieju): { name: "data+err", r => new iotest.DataErrReader(r) }, - // { name: "timeout", fn: r => new iotest.TimeoutReader(r) }, -]; - -// Call read to accumulate the text of a file -async function reads(buf: BufReader, m: number): Promise { - const b = new Uint8Array(1000); - let nb = 0; - while (true) { - const result = await buf.read(b.subarray(nb, nb + m)); - if (result === null) { - break; - } - nb += result; - } - const decoder = new TextDecoder(); - return decoder.decode(b.subarray(0, nb)); -} - -interface NamedBufReader { - name: string; - fn: (r: BufReader) => Promise; -} - -const bufreaders: NamedBufReader[] = [ - { name: "1", fn: (b: BufReader): Promise => reads(b, 1) }, - { name: "2", fn: (b: BufReader): Promise => reads(b, 2) }, - { name: "3", fn: (b: BufReader): Promise => reads(b, 3) }, - { name: "4", fn: (b: BufReader): Promise => reads(b, 4) }, - { name: "5", fn: (b: BufReader): Promise => reads(b, 5) }, - { name: "7", fn: (b: BufReader): Promise => reads(b, 7) }, - { name: "bytes", fn: readBytes }, - // { name: "lines", fn: readLines }, -]; - -const MIN_READ_BUFFER_SIZE = 16; -const bufsizes: number[] = [ - 0, - MIN_READ_BUFFER_SIZE, - 23, - 32, - 46, - 64, - 93, - 128, - 1024, - 4096, -]; - -Deno.test("bufioBufReader", async function () { - const texts = new Array(31); - let str = ""; - let all = ""; - for (let i = 0; i < texts.length - 1; i++) { - texts[i] = str + "\n"; - all += texts[i]; - str += String.fromCharCode((i % 26) + 97); - } - texts[texts.length - 1] = all; - - for (const text of texts) { - for (const readmaker of readMakers) { - for (const bufreader of bufreaders) { - for (const bufsize of bufsizes) { - const read = readmaker.fn(new StringReader(text)); - const buf = new BufReader(read, bufsize); - const s = await bufreader.fn(buf); - const debugStr = `reader=${readmaker.name} ` + - `fn=${bufreader.name} bufsize=${bufsize} want=${text} got=${s}`; - assertEquals(s, text, debugStr); - } - } - } - } -}); - -Deno.test("bufioBufferFull", async function () { - const longString = - "And now, hello, world! It is the time for all good men to come to the" + - " aid of their party"; - const buf = new BufReader(new StringReader(longString), MIN_READ_BUFFER_SIZE); - const decoder = new TextDecoder(); - - try { - await buf.readSlice("!".charCodeAt(0)); - fail("readSlice should throw"); - } catch (err) { - assert(err instanceof BufferFullError); - assert(err.partial instanceof Uint8Array); - assertEquals(decoder.decode(err.partial), "And now, hello, "); - } - - const line = await buf.readSlice("!".charCodeAt(0)); - assert(line !== null); - const actual = decoder.decode(line); - assertEquals(actual, "world!"); -}); - -Deno.test("bufioReadString", async function () { - const string = "And now, hello world!"; - const buf = new BufReader(new StringReader(string), MIN_READ_BUFFER_SIZE); - - const line = await buf.readString(","); - assert(line !== null); - assertEquals(line, "And now,"); - assertEquals(line.length, 8); - - const line2 = await buf.readString(","); - assert(line2 !== null); - assertEquals(line2, " hello world!"); - - assertEquals(await buf.readString(","), null); - - try { - await buf.readString("deno"); - - fail("should throw"); - } catch (err) { - assert(err instanceof Error); - assert(err.message, "Delimiter should be a single character"); - } -}); - -const encoder = new TextEncoder(); - -const testInput = encoder.encode( - "012\n345\n678\n9ab\ncde\nfgh\nijk\nlmn\nopq\nrst\nuvw\nxy", -); -const testInputrn = encoder.encode( - "012\r\n345\r\n678\r\n9ab\r\ncde\r\nfgh\r\nijk\r\nlmn\r\nopq\r\nrst\r\n" + - "uvw\r\nxy\r\n\n\r\n", -); -const testOutput = encoder.encode("0123456789abcdefghijklmnopqrstuvwxy"); - -// TestReader wraps a Uint8Array and returns reads of a specific length. -class TestReader implements Deno.Reader { - constructor(private data: Uint8Array, private stride: number) {} - - read(buf: Uint8Array): Promise { - let nread = this.stride; - if (nread > this.data.byteLength) { - nread = this.data.byteLength; - } - if (nread > buf.byteLength) { - nread = buf.byteLength; - } - if (nread === 0) { - return Promise.resolve(null); - } - copy(this.data, buf as Uint8Array); - this.data = this.data.subarray(nread); - return Promise.resolve(nread); - } -} - -async function testReadLine(input: Uint8Array) { - for (let stride = 1; stride < 2; stride++) { - let done = 0; - const reader = new TestReader(input, stride); - const l = new BufReader(reader, input.byteLength + 1); - while (true) { - const r = await l.readLine(); - if (r === null) { - break; - } - const { line, more } = r; - assertEquals(more, false); - const want = testOutput.subarray(done, done + line.byteLength); - assertEquals( - line, - want, - `Bad line at stride ${stride}: want: ${want} got: ${line}`, - ); - done += line.byteLength; - } - assertEquals( - done, - testOutput.byteLength, - `readLine didn't return everything: got: ${done}, ` + - `want: ${testOutput} (stride: ${stride})`, - ); - } -} - -Deno.test("bufioReadLine", async function () { - await testReadLine(testInput); - await testReadLine(testInputrn); -}); - -Deno.test("bufioReadLineBadResource", async () => { - const file = await Deno.open("README.md"); - const bufReader = new BufReader(file); - file.close(); - await assertRejects(async () => { - await bufReader.readLine(); - }, Deno.errors.BadResource); -}); - -Deno.test("bufioReadLineBufferFullError", async () => { - const input = "@".repeat(5000) + "\n"; - const bufReader = new BufReader(new StringReader(input)); - const r = await bufReader.readLine(); - - assert(r !== null); - - const { line, more } = r; - assertEquals(more, true); - assertEquals(line, encoder.encode("@".repeat(4096))); -}); - -Deno.test("[io] readStringDelim basic", async () => { - const delim = "!#$%&()=~"; - const exp = [ - "", - "a", - "bc", - "def", - "", - "!", - "!#", - "!#$%&()=", - "#$%&()=~", - "", - "", - ]; - const str = exp.join(delim); - const arr: string[] = []; - for await (const v of readStringDelim(new StringReader(str), delim)) { - arr.push(v); - } - assertEquals(arr, exp); -}); - -Deno.test("[io] readStringDelim bigger delim than buf size", async () => { - // 0123456789... - const delim = Array.from({ length: 1025 }).map((_, i) => i % 10).join(""); - const exp = ["", "a", "bc", "def", "01", "012345678", "123456789", "", ""]; - const str = exp.join(delim); - const arr: string[] = []; - for await (const v of readStringDelim(new StringReader(str), delim)) { - arr.push(v); - } - assertEquals(arr, exp); -}); - -Deno.test("[io] readStringDelim delim=1213", async () => { - const delim = "1213"; - const exp = ["", "a", "bc", "def", "01", "012345678", "123456789", "", ""]; - const str = exp.join(delim); - const arr: string[] = []; - for await (const v of readStringDelim(new StringReader(str), "1213")) { - arr.push(v); - } - assertEquals(arr, exp); -}); - -Deno.test("bufioPeek", async function () { - const decoder = new TextDecoder(); - const p = new Uint8Array(10); - // string is 16 (minReadBufferSize) long. - const buf = new BufReader( - new StringReader("abcdefghijklmnop"), - MIN_READ_BUFFER_SIZE, - ); - - let actual = await buf.peek(1); - assert(actual !== null); - assertEquals(decoder.decode(actual), "a"); - - actual = await buf.peek(4); - assert(actual !== null); - assertEquals(decoder.decode(actual), "abcd"); - - try { - await buf.peek(32); - fail("peek() should throw"); - } catch (err) { - assert(err instanceof BufferFullError); - assert(err.partial instanceof Uint8Array); - assertEquals(decoder.decode(err.partial), "abcdefghijklmnop"); - } - - await buf.read(p.subarray(0, 3)); - assertEquals(decoder.decode(p.subarray(0, 3)), "abc"); - - actual = await buf.peek(1); - assert(actual !== null); - assertEquals(decoder.decode(actual), "d"); - - actual = await buf.peek(1); - assert(actual !== null); - assertEquals(decoder.decode(actual), "d"); - - actual = await buf.peek(1); - assert(actual !== null); - assertEquals(decoder.decode(actual), "d"); - - actual = await buf.peek(2); - assert(actual !== null); - assertEquals(decoder.decode(actual), "de"); - - const res = await buf.read(p.subarray(0, 3)); - assertEquals(decoder.decode(p.subarray(0, 3)), "def"); - assert(res !== null); - - actual = await buf.peek(4); - assert(actual !== null); - assertEquals(decoder.decode(actual), "ghij"); - - await buf.read(p); - assertEquals(decoder.decode(p), "ghijklmnop"); - - actual = await buf.peek(0); - assert(actual !== null); - assertEquals(decoder.decode(actual), ""); - - const r = await buf.peek(1); - assert(r === null); - /* TODO - Test for issue 3022, not exposing a reader's error on a successful Peek. - buf = NewReaderSize(dataAndEOFReader("abcd"), 32) - if s, err := buf.Peek(2); string(s) != "ab" || err != nil { - t.Errorf(`Peek(2) on "abcd", EOF = %q, %v; want "ab", nil`, string(s), err) - } - if s, err := buf.Peek(4); string(s) != "abcd" || err != nil { - t.Errorf( - `Peek(4) on "abcd", EOF = %q, %v; want "abcd", nil`, - string(s), - err - ) - } - if n, err := buf.Read(p[0:5]); string(p[0:n]) != "abcd" || err != nil { - t.Fatalf("Read after peek = %q, %v; want abcd, EOF", p[0:n], err) - } - if n, err := buf.Read(p[0:1]); string(p[0:n]) != "" || err != io.EOF { - t.Fatalf(`second Read after peek = %q, %v; want "", EOF`, p[0:n], err) - } - */ -}); - -Deno.test("bufioWriter", async function () { - const data = new Uint8Array(8192); - - for (let i = 0; i < data.byteLength; i++) { - data[i] = " ".charCodeAt(0) + (i % ("~".charCodeAt(0) - " ".charCodeAt(0))); - } - - const w = new Buffer(); - for (const nwrite of bufsizes) { - for (const bs of bufsizes) { - // Write nwrite bytes using buffer size bs. - // Check that the right amount makes it out - // and that the data is correct. - - w.reset(); - const buf = new BufWriter(w, bs); - - const context = `nwrite=${nwrite} bufsize=${bs}`; - const n = await buf.write(data.subarray(0, nwrite)); - assertEquals(n, nwrite, context); - - await buf.flush(); - - const written = w.bytes(); - assertEquals(written.byteLength, nwrite); - - for (let l = 0; l < written.byteLength; l++) { - assertEquals(written[l], data[l]); - } - } - } -}); - -Deno.test("bufioWriterSync", function () { - const data = new Uint8Array(8192); - - for (let i = 0; i < data.byteLength; i++) { - data[i] = " ".charCodeAt(0) + (i % ("~".charCodeAt(0) - " ".charCodeAt(0))); - } - - const w = new Buffer(); - for (const nwrite of bufsizes) { - for (const bs of bufsizes) { - // Write nwrite bytes using buffer size bs. - // Check that the right amount makes it out - // and that the data is correct. - - w.reset(); - const buf = new BufWriterSync(w, bs); - - const context = `nwrite=${nwrite} bufsize=${bs}`; - const n = buf.writeSync(data.subarray(0, nwrite)); - assertEquals(n, nwrite, context); - - buf.flush(); - - const written = w.bytes(); - assertEquals(written.byteLength, nwrite); - - for (let l = 0; l < written.byteLength; l++) { - assertEquals(written[l], data[l]); - } - } - } -}); - -Deno.test("bufReaderReadFull", async function () { - const enc = new TextEncoder(); - const dec = new TextDecoder(); - const text = "Hello World"; - const data = new Buffer(enc.encode(text)); - const bufr = new BufReader(data, 3); - { - const buf = new Uint8Array(6); - const r = await bufr.readFull(buf); - assert(r !== null); - assertEquals(r, buf); - assertEquals(dec.decode(buf), "Hello "); - } - { - const buf = new Uint8Array(6); - try { - await bufr.readFull(buf); - fail("readFull() should throw PartialReadError"); - } catch (err) { - assert(err instanceof PartialReadError); - assert(err.partial instanceof Uint8Array); - assertEquals(err.partial.length, 5); - assertEquals(dec.decode(buf.subarray(0, 5)), "World"); - } - } -}); - -Deno.test("readStringDelimAndLines", async function () { - const enc = new TextEncoder(); - const data = new Buffer( - enc.encode("Hello World\tHello World 2\tHello World 3"), - ); - const chunks_ = []; - - for await (const c of readStringDelim(data, "\t")) { - chunks_.push(c); - } - - assertEquals(chunks_.length, 3); - assertEquals(chunks_, ["Hello World", "Hello World 2", "Hello World 3"]); - - const linesData = new Buffer(enc.encode("0\n1\n2\n3\n4\n5\n6\n7\n8\n9")); - const linesDataWithTrailingNewLine = new Buffer(enc.encode("1\n2\n3\n")); - // consider data with windows newlines too - const linesDataWindows = new Buffer( - enc.encode("0\r\n1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n7\r\n8\r\n9"), - ); - const lines_ = []; - - for await (const l of readLines(linesData)) { - lines_.push(l); - } - - assertEquals(lines_.length, 10); - assertEquals(lines_, ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); - - lines_.length = 0; - for await (const l of readLines(linesDataWithTrailingNewLine)) { - lines_.push(l); - } - - assertEquals(lines_.length, 3); - assertEquals(lines_, ["1", "2", "3"]); // No empty line at the end - - // Now test for "windows" lines - lines_.length = 0; - for await (const l of readLines(linesDataWindows)) { - lines_.push(l); - } - assertEquals(lines_.length, 10); - assertEquals(lines_, ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); -}); - -Deno.test("readLinesWithEncodingISO-8859-15", async function () { - const lines_ = []; - const file_ = await Deno.open("./io/testdata/iso-8859-15.txt"); - - for await (const l of readLines(file_, { encoding: "iso-8859-15" })) { - lines_.push(l); - } - - file_.close(); - - assertEquals(lines_.length, 12); - assertEquals(lines_, [ - "\u0020!\"#$%&'()*+,-./", - "0123456789:;<=>?", - "@ABCDEFGHIJKLMNO", - "PQRSTUVWXYZ[\\]^_", - "`abcdefghijklmno", - "pqrstuvwxyz{|}~", - "\u00a0¡¢£€¥Š§š©ª«¬\u00ad®¯", - "°±²³Žµ¶·ž¹º»ŒœŸ¿", - "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏ", - "ÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞß", - "àáâãäåæçèéêëìíîï", - "ðñòóôõö÷øùúûüýþÿ", - ]); -}); - -/* TODO(kt3k): Enable this test -Deno.test( - "bufReaderShouldNotShareArrayBufferAcrossReads", - async function () { - const decoder = new TextDecoder(); - const data = "abcdefghijklmnopqrstuvwxyz"; - const bufSize = 25; - const b = new BufReader(new StringReader(data), bufSize); - - const r1 = (await b.readLine()) as ReadLineResult; - assert(r1 !== null); - assertEquals(decoder.decode(r1.line), "abcdefghijklmnopqrstuvwxy"); - - const r2 = (await b.readLine()) as ReadLineResult; - assert(r2 !== null); - assertEquals(decoder.decode(r2.line), "z"); - assert( - r1.line.buffer !== r2.line.buffer, - "array buffer should not be shared across reads", - ); - }, -); -*/ - -Deno.test({ - name: "Reset buffer after flush", - async fn() { - const stringWriter = new StringWriter(); - const bufWriter = new BufWriter(stringWriter); - const encoder = new TextEncoder(); - await bufWriter.write(encoder.encode("hello\nworld\nhow\nare\nyou?\n\n")); - await bufWriter.flush(); - await bufWriter.write(encoder.encode("foobar\n\n")); - await bufWriter.flush(); - const actual = stringWriter.toString(); - assertEquals(actual, "hello\nworld\nhow\nare\nyou?\n\nfoobar\n\n"); - }, -}); - -Deno.test({ - name: "Reset buffer after flush sync", - fn() { - const stringWriter = new StringWriter(); - const bufWriter = new BufWriterSync(stringWriter); - const encoder = new TextEncoder(); - bufWriter.writeSync(encoder.encode("hello\nworld\nhow\nare\nyou?\n\n")); - bufWriter.flush(); - bufWriter.writeSync(encoder.encode("foobar\n\n")); - bufWriter.flush(); - const actual = stringWriter.toString(); - assertEquals(actual, "hello\nworld\nhow\nare\nyou?\n\nfoobar\n\n"); - }, -}); - -Deno.test({ - name: "BufWriter.flush should write all bytes", - async fn() { - const bufSize = 16 * 1024; - const data = new Uint8Array(bufSize); - data.fill("a".charCodeAt(0)); - - const cache: Uint8Array[] = []; - const writer: Deno.Writer = { - write(p: Uint8Array): Promise { - cache.push(p.subarray(0, 1)); - - // Writer that only writes 1 byte at a time - return Promise.resolve(1); - }, - }; - - const bufWriter = new BufWriter(writer); - await bufWriter.write(data); - - await bufWriter.flush(); - const buf = new Uint8Array(cache.length); - for (let i = 0; i < cache.length; i++) buf.set(cache[i], i); - - assertEquals(data, buf); - }, -}); - -Deno.test({ - name: "BufWriterSync.flush should write all bytes", - fn() { - const bufSize = 16 * 1024; - const data = new Uint8Array(bufSize); - data.fill("a".charCodeAt(0)); - - const cache: Uint8Array[] = []; - const writer: Deno.WriterSync = { - writeSync(p: Uint8Array): number { - cache.push(p.subarray(0, 1)); - // Writer that only writes 1 byte at a time - return 1; - }, - }; - - const bufWriter = new BufWriterSync(writer); - bufWriter.writeSync(data); - - bufWriter.flush(); - const buf = new Uint8Array(cache.length); - for (let i = 0; i < cache.length; i++) buf.set(cache[i], i); - - assertEquals(data, buf); - }, -}); diff --git a/io/copy_n.ts b/io/copy_n.ts new file mode 100644 index 000000000000..3c317617a423 --- /dev/null +++ b/io/copy_n.ts @@ -0,0 +1,40 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { assert } from "../_util/asserts.ts"; +import type { Reader, Writer } from "./types.d.ts"; + +const DEFAULT_BUFFER_SIZE = 32 * 1024; + +/** + * Copy N size at the most. If read size is lesser than N, then returns nread + * @param r Reader + * @param dest Writer + * @param size Read size + */ +export async function copyN( + r: Reader, + dest: Writer, + size: number, +): Promise { + let bytesRead = 0; + let buf = new Uint8Array(DEFAULT_BUFFER_SIZE); + while (bytesRead < size) { + if (size - bytesRead < DEFAULT_BUFFER_SIZE) { + buf = new Uint8Array(size - bytesRead); + } + const result = await r.read(buf); + const nread = result ?? 0; + bytesRead += nread; + if (nread > 0) { + let n = 0; + while (n < nread) { + n += await dest.write(buf.slice(n, nread)); + } + assert(n === nread, "could not write"); + } + if (result === null) { + break; + } + } + return bytesRead; +} diff --git a/io/copy_n_test.ts b/io/copy_n_test.ts new file mode 100644 index 000000000000..b2a76110f8c8 --- /dev/null +++ b/io/copy_n_test.ts @@ -0,0 +1,37 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "../testing/asserts.ts"; +import { copyN } from "./copy_n.ts"; +import { Buffer } from "./buffer.ts"; +import { StringReader } from "./string_reader.ts"; + +Deno.test("testCopyN1", async function () { + const w = new Buffer(); + const r = new StringReader("abcdefghij"); + const n = await copyN(r, w, 3); + assertEquals(n, 3); + assertEquals(new TextDecoder().decode(w.bytes()), "abc"); +}); + +Deno.test("testCopyN2", async function () { + const w = new Buffer(); + const r = new StringReader("abcdefghij"); + const n = await copyN(r, w, 11); + assertEquals(n, 10); + assertEquals(new TextDecoder().decode(w.bytes()), "abcdefghij"); +}); + +Deno.test("copyNWriteAllData", async function () { + const tmpDir = await Deno.makeTempDir(); + const filepath = `${tmpDir}/data`; + const file = await Deno.open(filepath, { create: true, write: true }); + + const size = 16 * 1024 + 1; + const data = "a".repeat(32 * 1024); + const r = new StringReader(data); + const n = await copyN(r, file, size); // Over max file possible buffer + file.close(); + await Deno.remove(filepath); + + assertEquals(n, size); +}); diff --git a/io/files.ts b/io/files.ts index e6f98525d59b..35e15cc2aefd 100644 --- a/io/files.ts +++ b/io/files.ts @@ -1,10 +1,11 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -import { copy as copyBytes } from "../bytes/copy.ts"; -import { assert } from "../_util/asserts.ts"; - -const DEFAULT_BUFFER_SIZE = 32 * 1024; +import { + readRange as _readRange, + readRangeSync as _readRangeSync, +} from "./read_range.ts"; +/** @deprecated (will be removed after 0.170.0) Import from `std/io/read_range.ts` instead */ export interface ByteRange { /** The 0 based index of the start byte for a range. */ start: number; @@ -14,6 +15,8 @@ export interface ByteRange { } /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/read_range.ts` instead + * * Read a range of bytes from a file or other resource that is readable and * seekable. The range start and end are inclusive of the bytes within that * range. @@ -28,30 +31,11 @@ export interface ByteRange { * assertEquals(bytes.length, 10); * ``` */ -export async function readRange( - r: Deno.Reader & Deno.Seeker, - range: ByteRange, -): Promise { - // byte ranges are inclusive, so we have to add one to the end - let length = range.end - range.start + 1; - assert(length > 0, "Invalid byte range was passed."); - await r.seek(range.start, Deno.SeekMode.Start); - const result = new Uint8Array(length); - let off = 0; - while (length) { - const p = new Uint8Array(Math.min(length, DEFAULT_BUFFER_SIZE)); - const nread = await r.read(p); - assert(nread !== null, "Unexpected EOF reach while reading a range."); - assert(nread > 0, "Unexpected read of 0 bytes while reading a range."); - copyBytes(p, result, off); - off += nread; - length -= nread; - assert(length >= 0, "Unexpected length remaining after reading range."); - } - return result; -} +export const readRange = _readRange; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/read_range.ts` instead + * * Read a range of bytes synchronously from a file or other resource that is * readable and seekable. The range start and end are inclusive of the bytes * within that range. @@ -66,25 +50,4 @@ export async function readRange( * assertEquals(bytes.length, 10); * ``` */ -export function readRangeSync( - r: Deno.ReaderSync & Deno.SeekerSync, - range: ByteRange, -): Uint8Array { - // byte ranges are inclusive, so we have to add one to the end - let length = range.end - range.start + 1; - assert(length > 0, "Invalid byte range was passed."); - r.seekSync(range.start, Deno.SeekMode.Start); - const result = new Uint8Array(length); - let off = 0; - while (length) { - const p = new Uint8Array(Math.min(length, DEFAULT_BUFFER_SIZE)); - const nread = r.readSync(p); - assert(nread !== null, "Unexpected EOF reach while reading a range."); - assert(nread > 0, "Unexpected read of 0 bytes while reading a range."); - copyBytes(p, result, off); - off += nread; - length -= nread; - assert(length >= 0, "Unexpected length remaining after reading range."); - } - return result; -} +export const readRangeSync = _readRangeSync; diff --git a/io/limited_reader.ts b/io/limited_reader.ts new file mode 100644 index 000000000000..9832a562e2a1 --- /dev/null +++ b/io/limited_reader.ts @@ -0,0 +1,27 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +/** + * A `LimitedReader` reads from `reader` but limits the amount of data returned to just `limit` bytes. + * Each call to `read` updates `limit` to reflect the new amount remaining. + * `read` returns `null` when `limit` <= `0` or + * when the underlying `reader` returns `null`. + */ +export class LimitedReader implements Deno.Reader { + constructor(public reader: Deno.Reader, public limit: number) {} + + async read(p: Uint8Array): Promise { + if (this.limit <= 0) { + return null; + } + + if (p.length > this.limit) { + p = p.subarray(0, this.limit); + } + const n = await this.reader.read(p); + if (n == null) { + return null; + } + + this.limit -= n; + return n; + } +} diff --git a/io/limited_reader_test.ts b/io/limited_reader_test.ts new file mode 100644 index 000000000000..beb515292941 --- /dev/null +++ b/io/limited_reader_test.ts @@ -0,0 +1,33 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +import { assertEquals } from "../testing/asserts.ts"; +import { LimitedReader } from "./limited_reader.ts"; +import { StringWriter } from "./string_writer.ts"; +import { copy } from "../streams/copy.ts"; +import { readAll } from "../streams/read_all.ts"; +import { StringReader } from "./string_reader.ts"; + +Deno.test("ioLimitedReader", async function () { + const decoder = new TextDecoder(); + let sr = new StringReader("abc"); + let r = new LimitedReader(sr, 2); + let buffer = await readAll(r); + assertEquals(decoder.decode(buffer), "ab"); + assertEquals(decoder.decode(await readAll(sr)), "c"); + sr = new StringReader("abc"); + r = new LimitedReader(sr, 3); + buffer = await readAll(r); + assertEquals(decoder.decode(buffer), "abc"); + assertEquals((await readAll(r)).length, 0); + sr = new StringReader("abc"); + r = new LimitedReader(sr, 4); + buffer = await readAll(r); + assertEquals(decoder.decode(buffer), "abc"); + assertEquals((await readAll(r)).length, 0); +}); + +Deno.test("ioLimitedReader", async function () { + const rb = new StringReader("abc"); + const wb = new StringWriter(); + await copy(new LimitedReader(rb, -1), wb); + assertEquals(wb.toString(), ""); +}); diff --git a/io/mod.ts b/io/mod.ts index 6c6698477fdd..1a2086737691 100644 --- a/io/mod.ts +++ b/io/mod.ts @@ -9,13 +9,20 @@ * @module */ -export * from "./buffer.ts"; -export * from "./readers.ts"; -export { - copyN, - readInt, - readLong, - readShort, - sliceLongToBytes, -} from "./util.ts"; -export * from "./writers.ts"; +export * from "./buf_reader.ts"; +export * from "./buf_writer.ts"; +/** @deprecated (will be removed after 0.170.0) `export * from "./buffer.ts";` upon removal */ +export { Buffer } from "./buffer.ts"; +export * from "./copy_n.ts"; +export * from "./limited_reader.ts"; +export * from "./multi_reader.ts"; +export * from "./read_delim.ts"; +export * from "./read_int.ts"; +export * from "./read_lines.ts"; +export * from "./read_long.ts"; +export * from "./read_range.ts"; +export * from "./read_short.ts"; +export * from "./read_string_delim.ts"; +export * from "./slice_long_to_bytes.ts"; +export * from "./string_reader.ts"; +export * from "./string_writer.ts"; diff --git a/io/multi_reader.ts b/io/multi_reader.ts new file mode 100644 index 000000000000..9f3cbd311ba0 --- /dev/null +++ b/io/multi_reader.ts @@ -0,0 +1,22 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +/** Reader utility for combining multiple readers */ +export class MultiReader implements Deno.Reader { + readonly #readers: Deno.Reader[]; + #currentIndex = 0; + + constructor(readers: Deno.Reader[]) { + this.#readers = [...readers]; + } + + async read(p: Uint8Array): Promise { + const r = this.#readers[this.#currentIndex]; + if (!r) return null; + const result = await r.read(p); + if (result === null) { + this.#currentIndex++; + return 0; + } + return result; + } +} diff --git a/io/multi_reader_test.ts b/io/multi_reader_test.ts new file mode 100644 index 000000000000..07a7a8249b0b --- /dev/null +++ b/io/multi_reader_test.ts @@ -0,0 +1,18 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "../testing/asserts.ts"; +import { MultiReader } from "./multi_reader.ts"; +import { StringWriter } from "./string_writer.ts"; +import { copyN } from "./copy_n.ts"; +import { copy } from "../streams/copy.ts"; +import { StringReader } from "./string_reader.ts"; + +Deno.test("ioMultiReader", async function () { + const r = new MultiReader([new StringReader("abc"), new StringReader("def")]); + const w = new StringWriter(); + const n = await copyN(r, w, 4); + assertEquals(n, 4); + assertEquals(w.toString(), "abcd"); + await copy(r, w); + assertEquals(w.toString(), "abcdef"); +}); diff --git a/io/read_delim.ts b/io/read_delim.ts new file mode 100644 index 000000000000..f1e32cfc3701 --- /dev/null +++ b/io/read_delim.ts @@ -0,0 +1,79 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { BytesList } from "../bytes/bytes_list.ts"; +import type { Reader } from "./types.d.ts"; + +/** Generate longest proper prefix which is also suffix array. */ +function createLPS(pat: Uint8Array): Uint8Array { + const lps = new Uint8Array(pat.length); + lps[0] = 0; + let prefixEnd = 0; + let i = 1; + while (i < lps.length) { + if (pat[i] == pat[prefixEnd]) { + prefixEnd++; + lps[i] = prefixEnd; + i++; + } else if (prefixEnd === 0) { + lps[i] = 0; + i++; + } else { + prefixEnd = lps[prefixEnd - 1]; + } + } + return lps; +} + +/** Read delimited bytes from a Reader. */ +export async function* readDelim( + reader: Reader, + delim: Uint8Array, +): AsyncIterableIterator { + // Avoid unicode problems + const delimLen = delim.length; + const delimLPS = createLPS(delim); + const chunks = new BytesList(); + const bufSize = Math.max(1024, delimLen + 1); + + // Modified KMP + let inspectIndex = 0; + let matchIndex = 0; + while (true) { + const inspectArr = new Uint8Array(bufSize); + const result = await reader.read(inspectArr); + if (result === null) { + // Yield last chunk. + yield chunks.concat(); + return; + } else if (result < 0) { + // Discard all remaining and silently fail. + return; + } + chunks.add(inspectArr, 0, result); + let localIndex = 0; + while (inspectIndex < chunks.size()) { + if (inspectArr[localIndex] === delim[matchIndex]) { + inspectIndex++; + localIndex++; + matchIndex++; + if (matchIndex === delimLen) { + // Full match + const matchEnd = inspectIndex - delimLen; + const readyBytes = chunks.slice(0, matchEnd); + yield readyBytes; + // Reset match, different from KMP. + chunks.shift(inspectIndex); + inspectIndex = 0; + matchIndex = 0; + } + } else { + if (matchIndex === 0) { + inspectIndex++; + localIndex++; + } else { + matchIndex = delimLPS[matchIndex - 1]; + } + } + } + } +} diff --git a/io/read_int.ts b/io/read_int.ts new file mode 100644 index 000000000000..a954ecc9674b --- /dev/null +++ b/io/read_int.ts @@ -0,0 +1,16 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { type BufReader } from "./buf_reader.ts"; +import { readShort } from "./read_short.ts"; + +/** + * Read big endian 32bit integer from BufReader + * @param buf + */ +export async function readInt(buf: BufReader): Promise { + const high = await readShort(buf); + if (high === null) return null; + const low = await readShort(buf); + if (low === null) throw new Deno.errors.UnexpectedEof(); + return (high << 16) | low; +} diff --git a/io/read_int_test.ts b/io/read_int_test.ts new file mode 100644 index 000000000000..a5f9f3ea8747 --- /dev/null +++ b/io/read_int_test.ts @@ -0,0 +1,12 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "../testing/asserts.ts"; +import { readInt } from "./read_int.ts"; +import { BufReader } from "./buf_reader.ts"; +import { BinaryReader } from "./_test_common.ts"; + +Deno.test("testReadInt", async function () { + const r = new BinaryReader(new Uint8Array([0x12, 0x34, 0x56, 0x78])); + const int = await readInt(new BufReader(r)); + assertEquals(int, 0x12345678); +}); diff --git a/io/read_lines.ts b/io/read_lines.ts new file mode 100644 index 000000000000..7053a253269b --- /dev/null +++ b/io/read_lines.ts @@ -0,0 +1,48 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { type Reader } from "./types.d.ts"; +import { BufReader } from "./buf_reader.ts"; +import { concat } from "../bytes/concat.ts"; + +/** + * Read strings line-by-line from a Reader. + * + * @example + * ```ts + * import { readLines } from "https://deno.land/std@$STD_VERSION/io/read_lines.ts"; + * import * as path from "https://deno.land/std@$STD_VERSION/path/mod.ts"; + * + * const filename = path.join(Deno.cwd(), "std/io/README.md"); + * let fileReader = await Deno.open(filename); + * + * for await (let line of readLines(fileReader)) { + * console.log(line); + * } + * ``` + */ +export async function* readLines( + reader: Reader, + decoderOpts?: { + encoding?: string; + fatal?: boolean; + ignoreBOM?: boolean; + }, +): AsyncIterableIterator { + const bufReader = new BufReader(reader); + let chunks: Uint8Array[] = []; + const decoder = new TextDecoder(decoderOpts?.encoding, decoderOpts); + while (true) { + const res = await bufReader.readLine(); + if (!res) { + if (chunks.length > 0) { + yield decoder.decode(concat(...chunks)); + } + break; + } + chunks.push(res.line); + if (!res.more) { + yield decoder.decode(concat(...chunks)); + chunks = []; + } + } +} diff --git a/io/read_lines_test.ts b/io/read_lines_test.ts new file mode 100644 index 000000000000..961e62e94360 --- /dev/null +++ b/io/read_lines_test.ts @@ -0,0 +1,82 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +// This code has been ported almost directly from Go's src/bytes/buffer_test.go +// Copyright 2009 The Go Authors. All rights reserved. BSD license. +// https://github.com/golang/go/blob/master/LICENSE +import { assertEquals } from "../testing/asserts.ts"; +import { Buffer } from "./buffer.ts"; +import { readLines } from "./read_lines.ts"; +import { readStringDelim } from "./read_string_delim.ts"; + +/** @todo (iuioiua) Can these tests be separated? */ +Deno.test("readStringDelimAndLines", async function () { + const enc = new TextEncoder(); + const data = new Buffer( + enc.encode("Hello World\tHello World 2\tHello World 3"), + ); + const chunks_ = []; + + for await (const c of readStringDelim(data, "\t")) { + chunks_.push(c); + } + + assertEquals(chunks_.length, 3); + assertEquals(chunks_, ["Hello World", "Hello World 2", "Hello World 3"]); + + const linesData = new Buffer(enc.encode("0\n1\n2\n3\n4\n5\n6\n7\n8\n9")); + const linesDataWithTrailingNewLine = new Buffer(enc.encode("1\n2\n3\n")); + // consider data with windows newlines too + const linesDataWindows = new Buffer( + enc.encode("0\r\n1\r\n2\r\n3\r\n4\r\n5\r\n6\r\n7\r\n8\r\n9"), + ); + const lines_ = []; + + for await (const l of readLines(linesData)) { + lines_.push(l); + } + + assertEquals(lines_.length, 10); + assertEquals(lines_, ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); + + lines_.length = 0; + for await (const l of readLines(linesDataWithTrailingNewLine)) { + lines_.push(l); + } + + assertEquals(lines_.length, 3); + assertEquals(lines_, ["1", "2", "3"]); // No empty line at the end + + // Now test for "windows" lines + lines_.length = 0; + for await (const l of readLines(linesDataWindows)) { + lines_.push(l); + } + assertEquals(lines_.length, 10); + assertEquals(lines_, ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); +}); + +Deno.test("readLinesWithEncodingISO-8859-15", async function () { + const lines_ = []; + const file_ = await Deno.open("./io/testdata/iso-8859-15.txt"); + + for await (const l of readLines(file_, { encoding: "iso-8859-15" })) { + lines_.push(l); + } + + file_.close(); + + assertEquals(lines_.length, 12); + assertEquals(lines_, [ + "\u0020!\"#$%&'()*+,-./", + "0123456789:;<=>?", + "@ABCDEFGHIJKLMNO", + "PQRSTUVWXYZ[\\]^_", + "`abcdefghijklmno", + "pqrstuvwxyz{|}~", + "\u00a0¡¢£€¥Š§š©ª«¬\u00ad®¯", + "°±²³Žµ¶·ž¹º»ŒœŸ¿", + "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏ", + "ÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞß", + "àáâãäåæçèéêëìíîï", + "ðñòóôõö÷øùúûüýþÿ", + ]); +}); diff --git a/io/read_long.ts b/io/read_long.ts new file mode 100644 index 000000000000..88a672ca69a0 --- /dev/null +++ b/io/read_long.ts @@ -0,0 +1,25 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { type BufReader } from "./buf_reader.ts"; +import { readInt } from "./read_int.ts"; + +const MAX_SAFE_INTEGER = BigInt(Number.MAX_SAFE_INTEGER); + +/** + * Read big endian 64bit long from BufReader + * @param buf + */ +export async function readLong(buf: BufReader): Promise { + const high = await readInt(buf); + if (high === null) return null; + const low = await readInt(buf); + if (low === null) throw new Deno.errors.UnexpectedEof(); + const big = (BigInt(high) << 32n) | BigInt(low); + // We probably should provide a similar API that returns BigInt values. + if (big > MAX_SAFE_INTEGER) { + throw new RangeError( + "Long value too big to be represented as a JavaScript number.", + ); + } + return Number(big); +} diff --git a/io/read_long_test.ts b/io/read_long_test.ts new file mode 100644 index 000000000000..61ddf011f276 --- /dev/null +++ b/io/read_long_test.ts @@ -0,0 +1,22 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "../testing/asserts.ts"; +import { readLong } from "./read_long.ts"; +import { BufReader } from "./buf_reader.ts"; +import { BinaryReader } from "./_test_common.ts"; + +Deno.test("testReadLong", async function () { + const r = new BinaryReader( + new Uint8Array([0x00, 0x00, 0x00, 0x78, 0x12, 0x34, 0x56, 0x78]), + ); + const long = await readLong(new BufReader(r)); + assertEquals(long, 0x7812345678); +}); + +Deno.test("testReadLong2", async function () { + const r = new BinaryReader( + new Uint8Array([0, 0, 0, 0, 0x12, 0x34, 0x56, 0x78]), + ); + const long = await readLong(new BufReader(r)); + assertEquals(long, 0x12345678); +}); diff --git a/io/read_range.ts b/io/read_range.ts new file mode 100644 index 000000000000..72c3141c460d --- /dev/null +++ b/io/read_range.ts @@ -0,0 +1,90 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { copy as copyBytes } from "../bytes/copy.ts"; +import { assert } from "../_util/asserts.ts"; + +const DEFAULT_BUFFER_SIZE = 32 * 1024; + +export interface ByteRange { + /** The 0 based index of the start byte for a range. */ + start: number; + + /** The 0 based index of the end byte for a range, which is inclusive. */ + end: number; +} + +/** + * Read a range of bytes from a file or other resource that is readable and + * seekable. The range start and end are inclusive of the bytes within that + * range. + * + * ```ts + * import { assertEquals } from "https://deno.land/std@$STD_VERSION/testing/asserts.ts"; + * import { readRange } from "https://deno.land/std@$STD_VERSION/io/read_range.ts"; + * + * // Read the first 10 bytes of a file + * const file = await Deno.open("example.txt", { read: true }); + * const bytes = await readRange(file, { start: 0, end: 9 }); + * assertEquals(bytes.length, 10); + * ``` + */ +export async function readRange( + r: Deno.Reader & Deno.Seeker, + range: ByteRange, +): Promise { + // byte ranges are inclusive, so we have to add one to the end + let length = range.end - range.start + 1; + assert(length > 0, "Invalid byte range was passed."); + await r.seek(range.start, Deno.SeekMode.Start); + const result = new Uint8Array(length); + let off = 0; + while (length) { + const p = new Uint8Array(Math.min(length, DEFAULT_BUFFER_SIZE)); + const nread = await r.read(p); + assert(nread !== null, "Unexpected EOF reach while reading a range."); + assert(nread > 0, "Unexpected read of 0 bytes while reading a range."); + copyBytes(p, result, off); + off += nread; + length -= nread; + assert(length >= 0, "Unexpected length remaining after reading range."); + } + return result; +} + +/** + * Read a range of bytes synchronously from a file or other resource that is + * readable and seekable. The range start and end are inclusive of the bytes + * within that range. + * + * ```ts + * import { assertEquals } from "https://deno.land/std@$STD_VERSION/testing/asserts.ts"; + * import { readRangeSync } from "https://deno.land/std@$STD_VERSION/io/read_range.ts"; + * + * // Read the first 10 bytes of a file + * const file = Deno.openSync("example.txt", { read: true }); + * const bytes = readRangeSync(file, { start: 0, end: 9 }); + * assertEquals(bytes.length, 10); + * ``` + */ +export function readRangeSync( + r: Deno.ReaderSync & Deno.SeekerSync, + range: ByteRange, +): Uint8Array { + // byte ranges are inclusive, so we have to add one to the end + let length = range.end - range.start + 1; + assert(length > 0, "Invalid byte range was passed."); + r.seekSync(range.start, Deno.SeekMode.Start); + const result = new Uint8Array(length); + let off = 0; + while (length) { + const p = new Uint8Array(Math.min(length, DEFAULT_BUFFER_SIZE)); + const nread = r.readSync(p); + assert(nread !== null, "Unexpected EOF reach while reading a range."); + assert(nread > 0, "Unexpected read of 0 bytes while reading a range."); + copyBytes(p, result, off); + off += nread; + length -= nread; + assert(length >= 0, "Unexpected length remaining after reading range."); + } + return result; +} diff --git a/io/files_test.ts b/io/read_range_test.ts similarity index 98% rename from io/files_test.ts rename to io/read_range_test.ts index 1c4e064059cb..62c4a1e00aaa 100644 --- a/io/files_test.ts +++ b/io/read_range_test.ts @@ -7,7 +7,7 @@ import { assertRejects, assertThrows, } from "../testing/asserts.ts"; -import { readRange, readRangeSync } from "./files.ts"; +import { readRange, readRangeSync } from "./read_range.ts"; // N controls how many iterations of certain checks are performed. const N = 100; diff --git a/io/read_short.ts b/io/read_short.ts new file mode 100644 index 000000000000..001b5c6abf72 --- /dev/null +++ b/io/read_short.ts @@ -0,0 +1,15 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { type BufReader } from "./buf_reader.ts"; + +/** + * Read big endian 16bit short from BufReader + * @param buf + */ +export async function readShort(buf: BufReader): Promise { + const high = await buf.readByte(); + if (high === null) return null; + const low = await buf.readByte(); + if (low === null) throw new Deno.errors.UnexpectedEof(); + return (high << 8) | low; +} diff --git a/io/read_short_test.ts b/io/read_short_test.ts new file mode 100644 index 000000000000..68d481eb8f3d --- /dev/null +++ b/io/read_short_test.ts @@ -0,0 +1,12 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "../testing/asserts.ts"; +import { readShort } from "./read_short.ts"; +import { BufReader } from "./buf_reader.ts"; +import { BinaryReader } from "./_test_common.ts"; + +Deno.test("testReadShort", async function () { + const r = new BinaryReader(new Uint8Array([0x12, 0x34])); + const short = await readShort(new BufReader(r)); + assertEquals(short, 0x1234); +}); diff --git a/io/read_string_delim.ts b/io/read_string_delim.ts new file mode 100644 index 000000000000..1bd2dda88b1a --- /dev/null +++ b/io/read_string_delim.ts @@ -0,0 +1,36 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { type Reader } from "./types.d.ts"; +import { readDelim } from "./read_delim.ts"; + +/** + * Read Reader chunk by chunk, splitting based on delimiter. + * + * @example + * ```ts + * import { readStringDelim } from "https://deno.land/std@$STD_VERSION/io/read_string_delim.ts"; + * import * as path from "https://deno.land/std@$STD_VERSION/path/mod.ts"; + * + * const filename = path.join(Deno.cwd(), "std/io/README.md"); + * let fileReader = await Deno.open(filename); + * + * for await (let line of readStringDelim(fileReader, "\n")) { + * console.log(line); + * } + * ``` + */ +export async function* readStringDelim( + reader: Reader, + delim: string, + decoderOpts?: { + encoding?: string; + fatal?: boolean; + ignoreBOM?: boolean; + }, +): AsyncIterableIterator { + const encoder = new TextEncoder(); + const decoder = new TextDecoder(decoderOpts?.encoding, decoderOpts); + for await (const chunk of readDelim(reader, encoder.encode(delim))) { + yield decoder.decode(chunk); + } +} diff --git a/io/read_string_delim_test.ts b/io/read_string_delim_test.ts new file mode 100644 index 000000000000..2d7a9fe6400c --- /dev/null +++ b/io/read_string_delim_test.ts @@ -0,0 +1,53 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +// This code has been ported almost directly from Go's src/bytes/buffer_test.go +// Copyright 2009 The Go Authors. All rights reserved. BSD license. +// https://github.com/golang/go/blob/master/LICENSE +import { assertEquals } from "../testing/asserts.ts"; +import { readStringDelim } from "./read_string_delim.ts"; +import { StringReader } from "./string_reader.ts"; + +Deno.test("[io] readStringDelim basic", async () => { + const delim = "!#$%&()=~"; + const exp = [ + "", + "a", + "bc", + "def", + "", + "!", + "!#", + "!#$%&()=", + "#$%&()=~", + "", + "", + ]; + const str = exp.join(delim); + const arr: string[] = []; + for await (const v of readStringDelim(new StringReader(str), delim)) { + arr.push(v); + } + assertEquals(arr, exp); +}); + +Deno.test("[io] readStringDelim bigger delim than buf size", async () => { + // 0123456789... + const delim = Array.from({ length: 1025 }).map((_, i) => i % 10).join(""); + const exp = ["", "a", "bc", "def", "01", "012345678", "123456789", "", ""]; + const str = exp.join(delim); + const arr: string[] = []; + for await (const v of readStringDelim(new StringReader(str), delim)) { + arr.push(v); + } + assertEquals(arr, exp); +}); + +Deno.test("[io] readStringDelim delim=1213", async () => { + const delim = "1213"; + const exp = ["", "a", "bc", "def", "01", "012345678", "123456789", "", ""]; + const str = exp.join(delim); + const arr: string[] = []; + for await (const v of readStringDelim(new StringReader(str), "1213")) { + arr.push(v); + } + assertEquals(arr, exp); +}); diff --git a/io/readers.ts b/io/readers.ts index acbadf707e1c..a279914e00dc 100644 --- a/io/readers.ts +++ b/io/readers.ts @@ -4,9 +4,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -import { Buffer } from "./buffer.ts"; +import { StringReader as _StringReader } from "./string_reader.ts"; +import { MultiReader as _MultiReader } from "./multi_reader.ts"; +import { LimitedReader as _LimitedReader } from "./limited_reader.ts"; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/string_reader.ts` instead + * * Reader utility for strings. * * @example @@ -35,56 +39,21 @@ import { Buffer } from "./buffer.ts"; * abcdef * ``` */ -export class StringReader extends Buffer { - constructor(s: string) { - super(new TextEncoder().encode(s).buffer); - } -} - -/** Reader utility for combining multiple readers */ -export class MultiReader implements Deno.Reader { - readonly #readers: Deno.Reader[]; - #currentIndex = 0; - - constructor(readers: Deno.Reader[]) { - this.#readers = [...readers]; - } +export const StringReader = _StringReader; - async read(p: Uint8Array): Promise { - const r = this.#readers[this.#currentIndex]; - if (!r) return null; - const result = await r.read(p); - if (result === null) { - this.#currentIndex++; - return 0; - } - return result; - } -} +/** + * @deprecated (will be removed after 0.170.0) Import from `std/io/multi_reader.ts` instead + * + * Reader utility for combining multiple readers + */ +export const MultiReader = _MultiReader; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/limited_reader.ts` instead + * * A `LimitedReader` reads from `reader` but limits the amount of data returned to just `limit` bytes. * Each call to `read` updates `limit` to reflect the new amount remaining. * `read` returns `null` when `limit` <= `0` or * when the underlying `reader` returns `null`. */ -export class LimitedReader implements Deno.Reader { - constructor(public reader: Deno.Reader, public limit: number) {} - - async read(p: Uint8Array): Promise { - if (this.limit <= 0) { - return null; - } - - if (p.length > this.limit) { - p = p.subarray(0, this.limit); - } - const n = await this.reader.read(p); - if (n == null) { - return null; - } - - this.limit -= n; - return n; - } -} +export const LimitedReader = _LimitedReader; diff --git a/io/readers_test.ts b/io/readers_test.ts deleted file mode 100644 index 2f935928963b..000000000000 --- a/io/readers_test.ts +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -import { assertEquals } from "../testing/asserts.ts"; -import { LimitedReader, MultiReader, StringReader } from "./readers.ts"; -import { StringWriter } from "./writers.ts"; -import { copyN } from "./util.ts"; -import { copy } from "../streams/copy.ts"; -import { readAll } from "../streams/read_all.ts"; - -Deno.test("ioStringReader", async function () { - const r = new StringReader("abcdef"); - const res0 = await r.read(new Uint8Array(6)); - assertEquals(res0, 6); - const res1 = await r.read(new Uint8Array(6)); - assertEquals(res1, null); -}); - -Deno.test("ioStringReader", async function () { - const decoder = new TextDecoder(); - const r = new StringReader("abcdef"); - const buf = new Uint8Array(3); - const res1 = await r.read(buf); - assertEquals(res1, 3); - assertEquals(decoder.decode(buf), "abc"); - const res2 = await r.read(buf); - assertEquals(res2, 3); - assertEquals(decoder.decode(buf), "def"); - const res3 = await r.read(buf); - assertEquals(res3, null); - assertEquals(decoder.decode(buf), "def"); -}); - -Deno.test("ioMultiReader", async function () { - const r = new MultiReader([new StringReader("abc"), new StringReader("def")]); - const w = new StringWriter(); - const n = await copyN(r, w, 4); - assertEquals(n, 4); - assertEquals(w.toString(), "abcd"); - await copy(r, w); - assertEquals(w.toString(), "abcdef"); -}); - -Deno.test("ioLimitedReader", async function () { - const decoder = new TextDecoder(); - let sr = new StringReader("abc"); - let r = new LimitedReader(sr, 2); - let buffer = await readAll(r); - assertEquals(decoder.decode(buffer), "ab"); - assertEquals(decoder.decode(await readAll(sr)), "c"); - sr = new StringReader("abc"); - r = new LimitedReader(sr, 3); - buffer = await readAll(r); - assertEquals(decoder.decode(buffer), "abc"); - assertEquals((await readAll(r)).length, 0); - sr = new StringReader("abc"); - r = new LimitedReader(sr, 4); - buffer = await readAll(r); - assertEquals(decoder.decode(buffer), "abc"); - assertEquals((await readAll(r)).length, 0); -}); - -Deno.test("ioLimitedReader", async function () { - const rb = new StringReader("abc"); - const wb = new StringWriter(); - await copy(new LimitedReader(rb, -1), wb); - assertEquals(wb.toString(), ""); -}); diff --git a/io/slice_long_to_bytes.ts b/io/slice_long_to_bytes.ts new file mode 100644 index 000000000000..78777745905e --- /dev/null +++ b/io/slice_long_to_bytes.ts @@ -0,0 +1,18 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +/** + * Slice number into 64bit big endian byte array + * @param d The number to be sliced + * @param dest The sliced array + */ +export function sliceLongToBytes( + d: number, + dest = Array.from({ length: 8 }), +): number[] { + let big = BigInt(d); + for (let i = 0; i < 8; i++) { + dest[7 - i] = Number(big & 0xffn); + big >>= 8n; + } + return dest; +} diff --git a/io/slice_long_to_bytes_test.ts b/io/slice_long_to_bytes_test.ts new file mode 100644 index 000000000000..e055305fd3dc --- /dev/null +++ b/io/slice_long_to_bytes_test.ts @@ -0,0 +1,24 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +import { assertEquals } from "../testing/asserts.ts"; +import { readLong } from "./read_long.ts"; +import { sliceLongToBytes } from "./slice_long_to_bytes.ts"; +import { BufReader } from "./buf_reader.ts"; +import { BinaryReader } from "./_test_common.ts"; + +Deno.test("testSliceLongToBytes", function () { + const arr = sliceLongToBytes(0x1234567890abcdef); + const actual = readLong(new BufReader(new BinaryReader(new Uint8Array(arr)))); + const expected = readLong( + new BufReader( + new BinaryReader( + new Uint8Array([0x12, 0x34, 0x56, 0x78, 0x90, 0xab, 0xcd, 0xef]), + ), + ), + ); + assertEquals(actual, expected); +}); + +Deno.test("testSliceLongToBytes2", function () { + const arr = sliceLongToBytes(0x12345678); + assertEquals(arr, [0, 0, 0, 0, 0x12, 0x34, 0x56, 0x78]); +}); diff --git a/io/string_reader.ts b/io/string_reader.ts new file mode 100644 index 000000000000..5859840636a5 --- /dev/null +++ b/io/string_reader.ts @@ -0,0 +1,38 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { Buffer } from "./buffer.ts"; + +/** + * Reader utility for strings. + * + * @example + * ```ts + * import { StringReader } from "https://deno.land/std@$STD_VERSION/io/string_reader.ts"; + * + * const data = new Uint8Array(6); + * const r = new StringReader("abcdef"); + * const res0 = await r.read(data); + * const res1 = await r.read(new Uint8Array(6)); + * + * // Number of bytes read + * console.log(res0); // 6 + * console.log(res1); // null, no byte left to read. EOL + * + * // text + * + * console.log(new TextDecoder().decode(data)); // abcdef + * ``` + * + * **Output:** + * + * ```text + * 6 + * null + * abcdef + * ``` + */ +export class StringReader extends Buffer { + constructor(s: string) { + super(new TextEncoder().encode(s).buffer); + } +} diff --git a/io/string_reader_test.ts b/io/string_reader_test.ts new file mode 100644 index 000000000000..6881059f1f4d --- /dev/null +++ b/io/string_reader_test.ts @@ -0,0 +1,34 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "../testing/asserts.ts"; +import { StringReader } from "./string_reader.ts"; + +Deno.test("ioStringReader", async function () { + const r = new StringReader("abcdef"); + const res0 = await r.read(new Uint8Array(6)); + assertEquals(res0, 6); + const res1 = await r.read(new Uint8Array(6)); + assertEquals(res1, null); +}); + +Deno.test("ioStringReader", async function () { + const decoder = new TextDecoder(); + const r = new StringReader("abcdef"); + const buf = new Uint8Array(3); + const res1 = await r.read(buf); + assertEquals(res1, 3); + assertEquals(decoder.decode(buf), "abc"); + const res2 = await r.read(buf); + assertEquals(res2, 3); + assertEquals(decoder.decode(buf), "def"); + const res3 = await r.read(buf); + assertEquals(res3, null); + assertEquals(decoder.decode(buf), "def"); +}); + +Deno.test("testStringReaderEof", async function () { + const r = new StringReader("abc"); + assertEquals(await r.read(new Uint8Array()), 0); + assertEquals(await r.read(new Uint8Array(4)), 3); + assertEquals(await r.read(new Uint8Array(1)), null); +}); diff --git a/io/string_writer.ts b/io/string_writer.ts new file mode 100644 index 000000000000..f7e6ff639e14 --- /dev/null +++ b/io/string_writer.ts @@ -0,0 +1,73 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +// This module is browser compatible. + +import type { Writer, WriterSync } from "./types.d.ts"; + +const decoder = new TextDecoder(); + +/** + * Writer utility for buffering string chunks. + * + * @example + * ```ts + * import { + * copyN, + * StringReader, + * StringWriter, + * } from "https://deno.land/std@$STD_VERSION/io/mod.ts"; + * import { copy } from "https://deno.land/std@$STD_VERSION/streams/copy.ts"; + * + * const w = new StringWriter("base"); + * const r = new StringReader("0123456789"); + * await copyN(r, w, 4); // copy 4 bytes + * + * // Number of bytes read + * console.log(w.toString()); //base0123 + * + * await copy(r, w); // copy all + * console.log(w.toString()); // base0123456789 + * ``` + * + * **Output:** + * + * ```text + * base0123 + * base0123456789 + * ``` + */ +export class StringWriter implements Writer, WriterSync { + #chunks: Uint8Array[] = []; + #byteLength = 0; + #cache: string | undefined; + + constructor(private base: string = "") { + const c = new TextEncoder().encode(base); + this.#chunks.push(c); + this.#byteLength += c.byteLength; + } + + write(p: Uint8Array): Promise { + return Promise.resolve(this.writeSync(p)); + } + + writeSync(p: Uint8Array): number { + this.#chunks.push(new Uint8Array(p)); + this.#byteLength += p.byteLength; + this.#cache = undefined; + return p.byteLength; + } + + toString(): string { + if (this.#cache) { + return this.#cache; + } + const buf = new Uint8Array(this.#byteLength); + let offs = 0; + for (const chunk of this.#chunks) { + buf.set(chunk, offs); + offs += chunk.byteLength; + } + this.#cache = decoder.decode(buf); + return this.#cache; + } +} diff --git a/io/writers_test.ts b/io/string_writer_test.ts similarity index 88% rename from io/writers_test.ts rename to io/string_writer_test.ts index 92d02c6a6429..7b9a843826bb 100644 --- a/io/writers_test.ts +++ b/io/string_writer_test.ts @@ -1,8 +1,8 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. import { assertEquals } from "../testing/asserts.ts"; -import { StringWriter } from "./writers.ts"; -import { StringReader } from "./readers.ts"; -import { copyN } from "./util.ts"; +import { StringWriter } from "./string_writer.ts"; +import { StringReader } from "./string_reader.ts"; +import { copyN } from "./copy_n.ts"; import { copy } from "../streams/copy.ts"; Deno.test("ioStringWriter", async function () { diff --git a/io/test.ts b/io/test.ts deleted file mode 100644 index aabd7ef0a9f7..000000000000 --- a/io/test.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -import "./mod.ts"; diff --git a/io/util.ts b/io/util.ts index 19f1b8f56a77..f56dd0c10581 100644 --- a/io/util.ts +++ b/io/util.ts @@ -1,102 +1,49 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -import { assert } from "../_util/asserts.ts"; -import type { BufReader } from "./buffer.ts"; -import type { Reader, Writer } from "./types.d.ts"; - -const DEFAULT_BUFFER_SIZE = 32 * 1024; +import { copyN as _copyN } from "./copy_n.ts"; +import { readShort as _readShort } from "./read_short.ts"; +import { readInt as _readInt } from "./read_int.ts"; +import { readLong as _readLong } from "./read_long.ts"; +import { sliceLongToBytes as _sliceLongToBytes } from "./slice_long_to_bytes.ts"; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/copy_reader.ts` instead + * * Copy N size at the most. If read size is lesser than N, then returns nread * @param r Reader * @param dest Writer * @param size Read size */ -export async function copyN( - r: Reader, - dest: Writer, - size: number, -): Promise { - let bytesRead = 0; - let buf = new Uint8Array(DEFAULT_BUFFER_SIZE); - while (bytesRead < size) { - if (size - bytesRead < DEFAULT_BUFFER_SIZE) { - buf = new Uint8Array(size - bytesRead); - } - const result = await r.read(buf); - const nread = result ?? 0; - bytesRead += nread; - if (nread > 0) { - let n = 0; - while (n < nread) { - n += await dest.write(buf.slice(n, nread)); - } - assert(n === nread, "could not write"); - } - if (result === null) { - break; - } - } - return bytesRead; -} +export const copyN = _copyN; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/read_short.ts` instead + * * Read big endian 16bit short from BufReader * @param buf */ -export async function readShort(buf: BufReader): Promise { - const high = await buf.readByte(); - if (high === null) return null; - const low = await buf.readByte(); - if (low === null) throw new Deno.errors.UnexpectedEof(); - return (high << 8) | low; -} +export const readShort = _readShort; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/read_int.ts` instead + * * Read big endian 32bit integer from BufReader * @param buf */ -export async function readInt(buf: BufReader): Promise { - const high = await readShort(buf); - if (high === null) return null; - const low = await readShort(buf); - if (low === null) throw new Deno.errors.UnexpectedEof(); - return (high << 16) | low; -} - -const MAX_SAFE_INTEGER = BigInt(Number.MAX_SAFE_INTEGER); +export const readInt = _readInt; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/read_long.ts` instead + * * Read big endian 64bit long from BufReader * @param buf */ -export async function readLong(buf: BufReader): Promise { - const high = await readInt(buf); - if (high === null) return null; - const low = await readInt(buf); - if (low === null) throw new Deno.errors.UnexpectedEof(); - const big = (BigInt(high) << 32n) | BigInt(low); - // We probably should provide a similar API that returns BigInt values. - if (big > MAX_SAFE_INTEGER) { - throw new RangeError( - "Long value too big to be represented as a JavaScript number.", - ); - } - return Number(big); -} +export const readLong = _readLong; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/slice_long_to_bytes.ts` instead + * * Slice number into 64bit big endian byte array * @param d The number to be sliced * @param dest The sliced array */ -export function sliceLongToBytes( - d: number, - dest = Array.from({ length: 8 }), -): number[] { - let big = BigInt(d); - for (let i = 0; i < 8; i++) { - dest[7 - i] = Number(big & 0xffn); - big >>= 8n; - } - return dest; -} +export const sliceLongToBytes = _sliceLongToBytes; diff --git a/io/util_test.ts b/io/util_test.ts deleted file mode 100644 index addd27d478d1..000000000000 --- a/io/util_test.ts +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -import { assertEquals } from "../testing/asserts.ts"; -import { - copyN, - readInt, - readLong, - readShort, - sliceLongToBytes, -} from "./util.ts"; -import { Buffer, BufReader } from "./buffer.ts"; -import { StringReader } from "./readers.ts"; -import type { Reader } from "./types.d.ts"; - -class BinaryReader implements Reader { - index = 0; - - constructor(private bytes: Uint8Array = new Uint8Array(0)) {} - - read(p: Uint8Array): Promise { - p.set(this.bytes.subarray(this.index, p.byteLength)); - this.index += p.byteLength; - return Promise.resolve(p.byteLength); - } -} - -Deno.test("testReadShort", async function () { - const r = new BinaryReader(new Uint8Array([0x12, 0x34])); - const short = await readShort(new BufReader(r)); - assertEquals(short, 0x1234); -}); - -Deno.test("testReadInt", async function () { - const r = new BinaryReader(new Uint8Array([0x12, 0x34, 0x56, 0x78])); - const int = await readInt(new BufReader(r)); - assertEquals(int, 0x12345678); -}); - -Deno.test("testReadLong", async function () { - const r = new BinaryReader( - new Uint8Array([0x00, 0x00, 0x00, 0x78, 0x12, 0x34, 0x56, 0x78]), - ); - const long = await readLong(new BufReader(r)); - assertEquals(long, 0x7812345678); -}); - -Deno.test("testReadLong2", async function () { - const r = new BinaryReader( - new Uint8Array([0, 0, 0, 0, 0x12, 0x34, 0x56, 0x78]), - ); - const long = await readLong(new BufReader(r)); - assertEquals(long, 0x12345678); -}); - -Deno.test("testSliceLongToBytes", function () { - const arr = sliceLongToBytes(0x1234567890abcdef); - const actual = readLong(new BufReader(new BinaryReader(new Uint8Array(arr)))); - const expected = readLong( - new BufReader( - new BinaryReader( - new Uint8Array([0x12, 0x34, 0x56, 0x78, 0x90, 0xab, 0xcd, 0xef]), - ), - ), - ); - assertEquals(actual, expected); -}); - -Deno.test("testSliceLongToBytes2", function () { - const arr = sliceLongToBytes(0x12345678); - assertEquals(arr, [0, 0, 0, 0, 0x12, 0x34, 0x56, 0x78]); -}); - -Deno.test("testCopyN1", async function () { - const w = new Buffer(); - const r = new StringReader("abcdefghij"); - const n = await copyN(r, w, 3); - assertEquals(n, 3); - assertEquals(new TextDecoder().decode(w.bytes()), "abc"); -}); - -Deno.test("testCopyN2", async function () { - const w = new Buffer(); - const r = new StringReader("abcdefghij"); - const n = await copyN(r, w, 11); - assertEquals(n, 10); - assertEquals(new TextDecoder().decode(w.bytes()), "abcdefghij"); -}); - -Deno.test("copyNWriteAllData", async function () { - const tmpDir = await Deno.makeTempDir(); - const filepath = `${tmpDir}/data`; - const file = await Deno.open(filepath, { create: true, write: true }); - - const size = 16 * 1024 + 1; - const data = "a".repeat(32 * 1024); - const r = new StringReader(data); - const n = await copyN(r, file, size); // Over max file possible buffer - file.close(); - await Deno.remove(filepath); - - assertEquals(n, size); -}); - -Deno.test("testStringReaderEof", async function () { - const r = new StringReader("abc"); - assertEquals(await r.read(new Uint8Array()), 0); - assertEquals(await r.read(new Uint8Array(4)), 3); - assertEquals(await r.read(new Uint8Array(1)), null); -}); diff --git a/io/writers.ts b/io/writers.ts index f7e6ff639e14..587ec5028753 100644 --- a/io/writers.ts +++ b/io/writers.ts @@ -1,11 +1,11 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. // This module is browser compatible. -import type { Writer, WriterSync } from "./types.d.ts"; - -const decoder = new TextDecoder(); +import { StringWriter as _StringWriter } from "./string_writer.ts"; /** + * @deprecated (will be removed after 0.170.0) Import from `std/io/string_writer.ts` instead + * * Writer utility for buffering string chunks. * * @example @@ -35,39 +35,4 @@ const decoder = new TextDecoder(); * base0123456789 * ``` */ -export class StringWriter implements Writer, WriterSync { - #chunks: Uint8Array[] = []; - #byteLength = 0; - #cache: string | undefined; - - constructor(private base: string = "") { - const c = new TextEncoder().encode(base); - this.#chunks.push(c); - this.#byteLength += c.byteLength; - } - - write(p: Uint8Array): Promise { - return Promise.resolve(this.writeSync(p)); - } - - writeSync(p: Uint8Array): number { - this.#chunks.push(new Uint8Array(p)); - this.#byteLength += p.byteLength; - this.#cache = undefined; - return p.byteLength; - } - - toString(): string { - if (this.#cache) { - return this.#cache; - } - const buf = new Uint8Array(this.#byteLength); - let offs = 0; - for (const chunk of this.#chunks) { - buf.set(chunk, offs); - offs += chunk.byteLength; - } - this.#cache = decoder.decode(buf); - return this.#cache; - } -} +export const StringWriter = _StringWriter; diff --git a/log/handlers.ts b/log/handlers.ts index 195540e2b82f..fd639a8d2c41 100644 --- a/log/handlers.ts +++ b/log/handlers.ts @@ -3,7 +3,7 @@ import { getLevelByName, LevelName, LogLevels } from "./levels.ts"; import type { LogRecord } from "./logger.ts"; import { blue, bold, red, yellow } from "../fmt/colors.ts"; import { exists, existsSync } from "../fs/exists.ts"; -import { BufWriterSync } from "../io/buffer.ts"; +import { BufWriterSync } from "../io/buf_writer.ts"; const DEFAULT_FORMATTER = "{levelName} {msg}"; export type FormatterFunction = (logRecord: LogRecord) => string;