-
Notifications
You must be signed in to change notification settings - Fork 5.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(node): buffer isUtf8/isAscii (#23928)
Fixes: #23657 Implements `isUtf8` and `isAscii` as ops.
- Loading branch information
Showing
11 changed files
with
219 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. | ||
|
||
use deno_core::op2; | ||
|
||
#[op2(fast)] | ||
pub fn op_is_ascii(#[buffer] buf: &[u8]) -> bool { | ||
buf.is_ascii() | ||
} | ||
|
||
#[op2(fast)] | ||
pub fn op_is_utf8(#[buffer] buf: &[u8]) -> bool { | ||
std::str::from_utf8(buf).is_ok() | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -7,6 +7,8 @@ export { | |
Buffer, | ||
constants, | ||
default, | ||
isAscii, | ||
isUtf8, | ||
kMaxLength, | ||
kStringMaxLength, | ||
SlowBuffer, | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
// deno-fmt-ignore-file | ||
// deno-lint-ignore-file | ||
|
||
// Copyright Joyent and Node contributors. All rights reserved. MIT license. | ||
// Taken from Node 18.12.1 | ||
// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. | ||
|
||
'use strict'; | ||
|
||
require('../common'); | ||
const assert = require('assert'); | ||
const { isAscii, Buffer } = require('buffer'); | ||
const { TextEncoder } = require('util'); | ||
|
||
const encoder = new TextEncoder(); | ||
|
||
assert.strictEqual(isAscii(encoder.encode('hello')), true); | ||
assert.strictEqual(isAscii(encoder.encode('ğ')), false); | ||
assert.strictEqual(isAscii(Buffer.from([])), true); | ||
|
||
[ | ||
undefined, | ||
'', 'hello', | ||
false, true, | ||
0, 1, | ||
0n, 1n, | ||
Symbol(), | ||
() => {}, | ||
{}, [], null, | ||
].forEach((input) => { | ||
assert.throws( | ||
() => { isAscii(input); }, | ||
{ | ||
code: 'ERR_INVALID_ARG_TYPE', | ||
}, | ||
); | ||
}); | ||
|
||
{ | ||
// Test with detached array buffers | ||
const arrayBuffer = new ArrayBuffer(1024); | ||
structuredClone(arrayBuffer, { transfer: [arrayBuffer] }); | ||
assert.throws( | ||
() => { isAscii(arrayBuffer); }, | ||
{ | ||
code: 'ERR_INVALID_STATE' | ||
} | ||
); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
// deno-fmt-ignore-file | ||
// deno-lint-ignore-file | ||
|
||
// Copyright Joyent and Node contributors. All rights reserved. MIT license. | ||
// Taken from Node 18.12.1 | ||
// This file is automatically generated by `tests/node_compat/runner/setup.ts`. Do not modify this file manually. | ||
|
||
'use strict'; | ||
|
||
require('../common'); | ||
const assert = require('assert'); | ||
const { isUtf8, Buffer } = require('buffer'); | ||
const { TextEncoder } = require('util'); | ||
|
||
const encoder = new TextEncoder(); | ||
|
||
assert.strictEqual(isUtf8(encoder.encode('hello')), true); | ||
assert.strictEqual(isUtf8(encoder.encode('ğ')), true); | ||
assert.strictEqual(isUtf8(Buffer.from([])), true); | ||
|
||
// Taken from test/fixtures/wpt/encoding/textdecoder-fatal.any.js | ||
[ | ||
[0xFF], // 'invalid code' | ||
[0xC0], // 'ends early' | ||
[0xE0], // 'ends early 2' | ||
[0xC0, 0x00], // 'invalid trail' | ||
[0xC0, 0xC0], // 'invalid trail 2' | ||
[0xE0, 0x00], // 'invalid trail 3' | ||
[0xE0, 0xC0], // 'invalid trail 4' | ||
[0xE0, 0x80, 0x00], // 'invalid trail 5' | ||
[0xE0, 0x80, 0xC0], // 'invalid trail 6' | ||
[0xFC, 0x80, 0x80, 0x80, 0x80, 0x80], // '> 0x10FFFF' | ||
[0xFE, 0x80, 0x80, 0x80, 0x80, 0x80], // 'obsolete lead byte' | ||
|
||
// Overlong encodings | ||
[0xC0, 0x80], // 'overlong U+0000 - 2 bytes' | ||
[0xE0, 0x80, 0x80], // 'overlong U+0000 - 3 bytes' | ||
[0xF0, 0x80, 0x80, 0x80], // 'overlong U+0000 - 4 bytes' | ||
[0xF8, 0x80, 0x80, 0x80, 0x80], // 'overlong U+0000 - 5 bytes' | ||
[0xFC, 0x80, 0x80, 0x80, 0x80, 0x80], // 'overlong U+0000 - 6 bytes' | ||
|
||
[0xC1, 0xBF], // 'overlong U+007F - 2 bytes' | ||
[0xE0, 0x81, 0xBF], // 'overlong U+007F - 3 bytes' | ||
[0xF0, 0x80, 0x81, 0xBF], // 'overlong U+007F - 4 bytes' | ||
[0xF8, 0x80, 0x80, 0x81, 0xBF], // 'overlong U+007F - 5 bytes' | ||
[0xFC, 0x80, 0x80, 0x80, 0x81, 0xBF], // 'overlong U+007F - 6 bytes' | ||
|
||
[0xE0, 0x9F, 0xBF], // 'overlong U+07FF - 3 bytes' | ||
[0xF0, 0x80, 0x9F, 0xBF], // 'overlong U+07FF - 4 bytes' | ||
[0xF8, 0x80, 0x80, 0x9F, 0xBF], // 'overlong U+07FF - 5 bytes' | ||
[0xFC, 0x80, 0x80, 0x80, 0x9F, 0xBF], // 'overlong U+07FF - 6 bytes' | ||
|
||
[0xF0, 0x8F, 0xBF, 0xBF], // 'overlong U+FFFF - 4 bytes' | ||
[0xF8, 0x80, 0x8F, 0xBF, 0xBF], // 'overlong U+FFFF - 5 bytes' | ||
[0xFC, 0x80, 0x80, 0x8F, 0xBF, 0xBF], // 'overlong U+FFFF - 6 bytes' | ||
|
||
[0xF8, 0x84, 0x8F, 0xBF, 0xBF], // 'overlong U+10FFFF - 5 bytes' | ||
[0xFC, 0x80, 0x84, 0x8F, 0xBF, 0xBF], // 'overlong U+10FFFF - 6 bytes' | ||
|
||
// UTF-16 surrogates encoded as code points in UTF-8 | ||
[0xED, 0xA0, 0x80], // 'lead surrogate' | ||
[0xED, 0xB0, 0x80], // 'trail surrogate' | ||
[0xED, 0xA0, 0x80, 0xED, 0xB0, 0x80], // 'surrogate pair' | ||
].forEach((input) => { | ||
assert.strictEqual(isUtf8(Buffer.from(input)), false); | ||
}); | ||
|
||
[ | ||
null, | ||
undefined, | ||
'hello', | ||
true, | ||
false, | ||
].forEach((input) => { | ||
assert.throws( | ||
() => { isUtf8(input); }, | ||
{ | ||
code: 'ERR_INVALID_ARG_TYPE', | ||
}, | ||
); | ||
}); | ||
|
||
{ | ||
// Test with detached array buffers | ||
const arrayBuffer = new ArrayBuffer(1024); | ||
structuredClone(arrayBuffer, { transfer: [arrayBuffer] }); | ||
assert.throws( | ||
() => { isUtf8(arrayBuffer); }, | ||
{ | ||
code: 'ERR_INVALID_STATE' | ||
} | ||
); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters