feat: string pack type to flexible

src/__test__/packer.test.ts

@@ -21,4 +21,15 @@ describe('packer', () => {
       packer('>f' as string)(new Uint8Array(4), 0, '2')
     ).toThrowError()
   })
+
+  test('pack with array', () => {
+    const b = new Uint8Array(10)
+    packer('>10s')(b, 0, [3, 4, 5])
+    expect(b).toStrictEqual(Uint8Array.of(3, 4, 5, 0, 0, 0, 0, 0, 0, 0))
+  })
+  test('pack with string', () => {
+    const b = new Uint8Array(10)
+    packer('>10s')(b, 0, 'abcd')
+    expect(b).toStrictEqual(Uint8Array.of(97, 98, 99, 100, 0, 0, 0, 0, 0, 0))
+  })
 })

src/__test__/struct.test.ts

@@ -473,6 +473,35 @@ test('default export', () => {
   )
 })
 
+test('pack with string', () => {
+  expect(struct.pack('6s', '123456')).toStrictEqual(
+    Uint8Array.from([49, 50, 51, 52, 53, 54])
+  )
+  expect(struct.pack('4s', '💩')).toStrictEqual(
+    Uint8Array.from([0xf0, 0x9f, 0x92, 0xa9])
+  )
+
+  expect(struct.pack('2s', '¥')).toStrictEqual(Uint8Array.from([0xc2, 0xa5]))
+
+  expect(struct.pack('3s', 'あ')).toStrictEqual(
+    Uint8Array.from([0xe3, 0x81, 0x82])
+  )
+  expect(struct.pack('3s', 'あ')).toStrictEqual(
+    Uint8Array.from([0xe3, 0x81, 0x82])
+  )
+  expect(struct.pack('5s', '𠮷')).toStrictEqual(
+    Uint8Array.from([0xf0, 0xa0, 0xae, 0xb7, 0])
+  )
+})
+test('pack with invalid string', () => {
+  expect(struct.pack('6s', String.fromCharCode(0xd801))).toStrictEqual(
+    Uint8Array.from([0xef, 0xbf, 0xbd, 0, 0, 0])
+  )
+  expect(struct.pack('6s', String.fromCharCode(0xd801, 0xdb00))).toStrictEqual(
+    Uint8Array.from([0xef, 0xbf, 0xbd, 0xef, 0xbf, 0xbd])
+  )
+})
+
 describe('class interface', () => {
   test('BigInt', () => {
     const s = new Struct('2Q2q')

src/__test__/typetest.test.ts

@@ -9,7 +9,7 @@ test('typecheck', () => {
 
 test('@2c', () => {
   const st = new Struct('@2c')
-  const v = st.unpack(st.pack('s', 'v'))
+  const v: [string, string] = st.unpack(st.pack('s', 'v'))
   expect(v).toStrictEqual(['s', 'v'])
 })
 test('@2l', () => {

src/packer.ts

@@ -1,11 +1,12 @@
-import { FormatTokenTuple } from './struct_type'
+import { FormatTokenTupleForWrite } from './struct_type'
 import { splitTokens, formatOrder, Token } from './format'
 import { WriteDataViewStream } from './stream'
+import { textencode } from './textencode'
 
 export type Packer<T extends string> = (
   buffer: Uint8Array,
   offset: number,
-  ...args: FormatTokenTuple<T>
+  ...args: FormatTokenTupleForWrite<T>
 ) => void
 export const packer = <T extends string>(format: T): Packer<T> => {
   const f = formatOrder(format)
@@ -14,7 +15,7 @@ export const packer = <T extends string>(format: T): Packer<T> => {
   return (
     buffer: Uint8Array,
     bufferOffset: number,
-    ...args: FormatTokenTuple<T>
+    ...args: FormatTokenTupleForWrite<T>
   ) => {
     const stream = new WriteDataViewStream(buffer, bufferOffset, f.le)
 
@@ -45,11 +46,12 @@ export const packer = <T extends string>(format: T): Packer<T> => {
     }
     const writeString = (t: Token): void => {
       const v = args.shift()
-      if (!(v instanceof Uint8Array)) {
+      const b = (typeof v === 'string' ? textencode(v) : v) as ArrayLike<number>
+      if (b.length === undefined) {
         throw Error('Invalid Argument type')
       }
       for (let i = 0; i < t.count; i++) {
-        stream.writeUInt8(v[i] || 0)
+        stream.writeUInt8(b[i] || 0)
       }
     }
     const checkSIntRange = (

src/struct.ts

@@ -1,12 +1,12 @@
-import { FormatTokenTuple } from './struct_type'
+import { FormatTokenTuple, FormatTokenTupleForWrite } from './struct_type'
 import { splitTokens, formatOrder } from './format'
 
 import { packer, Packer } from './packer'
 import { unpacker, UnPacker } from './unpacker'
 
 export const pack = <T extends string>(
   format: T,
-  ...args: FormatTokenTuple<T>
+  ...args: FormatTokenTupleForWrite<T>
 ): Uint8Array => {
   const size = calcsize(format)
   const buffer = new Uint8Array(size)
@@ -17,7 +17,7 @@ export const pack_into = <T extends string>(
   format: T,
   buffer: Uint8Array,
   offset: number,
-  ...args: FormatTokenTuple<T>
+  ...args: FormatTokenTupleForWrite<T>
 ): void => {
   packer(format)(buffer, offset, ...args)
 }
@@ -54,15 +54,15 @@ export class Struct<T extends string> {
     this.packer = packer(format)
     this.unpacker = unpacker(format)
   }
-  pack(...arg: FormatTokenTuple<T>): Uint8Array {
+  pack(...arg: FormatTokenTupleForWrite<T>): Uint8Array {
     const buffer = new Uint8Array(this.size)
     this.packer(buffer, 0, ...arg)
     return buffer
   }
   pack_into(
     buffer: Uint8Array,
     offset: number,
-    ...arg: FormatTokenTuple<T>
+    ...arg: FormatTokenTupleForWrite<T>
   ): void {
     if (buffer.length - offset < this.size) {
       throw Error('Not enough buffer.')

src/struct_type.ts

@@ -82,46 +82,61 @@ type NormalizeCountDigits<CountDigits extends string[]> = CountDigits extends []
   ? ['1']
   : CountDigits
 
-type ChooseTypeTuple<
-  T extends FormatChar,
-  StandardSize extends boolean
-> = StandardSize extends true
-  ? StandardTokenCharTypeMap[T] extends void
-    ? []
-    : [StandardTokenCharTypeMap[T]]
-  : NativeTokenCharTypeMap[T] extends void
-  ? []
-  : [NativeTokenCharTypeMap[T]]
-type ChooseType<
+type ChooseUnpackType<
   T extends FormatChar,
   StandardSize extends boolean
 > = StandardSize extends true
   ? StandardTokenCharTypeMap[T]
   : NativeTokenCharTypeMap[T]
 
+type TransformPackType<T> = T extends Uint8Array
+  ? string | ArrayLike<number>
+  : T
+
+type ChooseType<
+  T extends FormatChar,
+  StandardSize extends boolean,
+  forPack extends boolean
+> = forPack extends true
+  ? TransformPackType<ChooseUnpackType<T, StandardSize>>
+  : ChooseUnpackType<T, StandardSize>
+
+type ChooseTypeTuple<
+  T extends FormatChar,
+  StandardSize extends boolean,
+  forPack extends boolean
+> = ChooseType<T, StandardSize, forPack> extends void
+  ? []
+  : [ChooseType<T, StandardSize, forPack>]
+
 type ToTypeTuple<
   T extends FormatChar,
   CountDigits extends string[],
-  StandardSize extends boolean
+  StandardSize extends boolean,
+  forPack extends boolean
 > = T extends LengthFormatChar
-  ? ChooseTypeTuple<T, StandardSize>
+  ? ChooseTypeTuple<T, StandardSize, forPack>
   : T extends RepeatableFormatChar
-  ? RepeatA<ChooseType<T, StandardSize>, NormalizeCountDigits<CountDigits>>
+  ? RepeatA<
+      ChooseType<T, StandardSize, forPack>,
+      NormalizeCountDigits<CountDigits>
+    >
   : never
 
 type ParseTokenSub<
   T extends string,
   StandardSize extends boolean,
+  forPack extends boolean,
   CountDigitsHolder extends string[] = []
 > = T extends ''
   ? []
   : First<T> extends infer D
   ? D extends Digit
-    ? ParseTokenSub<Tail<T>, StandardSize, [...CountDigitsHolder, D]>
+    ? ParseTokenSub<Tail<T>, StandardSize, forPack, [...CountDigitsHolder, D]>
     : D extends FormatChar
     ? {
-        type: ToTypeTuple<D, CountDigitsHolder, StandardSize>
-        next: ParseTokenSub<Tail<T>, StandardSize>
+        type: ToTypeTuple<D, CountDigitsHolder, StandardSize, forPack>
+        next: ParseTokenSub<Tail<T>, StandardSize, forPack>
       }
     : never
   : never
@@ -145,9 +160,11 @@ type Flatten2<T> = T extends {
     : T
   : T
 
-type ParseToken<T extends string, StandardSize extends boolean> = Flatten<
-  ParseTokenSub<T, StandardSize>
->
+type ParseToken<
+  T extends string,
+  StandardSize extends boolean,
+  forPack extends boolean
+> = Flatten<ParseTokenSub<T, StandardSize, forPack>>
 
 type IsStandardTypeFormat<T extends string> = T extends `${
   | '<'
@@ -157,8 +174,16 @@ type IsStandardTypeFormat<T extends string> = T extends `${
   ? true
   : false
 
-export type FormatTokenTuple<T extends string> = string extends T
+export type FormatTokenTuple<
+  T extends string,
+  forPack extends boolean = false
+> = string extends T
   ? TokenType[]
   : T extends `${OrderType}${infer U}`
-  ? ParseToken<U, IsStandardTypeFormat<T>>
+  ? ParseToken<U, IsStandardTypeFormat<T>, forPack>
   : never
+
+export type FormatTokenTupleForWrite<T extends string> = FormatTokenTuple<
+  T,
+  true
+>

src/textencode.ts

@@ -0,0 +1,55 @@
+/// https://developer.mozilla.org/ja/docs/Web/API/TextEncoder
+export function textencode(str: string): Uint8Array {
+  const len = str.length
+  let resPos = -1
+  // The Uint8Array's length must be at least 3x the length of the string because an invalid UTF-16
+  //  takes up the equivelent space of 3 UTF-8 characters to encode it properly. However, Array's
+  //  have an auto expanding length and 1.5x should be just the right balance for most uses.
+  const resArr = new Uint8Array(len * 3)
+  for (let i = 0; i !== len; ) {
+    const firstcode = str.charCodeAt(i)
+    i += 1
+    if (firstcode >= 0xd800 && firstcode <= 0xdbff) {
+      if (i === len) {
+        resArr[(resPos += 1)] = 0xef /*0b11101111*/
+        resArr[(resPos += 1)] = 0xbf /*0b10111111*/
+        resArr[(resPos += 1)] = 0xbd /*0b10111101*/
+        break
+      }
+      // https://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae
+      const nextcode = str.charCodeAt(i)
+      if (nextcode >= 0xdc00 && nextcode <= 0xdfff) {
+        const codepoint =
+          (firstcode - 0xd800) * 0x400 + nextcode - 0xdc00 + 0x10000
+        i += 1
+        resArr[(resPos += 1)] = (0x1e /*0b11110*/ << 3) | (codepoint >>> 18)
+        resArr[(resPos += 1)] =
+          (0x2 /*0b10*/ << 6) | ((codepoint >>> 12) & 0x3f) /*0b00111111*/
+        resArr[(resPos += 1)] =
+          (0x2 /*0b10*/ << 6) | ((codepoint >>> 6) & 0x3f) /*0b00111111*/
+        resArr[(resPos += 1)] =
+          (0x2 /*0b10*/ << 6) | (codepoint & 0x3f) /*0b00111111*/
+        continue
+      } else {
+        resArr[(resPos += 1)] = 0xef /*0b11101111*/
+        resArr[(resPos += 1)] = 0xbf /*0b10111111*/
+        resArr[(resPos += 1)] = 0xbd /*0b10111101*/
+        continue
+      }
+    }
+    if (firstcode <= 0x007f) {
+      resArr[(resPos += 1)] = (0x0 /*0b0*/ << 7) | firstcode
+    } else if (firstcode <= 0x07ff) {
+      resArr[(resPos += 1)] = (0x6 /*0b110*/ << 5) | (firstcode >>> 6)
+      resArr[(resPos += 1)] =
+        (0x2 /*0b10*/ << 6) | (firstcode & 0x3f) /*0b00111111*/
+    } else {
+      resArr[(resPos += 1)] = (0xe /*0b1110*/ << 4) | (firstcode >>> 12)
+      resArr[(resPos += 1)] =
+        (0x2 /*0b10*/ << 6) | ((firstcode >>> 6) & 0x3f) /*0b00111111*/
+      resArr[(resPos += 1)] =
+        (0x2 /*0b10*/ << 6) | (firstcode & 0x3f) /*0b00111111*/
+    }
+  }
+  return resArr.subarray(0, resPos + 1)
+}