feat(core): support Uint8Array (#4419)

The goal is to work properly with entities that have properties of type
Uint8Array. Those properties should automatically be stored as binary
data and converted back to Uint8Array without specifying a type
(analogous to Buffer).

Also when converting entities to DTOs Uint8Array types should be kept
unchanged.

Closes #4418

docs/docs/custom-types.md

@@ -237,6 +237,7 @@ export const types = {
   datetime: DateTimeType,
   bigint: BigIntType,
   blob: BlobType,
+  uint8array: Uint8ArrayType,
   array: ArrayType,
   enumArray: EnumArrayType,
   enum: EnumType,
@@ -291,6 +292,17 @@ Blob type can be used to store binary data in the database.
 blob?: Buffer;
 ```
 
+### Uint8ArrayType
+
+Uint8Array type can be used to store binary data in the database.
+
+> `Uint8ArrayType` will be used automatically if you specify the type hint as `Uint8Array`. This means that the following example should work even without the explicit `type: Uint8ArrayType` option (with both reflect-metadata and ts-morph providers).
+
+```ts
+@Property({ type: Uint8ArrayType, nullable: true })
+blob?: Uint8Array;
+```
+
 ### JsonType
 
 To store objects we can use `JsonType`. As some drivers are handling objects automatically and some don't, this type will handle the serialization in a driver independent way (calling `parse` and `stringify` only when needed).

packages/core/src/platforms/Platform.ts

@@ -8,7 +8,7 @@ import type { EntityManager } from '../EntityManager';
 import type { Configuration } from '../utils/Configuration';
 import type { IDatabaseDriver } from '../drivers/IDatabaseDriver';
 import {
-  ArrayType, BigIntType, BlobType, BooleanType, DateType, DecimalType, DoubleType, JsonType, SmallIntType, TimeType,
+  ArrayType, BigIntType, BlobType, Uint8ArrayType, BooleanType, DateType, DecimalType, DoubleType, JsonType, SmallIntType, TimeType,
   TinyIntType, Type, UuidType, StringType, IntegerType, FloatType, DateTimeType, TextType, EnumType, UnknownType, MediumIntType,
 } from '../types';
 import { parseJsonSafe, Utils } from '../utils/Utils';
@@ -262,6 +262,7 @@ export abstract class Platform {
       case 'boolean': return Type.getType(BooleanType);
       case 'blob':
       case 'buffer': return Type.getType(BlobType);
+      case 'uint8array': return Type.getType(Uint8ArrayType);
       case 'uuid': return Type.getType(UuidType);
       case 'date': return Type.getType(DateType);
       case 'datetime': return Type.getType(DateTimeType);

packages/core/src/types/BlobType.ts

@@ -1,8 +1,6 @@
-import { Type } from './Type';
-import type { Platform } from '../platforms';
-import type { EntityProperty } from '../typings';
+import { Uint8ArrayType } from './Uint8ArrayType';
 
-export class BlobType extends Type<Buffer | null> {
+export class BlobType extends Uint8ArrayType {
 
   convertToDatabaseValue(value: Buffer): Buffer {
     return value;
@@ -22,16 +20,4 @@ export class BlobType extends Type<Buffer | null> {
     return Buffer.from(value);
   }
 
-  compareAsType(): string {
-    return 'Buffer';
-  }
-
-  ensureComparable(): boolean {
-    return false;
-  }
-
-  getColumnType(prop: EntityProperty, platform: Platform): string {
-    return platform.getBlobDeclarationSQL();
-  }
-
 }

packages/core/src/types/Uint8ArrayType.ts

@@ -0,0 +1,42 @@
+import { Type } from './Type';
+import type { Platform } from '../platforms';
+import type { EntityProperty } from '../typings';
+
+export class Uint8ArrayType extends Type<Uint8Array | null> {
+
+  convertToDatabaseValue(value: Uint8Array): Buffer {
+    return Buffer.from(value);
+  }
+
+  convertToJSValue(value: Buffer): Uint8Array | null {
+    if (!value) {
+      return value;
+    }
+
+    /* istanbul ignore else */
+    if (value as unknown instanceof Buffer) {
+      return new Uint8Array(value);
+    }
+
+    /* istanbul ignore else */
+    if (value.buffer instanceof Buffer) {
+      return new Uint8Array(value.buffer);
+    }
+
+    /* istanbul ignore next */
+    return new Uint8Array(Buffer.from(value));
+  }
+
+  compareAsType(): string {
+    return 'Buffer';
+  }
+
+  ensureComparable(): boolean {
+    return false;
+  }
+
+  getColumnType(prop: EntityProperty, platform: Platform): string {
+    return platform.getBlobDeclarationSQL();
+  }
+
+}

packages/core/src/types/index.ts

@@ -4,6 +4,7 @@ import { TimeType } from './TimeType';
 import { DateTimeType } from './DateTimeType';
 import { BigIntType } from './BigIntType';
 import { BlobType } from './BlobType';
+import { Uint8ArrayType } from './Uint8ArrayType';
 import { ArrayType } from './ArrayType';
 import { EnumArrayType } from './EnumArrayType';
 import { EnumType } from './EnumType';
@@ -22,7 +23,7 @@ import { TextType } from './TextType';
 import { UnknownType } from './UnknownType';
 
 export {
-  Type, DateType, TimeType, DateTimeType, BigIntType, BlobType, ArrayType, EnumArrayType, EnumType,
+  Type, DateType, TimeType, DateTimeType, BigIntType, BlobType, Uint8ArrayType, ArrayType, EnumArrayType, EnumType,
   JsonType, IntegerType, SmallIntType, TinyIntType, MediumIntType, FloatType, DoubleType, BooleanType, DecimalType,
   StringType, UuidType, TextType, UnknownType, TransformContext,
 };
@@ -33,6 +34,7 @@ export const types = {
   datetime: DateTimeType,
   bigint: BigIntType,
   blob: BlobType,
+  uint8array: Uint8ArrayType,
   array: ArrayType,
   enumArray: EnumArrayType,
   enum: EnumType,

packages/core/src/typings.ts

@@ -58,7 +58,7 @@ export type PrimaryProperty<T> = T extends { [PrimaryKeyProp]?: infer PK }
 export type IPrimaryKeyValue = number | string | bigint | Date | { toHexString(): string };
 export type IPrimaryKey<T extends IPrimaryKeyValue = IPrimaryKeyValue> = T;
 
-export type Scalar = boolean | number | string | bigint | symbol | Date | RegExp | Buffer | { toHexString(): string };
+export type Scalar = boolean | number | string | bigint | symbol | Date | RegExp | Uint8Array | { toHexString(): string };
 
 export type ExpandScalar<T> = null | (T extends string
   ? T | RegExp

tests/EntityManager.mongo.test.ts

@@ -2101,6 +2101,7 @@ describe('EntityManagerMongo', () => {
 
     const bar = FooBar.create('b1');
     bar.blob = Buffer.from([1, 2, 3, 4, 5]);
+    bar.blob2 = new Uint8Array([1, 2, 3, 4, 5]);
     bar.array = [];
     bar.object = { foo: 'bar', bar: 3 };
     await orm.em.persistAndFlush(bar);
@@ -2109,6 +2110,8 @@ describe('EntityManagerMongo', () => {
     const b1 = await orm.em.findOneOrFail(FooBar, bar.id);
     expect(b1.blob).toEqual(Buffer.from([1, 2, 3, 4, 5]));
     expect(b1.blob).toBeInstanceOf(Buffer);
+    expect(b1.blob2).toEqual(new Uint8Array([1, 2, 3, 4, 5]));
+    expect(b1.blob2).toBeInstanceOf(Uint8Array);
     expect(b1.array).toEqual([]);
     expect(b1.array).toBeInstanceOf(Array);
     expect(b1.object).toEqual({ foo: 'bar', bar: 3 });

tests/EntityManager.mysql.test.ts

@@ -2328,6 +2328,7 @@ describe('EntityManagerMySql', () => {
 
     const bar = FooBar2.create('b1');
     bar.blob = Buffer.from([1, 2, 3, 4, 5]);
+    bar.blob2 = new Uint8Array([1, 2, 3, 4, 5]);
     bar.array = [];
     bar.objectProperty = { foo: 'bar', bar: 3 };
     await orm.em.persistAndFlush(bar);
@@ -2336,6 +2337,8 @@ describe('EntityManagerMySql', () => {
     const b1 = await orm.em.findOneOrFail(FooBar2, bar.id);
     expect(b1.blob).toEqual(Buffer.from([1, 2, 3, 4, 5]));
     expect(b1.blob).toBeInstanceOf(Buffer);
+    expect(b1.blob2).toEqual(new Uint8Array([1, 2, 3, 4, 5]));
+    expect(b1.blob2).toBeInstanceOf(Uint8Array);
     expect(b1.array).toEqual([]);
     expect(b1.array).toBeInstanceOf(Array);
     expect(b1.objectProperty).toEqual({ foo: 'bar', bar: 3 });

tests/EntityManager.postgre.test.ts

@@ -1715,6 +1715,7 @@ describe('EntityManagerPostgre', () => {
 
     const bar = FooBar2.create('b1 "b" \'1\'');
     bar.blob = Buffer.from([1, 2, 3, 4, 5]);
+    bar.blob2 = new Uint8Array([1, 2, 3, 4, 5]);
     bar.array = [];
     bar.objectProperty = { foo: `bar 'lol' baz "foo"`, bar: 3 };
     await orm.em.persistAndFlush(bar);
@@ -1723,6 +1724,8 @@ describe('EntityManagerPostgre', () => {
     const b1 = await orm.em.findOneOrFail(FooBar2, bar.id);
     expect(b1.blob).toEqual(Buffer.from([1, 2, 3, 4, 5]));
     expect(b1.blob).toBeInstanceOf(Buffer);
+    expect(b1.blob2).toEqual(new Uint8Array([1, 2, 3, 4, 5]));
+    expect(b1.blob2).toBeInstanceOf(Uint8Array);
     expect(b1.array).toEqual([]);
     expect(b1.array).toBeInstanceOf(Array);
     expect(b1.objectProperty).toEqual({ foo: `bar 'lol' baz "foo"`, bar: 3 });

tests/EntityManager.sqlite2.test.ts

@@ -1033,6 +1033,7 @@ describe.each(['sqlite', 'better-sqlite'] as const)('EntityManager (%s)', driver
 
     const bar = orm.em.create(FooBar4, { name: 'b1 \'the bad\' lol' });
     bar.blob = Buffer.from([1, 2, 3, 4, 5]);
+    bar.blob2 = new Uint8Array([1, 2, 3, 4, 5]);
     bar.array = [];
     bar.object = { foo: 'bar "lol" \'wut\' escaped', bar: 3 };
     await orm.em.persistAndFlush(bar);
@@ -1041,6 +1042,8 @@ describe.each(['sqlite', 'better-sqlite'] as const)('EntityManager (%s)', driver
     const b1 = await orm.em.findOneOrFail(FooBar4, bar.id);
     expect(b1.blob).toEqual(Buffer.from([1, 2, 3, 4, 5]));
     expect(b1.blob).toBeInstanceOf(Buffer);
+    expect(b1.blob2).toEqual(new Uint8Array([1, 2, 3, 4, 5]));
+    expect(b1.blob2).toBeInstanceOf(Uint8Array);
     expect(b1.array).toEqual([]);
     expect(b1.array).toBeInstanceOf(Array);
     expect(b1.object).toEqual({ foo: 'bar "lol" \'wut\' escaped', bar: 3 });

tests/entities-schema/FooBar4.ts

@@ -1,5 +1,5 @@
 import type { OptionalProps } from '@mikro-orm/core';
-import { ArrayType, BlobType, EntitySchema, JsonType } from '@mikro-orm/core';
+import { ArrayType, BlobType, EntitySchema, JsonType, Uint8ArrayType } from '@mikro-orm/core';
 import type { IFooBaz4, IBaseEntity5 } from './index';
 
 export interface IFooBar4 extends Omit<IBaseEntity5, typeof OptionalProps> {
@@ -9,6 +9,7 @@ export interface IFooBar4 extends Omit<IBaseEntity5, typeof OptionalProps> {
   fooBar?: IFooBar4;
   version: number;
   blob?: Buffer;
+  blob2?: Uint8Array;
   array?: number[];
   object?: { foo: string; bar: number } | any;
   virtual?: string;
@@ -23,6 +24,7 @@ export const FooBar4 = new EntitySchema<IFooBar4, IBaseEntity5>({
     fooBar: { reference: '1:1', entity: 'FooBar4', nullable: true },
     version: { type: 'number', version: true },
     blob: { type: BlobType, nullable: true },
+    blob2: { type: Uint8ArrayType, nullable: true },
     array: { customType: new ArrayType(i => +i), nullable: true },
     object: { type: JsonType, nullable: true },
     virtual: { type: String, persist: false },

tests/entities-sql/FooBar2.ts

@@ -29,6 +29,9 @@ export class FooBar2 extends BaseEntity22 {
   @Property({ nullable: true })
   blob?: Buffer;
 
+  @Property({ nullable: true })
+  blob2?: Uint8Array;
+
   @Property({ type: 'number[]', nullable: true })
   array?: number[];
 

tests/entities/FooBar.ts

@@ -36,6 +36,9 @@ export default class FooBar {
   @Property({ nullable: true })
   blob?: Buffer;
 
+  @Property({ nullable: true })
+  blob2?: Uint8Array;
+
   @Property({ type: new ArrayType(i => +i), nullable: true })
   array?: number[];