diff --git a/README.markdown b/README.markdown index 2703dff59..0e909b1eb 100644 --- a/README.markdown +++ b/README.markdown @@ -354,6 +354,10 @@ Generated code will be placed in the Gradle build directory. The default behavior is `useExactTypes=true`, which makes `fromPartial` use Exact type for its argument to make TypeScript reject any unknown properties. +- With `--ts_proto_opt=delimitedMethods=true`, ts-proto will generate `Message.encodeDelimited` which will prefix the output binary with its length, and `Message.decodeDelimited` which will decode binaries prefixed with a length. This is helpful in cases where you want to stream multiple messages, you can read more about that approach [here](https://developers.google.com/protocol-buffers/docs/techniques#streaming) + + (Requires `outputEncodeMethods=true`, which is true by default.) + ### Only Types If you're looking for `ts-proto` to generate only types for your Protobuf types then passing all three of `outputEncodeMethods`, `outputJsonMethods`, and `outputClientImpl` as `false` is probably what you want, i.e.: diff --git a/integration/delimited-methods/delimited-methods-test.ts b/integration/delimited-methods/delimited-methods-test.ts new file mode 100644 index 000000000..921f3b2e8 --- /dev/null +++ b/integration/delimited-methods/delimited-methods-test.ts @@ -0,0 +1,85 @@ +import { Reader, Writer } from 'protobufjs'; +import { AnotherSimple, Simple } from './delimited-methods'; + +describe('delimited-methods', () => { + // normal size + const messageA = Simple.fromPartial({ + age: 42, + name: 'John Doe', + children: ['Jane', 'Jack', 'Joe'], + }); + + // big size + const messageB = Simple.fromPartial({ + age: 2147483647, // max int32 + name: 'A Very Long Name', + children: ['Jane', 'Jack', 'Joe', 'Jill', 'Jane Jr.', 'Jack Jr.', 'Joe Jr.', 'Jill Jr.'], + }); + + // minimum size + const messageC = Simple.fromPartial({}); + + // different message type + const messageD = AnotherSimple.fromPartial({ + num: 2147483.75, + str: 'A Very Long Name', + }); + + it('encodes with a length delimiter', () => { + const encoded = Simple.encodeDelimited(messageA).finish(); + + // -1 for the length delimiter + const length = encoded.length - 1; + const delimiter = encoded[0]; + + expect(length).toEqual(delimiter); + }); + + it('decodes a length-delimited message', () => { + const encoded = Simple.encodeDelimited(messageA).finish(); + const decoded = Simple.decodeDelimited(encoded); + + expect(decoded).toEqual(messageA); + }); + + it('decodes a stream of same length-delimited messages', () => { + const writer = new Writer(); + const messages = [messageA, messageB, messageC, messageB]; + + messages.forEach((msg) => Simple.encodeDelimited(msg, writer)); + + const stream = writer.finish(); + + const reader = new Reader(stream); + const decodedMessages = messages.map(() => Simple.decodeDelimited(reader)); + + expect(decodedMessages).toEqual(messages); + }); + + /** + * Requires knowing the type order of messages beforehand + * + * Could also be done programmatically by checking for unique properties + */ + it('decodes a stream of different length-delimited messages', () => { + const writer = new Writer(); + const messages = [messageA, messageD, messageC, messageD] as const; + + Simple.encodeDelimited(messages[0], writer); + AnotherSimple.encodeDelimited(messages[1], writer); + Simple.encodeDelimited(messages[2], writer); + AnotherSimple.encodeDelimited(messages[3], writer); + + const stream = writer.finish(); + + const reader = new Reader(stream); + const decodedMessages = [ + Simple.decodeDelimited(reader), + AnotherSimple.decodeDelimited(reader), + Simple.decodeDelimited(reader), + AnotherSimple.decodeDelimited(reader), + ]; + + expect(decodedMessages).toEqual(messages); + }); +}); diff --git a/integration/delimited-methods/delimited-methods.bin b/integration/delimited-methods/delimited-methods.bin new file mode 100644 index 000000000..5d97506d2 Binary files /dev/null and b/integration/delimited-methods/delimited-methods.bin differ diff --git a/integration/delimited-methods/delimited-methods.proto b/integration/delimited-methods/delimited-methods.proto new file mode 100644 index 000000000..e6b457821 --- /dev/null +++ b/integration/delimited-methods/delimited-methods.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +message Simple { + string name = 1; + int32 age = 2; + repeated string children = 3; +} + +message AnotherSimple { + float num = 1; + string str = 2; +} diff --git a/integration/delimited-methods/delimited-methods.ts b/integration/delimited-methods/delimited-methods.ts new file mode 100644 index 000000000..028befeed --- /dev/null +++ b/integration/delimited-methods/delimited-methods.ts @@ -0,0 +1,189 @@ +/* eslint-disable */ +import { util, configure, Writer, Reader } from 'protobufjs/minimal'; +import * as Long from 'long'; + +export const protobufPackage = ''; + +export interface Simple { + name: string; + age: number; + children: string[]; +} + +export interface AnotherSimple { + num: number; + str: string; +} + +function createBaseSimple(): Simple { + return { name: '', age: 0, children: [] }; +} + +export const Simple = { + encode(message: Simple, writer: Writer = Writer.create()): Writer { + if (message.name !== '') { + writer.uint32(10).string(message.name); + } + if (message.age !== 0) { + writer.uint32(16).int32(message.age); + } + for (const v of message.children) { + writer.uint32(26).string(v!); + } + return writer; + }, + + decode(input: Reader | Uint8Array, length?: number): Simple { + const reader = input instanceof Reader ? input : new Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseSimple(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.name = reader.string(); + break; + case 2: + message.age = reader.int32(); + break; + case 3: + message.children.push(reader.string()); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }, + + encodeDelimited(message: Simple, writer: Writer = Writer.create()): Writer { + return this.encode(message, writer.fork()).ldelim(); + }, + + decodeDelimited(input: Reader | Uint8Array): Simple { + const reader = input instanceof Reader ? input : new Reader(input); + const length = reader.uint32(); + return this.decode(reader, length); + }, + + fromJSON(object: any): Simple { + const message = createBaseSimple(); + message.name = object.name !== undefined && object.name !== null ? String(object.name) : ''; + message.age = object.age !== undefined && object.age !== null ? Number(object.age) : 0; + message.children = (object.children ?? []).map((e: any) => String(e)); + return message; + }, + + toJSON(message: Simple): unknown { + const obj: any = {}; + message.name !== undefined && (obj.name = message.name); + message.age !== undefined && (obj.age = Math.round(message.age)); + if (message.children) { + obj.children = message.children.map((e) => e); + } else { + obj.children = []; + } + return obj; + }, + + fromPartial, I>>(object: I): Simple { + const message = createBaseSimple(); + message.name = object.name ?? ''; + message.age = object.age ?? 0; + message.children = object.children?.map((e) => e) || []; + return message; + }, +}; + +function createBaseAnotherSimple(): AnotherSimple { + return { num: 0, str: '' }; +} + +export const AnotherSimple = { + encode(message: AnotherSimple, writer: Writer = Writer.create()): Writer { + if (message.num !== 0) { + writer.uint32(13).float(message.num); + } + if (message.str !== '') { + writer.uint32(18).string(message.str); + } + return writer; + }, + + decode(input: Reader | Uint8Array, length?: number): AnotherSimple { + const reader = input instanceof Reader ? input : new Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseAnotherSimple(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.num = reader.float(); + break; + case 2: + message.str = reader.string(); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }, + + encodeDelimited(message: AnotherSimple, writer: Writer = Writer.create()): Writer { + return this.encode(message, writer.fork()).ldelim(); + }, + + decodeDelimited(input: Reader | Uint8Array): AnotherSimple { + const reader = input instanceof Reader ? input : new Reader(input); + const length = reader.uint32(); + return this.decode(reader, length); + }, + + fromJSON(object: any): AnotherSimple { + const message = createBaseAnotherSimple(); + message.num = object.num !== undefined && object.num !== null ? Number(object.num) : 0; + message.str = object.str !== undefined && object.str !== null ? String(object.str) : ''; + return message; + }, + + toJSON(message: AnotherSimple): unknown { + const obj: any = {}; + message.num !== undefined && (obj.num = message.num); + message.str !== undefined && (obj.str = message.str); + return obj; + }, + + fromPartial, I>>(object: I): AnotherSimple { + const message = createBaseAnotherSimple(); + message.num = object.num ?? 0; + message.str = object.str ?? ''; + return message; + }, +}; + +type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined; + +export type DeepPartial = T extends Builtin + ? T + : T extends Array + ? Array> + : T extends ReadonlyArray + ? ReadonlyArray> + : T extends {} + ? { [K in keyof T]?: DeepPartial } + : Partial; + +type KeysOfUnion = T extends T ? keyof T : never; +export type Exact = P extends Builtin + ? P + : P & { [K in keyof P]: Exact } & Record>, never>; + +// If you get a compile-error about 'Constructor and ... have no overlap', +// add '--ts_proto_opt=esModuleInterop=true' as a flag when calling 'protoc'. +if (util.Long !== Long) { + util.Long = Long as any; + configure(); +} diff --git a/integration/delimited-methods/parameters.txt b/integration/delimited-methods/parameters.txt new file mode 100644 index 000000000..d74bf74a9 --- /dev/null +++ b/integration/delimited-methods/parameters.txt @@ -0,0 +1 @@ +outputEncodeMethods=true,delimitedMethods=true diff --git a/src/generate-type-registry.ts b/src/generate-type-registry.ts index 1ee8fd999..62debd12b 100644 --- a/src/generate-type-registry.ts +++ b/src/generate-type-registry.ts @@ -32,6 +32,10 @@ function generateMessageType(ctx: Context): Code { if (ctx.options.outputEncodeMethods) { chunks.push(code`encode(message: Message, writer?: ${Writer}): ${Writer};`); chunks.push(code`decode(input: ${Reader} | Uint8Array, length?: number): Message;`); + if (ctx.options.delimitedMethods) { + chunks.push(code`encodeDelimited(message: Message, writer?: ${Writer}): ${Writer};`); + chunks.push(code`decodeDelimited(input: ${Reader} | Uint8Array): Message;`); + } } if (ctx.options.outputJsonMethods) { diff --git a/src/main.ts b/src/main.ts index 6dfe84e89..0c09ad9e7 100644 --- a/src/main.ts +++ b/src/main.ts @@ -159,6 +159,10 @@ export function generateFile(ctx: Context, fileDesc: FileDescriptorProto): [stri if (options.outputEncodeMethods) { staticMembers.push(generateEncode(ctx, fullName, message)); staticMembers.push(generateDecode(ctx, fullName, message)); + if (options.delimitedMethods) { + staticMembers.push(generateEncodeDelimited(ctx, fullName, message)); + staticMembers.push(generateDecodeDelimited(ctx, fullName, message)); + } } if (options.outputJsonMethods) { staticMembers.push(generateFromJson(ctx, fullName, message)); @@ -811,6 +815,24 @@ function generateDecode(ctx: Context, fullName: string, messageDesc: DescriptorP return joinCode(chunks, { on: '\n' }); } +function generateDecodeDelimited(ctx: Context, fullName: string, messageDesc: DescriptorProto): Code { + const { options, utils, typeMap } = ctx; + const chunks: Code[] = []; + + // create the basic function declaration + chunks.push(code` + decodeDelimited( + input: ${Reader} | Uint8Array, + ): ${fullName} { + const reader = input instanceof ${Reader} ? input : new ${Reader}(input); + const length = reader.uint32(); + `); + chunks.push(code`return this.decode(reader, length);`); + + chunks.push(code`}`); + return joinCode(chunks, { on: '\n' }); +} + const Writer = imp('Writer@protobufjs/minimal'); const Reader = imp('Reader@protobufjs/minimal'); @@ -978,6 +1000,24 @@ function generateEncode(ctx: Context, fullName: string, messageDesc: DescriptorP return joinCode(chunks, { on: '\n' }); } +/** Creates a function to encode a message by loop overing the tags then delimits the encoding with a length prefix */ +function generateEncodeDelimited(ctx: Context, fullName: string, messageDesc: DescriptorProto): Code { + const { options, utils, typeMap } = ctx; + const chunks: Code[] = []; + + // create the basic function declaration + chunks.push(code` + encodeDelimited( + message: ${fullName}, + writer: ${Writer} = ${Writer}.create(), + ): ${Writer} { + `); + + chunks.push(code`return this.encode(message, writer.fork()).ldelim();`); + chunks.push(code`}`); + return joinCode(chunks, { on: '\n' }); +} + /** * Creates a function to decode a message from JSON. * diff --git a/src/options.ts b/src/options.ts index 380718f13..e8163c19a 100644 --- a/src/options.ts +++ b/src/options.ts @@ -59,6 +59,7 @@ export type Options = { onlyTypes: boolean; emitImportedFiles: boolean; useExactTypes: boolean; + delimitedMethods: boolean; }; export function defaultOptions(): Options { @@ -92,6 +93,7 @@ export function defaultOptions(): Options { onlyTypes: false, emitImportedFiles: true, useExactTypes: true, + delimitedMethods: false, }; } diff --git a/tests/options-test.ts b/tests/options-test.ts index 6f82a0818..0f7280756 100644 --- a/tests/options-test.ts +++ b/tests/options-test.ts @@ -8,6 +8,7 @@ describe('options', () => { "addNestjsRestParameter": false, "constEnums": false, "context": false, + "delimitedMethods": false, "emitImportedFiles": true, "enumsAsLiterals": false, "env": "both",