feat: delimitedMethods option

README.markdown

@@ -354,6 +354,10 @@ Generated code will be placed in the Gradle build directory.
 
   The default behavior is `useExactTypes=true`, which makes `fromPartial` use Exact type for its argument to make TypeScript reject any unknown properties.
 
+- With `--ts_proto_opt=delimitedMethods=true`, ts-proto will generate `Message.encodeDelimited` which will prefix the output binary with its length, and `Message.decodeDelimited` which will decode binaries prefixed with a length. This is helpful in cases where you want to stream multiple messages, you can read more about that approach [here](https://developers.google.com/protocol-buffers/docs/techniques#streaming)
+
+  (Requires `outputEncodeMethods=true`, which is true by default.)
+
 ### Only Types
 
 If you're looking for `ts-proto` to generate only types for your Protobuf types then passing all three of `outputEncodeMethods`, `outputJsonMethods`, and `outputClientImpl` as `false` is probably what you want, i.e.:

integration/delimited-methods/delimited-methods-test.ts

@@ -0,0 +1,85 @@
+import { Reader, Writer } from 'protobufjs';
+import { AnotherSimple, Simple } from './delimited-methods';
+
+describe('delimited-methods', () => {
+  // normal size
+  const messageA = Simple.fromPartial({
+    age: 42,
+    name: 'John Doe',
+    children: ['Jane', 'Jack', 'Joe'],
+  });
+
+  // big size
+  const messageB = Simple.fromPartial({
+    age: 2147483647, // max int32
+    name: 'A Very Long Name',
+    children: ['Jane', 'Jack', 'Joe', 'Jill', 'Jane Jr.', 'Jack Jr.', 'Joe Jr.', 'Jill Jr.'],
+  });
+
+  // minimum size
+  const messageC = Simple.fromPartial({});
+
+  // different message type
+  const messageD = AnotherSimple.fromPartial({
+    num: 2147483.75,
+    str: 'A Very Long Name',
+  });
+
+  it('encodes with a length delimiter', () => {
+    const encoded = Simple.encodeDelimited(messageA).finish();
+
+    // -1 for the length delimiter
+    const length = encoded.length - 1;
+    const delimiter = encoded[0];
+
+    expect(length).toEqual(delimiter);
+  });
+
+  it('decodes a length-delimited message', () => {
+    const encoded = Simple.encodeDelimited(messageA).finish();
+    const decoded = Simple.decodeDelimited(encoded);
+
+    expect(decoded).toEqual(messageA);
+  });
+
+  it('decodes a stream of same length-delimited messages', () => {
+    const writer = new Writer();
+    const messages = [messageA, messageB, messageC, messageB];
+
+    messages.forEach((msg) => Simple.encodeDelimited(msg, writer));
+
+    const stream = writer.finish();
+
+    const reader = new Reader(stream);
+    const decodedMessages = messages.map(() => Simple.decodeDelimited(reader));
+
+    expect(decodedMessages).toEqual(messages);
+  });
+
+  /**
+   * Requires knowing the type order of messages beforehand
+   *
+   * Could also be done programmatically by checking for unique properties
+   */
+  it('decodes a stream of different length-delimited messages', () => {
+    const writer = new Writer();
+    const messages = [messageA, messageD, messageC, messageD] as const;
+
+    Simple.encodeDelimited(messages[0], writer);
+    AnotherSimple.encodeDelimited(messages[1], writer);
+    Simple.encodeDelimited(messages[2], writer);
+    AnotherSimple.encodeDelimited(messages[3], writer);
+
+    const stream = writer.finish();
+
+    const reader = new Reader(stream);
+    const decodedMessages = [
+      Simple.decodeDelimited(reader),
+      AnotherSimple.decodeDelimited(reader),
+      Simple.decodeDelimited(reader),
+      AnotherSimple.decodeDelimited(reader),
+    ];
+
+    expect(decodedMessages).toEqual(messages);
+  });
+});

integration/delimited-methods/delimited-methods.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+
+message Simple {
+  string name = 1;
+  int32 age = 2;
+  repeated string children = 3;
+}
+
+message AnotherSimple {
+  float num = 1;
+  string str = 2;
+}

integration/delimited-methods/delimited-methods.ts

@@ -0,0 +1,189 @@
+/* eslint-disable */
+import { util, configure, Writer, Reader } from 'protobufjs/minimal';
+import * as Long from 'long';
+
+export const protobufPackage = '';
+
+export interface Simple {
+  name: string;
+  age: number;
+  children: string[];
+}
+
+export interface AnotherSimple {
+  num: number;
+  str: string;
+}
+
+function createBaseSimple(): Simple {
+  return { name: '', age: 0, children: [] };
+}
+
+export const Simple = {
+  encode(message: Simple, writer: Writer = Writer.create()): Writer {
+    if (message.name !== '') {
+      writer.uint32(10).string(message.name);
+    }
+    if (message.age !== 0) {
+      writer.uint32(16).int32(message.age);
+    }
+    for (const v of message.children) {
+      writer.uint32(26).string(v!);
+    }
+    return writer;
+  },
+
+  decode(input: Reader | Uint8Array, length?: number): Simple {
+    const reader = input instanceof Reader ? input : new Reader(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseSimple();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          message.name = reader.string();
+          break;
+        case 2:
+          message.age = reader.int32();
+          break;
+        case 3:
+          message.children.push(reader.string());
+          break;
+        default:
+          reader.skipType(tag & 7);
+          break;
+      }
+    }
+    return message;
+  },
+
+  encodeDelimited(message: Simple, writer: Writer = Writer.create()): Writer {
+    return this.encode(message, writer.fork()).ldelim();
+  },
+
+  decodeDelimited(input: Reader | Uint8Array): Simple {
+    const reader = input instanceof Reader ? input : new Reader(input);
+    const length = reader.uint32();
+    return this.decode(reader, length);
+  },
+
+  fromJSON(object: any): Simple {
+    const message = createBaseSimple();
+    message.name = object.name !== undefined && object.name !== null ? String(object.name) : '';
+    message.age = object.age !== undefined && object.age !== null ? Number(object.age) : 0;
+    message.children = (object.children ?? []).map((e: any) => String(e));
+    return message;
+  },
+
+  toJSON(message: Simple): unknown {
+    const obj: any = {};
+    message.name !== undefined && (obj.name = message.name);
+    message.age !== undefined && (obj.age = Math.round(message.age));
+    if (message.children) {
+      obj.children = message.children.map((e) => e);
+    } else {
+      obj.children = [];
+    }
+    return obj;
+  },
+
+  fromPartial<I extends Exact<DeepPartial<Simple>, I>>(object: I): Simple {
+    const message = createBaseSimple();
+    message.name = object.name ?? '';
+    message.age = object.age ?? 0;
+    message.children = object.children?.map((e) => e) || [];
+    return message;
+  },
+};
+
+function createBaseAnotherSimple(): AnotherSimple {
+  return { num: 0, str: '' };
+}
+
+export const AnotherSimple = {
+  encode(message: AnotherSimple, writer: Writer = Writer.create()): Writer {
+    if (message.num !== 0) {
+      writer.uint32(13).float(message.num);
+    }
+    if (message.str !== '') {
+      writer.uint32(18).string(message.str);
+    }
+    return writer;
+  },
+
+  decode(input: Reader | Uint8Array, length?: number): AnotherSimple {
+    const reader = input instanceof Reader ? input : new Reader(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseAnotherSimple();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          message.num = reader.float();
+          break;
+        case 2:
+          message.str = reader.string();
+          break;
+        default:
+          reader.skipType(tag & 7);
+          break;
+      }
+    }
+    return message;
+  },
+
+  encodeDelimited(message: AnotherSimple, writer: Writer = Writer.create()): Writer {
+    return this.encode(message, writer.fork()).ldelim();
+  },
+
+  decodeDelimited(input: Reader | Uint8Array): AnotherSimple {
+    const reader = input instanceof Reader ? input : new Reader(input);
+    const length = reader.uint32();
+    return this.decode(reader, length);
+  },
+
+  fromJSON(object: any): AnotherSimple {
+    const message = createBaseAnotherSimple();
+    message.num = object.num !== undefined && object.num !== null ? Number(object.num) : 0;
+    message.str = object.str !== undefined && object.str !== null ? String(object.str) : '';
+    return message;
+  },
+
+  toJSON(message: AnotherSimple): unknown {
+    const obj: any = {};
+    message.num !== undefined && (obj.num = message.num);
+    message.str !== undefined && (obj.str = message.str);
+    return obj;
+  },
+
+  fromPartial<I extends Exact<DeepPartial<AnotherSimple>, I>>(object: I): AnotherSimple {
+    const message = createBaseAnotherSimple();
+    message.num = object.num ?? 0;
+    message.str = object.str ?? '';
+    return message;
+  },
+};
+
+type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
+
+export type DeepPartial<T> = T extends Builtin
+  ? T
+  : T extends Array<infer U>
+  ? Array<DeepPartial<U>>
+  : T extends ReadonlyArray<infer U>
+  ? ReadonlyArray<DeepPartial<U>>
+  : T extends {}
+  ? { [K in keyof T]?: DeepPartial<T[K]> }
+  : Partial<T>;
+
+type KeysOfUnion<T> = T extends T ? keyof T : never;
+export type Exact<P, I extends P> = P extends Builtin
+  ? P
+  : P & { [K in keyof P]: Exact<P[K], I[K]> } & Record<Exclude<keyof I, KeysOfUnion<P>>, never>;
+
+// If you get a compile-error about 'Constructor<Long> and ... have no overlap',
+// add '--ts_proto_opt=esModuleInterop=true' as a flag when calling 'protoc'.
+if (util.Long !== Long) {
+  util.Long = Long as any;
+  configure();
+}

integration/delimited-methods/parameters.txt

@@ -0,0 +1 @@
+outputEncodeMethods=true,delimitedMethods=true

src/generate-type-registry.ts

@@ -32,6 +32,10 @@ function generateMessageType(ctx: Context): Code {
   if (ctx.options.outputEncodeMethods) {
     chunks.push(code`encode(message: Message, writer?: ${Writer}): ${Writer};`);
     chunks.push(code`decode(input: ${Reader} | Uint8Array, length?: number): Message;`);
+    if (ctx.options.delimitedMethods) {
+      chunks.push(code`encodeDelimited(message: Message, writer?: ${Writer}): ${Writer};`);
+      chunks.push(code`decodeDelimited(input: ${Reader} | Uint8Array): Message;`);
+    }
   }
 
   if (ctx.options.outputJsonMethods) {

src/main.ts

@@ -159,6 +159,10 @@ export function generateFile(ctx: Context, fileDesc: FileDescriptorProto): [stri
         if (options.outputEncodeMethods) {
           staticMembers.push(generateEncode(ctx, fullName, message));
           staticMembers.push(generateDecode(ctx, fullName, message));
+          if (options.delimitedMethods) {
+            staticMembers.push(generateEncodeDelimited(ctx, fullName, message));
+            staticMembers.push(generateDecodeDelimited(ctx, fullName, message));
+          }
         }
         if (options.outputJsonMethods) {
           staticMembers.push(generateFromJson(ctx, fullName, message));
@@ -811,6 +815,24 @@ function generateDecode(ctx: Context, fullName: string, messageDesc: DescriptorP
   return joinCode(chunks, { on: '\n' });
 }
 
+function generateDecodeDelimited(ctx: Context, fullName: string, messageDesc: DescriptorProto): Code {
+  const { options, utils, typeMap } = ctx;
+  const chunks: Code[] = [];
+
+  // create the basic function declaration
+  chunks.push(code`
+    decodeDelimited(
+      input: ${Reader} | Uint8Array,
+    ): ${fullName} {
+      const reader = input instanceof ${Reader} ? input : new ${Reader}(input);
+      const length = reader.uint32();
+  `);
+  chunks.push(code`return this.decode(reader, length);`);
+
+  chunks.push(code`}`);
+  return joinCode(chunks, { on: '\n' });
+}
+
 const Writer = imp('Writer@protobufjs/minimal');
 const Reader = imp('Reader@protobufjs/minimal');
 
@@ -978,6 +1000,24 @@ function generateEncode(ctx: Context, fullName: string, messageDesc: DescriptorP
   return joinCode(chunks, { on: '\n' });
 }
 
+/** Creates a function to encode a message by loop overing the tags then delimits the encoding with a length prefix */
+function generateEncodeDelimited(ctx: Context, fullName: string, messageDesc: DescriptorProto): Code {
+  const { options, utils, typeMap } = ctx;
+  const chunks: Code[] = [];
+
+  // create the basic function declaration
+  chunks.push(code`
+    encodeDelimited(
+      message: ${fullName},
+      writer: ${Writer} = ${Writer}.create(),
+    ): ${Writer} {
+  `);
+
+  chunks.push(code`return this.encode(message, writer.fork()).ldelim();`);
+  chunks.push(code`}`);
+  return joinCode(chunks, { on: '\n' });
+}
+
 /**
  * Creates a function to decode a message from JSON.
  *

src/options.ts

@@ -59,6 +59,7 @@ export type Options = {
   onlyTypes: boolean;
   emitImportedFiles: boolean;
   useExactTypes: boolean;
+  delimitedMethods: boolean;
 };
 
 export function defaultOptions(): Options {
@@ -92,6 +93,7 @@ export function defaultOptions(): Options {
     onlyTypes: false,
     emitImportedFiles: true,
     useExactTypes: true,
+    delimitedMethods: false,
   };
 }
 

tests/options-test.ts

@@ -8,6 +8,7 @@ describe('options', () => {
         "addNestjsRestParameter": false,
         "constEnums": false,
         "context": false,
+        "delimitedMethods": false,
         "emitImportedFiles": true,
         "enumsAsLiterals": false,
         "env": "both",