From 903b216238db025e24ec3cfb2d20063aec1a40ed Mon Sep 17 00:00:00 2001 From: Bouke Versteegh Date: Mon, 7 Mar 2022 11:05:39 +0100 Subject: [PATCH] feat: represent field masks as `string[]` (#525) * feat: field masks represented as string[] * chore: allow installation from branch --- README.markdown | 31 ++++----- integration/fieldmask/fieldmask-test.ts | 45 ++++++++++--- integration/fieldmask/fieldmask.ts | 13 ++-- .../fieldmask/google/protobuf/field_mask.ts | 25 ++++--- package.json | 1 + src/main.ts | 67 ++++++++++++++++--- src/types.ts | 2 + 7 files changed, 132 insertions(+), 52 deletions(-) diff --git a/README.markdown b/README.markdown index 046f5a1b0..194dfed17 100644 --- a/README.markdown +++ b/README.markdown @@ -622,24 +622,19 @@ Their interpretation is defined by the Protobuf specification, and libraries are `ts-proto` currently automatically converts these messages to their corresponding native types. -- Wrapper Types: - - * [google.protobuf.DoubleValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#DoubleValue) ⇆ `number | undefined` - * [google.protobuf.FloatValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#FloatValue) ⇆ `number | undefined` - * [google.protobuf.Int64Value](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#Int64Value) ⇆ `number | undefined` - * [google.protobuf.UInt64Value](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#UInt64Value) ⇆ `number | undefined` - * [google.protobuf.Int32Value](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#Int32Value) ⇆ `number | undefined` - * [google.protobuf.UInt32Value](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#UInt32Value) ⇆ `number | undefined` - * [google.protobuf.BoolValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#BoolValue) ⇆ `boolean | undefined` - * [google.protobuf.StringValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#StringValue) ⇆ `string | undefined` - * [google.protobuf.BytesValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.BytesValue) ⇆ `Uint8Array | undefined` - -- JSON Types (Struct Types): - - * [google.protobuf.Value](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#Value) ⇆ `any | undefined` (i.e. `number | string | boolean | null | array | object`) - * [google.protobuf.ListValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#ListValue) ⇆ `any[]` - * [google.protobuf.Struct](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#Struct) ⇆ `{ [key: string]: any } | undefined` - * [google.protobuf.FieldMask](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#fieldmask) ⇆ `string[]` (only in the JSON, `FieldMask` is still a message) + * [google.protobuf.BoolValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#boolvalue) ⇆ `boolean` + * [google.protobuf.BytesValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#bytesvalue) ⇆ `Uint8Array` + * [google.protobuf.DoubleValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#doublevalue) ⇆ `number` + * [google.protobuf.FieldMask](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#fieldmask) ⇆ `string[]` + * [google.protobuf.FloatValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#floatvalue) ⇆ `number` + * [google.protobuf.Int32Value](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#int32value) ⇆ `number` + * [google.protobuf.Int64Value](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#int64value) ⇆ `number` + * [google.protobuf.ListValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#listvalue) ⇆ `any[]` + * [google.protobuf.UInt32Value](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#uint32value) ⇆ `number` + * [google.protobuf.UInt64Value](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#uint64value) ⇆ `number` + * [google.protobuf.StringValue](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#stringvalue) ⇆ `string` + * [google.protobuf.Value](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#value) ⇆ `any` (i.e. `number | string | boolean | null | array | object`) + * [google.protobuf.Struct](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#struct) ⇆ `{ [key: string]: any }` ## Wrapper Types diff --git a/integration/fieldmask/fieldmask-test.ts b/integration/fieldmask/fieldmask-test.ts index 2be785af4..10f00c15c 100644 --- a/integration/fieldmask/fieldmask-test.ts +++ b/integration/fieldmask/fieldmask-test.ts @@ -5,23 +5,50 @@ let data = { }; describe('fieldmask', () => { - it('can decode JSON', () => { + it('can decode canonical JSON', () => { const f = FieldMaskMessage.fromJSON(data); expect(f).toMatchInlineSnapshot(` Object { - "fieldMask": Object { - "paths": Array [ - "a", - "b", - "c.d", - ], - }, + "fieldMask": Array [ + "a", + "b", + "c.d", + ], + } + `); + }); + + it('can decode non-canonical JSON', () => { + const f = FieldMaskMessage.fromJSON({ + fieldMask: { + paths: ['a', 'b', 'c.d'], + } + }); + expect(f).toMatchInlineSnapshot(` + Object { + "fieldMask": Array [ + "a", + "b", + "c.d", + ], } `); }); it('can encode JSON', () => { - const f = FieldMaskMessage.toJSON({ fieldMask: { paths: ['a', 'b', 'c.d'] } }); + const f = FieldMaskMessage.toJSON({ fieldMask: ['a', 'b', 'c.d'] }); expect(f).toEqual(data); }); + + it('skips empty paths', () => { + const f = FieldMaskMessage.fromJSON({fieldMask: 'a,,c.d'}); + expect(f).toMatchInlineSnapshot(` + Object { + "fieldMask": Array [ + "a", + "c.d", + ], + } + `); + }); }); diff --git a/integration/fieldmask/fieldmask.ts b/integration/fieldmask/fieldmask.ts index cea74768c..75af744fa 100644 --- a/integration/fieldmask/fieldmask.ts +++ b/integration/fieldmask/fieldmask.ts @@ -6,7 +6,7 @@ import { FieldMask } from './google/protobuf/field_mask'; export const protobufPackage = ''; export interface FieldMaskMessage { - fieldMask: FieldMask | undefined; + fieldMask: string[] | undefined; } function createBaseFieldMaskMessage(): FieldMaskMessage { @@ -16,7 +16,7 @@ function createBaseFieldMaskMessage(): FieldMaskMessage { export const FieldMaskMessage = { encode(message: FieldMaskMessage, writer: Writer = Writer.create()): Writer { if (message.fieldMask !== undefined) { - FieldMask.encode(message.fieldMask, writer.uint32(10).fork()).ldelim(); + FieldMask.encode(FieldMask.wrap(message.fieldMask), writer.uint32(10).fork()).ldelim(); } return writer; }, @@ -29,7 +29,7 @@ export const FieldMaskMessage = { const tag = reader.uint32(); switch (tag >>> 3) { case 1: - message.fieldMask = FieldMask.decode(reader, reader.uint32()); + message.fieldMask = FieldMask.unwrap(FieldMask.decode(reader, reader.uint32())); break; default: reader.skipType(tag & 7); @@ -41,20 +41,19 @@ export const FieldMaskMessage = { fromJSON(object: any): FieldMaskMessage { return { - fieldMask: isSet(object.fieldMask) ? { paths: object.fieldMask.split(',') } : undefined, + fieldMask: isSet(object.fieldMask) ? FieldMask.unwrap(FieldMask.fromJSON(object.fieldMask)) : undefined, }; }, toJSON(message: FieldMaskMessage): unknown { const obj: any = {}; - message.fieldMask !== undefined && (obj.fieldMask = message.fieldMask.paths.join()); + message.fieldMask !== undefined && (obj.fieldMask = FieldMask.toJSON(FieldMask.wrap(message.fieldMask))); return obj; }, fromPartial, I>>(object: I): FieldMaskMessage { const message = createBaseFieldMaskMessage(); - message.fieldMask = - object.fieldMask !== undefined && object.fieldMask !== null ? FieldMask.fromPartial(object.fieldMask) : undefined; + message.fieldMask = object.fieldMask ?? undefined; return message; }, }; diff --git a/integration/fieldmask/google/protobuf/field_mask.ts b/integration/fieldmask/google/protobuf/field_mask.ts index e656bf95f..0b040f2b9 100644 --- a/integration/fieldmask/google/protobuf/field_mask.ts +++ b/integration/fieldmask/google/protobuf/field_mask.ts @@ -242,18 +242,17 @@ export const FieldMask = { fromJSON(object: any): FieldMask { return { - paths: Array.isArray(object?.paths) ? object.paths.map((e: any) => String(e)) : [], + paths: + typeof object === 'string' + ? object.split(',').filter(Boolean) + : Array.isArray(object?.paths) + ? object.paths.map(String) + : [], }; }, - toJSON(message: FieldMask): unknown { - const obj: any = {}; - if (message.paths) { - obj.paths = message.paths.map((e) => e); - } else { - obj.paths = []; - } - return obj; + toJSON(message: FieldMask): string { + return message.paths.join(','); }, fromPartial, I>>(object: I): FieldMask { @@ -261,6 +260,14 @@ export const FieldMask = { message.paths = object.paths?.map((e) => e) || []; return message; }, + + wrap(paths: string[]): FieldMask { + return { paths: paths }; + }, + + unwrap(message: FieldMask): string[] { + return message.paths; + }, }; type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined; diff --git a/package.json b/package.json index dc37e2202..931a8219c 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "build": "yarn tsc", "build:test": "yarn proto2bin && yarn proto2pbjs && yarn bin2ts", "build:test:local": "yarn proto2bin:local && yarn proto2pbjs:local && yarn bin2ts:local", + "prepare": "yarn build", "proto2bin": "docker-compose run --rm protoc update-bins.sh", "proto2bin-node": "docker-compose run --rm node update-bins.sh", "proto2pbjs": "docker-compose run --rm protoc pbjs.sh", diff --git a/src/main.ts b/src/main.ts index 9dae98a87..d7319af64 100644 --- a/src/main.ts +++ b/src/main.ts @@ -13,6 +13,7 @@ import { isBytesValueType, isEnum, isFieldMaskType, + isFieldMaskTypeName, isListValueType, isListValueTypeName, isLong, @@ -164,8 +165,8 @@ export function generateFile(ctx: Context, fileDesc: FileDescriptorProto): [stri staticMembers.push(generateDecode(ctx, fullName, message)); } if (options.outputJsonMethods) { - staticMembers.push(generateFromJson(ctx, fullName, message)); - staticMembers.push(generateToJson(ctx, fullName, message)); + staticMembers.push(generateFromJson(ctx, fullName, fullTypeName, message)); + staticMembers.push(generateToJson(ctx, fullName, fullTypeName, message)); } if (options.outputPartialMethods) { staticMembers.push(generateFromPartial(ctx, fullName, message)); @@ -828,7 +829,7 @@ function generateDecode(ctx: Context, fullName: string, messageDesc: DescriptorP } else if (isValueType(ctx, field)) { const type = basicTypeName(ctx, field, { keepValueType: true }); const unwrap = (decodedValue: any): Code => { - if (isListValueType(field) || isStructType(field) || isAnyValueType(field)) { + if (isListValueType(field) || isStructType(field) || isAnyValueType(field) || isFieldMaskType(field)) { return code`${type}.unwrap(${decodedValue})`; } return code`${decodedValue}.value`; @@ -953,7 +954,7 @@ function generateEncode(ctx: Context, fullName: string, messageDesc: DescriptorP const type = basicTypeName(ctx, field, { keepValueType: true }); const wrappedValue = (place: string): Code => { - if (isAnyValueType(field) || isListValueType(field) || isStructType(field)) { + if (isAnyValueType(field) || isListValueType(field) || isStructType(field) || isFieldMaskType(field)) { return code`${type}.wrap(${place})`; } return code`{${maybeTypeField} value: ${place}!}`; @@ -1103,7 +1104,7 @@ function generateEncode(ctx: Context, fullName: string, messageDesc: DescriptorP * This is very similar to decode, we loop through looking for properties, with * a few special cases for https://developers.google.com/protocol-buffers/docs/proto3#json. * */ -function generateFromJson(ctx: Context, fullName: string, messageDesc: DescriptorProto): Code { +function generateFromJson(ctx: Context, fullName: string, fullTypeName: string, messageDesc: DescriptorProto): Code { const { options, utils, typeMap } = ctx; const chunks: Code[] = []; @@ -1121,6 +1122,16 @@ function generateFromJson(ctx: Context, fullName: string, messageDesc: Descripto messageDesc.field.filter(isWithinOneOf).filter((field) => field.oneofIndex === oneofIndex) ); + const canonicalFromJson: { [key: string]: { [field: string]: (from: string) => Code } } = { + ['google.protobuf.FieldMask']: { + paths: (from: string) => code`typeof(${from}) === 'string' + ? ${from}.split(",").filter(Boolean) + : Array.isArray(${from}?.paths) + ? ${from}.paths.map(String) + : []`, + }, + }; + // add a check for each incoming field messageDesc.field.forEach((field) => { const fieldName = maybeSnakeToCamel(field.name, options); @@ -1160,7 +1171,8 @@ function generateFromJson(ctx: Context, fullName: string, messageDesc: Descripto } else if (isAnyValueType(field) || isStructType(field)) { return code`${from}`; } else if (isFieldMaskType(field)) { - return code`{paths: ${from}.split(",")}`; + const type = basicTypeName(ctx, field, { keepValueType: true }); + return code`${type}.unwrap(${type}.fromJSON(${from}))`; } else if (isListValueType(field)) { return code`[...${from}]`; } else if (isValueType(ctx, field)) { @@ -1219,7 +1231,9 @@ function generateFromJson(ctx: Context, fullName: string, messageDesc: Descripto }; // and then use the snippet to handle repeated fields if necessary - if (isRepeated(field)) { + if (canonicalFromJson[fullTypeName]?.[fieldName]) { + chunks.push(code`${fieldName}: ${canonicalFromJson[fullTypeName][fieldName]('object')},`); + } else if (isRepeated(field)) { if (isMapType(ctx, messageDesc, field)) { const fieldType = toTypeName(ctx, messageDesc, field); const i = maybeCastToNumber(ctx, messageDesc, field, 'key'); @@ -1291,10 +1305,32 @@ function generateFromJson(ctx: Context, fullName: string, messageDesc: Descripto return joinCode(chunks, { on: '\n' }); } -function generateToJson(ctx: Context, fullName: string, messageDesc: DescriptorProto): Code { +function generateCanonicalToJson(fullName: string, fullProtobufTypeName: string): Code | undefined { + if (isFieldMaskTypeName(fullProtobufTypeName)) { + return code` + toJSON(message: ${fullName}): string { + return message.paths.join(','); + } + `; + } + return undefined; +} + +function generateToJson( + ctx: Context, + fullName: string, + fullProtobufTypeName: string, + messageDesc: DescriptorProto +): Code { const { options, utils, typeMap } = ctx; const chunks: Code[] = []; + const canonicalToJson = generateCanonicalToJson(fullName, fullProtobufTypeName); + if (canonicalToJson) { + chunks.push(canonicalToJson); + return joinCode(chunks, { on: '\n' }); + } + // create the basic function declaration chunks.push(code` toJSON(${messageDesc.field.length > 0 ? 'message' : '_'}: ${fullName}): unknown { @@ -1352,7 +1388,8 @@ function generateToJson(ctx: Context, fullName: string, messageDesc: DescriptorP } else if (isAnyValueType(field)) { return code`${from}`; } else if (isFieldMaskType(field)) { - return code`${from}.paths.join()`; + const type = basicTypeName(ctx, field, { keepValueType: true }); + return code`${type}.toJSON(${type}.wrap(${from}))`; } else if (isMessage(field) && !isValueType(ctx, field) && !isMapType(ctx, messageDesc, field)) { const type = basicTypeName(ctx, field, { keepValueType: true }); return code`${from} ? ${type}.toJSON(${from}) : ${defaultValue(ctx, field)}`; @@ -1607,6 +1644,12 @@ function generateWrap(ctx: Context, fullProtoTypeName: string): Code[] { }`); } + if (isFieldMaskTypeName(fullProtoTypeName)) { + chunks.push(code`wrap(paths: string[]): FieldMask { + return {paths: paths}; + }`); + } + return chunks; } @@ -1667,6 +1710,12 @@ function generateUnwrap(ctx: Context, fullProtoTypeName: string): Code[] { }`); } + if (isFieldMaskTypeName(fullProtoTypeName)) { + chunks.push(code`unwrap(message: FieldMask): string[] { + return message.paths; + }`); + } + return chunks; } diff --git a/src/types.ts b/src/types.ts index 29419d7ee..6f96527ea 100644 --- a/src/types.ts +++ b/src/types.ts @@ -484,6 +484,8 @@ export function valueTypeName(ctx: Context, typeName: string): Code | undefined return code`any`; case '.google.protobuf.Struct': return code`{[key: string]: any}`; + case '.google.protobuf.FieldMask': + return code`string[]`; default: return undefined; }