diff --git a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/avro.proto b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/avro.proto index 15de2db5468..e1ecb667b61 100644 --- a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/avro.proto +++ b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/avro.proto @@ -39,3 +39,18 @@ message AvroRows { // Please use the format-independent ReadRowsResponse.row_count instead. int64 row_count = 2 [deprecated = true]; } + +// Contains options specific to Avro Serialization. +message AvroSerializationOptions { + // Enable displayName attribute in Avro schema. + // + // The Avro specification requires field names to be alphanumeric. By + // default, in cases when column names do not conform to these requirements + // (e.g. non-ascii unicode codepoints) and Avro is requested as an output + // format, the CreateReadSession call will fail. + // + // Setting this field to true, populates avro field names with a placeholder + // value and populates a "displayName" attribute for every avro field with the + // original column name. + bool enable_display_name_attribute = 1; +} diff --git a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto index fd1e25b65fd..3735c73d670 100644 --- a/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto +++ b/packages/google-cloud-bigquery-storage/protos/google/cloud/bigquery/storage/v1/stream.proto @@ -59,10 +59,53 @@ message ReadSession { // Options dictating how we read a table. message TableReadOptions { - // Names of the fields in the table that should be read. If empty, all - // fields will be read. If the specified field is a nested field, all - // the sub-fields in the field will be selected. The output field order is - // unrelated to the order of fields in selected_fields. + // Optional. The names of the fields in the table to be returned. If no + // field names are specified, then all fields in the table are returned. + // + // Nested fields -- the child elements of a STRUCT field -- can be selected + // individually using their fully-qualified names, and will be returned as + // record fields containing only the selected nested fields. If a STRUCT + // field is specified in the selected fields list, all of the child elements + // will be returned. + // + // As an example, consider a table with the following schema: + // + // { + // "name": "struct_field", + // "type": "RECORD", + // "mode": "NULLABLE", + // "fields": [ + // { + // "name": "string_field1", + // "type": "STRING", + // . "mode": "NULLABLE" + // }, + // { + // "name": "string_field2", + // "type": "STRING", + // "mode": "NULLABLE" + // } + // ] + // } + // + // Specifying "struct_field" in the selected fields list will result in a + // read session schema with the following logical structure: + // + // struct_field { + // string_field1 + // string_field2 + // } + // + // Specifying "struct_field.string_field1" in the selected fields list will + // result in a read session schema with the following logical structure: + // + // struct_field { + // string_field1 + // } + // + // The order of the fields in the read session schema is derived from the + // table schema and does not correspond to the order in which the fields are + // specified in this list. repeated string selected_fields = 1; // SQL text filtering statement, similar to a WHERE clause in a query. @@ -80,6 +123,9 @@ message ReadSession { oneof output_format_serialization_options { // Optional. Options specific to the Apache Arrow output format. ArrowSerializationOptions arrow_serialization_options = 3 [(google.api.field_behavior) = OPTIONAL]; + + // Optional. Options specific to the Apache Avro output format + AvroSerializationOptions avro_serialization_options = 4 [(google.api.field_behavior) = OPTIONAL]; } } diff --git a/packages/google-cloud-bigquery-storage/protos/protos.d.ts b/packages/google-cloud-bigquery-storage/protos/protos.d.ts index 197d0abe8c4..bb841e4c0d3 100644 --- a/packages/google-cloud-bigquery-storage/protos/protos.d.ts +++ b/packages/google-cloud-bigquery-storage/protos/protos.d.ts @@ -501,6 +501,96 @@ export namespace google { public toJSON(): { [k: string]: any }; } + /** Properties of an AvroSerializationOptions. */ + interface IAvroSerializationOptions { + + /** AvroSerializationOptions enableDisplayNameAttribute */ + enableDisplayNameAttribute?: (boolean|null); + } + + /** Represents an AvroSerializationOptions. */ + class AvroSerializationOptions implements IAvroSerializationOptions { + + /** + * Constructs a new AvroSerializationOptions. + * @param [properties] Properties to set + */ + constructor(properties?: google.cloud.bigquery.storage.v1.IAvroSerializationOptions); + + /** AvroSerializationOptions enableDisplayNameAttribute. */ + public enableDisplayNameAttribute: boolean; + + /** + * Creates a new AvroSerializationOptions instance using the specified properties. + * @param [properties] Properties to set + * @returns AvroSerializationOptions instance + */ + public static create(properties?: google.cloud.bigquery.storage.v1.IAvroSerializationOptions): google.cloud.bigquery.storage.v1.AvroSerializationOptions; + + /** + * Encodes the specified AvroSerializationOptions message. Does not implicitly {@link google.cloud.bigquery.storage.v1.AvroSerializationOptions.verify|verify} messages. + * @param message AvroSerializationOptions message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encode(message: google.cloud.bigquery.storage.v1.IAvroSerializationOptions, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Encodes the specified AvroSerializationOptions message, length delimited. Does not implicitly {@link google.cloud.bigquery.storage.v1.AvroSerializationOptions.verify|verify} messages. + * @param message AvroSerializationOptions message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encodeDelimited(message: google.cloud.bigquery.storage.v1.IAvroSerializationOptions, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Decodes an AvroSerializationOptions message from the specified reader or buffer. + * @param reader Reader or buffer to decode from + * @param [length] Message length if known beforehand + * @returns AvroSerializationOptions + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): google.cloud.bigquery.storage.v1.AvroSerializationOptions; + + /** + * Decodes an AvroSerializationOptions message from the specified reader or buffer, length delimited. + * @param reader Reader or buffer to decode from + * @returns AvroSerializationOptions + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decodeDelimited(reader: ($protobuf.Reader|Uint8Array)): google.cloud.bigquery.storage.v1.AvroSerializationOptions; + + /** + * Verifies an AvroSerializationOptions message. + * @param message Plain object to verify + * @returns `null` if valid, otherwise the reason why it is not + */ + public static verify(message: { [k: string]: any }): (string|null); + + /** + * Creates an AvroSerializationOptions message from a plain object. Also converts values to their respective internal types. + * @param object Plain object + * @returns AvroSerializationOptions + */ + public static fromObject(object: { [k: string]: any }): google.cloud.bigquery.storage.v1.AvroSerializationOptions; + + /** + * Creates a plain object from an AvroSerializationOptions message. Also converts values to other types if specified. + * @param message AvroSerializationOptions + * @param [options] Conversion options + * @returns Plain object + */ + public static toObject(message: google.cloud.bigquery.storage.v1.AvroSerializationOptions, options?: $protobuf.IConversionOptions): { [k: string]: any }; + + /** + * Converts this AvroSerializationOptions to JSON. + * @returns JSON object + */ + public toJSON(): { [k: string]: any }; + } + /** Properties of a ProtoSchema. */ interface IProtoSchema { @@ -3382,6 +3472,9 @@ export namespace google { /** TableReadOptions arrowSerializationOptions */ arrowSerializationOptions?: (google.cloud.bigquery.storage.v1.IArrowSerializationOptions|null); + + /** TableReadOptions avroSerializationOptions */ + avroSerializationOptions?: (google.cloud.bigquery.storage.v1.IAvroSerializationOptions|null); } /** Represents a TableReadOptions. */ @@ -3402,8 +3495,11 @@ export namespace google { /** TableReadOptions arrowSerializationOptions. */ public arrowSerializationOptions?: (google.cloud.bigquery.storage.v1.IArrowSerializationOptions|null); + /** TableReadOptions avroSerializationOptions. */ + public avroSerializationOptions?: (google.cloud.bigquery.storage.v1.IAvroSerializationOptions|null); + /** TableReadOptions outputFormatSerializationOptions. */ - public outputFormatSerializationOptions?: "arrowSerializationOptions"; + public outputFormatSerializationOptions?: ("arrowSerializationOptions"|"avroSerializationOptions"); /** * Creates a new TableReadOptions instance using the specified properties. diff --git a/packages/google-cloud-bigquery-storage/protos/protos.js b/packages/google-cloud-bigquery-storage/protos/protos.js index 32551b9ff33..5996556813f 100644 --- a/packages/google-cloud-bigquery-storage/protos/protos.js +++ b/packages/google-cloud-bigquery-storage/protos/protos.js @@ -1145,6 +1145,193 @@ return AvroRows; })(); + v1.AvroSerializationOptions = (function() { + + /** + * Properties of an AvroSerializationOptions. + * @memberof google.cloud.bigquery.storage.v1 + * @interface IAvroSerializationOptions + * @property {boolean|null} [enableDisplayNameAttribute] AvroSerializationOptions enableDisplayNameAttribute + */ + + /** + * Constructs a new AvroSerializationOptions. + * @memberof google.cloud.bigquery.storage.v1 + * @classdesc Represents an AvroSerializationOptions. + * @implements IAvroSerializationOptions + * @constructor + * @param {google.cloud.bigquery.storage.v1.IAvroSerializationOptions=} [properties] Properties to set + */ + function AvroSerializationOptions(properties) { + if (properties) + for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) + if (properties[keys[i]] != null) + this[keys[i]] = properties[keys[i]]; + } + + /** + * AvroSerializationOptions enableDisplayNameAttribute. + * @member {boolean} enableDisplayNameAttribute + * @memberof google.cloud.bigquery.storage.v1.AvroSerializationOptions + * @instance + */ + AvroSerializationOptions.prototype.enableDisplayNameAttribute = false; + + /** + * Creates a new AvroSerializationOptions instance using the specified properties. + * @function create + * @memberof google.cloud.bigquery.storage.v1.AvroSerializationOptions + * @static + * @param {google.cloud.bigquery.storage.v1.IAvroSerializationOptions=} [properties] Properties to set + * @returns {google.cloud.bigquery.storage.v1.AvroSerializationOptions} AvroSerializationOptions instance + */ + AvroSerializationOptions.create = function create(properties) { + return new AvroSerializationOptions(properties); + }; + + /** + * Encodes the specified AvroSerializationOptions message. Does not implicitly {@link google.cloud.bigquery.storage.v1.AvroSerializationOptions.verify|verify} messages. + * @function encode + * @memberof google.cloud.bigquery.storage.v1.AvroSerializationOptions + * @static + * @param {google.cloud.bigquery.storage.v1.IAvroSerializationOptions} message AvroSerializationOptions message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + AvroSerializationOptions.encode = function encode(message, writer) { + if (!writer) + writer = $Writer.create(); + if (message.enableDisplayNameAttribute != null && Object.hasOwnProperty.call(message, "enableDisplayNameAttribute")) + writer.uint32(/* id 1, wireType 0 =*/8).bool(message.enableDisplayNameAttribute); + return writer; + }; + + /** + * Encodes the specified AvroSerializationOptions message, length delimited. Does not implicitly {@link google.cloud.bigquery.storage.v1.AvroSerializationOptions.verify|verify} messages. + * @function encodeDelimited + * @memberof google.cloud.bigquery.storage.v1.AvroSerializationOptions + * @static + * @param {google.cloud.bigquery.storage.v1.IAvroSerializationOptions} message AvroSerializationOptions message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + AvroSerializationOptions.encodeDelimited = function encodeDelimited(message, writer) { + return this.encode(message, writer).ldelim(); + }; + + /** + * Decodes an AvroSerializationOptions message from the specified reader or buffer. + * @function decode + * @memberof google.cloud.bigquery.storage.v1.AvroSerializationOptions + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {google.cloud.bigquery.storage.v1.AvroSerializationOptions} AvroSerializationOptions + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + AvroSerializationOptions.decode = function decode(reader, length) { + if (!(reader instanceof $Reader)) + reader = $Reader.create(reader); + var end = length === undefined ? reader.len : reader.pos + length, message = new $root.google.cloud.bigquery.storage.v1.AvroSerializationOptions(); + while (reader.pos < end) { + var tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.enableDisplayNameAttribute = reader.bool(); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Decodes an AvroSerializationOptions message from the specified reader or buffer, length delimited. + * @function decodeDelimited + * @memberof google.cloud.bigquery.storage.v1.AvroSerializationOptions + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @returns {google.cloud.bigquery.storage.v1.AvroSerializationOptions} AvroSerializationOptions + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + AvroSerializationOptions.decodeDelimited = function decodeDelimited(reader) { + if (!(reader instanceof $Reader)) + reader = new $Reader(reader); + return this.decode(reader, reader.uint32()); + }; + + /** + * Verifies an AvroSerializationOptions message. + * @function verify + * @memberof google.cloud.bigquery.storage.v1.AvroSerializationOptions + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + AvroSerializationOptions.verify = function verify(message) { + if (typeof message !== "object" || message === null) + return "object expected"; + if (message.enableDisplayNameAttribute != null && message.hasOwnProperty("enableDisplayNameAttribute")) + if (typeof message.enableDisplayNameAttribute !== "boolean") + return "enableDisplayNameAttribute: boolean expected"; + return null; + }; + + /** + * Creates an AvroSerializationOptions message from a plain object. Also converts values to their respective internal types. + * @function fromObject + * @memberof google.cloud.bigquery.storage.v1.AvroSerializationOptions + * @static + * @param {Object.} object Plain object + * @returns {google.cloud.bigquery.storage.v1.AvroSerializationOptions} AvroSerializationOptions + */ + AvroSerializationOptions.fromObject = function fromObject(object) { + if (object instanceof $root.google.cloud.bigquery.storage.v1.AvroSerializationOptions) + return object; + var message = new $root.google.cloud.bigquery.storage.v1.AvroSerializationOptions(); + if (object.enableDisplayNameAttribute != null) + message.enableDisplayNameAttribute = Boolean(object.enableDisplayNameAttribute); + return message; + }; + + /** + * Creates a plain object from an AvroSerializationOptions message. Also converts values to other types if specified. + * @function toObject + * @memberof google.cloud.bigquery.storage.v1.AvroSerializationOptions + * @static + * @param {google.cloud.bigquery.storage.v1.AvroSerializationOptions} message AvroSerializationOptions + * @param {$protobuf.IConversionOptions} [options] Conversion options + * @returns {Object.} Plain object + */ + AvroSerializationOptions.toObject = function toObject(message, options) { + if (!options) + options = {}; + var object = {}; + if (options.defaults) + object.enableDisplayNameAttribute = false; + if (message.enableDisplayNameAttribute != null && message.hasOwnProperty("enableDisplayNameAttribute")) + object.enableDisplayNameAttribute = message.enableDisplayNameAttribute; + return object; + }; + + /** + * Converts this AvroSerializationOptions to JSON. + * @function toJSON + * @memberof google.cloud.bigquery.storage.v1.AvroSerializationOptions + * @instance + * @returns {Object.} JSON object + */ + AvroSerializationOptions.prototype.toJSON = function toJSON() { + return this.constructor.toObject(this, $protobuf.util.toJSONOptions); + }; + + return AvroSerializationOptions; + })(); + v1.ProtoSchema = (function() { /** @@ -7846,6 +8033,7 @@ * @property {Array.|null} [selectedFields] TableReadOptions selectedFields * @property {string|null} [rowRestriction] TableReadOptions rowRestriction * @property {google.cloud.bigquery.storage.v1.IArrowSerializationOptions|null} [arrowSerializationOptions] TableReadOptions arrowSerializationOptions + * @property {google.cloud.bigquery.storage.v1.IAvroSerializationOptions|null} [avroSerializationOptions] TableReadOptions avroSerializationOptions */ /** @@ -7888,17 +8076,25 @@ */ TableReadOptions.prototype.arrowSerializationOptions = null; + /** + * TableReadOptions avroSerializationOptions. + * @member {google.cloud.bigquery.storage.v1.IAvroSerializationOptions|null|undefined} avroSerializationOptions + * @memberof google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions + * @instance + */ + TableReadOptions.prototype.avroSerializationOptions = null; + // OneOf field names bound to virtual getters and setters var $oneOfFields; /** * TableReadOptions outputFormatSerializationOptions. - * @member {"arrowSerializationOptions"|undefined} outputFormatSerializationOptions + * @member {"arrowSerializationOptions"|"avroSerializationOptions"|undefined} outputFormatSerializationOptions * @memberof google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions * @instance */ Object.defineProperty(TableReadOptions.prototype, "outputFormatSerializationOptions", { - get: $util.oneOfGetter($oneOfFields = ["arrowSerializationOptions"]), + get: $util.oneOfGetter($oneOfFields = ["arrowSerializationOptions", "avroSerializationOptions"]), set: $util.oneOfSetter($oneOfFields) }); @@ -7933,6 +8129,8 @@ writer.uint32(/* id 2, wireType 2 =*/18).string(message.rowRestriction); if (message.arrowSerializationOptions != null && Object.hasOwnProperty.call(message, "arrowSerializationOptions")) $root.google.cloud.bigquery.storage.v1.ArrowSerializationOptions.encode(message.arrowSerializationOptions, writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim(); + if (message.avroSerializationOptions != null && Object.hasOwnProperty.call(message, "avroSerializationOptions")) + $root.google.cloud.bigquery.storage.v1.AvroSerializationOptions.encode(message.avroSerializationOptions, writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim(); return writer; }; @@ -7978,6 +8176,9 @@ case 3: message.arrowSerializationOptions = $root.google.cloud.bigquery.storage.v1.ArrowSerializationOptions.decode(reader, reader.uint32()); break; + case 4: + message.avroSerializationOptions = $root.google.cloud.bigquery.storage.v1.AvroSerializationOptions.decode(reader, reader.uint32()); + break; default: reader.skipType(tag & 7); break; @@ -8032,6 +8233,16 @@ return "arrowSerializationOptions." + error; } } + if (message.avroSerializationOptions != null && message.hasOwnProperty("avroSerializationOptions")) { + if (properties.outputFormatSerializationOptions === 1) + return "outputFormatSerializationOptions: multiple values"; + properties.outputFormatSerializationOptions = 1; + { + var error = $root.google.cloud.bigquery.storage.v1.AvroSerializationOptions.verify(message.avroSerializationOptions); + if (error) + return "avroSerializationOptions." + error; + } + } return null; }; @@ -8061,6 +8272,11 @@ throw TypeError(".google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions.arrowSerializationOptions: object expected"); message.arrowSerializationOptions = $root.google.cloud.bigquery.storage.v1.ArrowSerializationOptions.fromObject(object.arrowSerializationOptions); } + if (object.avroSerializationOptions != null) { + if (typeof object.avroSerializationOptions !== "object") + throw TypeError(".google.cloud.bigquery.storage.v1.ReadSession.TableReadOptions.avroSerializationOptions: object expected"); + message.avroSerializationOptions = $root.google.cloud.bigquery.storage.v1.AvroSerializationOptions.fromObject(object.avroSerializationOptions); + } return message; }; @@ -8093,6 +8309,11 @@ if (options.oneofs) object.outputFormatSerializationOptions = "arrowSerializationOptions"; } + if (message.avroSerializationOptions != null && message.hasOwnProperty("avroSerializationOptions")) { + object.avroSerializationOptions = $root.google.cloud.bigquery.storage.v1.AvroSerializationOptions.toObject(message.avroSerializationOptions, options); + if (options.oneofs) + object.outputFormatSerializationOptions = "avroSerializationOptions"; + } return object; }; diff --git a/packages/google-cloud-bigquery-storage/protos/protos.json b/packages/google-cloud-bigquery-storage/protos/protos.json index 25abbfdca79..389ea6f3f92 100644 --- a/packages/google-cloud-bigquery-storage/protos/protos.json +++ b/packages/google-cloud-bigquery-storage/protos/protos.json @@ -83,6 +83,14 @@ } } }, + "AvroSerializationOptions": { + "fields": { + "enableDisplayNameAttribute": { + "type": "bool", + "id": 1 + } + } + }, "ProtoSchema": { "fields": { "protoDescriptor": { @@ -808,7 +816,8 @@ "oneofs": { "outputFormatSerializationOptions": { "oneof": [ - "arrowSerializationOptions" + "arrowSerializationOptions", + "avroSerializationOptions" ] } }, @@ -828,6 +837,13 @@ "options": { "(google.api.field_behavior)": "OPTIONAL" } + }, + "avroSerializationOptions": { + "type": "AvroSerializationOptions", + "id": 4, + "options": { + "(google.api.field_behavior)": "OPTIONAL" + } } } }