From ae59c7f957660e08cd5965b5e67694fa1ccc0057 Mon Sep 17 00:00:00 2001 From: Sri Harsha CH <57220027+harshachinta@users.noreply.github.com> Date: Wed, 15 May 2024 11:59:17 +0530 Subject: [PATCH] feat: add support for Proto columns (#1991) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Support For Proto Column in Spanner (#1829) * feat: Generated Proto Changes Changes need to be reverted. * feat: Implementation for Proto Message & Enum -Adding Logic for Serialization & Deserialization -New Type Codes and utilities * feat: Proto static files and typings and generated descriptors * sample: Adding DML, DQL, DML, table insert & read samples. * style: Lint * test: Adding unit tests * refactor: minor refactoring * refactor: minor refactoring * test: Adding integration tests * docs: Adding docs * test: Adding sample Integration Tests * refactor: Minor refactoring and updating comments/docs. * test: Making test fixes * refactor: Minor refactoring and lint fixes * refactor: Minor refactoring and lint fixes * Updating docs and minor changes. * test: fixing test * refactor: minor changes * refactor: minor refactoring * docs: Updating docs & comments * feat(spanner): fix lint * fix(spanner: lint * fix(spanner): lint * fix(spanner): lint * fix(spanner): lint * feat(spanner): fix db name * feat(spanner): remove it.only * feat(spanner): fix tests lint * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat(spanner): remove samples and sample tests * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat(spanner): update schema to avoid reserved keyword * feat(spanner): add samples and sample tests * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat(spanner): code refactoring --------- Co-authored-by: Gaurav Purohit Co-authored-by: Owl Bot --- README.md | 4 + package.json | 2 +- samples/README.md | 72 +++++ samples/package.json | 3 +- samples/proto-query-data.js | 100 ++++++ samples/proto-type-add-column.js | 90 ++++++ samples/proto-update-data-dml.js | 128 ++++++++ samples/proto-update-data.js | 120 ++++++++ samples/resource/descriptors.pb | Bin 0 -> 251 bytes samples/resource/singer.d.ts | 163 ++++++++++ samples/resource/singer.js | 451 ++++++++++++++++++++++++++++ samples/resource/singer.proto | 31 ++ samples/system-test/spanner.test.js | 76 +++++ src/codec.ts | 172 ++++++++++- src/index.ts | 60 ++++ src/partial-result-stream.ts | 51 +++- src/transaction.ts | 61 +++- system-test/spanner.ts | 275 ++++++++++++++++- test/codec.ts | 172 +++++++++++ test/data/descriptors.pb | Bin 0 -> 251 bytes test/data/singer.d.ts | 163 ++++++++++ test/data/singer.js | 451 ++++++++++++++++++++++++++++ test/data/singer.proto | 31 ++ test/index.ts | 62 ++++ test/transaction.ts | 2 + 25 files changed, 2724 insertions(+), 16 deletions(-) create mode 100644 samples/proto-query-data.js create mode 100644 samples/proto-type-add-column.js create mode 100644 samples/proto-update-data-dml.js create mode 100644 samples/proto-update-data.js create mode 100644 samples/resource/descriptors.pb create mode 100644 samples/resource/singer.d.ts create mode 100644 samples/resource/singer.js create mode 100644 samples/resource/singer.proto create mode 100644 test/data/descriptors.pb create mode 100644 test/data/singer.d.ts create mode 100644 test/data/singer.js create mode 100644 test/data/singer.proto diff --git a/README.md b/README.md index e40782b4e..12878220b 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,10 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-spanner/tre | Alters a sequence in a PostgreSQL database. | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/pg-sequence-alter.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/pg-sequence-alter.js,samples/README.md) | | Creates sequence in PostgreSQL database. | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/pg-sequence-create.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/pg-sequence-create.js,samples/README.md) | | Drops a sequence in PostgreSQL database. | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/pg-sequence-drop.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/pg-sequence-drop.js,samples/README.md) | +| Proto-query-data | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/proto-query-data.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/proto-query-data.js,samples/README.md) | +| Creates a new database with a proto column and enum | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/proto-type-add-column.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/proto-type-add-column.js,samples/README.md) | +| Proto-update-data-dml | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/proto-update-data-dml.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/proto-update-data-dml.js,samples/README.md) | +| Proto-update-data | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/proto-update-data.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/proto-update-data.js,samples/README.md) | | Queryoptions | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/queryoptions.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/queryoptions.js,samples/README.md) | | Quickstart | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/quickstart.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/quickstart.js,samples/README.md) | | Read data with database role | [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/read-data-with-database-role.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/read-data-with-database-role.js,samples/README.md) | diff --git a/package.json b/package.json index c6ed8b666..40edf4761 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "ycsb": "node ./benchmark/ycsb.js run -P ./benchmark/workloada -p table=usertable -p cloudspanner.instance=ycsb-instance -p operationcount=100 -p cloudspanner.database=ycsb", "fix": "gts fix", "clean": "gts clean", - "compile": "tsc -p . && cp -r protos build", + "compile": "tsc -p . && cp -r protos build && cp -r test/data build/test", "prepare": "npm run compile-protos && npm run compile", "pretest": "npm run compile", "presystem-test": "npm run compile", diff --git a/samples/README.md b/samples/README.md index 5e99af7cf..d60b9cdc2 100644 --- a/samples/README.md +++ b/samples/README.md @@ -92,6 +92,10 @@ and automatic, synchronous replication for high availability. * [Alters a sequence in a PostgreSQL database.](#alters-a-sequence-in-a-postgresql-database.) * [Creates sequence in PostgreSQL database.](#creates-sequence-in-postgresql-database.) * [Drops a sequence in PostgreSQL database.](#drops-a-sequence-in-postgresql-database.) + * [Proto-query-data](#proto-query-data) + * [Creates a new database with a proto column and enum](#creates-a-new-database-with-a-proto-column-and-enum) + * [Proto-update-data-dml](#proto-update-data-dml) + * [Proto-update-data](#proto-update-data) * [Queryoptions](#queryoptions) * [Quickstart](#quickstart) * [Read data with database role](#read-data-with-database-role) @@ -1455,6 +1459,74 @@ __Usage:__ +### Proto-query-data + +View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/proto-query-data.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/proto-query-data.js,samples/README.md) + +__Usage:__ + + +`node samples/proto-query-data.js` + + +----- + + + + +### Creates a new database with a proto column and enum + +View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/proto-type-add-column.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/proto-type-add-column.js,samples/README.md) + +__Usage:__ + + +`node proto-type-add-column.js ` + + +----- + + + + +### Proto-update-data-dml + +View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/proto-update-data-dml.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/proto-update-data-dml.js,samples/README.md) + +__Usage:__ + + +`node samples/proto-update-data-dml.js` + + +----- + + + + +### Proto-update-data + +View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/proto-update-data.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-spanner&page=editor&open_in_editor=samples/proto-update-data.js,samples/README.md) + +__Usage:__ + + +`node samples/proto-update-data.js` + + +----- + + + + ### Queryoptions View the [source code](https://github.com/googleapis/nodejs-spanner/blob/main/samples/queryoptions.js). diff --git a/samples/package.json b/samples/package.json index ea6e99d57..993669bdb 100644 --- a/samples/package.json +++ b/samples/package.json @@ -18,7 +18,8 @@ "@google-cloud/kms": "^4.0.0", "@google-cloud/precise-date": "^4.0.0", "@google-cloud/spanner": "^7.7.0", - "yargs": "^17.0.0" + "yargs": "^17.0.0", + "protobufjs": "^7.0.0" }, "devDependencies": { "chai": "^4.2.0", diff --git a/samples/proto-query-data.js b/samples/proto-query-data.js new file mode 100644 index 000000000..3ee51cd34 --- /dev/null +++ b/samples/proto-query-data.js @@ -0,0 +1,100 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +// eslint-disable-next-line node/no-unpublished-require +const singer = require('./resource/singer.js'); +const music = singer.examples.spanner.music; + +function main( + instanceId = 'my-instance', + databaseId = 'my-database', + projectId = 'my-project-id' +) { + // [START spanner_query_with_proto_types_parameter] + /** + * TODO(developer): Uncomment these variables before running the sample. + */ + // const instanceId = 'my-instance'; + // const databaseId = 'my-database'; + // const projectId = 'my-project-id'; + + // Imports the Google Cloud Spanner client library + const {Spanner} = require('@google-cloud/spanner'); + + // Instantiates a client + const spanner = new Spanner({ + projectId: projectId, + }); + + async function queryDataWithProtoTypes() { + // Gets a reference to a Cloud Spanner instance and database. + const instance = spanner.instance(instanceId); + const database = instance.database(databaseId); + + const query = { + sql: `SELECT SingerId, + SingerInfo, + SingerInfo.nationality, + SingerInfoArray, + SingerGenre, + SingerGenreArray + FROM Singers + WHERE SingerInfo.nationality = @country + and SingerGenre=@singerGenre`, + params: { + country: 'Country2', + singerGenre: music.Genre.FOLK, + }, + /* `columnsMetadata` is an optional parameter and is used to deserialize the + proto message and enum object back from bytearray and int respectively. + If columnsMetadata is not passed for proto messages and enums, then the data + types for these columns will be bytes and int respectively. */ + columnsMetadata: { + SingerInfo: music.SingerInfo, + SingerInfoArray: music.SingerInfo, + SingerGenre: music.Genre, + SingerGenreArray: music.Genre, + }, + }; + + // Queries rows from the Singers table. + try { + const [rows] = await database.run(query); + + rows.forEach(row => { + const json = row.toJSON(); + console.log( + `SingerId: ${json.SingerId}, SingerInfo: ${json.SingerInfo}, SingerGenre: ${json.SingerGenre}, + SingerInfoArray: ${json.SingerInfoArray}, SingerGenreArray: ${json.SingerGenreArray}` + ); + }); + } catch (err) { + console.error('ERROR:', err); + } finally { + // Close the database when finished. + database.close(); + } + } + + queryDataWithProtoTypes(); + // [END spanner_query_with_proto_types_parameter] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/proto-type-add-column.js b/samples/proto-type-add-column.js new file mode 100644 index 000000000..76c77aa5a --- /dev/null +++ b/samples/proto-type-add-column.js @@ -0,0 +1,90 @@ +/** + * Copyright 2024 Google LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// sample-metadata: +// title: Creates a new database with a proto column and enum +// usage: node proto-type-add-column.js + +'use strict'; + +const fs = require('fs'); + +function main( + instanceId = 'my-instance', + databaseId = 'my-database', + projectId = 'my-project-id' +) { + // [START spanner_add_proto_type_columns] + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // const projectId = 'my-project-id'; + // const instanceId = 'my-instance-id'; + // const databaseId = 'my-database-id'; + + // Imports the Google Cloud client library + const {Spanner} = require('@google-cloud/spanner'); + + // Creates a client + const spanner = new Spanner({ + projectId: projectId, + }); + + const databaseAdminClient = spanner.getDatabaseAdminClient(); + async function protoTypeAddColumn() { + // Adds a new Proto Message column and Proto Enum column to the Singers table. + + const request = [ + `CREATE PROTO BUNDLE ( + examples.spanner.music.SingerInfo, + examples.spanner.music.Genre, + )`, + 'ALTER TABLE Singers ADD COLUMN SingerInfo examples.spanner.music.SingerInfo', + 'ALTER TABLE Singers ADD COLUMN SingerInfoArray ARRAY', + 'ALTER TABLE Singers ADD COLUMN SingerGenre examples.spanner.music.Genre', + 'ALTER TABLE Singers ADD COLUMN SingerGenreArray ARRAY', + ]; + + // Read a proto descriptor file and convert it to a base64 string + const protoDescriptor = fs + .readFileSync('./resource/descriptors.pb') + .toString('base64'); + + // Alter existing table to add a column. + const [operation] = await databaseAdminClient.updateDatabaseDdl({ + database: databaseAdminClient.databasePath( + projectId, + instanceId, + databaseId + ), + statements: request, + protoDescriptors: protoDescriptor, + }); + + console.log(`Waiting for operation on ${databaseId} to complete...`); + await operation.promise(); + console.log( + `Altered table "Singers" on database ${databaseId} on instance ${instanceId} with proto descriptors.` + ); + } + protoTypeAddColumn(); + // [END spanner_add_proto_type_columns] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/proto-update-data-dml.js b/samples/proto-update-data-dml.js new file mode 100644 index 000000000..e5f9f7ac8 --- /dev/null +++ b/samples/proto-update-data-dml.js @@ -0,0 +1,128 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +// eslint-disable-next-line node/no-unpublished-require +const singer = require('./resource/singer.js'); +const music = singer.examples.spanner.music; + +function main( + instanceId = 'my-instance', + databaseId = 'my-database', + projectId = 'my-project-id' +) { + // [START spanner_update_data_with_proto_types_with_dml] + /** + * TODO(developer): Uncomment these variables before running the sample. + */ + // const instanceId = 'my-instance'; + // const databaseId = 'my-database'; + // const projectId = 'my-project-id'; + + // Imports the Google Cloud Spanner client library + const {Spanner} = require('@google-cloud/spanner'); + + // Instantiates a client + const spanner = new Spanner({ + projectId: projectId, + }); + + async function updateDataUsingDmlWithProtoTypes() { + /* + Updates Singers tables in the database with the ProtoMessage + and ProtoEnum column. + This updates the `SingerInfo`, `SingerInfoArray`, `SingerGenre` and + SingerGenreArray` columns which must be created before running this sample. + You can add the column by running the `add_proto_type_columns` sample or + by running this DDL statement against your database: + + ALTER TABLE Singers ADD COLUMN SingerInfo examples.spanner.music.SingerInfo\n + ALTER TABLE Singers ADD COLUMN SingerInfoArray ARRAY\n + ALTER TABLE Singers ADD COLUMN SingerGenre examples.spanner.music.Genre\n + ALTER TABLE Singers ADD COLUMN SingerGenreArray ARRAY\n + */ + + // Gets a reference to a Cloud Spanner instance and database. + const instance = spanner.instance(instanceId); + const database = instance.database(databaseId); + + const genre = music.Genre.ROCK; + const singerInfo = music.SingerInfo.create({ + singerId: 1, + genre: genre, + birthDate: 'January', + nationality: 'Country1', + }); + + const protoMessage = Spanner.protoMessage({ + value: singerInfo, + messageFunction: music.SingerInfo, + fullName: 'examples.spanner.music.SingerInfo', + }); + + const protoEnum = Spanner.protoEnum({ + value: genre, + enumObject: music.Genre, + fullName: 'examples.spanner.music.Genre', + }); + + database.runTransaction(async (err, transaction) => { + if (err) { + console.error(err); + return; + } + try { + const [, stats] = await transaction.run({ + sql: `UPDATE Singers SET SingerInfo=@singerInfo, SingerInfoArray=@singerInfoArray, + SingerGenre=@singerGenre, SingerGenreArray=@singerGenreArray WHERE SingerId = 1`, + params: { + singerInfo: protoMessage, + singerInfoArray: [protoMessage, null], + singerGenre: genre, + singerGenreArray: [protoEnum, null], + }, + }); + + const rowCount = stats[stats.rowCount]; + console.log(`${rowCount} record updated.`); + + const [, stats1] = await transaction.run({ + sql: 'UPDATE Singers SET SingerInfo.nationality=@singerNationality WHERE SingerId = 1', + params: { + singerNationality: 'Country2', + }, + }); + const rowCount1 = stats1[stats1.rowCount]; + console.log(`${rowCount1} record updated.`); + + await transaction.commit(); + } catch (err) { + console.error('ERROR:', err); + } finally { + // Close the database when finished. + database.close(); + } + }); + } + + updateDataUsingDmlWithProtoTypes(); + // [END spanner_update_data_with_proto_types_with_dml] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/proto-update-data.js b/samples/proto-update-data.js new file mode 100644 index 000000000..004e5938a --- /dev/null +++ b/samples/proto-update-data.js @@ -0,0 +1,120 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +// eslint-disable-next-line node/no-unpublished-require +const singer = require('./resource/singer.js'); +const music = singer.examples.spanner.music; + +function main( + instanceId = 'my-instance', + databaseId = 'my-database', + projectId = 'my-project-id' +) { + // [START spanner_update_data_with_proto_types] + /** + * TODO(developer): Uncomment these variables before running the sample. + */ + // const instanceId = 'my-instance'; + // const databaseId = 'my-database'; + // const projectId = 'my-project-id'; + + // Imports the Google Cloud Spanner client library + const {Spanner} = require('@google-cloud/spanner'); + + // Instantiates a client + const spanner = new Spanner({ + projectId: projectId, + }); + + async function updateDataWithProtoTypes() { + /* + Updates Singers tables in the database with the ProtoMessage + and ProtoEnum column. + This updates the `SingerInfo`, `SingerInfoArray`, `SingerGenre` and + SingerGenreArray` columns which must be created before running this sample. + You can add the column by running the `add_proto_type_columns` sample or + by running this DDL statement against your database: + + ALTER TABLE Singers ADD COLUMN SingerInfo examples.spanner.music.SingerInfo\n + ALTER TABLE Singers ADD COLUMN SingerInfoArray ARRAY\n + ALTER TABLE Singers ADD COLUMN SingerGenre examples.spanner.music.Genre\n + ALTER TABLE Singers ADD COLUMN SingerGenreArray ARRAY\n + */ + + // Gets a reference to a Cloud Spanner instance and database. + const instance = spanner.instance(instanceId); + const database = instance.database(databaseId); + + const genre = music.Genre.FOLK; + const singerInfo = music.SingerInfo.create({ + singerId: 2, + genre: genre, + birthDate: 'February', + nationality: 'Country2', + }); + + const protoMessage = Spanner.protoMessage({ + value: singerInfo, + messageFunction: music.SingerInfo, + fullName: 'examples.spanner.music.SingerInfo', + }); + + const protoEnum = Spanner.protoEnum({ + value: genre, + enumObject: music.Genre, + fullName: 'examples.spanner.music.Genre', + }); + + // Get a reference to the Singers table + const table = database.table('Singers'); + + const data = [ + { + SingerId: 2, + SingerInfo: protoMessage, + SingerInfoArray: [protoMessage], + SingerGenre: protoEnum, + SingerGenreArray: [protoEnum], + }, + { + SingerId: 3, + SingerInfo: null, + SingerInfoArray: null, + SingerGenre: null, + SingerGenreArray: null, + }, + ]; + + try { + await table.update(data); + console.log('Data updated.'); + } catch (err) { + console.error('ERROR:', err); + } finally { + // Close the database when finished. + await database.close(); + } + } + + updateDataWithProtoTypes(); + // [END spanner_update_data_with_proto_types] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); +main(...process.argv.slice(2)); diff --git a/samples/resource/descriptors.pb b/samples/resource/descriptors.pb new file mode 100644 index 0000000000000000000000000000000000000000..d4c018f3a3c21b18f68820eeab130d8195064e81 GIT binary patch literal 251 zcmd=3!N|o^oSB!NTBKJ{lwXoBB$ir{m|KvOTC7)GkeHVT6wfU!&P-OC&&b6U3|8ow zmzFOi&BY1P7N40S!KlEf!5qW^5%5eAlI7w`$}B3$h)+o@NtIv%%5nyAf<;__0zwL0 z+>> 3) { + case 1: { + message.singerId = reader.int64(); + break; + } + case 2: { + message.birthDate = reader.string(); + break; + } + case 3: { + message.nationality = reader.string(); + break; + } + case 4: { + message.genre = reader.int32(); + break; + } + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Decodes a SingerInfo message from the specified reader or buffer, length delimited. + * @function decodeDelimited + * @memberof examples.spanner.music.SingerInfo + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @returns {examples.spanner.music.SingerInfo} SingerInfo + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + SingerInfo.decodeDelimited = function decodeDelimited(reader) { + if (!(reader instanceof $Reader)) reader = new $Reader(reader); + return this.decode(reader, reader.uint32()); + }; + + /** + * Verifies a SingerInfo message. + * @function verify + * @memberof examples.spanner.music.SingerInfo + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + SingerInfo.verify = function verify(message) { + if (typeof message !== 'object' || message === null) + return 'object expected'; + if (message.singerId !== null && message.hasOwnProperty('singerId')) + if ( + !$util.isInteger(message.singerId) && + !( + message.singerId && + $util.isInteger(message.singerId.low) && + $util.isInteger(message.singerId.high) + ) + ) + return 'singerId: integer|Long expected'; + if (message.birthDate !== null && message.hasOwnProperty('birthDate')) + if (!$util.isString(message.birthDate)) + return 'birthDate: string expected'; + if ( + message.nationality !== null && + message.hasOwnProperty('nationality') + ) + if (!$util.isString(message.nationality)) + return 'nationality: string expected'; + if (message.genre !== null && message.hasOwnProperty('genre')) + switch (message.genre) { + default: + return 'genre: enum value expected'; + case 0: + case 1: + case 2: + case 3: + break; + } + return null; + }; + + /** + * Creates a SingerInfo message from a plain object. Also converts values to their respective internal types. + * @function fromObject + * @memberof examples.spanner.music.SingerInfo + * @static + * @param {Object.} object Plain object + * @returns {examples.spanner.music.SingerInfo} SingerInfo + */ + SingerInfo.fromObject = function fromObject(object) { + if (object instanceof $root.examples.spanner.music.SingerInfo) + return object; + var message = new $root.examples.spanner.music.SingerInfo(); + if (object.singerId !== null) + if ($util.Long) + (message.singerId = $util.Long.fromValue( + object.singerId + )).unsigned = false; + else if (typeof object.singerId === 'string') + message.singerId = parseInt(object.singerId, 10); + else if (typeof object.singerId === 'number') + message.singerId = object.singerId; + else if (typeof object.singerId === 'object') + message.singerId = new $util.LongBits( + object.singerId.low >>> 0, + object.singerId.high >>> 0 + ).toNumber(); + if (object.birthDate !== null) + message.birthDate = String(object.birthDate); + if (object.nationality !== null) + message.nationality = String(object.nationality); + switch (object.genre) { + default: + if (typeof object.genre === 'number') { + message.genre = object.genre; + break; + } + break; + case 'POP': + case 0: + message.genre = 0; + break; + case 'JAZZ': + case 1: + message.genre = 1; + break; + case 'FOLK': + case 2: + message.genre = 2; + break; + case 'ROCK': + case 3: + message.genre = 3; + break; + } + return message; + }; + + /** + * Creates a plain object from a SingerInfo message. Also converts values to other types if specified. + * @function toObject + * @memberof examples.spanner.music.SingerInfo + * @static + * @param {examples.spanner.music.SingerInfo} message SingerInfo + * @param {$protobuf.IConversionOptions} [options] Conversion options + * @returns {Object.} Plain object + */ + SingerInfo.toObject = function toObject(message, options) { + if (!options) options = {}; + var object = {}; + if (options.defaults) { + if ($util.Long) { + var long = new $util.Long(0, 0, false); + object.singerId = + options.longs === String + ? long.toString() + : options.longs === Number + ? long.toNumber() + : long; + } else object.singerId = options.longs === String ? '0' : 0; + object.birthDate = ''; + object.nationality = ''; + object.genre = options.enums === String ? 'POP' : 0; + } + if (message.singerId !== null && message.hasOwnProperty('singerId')) + if (typeof message.singerId === 'number') + object.singerId = + options.longs === String + ? String(message.singerId) + : message.singerId; + else + object.singerId = + options.longs === String + ? $util.Long.prototype.toString.call(message.singerId) + : options.longs === Number + ? new $util.LongBits( + message.singerId.low >>> 0, + message.singerId.high >>> 0 + ).toNumber() + : message.singerId; + if (message.birthDate !== null && message.hasOwnProperty('birthDate')) + object.birthDate = message.birthDate; + if ( + message.nationality !== null && + message.hasOwnProperty('nationality') + ) + object.nationality = message.nationality; + if (message.genre !== null && message.hasOwnProperty('genre')) + object.genre = + options.enums === String + ? $root.examples.spanner.music.Genre[message.genre] === + undefined + ? message.genre + : $root.examples.spanner.music.Genre[message.genre] + : message.genre; + return object; + }; + + /** + * Converts this SingerInfo to JSON. + * @function toJSON + * @memberof examples.spanner.music.SingerInfo + * @instance + * @returns {Object.} JSON object + */ + SingerInfo.prototype.toJSON = function toJSON() { + return this.constructor.toObject(this, $protobuf.util.toJSONOptions); + }; + + /** + * Gets the default type url for SingerInfo + * @function getTypeUrl + * @memberof examples.spanner.music.SingerInfo + * @static + * @param {string} [typeUrlPrefix] your custom typeUrlPrefix(default "type.googleapis.com") + * @returns {string} The default type url + */ + SingerInfo.getTypeUrl = function getTypeUrl(typeUrlPrefix) { + if (typeUrlPrefix === undefined) { + typeUrlPrefix = 'type.googleapis.com'; + } + return typeUrlPrefix + '/examples.spanner.music.SingerInfo'; + }; + + return SingerInfo; + })(); + + /** + * Genre enum. + * @name examples.spanner.music.Genre + * @enum {number} + * @property {number} POP=0 POP value + * @property {number} JAZZ=1 JAZZ value + * @property {number} FOLK=2 FOLK value + * @property {number} ROCK=3 ROCK value + */ + music.Genre = (function () { + var valuesById = {}, + values = Object.create(valuesById); + values[(valuesById[0] = 'POP')] = 0; + values[(valuesById[1] = 'JAZZ')] = 1; + values[(valuesById[2] = 'FOLK')] = 2; + values[(valuesById[3] = 'ROCK')] = 3; + return values; + })(); + + return music; + })(); + + return spanner; + })(); + + return examples; +})(); + +module.exports = $root; diff --git a/samples/resource/singer.proto b/samples/resource/singer.proto new file mode 100644 index 000000000..d4e82bfc7 --- /dev/null +++ b/samples/resource/singer.proto @@ -0,0 +1,31 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package examples.spanner.music; + +message SingerInfo { + optional int64 singer_id = 1; + optional string birth_date = 2; + optional string nationality = 3; + optional Genre genre = 4; +} + +enum Genre { + POP = 0; + JAZZ = 1; + FOLK = 2; + ROCK = 3; +} diff --git a/samples/system-test/spanner.test.js b/samples/system-test/spanner.test.js index 3d18b8493..8ce55029b 100644 --- a/samples/system-test/spanner.test.js +++ b/samples/system-test/spanner.test.js @@ -67,6 +67,7 @@ const VERSION_RETENTION_DATABASE_ID = `test-database-${CURRENT_TIME}-v`; const ENCRYPTED_DATABASE_ID = `test-database-${CURRENT_TIME}-enc`; const DEFAULT_LEADER_DATABASE_ID = `test-database-${CURRENT_TIME}-dl`; const SEQUENCE_DATABASE_ID = `test-seq-database-${CURRENT_TIME}-r`; +const PROTO_DATABASE_ID = `test-db${CURRENT_TIME}-proto1`; const BACKUP_ID = `test-backup-${CURRENT_TIME}`; const COPY_BACKUP_ID = `test-copy-backup-${CURRENT_TIME}`; const ENCRYPTED_BACKUP_ID = `test-backup-${CURRENT_TIME}-enc`; @@ -2052,4 +2053,79 @@ describe('Autogenerated Admin Clients', () => { ); }); }); + + describe('proto columns', () => { + before(async () => { + // Setup database for Proto columns + const databaseAdminClient = spanner.getDatabaseAdminClient(); + const [operation] = await databaseAdminClient.createDatabase({ + createStatement: 'CREATE DATABASE `' + PROTO_DATABASE_ID + '`', + extraStatements: [ + ` + CREATE TABLE Singers ( + SingerId INT64 NOT NULL, + FirstName STRING(1024), + LastName STRING(1024), + ) PRIMARY KEY (SingerId)`, + `CREATE TABLE Albums ( + SingerId INT64 NOT NULL, + AlbumId INT64 NOT NULL, + AlbumTitle STRING(MAX) + ) PRIMARY KEY (SingerId, AlbumId), + INTERLEAVE IN PARENT Singers ON DELETE CASCADE`, + ], + parent: databaseAdminClient.instancePath(PROJECT_ID, INSTANCE_ID), + }); + + console.log( + `Waiting for creation of ${PROTO_DATABASE_ID} to complete...` + ); + await operation.promise(); + console.log( + `Created database ${PROTO_DATABASE_ID} on instance ${INSTANCE_ID}.` + ); + + // Insert seed data into the database tables + execSync( + `${crudCmd} insert ${INSTANCE_ID} ${PROTO_DATABASE_ID} ${PROJECT_ID}` + ); + }); + + after(async () => { + await spanner.instance(INSTANCE_ID).database(PROTO_DATABASE_ID).delete(); + }); + + it('should add proto message and enum columns', async () => { + const output = execSync( + `node proto-type-add-column.js "${INSTANCE_ID}" "${PROTO_DATABASE_ID}" ${PROJECT_ID}` + ); + assert.match( + output, + new RegExp( + `Altered table "Singers" on database ${PROTO_DATABASE_ID} on instance ${INSTANCE_ID} with proto descriptors.` + ) + ); + }); + + it('update data with proto message and enum columns', async () => { + const output = execSync( + `node proto-update-data.js "${INSTANCE_ID}" "${PROTO_DATABASE_ID}" ${PROJECT_ID}` + ); + assert.match(output, new RegExp('Data updated')); + }); + + it('update data with proto message and enum columns using DML', async () => { + const output = execSync( + `node proto-update-data-dml.js "${INSTANCE_ID}" "${PROTO_DATABASE_ID}" ${PROJECT_ID}` + ); + assert.include(output, '1 record updated.'); + }); + + it('query data with proto message and enum columns', async () => { + const output = execSync( + `node proto-query-data.js "${INSTANCE_ID}" "${PROTO_DATABASE_ID}" ${PROJECT_ID}` + ); + assert.match(output, new RegExp('SingerId: 2')); + }); + }); }); diff --git a/src/codec.ts b/src/codec.ts index 53c643200..1c49d8599 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -30,6 +30,40 @@ export interface Field { value: Value; } +export interface IProtoMessageParams { + // Supports proto message serialized binary data as a `Buffer` or pass a + // message object created using the functions generated by protobufjs-cli. + value: object; + // Fully qualified name of the proto representing the message definition + fullName: string; + /** + * Provide a First Class function that includes nested functions named + * "encode" for serialization and "decode" for deserialization of the proto + * message. The function should be sourced from the JS file generated by + * protobufjs-cli for the proto message. + */ + messageFunction?: Function; +} + +export interface IProtoEnumParams { + // Supports proto enum integer constant or pass the enum string + // the functions generated by protobufjs-cli. + value: string | number; + // Fully qualified name of the proto representing the enum definition + fullName: string; + /** + * An object containing enum string to id mapping. + * @example: { POP: 0, JAZZ: 1, FOLK: 2, ROCK: 3 } + * + * The object should be sourced from the JS file generated by + * protobufjs-cli for the proto message. Additionally, please review the + * sample at {@link www.samples.com} for guidance. + * + * ToDo: Update the link + */ + enumObject?: object; +} + export interface Json { [field: string]: Value; } @@ -246,6 +280,83 @@ export class PGNumeric { } } +/** + * @typedef ProtoMessage + * @see Spanner.protoMessage + */ +export class ProtoMessage { + value: Buffer; + fullName: string; + messageFunction?: Function; + + constructor(protoMessageParams: IProtoMessageParams) { + this.fullName = protoMessageParams.fullName; + this.messageFunction = protoMessageParams.messageFunction; + + if (protoMessageParams.value instanceof Buffer) { + this.value = protoMessageParams.value; + } else if (protoMessageParams.messageFunction) { + this.value = protoMessageParams.messageFunction['encode']( + protoMessageParams.value + ).finish(); + } else { + throw new GoogleError(`protoMessageParams cannot be used to construct + the ProtoMessage. Pass the serialized buffer of the + proto message as the value or provide the message object along with the + corresponding messageFunction generated by protobufjs-cli.`); + } + } + + toJSON(): string { + if (this.messageFunction) { + return this.messageFunction['toObject']( + this.messageFunction['decode'](this.value) + ); + } + return this.value.toString(); + } +} + +/** + * @typedef ProtoEnum + * @see Spanner.protoEnum + */ +export class ProtoEnum { + value: string; + fullName: string; + enumObject?: object; + + constructor(protoEnumParams: IProtoEnumParams) { + this.fullName = protoEnumParams.fullName; + this.enumObject = protoEnumParams.enumObject; + + /** + * @code{IProtoEnumParams} can accept either a number or a string as a value so + * converting to string and checking whether it's numeric using regex. + */ + if (/^\d+$/.test(protoEnumParams.value.toString())) { + this.value = protoEnumParams.value.toString(); + } else if ( + protoEnumParams.enumObject && + protoEnumParams.enumObject[protoEnumParams.value] + ) { + this.value = protoEnumParams.enumObject[protoEnumParams.value]; + } else { + throw new GoogleError(`protoEnumParams cannot be used for constructing the + ProtoEnum. Pass the number as the value or provide the enum string + constant as the value along with the corresponding enumObject generated + by protobufjs-cli.`); + } + } + + toJSON(): string { + if (this.enumObject) { + return Object.getPrototypeOf(this.enumObject)[this.value]; + } + return this.value.toString(); + } +} + /** * @typedef PGJsonb * @see Spanner.pgJsonb @@ -367,6 +478,10 @@ function convertValueToJson(value: Value, options: JSONOptions): Value { return value.map(child => convertValueToJson(child, options)); } + if (value instanceof ProtoMessage || value instanceof ProtoEnum) { + return value.toJSON(); + } + return value; } @@ -377,9 +492,14 @@ function convertValueToJson(value: Value, options: JSONOptions): Value { * * @param {*} value Value to decode * @param {object[]} type Value type object. + * @param columnMetadata Optional parameter to deserialize data * @returns {*} */ -function decode(value: Value, type: spannerClient.spanner.v1.Type): Value { +function decode( + value: Value, + type: spannerClient.spanner.v1.Type, + columnMetadata?: object +): Value { if (is.null(value)) { return null; } @@ -392,6 +512,23 @@ function decode(value: Value, type: spannerClient.spanner.v1.Type): Value { case 'BYTES': decoded = Buffer.from(decoded, 'base64'); break; + case spannerClient.spanner.v1.TypeCode.PROTO: + case 'PROTO': + decoded = Buffer.from(decoded, 'base64'); + decoded = new ProtoMessage({ + value: decoded, + fullName: type.protoTypeFqn, + messageFunction: columnMetadata as Function, + }); + break; + case spannerClient.spanner.v1.TypeCode.ENUM: + case 'ENUM': + decoded = new ProtoEnum({ + value: decoded, + fullName: type.protoTypeFqn, + enumObject: columnMetadata as object, + }); + break; case spannerClient.spanner.v1.TypeCode.FLOAT32: case 'FLOAT32': decoded = new Float32(decoded); @@ -449,7 +586,8 @@ function decode(value: Value, type: spannerClient.spanner.v1.Type): Value { decoded = decoded.map(value => { return decode( value, - type.arrayElementType! as spannerClient.spanner.v1.Type + type.arrayElementType! as spannerClient.spanner.v1.Type, + columnMetadata ); }); break; @@ -458,7 +596,8 @@ function decode(value: Value, type: spannerClient.spanner.v1.Type): Value { fields = type.structType!.fields!.map(({name, type}, index) => { const value = decode( (!Array.isArray(decoded) && decoded[name!]) || decoded[index], - type as spannerClient.spanner.v1.Type + type as spannerClient.spanner.v1.Type, + columnMetadata ); return {name, value}; }); @@ -518,6 +657,14 @@ function encodeValue(value: Value): Value { return value.toString('base64'); } + if (value instanceof ProtoMessage) { + return value.value.toString('base64'); + } + + if (value instanceof ProtoEnum) { + return value.value; + } + if (value instanceof Struct) { return Array.from(value).map(field => encodeValue(field.value)); } @@ -560,6 +707,8 @@ const TypeCode: { bytes: 'BYTES', json: 'JSON', jsonb: 'JSON', + proto: 'PROTO', + enum: 'ENUM', array: 'ARRAY', struct: 'STRUCT', }; @@ -574,6 +723,7 @@ export interface Type { type: string; fields?: FieldType[]; child?: Type; + fullName?: string; } interface FieldType extends Type { @@ -595,6 +745,8 @@ interface FieldType extends Type { * - string * - bytes * - json + * - proto + * - enum * - timestamp * - date * - struct @@ -650,6 +802,14 @@ function getType(value: Value): Type { return {type: 'pgOid'}; } + if (value instanceof ProtoMessage) { + return {type: 'proto', fullName: value.fullName}; + } + + if (value instanceof ProtoEnum) { + return {type: 'enum', fullName: value.fullName}; + } + if (is.boolean(value)) { return {type: 'bool'}; } @@ -776,6 +936,10 @@ function createTypeObject( code, } as spannerClient.spanner.v1.Type; + if (code === 'PROTO' || code === 'ENUM') { + type.protoTypeFqn = config.fullName || ''; + } + if (code === 'ARRAY') { type.arrayElementType = codec.createTypeObject(config.child); } @@ -811,6 +975,8 @@ export const codec = { Numeric, PGNumeric, PGJsonb, + ProtoMessage, + ProtoEnum, PGOid, convertFieldsToJson, decode, diff --git a/src/index.ts b/src/index.ts index 62e3fc23d..71370fce2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -34,6 +34,10 @@ import { PGJsonb, SpannerDate, Struct, + ProtoMessage, + ProtoEnum, + IProtoMessageParams, + IProtoEnumParams, } from './codec'; import {Backup} from './backup'; import {Database} from './database'; @@ -1772,6 +1776,62 @@ class Spanner extends GrpcService { static pgJsonb(value): PGJsonb { return new codec.PGJsonb(value); } + + /** + * @typedef IProtoMessageParams + * @property {object} value Proto Message value as serialized-buffer or message object. + * @property {string} fullName Fully-qualified path name of proto message. + * @property {Function} [messageFunction] Function generated by protobufs containing + * helper methods for deserializing and serializing messages. + */ + /** + * Helper function to get a Cloud Spanner proto Message object. + * + * @param {IProtoMessageParams} params The proto message value params + * @returns {ProtoMessage} + * + * @example + * ``` + * const {Spanner} = require('@google-cloud/spanner'); + * const protoMessage = Spanner.protoMessage({ + * value: singerInfo, + * messageFunction: music.SingerInfo, + * fullName: "examples.spanner.music.SingerInfo" + * }); + * ``` + */ + static protoMessage(params: IProtoMessageParams): ProtoMessage { + return new codec.ProtoMessage(params); + } + + /** + * @typedef IProtoEnumParams + * @property {string | number} value Proto Enum value as a string constant or + * an integer constant. + * @property {string} fullName Fully-qualified path name of proto enum. + * @property {object} [enumObject] An enum object generated by protobufjs-cli. + */ + /** + * Helper function to get a Cloud Spanner proto enum object. + * + * @param {IProtoEnumParams} params The proto enum value params in the format of + * @code{IProtoEnumParams} + * @returns {ProtoEnum} + * + * @example + * ``` + * const {Spanner} = require('@google-cloud/spanner'); + * const protoEnum = Spanner.protoEnum({ + * value: 'ROCK', + * enumObject: music.Genre, + * fullName: "examples.spanner.music.Genre" + * }); + * ``` + */ + static protoEnum(params: IProtoEnumParams): ProtoEnum { + return new codec.ProtoEnum(params); + } + /** * Helper function to get a Cloud Spanner Struct object. * diff --git a/src/partial-result-stream.ts b/src/partial-result-stream.ts index e03539219..616cdc88b 100644 --- a/src/partial-result-stream.ts +++ b/src/partial-result-stream.ts @@ -50,11 +50,52 @@ interface RequestFunction { * that it is not ready for any more data. Increase this value if you * experience 'Stream is still not ready to receive data' errors as a * result of a slow writer in your receiving stream. + * @property {object} [columnsMetadata] An object map that can be used to pass + * additional properties for each column type which can help in deserializing + * the data coming from backend. (Eg: We need to pass Proto Function and Enum + * map to deserialize proto messages and enums, respectively.) */ export interface RowOptions { json?: boolean; jsonOptions?: JSONOptions; maxResumeRetries?: number; + /** + * An object where column names as keys and custom objects as corresponding + * values for deserialization. It's specifically useful for data types like + * protobuf where deserialization logic is on user-specific code. When provided, + * the custom object enables deserialization of backend-received column data. + * If not provided, data remains serialized as buffer for Proto Messages and + * integer for Proto Enums. + * + * @example + * To obtain Proto Messages and Proto Enums as JSON objects, you must supply + * additional metadata. This metadata should include the protobufjs-cli + * generated proto message function and enum object. It encompasses the essential + * logic for proper data deserialization. + * + * Eg: To read data from Proto Columns in json format using DQL, you should pass + * columnsMetadata where key is the name of the column and value is the protobufjs-cli + * generated proto message function and enum object. + * + * const query = { + * sql: `SELECT SingerId, + * FirstName, + * LastName, + * SingerInfo, + * SingerGenre, + * SingerInfoArray, + * SingerGenreArray + * FROM Singers + * WHERE SingerId = 6`, + * columnsMetadata: { + * SingerInfo: music.SingerInfo, + * SingerInfoArray: music.SingerInfo, + * SingerGenre: music.Genre, + * SingerGenreArray: music.Genre, + * }, + * }; + */ + columnsMetadata?: object; } /** @@ -334,7 +375,15 @@ export class PartialResultStream extends Transform implements ResultEvents { private _createRow(values: Value[]): Row { const fields = values.map((value, index) => { const {name, type} = this._fields[index]; - return {name, value: codec.decode(value, type as google.spanner.v1.Type)}; + const columnMetadata = this._options.columnsMetadata?.[name]; + return { + name, + value: codec.decode( + value, + type as google.spanner.v1.Type, + columnMetadata + ), + }; }); Object.defineProperty(fields, 'toJSON', { diff --git a/src/transaction.ts b/src/transaction.ts index 11eb03657..f7773bd96 100644 --- a/src/transaction.ts +++ b/src/transaction.ts @@ -64,6 +64,43 @@ export interface RequestOptions { jsonOptions?: JSONOptions; gaxOptions?: CallOptions; maxResumeRetries?: number; + /** + * An object where column names as keys and custom objects as corresponding + * values for deserialization. This is only needed for proto columns + * where deserialization logic is on user-specific code. When provided, + * the custom object enables deserialization of backend-received column data. + * If not provided, data remains serialized as buffer for Proto Messages and + * integer for Proto Enums. + * + * @example + * To obtain Proto Messages and Proto Enums as JSON objects, you must supply + * additional metadata. This metadata should include the protobufjs-cli + * generated proto message function and enum object. It encompasses the essential + * logic for proper data deserialization. + * + * Eg: To read data from Proto Columns in json format using DQL, you should pass + * columnsMetadata where key is the name of the column and value is the protobufjs-cli + * generated proto message function and enum object. + * + * const query = { + * sql: `SELECT SingerId, + * FirstName, + * LastName, + * SingerInfo, + * SingerGenre, + * SingerInfoArray, + * SingerGenreArray + * FROM Singers + * WHERE SingerId = 6`, + * columnsMetadata: { + * SingerInfo: music.SingerInfo, + * SingerInfoArray: music.SingerInfo, + * SingerGenre: music.Genre, + * SingerGenreArray: music.Genre, + * }, + * }; + */ + columnsMetadata?: object; } export interface CommitOptions { @@ -583,8 +620,14 @@ export class Snapshot extends EventEmitter { table: string, request = {} as ReadRequest ): PartialResultStream { - const {gaxOptions, json, jsonOptions, maxResumeRetries, requestOptions} = - request; + const { + gaxOptions, + json, + jsonOptions, + maxResumeRetries, + requestOptions, + columnsMetadata, + } = request; const keySet = Snapshot.encodeKeySet(request); const transaction: spannerClient.spanner.v1.ITransactionSelector = {}; @@ -610,6 +653,7 @@ export class Snapshot extends EventEmitter { delete request.ranges; delete request.requestOptions; delete request.directedReadOptions; + delete request.columnsMetadata; const reqOpts: spannerClient.spanner.v1.IReadRequest = Object.assign( request, @@ -654,6 +698,7 @@ export class Snapshot extends EventEmitter { json, jsonOptions, maxResumeRetries, + columnsMetadata, }) ?.on('response', response => { if (response.metadata && response.metadata!.transaction && !this.id) { @@ -1077,8 +1122,14 @@ export class Snapshot extends EventEmitter { query.queryOptions ); - const {gaxOptions, json, jsonOptions, maxResumeRetries, requestOptions} = - query; + const { + gaxOptions, + json, + jsonOptions, + maxResumeRetries, + requestOptions, + columnsMetadata, + } = query; let reqOpts; const directedReadOptions = this._getDirectedReadOptions( @@ -1103,6 +1154,7 @@ export class Snapshot extends EventEmitter { delete query.requestOptions; delete query.types; delete query.directedReadOptions; + delete query.columnsMetadata; reqOpts = Object.assign(query, { session: this.session.formattedName_!, @@ -1152,6 +1204,7 @@ export class Snapshot extends EventEmitter { json, jsonOptions, maxResumeRetries, + columnsMetadata, }) .on('response', response => { if (response.metadata && response.metadata!.transaction && !this.id) { diff --git a/system-test/spanner.ts b/system-test/spanner.ts index 041920a77..2a4df13aa 100644 --- a/system-test/spanner.ts +++ b/system-test/spanner.ts @@ -46,6 +46,11 @@ import {google} from '../protos/protos'; import CreateDatabaseMetadata = google.spanner.admin.database.v1.CreateDatabaseMetadata; import CreateBackupMetadata = google.spanner.admin.database.v1.CreateBackupMetadata; import CreateInstanceConfigMetadata = google.spanner.admin.instance.v1.CreateInstanceConfigMetadata; +const singer = require('../test/data/singer'); +const music = singer.examples.spanner.music; +import {util} from 'protobufjs'; +import Long = util.Long; +const fs = require('fs'); const SKIP_BACKUPS = process.env.SKIP_BACKUPS; const SKIP_FGAC_TESTS = (process.env.SKIP_FGAC_TESTS || 'false').toLowerCase(); @@ -124,15 +129,42 @@ describe('Spanner', () => { `Not creating temp instance, using + ${instance.formattedName_}...` ); } - const [, googleSqlOperation1] = await DATABASE.create({ - schema: ` + + if (IS_EMULATOR_ENABLED) { + const [, googleSqlOperation1] = await DATABASE.create({ + schema: ` CREATE TABLE ${TABLE_NAME} ( SingerId STRING(1024) NOT NULL, Name STRING(1024), ) PRIMARY KEY(SingerId)`, - gaxOptions: GAX_OPTIONS, - }); - await googleSqlOperation1.promise(); + gaxOptions: GAX_OPTIONS, + }); + await googleSqlOperation1.promise(); + } else { + // Reading proto descriptor file + const protoDescriptor = fs + .readFileSync('test/data/descriptors.pb') + .toString('base64'); + + const [, googleSqlOperation1] = await DATABASE.create({ + schema: [ + ` + CREATE PROTO BUNDLE ( + examples.spanner.music.SingerInfo, + examples.spanner.music.Genre, + )`, + ` + CREATE TABLE ${TABLE_NAME} ( + SingerId STRING(1024) NOT NULL, + Name STRING(1024), + ) PRIMARY KEY(SingerId)`, + ], + gaxOptions: GAX_OPTIONS, + protoDescriptors: protoDescriptor, + }); + + await googleSqlOperation1.promise(); + } RESOURCES_TO_CLEAN.push(DATABASE); const [, googleSqlOperation2] = await DATABASE_DROP_PROTECTION.create({ @@ -346,7 +378,20 @@ describe('Spanner', () => { const googleSqlTable = DATABASE.table(TABLE_NAME); const postgreSqlTable = PG_DATABASE.table(TABLE_NAME); - function insert(insertData, dialect, callback) { + /** + * + * @param insertData data to insert + * @param dialect sql dialect + * @param callback + * @param columnsMetadataForRead Optional parameter use for read/query for + * deserializing Proto messages and enum + */ + function insert( + insertData, + dialect, + callback, + columnsMetadataForRead?: {} + ) { const id = generateName('id'); insertData.Key = id; @@ -357,6 +402,7 @@ describe('Spanner', () => { params: { id, }, + columnsMetadata: columnsMetadataForRead, }; let database = DATABASE; if (dialect === Spanner.POSTGRESQL) { @@ -430,6 +476,8 @@ describe('Spanner', () => { NumericValue NUMERIC, StringValue STRING( MAX), TimestampValue TIMESTAMP, + ProtoMessageValue examples.spanner.music.SingerInfo, + ProtoEnumValue examples.spanner.music.Genre, BytesArray ARRAY, BoolArray ARRAY, DateArray ARRAY< DATE >, @@ -439,6 +487,8 @@ describe('Spanner', () => { NumericArray ARRAY< NUMERIC >, StringArray ARRAY, TimestampArray ARRAY< TIMESTAMP >, + ProtoMessageArray ARRAY, + ProtoEnumArray ARRAY, CommitTimestamp TIMESTAMP OPTIONS (allow_commit_timestamp= true) ) PRIMARY KEY (Key) ` @@ -1844,6 +1894,219 @@ describe('Spanner', () => { }); }); + describe('protoMessage', () => { + before(async function () { + if (IS_EMULATOR_ENABLED) { + this.skip(); + } + }); + + const protoMessageParams = { + value: music.SingerInfo.create({ + singerId: new Long(1), + genre: music.Genre.POP, + birthDate: 'January', + nationality: 'Country1', + }), + messageFunction: music.SingerInfo, + fullName: 'examples.spanner.music.SingerInfo', + }; + + it('GOOGLE_STANDARD_SQL should write protoMessage values', done => { + const value = Spanner.protoMessage(protoMessageParams); + insert( + {ProtoMessageValue: value}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.deepStrictEqual( + row.toJSON().ProtoMessageValue, + music.SingerInfo.toObject(protoMessageParams.value) + ); + done(); + }, + {ProtoMessageValue: music.SingerInfo} + ); + }); + + it('GOOGLE_STANDARD_SQL should write bytes in the protoMessage column', done => { + const value = music.SingerInfo.encode( + protoMessageParams.value + ).finish(); + insert( + {ProtoMessageValue: value}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.deepStrictEqual( + row.toJSON().ProtoMessageValue, + value.toString() + ); + done(); + } + ); + }); + + it('GOOGLE_STANDARD_SQL should write null in the protoMessage column', done => { + insert( + {ProtoMessageValue: null}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.equal(row.toJSON().ProtoMessageValue, null); + done(); + } + ); + }); + + it('GOOGLE_STANDARD_SQL should write protoMessageArray', done => { + const value = Spanner.protoMessage(protoMessageParams); + insert( + {ProtoMessageArray: [value]}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.deepStrictEqual(row.toJSON().ProtoMessageArray, [ + music.SingerInfo.toObject(protoMessageParams.value), + ]); + done(); + }, + {ProtoMessageArray: music.SingerInfo} + ); + }); + + it('GOOGLE_STANDARD_SQL should write bytes array in the protoMessageArray column', done => { + const value = music.SingerInfo.encode( + protoMessageParams.value + ).finish(); + insert( + {ProtoMessageArray: [value]}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.deepStrictEqual(row.toJSON().ProtoMessageArray, [ + value.toString(), + ]); + done(); + } + ); + }); + + it('GOOGLE_STANDARD_SQL should write null in the protoMessageArray column', done => { + insert( + {ProtoMessageArray: null}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.equal(row.toJSON().ProtoMessageArray, null); + done(); + } + ); + }); + }); + + describe('protoEnum', () => { + before(async function () { + if (IS_EMULATOR_ENABLED) { + this.skip(); + } + }); + + const enumParams = { + value: music.Genre.JAZZ, + enumObject: music.Genre, + fullName: 'examples.spanner.music.Genre', + }; + + it('GOOGLE_STANDARD_SQL should write protoEnum values', done => { + const value = Spanner.protoEnum(enumParams); + insert( + {ProtoEnumValue: value}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.deepStrictEqual( + row.toJSON().ProtoEnumValue, + Object.getPrototypeOf(music.Genre)[enumParams.value] + ); + done(); + }, + {ProtoEnumValue: music.Genre} + ); + }); + + it('GOOGLE_STANDARD_SQL should write int in the protoEnum column', done => { + const value = 2; + insert( + {ProtoEnumValue: value}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.deepStrictEqual( + row.toJSON().ProtoEnumValue, + value.toString() + ); + done(); + } + ); + }); + + it('GOOGLE_STANDARD_SQL should write null in the protoEnum column', done => { + insert( + {ProtoEnumValue: null}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.equal(row.toJSON().ProtoEnumValue, null); + done(); + } + ); + }); + + it('GOOGLE_STANDARD_SQL should write protoEnumArray', done => { + const value = Spanner.protoEnum(enumParams); + insert( + {ProtoEnumArray: [value]}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.deepStrictEqual(row.toJSON().ProtoEnumArray, [ + Object.getPrototypeOf(music.Genre)[enumParams.value], + ]); + done(); + }, + {ProtoEnumArray: music.Genre} + ); + }); + + it('GOOGLE_STANDARD_SQL should write int array in the protoEnumArray column', done => { + const value = 3; + insert( + {ProtoEnumArray: [value]}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.deepStrictEqual(row.toJSON().ProtoEnumArray, [ + value.toString(), + ]); + done(); + } + ); + }); + + it('GOOGLE_STANDARD_SQL should write null in the protoEnumArray column', done => { + insert( + {ProtoEnumArray: null}, + Spanner.GOOGLE_STANDARD_SQL, + (err, row) => { + assert.ifError(err); + assert.equal(row.toJSON().ProtoEnumArray, null); + done(); + } + ); + }); + }); + describe('jsonb', () => { before(async function () { if (IS_EMULATOR_ENABLED) { diff --git a/test/codec.ts b/test/codec.ts index 73b9f2132..9a925be86 100644 --- a/test/codec.ts +++ b/test/codec.ts @@ -22,6 +22,12 @@ import {Big} from 'big.js'; import {PreciseDate} from '@google-cloud/precise-date'; import {GrpcService} from '../src/common-grpc/service'; import {google} from '../protos/protos'; +import {GoogleError} from 'google-gax'; +import {util} from 'protobufjs'; +import Long = util.Long; +const singer = require('./data/singer'); +const music = singer.examples.spanner.music; +const is = require('is'); describe('codec', () => { let codec; @@ -302,6 +308,100 @@ describe('codec', () => { }); }); + describe('ProtoMessage', () => { + const protoMessageParams = { + value: music.SingerInfo.create({ + singerId: new Long(1), + genre: music.Genre.POP, + birthDate: 'January', + nationality: 'Country1', + }), + messageFunction: music.SingerInfo, + fullName: 'examples.spanner.music.SingerInfo', + }; + + it('should store value as buffer', () => { + const protoMessage = new codec.ProtoMessage(protoMessageParams); + assert(Buffer.isBuffer(protoMessage.value)); + }); + + it('should throw an error when value is not object and protoMessage is not passed', () => { + assert.throws( + () => { + new codec.ProtoMessage({ + value: { + singerId: 1, + genre: music.Genre.POP, + birthDate: 'January', + }, + fullName: 'examples.spanner.music.SingerInfo', + }); + }, + new GoogleError(`protoMessageParams cannot be used to construct + the ProtoMessage. Pass the serialized buffer of the + proto message as the value or provide the message object along with the + corresponding messageFunction generated by protobufjs-cli.`) + ); + }); + + it('toJSON with messageFunction', () => { + assert.deepEqual( + new codec.ProtoMessage(protoMessageParams).toJSON(), + music.SingerInfo.toObject(protoMessageParams.value) + ); + }); + + it('toJSON without messageFunction', () => { + const message = new codec.ProtoMessage({ + value: music.SingerInfo.encode(protoMessageParams.value).finish(), + fullName: 'examples.spanner.music.SingerInfo', + }); + assert.deepEqual(message.toJSON(), message.value.toString()); + }); + }); + + describe('ProtoEnum', () => { + const enumParams = { + value: music.Genre.JAZZ, + enumObject: music.Genre, + fullName: 'examples.spanner.music.Genre', + }; + + it('should store value as string', () => { + const protoEnum = new codec.ProtoEnum(enumParams); + assert(is.string(protoEnum.value)); + }); + + it('should throw an error when value is non numeric string and enumObject is not passed', () => { + assert.throws( + () => { + new codec.ProtoEnum({ + value: 'POP', + fullName: 'examples.spanner.music.Genre', + }); + }, + new GoogleError(`protoEnumParams cannot be used for constructing the + ProtoEnum. Pass the number as the value or provide the enum string + constant as the value along with the corresponding enumObject generated + by protobufjs-cli.`) + ); + }); + + it('toJSON with enumObject', () => { + assert.deepEqual(new codec.ProtoEnum(enumParams).toJSON(), 'JAZZ'); + }); + + it('toJSON without enumObject', () => { + assert.deepEqual( + new codec.ProtoEnum({ + value: music.Genre.JAZZ, + fullName: 'examples.spanner.music.Genre', + }).toJSON(), + 1 + ); + }); + }); + describe('Struct', () => { describe('toJSON', () => { it('should covert the struct to JSON', () => { @@ -544,6 +644,42 @@ describe('codec', () => { assert.deepStrictEqual(decoded, expected); }); + it('should decode ProtoMessage', () => { + const expected = new codec.ProtoMessage({ + value: music.SingerInfo.create({ + singerId: 1, + genre: music.Genre.POP, + birthDate: 'January', + nationality: 'Country1', + }), + messageFunction: music.SingerInfo, + fullName: 'examples.spanner.music.SingerInfo', + }); + const encoded = expected.value.toString('base64'); + + const decoded = codec.decode( + encoded, + { + code: google.spanner.v1.TypeCode.PROTO, + protoTypeFqn: 'examples.spanner.music.SingerInfo', + }, + music.SingerInfo + ); + + assert.deepStrictEqual(decoded, expected); + }); + + it('should decode ProtoEnum', () => { + const expected = Buffer.from('bytes value'); + const encoded = expected.toString('base64'); + + const decoded = codec.decode(encoded, { + code: google.spanner.v1.TypeCode.BYTES, + }); + + assert.deepStrictEqual(decoded, expected); + }); + it.skip('should decode FLOAT32', () => { const value = 'Infinity'; @@ -804,6 +940,42 @@ describe('codec', () => { assert.strictEqual(encoded, value.toString('base64')); }); + it('should encode ProtoMessage', () => { + const genre = music.Genre.ROCK; + const singerInfo = music.SingerInfo.create({ + singerId: 1, + genre: genre, + birthDate: 'January', + nationality: 'Country1', + }); + + const protoMessage = new codec.ProtoMessage({ + value: singerInfo, + messageFunction: music.SingerInfo, + fullName: 'examples.spanner.music.SingerInfo', + }); + + const encoded = codec.encode(protoMessage); + + assert.strictEqual( + encoded, + music.SingerInfo.encode(singerInfo).finish().toString('base64') + ); + }); + + it('should encode ProtoEnum', () => { + const genre = music.Genre.ROCK; + const protoEnum = new codec.ProtoEnum({ + value: genre, + enumObject: music.Genre, + fullName: 'examples.spanner.music.Genre', + }); + + const encoded = codec.encode(protoEnum); + + assert.strictEqual(encoded, genre.toString()); + }); + it('should encode structs', () => { const value = codec.Struct.fromJSON({a: 'b', c: 'd'}); const encoded = codec.encode(value); diff --git a/test/data/descriptors.pb b/test/data/descriptors.pb new file mode 100644 index 0000000000000000000000000000000000000000..d4c018f3a3c21b18f68820eeab130d8195064e81 GIT binary patch literal 251 zcmd=3!N|o^oSB!NTBKJ{lwXoBB$ir{m|KvOTC7)GkeHVT6wfU!&P-OC&&b6U3|8ow zmzFOi&BY1P7N40S!KlEf!5qW^5%5eAlI7w`$}B3$h)+o@NtIv%%5nyAf<;__0zwL0 z+>> 3) { + case 1: { + message.singerId = reader.int64(); + break; + } + case 2: { + message.birthDate = reader.string(); + break; + } + case 3: { + message.nationality = reader.string(); + break; + } + case 4: { + message.genre = reader.int32(); + break; + } + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Decodes a SingerInfo message from the specified reader or buffer, length delimited. + * @function decodeDelimited + * @memberof examples.spanner.music.SingerInfo + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @returns {examples.spanner.music.SingerInfo} SingerInfo + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + SingerInfo.decodeDelimited = function decodeDelimited(reader) { + if (!(reader instanceof $Reader)) reader = new $Reader(reader); + return this.decode(reader, reader.uint32()); + }; + + /** + * Verifies a SingerInfo message. + * @function verify + * @memberof examples.spanner.music.SingerInfo + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + SingerInfo.verify = function verify(message) { + if (typeof message !== 'object' || message === null) + return 'object expected'; + if (message.singerId !== null && message.hasOwnProperty('singerId')) + if ( + !$util.isInteger(message.singerId) && + !( + message.singerId && + $util.isInteger(message.singerId.low) && + $util.isInteger(message.singerId.high) + ) + ) + return 'singerId: integer|Long expected'; + if (message.birthDate !== null && message.hasOwnProperty('birthDate')) + if (!$util.isString(message.birthDate)) + return 'birthDate: string expected'; + if ( + message.nationality !== null && + message.hasOwnProperty('nationality') + ) + if (!$util.isString(message.nationality)) + return 'nationality: string expected'; + if (message.genre !== null && message.hasOwnProperty('genre')) + switch (message.genre) { + default: + return 'genre: enum value expected'; + case 0: + case 1: + case 2: + case 3: + break; + } + return null; + }; + + /** + * Creates a SingerInfo message from a plain object. Also converts values to their respective internal types. + * @function fromObject + * @memberof examples.spanner.music.SingerInfo + * @static + * @param {Object.} object Plain object + * @returns {examples.spanner.music.SingerInfo} SingerInfo + */ + SingerInfo.fromObject = function fromObject(object) { + if (object instanceof $root.examples.spanner.music.SingerInfo) + return object; + var message = new $root.examples.spanner.music.SingerInfo(); + if (object.singerId !== null) + if ($util.Long) + (message.singerId = $util.Long.fromValue( + object.singerId + )).unsigned = false; + else if (typeof object.singerId === 'string') + message.singerId = parseInt(object.singerId, 10); + else if (typeof object.singerId === 'number') + message.singerId = object.singerId; + else if (typeof object.singerId === 'object') + message.singerId = new $util.LongBits( + object.singerId.low >>> 0, + object.singerId.high >>> 0 + ).toNumber(); + if (object.birthDate !== null) + message.birthDate = String(object.birthDate); + if (object.nationality !== null) + message.nationality = String(object.nationality); + switch (object.genre) { + default: + if (typeof object.genre === 'number') { + message.genre = object.genre; + break; + } + break; + case 'POP': + case 0: + message.genre = 0; + break; + case 'JAZZ': + case 1: + message.genre = 1; + break; + case 'FOLK': + case 2: + message.genre = 2; + break; + case 'ROCK': + case 3: + message.genre = 3; + break; + } + return message; + }; + + /** + * Creates a plain object from a SingerInfo message. Also converts values to other types if specified. + * @function toObject + * @memberof examples.spanner.music.SingerInfo + * @static + * @param {examples.spanner.music.SingerInfo} message SingerInfo + * @param {$protobuf.IConversionOptions} [options] Conversion options + * @returns {Object.} Plain object + */ + SingerInfo.toObject = function toObject(message, options) { + if (!options) options = {}; + var object = {}; + if (options.defaults) { + if ($util.Long) { + var long = new $util.Long(0, 0, false); + object.singerId = + options.longs === String + ? long.toString() + : options.longs === Number + ? long.toNumber() + : long; + } else object.singerId = options.longs === String ? '0' : 0; + object.birthDate = ''; + object.nationality = ''; + object.genre = options.enums === String ? 'POP' : 0; + } + if (message.singerId !== null && message.hasOwnProperty('singerId')) + if (typeof message.singerId === 'number') + object.singerId = + options.longs === String + ? String(message.singerId) + : message.singerId; + else + object.singerId = + options.longs === String + ? $util.Long.prototype.toString.call(message.singerId) + : options.longs === Number + ? new $util.LongBits( + message.singerId.low >>> 0, + message.singerId.high >>> 0 + ).toNumber() + : message.singerId; + if (message.birthDate !== null && message.hasOwnProperty('birthDate')) + object.birthDate = message.birthDate; + if ( + message.nationality !== null && + message.hasOwnProperty('nationality') + ) + object.nationality = message.nationality; + if (message.genre !== null && message.hasOwnProperty('genre')) + object.genre = + options.enums === String + ? $root.examples.spanner.music.Genre[message.genre] === + undefined + ? message.genre + : $root.examples.spanner.music.Genre[message.genre] + : message.genre; + return object; + }; + + /** + * Converts this SingerInfo to JSON. + * @function toJSON + * @memberof examples.spanner.music.SingerInfo + * @instance + * @returns {Object.} JSON object + */ + SingerInfo.prototype.toJSON = function toJSON() { + return this.constructor.toObject(this, $protobuf.util.toJSONOptions); + }; + + /** + * Gets the default type url for SingerInfo + * @function getTypeUrl + * @memberof examples.spanner.music.SingerInfo + * @static + * @param {string} [typeUrlPrefix] your custom typeUrlPrefix(default "type.googleapis.com") + * @returns {string} The default type url + */ + SingerInfo.getTypeUrl = function getTypeUrl(typeUrlPrefix) { + if (typeUrlPrefix === undefined) { + typeUrlPrefix = 'type.googleapis.com'; + } + return typeUrlPrefix + '/examples.spanner.music.SingerInfo'; + }; + + return SingerInfo; + })(); + + /** + * Genre enum. + * @name examples.spanner.music.Genre + * @enum {number} + * @property {number} POP=0 POP value + * @property {number} JAZZ=1 JAZZ value + * @property {number} FOLK=2 FOLK value + * @property {number} ROCK=3 ROCK value + */ + music.Genre = (function () { + var valuesById = {}, + values = Object.create(valuesById); + values[(valuesById[0] = 'POP')] = 0; + values[(valuesById[1] = 'JAZZ')] = 1; + values[(valuesById[2] = 'FOLK')] = 2; + values[(valuesById[3] = 'ROCK')] = 3; + return values; + })(); + + return music; + })(); + + return spanner; + })(); + + return examples; +})(); + +module.exports = $root; diff --git a/test/data/singer.proto b/test/data/singer.proto new file mode 100644 index 000000000..d4e82bfc7 --- /dev/null +++ b/test/data/singer.proto @@ -0,0 +1,31 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package examples.spanner.music; + +message SingerInfo { + optional int64 singer_id = 1; + optional string birth_date = 2; + optional string nationality = 3; + optional Genre genre = 4; +} + +enum Genre { + POP = 0; + JAZZ = 1; + FOLK = 2; + ROCK = 3; +} diff --git a/test/index.ts b/test/index.ts index dd2b4b7b9..636bb845e 100644 --- a/test/index.ts +++ b/test/index.ts @@ -38,6 +38,8 @@ import { GetInstancesOptions, } from '../src'; import {CLOUD_RESOURCE_HEADER} from '../src/common'; +const singer = require('./data/singer'); +const music = singer.examples.spanner.music; // Verify that CLOUD_RESOURCE_HEADER is set to a correct value. assert.strictEqual(CLOUD_RESOURCE_HEADER, 'google-cloud-resource-prefix'); @@ -619,6 +621,66 @@ describe('Spanner', () => { }); }); + describe('protoMessage', () => { + it('should create a ProtoMessage instance', () => { + const protoMessageParams = { + value: music.SingerInfo.create({ + singerId: 2, + genre: music.Genre.POP, + birthDate: 'January', + }), + messageFunction: music.SingerInfo, + fullName: 'examples.spanner.music.SingerInfo', + }; + + const customValue = { + value: { + singerId: 2, + genre: music.Genre.POP, + birthDate: 'January', + }, + messageFunction: music.SingerInfo, + fullName: 'examples.spanner.music.SingerInfo', + }; + + fakeCodec.ProtoMessage = class { + constructor(value_) { + assert.strictEqual(value_, protoMessageParams); + return customValue; + } + }; + + const protoMessage = Spanner.protoMessage(protoMessageParams); + assert.strictEqual(protoMessage, customValue); + }); + }); + + describe('protoEnum', () => { + it('should create a ProtoEnum instance', () => { + const enumParams = { + value: music.Genre.JAZZ, + enumObject: music.Genre, + fullName: 'examples.spanner.music.Genre', + }; + + const customValue = { + value: music.Genre.JAZZ, + enumObject: music.Genre, + fullName: 'examples.spanner.music.Genre', + }; + + fakeCodec.ProtoEnum = class { + constructor(value_) { + assert.strictEqual(value_, enumParams); + return customValue; + } + }; + + const protoEnum = Spanner.protoEnum(enumParams); + assert.strictEqual(protoEnum, customValue); + }); + }); + describe('createInstance', () => { const NAME = 'instance-name'; let PATH; diff --git a/test/transaction.ts b/test/transaction.ts index 11a8647e0..392370ab7 100644 --- a/test/transaction.ts +++ b/test/transaction.ts @@ -395,6 +395,7 @@ describe('Transaction', () => { json: true, jsonOptions: {a: 'b'}, maxResumeRetries: 10, + columnsMetadata: {column1: {test: 'ss'}, column2: Function}, }; snapshot.createReadStream(TABLE, fakeOptions); @@ -769,6 +770,7 @@ describe('Transaction', () => { json: true, jsonOptions: {a: 'b'}, maxResumeRetries: 10, + columnsMetadata: {column1: {test: 'ss'}, column2: Function}, }; const fakeQuery = Object.assign({}, QUERY, expectedOptions);