From e368301b0fdce81054cd1283108e90c77b733239 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 18 May 2022 11:58:02 -0400 Subject: [PATCH 01/51] Getting grpc set up for tests --- test/grpc-mock.ts | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 test/grpc-mock.ts diff --git a/test/grpc-mock.ts b/test/grpc-mock.ts new file mode 100644 index 000000000..9ec0f1634 --- /dev/null +++ b/test/grpc-mock.ts @@ -0,0 +1,56 @@ +import {describe, it} from 'mocha'; +import {Bigtable} from '../src'; +import * as v2 from '../src/v2'; +import {PassThrough} from 'stream'; + +import * as protos from '../protos/protos'; +import jsonProtos = require('../protos/protos.json'); + +import grpc = require('@grpc/grpc-js'); +import protoLoader = require('@grpc/proto-loader'); +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +const packageDefinition = protoLoader.fromJSON(jsonProtos, { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true, +}); +const readRows = (arg1: any, arg2: any) => { + console.log(arg1, arg2); +}; +const proto = grpc.loadPackageDefinition(packageDefinition); +const server = new grpc.Server(); +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +const service = proto.google.bigtable.v2.Bigtable.service; +server.addService(service, { + ReadRows: readRows, +}); +server.bindAsync( + 'localhost:1234', + grpc.ServerCredentials.createInsecure(), + () => { + server.start(); + } +); + +describe('Bigtable/Grpc-mock', () => { + it('error should not change when emitted from gax', done => { + const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); + const clientConfig = 'BigtableClient'; + const gaxClient = new v2[clientConfig]({}); + const stream = new PassThrough({objectMode: true}); + gaxClient.innerApiCalls.readRows = () => { + return stream; + }; + const table = bigtable.instance('fake-instance').table('fake-table'); + const readStream = table.createReadStream({}); + readStream.on('error', err => { + console.log(err); + done(); + }); + console.log('test'); + }); +}); From eb94439115d774f3e38918fbf4b52e839a4b6f24 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 18 May 2022 17:42:58 -0400 Subject: [PATCH 02/51] test for error sent through gax --- test/grpc-mock.ts | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/test/grpc-mock.ts b/test/grpc-mock.ts index 9ec0f1634..34ea6e1ea 100644 --- a/test/grpc-mock.ts +++ b/test/grpc-mock.ts @@ -2,12 +2,13 @@ import {describe, it} from 'mocha'; import {Bigtable} from '../src'; import * as v2 from '../src/v2'; import {PassThrough} from 'stream'; +import * as assert from 'assert'; -import * as protos from '../protos/protos'; import jsonProtos = require('../protos/protos.json'); import grpc = require('@grpc/grpc-js'); import protoLoader = require('@grpc/proto-loader'); +import {GoogleError} from 'google-gax'; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore const packageDefinition = protoLoader.fromJSON(jsonProtos, { @@ -17,8 +18,26 @@ const packageDefinition = protoLoader.fromJSON(jsonProtos, { defaults: true, oneofs: true, }); -const readRows = (arg1: any, arg2: any) => { - console.log(arg1, arg2); +const errorDetails = + 'Table not found: projects/my-project/instances/my-instance/tables/my-table'; +const readRows = (call: any, example: any) => { + // const internalRepr = new Map(); + // internalRepr.set('grpc-server-stats-bin', [ + // Buffer.from([0, 0, 201, 39, 110, 3, 0, 0, 0, 0]), + // ]); + // call.emit('error', { + // code: 5, + // details: + // 'Table not found: projects/my-project/instances/my-instance/tables/my-table', + // metadata: { + // internalRepr, + // options: {}, + // }, + // }); + call.emit('error', { + code: 5, + details: errorDetails, + }); }; const proto = grpc.loadPackageDefinition(packageDefinition); const server = new grpc.Server(); @@ -37,7 +56,7 @@ server.bindAsync( ); describe('Bigtable/Grpc-mock', () => { - it('error should not change when emitted from gax', done => { + it('should produce human readable error when passing through gax', done => { const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); const clientConfig = 'BigtableClient'; const gaxClient = new v2[clientConfig]({}); @@ -47,10 +66,15 @@ describe('Bigtable/Grpc-mock', () => { }; const table = bigtable.instance('fake-instance').table('fake-table'); const readStream = table.createReadStream({}); - readStream.on('error', err => { - console.log(err); + readStream.on('error', (err: GoogleError) => { + const {code, statusDetails, message} = err; + assert.strictEqual(statusDetails, errorDetails); + assert.strictEqual(code, 5); + assert.strictEqual( + message, + '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' + ); done(); }); - console.log('test'); }); }); From 95a281cd0a3c6b26628708a2840bb2d783fa455f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 19 May 2022 14:03:08 -0400 Subject: [PATCH 03/51] Trying other things --- test/grpc-mock.ts | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/test/grpc-mock.ts b/test/grpc-mock.ts index 34ea6e1ea..84b4324d5 100644 --- a/test/grpc-mock.ts +++ b/test/grpc-mock.ts @@ -21,10 +21,23 @@ const packageDefinition = protoLoader.fromJSON(jsonProtos, { const errorDetails = 'Table not found: projects/my-project/instances/my-instance/tables/my-table'; const readRows = (call: any, example: any) => { + // const metadata = new grpc.Metadata(); + // metadata.set( + // 'grpc-server-stats-bin',Buffer.from('AAKENLPQKNSALSDFJ') + // Buffer.from([0, 0, 201, 39, 110, 3, 0, 0, 0, 0]).toString() + // ); + const metadata = new grpc.Metadata(); + metadata.set('grpc-server-stats-bin', Buffer.from('AAKENLPQKNSALSDFJ')); // const internalRepr = new Map(); // internalRepr.set('grpc-server-stats-bin', [ // Buffer.from([0, 0, 201, 39, 110, 3, 0, 0, 0, 0]), // ]); + call.emit('error', { + code: 5, + details: + 'Table not found: projects/my-project/instances/my-instance/tables/my-table', + metadata, + }); // call.emit('error', { // code: 5, // details: @@ -34,10 +47,10 @@ const readRows = (call: any, example: any) => { // options: {}, // }, // }); - call.emit('error', { - code: 5, - details: errorDetails, - }); + // call.emit('error', { + // code: 5, + // details: errorDetails, + // }); }; const proto = grpc.loadPackageDefinition(packageDefinition); const server = new grpc.Server(); From cdf81494e8ebbe6896420f071a9d8146e1654fed Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 19 May 2022 14:15:53 -0400 Subject: [PATCH 04/51] Group everything into describe blocks. --- test/grpc-mock.ts | 98 ++++++++++++++++++----------------------------- 1 file changed, 37 insertions(+), 61 deletions(-) diff --git a/test/grpc-mock.ts b/test/grpc-mock.ts index 84b4324d5..b10eceb5e 100644 --- a/test/grpc-mock.ts +++ b/test/grpc-mock.ts @@ -1,74 +1,50 @@ -import {describe, it} from 'mocha'; +import {before, describe, it} from 'mocha'; import {Bigtable} from '../src'; import * as v2 from '../src/v2'; import {PassThrough} from 'stream'; import * as assert from 'assert'; - import jsonProtos = require('../protos/protos.json'); - import grpc = require('@grpc/grpc-js'); import protoLoader = require('@grpc/proto-loader'); import {GoogleError} from 'google-gax'; -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore -const packageDefinition = protoLoader.fromJSON(jsonProtos, { - keepCase: true, - longs: String, - enums: String, - defaults: true, - oneofs: true, -}); -const errorDetails = - 'Table not found: projects/my-project/instances/my-instance/tables/my-table'; -const readRows = (call: any, example: any) => { - // const metadata = new grpc.Metadata(); - // metadata.set( - // 'grpc-server-stats-bin',Buffer.from('AAKENLPQKNSALSDFJ') - // Buffer.from([0, 0, 201, 39, 110, 3, 0, 0, 0, 0]).toString() - // ); - const metadata = new grpc.Metadata(); - metadata.set('grpc-server-stats-bin', Buffer.from('AAKENLPQKNSALSDFJ')); - // const internalRepr = new Map(); - // internalRepr.set('grpc-server-stats-bin', [ - // Buffer.from([0, 0, 201, 39, 110, 3, 0, 0, 0, 0]), - // ]); - call.emit('error', { - code: 5, - details: - 'Table not found: projects/my-project/instances/my-instance/tables/my-table', - metadata, - }); - // call.emit('error', { - // code: 5, - // details: - // 'Table not found: projects/my-project/instances/my-instance/tables/my-table', - // metadata: { - // internalRepr, - // options: {}, - // }, - // }); - // call.emit('error', { - // code: 5, - // details: errorDetails, - // }); -}; -const proto = grpc.loadPackageDefinition(packageDefinition); -const server = new grpc.Server(); -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore -const service = proto.google.bigtable.v2.Bigtable.service; -server.addService(service, { - ReadRows: readRows, -}); -server.bindAsync( - 'localhost:1234', - grpc.ServerCredentials.createInsecure(), - () => { - server.start(); - } -); describe('Bigtable/Grpc-mock', () => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const packageDefinition = protoLoader.fromJSON(jsonProtos, { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true, + }); + const errorDetails = + 'Table not found: projects/my-project/instances/my-instance/tables/my-table'; + const readRows = (stream: any) => { + stream.emit('error', { + code: 5, + details: errorDetails, + }); + }; + const proto = grpc.loadPackageDefinition(packageDefinition); + const server = new grpc.Server(); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const service = proto.google.bigtable.v2.Bigtable.service; + + before(async () => { + server.addService(service, { + ReadRows: readRows, + }); + server.bindAsync( + 'localhost:1234', + grpc.ServerCredentials.createInsecure(), + () => { + server.start(); + } + ); + }); + it('should produce human readable error when passing through gax', done => { const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); const clientConfig = 'BigtableClient'; From 091b08cb58df8126433c9fe4aba226014543fb64 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 19 May 2022 14:48:06 -0400 Subject: [PATCH 05/51] Add Google header --- test/grpc-mock.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/grpc-mock.ts b/test/grpc-mock.ts index b10eceb5e..8c448a862 100644 --- a/test/grpc-mock.ts +++ b/test/grpc-mock.ts @@ -1,3 +1,21 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + import {before, describe, it} from 'mocha'; import {Bigtable} from '../src'; import * as v2 from '../src/v2'; From 7d5fa9c11e906ca334415a35ae8533715ebeb290 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 20 May 2022 11:29:30 -0400 Subject: [PATCH 06/51] Added more tests --- test/grpc-mock.ts | 152 +++++++++++++++++++++++++++++++++------------- 1 file changed, 109 insertions(+), 43 deletions(-) diff --git a/test/grpc-mock.ts b/test/grpc-mock.ts index 8c448a862..046fc8490 100644 --- a/test/grpc-mock.ts +++ b/test/grpc-mock.ts @@ -18,8 +18,6 @@ import {before, describe, it} from 'mocha'; import {Bigtable} from '../src'; -import * as v2 from '../src/v2'; -import {PassThrough} from 'stream'; import * as assert from 'assert'; import jsonProtos = require('../protos/protos.json'); import grpc = require('@grpc/grpc-js'); @@ -36,52 +34,120 @@ describe('Bigtable/Grpc-mock', () => { defaults: true, oneofs: true, }); - const errorDetails = - 'Table not found: projects/my-project/instances/my-instance/tables/my-table'; - const readRows = (stream: any) => { - stream.emit('error', { - code: 5, - details: errorDetails, - }); - }; const proto = grpc.loadPackageDefinition(packageDefinition); const server = new grpc.Server(); // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore const service = proto.google.bigtable.v2.Bigtable.service; - - before(async () => { - server.addService(service, { - ReadRows: readRows, - }); - server.bindAsync( - 'localhost:1234', - grpc.ServerCredentials.createInsecure(), - () => { - server.start(); - } - ); - }); - - it('should produce human readable error when passing through gax', done => { - const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); - const clientConfig = 'BigtableClient'; - const gaxClient = new v2[clientConfig]({}); - const stream = new PassThrough({objectMode: true}); - gaxClient.innerApiCalls.readRows = () => { - return stream; - }; - const table = bigtable.instance('fake-instance').table('fake-table'); - const readStream = table.createReadStream({}); - readStream.on('error', (err: GoogleError) => { - const {code, statusDetails, message} = err; - assert.strictEqual(statusDetails, errorDetails); - assert.strictEqual(code, 5); - assert.strictEqual( - message, - '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' - ); - done(); + server.bindAsync( + 'localhost:1234', + grpc.ServerCredentials.createInsecure(), + () => { + server.start(); + } + ); + describe('with the bigtable data client', () => { + describe('sends errors through a streaming request', () => { + const errorDetails = + 'Table not found: projects/my-project/instances/my-instance/tables/my-table'; + const emitTableNotExistsError = (stream: any) => { + stream.emit('error', { + code: 5, + details: errorDetails, + }); + }; + describe('with ReadRows service', () => { + before(async () => { + server.addService(service, { + ReadRows: emitTableNotExistsError, + }); + }); + after(async () => { + server.removeService(service); + }); + it('should produce human readable error when passing through gax', done => { + const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); + const table = bigtable.instance('fake-instance').table('fake-table'); + const readStream = table.createReadStream({}); + readStream.on('error', (err: GoogleError) => { + const {code, statusDetails, message} = err; + assert.strictEqual(statusDetails, errorDetails); + assert.strictEqual(code, 5); + assert.strictEqual( + message, + '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' + ); + done(); + }); + }); + }); + describe('with mutateRows service through insert', () => { + before(async () => { + server.addService(service, { + mutateRows: emitTableNotExistsError, + }); + }); + after(async () => { + server.removeService(service); + }); + it('should produce human readable error when passing through gax', async () => { + const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); + const table = bigtable.instance('fake-instance').table('fake-table'); + const timestamp = new Date(); + const rowsToInsert = [ + { + key: 'r2', + data: { + cf1: { + c1: { + value: 'test-value2', + labels: [], + timestamp, + }, + }, + }, + }, + ]; + try { + await table.insert(rowsToInsert); + assert.fail(); + } catch (err) { + const {code, statusDetails, message} = err; + assert.strictEqual(statusDetails, errorDetails); + assert.strictEqual(code, 5); + assert.strictEqual( + message, + '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' + ); + } + }); + }); + describe('with sampleRowKeys', () => { + before(async () => { + server.addService(service, { + sampleRowKeys: emitTableNotExistsError, + }); + }); + after(async () => { + server.removeService(service); + }); + it('should produce human readable error when passing through gax', async () => { + const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); + const table = bigtable.instance('fake-instance').table('fake-table'); + try { + await table.sampleRowKeys({}); + assert.fail(); + } catch (err) { + const {code, statusDetails, message} = err; + assert.strictEqual(statusDetails, errorDetails); + assert.strictEqual(code, 5); + assert.strictEqual( + message, + '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' + ); + } + }); + }); }); }); }); From 16989020a91f6bfec012428ada3f13724a2fd165 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 20 May 2022 16:20:40 -0400 Subject: [PATCH 07/51] Create mock service files --- src/util/mock-servers/mock-server.ts | 56 +++++++++++++++++++ src/util/mock-servers/mock-service.ts | 34 +++++++++++ .../bigtable-client-mock-service.ts | 38 +++++++++++++ test/grpc-mock.ts | 50 +++++------------ 4 files changed, 142 insertions(+), 36 deletions(-) create mode 100644 src/util/mock-servers/mock-server.ts create mode 100644 src/util/mock-servers/mock-service.ts create mode 100644 src/util/mock-servers/service-implementations/bigtable-client-mock-service.ts diff --git a/src/util/mock-servers/mock-server.ts b/src/util/mock-servers/mock-server.ts new file mode 100644 index 000000000..d143759e5 --- /dev/null +++ b/src/util/mock-servers/mock-server.ts @@ -0,0 +1,56 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + +import {grpc} from 'google-gax'; + +const DEFAULT_PORT = 1234; + +export class MockServer { + port: number; + services: Set = new Set(); + server: grpc.Server; + + constructor(port?: string | number | undefined) { + this.port = port ? Number(port) : DEFAULT_PORT; + const server = new grpc.Server(); + server.bindAsync( + `localhost:${this.port.toString()}`, + grpc.ServerCredentials.createInsecure(), + () => { + server.start(); + } + ); + this.server = server; + } + + getPort(): number { + return this.port; + } + + setService( + service: grpc.ServiceDefinition, + implementation: grpc.UntypedServiceImplementation + ) { + if (this.services.has(service)) { + this.server.removeService(service); + } else { + this.services.add(service); + } + this.server.addService(service, implementation); + } +} diff --git a/src/util/mock-servers/mock-service.ts b/src/util/mock-servers/mock-service.ts new file mode 100644 index 000000000..37c25d433 --- /dev/null +++ b/src/util/mock-servers/mock-service.ts @@ -0,0 +1,34 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + +import {grpc} from 'google-gax'; + +import {MockServer} from './mock-server'; + +export abstract class MockService { + abstract service: grpc.ServiceDefinition; + server: MockServer; + + constructor(server: MockServer) { + this.server = server; + } + + setService(implementation: grpc.UntypedServiceImplementation) { + this.server.setService(this.service, implementation); + } +} diff --git a/src/util/mock-servers/service-implementations/bigtable-client-mock-service.ts b/src/util/mock-servers/service-implementations/bigtable-client-mock-service.ts new file mode 100644 index 000000000..614593252 --- /dev/null +++ b/src/util/mock-servers/service-implementations/bigtable-client-mock-service.ts @@ -0,0 +1,38 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + +import grpc = require('@grpc/grpc-js'); +import jsonProtos = require('../../../../protos/protos.json'); +import protoLoader = require('@grpc/proto-loader'); +import {MockService} from '../mock-service'; + +const packageDefinition = protoLoader.fromJSON(jsonProtos, { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true, +}); +const proto = grpc.loadPackageDefinition(packageDefinition); + +export class BigtableClientMockService extends MockService { + service = + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + proto.google.bigtable.v2.Bigtable.service; +} diff --git a/test/grpc-mock.ts b/test/grpc-mock.ts index 046fc8490..1baae9aa4 100644 --- a/test/grpc-mock.ts +++ b/test/grpc-mock.ts @@ -19,38 +19,25 @@ import {before, describe, it} from 'mocha'; import {Bigtable} from '../src'; import * as assert from 'assert'; -import jsonProtos = require('../protos/protos.json'); -import grpc = require('@grpc/grpc-js'); -import protoLoader = require('@grpc/proto-loader'); + import {GoogleError} from 'google-gax'; +import {MockServer} from '../src/util/mock-servers/mock-server'; +import {BigtableClientMockService} from '../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; +import {MockService} from '../src/util/mock-servers/mock-service'; + +// TODO: Test server shuts down describe('Bigtable/Grpc-mock', () => { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const packageDefinition = protoLoader.fromJSON(jsonProtos, { - keepCase: true, - longs: String, - enums: String, - defaults: true, - oneofs: true, - }); - const proto = grpc.loadPackageDefinition(packageDefinition); - const server = new grpc.Server(); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const service = proto.google.bigtable.v2.Bigtable.service; - server.bindAsync( - 'localhost:1234', - grpc.ServerCredentials.createInsecure(), - () => { - server.start(); - } - ); + const server: MockServer = new MockServer(); + describe('with the bigtable data client', () => { + const service: MockService = new BigtableClientMockService(server); + describe('sends errors through a streaming request', () => { const errorDetails = 'Table not found: projects/my-project/instances/my-instance/tables/my-table'; const emitTableNotExistsError = (stream: any) => { + // TODO: Replace stream with type stream.emit('error', { code: 5, details: errorDetails, @@ -58,13 +45,10 @@ describe('Bigtable/Grpc-mock', () => { }; describe('with ReadRows service', () => { before(async () => { - server.addService(service, { + service.setService({ ReadRows: emitTableNotExistsError, }); }); - after(async () => { - server.removeService(service); - }); it('should produce human readable error when passing through gax', done => { const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); const table = bigtable.instance('fake-instance').table('fake-table'); @@ -83,13 +67,10 @@ describe('Bigtable/Grpc-mock', () => { }); describe('with mutateRows service through insert', () => { before(async () => { - server.addService(service, { + service.setService({ mutateRows: emitTableNotExistsError, }); }); - after(async () => { - server.removeService(service); - }); it('should produce human readable error when passing through gax', async () => { const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); const table = bigtable.instance('fake-instance').table('fake-table'); @@ -124,13 +105,10 @@ describe('Bigtable/Grpc-mock', () => { }); describe('with sampleRowKeys', () => { before(async () => { - server.addService(service, { + service.setService({ sampleRowKeys: emitTableNotExistsError, }); }); - after(async () => { - server.removeService(service); - }); it('should produce human readable error when passing through gax', async () => { const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); const table = bigtable.instance('fake-instance').table('fake-table'); From 47eb8d885fc8260a8dd89fb3bbcea52239b1968c Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 20 May 2022 16:34:32 -0400 Subject: [PATCH 08/51] refactor a check --- src/util/mock-servers/mock-server.ts | 10 ++----- test/grpc-mock.ts | 45 ++++++++++------------------ 2 files changed, 19 insertions(+), 36 deletions(-) diff --git a/src/util/mock-servers/mock-server.ts b/src/util/mock-servers/mock-server.ts index d143759e5..16db56363 100644 --- a/src/util/mock-servers/mock-server.ts +++ b/src/util/mock-servers/mock-server.ts @@ -21,15 +21,15 @@ import {grpc} from 'google-gax'; const DEFAULT_PORT = 1234; export class MockServer { - port: number; + port: string; services: Set = new Set(); server: grpc.Server; constructor(port?: string | number | undefined) { - this.port = port ? Number(port) : DEFAULT_PORT; + this.port = Number(port ? port : DEFAULT_PORT).toString(); const server = new grpc.Server(); server.bindAsync( - `localhost:${this.port.toString()}`, + `localhost:${this.port}`, grpc.ServerCredentials.createInsecure(), () => { server.start(); @@ -38,10 +38,6 @@ export class MockServer { this.server = server; } - getPort(): number { - return this.port; - } - setService( service: grpc.ServiceDefinition, implementation: grpc.UntypedServiceImplementation diff --git a/test/grpc-mock.ts b/test/grpc-mock.ts index 1baae9aa4..e14aaaa24 100644 --- a/test/grpc-mock.ts +++ b/test/grpc-mock.ts @@ -29,10 +29,12 @@ import {MockService} from '../src/util/mock-servers/mock-service'; describe('Bigtable/Grpc-mock', () => { const server: MockServer = new MockServer(); - + const bigtable = new Bigtable({ + apiEndpoint: `localhost:${server.port}`, + }); + const table = bigtable.instance('fake-instance').table('fake-table'); describe('with the bigtable data client', () => { const service: MockService = new BigtableClientMockService(server); - describe('sends errors through a streaming request', () => { const errorDetails = 'Table not found: projects/my-project/instances/my-instance/tables/my-table'; @@ -43,6 +45,15 @@ describe('Bigtable/Grpc-mock', () => { details: errorDetails, }); }; + function checkTableNotExistError(err: GoogleError) { + const {code, statusDetails, message} = err; + assert.strictEqual(statusDetails, errorDetails); + assert.strictEqual(code, 5); + assert.strictEqual( + message, + '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' + ); + } describe('with ReadRows service', () => { before(async () => { service.setService({ @@ -50,17 +61,9 @@ describe('Bigtable/Grpc-mock', () => { }); }); it('should produce human readable error when passing through gax', done => { - const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); - const table = bigtable.instance('fake-instance').table('fake-table'); const readStream = table.createReadStream({}); readStream.on('error', (err: GoogleError) => { - const {code, statusDetails, message} = err; - assert.strictEqual(statusDetails, errorDetails); - assert.strictEqual(code, 5); - assert.strictEqual( - message, - '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' - ); + checkTableNotExistError(err); done(); }); }); @@ -72,8 +75,6 @@ describe('Bigtable/Grpc-mock', () => { }); }); it('should produce human readable error when passing through gax', async () => { - const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); - const table = bigtable.instance('fake-instance').table('fake-table'); const timestamp = new Date(); const rowsToInsert = [ { @@ -93,13 +94,7 @@ describe('Bigtable/Grpc-mock', () => { await table.insert(rowsToInsert); assert.fail(); } catch (err) { - const {code, statusDetails, message} = err; - assert.strictEqual(statusDetails, errorDetails); - assert.strictEqual(code, 5); - assert.strictEqual( - message, - '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' - ); + checkTableNotExistError(err); } }); }); @@ -110,19 +105,11 @@ describe('Bigtable/Grpc-mock', () => { }); }); it('should produce human readable error when passing through gax', async () => { - const bigtable = new Bigtable({apiEndpoint: 'localhost:1234'}); - const table = bigtable.instance('fake-instance').table('fake-table'); try { await table.sampleRowKeys({}); assert.fail(); } catch (err) { - const {code, statusDetails, message} = err; - assert.strictEqual(statusDetails, errorDetails); - assert.strictEqual(code, 5); - assert.strictEqual( - message, - '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' - ); + checkTableNotExistError(err); } }); }); From b9d82f82e249a3595d91c073d20d1383f08faecc Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 24 May 2022 17:22:09 -0400 Subject: [PATCH 09/51] mock server tests --- src/util/mock-servers/mock-server.ts | 17 +++++-- test/{grpc-mock.ts => errors.ts} | 29 +++++++++--- test/util/mock-server.ts | 69 ++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 10 deletions(-) rename test/{grpc-mock.ts => errors.ts} (87%) create mode 100644 test/util/mock-server.ts diff --git a/src/util/mock-servers/mock-server.ts b/src/util/mock-servers/mock-server.ts index 16db56363..c4ebb9779 100644 --- a/src/util/mock-servers/mock-server.ts +++ b/src/util/mock-servers/mock-server.ts @@ -25,17 +25,22 @@ export class MockServer { services: Set = new Set(); server: grpc.Server; - constructor(port?: string | number | undefined) { - this.port = Number(port ? port : DEFAULT_PORT).toString(); + constructor( + callback?: (port: string) => void, + port?: string | number | undefined + ) { + const portString = Number(port ? port : DEFAULT_PORT).toString(); + this.port = portString; const server = new grpc.Server(); + this.server = server; server.bindAsync( `localhost:${this.port}`, grpc.ServerCredentials.createInsecure(), () => { server.start(); + callback ? callback(portString) : undefined; } ); - this.server = server; } setService( @@ -49,4 +54,10 @@ export class MockServer { } this.server.addService(service, implementation); } + + shutdown(callback: (err?: Error) => void) { + this.server.tryShutdown((err?: Error) => { + callback(err); + }); + } } diff --git a/test/grpc-mock.ts b/test/errors.ts similarity index 87% rename from test/grpc-mock.ts rename to test/errors.ts index e14aaaa24..c08ae1090 100644 --- a/test/grpc-mock.ts +++ b/test/errors.ts @@ -25,16 +25,27 @@ import {MockServer} from '../src/util/mock-servers/mock-server'; import {BigtableClientMockService} from '../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; import {MockService} from '../src/util/mock-servers/mock-service'; -// TODO: Test server shuts down +describe('Bigtable/Errors', () => { + let server: MockServer; + let bigtable: Bigtable; + let table: any; -describe('Bigtable/Grpc-mock', () => { - const server: MockServer = new MockServer(); - const bigtable = new Bigtable({ - apiEndpoint: `localhost:${server.port}`, + before(done => { + server = new MockServer(() => { + bigtable = new Bigtable({ + apiEndpoint: `localhost:${server.port}`, + }); + table = bigtable.instance('fake-instance').table('fake-table'); + done(); + }); }); - const table = bigtable.instance('fake-instance').table('fake-table'); + describe('with the bigtable data client', () => { - const service: MockService = new BigtableClientMockService(server); + let service: MockService; + before(async () => { + service = new BigtableClientMockService(server); + }); + describe('sends errors through a streaming request', () => { const errorDetails = 'Table not found: projects/my-project/instances/my-instance/tables/my-table'; @@ -115,4 +126,8 @@ describe('Bigtable/Grpc-mock', () => { }); }); }); + + after(async () => { + server.shutdown(() => {}); + }); }); diff --git a/test/util/mock-server.ts b/test/util/mock-server.ts new file mode 100644 index 000000000..d1406b4c5 --- /dev/null +++ b/test/util/mock-server.ts @@ -0,0 +1,69 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + +import {describe, it} from 'mocha'; +import {MockServer} from '../../src/util/mock-servers/mock-server'; +import * as net from 'net'; +import * as assert from 'assert'; + +interface ServerError { + code: string; +} + +// TODO: Test server shuts down + +describe('Bigtable/Mock-Server', () => { + const inputPort = '1234'; + let server: MockServer; + function checkPort(port: string, inUse: boolean, callback: () => void) { + const netServer = net.createServer(); + netServer.once('error', (err: ServerError) => { + if (inUse) { + assert.strictEqual(err.code, 'EADDRINUSE'); + } else { + assert.notEqual(err.code, 'EADDRINUSE'); + } + netServer.close(); + callback(); + }); + netServer.once('listening', () => { + // close the server if listening doesn't fail + netServer.close(); + }); + // netServer.listen(port); + netServer.listen(`localhost:${port}`); + } + before(done => { + checkPort(inputPort, false, done); + }); + describe('Ensure server shuts down properly when destroyed', () => { + it('should start a mock server', done => { + server = new MockServer(port => { + checkPort(port, true, done); + }, inputPort); + }); + }); + after(done => { + checkPort(server.port, true, () => { + server.shutdown((err?: Error) => { + assert.deepStrictEqual(err, undefined); + checkPort(server.port, false, done); + }); + }); + }); +}); From 3256cdc6ca8cc85b076553880069b33c084f33ea Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 24 May 2022 17:24:25 -0400 Subject: [PATCH 10/51] undo tests --- test/errors.ts | 1 - test/util/mock-server.ts | 4 ---- 2 files changed, 5 deletions(-) diff --git a/test/errors.ts b/test/errors.ts index c08ae1090..e31130d80 100644 --- a/test/errors.ts +++ b/test/errors.ts @@ -126,7 +126,6 @@ describe('Bigtable/Errors', () => { }); }); }); - after(async () => { server.shutdown(() => {}); }); diff --git a/test/util/mock-server.ts b/test/util/mock-server.ts index d1406b4c5..1e0d8eab4 100644 --- a/test/util/mock-server.ts +++ b/test/util/mock-server.ts @@ -48,9 +48,6 @@ describe('Bigtable/Mock-Server', () => { // netServer.listen(port); netServer.listen(`localhost:${port}`); } - before(done => { - checkPort(inputPort, false, done); - }); describe('Ensure server shuts down properly when destroyed', () => { it('should start a mock server', done => { server = new MockServer(port => { @@ -62,7 +59,6 @@ describe('Bigtable/Mock-Server', () => { checkPort(server.port, true, () => { server.shutdown((err?: Error) => { assert.deepStrictEqual(err, undefined); - checkPort(server.port, false, done); }); }); }); From b25068a8f0823ac65b4b07d7a0c57df1b6bdb00a Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 25 May 2022 14:01:10 -0400 Subject: [PATCH 11/51] Pass metadata through --- test/errors.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/errors.ts b/test/errors.ts index e31130d80..b0a7568de 100644 --- a/test/errors.ts +++ b/test/errors.ts @@ -20,7 +20,7 @@ import {before, describe, it} from 'mocha'; import {Bigtable} from '../src'; import * as assert from 'assert'; -import {GoogleError} from 'google-gax'; +import {GoogleError, grpc} from 'google-gax'; import {MockServer} from '../src/util/mock-servers/mock-server'; import {BigtableClientMockService} from '../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; import {MockService} from '../src/util/mock-servers/mock-service'; @@ -51,9 +51,15 @@ describe('Bigtable/Errors', () => { 'Table not found: projects/my-project/instances/my-instance/tables/my-table'; const emitTableNotExistsError = (stream: any) => { // TODO: Replace stream with type + const metadata = new grpc.Metadata(); + metadata.set( + 'grpc-server-stats-bin', + Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]) + ); stream.emit('error', { code: 5, details: errorDetails, + metadata, }); }; function checkTableNotExistError(err: GoogleError) { From 6ef1bd4d34cd9d36bcf21ce4f2064c34aeb671a5 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 26 May 2022 15:06:58 -0400 Subject: [PATCH 12/51] build fix --- test/errors.ts | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/test/errors.ts b/test/errors.ts index b0a7568de..d64ab7b03 100644 --- a/test/errors.ts +++ b/test/errors.ts @@ -25,6 +25,13 @@ import {MockServer} from '../src/util/mock-servers/mock-server'; import {BigtableClientMockService} from '../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; import {MockService} from '../src/util/mock-servers/mock-service'; +function isGoogleError(error: any): error is GoogleError { + return ( + error.parseGRPCStatusDetails !== undefined && + error.parseHttpError !== undefined + ); +} + describe('Bigtable/Errors', () => { let server: MockServer; let bigtable: Bigtable; @@ -62,14 +69,20 @@ describe('Bigtable/Errors', () => { metadata, }); }; - function checkTableNotExistError(err: GoogleError) { - const {code, statusDetails, message} = err; - assert.strictEqual(statusDetails, errorDetails); - assert.strictEqual(code, 5); - assert.strictEqual( - message, - '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' - ); + function checkTableNotExistError(err: any) { + if (isGoogleError(err)) { + const {code, statusDetails, message} = err; + assert.strictEqual(statusDetails, errorDetails); + assert.strictEqual(code, 5); + assert.strictEqual( + message, + '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' + ); + } else { + assert.fail( + 'Errors checked using this function should all be GoogleErrors' + ); + } } describe('with ReadRows service', () => { before(async () => { @@ -109,10 +122,11 @@ describe('Bigtable/Errors', () => { ]; try { await table.insert(rowsToInsert); - assert.fail(); } catch (err) { checkTableNotExistError(err); + return; } + assert.fail('An error should have been thrown by the stream'); }); }); describe('with sampleRowKeys', () => { @@ -124,10 +138,11 @@ describe('Bigtable/Errors', () => { it('should produce human readable error when passing through gax', async () => { try { await table.sampleRowKeys({}); - assert.fail(); } catch (err) { checkTableNotExistError(err); + return; } + assert.fail('An error should have been thrown by the stream'); }); }); }); From 8ae117a699ea7851f9c0e010cb30e20478cde5ce Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 27 May 2022 15:07:40 -0400 Subject: [PATCH 13/51] work in progress --- test/errors.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/errors.ts b/test/errors.ts index d64ab7b03..bbbee50b7 100644 --- a/test/errors.ts +++ b/test/errors.ts @@ -71,13 +71,9 @@ describe('Bigtable/Errors', () => { }; function checkTableNotExistError(err: any) { if (isGoogleError(err)) { - const {code, statusDetails, message} = err; - assert.strictEqual(statusDetails, errorDetails); + const {code, message} = err; assert.strictEqual(code, 5); - assert.strictEqual( - message, - '5 NOT_FOUND: Table not found: projects/my-project/instances/my-instance/tables/my-table' - ); + assert.strictEqual(message, `5 NOT_FOUND: ${errorDetails}`); } else { assert.fail( 'Errors checked using this function should all be GoogleErrors' From e8cd776008d6eefda0ed4bdea1b5c1a0d79242b5 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 27 May 2022 16:37:25 -0400 Subject: [PATCH 14/51] Add service error check and change test --- test/errors.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/errors.ts b/test/errors.ts index bbbee50b7..33721b743 100644 --- a/test/errors.ts +++ b/test/errors.ts @@ -20,15 +20,16 @@ import {before, describe, it} from 'mocha'; import {Bigtable} from '../src'; import * as assert from 'assert'; -import {GoogleError, grpc} from 'google-gax'; +import {GoogleError, grpc, ServiceError} from 'google-gax'; import {MockServer} from '../src/util/mock-servers/mock-server'; import {BigtableClientMockService} from '../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; import {MockService} from '../src/util/mock-servers/mock-service'; -function isGoogleError(error: any): error is GoogleError { +function isServiceError(error: any): error is ServiceError { return ( - error.parseGRPCStatusDetails !== undefined && - error.parseHttpError !== undefined + error.code !== undefined && + error.details !== undefined && + error.metadata !== undefined ); } @@ -70,8 +71,9 @@ describe('Bigtable/Errors', () => { }); }; function checkTableNotExistError(err: any) { - if (isGoogleError(err)) { - const {code, message} = err; + if (isServiceError(err)) { + const {code, message, details} = err; + assert.strictEqual(details, errorDetails); assert.strictEqual(code, 5); assert.strictEqual(message, `5 NOT_FOUND: ${errorDetails}`); } else { From 148bd421ae2bd483526a10508255f570858d574c Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 31 May 2022 14:40:07 -0400 Subject: [PATCH 15/51] Remove TODO and add done hook. --- test/util/mock-server.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/util/mock-server.ts b/test/util/mock-server.ts index 1e0d8eab4..0d3b7ce7e 100644 --- a/test/util/mock-server.ts +++ b/test/util/mock-server.ts @@ -25,8 +25,6 @@ interface ServerError { code: string; } -// TODO: Test server shuts down - describe('Bigtable/Mock-Server', () => { const inputPort = '1234'; let server: MockServer; @@ -59,6 +57,7 @@ describe('Bigtable/Mock-Server', () => { checkPort(server.port, true, () => { server.shutdown((err?: Error) => { assert.deepStrictEqual(err, undefined); + done(); }); }); }); From d3d9fb6d8ac7a807699c3603fa4a9ff2b967bbe2 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 13 Jun 2022 13:19:48 -0400 Subject: [PATCH 16/51] Logs for debugging --- test/util/mock-server.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/util/mock-server.ts b/test/util/mock-server.ts index 0d3b7ce7e..f0083ad1c 100644 --- a/test/util/mock-server.ts +++ b/test/util/mock-server.ts @@ -30,7 +30,9 @@ describe('Bigtable/Mock-Server', () => { let server: MockServer; function checkPort(port: string, inUse: boolean, callback: () => void) { const netServer = net.createServer(); + console.log('Starting server'); netServer.once('error', (err: ServerError) => { + console.log('error'); if (inUse) { assert.strictEqual(err.code, 'EADDRINUSE'); } else { From 715339919fa9acb0f054b6dceda520ae9be50d59 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 13 Jun 2022 14:45:39 -0400 Subject: [PATCH 17/51] use the tcp-port-used library instead --- package.json | 1 + test/util/mock-server.ts | 25 ++++--------------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index 5238bccd4..660b9b536 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "pack-n-play": "^1.0.0-2", "proxyquire": "^2.0.0", "sinon": "^14.0.0", + "tcp-port-used": "^1.0.2", "ts-loader": "^9.0.0", "typescript": "^4.6.4", "uuid": "^8.0.0", diff --git a/test/util/mock-server.ts b/test/util/mock-server.ts index f0083ad1c..d1ef4474d 100644 --- a/test/util/mock-server.ts +++ b/test/util/mock-server.ts @@ -18,35 +18,18 @@ import {describe, it} from 'mocha'; import {MockServer} from '../../src/util/mock-servers/mock-server'; -import * as net from 'net'; import * as assert from 'assert'; -interface ServerError { - code: string; -} +const tcpPortUsed = require('tcp-port-used'); describe('Bigtable/Mock-Server', () => { const inputPort = '1234'; let server: MockServer; function checkPort(port: string, inUse: boolean, callback: () => void) { - const netServer = net.createServer(); - console.log('Starting server'); - netServer.once('error', (err: ServerError) => { - console.log('error'); - if (inUse) { - assert.strictEqual(err.code, 'EADDRINUSE'); - } else { - assert.notEqual(err.code, 'EADDRINUSE'); - } - netServer.close(); + tcpPortUsed.check(parseInt(port), 'localhost').then((isInUse: boolean) => { + assert.strictEqual(isInUse, inUse); callback(); }); - netServer.once('listening', () => { - // close the server if listening doesn't fail - netServer.close(); - }); - // netServer.listen(port); - netServer.listen(`localhost:${port}`); } describe('Ensure server shuts down properly when destroyed', () => { it('should start a mock server', done => { @@ -59,7 +42,7 @@ describe('Bigtable/Mock-Server', () => { checkPort(server.port, true, () => { server.shutdown((err?: Error) => { assert.deepStrictEqual(err, undefined); - done(); + checkPort(server.port, false, done); }); }); }); From 32478203f2a328e62043914e1cbc529253b50285 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 17 Jun 2022 17:29:01 -0400 Subject: [PATCH 18/51] use await --- test/util/mock-server.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/util/mock-server.ts b/test/util/mock-server.ts index d1ef4474d..920673579 100644 --- a/test/util/mock-server.ts +++ b/test/util/mock-server.ts @@ -25,11 +25,13 @@ const tcpPortUsed = require('tcp-port-used'); describe('Bigtable/Mock-Server', () => { const inputPort = '1234'; let server: MockServer; - function checkPort(port: string, inUse: boolean, callback: () => void) { - tcpPortUsed.check(parseInt(port), 'localhost').then((isInUse: boolean) => { - assert.strictEqual(isInUse, inUse); - callback(); - }); + async function checkPort(port: string, inUse: boolean, callback: () => void) { + const isInUse: boolean = await tcpPortUsed.check( + parseInt(port), + 'localhost' + ); + assert.strictEqual(isInUse, inUse); + callback(); } describe('Ensure server shuts down properly when destroyed', () => { it('should start a mock server', done => { From d52f50450582352af1018e436f1d56a1bab82ee3 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 27 Jun 2022 11:36:50 -0400 Subject: [PATCH 19/51] Set of tests for retriable errors --- __snapshots__/read-rows.js | 71 ++++++++++++++++++++++ test/server/read-rows.ts | 117 +++++++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 __snapshots__/read-rows.js create mode 100644 test/server/read-rows.ts diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js new file mode 100644 index 000000000..c9c884235 --- /dev/null +++ b/__snapshots__/read-rows.js @@ -0,0 +1,71 @@ +exports[ + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with retryable errors 1' +] = { + code: 14, + callCount: 4, + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, +}; + +exports[ + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with retryable errors 2' +] = { + code: 10, + callCount: 4, + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, +}; + +exports[ + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with retryable errors 3' +] = { + code: 8, + callCount: 4, + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, +}; + +exports[ + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with retryable errors 4' +] = { + code: 4, + callCount: 4, + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, +}; diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts new file mode 100644 index 000000000..7faace42b --- /dev/null +++ b/test/server/read-rows.ts @@ -0,0 +1,117 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + +import {before, describe, it} from 'mocha'; +import {Bigtable} from '../../src'; +import * as assert from 'assert'; + +import {GoogleError, grpc, ServiceError} from 'google-gax'; +import {MockServer} from '../../src/util/mock-servers/mock-server'; +import {BigtableClientMockService} from '../../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; +import {MockService} from '../../src/util/mock-servers/mock-service'; +const snapshot = require('snap-shot-it'); +import {check} from 'linkinator'; +import * as gax from 'google-gax'; + +function isServiceError(error: any): error is ServiceError { + return ( + error.code !== undefined && + error.details !== undefined && + error.metadata !== undefined + ); +} + +describe('Bigtable/ReadRows', () => { + let server: MockServer; + let service: MockService; + let bigtable: Bigtable; + let table: any; + + before(done => { + server = new MockServer(() => { + bigtable = new Bigtable({ + apiEndpoint: `localhost:${server.port}`, + }); + // TODO: Replace this with generated Ids so that we don't have flaky tests + table = bigtable.instance('fake-instance').table('fake-table'); + service = new BigtableClientMockService(server); + done(); + }); + }); + + describe('with a mock server that always sends an error back', () => { + // Define the standard call here + describe('where the error is retryable', () => { + function checkRequest(code: grpc.status, callback: () => void) { + const errorDetails = 'Details for a particular type of error'; + let request: any = null; + let callCount = 0; + const emitError = (stream: any) => { + const streamRequest = stream.request; + if (request) { + // This ensures that every call to the server is the same + assert.deepStrictEqual(request, streamRequest); + } else { + request = streamRequest; + } + callCount++; + stream.emit('error', { + code, + details: errorDetails, + }); + }; + service.setService({ + // Abstraction: Always emit error + ReadRows: emitError, + }); + table.createReadStream({}).on('error', (error: ServiceError) => { + snapshot({ + code, + callCount, + request, + }); + callback(); + }); + } + function checkForCodes(codes: Array, callback: () => void) { + function withNextCode() { + const code = codes.pop(); + if (code) { + checkRequest(code, withNextCode); + } else { + callback(); + } + } + withNextCode(); + } + it('should ensure correct behavior with retryable errors', done => { + const status = grpc.status; + const codes: Array = [ + status.DEADLINE_EXCEEDED, + status.RESOURCE_EXHAUSTED, + status.ABORTED, + status.UNAVAILABLE, + ]; + checkForCodes(codes, done); + }); + }); + }); + after(async () => { + server.shutdown(() => {}); + }); +}); From e23127c2a95a4566eed04e9ac82dbc4dc038add8 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 27 Jun 2022 15:52:00 -0400 Subject: [PATCH 20/51] Add modularity --- __snapshots__/read-rows.js | 16 ++-- .../service-testers/check-retry-snapshots.ts | 41 +++++++++++ test/server/read-rows.ts | 73 ++++++------------- 3 files changed, 71 insertions(+), 59 deletions(-) create mode 100644 src/util/mock-servers/service-testers/check-retry-snapshots.ts diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index c9c884235..fa706805b 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -1,7 +1,7 @@ exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with retryable errors 1' + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with deadline exceeded error 1' ] = { - code: 14, + code: 4, callCount: 4, request: { tableName: @@ -17,9 +17,9 @@ exports[ }; exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with retryable errors 2' + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with resource exhausted error 1' ] = { - code: 10, + code: 8, callCount: 4, request: { tableName: @@ -35,9 +35,9 @@ exports[ }; exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with retryable errors 3' + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with aborted error 1' ] = { - code: 8, + code: 10, callCount: 4, request: { tableName: @@ -53,9 +53,9 @@ exports[ }; exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with retryable errors 4' + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with unavailable error 1' ] = { - code: 4, + code: 14, callCount: 4, request: { tableName: diff --git a/src/util/mock-servers/service-testers/check-retry-snapshots.ts b/src/util/mock-servers/service-testers/check-retry-snapshots.ts new file mode 100644 index 000000000..7720749d1 --- /dev/null +++ b/src/util/mock-servers/service-testers/check-retry-snapshots.ts @@ -0,0 +1,41 @@ +import {grpc, ServiceError} from 'google-gax'; +import * as assert from 'assert'; +import * as snapshot from 'snap-shot-it'; +import {MockService} from '../mock-service'; + +export function checkRetrySnapshots( + service: MockService, + table: any, + code: grpc.status, + callback: () => void +) { + const errorDetails = 'Details for a particular type of error'; + let request: any = null; + let callCount = 0; + const emitError = (stream: any) => { + const streamRequest = stream.request; + if (request) { + // This ensures that every call to the server is the same + assert.deepStrictEqual(request, streamRequest); + } else { + request = streamRequest; + } + callCount++; + stream.emit('error', { + code, + details: errorDetails, + }); + }; + service.setService({ + // Abstraction: Always emit error + ReadRows: emitError, + }); + table.createReadStream({}).on('error', (error: ServiceError) => { + snapshot({ + code, + callCount, + request, + }); + callback(); + }); +} \ No newline at end of file diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 7faace42b..44ec542c2 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -27,6 +27,7 @@ import {MockService} from '../../src/util/mock-servers/mock-service'; const snapshot = require('snap-shot-it'); import {check} from 'linkinator'; import * as gax from 'google-gax'; +import {checkRetrySnapshots} from '../../src/util/mock-servers/service-testers/check-retry-snapshots'; function isServiceError(error: any): error is ServiceError { return ( @@ -57,57 +58,27 @@ describe('Bigtable/ReadRows', () => { describe('with a mock server that always sends an error back', () => { // Define the standard call here describe('where the error is retryable', () => { - function checkRequest(code: grpc.status, callback: () => void) { - const errorDetails = 'Details for a particular type of error'; - let request: any = null; - let callCount = 0; - const emitError = (stream: any) => { - const streamRequest = stream.request; - if (request) { - // This ensures that every call to the server is the same - assert.deepStrictEqual(request, streamRequest); - } else { - request = streamRequest; - } - callCount++; - stream.emit('error', { - code, - details: errorDetails, - }); - }; - service.setService({ - // Abstraction: Always emit error - ReadRows: emitError, - }); - table.createReadStream({}).on('error', (error: ServiceError) => { - snapshot({ - code, - callCount, - request, - }); - callback(); - }); - } - function checkForCodes(codes: Array, callback: () => void) { - function withNextCode() { - const code = codes.pop(); - if (code) { - checkRequest(code, withNextCode); - } else { - callback(); - } - } - withNextCode(); - } - it('should ensure correct behavior with retryable errors', done => { - const status = grpc.status; - const codes: Array = [ - status.DEADLINE_EXCEEDED, - status.RESOURCE_EXHAUSTED, - status.ABORTED, - status.UNAVAILABLE, - ]; - checkForCodes(codes, done); + it('should ensure correct behavior with deadline exceeded error', done => { + checkRetrySnapshots( + service, + table, + grpc.status.DEADLINE_EXCEEDED, + done + ); + }); + it('should ensure correct behavior with resource exhausted error', done => { + checkRetrySnapshots( + service, + table, + grpc.status.RESOURCE_EXHAUSTED, + done + ); + }); + it('should ensure correct behavior with aborted error', done => { + checkRetrySnapshots(service, table, grpc.status.ABORTED, done); + }); + it('should ensure correct behavior with unavailable error', done => { + checkRetrySnapshots(service, table, grpc.status.UNAVAILABLE, done); }); }); }); From 05f9910482dc98603e48ce54d8aafe8d1b15ae86 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 27 Jun 2022 16:01:19 -0400 Subject: [PATCH 21/51] small refactor step --- .../service-testers/check-retry-snapshots.ts | 2 +- test/server/read-rows.ts | 22 ++++++------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/util/mock-servers/service-testers/check-retry-snapshots.ts b/src/util/mock-servers/service-testers/check-retry-snapshots.ts index 7720749d1..daaec1f9a 100644 --- a/src/util/mock-servers/service-testers/check-retry-snapshots.ts +++ b/src/util/mock-servers/service-testers/check-retry-snapshots.ts @@ -38,4 +38,4 @@ export function checkRetrySnapshots( }); callback(); }); -} \ No newline at end of file +} diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 44ec542c2..4032305d7 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -56,29 +56,21 @@ describe('Bigtable/ReadRows', () => { }); describe('with a mock server that always sends an error back', () => { - // Define the standard call here + function checkRetryWithServer(code: grpc.status, callback: () => void) { + checkRetrySnapshots(service, table, code, callback); + } describe('where the error is retryable', () => { it('should ensure correct behavior with deadline exceeded error', done => { - checkRetrySnapshots( - service, - table, - grpc.status.DEADLINE_EXCEEDED, - done - ); + checkRetryWithServer(grpc.status.DEADLINE_EXCEEDED, done); }); it('should ensure correct behavior with resource exhausted error', done => { - checkRetrySnapshots( - service, - table, - grpc.status.RESOURCE_EXHAUSTED, - done - ); + checkRetryWithServer(grpc.status.RESOURCE_EXHAUSTED, done); }); it('should ensure correct behavior with aborted error', done => { - checkRetrySnapshots(service, table, grpc.status.ABORTED, done); + checkRetryWithServer(grpc.status.ABORTED, done); }); it('should ensure correct behavior with unavailable error', done => { - checkRetrySnapshots(service, table, grpc.status.UNAVAILABLE, done); + checkRetryWithServer(grpc.status.UNAVAILABLE, done); }); }); }); From dcf21eb4b3ef2ce842e03c8ad5b71d2cd4b0f846 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 27 Jun 2022 16:50:38 -0400 Subject: [PATCH 22/51] A step toward the right abstraction --- .../service-testers/check-retry-snapshots.ts | 29 +++----------- .../service-testers/service-handler.ts | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+), 24 deletions(-) create mode 100644 src/util/mock-servers/service-testers/service-handler.ts diff --git a/src/util/mock-servers/service-testers/check-retry-snapshots.ts b/src/util/mock-servers/service-testers/check-retry-snapshots.ts index daaec1f9a..a9ef4163e 100644 --- a/src/util/mock-servers/service-testers/check-retry-snapshots.ts +++ b/src/util/mock-servers/service-testers/check-retry-snapshots.ts @@ -1,7 +1,7 @@ import {grpc, ServiceError} from 'google-gax'; -import * as assert from 'assert'; import * as snapshot from 'snap-shot-it'; import {MockService} from '../mock-service'; +import {ServiceHandler} from './service-handler'; export function checkRetrySnapshots( service: MockService, @@ -9,32 +9,13 @@ export function checkRetrySnapshots( code: grpc.status, callback: () => void ) { - const errorDetails = 'Details for a particular type of error'; - let request: any = null; - let callCount = 0; - const emitError = (stream: any) => { - const streamRequest = stream.request; - if (request) { - // This ensures that every call to the server is the same - assert.deepStrictEqual(request, streamRequest); - } else { - request = streamRequest; - } - callCount++; - stream.emit('error', { - code, - details: errorDetails, - }); - }; - service.setService({ - // Abstraction: Always emit error - ReadRows: emitError, - }); + const serviceHandler = new ServiceHandler(code); + serviceHandler.setupService(service); table.createReadStream({}).on('error', (error: ServiceError) => { snapshot({ code, - callCount, - request, + callCount: serviceHandler.callCount, + request: serviceHandler.request, }); callback(); }); diff --git a/src/util/mock-servers/service-testers/service-handler.ts b/src/util/mock-servers/service-testers/service-handler.ts new file mode 100644 index 000000000..6eb8a7586 --- /dev/null +++ b/src/util/mock-servers/service-testers/service-handler.ts @@ -0,0 +1,39 @@ +import {grpc} from 'google-gax'; +import {MockService} from '../mock-service'; +import * as assert from 'assert'; + +export class ServiceHandler { + code: grpc.status; + request: any = null; + callCount = 0; + + constructor(code: grpc.status) { + this.code = code; + } + + handler(call: any) { + const errorDetails = 'Details for a particular type of error'; + call.emit('error', { + code: this.code, + details: errorDetails, + }); + } + + setupService(service: MockService) { + const handleRpcCall = (call: any) => { + const streamRequest = call.request; + if (this.request) { + // This ensures that every call to the server is the same + assert.deepStrictEqual(this.request, streamRequest); + } else { + this.request = streamRequest; + } + this.callCount++; + this.handler(call); + }; + service.setService({ + // Abstraction: Always emit error + ReadRows: handleRpcCall, + }); + } +} From 0150022ce8234e80cf12770aecfde664f7d25083 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 27 Jun 2022 17:16:39 -0400 Subject: [PATCH 23/51] send error handler abstraction --- .../service-testers/check-retry-snapshots.ts | 4 ++-- .../implementation/send-error-handler.ts | 21 +++++++++++++++++++ .../{ => service-handlers}/service-handler.ts | 18 +++------------- 3 files changed, 26 insertions(+), 17 deletions(-) create mode 100644 src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts rename src/util/mock-servers/service-testers/{ => service-handlers}/service-handler.ts (61%) diff --git a/src/util/mock-servers/service-testers/check-retry-snapshots.ts b/src/util/mock-servers/service-testers/check-retry-snapshots.ts index a9ef4163e..fffa19e1c 100644 --- a/src/util/mock-servers/service-testers/check-retry-snapshots.ts +++ b/src/util/mock-servers/service-testers/check-retry-snapshots.ts @@ -1,7 +1,7 @@ import {grpc, ServiceError} from 'google-gax'; import * as snapshot from 'snap-shot-it'; import {MockService} from '../mock-service'; -import {ServiceHandler} from './service-handler'; +import {SendErrorHandler} from './service-handlers/implementation/send-error-handler'; export function checkRetrySnapshots( service: MockService, @@ -9,7 +9,7 @@ export function checkRetrySnapshots( code: grpc.status, callback: () => void ) { - const serviceHandler = new ServiceHandler(code); + const serviceHandler = new SendErrorHandler(code); serviceHandler.setupService(service); table.createReadStream({}).on('error', (error: ServiceError) => { snapshot({ diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts new file mode 100644 index 000000000..561f80f3a --- /dev/null +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -0,0 +1,21 @@ +import {grpc} from 'google-gax'; +import {ServiceHandler} from '../service-handler'; + +export class SendErrorHandler extends ServiceHandler { + code: grpc.status; + request: any = null; + callCount = 0; + + constructor(code: grpc.status) { + super(); + this.code = code; + } + + handler(call: any) { + const errorDetails = 'Details for a particular type of error'; + call.emit('error', { + code: this.code, + details: errorDetails, + }); + } +} diff --git a/src/util/mock-servers/service-testers/service-handler.ts b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts similarity index 61% rename from src/util/mock-servers/service-testers/service-handler.ts rename to src/util/mock-servers/service-testers/service-handlers/service-handler.ts index 6eb8a7586..617eefa94 100644 --- a/src/util/mock-servers/service-testers/service-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts @@ -1,23 +1,11 @@ -import {grpc} from 'google-gax'; -import {MockService} from '../mock-service'; +import {MockService} from '../../mock-service'; import * as assert from 'assert'; -export class ServiceHandler { - code: grpc.status; +export abstract class ServiceHandler { request: any = null; callCount = 0; - constructor(code: grpc.status) { - this.code = code; - } - - handler(call: any) { - const errorDetails = 'Details for a particular type of error'; - call.emit('error', { - code: this.code, - details: errorDetails, - }); - } + abstract handler(call: any): void; setupService(service: MockService) { const handleRpcCall = (call: any) => { From 93bfea3313b7200ce213152af66c3987a815f3e9 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Mon, 27 Jun 2022 17:28:11 -0400 Subject: [PATCH 24/51] pass endpoint into constructor --- .../service-testers/check-retry-snapshots.ts | 2 +- .../implementation/send-error-handler.ts | 9 ++++----- .../service-handlers/service-handler.ts | 17 +++++++++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/util/mock-servers/service-testers/check-retry-snapshots.ts b/src/util/mock-servers/service-testers/check-retry-snapshots.ts index fffa19e1c..926daad37 100644 --- a/src/util/mock-servers/service-testers/check-retry-snapshots.ts +++ b/src/util/mock-servers/service-testers/check-retry-snapshots.ts @@ -9,7 +9,7 @@ export function checkRetrySnapshots( code: grpc.status, callback: () => void ) { - const serviceHandler = new SendErrorHandler(code); + const serviceHandler = new SendErrorHandler('ReadRows', code); serviceHandler.setupService(service); table.createReadStream({}).on('error', (error: ServiceError) => { snapshot({ diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts index 561f80f3a..4472b3be9 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -6,16 +6,15 @@ export class SendErrorHandler extends ServiceHandler { request: any = null; callCount = 0; - constructor(code: grpc.status) { - super(); + constructor(endpoint: string, code: grpc.status) { + super(endpoint); this.code = code; } - handler(call: any) { - const errorDetails = 'Details for a particular type of error'; + callHandler(call: any) { call.emit('error', { code: this.code, - details: errorDetails, + details: 'Details for a particular type of error', }); } } diff --git a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts index 617eefa94..c153efc5e 100644 --- a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts @@ -4,24 +4,29 @@ import * as assert from 'assert'; export abstract class ServiceHandler { request: any = null; callCount = 0; + endpoint: string; - abstract handler(call: any): void; + protected constructor(endpoint: string) { + this.endpoint = endpoint; + } + + abstract callHandler(call: any): void; setupService(service: MockService) { const handleRpcCall = (call: any) => { - const streamRequest = call.request; + const callRequest = call.request; if (this.request) { // This ensures that every call to the server is the same - assert.deepStrictEqual(this.request, streamRequest); + assert.deepStrictEqual(this.request, callRequest); } else { - this.request = streamRequest; + this.request = callRequest; } this.callCount++; - this.handler(call); + this.callHandler(call); }; service.setService({ // Abstraction: Always emit error - ReadRows: handleRpcCall, + [this.endpoint]: handleRpcCall, }); } } From 7a4dbcac1a065ce2eb16e171848579e47cef2739 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 28 Jun 2022 11:46:46 -0400 Subject: [PATCH 25/51] expand class hierarchy --- .../service-testers/check-retry-snapshots.ts | 11 +++-- .../implementation/same-call-handler.ts | 35 +++++++++++++++ .../implementation/send-error-handler.ts | 17 +++++-- .../service-handlers/service-handler.ts | 44 +++++++------------ test/server/read-rows.ts | 10 ++--- 5 files changed, 73 insertions(+), 44 deletions(-) create mode 100644 src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts diff --git a/src/util/mock-servers/service-testers/check-retry-snapshots.ts b/src/util/mock-servers/service-testers/check-retry-snapshots.ts index 926daad37..0b93d987f 100644 --- a/src/util/mock-servers/service-testers/check-retry-snapshots.ts +++ b/src/util/mock-servers/service-testers/check-retry-snapshots.ts @@ -1,19 +1,18 @@ import {grpc, ServiceError} from 'google-gax'; import * as snapshot from 'snap-shot-it'; -import {MockService} from '../mock-service'; -import {SendErrorHandler} from './service-handlers/implementation/send-error-handler'; +import {SameCallHandler} from './service-handlers/implementation/same-call-handler'; export function checkRetrySnapshots( - service: MockService, + serviceHandler: SameCallHandler, table: any, code: grpc.status, callback: () => void ) { - const serviceHandler = new SendErrorHandler('ReadRows', code); - serviceHandler.setupService(service); + serviceHandler.setupService(); + // TODO: Abstract out the stream getter table.createReadStream({}).on('error', (error: ServiceError) => { snapshot({ - code, + code, // TODO: Replace with input callCount: serviceHandler.callCount, request: serviceHandler.request, }); diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts new file mode 100644 index 000000000..e7695c337 --- /dev/null +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts @@ -0,0 +1,35 @@ +import {MockService} from '../../../mock-service'; +import * as assert from 'assert'; +import {ServiceHandler} from '../service-handler'; + +export abstract class SameCallHandler extends ServiceHandler { + service: MockService; + request: any = null; + callCount = 0; + endpoint: string; + + protected constructor(service: MockService, endpoint: string) { + super(); + this.endpoint = endpoint; + this.service = service; + } + + setupService(): void { + const handleRpcCall = (call: any) => { + // TODO: Make an abstraction of this + const callRequest = call.request; + if (this.request) { + // This ensures that every call to the server is the same + assert.deepStrictEqual(this.request, callRequest); + } else { + this.request = callRequest; + } + this.callCount++; + this.callHandler(call); + }; + this.service.setService({ + // Abstraction: Always emit error + [this.endpoint]: handleRpcCall, + }); + } +} diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts index 4472b3be9..ead98e026 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -1,13 +1,14 @@ import {grpc} from 'google-gax'; -import {ServiceHandler} from '../service-handler'; +import {SameCallHandler} from './same-call-handler'; +import {MockService} from '../../../mock-service'; -export class SendErrorHandler extends ServiceHandler { +export class SendErrorHandler extends SameCallHandler { code: grpc.status; request: any = null; callCount = 0; - constructor(endpoint: string, code: grpc.status) { - super(endpoint); + constructor(service: MockService, endpoint: string, code: grpc.status) { + super(service, endpoint); this.code = code; } @@ -17,4 +18,12 @@ export class SendErrorHandler extends ServiceHandler { details: 'Details for a particular type of error', }); } + + snapshotOutput(): any { + return { + request: this.request, + code: this.code, + callCount: this.callCount, + }; + } } diff --git a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts index c153efc5e..592809040 100644 --- a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts @@ -1,32 +1,20 @@ -import {MockService} from '../../mock-service'; -import * as assert from 'assert'; - export abstract class ServiceHandler { - request: any = null; - callCount = 0; - endpoint: string; - - protected constructor(endpoint: string) { - this.endpoint = endpoint; - } - + /* + callHandler accepts a grpc call and provides behaviour for that grpc call + which may involve sending errors or data back to the client for + example. + */ abstract callHandler(call: any): void; - setupService(service: MockService) { - const handleRpcCall = (call: any) => { - const callRequest = call.request; - if (this.request) { - // This ensures that every call to the server is the same - assert.deepStrictEqual(this.request, callRequest); - } else { - this.request = callRequest; - } - this.callCount++; - this.callHandler(call); - }; - service.setService({ - // Abstraction: Always emit error - [this.endpoint]: handleRpcCall, - }); - } + /* + snapshotOutput is used to provide a custom json object that represents the + results of the test that was run with this service handler. + */ + abstract snapshotOutput(): any; + + /* + setupService is called to setup the service we use for collecting data about + a running test. + */ + abstract setupService(): void; } diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 4032305d7..2812af874 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -18,16 +18,13 @@ import {before, describe, it} from 'mocha'; import {Bigtable} from '../../src'; -import * as assert from 'assert'; -import {GoogleError, grpc, ServiceError} from 'google-gax'; +import {grpc, ServiceError} from 'google-gax'; import {MockServer} from '../../src/util/mock-servers/mock-server'; import {BigtableClientMockService} from '../../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; import {MockService} from '../../src/util/mock-servers/mock-service'; -const snapshot = require('snap-shot-it'); -import {check} from 'linkinator'; -import * as gax from 'google-gax'; import {checkRetrySnapshots} from '../../src/util/mock-servers/service-testers/check-retry-snapshots'; +import {SendErrorHandler} from '../../src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler'; function isServiceError(error: any): error is ServiceError { return ( @@ -57,7 +54,8 @@ describe('Bigtable/ReadRows', () => { describe('with a mock server that always sends an error back', () => { function checkRetryWithServer(code: grpc.status, callback: () => void) { - checkRetrySnapshots(service, table, code, callback); + const serviceHandler = new SendErrorHandler(service, 'ReadRows', code); + checkRetrySnapshots(serviceHandler, table, code, callback); } describe('where the error is retryable', () => { it('should ensure correct behavior with deadline exceeded error', done => { From 7ee6ab807f9402868ca906f27dd7eb1cb8c79906 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 28 Jun 2022 11:52:48 -0400 Subject: [PATCH 26/51] Abstract snapshot output --- .../mock-servers/service-testers/check-retry-snapshots.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/util/mock-servers/service-testers/check-retry-snapshots.ts b/src/util/mock-servers/service-testers/check-retry-snapshots.ts index 0b93d987f..4f1ca439d 100644 --- a/src/util/mock-servers/service-testers/check-retry-snapshots.ts +++ b/src/util/mock-servers/service-testers/check-retry-snapshots.ts @@ -11,11 +11,7 @@ export function checkRetrySnapshots( serviceHandler.setupService(); // TODO: Abstract out the stream getter table.createReadStream({}).on('error', (error: ServiceError) => { - snapshot({ - code, // TODO: Replace with input - callCount: serviceHandler.callCount, - request: serviceHandler.request, - }); + snapshot(serviceHandler.snapshotOutput()); callback(); }); } From df82957dcd9bc9dd4bfdb8ecd58b97b31cb2f303 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 28 Jun 2022 11:56:06 -0400 Subject: [PATCH 27/51] remove code parameter --- src/util/mock-servers/service-testers/check-retry-snapshots.ts | 1 - test/server/read-rows.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/util/mock-servers/service-testers/check-retry-snapshots.ts b/src/util/mock-servers/service-testers/check-retry-snapshots.ts index 4f1ca439d..80b43cc77 100644 --- a/src/util/mock-servers/service-testers/check-retry-snapshots.ts +++ b/src/util/mock-servers/service-testers/check-retry-snapshots.ts @@ -5,7 +5,6 @@ import {SameCallHandler} from './service-handlers/implementation/same-call-handl export function checkRetrySnapshots( serviceHandler: SameCallHandler, table: any, - code: grpc.status, callback: () => void ) { serviceHandler.setupService(); diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 2812af874..ba85060c3 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -55,7 +55,7 @@ describe('Bigtable/ReadRows', () => { describe('with a mock server that always sends an error back', () => { function checkRetryWithServer(code: grpc.status, callback: () => void) { const serviceHandler = new SendErrorHandler(service, 'ReadRows', code); - checkRetrySnapshots(serviceHandler, table, code, callback); + checkRetrySnapshots(serviceHandler, table, callback); } describe('where the error is retryable', () => { it('should ensure correct behavior with deadline exceeded error', done => { From 05a2996c2943608a052518bed712d2af7d887748 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 28 Jun 2022 13:12:03 -0400 Subject: [PATCH 28/51] service tester done --- .../service-testers/check-retry-snapshots.ts | 16 ----------- .../implementation/read-rows-fetcher.ts | 16 +++++++++++ .../stream-fetchers/stream-fetcher.ts | 5 ++++ .../service-testers/stream-tester.ts | 22 +++++++++++++++ test/server/read-rows.ts | 27 ++++++++++--------- 5 files changed, 57 insertions(+), 29 deletions(-) delete mode 100644 src/util/mock-servers/service-testers/check-retry-snapshots.ts create mode 100644 src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts create mode 100644 src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts create mode 100644 src/util/mock-servers/service-testers/stream-tester.ts diff --git a/src/util/mock-servers/service-testers/check-retry-snapshots.ts b/src/util/mock-servers/service-testers/check-retry-snapshots.ts deleted file mode 100644 index 80b43cc77..000000000 --- a/src/util/mock-servers/service-testers/check-retry-snapshots.ts +++ /dev/null @@ -1,16 +0,0 @@ -import {grpc, ServiceError} from 'google-gax'; -import * as snapshot from 'snap-shot-it'; -import {SameCallHandler} from './service-handlers/implementation/same-call-handler'; - -export function checkRetrySnapshots( - serviceHandler: SameCallHandler, - table: any, - callback: () => void -) { - serviceHandler.setupService(); - // TODO: Abstract out the stream getter - table.createReadStream({}).on('error', (error: ServiceError) => { - snapshot(serviceHandler.snapshotOutput()); - callback(); - }); -} diff --git a/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts b/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts new file mode 100644 index 000000000..003ff0735 --- /dev/null +++ b/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts @@ -0,0 +1,16 @@ +import {Table} from '../../../../../table'; +import internal = require('stream'); +import {StreamFetcher} from '../stream-fetcher'; + +export class ReadRowsFetcher extends StreamFetcher { + table: Table; + + constructor(table: Table) { + super(); + this.table = table; + } + + fetchStream(): internal.PassThrough { + return this.table.createReadStream({}); + } +} diff --git a/src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts b/src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts new file mode 100644 index 000000000..fec7d79cd --- /dev/null +++ b/src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts @@ -0,0 +1,5 @@ +import internal = require('stream'); + +export abstract class StreamFetcher { + abstract fetchStream(): internal.PassThrough; +} diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/src/util/mock-servers/service-testers/stream-tester.ts new file mode 100644 index 000000000..6612ab919 --- /dev/null +++ b/src/util/mock-servers/service-testers/stream-tester.ts @@ -0,0 +1,22 @@ +import {ServiceHandler} from './service-handlers/service-handler'; +import {StreamFetcher} from './stream-fetchers/stream-fetcher'; +import {ServiceError} from 'google-gax'; +import * as snapshot from 'snap-shot-it'; + +export class StreamTester { + serviceHandler: ServiceHandler; + streamFetcher: StreamFetcher; + + constructor(serviceHandler: ServiceHandler, streamFetcher: StreamFetcher) { + this.serviceHandler = serviceHandler; + this.streamFetcher = streamFetcher; + } + + checkSnapshots(callback: () => void): void { + this.serviceHandler.setupService(); + this.streamFetcher.fetchStream().on('error', (error: ServiceError) => { + snapshot(this.serviceHandler.snapshotOutput()); + callback(); + }); + } +} diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index ba85060c3..5fcee5287 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -19,26 +19,21 @@ import {before, describe, it} from 'mocha'; import {Bigtable} from '../../src'; -import {grpc, ServiceError} from 'google-gax'; +import {grpc} from 'google-gax'; import {MockServer} from '../../src/util/mock-servers/mock-server'; import {BigtableClientMockService} from '../../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; import {MockService} from '../../src/util/mock-servers/mock-service'; -import {checkRetrySnapshots} from '../../src/util/mock-servers/service-testers/check-retry-snapshots'; import {SendErrorHandler} from '../../src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler'; - -function isServiceError(error: any): error is ServiceError { - return ( - error.code !== undefined && - error.details !== undefined && - error.metadata !== undefined - ); -} +import {StreamFetcher} from '../../src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher'; +import {ReadRowsFetcher} from '../../src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher'; +import {StreamTester} from '../../src/util/mock-servers/service-testers/stream-tester'; +import {ServiceHandler} from '../../src/util/mock-servers/service-testers/service-handlers/service-handler'; describe('Bigtable/ReadRows', () => { let server: MockServer; let service: MockService; let bigtable: Bigtable; - let table: any; + let streamFetcher: StreamFetcher; before(done => { server = new MockServer(() => { @@ -46,16 +41,22 @@ describe('Bigtable/ReadRows', () => { apiEndpoint: `localhost:${server.port}`, }); // TODO: Replace this with generated Ids so that we don't have flaky tests - table = bigtable.instance('fake-instance').table('fake-table'); + const table = bigtable.instance('fake-instance').table('fake-table'); + streamFetcher = new ReadRowsFetcher(table); service = new BigtableClientMockService(server); done(); }); }); + function getStreamTester(serviceHandler: ServiceHandler) { + return new StreamTester(serviceHandler, streamFetcher); + } + describe('with a mock server that always sends an error back', () => { function checkRetryWithServer(code: grpc.status, callback: () => void) { const serviceHandler = new SendErrorHandler(service, 'ReadRows', code); - checkRetrySnapshots(serviceHandler, table, callback); + const streamTester = getStreamTester(serviceHandler); + streamTester.checkSnapshots(callback); } describe('where the error is retryable', () => { it('should ensure correct behavior with deadline exceeded error', done => { From 516b5c1cbe410dd3e7bd2e072281c9b9b53ba0af Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 28 Jun 2022 13:28:10 -0400 Subject: [PATCH 29/51] Add licenses --- .../implementation/same-call-handler.ts | 18 ++++++++++++++++++ .../implementation/send-error-handler.ts | 18 ++++++++++++++++++ .../service-handlers/service-handler.ts | 18 ++++++++++++++++++ .../implementation/read-rows-fetcher.ts | 18 ++++++++++++++++++ .../stream-fetchers/stream-fetcher.ts | 18 ++++++++++++++++++ .../service-testers/stream-tester.ts | 18 ++++++++++++++++++ 6 files changed, 108 insertions(+) diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts index e7695c337..3cd5cbbfa 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts @@ -1,3 +1,21 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + import {MockService} from '../../../mock-service'; import * as assert from 'assert'; import {ServiceHandler} from '../service-handler'; diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts index ead98e026..b1117356e 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -1,3 +1,21 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + import {grpc} from 'google-gax'; import {SameCallHandler} from './same-call-handler'; import {MockService} from '../../../mock-service'; diff --git a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts index 592809040..e5eb1e298 100644 --- a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts @@ -1,3 +1,21 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + export abstract class ServiceHandler { /* callHandler accepts a grpc call and provides behaviour for that grpc call diff --git a/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts b/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts index 003ff0735..7fca546da 100644 --- a/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts +++ b/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts @@ -1,3 +1,21 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + import {Table} from '../../../../../table'; import internal = require('stream'); import {StreamFetcher} from '../stream-fetcher'; diff --git a/src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts b/src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts index fec7d79cd..1b9795af5 100644 --- a/src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts +++ b/src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts @@ -1,3 +1,21 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + import internal = require('stream'); export abstract class StreamFetcher { diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/src/util/mock-servers/service-testers/stream-tester.ts index 6612ab919..e32e1f487 100644 --- a/src/util/mock-servers/service-testers/stream-tester.ts +++ b/src/util/mock-servers/service-testers/stream-tester.ts @@ -1,3 +1,21 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + import {ServiceHandler} from './service-handlers/service-handler'; import {StreamFetcher} from './stream-fetchers/stream-fetcher'; import {ServiceError} from 'google-gax'; From 54f67cd41919562e754573c50f37ea35c48d408d Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 29 Jun 2022 10:00:49 -0400 Subject: [PATCH 30/51] new snapshot mechanism --- .../implementation/read-rows-fetcher.ts | 8 +- .../service-testers/stream-tester.ts | 26 ++++- test/server/read-rows.ts | 102 ++++++++++++++++-- test/server/test-options.ts | 68 ++++++++++++ 4 files changed, 191 insertions(+), 13 deletions(-) create mode 100644 test/server/test-options.ts diff --git a/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts b/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts index 7fca546da..24e56bcc1 100644 --- a/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts +++ b/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts @@ -16,19 +16,21 @@ // ** https://github.com/googleapis/gapic-generator-typescript ** // ** All changes to this file may be overwritten. ** -import {Table} from '../../../../../table'; +import {GetRowsOptions, Table} from '../../../../../table'; import internal = require('stream'); import {StreamFetcher} from '../stream-fetcher'; export class ReadRowsFetcher extends StreamFetcher { table: Table; + opts: GetRowsOptions; - constructor(table: Table) { + constructor(table: Table, opts?: GetRowsOptions) { super(); + this.opts = opts ?? {}; this.table = table; } fetchStream(): internal.PassThrough { - return this.table.createReadStream({}); + return this.table.createReadStream(this.opts); } } diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/src/util/mock-servers/service-testers/stream-tester.ts index e32e1f487..de00c13c0 100644 --- a/src/util/mock-servers/service-testers/stream-tester.ts +++ b/src/util/mock-servers/service-testers/stream-tester.ts @@ -20,6 +20,9 @@ import {ServiceHandler} from './service-handlers/service-handler'; import {StreamFetcher} from './stream-fetchers/stream-fetcher'; import {ServiceError} from 'google-gax'; import * as snapshot from 'snap-shot-it'; +import {Row} from '../../../row'; + +const concat = require('concat-stream'); export class StreamTester { serviceHandler: ServiceHandler; @@ -32,9 +35,24 @@ export class StreamTester { checkSnapshots(callback: () => void): void { this.serviceHandler.setupService(); - this.streamFetcher.fetchStream().on('error', (error: ServiceError) => { - snapshot(this.serviceHandler.snapshotOutput()); - callback(); - }); + this.streamFetcher + .fetchStream() + .on('error', (error: ServiceError) => { + snapshot({ + result: 'error', + output: this.serviceHandler.snapshotOutput(), + }); + callback(); + }) + .pipe( + concat((rows: Row[]) => { + snapshot({ + result: 'data', + output: this.serviceHandler.snapshotOutput(), + data: rows, + }); + callback(); + }) + ); } } diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 5fcee5287..860906c32 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -17,23 +17,24 @@ // ** All changes to this file may be overwritten. ** import {before, describe, it} from 'mocha'; -import {Bigtable} from '../../src'; +import {Bigtable, GetRowsOptions} from '../../src'; import {grpc} from 'google-gax'; import {MockServer} from '../../src/util/mock-servers/mock-server'; import {BigtableClientMockService} from '../../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; import {MockService} from '../../src/util/mock-servers/mock-service'; import {SendErrorHandler} from '../../src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler'; -import {StreamFetcher} from '../../src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher'; import {ReadRowsFetcher} from '../../src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher'; import {StreamTester} from '../../src/util/mock-servers/service-testers/stream-tester'; import {ServiceHandler} from '../../src/util/mock-servers/service-testers/service-handlers/service-handler'; +import {Table} from '../../src/table'; +import {testGaxOptions} from './test-options'; describe('Bigtable/ReadRows', () => { let server: MockServer; let service: MockService; let bigtable: Bigtable; - let streamFetcher: StreamFetcher; + let table: Table; before(done => { server = new MockServer(() => { @@ -41,14 +42,17 @@ describe('Bigtable/ReadRows', () => { apiEndpoint: `localhost:${server.port}`, }); // TODO: Replace this with generated Ids so that we don't have flaky tests - const table = bigtable.instance('fake-instance').table('fake-table'); - streamFetcher = new ReadRowsFetcher(table); + table = bigtable.instance('fake-instance').table('fake-table'); service = new BigtableClientMockService(server); done(); }); }); - function getStreamTester(serviceHandler: ServiceHandler) { + function getStreamTester( + serviceHandler: ServiceHandler, + opts?: GetRowsOptions + ) { + const streamFetcher = new ReadRowsFetcher(table, opts); return new StreamTester(serviceHandler, streamFetcher); } @@ -72,8 +76,94 @@ describe('Bigtable/ReadRows', () => { checkRetryWithServer(grpc.status.UNAVAILABLE, done); }); }); + describe('where the error is not retryable', () => { + it('should ensure correct behavior with cancelled error', done => { + checkRetryWithServer(grpc.status.CANCELLED, done); + }); + it('should ensure correct behavior with internal error', done => { + checkRetryWithServer(grpc.status.INTERNAL, done); + }); + it('should ensure correct behavior with invalid argument error', done => { + checkRetryWithServer(grpc.status.INVALID_ARGUMENT, done); + }); + }); + describe('with a deadline exceeded error and different createReadStream arguments', () => { + const serviceHandler = new SendErrorHandler( + service, + 'ReadRows', + grpc.status.DEADLINE_EXCEEDED + ); + function checkWithOptions(opts: any, callback: () => void) { + const streamTester = getStreamTester(serviceHandler, opts); + streamTester.checkSnapshots(callback); + } + it('should pass checks with an empty request', done => { + checkWithOptions({}, done); + }); + it('should pass checks with a decode value set', done => { + checkWithOptions({decode: true}, done); + }); + it('should pass checks with an encoding value set', done => { + // TODO: encoding + checkWithOptions({encoding: 'test-encoding'}, done); + }); + it('should pass checks with start and end values', done => { + checkWithOptions( + { + start: 'test-start', + end: 'test-end', + }, + done + ); + }); + it('should pass checks with keys', done => { + checkWithOptions({keys: ['test-key-1', 'test-key-2']}, done); + }); + it('should pass checks with a filter', done => { + checkWithOptions({filter: [{}]}, done); + }); + it('should pass checks with a limit', done => { + checkWithOptions({limit: 10}, done); + }); + it('should pass checks with a prefix', done => { + checkWithOptions({prefix: 'test-prefix'}, done); + }); + it('should pass checks with prefixes', done => { + checkWithOptions({prefixes: ['test-prefix1', 'test-prefix2']}, done); + }); + it('should pass checks with a list of ranges', done => { + checkWithOptions( + { + ranges: [ + { + start: 'test-start-1', + end: 'test-end-1', + }, + { + start: 'test-start-2', + end: 'test-end-2', + }, + ], + }, + done + ); + }); + it('should pass checks with gaxOptions', done => { + // TODO: Add the retry parameter + checkWithOptions( + { + gaxOptions: testGaxOptions, + }, + done + ); + }); + }); }); after(async () => { server.shutdown(() => {}); }); }); + +// TODO: Think of interesting cases for the shouldRetryFn +// TODO: Change the test framework so that it saves each different request +// TODO: and then records the order that they occur in diff --git a/test/server/test-options.ts b/test/server/test-options.ts new file mode 100644 index 000000000..1b1659b0c --- /dev/null +++ b/test/server/test-options.ts @@ -0,0 +1,68 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + +export const testGaxOptions = { + timeout: 1000, + retry: { + retryCodes: [3, 5, 7, 12], + backoffSettings: { + maxRetries: 41, + initialRetryDelayMillis: 401, + retryDelayMultiplier: 42, + maxRetryDelayMillis: 402, + initialRpcTimeoutMillis: 403, + maxRpcTimeoutMillis: 404, + rpcTimeoutMultiplier: 43, + }, + }, + autoPaginate: true, + maxResults: 23, + maxRetries: 24, + otherArgs: { + otherArg1: 217, + }, + bundleOptions: { + elementCountLimit: 71, + requestByteLimit: 72, + elementCountThreshold: 73, + requestByteThreshold: 74, + delayThreshold: 75, + }, + isBundling: false, + longRunning: { + maxRetries: 5, + initialRetryDelayMillis: 501, + retryDelayMultiplier: 52, + maxRetryDelayMillis: 502, + initialRpcTimeoutMillis: 503, + maxRpcTimeoutMillis: 504, + totalTimeoutMillis: 505, + rpcTimeoutMultiplier: 53, + }, + apiName: 'test-apiName', + retryRequestOptions: { + objectMode: true, + request: 'test-request', + retries: 38, + noResponseRetries: 39, + currentRetryAttempt: 35, + shouldRetryFn: () => { + return true; + }, + }, +}; From 32b3da583f9fc94a74a91d2ce9d96daa96b6849b Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 29 Jun 2022 10:04:22 -0400 Subject: [PATCH 31/51] new snapshots --- __snapshots__/read-rows.js | 164 +++++++++++++++++++++++++++---------- 1 file changed, 120 insertions(+), 44 deletions(-) diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index fa706805b..e44ce7673 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -1,71 +1,147 @@ exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with deadline exceeded error 1' ] = { - code: 4, - callCount: 4, - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + result: 'error', + output: { + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', }, - filter: null, - rowsLimit: '0', - appProfileId: '', + code: 4, + callCount: 4, }, }; exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with resource exhausted error 1' ] = { - code: 8, - callCount: 4, - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + result: 'error', + output: { + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', }, - filter: null, - rowsLimit: '0', - appProfileId: '', + code: 8, + callCount: 4, }, }; exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with aborted error 1' ] = { - code: 10, - callCount: 4, - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + result: 'error', + output: { + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', }, - filter: null, - rowsLimit: '0', - appProfileId: '', + code: 10, + callCount: 4, }, }; exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with unavailable error 1' ] = { - code: 14, - callCount: 4, - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + result: 'error', + output: { + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', }, - filter: null, - rowsLimit: '0', - appProfileId: '', + code: 14, + callCount: 4, + }, +}; + +exports[ + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with cancelled error 1' +] = { + result: 'data', + output: { + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + code: 1, + callCount: 1, + }, + data: [], +}; + +exports[ + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with internal error 1' +] = { + result: 'error', + output: { + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + code: 13, + callCount: 1, + }, +}; + +exports[ + 'Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with invalid argument error 1' +] = { + result: 'error', + output: { + request: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + code: 3, + callCount: 1, }, }; From 1230ee7d8e4287fe13f935c5f5835a8309f0495a Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 29 Jun 2022 11:20:44 -0400 Subject: [PATCH 32/51] Change input/output snapshots --- __snapshots__/read-rows.js | 203 +++++++++++------- package.json | 3 +- .../implementation/same-call-handler.ts | 25 ++- .../implementation/send-error-handler.ts | 8 +- 4 files changed, 153 insertions(+), 86 deletions(-) diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index e44ce7673..8f41d501c 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -3,19 +3,26 @@ exports[ ] = { result: 'error', output: { - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + input: { + code: 4, + }, + output: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, }, - filter: null, - rowsLimit: '0', - appProfileId: '', + requestOrder: [0, 0, 0, 0], + callCount: 4, }, - code: 4, - callCount: 4, }, }; @@ -24,19 +31,26 @@ exports[ ] = { result: 'error', output: { - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + input: { + code: 8, + }, + output: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, }, - filter: null, - rowsLimit: '0', - appProfileId: '', + requestOrder: [0, 0, 0, 0], + callCount: 4, }, - code: 8, - callCount: 4, }, }; @@ -45,19 +59,26 @@ exports[ ] = { result: 'error', output: { - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + input: { + code: 10, + }, + output: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, }, - filter: null, - rowsLimit: '0', - appProfileId: '', + requestOrder: [0, 0, 0, 0], + callCount: 4, }, - code: 10, - callCount: 4, }, }; @@ -66,19 +87,26 @@ exports[ ] = { result: 'error', output: { - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + input: { + code: 14, + }, + output: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, }, - filter: null, - rowsLimit: '0', - appProfileId: '', + requestOrder: [0, 0, 0, 0], + callCount: 4, }, - code: 14, - callCount: 4, }, }; @@ -87,19 +115,26 @@ exports[ ] = { result: 'data', output: { - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + input: { + code: 1, + }, + output: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, }, - filter: null, - rowsLimit: '0', - appProfileId: '', + requestOrder: [0], + callCount: 1, }, - code: 1, - callCount: 1, }, data: [], }; @@ -109,19 +144,26 @@ exports[ ] = { result: 'error', output: { - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + input: { + code: 13, + }, + output: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, }, - filter: null, - rowsLimit: '0', - appProfileId: '', + requestOrder: [0], + callCount: 1, }, - code: 13, - callCount: 1, }, }; @@ -130,18 +172,25 @@ exports[ ] = { result: 'error', output: { - request: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + input: { + code: 3, + }, + output: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, }, - filter: null, - rowsLimit: '0', - appProfileId: '', + requestOrder: [0], + callCount: 1, }, - code: 3, - callCount: 1, }, }; diff --git a/package.json b/package.json index 269e03ca5..31749ebb1 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "@google-cloud/promisify": "^3.0.0", "arrify": "^2.0.0", "concat-stream": "^2.0.0", + "deep-equal": "^2.0.5", "dot-prop": "^6.0.0", "escape-string-regexp": "^4.0.0", "extend": "^3.0.2", @@ -87,8 +88,8 @@ "pack-n-play": "^1.0.0-2", "proxyquire": "^2.0.0", "sinon": "^14.0.0", - "tcp-port-used": "^1.0.2", "snap-shot-it": "^7.9.1", + "tcp-port-used": "^1.0.2", "ts-loader": "^9.0.0", "typescript": "^4.6.4", "uuid": "^8.0.0", diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts index 3cd5cbbfa..8a1bd47a8 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts @@ -17,12 +17,15 @@ // ** All changes to this file may be overwritten. ** import {MockService} from '../../../mock-service'; -import * as assert from 'assert'; import {ServiceHandler} from '../service-handler'; +const equal = require('deep-equal'); + export abstract class SameCallHandler extends ServiceHandler { service: MockService; request: any = null; + requestList: any[] = []; + requestOrder: number[] = []; callCount = 0; endpoint: string; @@ -36,12 +39,16 @@ export abstract class SameCallHandler extends ServiceHandler { const handleRpcCall = (call: any) => { // TODO: Make an abstraction of this const callRequest = call.request; - if (this.request) { - // This ensures that every call to the server is the same - assert.deepStrictEqual(this.request, callRequest); + const requestIndex = this.requestList.findIndex(request => { + return equal(request, callRequest); + }); + if (requestIndex === -1) { + this.requestList.push(callRequest); + this.requestOrder.push(this.requestList.length - 1); } else { - this.request = callRequest; + this.requestOrder.push(requestIndex); } + this.request = callRequest; this.callCount++; this.callHandler(call); }; @@ -50,4 +57,12 @@ export abstract class SameCallHandler extends ServiceHandler { [this.endpoint]: handleRpcCall, }); } + + requests() { + return { + requests: Object.assign({}, this.requestList), + requestOrder: this.requestOrder, + callCount: this.callCount, + }; + } } diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts index b1117356e..f9cc67787 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -25,6 +25,7 @@ export class SendErrorHandler extends SameCallHandler { request: any = null; callCount = 0; + // TODO: service and endpoint should be bundled into one object. constructor(service: MockService, endpoint: string, code: grpc.status) { super(service, endpoint); this.code = code; @@ -39,9 +40,10 @@ export class SendErrorHandler extends SameCallHandler { snapshotOutput(): any { return { - request: this.request, - code: this.code, - callCount: this.callCount, + input: { + code: this.code, + }, + output: this.requests(), }; } } From 85f592bfa20de33e647e0a0eae521a01cc772899 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 29 Jun 2022 11:43:06 -0400 Subject: [PATCH 33/51] restructure snapshot to logical input / output --- __snapshots__/read-rows.js | 92 +++++++++++-------- .../implementation/send-error-handler.ts | 7 +- .../service-handlers/service-handler.ts | 2 +- .../service-testers/stream-tester.ts | 12 +-- 4 files changed, 65 insertions(+), 48 deletions(-) diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index 8f41d501c..87ed3b87d 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -1,12 +1,14 @@ exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with deadline exceeded error 1' ] = { - result: 'error', + input: { + code: 4, + }, output: { - input: { - code: 4, + results: { + result: 'error', }, - output: { + requestData: { requests: { 0: { tableName: @@ -29,12 +31,14 @@ exports[ exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with resource exhausted error 1' ] = { - result: 'error', + input: { + code: 8, + }, output: { - input: { - code: 8, + results: { + result: 'error', }, - output: { + requestData: { requests: { 0: { tableName: @@ -57,12 +61,14 @@ exports[ exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with aborted error 1' ] = { - result: 'error', + input: { + code: 10, + }, output: { - input: { - code: 10, + results: { + result: 'error', }, - output: { + requestData: { requests: { 0: { tableName: @@ -85,12 +91,14 @@ exports[ exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with unavailable error 1' ] = { - result: 'error', + input: { + code: 14, + }, output: { - input: { - code: 14, + results: { + result: 'error', }, - output: { + requestData: { requests: { 0: { tableName: @@ -113,41 +121,47 @@ exports[ exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with cancelled error 1' ] = { - result: 'data', output: { input: { code: 1, }, output: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], + results: { + result: 'data', + data: [], + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', }, - filter: null, - rowsLimit: '0', - appProfileId: '', }, + requestOrder: [0], + callCount: 1, }, - requestOrder: [0], - callCount: 1, }, }, - data: [], }; exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with internal error 1' ] = { - result: 'error', + input: { + code: 13, + }, output: { - input: { - code: 13, + results: { + result: 'error', }, - output: { + requestData: { requests: { 0: { tableName: @@ -170,12 +184,14 @@ exports[ exports[ 'Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with invalid argument error 1' ] = { - result: 'error', + input: { + code: 3, + }, output: { - input: { - code: 3, + results: { + result: 'error', }, - output: { + requestData: { requests: { 0: { tableName: diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts index f9cc67787..2010d5eb6 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -38,12 +38,15 @@ export class SendErrorHandler extends SameCallHandler { }); } - snapshotOutput(): any { + snapshotOutput(results: any): any { return { input: { code: this.code, }, - output: this.requests(), + output: { + results, + requestData: this.requests(), + }, }; } } diff --git a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts index e5eb1e298..26512bcba 100644 --- a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts @@ -28,7 +28,7 @@ export abstract class ServiceHandler { snapshotOutput is used to provide a custom json object that represents the results of the test that was run with this service handler. */ - abstract snapshotOutput(): any; + abstract snapshotOutput(results: any): any; /* setupService is called to setup the service we use for collecting data about diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/src/util/mock-servers/service-testers/stream-tester.ts index de00c13c0..ba8cecb16 100644 --- a/src/util/mock-servers/service-testers/stream-tester.ts +++ b/src/util/mock-servers/service-testers/stream-tester.ts @@ -38,18 +38,16 @@ export class StreamTester { this.streamFetcher .fetchStream() .on('error', (error: ServiceError) => { - snapshot({ - result: 'error', - output: this.serviceHandler.snapshotOutput(), - }); + snapshot(this.serviceHandler.snapshotOutput({result: 'error'})); callback(); }) .pipe( concat((rows: Row[]) => { snapshot({ - result: 'data', - output: this.serviceHandler.snapshotOutput(), - data: rows, + output: this.serviceHandler.snapshotOutput({ + result: 'data', + data: rows, + }), }); callback(); }) From 46cb7f284783596042143077aa15990d66b33728 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 29 Jun 2022 11:51:03 -0400 Subject: [PATCH 34/51] Small refactors --- .../implementation/send-error-handler.ts | 2 +- .../service-handlers/service-handler.ts | 2 +- .../mock-servers/service-testers/stream-tester.ts | 13 +++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts index 2010d5eb6..a2b669930 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -38,7 +38,7 @@ export class SendErrorHandler extends SameCallHandler { }); } - snapshotOutput(results: any): any { + snapshot(results: any): any { return { input: { code: this.code, diff --git a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts index 26512bcba..01a0d6bcf 100644 --- a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts @@ -28,7 +28,7 @@ export abstract class ServiceHandler { snapshotOutput is used to provide a custom json object that represents the results of the test that was run with this service handler. */ - abstract snapshotOutput(results: any): any; + abstract snapshot(results: any): any; /* setupService is called to setup the service we use for collecting data about diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/src/util/mock-servers/service-testers/stream-tester.ts index ba8cecb16..fca35d3e8 100644 --- a/src/util/mock-servers/service-testers/stream-tester.ts +++ b/src/util/mock-servers/service-testers/stream-tester.ts @@ -34,20 +34,21 @@ export class StreamTester { } checkSnapshots(callback: () => void): void { + const collectSnapshot = (results: any) => { + this.serviceHandler.snapshot(results); + }; this.serviceHandler.setupService(); this.streamFetcher .fetchStream() .on('error', (error: ServiceError) => { - snapshot(this.serviceHandler.snapshotOutput({result: 'error'})); + collectSnapshot({result: 'error'}); callback(); }) .pipe( concat((rows: Row[]) => { - snapshot({ - output: this.serviceHandler.snapshotOutput({ - result: 'data', - data: rows, - }), + collectSnapshot({ + result: 'data', + data: rows, }); callback(); }) From 1e33d959a4be1b9f714c5dfd8bb3b1aa56c5da11 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 29 Jun 2022 16:44:27 -0400 Subject: [PATCH 35/51] In a state with passing tests --- __snapshots__/read-rows.js | 267 +++++++++++++++--- .../implementation/send-error-handler.ts | 16 +- .../service-testers/stream-tester.ts | 4 +- test/server/read-rows.ts | 24 +- 4 files changed, 258 insertions(+), 53 deletions(-) diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index 87ed3b87d..01144372e 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -1,8 +1,9 @@ exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with deadline exceeded error 1' + 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with an empty request 1' ] = { input: { code: 4, + message: {}, }, output: { results: { @@ -29,10 +30,13 @@ exports[ }; exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with resource exhausted error 1' + 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a decode value set 1' ] = { input: { - code: 8, + code: 4, + message: { + decode: true, + }, }, output: { results: { @@ -59,10 +63,13 @@ exports[ }; exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with aborted error 1' + 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with an encoding value set 1' ] = { input: { - code: 10, + code: 4, + message: { + encoding: 'test-encoding', + }, }, output: { results: { @@ -89,10 +96,14 @@ exports[ }; exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with unavailable error 1' + 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with start and end values 1' ] = { input: { - code: 14, + code: 4, + message: { + start: 'test-start', + end: 'test-end', + }, }, output: { results: { @@ -105,7 +116,20 @@ exports[ 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', rows: { rowKeys: [], - rowRanges: [{}], + rowRanges: [ + { + startKeyClosed: { + type: 'Buffer', + data: [116, 101, 115, 116, 45, 115, 116, 97, 114, 116], + }, + startKey: 'startKeyClosed', + endKeyClosed: { + type: 'Buffer', + data: [116, 101, 115, 116, 45, 101, 110, 100], + }, + endKey: 'endKeyClosed', + }, + ], }, filter: null, rowsLimit: '0', @@ -119,43 +143,55 @@ exports[ }; exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with cancelled error 1' + 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with keys 1' ] = { + input: { + code: 4, + message: { + keys: ['test-key-1', 'test-key-2'], + }, + }, output: { - input: { - code: 1, + results: { + result: 'error', }, - output: { - results: { - result: 'data', - data: [], - }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [ + { + type: 'Buffer', + data: [116, 101, 115, 116, 45, 107, 101, 121, 45, 49], + }, + { + type: 'Buffer', + data: [116, 101, 115, 116, 45, 107, 101, 121, 45, 50], + }, + ], + rowRanges: [], }, + filter: null, + rowsLimit: '0', + appProfileId: '', }, - requestOrder: [0], - callCount: 1, }, + requestOrder: [0, 0, 0, 0], + callCount: 4, }, }, }; exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with internal error 1' + 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a limit 1' ] = { input: { - code: 13, + code: 4, + message: { + limit: 10, + }, }, output: { results: { @@ -171,21 +207,70 @@ exports[ rowRanges: [{}], }, filter: null, + rowsLimit: '10', + appProfileId: '', + }, + }, + requestOrder: [0, 0, 0, 0], + callCount: 4, + }, + }, +}; + +exports[ + 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a prefix 1' +] = { + input: { + code: 4, + message: { + prefix: 'test-prefix', + }, + }, + output: { + results: { + result: 'error', + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [ + { + startKeyClosed: { + type: 'Buffer', + data: [116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 120], + }, + startKey: 'startKeyClosed', + endKeyOpen: { + type: 'Buffer', + data: [116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 121], + }, + endKey: 'endKeyOpen', + }, + ], + }, + filter: null, rowsLimit: '0', appProfileId: '', }, }, - requestOrder: [0], - callCount: 1, + requestOrder: [0, 0, 0, 0], + callCount: 4, }, }, }; exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with invalid argument error 1' + 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with prefixes 1' ] = { input: { - code: 3, + code: 4, + message: { + prefixes: ['test-prefix1', 'test-prefix2'], + }, }, output: { results: { @@ -198,15 +283,119 @@ exports[ 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', rows: { rowKeys: [], - rowRanges: [{}], + rowRanges: [ + { + startKeyClosed: { + type: 'Buffer', + data: [ + 116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 120, 49, + ], + }, + startKey: 'startKeyClosed', + endKeyOpen: { + type: 'Buffer', + data: [ + 116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 120, 50, + ], + }, + endKey: 'endKeyOpen', + }, + { + startKeyClosed: { + type: 'Buffer', + data: [ + 116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 120, 50, + ], + }, + startKey: 'startKeyClosed', + endKeyOpen: { + type: 'Buffer', + data: [ + 116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 120, 51, + ], + }, + endKey: 'endKeyOpen', + }, + ], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + }, + requestOrder: [0, 0, 0, 0], + callCount: 4, + }, + }, +}; + +exports[ + 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a list of ranges 1' +] = { + input: { + code: 4, + message: { + ranges: [ + { + start: 'test-start-1', + end: 'test-end-1', + }, + { + start: 'test-start-2', + end: 'test-end-2', + }, + ], + }, + }, + output: { + results: { + result: 'error', + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [ + { + startKeyClosed: { + type: 'Buffer', + data: [ + 116, 101, 115, 116, 45, 115, 116, 97, 114, 116, 45, 49, + ], + }, + startKey: 'startKeyClosed', + endKeyClosed: { + type: 'Buffer', + data: [116, 101, 115, 116, 45, 101, 110, 100, 45, 49], + }, + endKey: 'endKeyClosed', + }, + { + startKeyClosed: { + type: 'Buffer', + data: [ + 116, 101, 115, 116, 45, 115, 116, 97, 114, 116, 45, 50, + ], + }, + startKey: 'startKeyClosed', + endKeyClosed: { + type: 'Buffer', + data: [116, 101, 115, 116, 45, 101, 110, 100, 45, 50], + }, + endKey: 'endKeyClosed', + }, + ], }, filter: null, rowsLimit: '0', appProfileId: '', }, }, - requestOrder: [0], - callCount: 1, + requestOrder: [0, 0, 0, 0], + callCount: 4, }, }, }; diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts index a2b669930..a796ea465 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -24,11 +24,18 @@ export class SendErrorHandler extends SameCallHandler { code: grpc.status; request: any = null; callCount = 0; + message: any; // TODO: service and endpoint should be bundled into one object. - constructor(service: MockService, endpoint: string, code: grpc.status) { + constructor( + service: MockService, + endpoint: string, + code: grpc.status, + message?: any + ) { super(service, endpoint); this.code = code; + this.message = Object.assign({}, message); } callHandler(call: any) { @@ -40,9 +47,10 @@ export class SendErrorHandler extends SameCallHandler { snapshot(results: any): any { return { - input: { - code: this.code, - }, + input: Object.assign( + {code: this.code}, + this.message ? {message: this.message} : null + ), output: { results, requestData: this.requests(), diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/src/util/mock-servers/service-testers/stream-tester.ts index fca35d3e8..c79cee096 100644 --- a/src/util/mock-servers/service-testers/stream-tester.ts +++ b/src/util/mock-servers/service-testers/stream-tester.ts @@ -19,10 +19,10 @@ import {ServiceHandler} from './service-handlers/service-handler'; import {StreamFetcher} from './stream-fetchers/stream-fetcher'; import {ServiceError} from 'google-gax'; -import * as snapshot from 'snap-shot-it'; import {Row} from '../../../row'; const concat = require('concat-stream'); +import * as snapshot from 'snap-shot-it'; export class StreamTester { serviceHandler: ServiceHandler; @@ -35,7 +35,7 @@ export class StreamTester { checkSnapshots(callback: () => void): void { const collectSnapshot = (results: any) => { - this.serviceHandler.snapshot(results); + snapshot(this.serviceHandler.snapshot(results)); }; this.serviceHandler.setupService(); this.streamFetcher diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 860906c32..11ae0247d 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -28,7 +28,7 @@ import {ReadRowsFetcher} from '../../src/util/mock-servers/service-testers/strea import {StreamTester} from '../../src/util/mock-servers/service-testers/stream-tester'; import {ServiceHandler} from '../../src/util/mock-servers/service-testers/service-handlers/service-handler'; import {Table} from '../../src/table'; -import {testGaxOptions} from './test-options'; +// import {testGaxOptions} from './test-options'; describe('Bigtable/ReadRows', () => { let server: MockServer; @@ -88,12 +88,16 @@ describe('Bigtable/ReadRows', () => { }); }); describe('with a deadline exceeded error and different createReadStream arguments', () => { - const serviceHandler = new SendErrorHandler( - service, - 'ReadRows', - grpc.status.DEADLINE_EXCEEDED - ); + function getServiceHandler(message: any) { + return new SendErrorHandler( + service, + 'ReadRows', + grpc.status.DEADLINE_EXCEEDED, + message + ); + } function checkWithOptions(opts: any, callback: () => void) { + const serviceHandler = getServiceHandler(opts); const streamTester = getStreamTester(serviceHandler, opts); streamTester.checkSnapshots(callback); } @@ -119,9 +123,11 @@ describe('Bigtable/ReadRows', () => { it('should pass checks with keys', done => { checkWithOptions({keys: ['test-key-1', 'test-key-2']}, done); }); + /* it('should pass checks with a filter', done => { checkWithOptions({filter: [{}]}, done); }); + */ it('should pass checks with a limit', done => { checkWithOptions({limit: 10}, done); }); @@ -148,6 +154,7 @@ describe('Bigtable/ReadRows', () => { done ); }); + /* it('should pass checks with gaxOptions', done => { // TODO: Add the retry parameter checkWithOptions( @@ -157,6 +164,7 @@ describe('Bigtable/ReadRows', () => { done ); }); + */ }); }); after(async () => { @@ -165,5 +173,5 @@ describe('Bigtable/ReadRows', () => { }); // TODO: Think of interesting cases for the shouldRetryFn -// TODO: Change the test framework so that it saves each different request -// TODO: and then records the order that they occur in +// TODO: Consider setting up the framework so that we take snapshots of values passed into createReadStream afterwards +// TODO: Adjust max retries From d379b45f007cde476f77c625a0c4eae102dbe77f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 5 Jul 2022 17:18:08 -0400 Subject: [PATCH 36/51] Adding event driven functionality to serve handler --- __snapshots__/read-rows.js | 386 +----------------- .../implementation/read-rows-handler.ts | 136 ++++++ .../implementation/same-call-handler.ts | 8 + .../implementation/send-error-handler.ts | 6 + .../service-handlers/service-handler.ts | 12 + .../service-testers/stream-tester.ts | 32 +- test/server/read-rows.ts | 47 ++- 7 files changed, 233 insertions(+), 394 deletions(-) create mode 100644 src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index 01144372e..53a1fcb36 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -1,201 +1,22 @@ exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with an empty request 1' + 'Bigtable/ReadRows with custom responses and createReadStream arguments should pass checks with a simple call 1' ] = { input: { - code: 4, - message: {}, - }, - output: { - results: { - result: 'error', - }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, - }, - requestOrder: [0, 0, 0, 0], - callCount: 4, - }, - }, -}; - -exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a decode value set 1' -] = { - input: { - code: 4, - message: { - decode: true, - }, - }, - output: { - results: { - result: 'error', - }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, - }, - requestOrder: [0, 0, 0, 0], - callCount: 4, - }, - }, -}; - -exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with an encoding value set 1' -] = { - input: { - code: 4, - message: { - encoding: 'test-encoding', - }, - }, - output: { - results: { - result: 'error', - }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, - }, - requestOrder: [0, 0, 0, 0], - callCount: 4, - }, - }, -}; - -exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with start and end values 1' -] = { - input: { - code: 4, - message: { - start: 'test-start', - end: 'test-end', - }, - }, - output: { - results: { - result: 'error', - }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ - { - startKeyClosed: { - type: 'Buffer', - data: [116, 101, 115, 116, 45, 115, 116, 97, 114, 116], - }, - startKey: 'startKeyClosed', - endKeyClosed: { - type: 'Buffer', - data: [116, 101, 115, 116, 45, 101, 110, 100], - }, - endKey: 'endKeyClosed', - }, - ], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, - }, - requestOrder: [0, 0, 0, 0], - callCount: 4, - }, - }, -}; - -exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with keys 1' -] = { - input: { - code: 4, - message: { - keys: ['test-key-1', 'test-key-2'], - }, - }, - output: { - results: { - result: 'error', - }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [ - { - type: 'Buffer', - data: [116, 101, 115, 116, 45, 107, 101, 121, 45, 49], - }, - { - type: 'Buffer', - data: [116, 101, 115, 116, 45, 107, 101, 121, 45, 50], - }, - ], - rowRanges: [], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, + responses: [ + { + row_keys: ['a', 'b', 'c'], + last_row_key: 'c', }, - requestOrder: [0, 0, 0, 0], - callCount: 4, - }, - }, -}; - -exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a limit 1' -] = { - input: { - code: 4, + ], message: { - limit: 10, + rowKeys: [], + rowRanges: [{}], }, }, output: { results: { - result: 'error', + result: 'data', + data: ['a', 'b', 'c'], }, requestData: { requests: { @@ -207,195 +28,12 @@ exports[ rowRanges: [{}], }, filter: null, - rowsLimit: '10', - appProfileId: '', - }, - }, - requestOrder: [0, 0, 0, 0], - callCount: 4, - }, - }, -}; - -exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a prefix 1' -] = { - input: { - code: 4, - message: { - prefix: 'test-prefix', - }, - }, - output: { - results: { - result: 'error', - }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ - { - startKeyClosed: { - type: 'Buffer', - data: [116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 120], - }, - startKey: 'startKeyClosed', - endKeyOpen: { - type: 'Buffer', - data: [116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 121], - }, - endKey: 'endKeyOpen', - }, - ], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, - }, - requestOrder: [0, 0, 0, 0], - callCount: 4, - }, - }, -}; - -exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with prefixes 1' -] = { - input: { - code: 4, - message: { - prefixes: ['test-prefix1', 'test-prefix2'], - }, - }, - output: { - results: { - result: 'error', - }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ - { - startKeyClosed: { - type: 'Buffer', - data: [ - 116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 120, 49, - ], - }, - startKey: 'startKeyClosed', - endKeyOpen: { - type: 'Buffer', - data: [ - 116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 120, 50, - ], - }, - endKey: 'endKeyOpen', - }, - { - startKeyClosed: { - type: 'Buffer', - data: [ - 116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 120, 50, - ], - }, - startKey: 'startKeyClosed', - endKeyOpen: { - type: 'Buffer', - data: [ - 116, 101, 115, 116, 45, 112, 114, 101, 102, 105, 120, 51, - ], - }, - endKey: 'endKeyOpen', - }, - ], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, - }, - requestOrder: [0, 0, 0, 0], - callCount: 4, - }, - }, -}; - -exports[ - 'Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a list of ranges 1' -] = { - input: { - code: 4, - message: { - ranges: [ - { - start: 'test-start-1', - end: 'test-end-1', - }, - { - start: 'test-start-2', - end: 'test-end-2', - }, - ], - }, - }, - output: { - results: { - result: 'error', - }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ - { - startKeyClosed: { - type: 'Buffer', - data: [ - 116, 101, 115, 116, 45, 115, 116, 97, 114, 116, 45, 49, - ], - }, - startKey: 'startKeyClosed', - endKeyClosed: { - type: 'Buffer', - data: [116, 101, 115, 116, 45, 101, 110, 100, 45, 49], - }, - endKey: 'endKeyClosed', - }, - { - startKeyClosed: { - type: 'Buffer', - data: [ - 116, 101, 115, 116, 45, 115, 116, 97, 114, 116, 45, 50, - ], - }, - startKey: 'startKeyClosed', - endKeyClosed: { - type: 'Buffer', - data: [116, 101, 115, 116, 45, 101, 110, 100, 45, 50], - }, - endKey: 'endKeyClosed', - }, - ], - }, - filter: null, rowsLimit: '0', appProfileId: '', }, }, - requestOrder: [0, 0, 0, 0], - callCount: 4, + requestOrder: [0], + callCount: 1, }, }, }; diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts new file mode 100644 index 000000000..6c74cf3e1 --- /dev/null +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -0,0 +1,136 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + +import {SameCallHandler} from './same-call-handler'; +import {MockService} from '../../../mock-service'; +import {Mutation} from '../../../../../mutation'; +import {Row} from '../../../../../row'; + +function rowResponse(rowKey: {}) { + return { + rowKey: Mutation.convertToBytes(rowKey), + familyName: {value: 'family'}, + qualifier: {value: Mutation.convertToBytes('qualifier')}, + valueSize: 0, + timestampMicros: 0, + labels: [], + commitRow: true, + value: Mutation.convertToBytes('value'), + }; +} + +export interface DataResponse { + row_keys: string[]; + last_row_key: string; + end_with_error: number; +} + +export interface ReadRowsResponse { + data?: DataResponse; + error_on_call?: number; +} + +export class ReadRowsHandler extends SameCallHandler { + responses: ReadRowsResponse[]; + request: any = null; + callCount = 0; + message: any; + lastCall: any; + + // TODO: service and endpoint should be bundled into one object. + constructor( + service: MockService, + endpoint: string, + responses: ReadRowsResponse[], + message?: any + ) { + super(service, endpoint); + this.responses = responses; + this.message = Object.assign({}, message); + } + + // TODO: Create interface for this. + callHandler(call: any) { + const lastResponse = this.responses[this.callCount - 1]; + // Send data if it is provided + const data = lastResponse.data; + if (data) { + // TODO: Issue warning if empty data is provided as stream may get stuck + const grpcResponse = { + chunks: data.row_keys.map(rowResponse), + lastScannedRowKey: Mutation.convertToBytes(data.last_row_key), + }; + call.write(grpcResponse); + } + // Send an error right away + const errorCode = lastResponse.error_on_call; + if (errorCode) { + call.emit('error', { + code: errorCode, + details: 'Details for a particular type of error', + }); + } + // Set a timer and send an error if we are confident that all data has been sent back to the user + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + function checkCollected() { + // Send the error if all data was collected + const lastIndex = self.data.length - 1; + const lastResponse = self.responses[self.callCount - 1]; + const lastResponseData = lastResponse.data; + if (lastResponseData) { + if (self.data[lastIndex].length === lastResponseData.row_keys.length) { + const errorCode = lastResponseData.end_with_error; + if (errorCode) { + call.emit('error', { + code: errorCode, + details: 'Details for a particular type of error', + }); + } + } else { + startTimer(); + } + } else { + throw Error('Response data should have been provided in the test'); + } + } + function startTimer() { + setTimeout(checkCollected, 2500); + } + startTimer(); + } + + addData(data: Row) { + // Add data collected from the stream + const lastIndex = this.data.length - 1; + this.data[lastIndex].push(data); + } + + snapshot(results: any): any { + return { + input: Object.assign( + {responses: this.responses}, + this.message ? {message: this.message} : null + ), + output: { + results, + requestData: this.requests(), + }, + }; + } +} diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts index 8a1bd47a8..229462ec4 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts @@ -18,6 +18,7 @@ import {MockService} from '../../../mock-service'; import {ServiceHandler} from '../service-handler'; +import {Row} from '../../../../../row'; const equal = require('deep-equal'); @@ -28,6 +29,7 @@ export abstract class SameCallHandler extends ServiceHandler { requestOrder: number[] = []; callCount = 0; endpoint: string; + data: Row[][] = []; protected constructor(service: MockService, endpoint: string) { super(); @@ -50,6 +52,7 @@ export abstract class SameCallHandler extends ServiceHandler { } this.request = callRequest; this.callCount++; + this.data.push([]); this.callHandler(call); }; this.service.setService({ @@ -58,8 +61,13 @@ export abstract class SameCallHandler extends ServiceHandler { }); } + getData() { + return this.data; + } + requests() { return { + data: this.data, requests: Object.assign({}, this.requestList), requestOrder: this.requestOrder, callCount: this.callCount, diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts index a796ea465..89b658739 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -19,6 +19,7 @@ import {grpc} from 'google-gax'; import {SameCallHandler} from './same-call-handler'; import {MockService} from '../../../mock-service'; +import {Row} from '../../../../../row'; export class SendErrorHandler extends SameCallHandler { code: grpc.status; @@ -45,6 +46,11 @@ export class SendErrorHandler extends SameCallHandler { }); } + addData(data: Row) { + const lastIndex = this.data.length - 1; + this.data[lastIndex].push(data); + } + snapshot(results: any): any { return { input: Object.assign( diff --git a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts index 01a0d6bcf..5739e5b98 100644 --- a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts @@ -16,6 +16,8 @@ // ** https://github.com/googleapis/gapic-generator-typescript ** // ** All changes to this file may be overwritten. ** +import {Row} from '../../../../row'; + export abstract class ServiceHandler { /* callHandler accepts a grpc call and provides behaviour for that grpc call @@ -35,4 +37,14 @@ export abstract class ServiceHandler { a running test. */ abstract setupService(): void; + + /* + addData is called to add data which will be reported in the snapshot later on. + */ + abstract addData(data: Row): void; + + /* + getData is called to get all data which was collected from requests. + */ + abstract getData(): Row[][]; } diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/src/util/mock-servers/service-testers/stream-tester.ts index c79cee096..72c6e4038 100644 --- a/src/util/mock-servers/service-testers/stream-tester.ts +++ b/src/util/mock-servers/service-testers/stream-tester.ts @@ -19,10 +19,9 @@ import {ServiceHandler} from './service-handlers/service-handler'; import {StreamFetcher} from './stream-fetchers/stream-fetcher'; import {ServiceError} from 'google-gax'; -import {Row} from '../../../row'; -const concat = require('concat-stream'); import * as snapshot from 'snap-shot-it'; +import {Row} from '../../../row'; export class StreamTester { serviceHandler: ServiceHandler; @@ -37,21 +36,26 @@ export class StreamTester { const collectSnapshot = (results: any) => { snapshot(this.serviceHandler.snapshot(results)); }; + const getData = (result: string) => { + return { + result, + data: this.serviceHandler + .getData() + .map(rows => rows.map(row => row.id)), + }; + }; this.serviceHandler.setupService(); - this.streamFetcher - .fetchStream() + const fetchedStream = this.streamFetcher.fetchStream(); + fetchedStream .on('error', (error: ServiceError) => { - collectSnapshot({result: 'error'}); + collectSnapshot(getData('error')); callback(); }) - .pipe( - concat((rows: Row[]) => { - collectSnapshot({ - result: 'data', - data: rows, - }); - callback(); - }) - ); + .on('data', (message: Row) => { + this.serviceHandler.addData(message); + }); + // TODO: Find a meaningful way to test stream ending + // TODO: We need to find a way to trigger the end of the test or errors + // after a certain amount of data has been sent back } } diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 11ae0247d..0802bb5c4 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -28,6 +28,10 @@ import {ReadRowsFetcher} from '../../src/util/mock-servers/service-testers/strea import {StreamTester} from '../../src/util/mock-servers/service-testers/stream-tester'; import {ServiceHandler} from '../../src/util/mock-servers/service-testers/service-handlers/service-handler'; import {Table} from '../../src/table'; +import { + ReadRowsHandler, + ReadRowsResponse, +} from '../../src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler'; // import {testGaxOptions} from './test-options'; describe('Bigtable/ReadRows', () => { @@ -55,7 +59,7 @@ describe('Bigtable/ReadRows', () => { const streamFetcher = new ReadRowsFetcher(table, opts); return new StreamTester(serviceHandler, streamFetcher); } - + /* describe('with a mock server that always sends an error back', () => { function checkRetryWithServer(code: grpc.status, callback: () => void) { const serviceHandler = new SendErrorHandler(service, 'ReadRows', code); @@ -123,11 +127,9 @@ describe('Bigtable/ReadRows', () => { it('should pass checks with keys', done => { checkWithOptions({keys: ['test-key-1', 'test-key-2']}, done); }); - /* it('should pass checks with a filter', done => { - checkWithOptions({filter: [{}]}, done); + checkWithOptions({filter: [{column: 'columnPrefix'}]}, done); }); - */ it('should pass checks with a limit', done => { checkWithOptions({limit: 10}, done); }); @@ -154,7 +156,7 @@ describe('Bigtable/ReadRows', () => { done ); }); - /* + ///* it('should pass checks with gaxOptions', done => { // TODO: Add the retry parameter checkWithOptions( @@ -164,7 +166,40 @@ describe('Bigtable/ReadRows', () => { done ); }); - */ + + }); + }); + */ + describe('with custom responses and createReadStream arguments', () => { + function getServiceHandler(responses: ReadRowsResponse[], message: any) { + return new ReadRowsHandler(service, 'ReadRows', responses, message); + } + function checkWithOptions( + responses: ReadRowsResponse[], + opts: any, + callback: () => void + ) { + const serviceHandler = getServiceHandler(responses, opts); + const streamTester = getStreamTester(serviceHandler, opts); + streamTester.checkSnapshots(callback); + } + it('should pass checks with a simple call', done => { + checkWithOptions( + [ + { + data: { + row_keys: ['a', 'b', 'c'], + last_row_key: 'c', + end_with_error: grpc.status.DEADLINE_EXCEEDED, + }, + }, + ], + { + rowKeys: [], + rowRanges: [{}], + }, + done + ); }); }); after(async () => { From 629c3d526883ea6122c5e5184553d89b61ee865c Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 6 Jul 2022 10:39:26 -0400 Subject: [PATCH 37/51] Attempt to avoid race condition --- .../implementation/read-rows-handler.ts | 19 ++++++++++++------- .../service-testers/stream-tester.ts | 2 ++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts index 6c74cf3e1..e2735af36 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -50,7 +50,6 @@ export class ReadRowsHandler extends SameCallHandler { request: any = null; callCount = 0; message: any; - lastCall: any; // TODO: service and endpoint should be bundled into one object. constructor( @@ -70,7 +69,6 @@ export class ReadRowsHandler extends SameCallHandler { // Send data if it is provided const data = lastResponse.data; if (data) { - // TODO: Issue warning if empty data is provided as stream may get stuck const grpcResponse = { chunks: data.row_keys.map(rowResponse), lastScannedRowKey: Mutation.convertToBytes(data.last_row_key), @@ -85,10 +83,16 @@ export class ReadRowsHandler extends SameCallHandler { details: 'Details for a particular type of error', }); } + call.emit('error', { + code: errorCode, + details: 'Details for a particular type of error', + }); // Set a timer and send an error if we are confident that all data has been sent back to the user // eslint-disable-next-line @typescript-eslint/no-this-alias + /* const self = this; - function checkCollected() { + const savedCall = call; + const checkCollected = () => { // Send the error if all data was collected const lastIndex = self.data.length - 1; const lastResponse = self.responses[self.callCount - 1]; @@ -97,7 +101,7 @@ export class ReadRowsHandler extends SameCallHandler { if (self.data[lastIndex].length === lastResponseData.row_keys.length) { const errorCode = lastResponseData.end_with_error; if (errorCode) { - call.emit('error', { + savedCall.emit('error', { code: errorCode, details: 'Details for a particular type of error', }); @@ -108,11 +112,12 @@ export class ReadRowsHandler extends SameCallHandler { } else { throw Error('Response data should have been provided in the test'); } - } - function startTimer() { + }; + const startTimer = () => { setTimeout(checkCollected, 2500); - } + }; startTimer(); + */ } addData(data: Row) { diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/src/util/mock-servers/service-testers/stream-tester.ts index 72c6e4038..5b6555438 100644 --- a/src/util/mock-servers/service-testers/stream-tester.ts +++ b/src/util/mock-servers/service-testers/stream-tester.ts @@ -48,10 +48,12 @@ export class StreamTester { const fetchedStream = this.streamFetcher.fetchStream(); fetchedStream .on('error', (error: ServiceError) => { + console.log('error received'); collectSnapshot(getData('error')); callback(); }) .on('data', (message: Row) => { + console.log('data received'); this.serviceHandler.addData(message); }); // TODO: Find a meaningful way to test stream ending From 0e61aab9a483d349ce8395ecbdae61ea78b4b348 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 6 Jul 2022 17:23:43 -0400 Subject: [PATCH 38/51] make a small call handler function --- .../implementation/read-rows-handler.ts | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts index e2735af36..d34f18a4b 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -64,32 +64,36 @@ export class ReadRowsHandler extends SameCallHandler { } // TODO: Create interface for this. + // callHandler(call: any) { + // const lastResponse = this.responses[this.callCount - 1]; + // // Send data if it is provided + // const data = lastResponse.data; + // if (data) { + // const grpcResponse = { + // chunks: data.row_keys.map(rowResponse), + // lastScannedRowKey: Mutation.convertToBytes(data.last_row_key), + // }; + // call.write(grpcResponse); + // } + // // Send an error right away + // const errorCode = lastResponse.error_on_call; + // if (errorCode) { + // call.emit('error', { + // code: errorCode, + // details: 'Details for a particular type of error', + // }); + // } callHandler(call: any) { - const lastResponse = this.responses[this.callCount - 1]; - // Send data if it is provided - const data = lastResponse.data; - if (data) { - const grpcResponse = { - chunks: data.row_keys.map(rowResponse), - lastScannedRowKey: Mutation.convertToBytes(data.last_row_key), - }; - call.write(grpcResponse); - } - // Send an error right away - const errorCode = lastResponse.error_on_call; - if (errorCode) { + setTimeout(() => { call.emit('error', { - code: errorCode, + code: 4, details: 'Details for a particular type of error', }); - } - call.emit('error', { - code: errorCode, - details: 'Details for a particular type of error', - }); - // Set a timer and send an error if we are confident that all data has been sent back to the user - // eslint-disable-next-line @typescript-eslint/no-this-alias - /* + }, 2500); + } + // Set a timer and send an error if we are confident that all data has been sent back to the user + // eslint-disable-next-line @typescript-eslint/no-this-alias + /* const self = this; const savedCall = call; const checkCollected = () => { @@ -118,7 +122,7 @@ export class ReadRowsHandler extends SameCallHandler { }; startTimer(); */ - } + // } addData(data: Row) { // Add data collected from the stream From 0f0129122cd90fd44eef39d61f07fe356885a5f7 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 7 Jul 2022 09:55:31 -0400 Subject: [PATCH 39/51] this works --- .../implementation/read-rows-handler.ts | 113 ++++++++++-------- 1 file changed, 65 insertions(+), 48 deletions(-) diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts index d34f18a4b..5d7d3fea0 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -62,67 +62,84 @@ export class ReadRowsHandler extends SameCallHandler { this.responses = responses; this.message = Object.assign({}, message); } - - // TODO: Create interface for this. - // callHandler(call: any) { - // const lastResponse = this.responses[this.callCount - 1]; - // // Send data if it is provided - // const data = lastResponse.data; - // if (data) { - // const grpcResponse = { - // chunks: data.row_keys.map(rowResponse), - // lastScannedRowKey: Mutation.convertToBytes(data.last_row_key), - // }; - // call.write(grpcResponse); - // } - // // Send an error right away - // const errorCode = lastResponse.error_on_call; - // if (errorCode) { - // call.emit('error', { - // code: errorCode, - // details: 'Details for a particular type of error', - // }); - // } + /* callHandler(call: any) { - setTimeout(() => { + const checkCollected = () => { + console.log('emit error'); call.emit('error', { code: 4, details: 'Details for a particular type of error', }); - }, 2500); + }; + const startTimer = () => { + setTimeout(checkCollected, 2500); + }; + startTimer(); } - // Set a timer and send an error if we are confident that all data has been sent back to the user - // eslint-disable-next-line @typescript-eslint/no-this-alias - /* - const self = this; - const savedCall = call; + */ + // TODO: Create interface for this. + callHandler(call: any) { + // const lastResponse = this.responses[this.callCount - 1]; + // Send data if it is provided + // const data = lastResponse.data; + // if (data) { + // const grpcResponse = { + // chunks: data.row_keys.map(rowResponse), + // lastScannedRowKey: Mutation.convertToBytes(data.last_row_key), + // }; + // call.write(grpcResponse); + // } + // Send an error right away + // const errorCode = lastResponse.error_on_call; + // if (errorCode) { + // call.emit('error', { + // code: errorCode, + // details: 'Details for a particular type of error', + // }); + // } + // callHandler(call: any) { + // const self = this; + // const sendError = () => { + // console.log(self.data); + // call.emit('error', { + // code: 4, + // details: 'Details for a particular type of error', + // }); + // }; + // setTimeout(sendError, 2500); + // } + // Set a timer and send an error if we are confident that all data has been sent back to the user + // eslint-disable-next-line @typescript-eslint/no-this-alias + // const self = this; const checkCollected = () => { // Send the error if all data was collected - const lastIndex = self.data.length - 1; - const lastResponse = self.responses[self.callCount - 1]; - const lastResponseData = lastResponse.data; - if (lastResponseData) { - if (self.data[lastIndex].length === lastResponseData.row_keys.length) { - const errorCode = lastResponseData.end_with_error; - if (errorCode) { - savedCall.emit('error', { - code: errorCode, - details: 'Details for a particular type of error', - }); - } - } else { - startTimer(); - } - } else { - throw Error('Response data should have been provided in the test'); - } + // const lastIndex = self.data.length - 1; + // const lastResponse = self.responses[self.callCount - 1]; + // const lastResponseData = lastResponse.data; + // if (lastResponseData) { + // console.log(self.data[lastIndex]); + // if (self.data[lastIndex].length === lastResponseData.row_keys.length) { + // const errorCode = lastResponseData.end_with_error; + // if (errorCode) { + // console.log('emit error'); + console.log('emit error'); + call.emit('error', { + code: 4, + details: 'Details for a particular type of error', + }); + // } + // } else { + // startTimer(); + // } + // } else { + // throw Error('Response data should have been provided in the test'); + //} }; const startTimer = () => { setTimeout(checkCollected, 2500); }; startTimer(); - */ - // } + } addData(data: Row) { // Add data collected from the stream From 054bdf830838be780f171131923a4d0c33100c76 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 7 Jul 2022 10:42:10 -0400 Subject: [PATCH 40/51] Working snapshot with time delay. --- .../implementation/read-rows-handler.ts | 63 +++++++++---------- .../implementation/same-call-handler.ts | 2 +- .../implementation/send-error-handler.ts | 2 +- .../service-handlers/service-handler.ts | 6 +- .../service-testers/stream-tester.ts | 6 +- 5 files changed, 37 insertions(+), 42 deletions(-) diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts index 5d7d3fea0..ea0e2b77b 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -79,16 +79,16 @@ export class ReadRowsHandler extends SameCallHandler { */ // TODO: Create interface for this. callHandler(call: any) { - // const lastResponse = this.responses[this.callCount - 1]; + const lastResponse = this.responses[this.callCount - 1]; // Send data if it is provided - // const data = lastResponse.data; - // if (data) { - // const grpcResponse = { - // chunks: data.row_keys.map(rowResponse), - // lastScannedRowKey: Mutation.convertToBytes(data.last_row_key), - // }; - // call.write(grpcResponse); - // } + const data = lastResponse.data; + if (data) { + const grpcResponse = { + chunks: data.row_keys.map(rowResponse), + lastScannedRowKey: Mutation.convertToBytes(data.last_row_key), + }; + call.write(grpcResponse); + } // Send an error right away // const errorCode = lastResponse.error_on_call; // if (errorCode) { @@ -110,30 +110,29 @@ export class ReadRowsHandler extends SameCallHandler { // } // Set a timer and send an error if we are confident that all data has been sent back to the user // eslint-disable-next-line @typescript-eslint/no-this-alias - // const self = this; + const self = this; const checkCollected = () => { // Send the error if all data was collected - // const lastIndex = self.data.length - 1; - // const lastResponse = self.responses[self.callCount - 1]; - // const lastResponseData = lastResponse.data; - // if (lastResponseData) { - // console.log(self.data[lastIndex]); - // if (self.data[lastIndex].length === lastResponseData.row_keys.length) { - // const errorCode = lastResponseData.end_with_error; - // if (errorCode) { - // console.log('emit error'); - console.log('emit error'); - call.emit('error', { - code: 4, - details: 'Details for a particular type of error', - }); - // } - // } else { - // startTimer(); - // } - // } else { - // throw Error('Response data should have been provided in the test'); - //} + const lastIndex = self.data.length - 1; + const lastResponse = self.responses[self.callCount - 1]; + const lastResponseData = lastResponse.data; + if (lastResponseData) { + console.log(self.data[lastIndex]); + if (self.data[lastIndex].length === lastResponseData.row_keys.length) { + const errorCode = lastResponseData.end_with_error; + if (errorCode) { + console.log('emit error'); + call.emit('error', { + code: 2, + details: 'Details for a particular type of error', + }); + } + } else { + startTimer(); + } + } else { + throw Error('Response data should have been provided in the test'); + } }; const startTimer = () => { setTimeout(checkCollected, 2500); @@ -141,7 +140,7 @@ export class ReadRowsHandler extends SameCallHandler { startTimer(); } - addData(data: Row) { + addData(data: string) { // Add data collected from the stream const lastIndex = this.data.length - 1; this.data[lastIndex].push(data); diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts index 229462ec4..ff8093b9a 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts @@ -29,7 +29,7 @@ export abstract class SameCallHandler extends ServiceHandler { requestOrder: number[] = []; callCount = 0; endpoint: string; - data: Row[][] = []; + data: string[][] = []; protected constructor(service: MockService, endpoint: string) { super(); diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts index 89b658739..0fc3ccefd 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -46,7 +46,7 @@ export class SendErrorHandler extends SameCallHandler { }); } - addData(data: Row) { + addData(data: string) { const lastIndex = this.data.length - 1; this.data[lastIndex].push(data); } diff --git a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts index 5739e5b98..742316b25 100644 --- a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/service-handler.ts @@ -16,8 +16,6 @@ // ** https://github.com/googleapis/gapic-generator-typescript ** // ** All changes to this file may be overwritten. ** -import {Row} from '../../../../row'; - export abstract class ServiceHandler { /* callHandler accepts a grpc call and provides behaviour for that grpc call @@ -41,10 +39,10 @@ export abstract class ServiceHandler { /* addData is called to add data which will be reported in the snapshot later on. */ - abstract addData(data: Row): void; + abstract addData(data: string): void; /* getData is called to get all data which was collected from requests. */ - abstract getData(): Row[][]; + abstract getData(): string[][]; } diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/src/util/mock-servers/service-testers/stream-tester.ts index 5b6555438..8f670ef3e 100644 --- a/src/util/mock-servers/service-testers/stream-tester.ts +++ b/src/util/mock-servers/service-testers/stream-tester.ts @@ -39,9 +39,7 @@ export class StreamTester { const getData = (result: string) => { return { result, - data: this.serviceHandler - .getData() - .map(rows => rows.map(row => row.id)), + data: this.serviceHandler.getData(), }; }; this.serviceHandler.setupService(); @@ -54,7 +52,7 @@ export class StreamTester { }) .on('data', (message: Row) => { console.log('data received'); - this.serviceHandler.addData(message); + this.serviceHandler.addData(message.id); }); // TODO: Find a meaningful way to test stream ending // TODO: We need to find a way to trigger the end of the test or errors From f97d4db9357f16c8fffd37eeefc11df6fa98bbd6 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 7 Jul 2022 10:45:32 -0400 Subject: [PATCH 41/51] new snapshot --- __snapshots__/read-rows.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index 53a1fcb36..6a361bba1 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -4,8 +4,11 @@ exports[ input: { responses: [ { - row_keys: ['a', 'b', 'c'], - last_row_key: 'c', + data: { + row_keys: ['a', 'b', 'c'], + last_row_key: 'c', + end_with_error: 4, + }, }, ], message: { @@ -15,10 +18,11 @@ exports[ }, output: { results: { - result: 'data', - data: ['a', 'b', 'c'], + result: 'error', + data: [['a', 'b', 'c']], }, requestData: { + data: [['a', 'b', 'c']], requests: { 0: { tableName: From f6240ffde05b2403a21ed91036b01744f86ed625 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 7 Jul 2022 11:16:16 -0400 Subject: [PATCH 42/51] remove unused imports --- .../service-handlers/implementation/read-rows-handler.ts | 1 - .../service-handlers/implementation/same-call-handler.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts index ea0e2b77b..1cf06452b 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -19,7 +19,6 @@ import {SameCallHandler} from './same-call-handler'; import {MockService} from '../../../mock-service'; import {Mutation} from '../../../../../mutation'; -import {Row} from '../../../../../row'; function rowResponse(rowKey: {}) { return { diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts index ff8093b9a..1c94bedb0 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts @@ -18,7 +18,6 @@ import {MockService} from '../../../mock-service'; import {ServiceHandler} from '../service-handler'; -import {Row} from '../../../../../row'; const equal = require('deep-equal'); From f1ae2bf49fa06c16d1222cc332920b783e234a2d Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 7 Jul 2022 13:27:23 -0400 Subject: [PATCH 43/51] Change snapshot structure --- __snapshots__/read-rows.js | 9 +++---- .../implementation/read-rows-handler.ts | 24 +++++++------------ .../implementation/same-call-handler.ts | 1 - test/server/read-rows.ts | 8 +++---- 4 files changed, 14 insertions(+), 28 deletions(-) diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index 6a361bba1..b4491a705 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -4,11 +4,9 @@ exports[ input: { responses: [ { - data: { - row_keys: ['a', 'b', 'c'], - last_row_key: 'c', - end_with_error: 4, - }, + row_keys: ['a', 'b', 'c'], + last_row_key: 'c', + end_with_error: 13, }, ], message: { @@ -22,7 +20,6 @@ exports[ data: [['a', 'b', 'c']], }, requestData: { - data: [['a', 'b', 'c']], requests: { 0: { tableName: diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts index 1cf06452b..4088599ac 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -33,17 +33,12 @@ function rowResponse(rowKey: {}) { }; } -export interface DataResponse { +export interface ReadRowsResponse { row_keys: string[]; last_row_key: string; end_with_error: number; } -export interface ReadRowsResponse { - data?: DataResponse; - error_on_call?: number; -} - export class ReadRowsHandler extends SameCallHandler { responses: ReadRowsResponse[]; request: any = null; @@ -79,12 +74,10 @@ export class ReadRowsHandler extends SameCallHandler { // TODO: Create interface for this. callHandler(call: any) { const lastResponse = this.responses[this.callCount - 1]; - // Send data if it is provided - const data = lastResponse.data; - if (data) { + if (lastResponse) { const grpcResponse = { - chunks: data.row_keys.map(rowResponse), - lastScannedRowKey: Mutation.convertToBytes(data.last_row_key), + chunks: lastResponse.row_keys.map(rowResponse), + lastScannedRowKey: Mutation.convertToBytes(lastResponse.last_row_key), }; call.write(grpcResponse); } @@ -114,15 +107,14 @@ export class ReadRowsHandler extends SameCallHandler { // Send the error if all data was collected const lastIndex = self.data.length - 1; const lastResponse = self.responses[self.callCount - 1]; - const lastResponseData = lastResponse.data; - if (lastResponseData) { + if (lastResponse) { console.log(self.data[lastIndex]); - if (self.data[lastIndex].length === lastResponseData.row_keys.length) { - const errorCode = lastResponseData.end_with_error; + if (self.data[lastIndex].length === lastResponse.row_keys.length) { + const errorCode = lastResponse.end_with_error; if (errorCode) { console.log('emit error'); call.emit('error', { - code: 2, + code: errorCode, details: 'Details for a particular type of error', }); } diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts index 1c94bedb0..1bcb30d30 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts @@ -66,7 +66,6 @@ export abstract class SameCallHandler extends ServiceHandler { requests() { return { - data: this.data, requests: Object.assign({}, this.requestList), requestOrder: this.requestOrder, callCount: this.callCount, diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 0802bb5c4..18cbd542b 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -187,11 +187,9 @@ describe('Bigtable/ReadRows', () => { checkWithOptions( [ { - data: { - row_keys: ['a', 'b', 'c'], - last_row_key: 'c', - end_with_error: grpc.status.DEADLINE_EXCEEDED, - }, + row_keys: ['a', 'b', 'c'], + last_row_key: 'c', + end_with_error: grpc.status.INTERNAL, }, ], { From 0c445c4807d40717058156b9abac2dca4e052fac Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 7 Jul 2022 13:30:35 -0400 Subject: [PATCH 44/51] Remove unused code --- .../implementation/read-rows-handler.ts | 35 +------------------ 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts index 4088599ac..867f1dbb7 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -56,21 +56,7 @@ export class ReadRowsHandler extends SameCallHandler { this.responses = responses; this.message = Object.assign({}, message); } - /* - callHandler(call: any) { - const checkCollected = () => { - console.log('emit error'); - call.emit('error', { - code: 4, - details: 'Details for a particular type of error', - }); - }; - const startTimer = () => { - setTimeout(checkCollected, 2500); - }; - startTimer(); - } - */ + // TODO: Create interface for this. callHandler(call: any) { const lastResponse = this.responses[this.callCount - 1]; @@ -81,25 +67,6 @@ export class ReadRowsHandler extends SameCallHandler { }; call.write(grpcResponse); } - // Send an error right away - // const errorCode = lastResponse.error_on_call; - // if (errorCode) { - // call.emit('error', { - // code: errorCode, - // details: 'Details for a particular type of error', - // }); - // } - // callHandler(call: any) { - // const self = this; - // const sendError = () => { - // console.log(self.data); - // call.emit('error', { - // code: 4, - // details: 'Details for a particular type of error', - // }); - // }; - // setTimeout(sendError, 2500); - // } // Set a timer and send an error if we are confident that all data has been sent back to the user // eslint-disable-next-line @typescript-eslint/no-this-alias const self = this; From 35bc274ab5dd223c7cd84d15c80d6909cf43dc79 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 7 Jul 2022 15:00:12 -0400 Subject: [PATCH 45/51] Added a couple more tests --- __snapshots__/read-rows.js | 112 ++++++++++++++++++ .../implementation/read-rows-handler.ts | 35 +++--- .../service-testers/stream-tester.ts | 10 +- test/server/read-rows.ts | 32 +++++ 4 files changed, 169 insertions(+), 20 deletions(-) diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index b4491a705..4256798d3 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -38,3 +38,115 @@ exports[ }, }, }; + +exports[ + 'Bigtable/ReadRows with custom responses and createReadStream arguments retries a failed read 1' +] = { + input: { + responses: [ + { + row_keys: ['a', 'b'], + last_row_key: 'c', + end_with_error: 4, + }, + { + row_keys: ['c'], + last_row_key: 'c', + }, + ], + message: { + rowKeys: [], + rowRanges: [{}], + }, + }, + output: { + results: { + result: 'end stream', + data: [['a', 'b'], ['c']], + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + 1: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [ + { + startKeyOpen: { + type: 'Buffer', + data: [99], + }, + startKey: 'startKeyOpen', + }, + ], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + }, + requestOrder: [0, 1], + callCount: 2, + }, + }, +}; + +exports[ + 'Bigtable/ReadRows with custom responses and createReadStream arguments fails after all available retries 1' +] = { + input: { + responses: [ + { + end_with_error: 4, + }, + { + end_with_error: 4, + }, + { + end_with_error: 4, + }, + { + end_with_error: 4, + }, + ], + message: { + rowKeys: [], + rowRanges: [{}], + }, + }, + output: { + results: { + result: 'error', + data: [[], [], [], []], + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + }, + requestOrder: [0, 0, 0, 0], + callCount: 4, + }, + }, +}; diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts index 867f1dbb7..7b47f9225 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -34,9 +34,9 @@ function rowResponse(rowKey: {}) { } export interface ReadRowsResponse { - row_keys: string[]; - last_row_key: string; - end_with_error: number; + row_keys?: string[]; + last_row_key?: string; + end_with_error?: number; } export class ReadRowsHandler extends SameCallHandler { @@ -60,7 +60,7 @@ export class ReadRowsHandler extends SameCallHandler { // TODO: Create interface for this. callHandler(call: any) { const lastResponse = this.responses[this.callCount - 1]; - if (lastResponse) { + if (lastResponse && lastResponse.row_keys) { const grpcResponse = { chunks: lastResponse.row_keys.map(rowResponse), lastScannedRowKey: Mutation.convertToBytes(lastResponse.last_row_key), @@ -70,23 +70,30 @@ export class ReadRowsHandler extends SameCallHandler { // Set a timer and send an error if we are confident that all data has been sent back to the user // eslint-disable-next-line @typescript-eslint/no-this-alias const self = this; + const endRequest = (lastResponse: any) => { + const errorCode = lastResponse.end_with_error; + if (errorCode) { + call.emit('error', { + code: errorCode, + details: 'Details for a particular type of error', + }); + } else { + call.end(); + } + }; const checkCollected = () => { // Send the error if all data was collected const lastIndex = self.data.length - 1; const lastResponse = self.responses[self.callCount - 1]; if (lastResponse) { - console.log(self.data[lastIndex]); - if (self.data[lastIndex].length === lastResponse.row_keys.length) { - const errorCode = lastResponse.end_with_error; - if (errorCode) { - console.log('emit error'); - call.emit('error', { - code: errorCode, - details: 'Details for a particular type of error', - }); + if (lastResponse.row_keys) { + if (self.data[lastIndex].length === lastResponse.row_keys.length) { + endRequest(lastResponse); + } else { + startTimer(); } } else { - startTimer(); + endRequest(lastResponse); } } else { throw Error('Response data should have been provided in the test'); diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/src/util/mock-servers/service-testers/stream-tester.ts index 8f670ef3e..2e647f91c 100644 --- a/src/util/mock-servers/service-testers/stream-tester.ts +++ b/src/util/mock-servers/service-testers/stream-tester.ts @@ -35,6 +35,7 @@ export class StreamTester { checkSnapshots(callback: () => void): void { const collectSnapshot = (results: any) => { snapshot(this.serviceHandler.snapshot(results)); + callback(); }; const getData = (result: string) => { return { @@ -46,16 +47,13 @@ export class StreamTester { const fetchedStream = this.streamFetcher.fetchStream(); fetchedStream .on('error', (error: ServiceError) => { - console.log('error received'); collectSnapshot(getData('error')); - callback(); }) .on('data', (message: Row) => { - console.log('data received'); this.serviceHandler.addData(message.id); + }) + .on('end', (message: Row) => { + collectSnapshot(getData('end stream')); }); - // TODO: Find a meaningful way to test stream ending - // TODO: We need to find a way to trigger the end of the test or errors - // after a certain amount of data has been sent back } } diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 18cbd542b..f78eb6db5 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -199,6 +199,38 @@ describe('Bigtable/ReadRows', () => { done ); }); + it('retries a failed read', done => { + checkWithOptions( + [ + { + row_keys: ['a', 'b'], + last_row_key: 'c', + end_with_error: grpc.status.DEADLINE_EXCEEDED, + }, + { + row_keys: ['c'], + last_row_key: 'c', + }, + ], + { + rowKeys: [], + rowRanges: [{}], + }, + done + ); + }); + it('fails after all available retries', done => { + checkWithOptions( + Array(4).fill({ + end_with_error: grpc.status.DEADLINE_EXCEEDED, + }), + { + rowKeys: [], + rowRanges: [{}], + }, + done + ); + }); }); after(async () => { server.shutdown(() => {}); From 0ddc3d785ed60f2cd3f902e52ec7427d947aaa8e Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 7 Jul 2022 15:06:33 -0400 Subject: [PATCH 46/51] Added another test for retries --- test/server/read-rows.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index f78eb6db5..deaf9632f 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -231,6 +231,18 @@ describe('Bigtable/ReadRows', () => { done ); }); + it('fails after all available retries', done => { + checkWithOptions( + Array(4).fill({ + end_with_error: grpc.status.DEADLINE_EXCEEDED, + }), + { + rowKeys: [], + rowRanges: [{}], + }, + done + ); + }); }); after(async () => { server.shutdown(() => {}); From 5dca0f78c4be5a076b765c183c145d17960749f8 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 7 Jul 2022 15:50:35 -0400 Subject: [PATCH 47/51] Four more tests --- __snapshots__/read-rows.js | 399 +++++++++++++++++++++++++++++++++++++ test/server/read-rows.ts | 63 +++++- 2 files changed, 458 insertions(+), 4 deletions(-) diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index 4256798d3..a05a5b2e2 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -150,3 +150,402 @@ exports[ }, }, }; + +exports[ + 'Bigtable/ReadRows with custom responses and createReadStream arguments resets the retry counter after a successful read 1' +] = { + input: { + responses: [ + { + row_keys: ['a'], + last_row_key: 'a', + end_with_error: 4, + }, + { + end_with_error: 4, + }, + { + end_with_error: 4, + }, + { + end_with_error: 4, + }, + { + row_keys: ['b'], + last_row_key: 'b', + end_with_error: 4, + }, + { + end_with_error: 4, + }, + { + end_with_error: 4, + }, + { + row_keys: ['c'], + last_row_key: 'b', + }, + ], + message: { + rowKeys: [], + rowRanges: [{}], + }, + }, + output: { + results: { + result: 'error', + data: [['a'], [], [], []], + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + 1: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [ + { + startKeyOpen: { + type: 'Buffer', + data: [97], + }, + startKey: 'startKeyOpen', + }, + ], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + }, + requestOrder: [0, 1, 1, 1], + callCount: 4, + }, + }, +}; + +exports[ + 'Bigtable/ReadRows with custom responses and createReadStream arguments moves the start point of a range being consumed 1' +] = { + input: { + responses: [ + { + row_keys: ['a', 'b'], + end_with_error: 4, + }, + { + row_keys: ['c'], + }, + ], + message: { + ranges: [ + { + start: { + value: 'b', + inclusive: false, + }, + end: 'z', + }, + ], + }, + }, + output: { + results: { + result: 'end stream', + data: [['a', 'b'], ['c']], + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [ + { + startKeyClosed: { + type: 'Buffer', + data: [97], + }, + startKey: 'startKeyClosed', + endKeyClosed: { + type: 'Buffer', + data: [122], + }, + endKey: 'endKeyClosed', + }, + ], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + 1: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [ + { + startKeyOpen: { + type: 'Buffer', + data: [98], + }, + startKey: 'startKeyOpen', + endKeyClosed: { + type: 'Buffer', + data: [122], + }, + endKey: 'endKeyClosed', + }, + ], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + }, + requestOrder: [0, 1], + callCount: 2, + }, + }, +}; + +exports[ + 'Bigtable/ReadRows with custom responses and createReadStream arguments removes ranges already consumed 1' +] = { + input: { + responses: [ + { + row_keys: ['a', 'b', 'c'], + end_with_error: 4, + }, + { + row_keys: ['x'], + }, + ], + message: { + ranges: [ + { + start: 'x', + end: 'z', + }, + ], + }, + }, + output: { + results: { + result: 'end stream', + data: [['a', 'b', 'c'], ['x']], + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [ + { + startKeyClosed: { + type: 'Buffer', + data: [97], + }, + startKey: 'startKeyClosed', + endKeyClosed: { + type: 'Buffer', + data: [99], + }, + endKey: 'endKeyClosed', + }, + { + startKeyClosed: { + type: 'Buffer', + data: [120], + }, + startKey: 'startKeyClosed', + endKeyClosed: { + type: 'Buffer', + data: [122], + }, + endKey: 'endKeyClosed', + }, + ], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + 1: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [ + { + startKeyClosed: { + type: 'Buffer', + data: [120], + }, + startKey: 'startKeyClosed', + endKeyClosed: { + type: 'Buffer', + data: [122], + }, + endKey: 'endKeyClosed', + }, + ], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + }, + requestOrder: [0, 1], + callCount: 2, + }, + }, +}; + +exports[ + 'Bigtable/ReadRows with custom responses and createReadStream arguments removes keys already read 1' +] = { + input: { + responses: [ + { + row_keys: ['a', 'b', 'c'], + end_with_error: 4, + }, + { + row_keys: ['x'], + }, + ], + message: { + keys: ['a', 'b', 'x'], + }, + }, + output: { + results: { + result: 'end stream', + data: [['a', 'b', 'c'], ['x']], + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [ + { + type: 'Buffer', + data: [97], + }, + { + type: 'Buffer', + data: [98], + }, + { + type: 'Buffer', + data: [120], + }, + ], + rowRanges: [], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + 1: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [ + { + type: 'Buffer', + data: [120], + }, + ], + rowRanges: [], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + }, + requestOrder: [0, 1], + callCount: 2, + }, + }, +}; + +exports[ + 'Bigtable/ReadRows with custom responses and createReadStream arguments adjust the limit based on the number of rows read 1' +] = { + input: { + responses: [ + { + row_keys: ['a', 'b'], + end_with_error: 4, + }, + { + row_keys: ['x'], + }, + ], + message: { + limit: 10, + }, + }, + output: { + results: { + result: 'end stream', + data: [['a', 'b'], ['x']], + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '10', + appProfileId: '', + }, + 1: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [ + { + startKeyOpen: { + type: 'Buffer', + data: [98], + }, + startKey: 'startKeyOpen', + }, + ], + }, + filter: null, + rowsLimit: '8', + appProfileId: '', + }, + }, + requestOrder: [0, 1], + callCount: 2, + }, + }, +}; diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index deaf9632f..8044e38d7 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -231,11 +231,18 @@ describe('Bigtable/ReadRows', () => { done ); }); - it('fails after all available retries', done => { + it('resets the retry counter after a successful read', done => { checkWithOptions( - Array(4).fill({ - end_with_error: grpc.status.DEADLINE_EXCEEDED, - }), + [ + {row_keys: ['a'], last_row_key: 'a', end_with_error: 4}, + {end_with_error: 4}, + {end_with_error: 4}, + {end_with_error: 4}, + {row_keys: ['b'], last_row_key: 'b', end_with_error: 4}, + {end_with_error: 4}, + {end_with_error: 4}, + {row_keys: ['c'], last_row_key: 'b'}, + ], { rowKeys: [], rowRanges: [{}], @@ -243,6 +250,54 @@ describe('Bigtable/ReadRows', () => { done ); }); + it('moves the start point of a range being consumed', done => { + checkWithOptions( + [{row_keys: ['a', 'b'], end_with_error: 4}, {row_keys: ['c']}], + { + ranges: [ + { + start: 'a', + end: 'z', + }, + ], + }, + done + ); + }); + it('removes ranges already consumed', done => { + checkWithOptions( + [{row_keys: ['a', 'b', 'c'], end_with_error: 4}, {row_keys: ['x']}], + { + ranges: [ + { + start: 'a', + end: 'c', + }, + { + start: 'x', + end: 'z', + }, + ], + }, + done + ); + }); + it('removes keys already read', done => { + checkWithOptions( + [{row_keys: ['a', 'b', 'c'], end_with_error: 4}, {row_keys: ['x']}], + { + keys: ['a', 'b', 'x'], + }, + done + ); + }); + it('adjust the limit based on the number of rows read', done => { + checkWithOptions( + [{row_keys: ['a', 'b'], end_with_error: 4}, {row_keys: ['x']}], + {limit: 10}, + done + ); + }); }); after(async () => { server.shutdown(() => {}); From 61df10f21f4f49680e7742a890cd58aadc055b66 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 8 Jul 2022 13:51:20 -0400 Subject: [PATCH 48/51] generated another snapshot --- __snapshots__/read-rows.js | 47 +++++++++++++++++++ .../implementation/read-rows-handler.ts | 2 +- test/server/read-rows.ts | 13 ++++- 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index a05a5b2e2..e031dc8e6 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -549,3 +549,50 @@ exports[ }, }, }; + +exports[ + 'Bigtable/ReadRows with custom responses and createReadStream arguments respects the max retries parameter 1' +] = { + input: { + responses: [ + { + end_with_error: 4, + }, + { + end_with_error: 4, + }, + { + end_with_error: 4, + }, + { + end_with_error: 4, + }, + ], + message: { + maxRetries: 2, + }, + }, + output: { + results: { + result: 'error', + data: [[], [], [], []], + }, + requestData: { + requests: { + 0: { + tableName: + 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', + rows: { + rowKeys: [], + rowRanges: [{}], + }, + filter: null, + rowsLimit: '0', + appProfileId: '', + }, + }, + requestOrder: [0, 0, 0, 0], + callCount: 4, + }, + }, +}; diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts index 7b47f9225..6c6042c29 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts +++ b/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -96,7 +96,7 @@ export class ReadRowsHandler extends SameCallHandler { endRequest(lastResponse); } } else { - throw Error('Response data should have been provided in the test'); + throw Error('Ran out of requests to send'); } }; const startTimer = () => { diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 8044e38d7..b04d2cb7c 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -298,6 +298,18 @@ describe('Bigtable/ReadRows', () => { done ); }); + it('respects the max retries parameter', done => { + // NOTE: This test snapshot currently shows that maxRetries is not respected on a per call basis because three requests are required. + checkWithOptions( + Array(4).fill({ + end_with_error: grpc.status.DEADLINE_EXCEEDED, + }), + { + maxRetries: 2, + }, + done + ); + }); }); after(async () => { server.shutdown(() => {}); @@ -306,4 +318,3 @@ describe('Bigtable/ReadRows', () => { // TODO: Think of interesting cases for the shouldRetryFn // TODO: Consider setting up the framework so that we take snapshots of values passed into createReadStream afterwards -// TODO: Adjust max retries From f28813d0fa3963acfc38182c312c4d12a254cd8e Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 8 Jul 2022 14:16:28 -0400 Subject: [PATCH 49/51] Comment out a test --- __snapshots__/read-rows.js | 2052 ++++++++++++++++++++++++++++-------- test/server/read-rows.ts | 8 +- 2 files changed, 1597 insertions(+), 463 deletions(-) diff --git a/__snapshots__/read-rows.js b/__snapshots__/read-rows.js index e031dc8e6..d320126df 100644 --- a/__snapshots__/read-rows.js +++ b/__snapshots__/read-rows.js @@ -1,598 +1,1734 @@ -exports[ - 'Bigtable/ReadRows with custom responses and createReadStream arguments should pass checks with a simple call 1' -] = { - input: { - responses: [ +exports['Bigtable/ReadRows with custom responses and createReadStream arguments should pass checks with a simple call 1'] = { + "input": { + "responses": [ { - row_keys: ['a', 'b', 'c'], - last_row_key: 'c', - end_with_error: 13, - }, + "row_keys": [ + "a", + "b", + "c" + ], + "last_row_key": "c", + "end_with_error": 13 + } ], - message: { - rowKeys: [], - rowRanges: [{}], - }, + "message": { + "rowKeys": [], + "rowRanges": [ + {} + ] + } }, - output: { - results: { - result: 'error', - data: [['a', 'b', 'c']], + "output": { + "results": { + "result": "error", + "data": [ + [ + "a", + "b", + "c" + ] + ] }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } }, - requestOrder: [0], - callCount: 1, - }, - }, -}; + "requestOrder": [ + 0 + ], + "callCount": 1 + } + } +} -exports[ - 'Bigtable/ReadRows with custom responses and createReadStream arguments retries a failed read 1' -] = { - input: { - responses: [ +exports['Bigtable/ReadRows with custom responses and createReadStream arguments retries a failed read 1'] = { + "input": { + "responses": [ { - row_keys: ['a', 'b'], - last_row_key: 'c', - end_with_error: 4, + "row_keys": [ + "a", + "b" + ], + "last_row_key": "c", + "end_with_error": 4 }, { - row_keys: ['c'], - last_row_key: 'c', - }, + "row_keys": [ + "c" + ], + "last_row_key": "c" + } ], - message: { - rowKeys: [], - rowRanges: [{}], - }, + "message": { + "rowKeys": [], + "rowRanges": [ + {} + ] + } }, - output: { - results: { - result: 'end stream', - data: [['a', 'b'], ['c']], + "output": { + "results": { + "result": "end stream", + "data": [ + [ + "a", + "b" + ], + [ + "c" + ] + ] }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" }, - 1: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ + "1": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ { - startKeyOpen: { - type: 'Buffer', - data: [99], + "startKeyOpen": { + "type": "Buffer", + "data": [ + 99 + ] }, - startKey: 'startKeyOpen', - }, - ], + "startKey": "startKeyOpen" + } + ] }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } }, - requestOrder: [0, 1], - callCount: 2, - }, - }, -}; + "requestOrder": [ + 0, + 1 + ], + "callCount": 2 + } + } +} -exports[ - 'Bigtable/ReadRows with custom responses and createReadStream arguments fails after all available retries 1' -] = { - input: { - responses: [ +exports['Bigtable/ReadRows with custom responses and createReadStream arguments fails after all available retries 1'] = { + "input": { + "responses": [ { - end_with_error: 4, + "end_with_error": 4 }, { - end_with_error: 4, + "end_with_error": 4 }, { - end_with_error: 4, + "end_with_error": 4 }, { - end_with_error: 4, - }, + "end_with_error": 4 + } ], - message: { - rowKeys: [], - rowRanges: [{}], - }, + "message": { + "rowKeys": [], + "rowRanges": [ + {} + ] + } }, - output: { - results: { - result: 'error', - data: [[], [], [], []], + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } }, - requestOrder: [0, 0, 0, 0], - callCount: 4, - }, - }, -}; + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} -exports[ - 'Bigtable/ReadRows with custom responses and createReadStream arguments resets the retry counter after a successful read 1' -] = { - input: { - responses: [ +exports['Bigtable/ReadRows with custom responses and createReadStream arguments resets the retry counter after a successful read 1'] = { + "input": { + "responses": [ { - row_keys: ['a'], - last_row_key: 'a', - end_with_error: 4, + "row_keys": [ + "a" + ], + "last_row_key": "a", + "end_with_error": 4 }, { - end_with_error: 4, + "end_with_error": 4 }, { - end_with_error: 4, + "end_with_error": 4 }, { - end_with_error: 4, + "end_with_error": 4 }, { - row_keys: ['b'], - last_row_key: 'b', - end_with_error: 4, + "row_keys": [ + "b" + ], + "last_row_key": "b", + "end_with_error": 4 }, { - end_with_error: 4, + "end_with_error": 4 }, { - end_with_error: 4, + "end_with_error": 4 }, { - row_keys: ['c'], - last_row_key: 'b', - }, + "row_keys": [ + "c" + ], + "last_row_key": "b" + } ], - message: { - rowKeys: [], - rowRanges: [{}], - }, + "message": { + "rowKeys": [], + "rowRanges": [ + {} + ] + } }, - output: { - results: { - result: 'error', - data: [['a'], [], [], []], + "output": { + "results": { + "result": "error", + "data": [ + [ + "a" + ], + [], + [], + [] + ] }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" }, - 1: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ + "1": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ { - startKeyOpen: { - type: 'Buffer', - data: [97], + "startKeyOpen": { + "type": "Buffer", + "data": [ + 97 + ] }, - startKey: 'startKeyOpen', - }, - ], + "startKey": "startKeyOpen" + } + ] }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } }, - requestOrder: [0, 1, 1, 1], - callCount: 4, - }, - }, -}; + "requestOrder": [ + 0, + 1, + 1, + 1 + ], + "callCount": 4 + } + } +} -exports[ - 'Bigtable/ReadRows with custom responses and createReadStream arguments moves the start point of a range being consumed 1' -] = { - input: { - responses: [ +exports['Bigtable/ReadRows with custom responses and createReadStream arguments moves the start point of a range being consumed 1'] = { + "input": { + "responses": [ { - row_keys: ['a', 'b'], - end_with_error: 4, + "row_keys": [ + "a", + "b" + ], + "end_with_error": 4 }, { - row_keys: ['c'], - }, + "row_keys": [ + "c" + ] + } ], - message: { - ranges: [ + "message": { + "ranges": [ { - start: { - value: 'b', - inclusive: false, + "start": { + "value": "b", + "inclusive": false }, - end: 'z', - }, - ], - }, + "end": "z" + } + ] + } }, - output: { - results: { - result: 'end stream', - data: [['a', 'b'], ['c']], + "output": { + "results": { + "result": "end stream", + "data": [ + [ + "a", + "b" + ], + [ + "c" + ] + ] }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ { - startKeyClosed: { - type: 'Buffer', - data: [97], + "startKeyClosed": { + "type": "Buffer", + "data": [ + 97 + ] }, - startKey: 'startKeyClosed', - endKeyClosed: { - type: 'Buffer', - data: [122], + "startKey": "startKeyClosed", + "endKeyClosed": { + "type": "Buffer", + "data": [ + 122 + ] }, - endKey: 'endKeyClosed', - }, - ], + "endKey": "endKeyClosed" + } + ] }, - filter: null, - rowsLimit: '0', - appProfileId: '', + "filter": null, + "rowsLimit": "0", + "appProfileId": "" }, - 1: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ + "1": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ { - startKeyOpen: { - type: 'Buffer', - data: [98], + "startKeyOpen": { + "type": "Buffer", + "data": [ + 98 + ] }, - startKey: 'startKeyOpen', - endKeyClosed: { - type: 'Buffer', - data: [122], + "startKey": "startKeyOpen", + "endKeyClosed": { + "type": "Buffer", + "data": [ + 122 + ] }, - endKey: 'endKeyClosed', - }, - ], + "endKey": "endKeyClosed" + } + ] }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } }, - requestOrder: [0, 1], - callCount: 2, - }, - }, -}; + "requestOrder": [ + 0, + 1 + ], + "callCount": 2 + } + } +} -exports[ - 'Bigtable/ReadRows with custom responses and createReadStream arguments removes ranges already consumed 1' -] = { - input: { - responses: [ +exports['Bigtable/ReadRows with custom responses and createReadStream arguments removes ranges already consumed 1'] = { + "input": { + "responses": [ { - row_keys: ['a', 'b', 'c'], - end_with_error: 4, + "row_keys": [ + "a", + "b", + "c" + ], + "end_with_error": 4 }, { - row_keys: ['x'], - }, + "row_keys": [ + "x" + ] + } ], - message: { - ranges: [ + "message": { + "ranges": [ { - start: 'x', - end: 'z', - }, - ], - }, + "start": "x", + "end": "z" + } + ] + } }, - output: { - results: { - result: 'end stream', - data: [['a', 'b', 'c'], ['x']], + "output": { + "results": { + "result": "end stream", + "data": [ + [ + "a", + "b", + "c" + ], + [ + "x" + ] + ] }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ { - startKeyClosed: { - type: 'Buffer', - data: [97], + "startKeyClosed": { + "type": "Buffer", + "data": [ + 97 + ] }, - startKey: 'startKeyClosed', - endKeyClosed: { - type: 'Buffer', - data: [99], + "startKey": "startKeyClosed", + "endKeyClosed": { + "type": "Buffer", + "data": [ + 99 + ] }, - endKey: 'endKeyClosed', + "endKey": "endKeyClosed" }, { - startKeyClosed: { - type: 'Buffer', - data: [120], + "startKeyClosed": { + "type": "Buffer", + "data": [ + 120 + ] }, - startKey: 'startKeyClosed', - endKeyClosed: { - type: 'Buffer', - data: [122], + "startKey": "startKeyClosed", + "endKeyClosed": { + "type": "Buffer", + "data": [ + 122 + ] }, - endKey: 'endKeyClosed', - }, - ], + "endKey": "endKeyClosed" + } + ] }, - filter: null, - rowsLimit: '0', - appProfileId: '', + "filter": null, + "rowsLimit": "0", + "appProfileId": "" }, - 1: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ + "1": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ { - startKeyClosed: { - type: 'Buffer', - data: [120], + "startKeyClosed": { + "type": "Buffer", + "data": [ + 120 + ] }, - startKey: 'startKeyClosed', - endKeyClosed: { - type: 'Buffer', - data: [122], + "startKey": "startKeyClosed", + "endKeyClosed": { + "type": "Buffer", + "data": [ + 122 + ] }, - endKey: 'endKeyClosed', - }, - ], + "endKey": "endKeyClosed" + } + ] }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } }, - requestOrder: [0, 1], - callCount: 2, - }, - }, -}; + "requestOrder": [ + 0, + 1 + ], + "callCount": 2 + } + } +} -exports[ - 'Bigtable/ReadRows with custom responses and createReadStream arguments removes keys already read 1' -] = { - input: { - responses: [ +exports['Bigtable/ReadRows with custom responses and createReadStream arguments removes keys already read 1'] = { + "input": { + "responses": [ { - row_keys: ['a', 'b', 'c'], - end_with_error: 4, + "row_keys": [ + "a", + "b", + "c" + ], + "end_with_error": 4 }, { - row_keys: ['x'], - }, + "row_keys": [ + "x" + ] + } ], - message: { - keys: ['a', 'b', 'x'], - }, + "message": { + "keys": [ + "a", + "b", + "x" + ] + } }, - output: { - results: { - result: 'end stream', - data: [['a', 'b', 'c'], ['x']], + "output": { + "results": { + "result": "end stream", + "data": [ + [ + "a", + "b", + "c" + ], + [ + "x" + ] + ] }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [ + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [ { - type: 'Buffer', - data: [97], + "type": "Buffer", + "data": [ + 97 + ] }, { - type: 'Buffer', - data: [98], + "type": "Buffer", + "data": [ + 98 + ] }, { - type: 'Buffer', - data: [120], - }, + "type": "Buffer", + "data": [ + 120 + ] + } ], - rowRanges: [], + "rowRanges": [] }, - filter: null, - rowsLimit: '0', - appProfileId: '', + "filter": null, + "rowsLimit": "0", + "appProfileId": "" }, - 1: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [ + "1": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [ { - type: 'Buffer', - data: [120], - }, + "type": "Buffer", + "data": [ + 120 + ] + } ], - rowRanges: [], + "rowRanges": [] }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } }, - requestOrder: [0, 1], - callCount: 2, - }, - }, -}; + "requestOrder": [ + 0, + 1 + ], + "callCount": 2 + } + } +} -exports[ - 'Bigtable/ReadRows with custom responses and createReadStream arguments adjust the limit based on the number of rows read 1' -] = { - input: { - responses: [ +exports['Bigtable/ReadRows with custom responses and createReadStream arguments adjust the limit based on the number of rows read 1'] = { + "input": { + "responses": [ { - row_keys: ['a', 'b'], - end_with_error: 4, + "row_keys": [ + "a", + "b" + ], + "end_with_error": 4 }, { - row_keys: ['x'], - }, + "row_keys": [ + "x" + ] + } ], - message: { - limit: 10, - }, + "message": { + "limit": 10 + } }, - output: { - results: { - result: 'end stream', - data: [['a', 'b'], ['x']], + "output": { + "results": { + "result": "end stream", + "data": [ + [ + "a", + "b" + ], + [ + "x" + ] + ] }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], - }, - filter: null, - rowsLimit: '10', - appProfileId: '', + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "10", + "appProfileId": "" }, - 1: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [ + "1": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ { - startKeyOpen: { - type: 'Buffer', - data: [98], + "startKeyOpen": { + "type": "Buffer", + "data": [ + 98 + ] }, - startKey: 'startKeyOpen', - }, - ], + "startKey": "startKeyOpen" + } + ] }, - filter: null, - rowsLimit: '8', - appProfileId: '', - }, + "filter": null, + "rowsLimit": "8", + "appProfileId": "" + } }, - requestOrder: [0, 1], - callCount: 2, - }, - }, -}; + "requestOrder": [ + 0, + 1 + ], + "callCount": 2 + } + } +} -exports[ - 'Bigtable/ReadRows with custom responses and createReadStream arguments respects the max retries parameter 1' -] = { - input: { - responses: [ +exports['Bigtable/ReadRows with custom responses and createReadStream arguments respects the max retries parameter 1'] = { + "input": { + "responses": [ { - end_with_error: 4, + "end_with_error": 4 }, { - end_with_error: 4, + "end_with_error": 4 }, { - end_with_error: 4, + "end_with_error": 4 }, { - end_with_error: 4, - }, + "end_with_error": 4 + } ], - message: { - maxRetries: 2, + "message": { + "maxRetries": 2 + } + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with deadline exceeded error 1'] = { + "input": { + "code": 4, + "message": {} }, - output: { - results: { - result: 'error', - data: [[], [], [], []], + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] }, - requestData: { - requests: { - 0: { - tableName: - 'projects/{{projectId}}/instances/fake-instance/tables/fake-table', - rows: { - rowKeys: [], - rowRanges: [{}], - }, - filter: null, - rowsLimit: '0', - appProfileId: '', - }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } }, - requestOrder: [0, 0, 0, 0], - callCount: 4, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with resource exhausted error 1'] = { + "input": { + "code": 8, + "message": {} + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with aborted error 1'] = { + "input": { + "code": 10, + "message": {} }, -}; + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back where the error is retryable should ensure correct behavior with unavailable error 1'] = { + "input": { + "code": 14, + "message": {} + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with cancelled error 1'] = { + "input": { + "code": 1, + "message": {} + }, + "output": { + "results": { + "result": "end stream", + "data": [ + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0 + ], + "callCount": 1 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with internal error 1'] = { + "input": { + "code": 13, + "message": {} + }, + "output": { + "results": { + "result": "error", + "data": [ + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0 + ], + "callCount": 1 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back where the error is not retryable should ensure correct behavior with invalid argument error 1'] = { + "input": { + "code": 3, + "message": {} + }, + "output": { + "results": { + "result": "error", + "data": [ + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0 + ], + "callCount": 1 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with an empty request 1'] = { + "input": { + "code": 4, + "message": {} + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a decode value set 1'] = { + "input": { + "code": 4, + "message": { + "decode": true + } + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with an encoding value set 1'] = { + "input": { + "code": 4, + "message": { + "encoding": "test-encoding" + } + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with start and end values 1'] = { + "input": { + "code": 4, + "message": { + "start": "test-start", + "end": "test-end" + } + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + { + "startKeyClosed": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 115, + 116, + 97, + 114, + 116 + ] + }, + "startKey": "startKeyClosed", + "endKeyClosed": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 101, + 110, + 100 + ] + }, + "endKey": "endKeyClosed" + } + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with keys 1'] = { + "input": { + "code": 4, + "message": { + "keys": [ + "test-key-1", + "test-key-2" + ] + } + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [ + { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 107, + 101, + 121, + 45, + 49 + ] + }, + { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 107, + 101, + 121, + 45, + 50 + ] + } + ], + "rowRanges": [] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a filter 1'] = { + "input": { + "code": 4, + "message": { + "filter": [ + { + "column": "columnPrefix" + } + ] + } + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": { + "columnQualifierRegexFilter": { + "type": "Buffer", + "data": [ + 99, + 111, + 108, + 117, + 109, + 110, + 80, + 114, + 101, + 102, + 105, + 120 + ] + }, + "filter": "columnQualifierRegexFilter" + }, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a limit 1'] = { + "input": { + "code": 4, + "message": { + "limit": 10 + } + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + {} + ] + }, + "filter": null, + "rowsLimit": "10", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a prefix 1'] = { + "input": { + "code": 4, + "message": { + "prefix": "test-prefix" + } + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + { + "startKeyClosed": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 112, + 114, + 101, + 102, + 105, + 120 + ] + }, + "startKey": "startKeyClosed", + "endKeyOpen": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 112, + 114, + 101, + 102, + 105, + 121 + ] + }, + "endKey": "endKeyOpen" + } + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with prefixes 1'] = { + "input": { + "code": 4, + "message": { + "prefixes": [ + "test-prefix1", + "test-prefix2" + ] + } + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + { + "startKeyClosed": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 112, + 114, + 101, + 102, + 105, + 120, + 49 + ] + }, + "startKey": "startKeyClosed", + "endKeyOpen": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 112, + 114, + 101, + 102, + 105, + 120, + 50 + ] + }, + "endKey": "endKeyOpen" + }, + { + "startKeyClosed": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 112, + 114, + 101, + 102, + 105, + 120, + 50 + ] + }, + "startKey": "startKeyClosed", + "endKeyOpen": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 112, + 114, + 101, + 102, + 105, + 120, + 51 + ] + }, + "endKey": "endKeyOpen" + } + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} + +exports['Bigtable/ReadRows with a mock server that always sends an error back with a deadline exceeded error and different createReadStream arguments should pass checks with a list of ranges 1'] = { + "input": { + "code": 4, + "message": { + "ranges": [ + { + "start": "test-start-1", + "end": "test-end-1" + }, + { + "start": "test-start-2", + "end": "test-end-2" + } + ] + } + }, + "output": { + "results": { + "result": "error", + "data": [ + [], + [], + [], + [] + ] + }, + "requestData": { + "requests": { + "0": { + "tableName": "projects/{{projectId}}/instances/fake-instance/tables/fake-table", + "rows": { + "rowKeys": [], + "rowRanges": [ + { + "startKeyClosed": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 115, + 116, + 97, + 114, + 116, + 45, + 49 + ] + }, + "startKey": "startKeyClosed", + "endKeyClosed": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 101, + 110, + 100, + 45, + 49 + ] + }, + "endKey": "endKeyClosed" + }, + { + "startKeyClosed": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 115, + 116, + 97, + 114, + 116, + 45, + 50 + ] + }, + "startKey": "startKeyClosed", + "endKeyClosed": { + "type": "Buffer", + "data": [ + 116, + 101, + 115, + 116, + 45, + 101, + 110, + 100, + 45, + 50 + ] + }, + "endKey": "endKeyClosed" + } + ] + }, + "filter": null, + "rowsLimit": "0", + "appProfileId": "" + } + }, + "requestOrder": [ + 0, + 0, + 0, + 0 + ], + "callCount": 4 + } + } +} diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index b04d2cb7c..9342ea68d 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -32,7 +32,7 @@ import { ReadRowsHandler, ReadRowsResponse, } from '../../src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler'; -// import {testGaxOptions} from './test-options'; +import {testGaxOptions} from './test-options'; describe('Bigtable/ReadRows', () => { let server: MockServer; @@ -59,7 +59,6 @@ describe('Bigtable/ReadRows', () => { const streamFetcher = new ReadRowsFetcher(table, opts); return new StreamTester(serviceHandler, streamFetcher); } - /* describe('with a mock server that always sends an error back', () => { function checkRetryWithServer(code: grpc.status, callback: () => void) { const serviceHandler = new SendErrorHandler(service, 'ReadRows', code); @@ -156,7 +155,7 @@ describe('Bigtable/ReadRows', () => { done ); }); - ///* + /* it('should pass checks with gaxOptions', done => { // TODO: Add the retry parameter checkWithOptions( @@ -166,10 +165,9 @@ describe('Bigtable/ReadRows', () => { done ); }); - + */ }); }); - */ describe('with custom responses and createReadStream arguments', () => { function getServiceHandler(responses: ReadRowsResponse[], message: any) { return new ReadRowsHandler(service, 'ReadRows', responses, message); From 5714dd6cf28dbe186501b65ee4439b3d6a0fac8f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 8 Jul 2022 15:23:08 -0400 Subject: [PATCH 50/51] Move the mock server file --- test/{util => }/mock-server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename test/{util => }/mock-server.ts (96%) diff --git a/test/util/mock-server.ts b/test/mock-server.ts similarity index 96% rename from test/util/mock-server.ts rename to test/mock-server.ts index 920673579..117a88e56 100644 --- a/test/util/mock-server.ts +++ b/test/mock-server.ts @@ -17,7 +17,7 @@ // ** All changes to this file may be overwritten. ** import {describe, it} from 'mocha'; -import {MockServer} from '../../src/util/mock-servers/mock-server'; +import {MockServer} from '../src/util/mock-servers/mock-server'; import * as assert from 'assert'; const tcpPortUsed = require('tcp-port-used'); From 776e6921c1a8870e65faff999d827d4bf2cc29f1 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Fri, 8 Jul 2022 16:56:16 -0400 Subject: [PATCH 51/51] Move test utilities --- test/errors.ts | 6 +++--- .../helpers}/mock-servers/mock-server.ts | 0 .../helpers}/mock-servers/mock-service.ts | 0 .../bigtable-client-mock-service.ts | 0 .../implementation/read-rows-handler.ts | 2 +- .../implementation/same-call-handler.ts | 0 .../implementation/send-error-handler.ts | 2 +- .../service-handlers/service-handler.ts | 0 .../implementation/read-rows-fetcher.ts | 2 +- .../stream-fetchers/stream-fetcher.ts | 0 .../service-testers/stream-tester.ts | 2 +- test/mock-server.ts | 2 +- test/server/read-rows.ts | 16 ++++++++-------- 13 files changed, 16 insertions(+), 16 deletions(-) rename {src/util => test/helpers}/mock-servers/mock-server.ts (100%) rename {src/util => test/helpers}/mock-servers/mock-service.ts (100%) rename {src/util => test/helpers}/mock-servers/service-implementations/bigtable-client-mock-service.ts (100%) rename {src/util => test/helpers}/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts (98%) rename {src/util => test/helpers}/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts (100%) rename {src/util => test/helpers}/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts (97%) rename {src/util => test/helpers}/mock-servers/service-testers/service-handlers/service-handler.ts (100%) rename {src/util => test/helpers}/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts (94%) rename {src/util => test/helpers}/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts (100%) rename {src/util => test/helpers}/mock-servers/service-testers/stream-tester.ts (97%) diff --git a/test/errors.ts b/test/errors.ts index 33721b743..f7b780600 100644 --- a/test/errors.ts +++ b/test/errors.ts @@ -21,9 +21,9 @@ import {Bigtable} from '../src'; import * as assert from 'assert'; import {GoogleError, grpc, ServiceError} from 'google-gax'; -import {MockServer} from '../src/util/mock-servers/mock-server'; -import {BigtableClientMockService} from '../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; -import {MockService} from '../src/util/mock-servers/mock-service'; +import {MockServer} from './helpers/mock-servers/mock-server'; +import {BigtableClientMockService} from './helpers/mock-servers/service-implementations/bigtable-client-mock-service'; +import {MockService} from './helpers/mock-servers/mock-service'; function isServiceError(error: any): error is ServiceError { return ( diff --git a/src/util/mock-servers/mock-server.ts b/test/helpers/mock-servers/mock-server.ts similarity index 100% rename from src/util/mock-servers/mock-server.ts rename to test/helpers/mock-servers/mock-server.ts diff --git a/src/util/mock-servers/mock-service.ts b/test/helpers/mock-servers/mock-service.ts similarity index 100% rename from src/util/mock-servers/mock-service.ts rename to test/helpers/mock-servers/mock-service.ts diff --git a/src/util/mock-servers/service-implementations/bigtable-client-mock-service.ts b/test/helpers/mock-servers/service-implementations/bigtable-client-mock-service.ts similarity index 100% rename from src/util/mock-servers/service-implementations/bigtable-client-mock-service.ts rename to test/helpers/mock-servers/service-implementations/bigtable-client-mock-service.ts diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts b/test/helpers/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts similarity index 98% rename from src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts rename to test/helpers/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts index 6c6042c29..9f2b8d339 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts +++ b/test/helpers/mock-servers/service-testers/service-handlers/implementation/read-rows-handler.ts @@ -18,7 +18,7 @@ import {SameCallHandler} from './same-call-handler'; import {MockService} from '../../../mock-service'; -import {Mutation} from '../../../../../mutation'; +import {Mutation} from '../../../../../../src/mutation'; function rowResponse(rowKey: {}) { return { diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts b/test/helpers/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts similarity index 100% rename from src/util/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts rename to test/helpers/mock-servers/service-testers/service-handlers/implementation/same-call-handler.ts diff --git a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts b/test/helpers/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts similarity index 97% rename from src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts rename to test/helpers/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts index 0fc3ccefd..7aa9545e5 100644 --- a/src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts +++ b/test/helpers/mock-servers/service-testers/service-handlers/implementation/send-error-handler.ts @@ -19,7 +19,7 @@ import {grpc} from 'google-gax'; import {SameCallHandler} from './same-call-handler'; import {MockService} from '../../../mock-service'; -import {Row} from '../../../../../row'; +import {Row} from '../../../../../../src/row'; export class SendErrorHandler extends SameCallHandler { code: grpc.status; diff --git a/src/util/mock-servers/service-testers/service-handlers/service-handler.ts b/test/helpers/mock-servers/service-testers/service-handlers/service-handler.ts similarity index 100% rename from src/util/mock-servers/service-testers/service-handlers/service-handler.ts rename to test/helpers/mock-servers/service-testers/service-handlers/service-handler.ts diff --git a/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts b/test/helpers/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts similarity index 94% rename from src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts rename to test/helpers/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts index 24e56bcc1..b44ed7901 100644 --- a/src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts +++ b/test/helpers/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher.ts @@ -16,7 +16,7 @@ // ** https://github.com/googleapis/gapic-generator-typescript ** // ** All changes to this file may be overwritten. ** -import {GetRowsOptions, Table} from '../../../../../table'; +import {GetRowsOptions, Table} from '../../../../../../src/table'; import internal = require('stream'); import {StreamFetcher} from '../stream-fetcher'; diff --git a/src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts b/test/helpers/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts similarity index 100% rename from src/util/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts rename to test/helpers/mock-servers/service-testers/stream-fetchers/stream-fetcher.ts diff --git a/src/util/mock-servers/service-testers/stream-tester.ts b/test/helpers/mock-servers/service-testers/stream-tester.ts similarity index 97% rename from src/util/mock-servers/service-testers/stream-tester.ts rename to test/helpers/mock-servers/service-testers/stream-tester.ts index 2e647f91c..d52de38e4 100644 --- a/src/util/mock-servers/service-testers/stream-tester.ts +++ b/test/helpers/mock-servers/service-testers/stream-tester.ts @@ -21,7 +21,7 @@ import {StreamFetcher} from './stream-fetchers/stream-fetcher'; import {ServiceError} from 'google-gax'; import * as snapshot from 'snap-shot-it'; -import {Row} from '../../../row'; +import {Row} from '../../../../src/row'; export class StreamTester { serviceHandler: ServiceHandler; diff --git a/test/mock-server.ts b/test/mock-server.ts index 117a88e56..56abc81a7 100644 --- a/test/mock-server.ts +++ b/test/mock-server.ts @@ -17,7 +17,7 @@ // ** All changes to this file may be overwritten. ** import {describe, it} from 'mocha'; -import {MockServer} from '../src/util/mock-servers/mock-server'; +import {MockServer} from './helpers/mock-servers/mock-server'; import * as assert from 'assert'; const tcpPortUsed = require('tcp-port-used'); diff --git a/test/server/read-rows.ts b/test/server/read-rows.ts index 9342ea68d..e1ddf7c2d 100644 --- a/test/server/read-rows.ts +++ b/test/server/read-rows.ts @@ -20,18 +20,18 @@ import {before, describe, it} from 'mocha'; import {Bigtable, GetRowsOptions} from '../../src'; import {grpc} from 'google-gax'; -import {MockServer} from '../../src/util/mock-servers/mock-server'; -import {BigtableClientMockService} from '../../src/util/mock-servers/service-implementations/bigtable-client-mock-service'; -import {MockService} from '../../src/util/mock-servers/mock-service'; -import {SendErrorHandler} from '../../src/util/mock-servers/service-testers/service-handlers/implementation/send-error-handler'; -import {ReadRowsFetcher} from '../../src/util/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher'; -import {StreamTester} from '../../src/util/mock-servers/service-testers/stream-tester'; -import {ServiceHandler} from '../../src/util/mock-servers/service-testers/service-handlers/service-handler'; +import {MockServer} from '../helpers/mock-servers/mock-server'; +import {BigtableClientMockService} from '../helpers/mock-servers/service-implementations/bigtable-client-mock-service'; +import {MockService} from '../helpers/mock-servers/mock-service'; +import {SendErrorHandler} from '../helpers/mock-servers/service-testers/service-handlers/implementation/send-error-handler'; +import {ReadRowsFetcher} from '../helpers/mock-servers/service-testers/stream-fetchers/implementation/read-rows-fetcher'; +import {StreamTester} from '../helpers/mock-servers/service-testers/stream-tester'; +import {ServiceHandler} from '../helpers/mock-servers/service-testers/service-handlers/service-handler'; import {Table} from '../../src/table'; import { ReadRowsHandler, ReadRowsResponse, -} from '../../src/util/mock-servers/service-testers/service-handlers/implementation/read-rows-handler'; +} from '../helpers/mock-servers/service-testers/service-handlers/implementation/read-rows-handler'; import {testGaxOptions} from './test-options'; describe('Bigtable/ReadRows', () => {