diff --git a/.circleci/config.yml b/.circleci/config.yml index 37be84d1..aa9db22e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,10 +3,12 @@ version: 2.1 executors: node-executor: working_directory: ~/repo - docker: - # Jest has issues with node > 16.11.0, please stick node version until the issues are resolved - # https://github.com/facebook/jest/issues/11956 - - image: cimg/node:16.10.0 + machine: + # What software on the machine https://discuss.circleci.com/t/linux-machine-executor-update-2022-july-q3-update/44873 + image: ubuntu-2204:2022.07.1 + # About DLC https://circleci.com/docs/docker-layer-caching + docker_layer_caching: true + resource_class: large parameters: workflow_name: @@ -134,9 +136,7 @@ jobs: - codecov-0.1.0_4653 - run: rm -rf ./coverage # Clear coverage folder - - run: yarn nx affected --target=test --ci --coverage --maxWorkers=2 --coverageReporters=lcov ${AFFECTED_ARGS} - # --maxWorkers=2 is required because we'll run virtual machine with 32 cores CPU (with actually 4 CPI), jest spawns lots of workers if we don't fix the worker size. - # https://support.circleci.com/hc/en-us/articles/360005442714-Your-test-tools-are-smart-and-that-s-a-problem-Learn-about-when-optimization-goes-wrong- + - run: yarn nx affected --target=test --ci --coverage --coverageReporters=lcov ${AFFECTED_ARGS} - run: name: upload reports with flags command: | @@ -163,9 +163,6 @@ jobs: <<: *set_env - restore_cache: <<: *yarn_cache - # Setup remote docker to build image - # https://circleci.com/docs/building-docker-images - - setup_remote_docker - run: name: NPM publish # running with yarn nx causes npm config issues diff --git a/jest.setup.ts b/jest.setup.ts index 93fab6a6..7c7fbb28 100644 --- a/jest.setup.ts +++ b/jest.setup.ts @@ -2,3 +2,10 @@ * Add reflect-metadata to collect Inversify IoC decorator information when running jest test. */ import 'reflect-metadata'; +/** + * https://github.com/prisma/prisma/issues/8558 + * in jest 27+, we'll have some issues with setImmediate function, this is a workaround + */ +global.setImmediate = + global.setImmediate || + ((fn: any, ...args: any) => global.setTimeout(fn, 0, ...args)); diff --git a/package.json b/package.json index 091c6b50..2374c5f3 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "dependencies": { "@koa/cors": "^3.3.0", "chokidar": "^3.5.3", + "bluebird": "^3.7.2", "bcryptjs": "^2.4.3", "bytes": "^3.1.2", "class-validator": "^0.13.2", @@ -31,6 +32,8 @@ "nunjucks": "^3.2.3", "openapi3-ts": "^2.0.2", "ora": "^5.4.1", + "pg": "^8.8.0", + "pg-cursor": "^2.7.4", "redoc": "2.0.0-rc.76", "reflect-metadata": "^0.1.13", "semver": "^7.3.7", @@ -48,6 +51,7 @@ "@nrwl/workspace": "14.0.3", "@types/bcryptjs": "^2.4.2", "@types/bytes": "^3.1.1", + "@types/dockerode": "^3.3.9", "@types/from2": "^2.3.1", "@types/glob": "^7.2.0", "@types/inquirer": "^8.0.0", @@ -63,6 +67,8 @@ "@types/lodash": "^4.14.182", "@types/md5": "^2.3.2", "@types/node": "16.11.7", + "@types/pg": "^8.6.5", + "@types/pg-cursor": "^2.7.0", "@types/semver": "^7.3.12", "@types/supertest": "^2.0.12", "@types/uuid": "^8.3.4", @@ -70,6 +76,7 @@ "@typescript-eslint/parser": "~5.18.0", "commitizen": "^4.2.5", "cz-conventional-changelog": "^3.3.0", + "dockerode": "^3.3.4", "eslint": "~8.12.0", "eslint-config-prettier": "8.1.0", "from2": "^2.3.0", diff --git a/packages/build/test/builder/profile.yaml b/packages/build/test/builder/profile.yaml index c8aa0072..e52a57fb 100644 --- a/packages/build/test/builder/profile.yaml +++ b/packages/build/test/builder/profile.yaml @@ -1,3 +1,3 @@ - name: test - type: pg + type: mock allow: '*' diff --git a/packages/core/src/lib/data-source/index.ts b/packages/core/src/lib/data-source/index.ts index 9e57f399..3806f67c 100644 --- a/packages/core/src/lib/data-source/index.ts +++ b/packages/core/src/lib/data-source/index.ts @@ -1,4 +1,4 @@ -import { PGDataSource } from './pg'; -export const builtInDataSource = [PGDataSource]; +import { MockDataSource } from './mock'; +export const builtInDataSource = [MockDataSource]; -export * from './pg'; +export * from './mock'; diff --git a/packages/core/src/lib/data-source/pg.ts b/packages/core/src/lib/data-source/mock.ts similarity index 88% rename from packages/core/src/lib/data-source/pg.ts rename to packages/core/src/lib/data-source/mock.ts index d13c0d0e..b3b6ee1f 100644 --- a/packages/core/src/lib/data-source/pg.ts +++ b/packages/core/src/lib/data-source/mock.ts @@ -9,8 +9,8 @@ import { } from '../../models/extensions'; @VulcanInternalExtension() -@VulcanExtensionId('pg') -export class PGDataSource extends DataSource { +@VulcanExtensionId('mock') +export class MockDataSource extends DataSource { // eslint-disable-next-line @typescript-eslint/no-unused-vars public async execute(options: ExecuteOptions): Promise { return { diff --git a/packages/extension-driver-duckdb/package.json b/packages/extension-driver-duckdb/package.json index 3b0b15cb..eee560eb 100644 --- a/packages/extension-driver-duckdb/package.json +++ b/packages/extension-driver-duckdb/package.json @@ -1,7 +1,7 @@ { "name": "@vulcan-sql/extension-driver-duckdb", "description": "duckdb driver for Vulcan SQL", - "version": "0.1.0-alpha.1", + "version": "0.2.0", "type": "commonjs", "publishConfig": { "access": "public" @@ -23,6 +23,6 @@ }, "license": "MIT", "peerDependencies": { - "@vulcan-sql/core": ">= 0.1.0 || >= 0.1.0-alpha.1" + "@vulcan-sql/core": "~0.2.0-0" } } diff --git a/packages/extension-driver-duckdb/project.json b/packages/extension-driver-duckdb/project.json index 4588058a..0122fb97 100644 --- a/packages/extension-driver-duckdb/project.json +++ b/packages/extension-driver-duckdb/project.json @@ -49,7 +49,7 @@ "publish": { "executor": "@nrwl/workspace:run-commands", "options": { - "command": "node ../../../tools/scripts/publish.mjs {args.tag}", + "command": "node ../../../tools/scripts/publish.mjs {args.tag} {args.version}", "cwd": "dist/packages/extension-driver-duckdb" }, "dependsOn": [ diff --git a/packages/extension-driver-pg/.eslintrc.json b/packages/extension-driver-pg/.eslintrc.json new file mode 100644 index 00000000..9d9c0db5 --- /dev/null +++ b/packages/extension-driver-pg/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/packages/extension-driver-pg/README.md b/packages/extension-driver-pg/README.md new file mode 100644 index 00000000..22154672 --- /dev/null +++ b/packages/extension-driver-pg/README.md @@ -0,0 +1,58 @@ +# extension-driver-pg + +[node-postgres](https://node-postgres.com/) driver for Vulcan SQL. + +## Install + +1. Install package + + ```sql + npm i @vulcan-sql/extension-driver-pg + ``` + +2. Update `vulcan.yaml`, enable the extension. + + ```yaml + extensions: + pg: '@vulcan-sql/extension-driver-pg' + ``` + +3. Create a new profile in `profiles.yaml` or in your profiles' paths. + + ```yaml + - name: pg # profile name + type: pg + connection: + # Optional: The max rows we should fetch once. + chunkSize: 100 + # Optional: Maximum number of clients the pool should contain. + max: 10 + # Optional: Number of milliseconds a client must sit idle in the pool and not be checked out before it is disconnected from the backend and discarded. + idleTimeoutMillis: 10000 + # Optional: Number of milliseconds to wait before timing out when connecting a new client by default this is 0 which means no timeout + connectionTimeoutMillis: 0 + # Optional: The user to connect to database. Default process.env.PGUSER || process.env.USER + user: string + # Optional: Password to connect to database. default process.env.PGPASSWORD + password: string + # Optional: Server host. default process.env.PGHOST + host: string + # Optional: Name of database. default process.env.PGDATABASE || user + database: string + # Optional: Server port. default process.env.PGPORT + port: 5432 + # Optional: Connection string. + connectionString: postgres://user:password@host:5432/database + # Optional: Passed directly to node.TLSSocket, supports all tls.connect options + ssl: false + # Optional: Number of milliseconds before a statement in query will time out, default is no timeout + statement_timeout: 0 + # Optional: Number of milliseconds before a query call will timeout, default is no timeout + query_timeout: 0 + # Optional: The name of the application that created this Client instance + application_name: string + # Optional: Number of milliseconds to wait for connection, default is no timeout + connectionTimeoutMillis: 0 + # Optional: Number of milliseconds before terminating any session with an open idle transaction, default is no timeout + idle_in_transaction_session_timeout: 0 + ``` diff --git a/packages/extension-driver-pg/jest.config.ts b/packages/extension-driver-pg/jest.config.ts new file mode 100644 index 00000000..d444cefd --- /dev/null +++ b/packages/extension-driver-pg/jest.config.ts @@ -0,0 +1,14 @@ +module.exports = { + displayName: 'extension-driver-pg', + preset: '../../jest.preset.ts', + globals: { + 'ts-jest': { + tsconfig: '/tsconfig.spec.json', + }, + }, + transform: { + '^.+\\.[tj]s$': 'ts-jest', + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: '../../coverage/packages/extension-driver-pg', +}; diff --git a/packages/extension-driver-pg/package.json b/packages/extension-driver-pg/package.json new file mode 100644 index 00000000..0d2382d4 --- /dev/null +++ b/packages/extension-driver-pg/package.json @@ -0,0 +1,29 @@ +{ + "name": "@vulcan-sql/extension-driver-pg", + "description": "PG driver for Vulcan SQL", + "version": "0.2.0", + "type": "commonjs", + "publishConfig": { + "access": "public" + }, + "keywords": [ + "vulcan", + "vulcan-sql", + "data", + "sql", + "database", + "data-warehouse", + "data-lake", + "api-builder", + "postgres", + "pg" + ], + "repository": { + "type": "git", + "url": "https://github.com/Canner/vulcan.git" + }, + "license": "MIT", + "peerDependencies": { + "@vulcan-sql/core": "~0.2.0-0" + } +} diff --git a/packages/extension-driver-pg/project.json b/packages/extension-driver-pg/project.json new file mode 100644 index 00000000..178a0abb --- /dev/null +++ b/packages/extension-driver-pg/project.json @@ -0,0 +1,64 @@ +{ + "root": "packages/extension-driver-pg", + "sourceRoot": "packages/extension-driver-pg/src", + "targets": { + "build": { + "executor": "@nrwl/workspace:run-commands", + "options": { + "command": "yarn ts-node ./tools/scripts/replaceAlias.ts extension-driver-pg" + }, + "dependsOn": [ + { + "projects": "self", + "target": "tsc" + } + ] + }, + "tsc": { + "executor": "@nrwl/js:tsc", + "outputs": ["{options.outputPath}"], + "options": { + "outputPath": "dist/packages/extension-driver-pg", + "main": "packages/extension-driver-pg/src/index.ts", + "tsConfig": "packages/extension-driver-pg/tsconfig.lib.json", + "assets": ["packages/extension-driver-pg/*.md"], + "buildableProjectDepsInPackageJsonType": "dependencies" + }, + "dependsOn": [ + { + "projects": "dependencies", + "target": "build" + } + ] + }, + "lint": { + "executor": "@nrwl/linter:eslint", + "outputs": ["{options.outputFile}"], + "options": { + "lintFilePatterns": ["packages/extension-driver-pg/**/*.ts"] + } + }, + "test": { + "executor": "@nrwl/jest:jest", + "outputs": ["coverage/packages/extension-driver-pg"], + "options": { + "jestConfig": "packages/extension-driver-pg/jest.config.ts", + "passWithNoTests": true + } + }, + "publish": { + "executor": "@nrwl/workspace:run-commands", + "options": { + "command": "node ../../../tools/scripts/publish.mjs {args.tag} {args.version}", + "cwd": "dist/packages/extension-driver-pg" + }, + "dependsOn": [ + { + "projects": "self", + "target": "build" + } + ] + } + }, + "tags": [] +} diff --git a/packages/extension-driver-pg/src/index.ts b/packages/extension-driver-pg/src/index.ts new file mode 100644 index 00000000..bdb725b2 --- /dev/null +++ b/packages/extension-driver-pg/src/index.ts @@ -0,0 +1,3 @@ +export * from './lib/pgDataSource'; +import { PGDataSource } from './lib/pgDataSource'; +export default [PGDataSource]; diff --git a/packages/extension-driver-pg/src/lib/pgDataSource.ts b/packages/extension-driver-pg/src/lib/pgDataSource.ts new file mode 100644 index 00000000..9643a22b --- /dev/null +++ b/packages/extension-driver-pg/src/lib/pgDataSource.ts @@ -0,0 +1,148 @@ +import { + DataResult, + DataSource, + ExecuteOptions, + RequestParameter, + VulcanExtensionId, +} from '@vulcan-sql/core'; +import { Pool, PoolConfig, QueryResult } from 'pg'; +import * as Cursor from 'pg-cursor'; +import { Readable } from 'stream'; +import { mapFromPGTypeId } from './typeMapper'; + +export interface PGOptions extends PoolConfig { + chunkSize?: number; +} + +@VulcanExtensionId('pg') +export class PGDataSource extends DataSource { + private logger = this.getLogger(); + private poolMapping = new Map(); + + public override async onActivate() { + const profiles = this.getProfiles().values(); + for (const profile of profiles) { + this.logger.debug( + `Initializing profile: ${profile.name} using pg driver` + ); + const pool = new Pool(profile.connection); + // https://node-postgres.com/api/pool#poolconnect + // When a client is sitting idly in the pool it can still emit errors because it is connected to a live backend. + // If the backend goes down or a network partition is encountered all the idle, connected clients in your application will emit an error through the pool's error event emitter. + pool.on('error', () => { + // Ignore the pool client error + }); + this.poolMapping.set(profile.name, { + pool: new Pool(profile.connection), + options: profile.connection, + }); + // Testing connection + await pool.query('SELECT 1;'); + this.logger.debug(`Profile ${profile.name} initialized`); + } + } + + public async execute({ + statement: sql, + bindParams, + profileName, + }: ExecuteOptions): Promise { + if (!this.poolMapping.has(profileName)) { + throw new Error(`Profile instance ${profileName} not found`); + } + const { pool, options } = this.poolMapping.get(profileName)!; + this.logger.debug(`Acquiring connection from ${profileName}`); + const client = await pool.connect(); + this.logger.debug(`Acquired connection from ${profileName}`); + try { + const cursor = client.query( + new Cursor(sql, Array.from(bindParams.values())) + ); + cursor.once('done', async () => { + this.logger.debug( + `Data fetched, release connection from ${profileName}` + ); + // It is important to close the cursor before releasing connection, or the connection might not able to handle next request. + await cursor.close(); + client.release(); + }); + // All promises MUST fulfilled in this function or we are not able to release the connection when error occurred + return await this.getResultFromCursor(cursor, options); + } catch (e: any) { + this.logger.debug( + `Errors occurred, release connection from ${profileName}` + ); + client.release(); + throw e; + } + } + + public async prepare({ parameterIndex }: RequestParameter) { + return `$${parameterIndex}`; + } + + public async destroy() { + for (const { pool } of this.poolMapping.values()) { + await pool.end(); + } + } + + private async getResultFromCursor( + cursor: Cursor, + options: PGOptions = {} + ): Promise { + const { chunkSize = 100 } = options; + const cursorRead = this.cursorRead.bind(this); + const firstChunk = await cursorRead(cursor, chunkSize); + // save first chunk in buffer for incoming requests + let bufferedRows = [...firstChunk.rows]; + let bufferReadIndex = 0; + const fetchNext = async () => { + if (bufferReadIndex >= bufferedRows.length) { + bufferedRows = (await cursorRead(cursor, chunkSize)).rows; + bufferReadIndex = 0; + } + return bufferedRows[bufferReadIndex++] || null; + }; + const stream = new Readable({ + objectMode: true, + read() { + fetchNext() + .then((row) => { + this.push(row); + }) + .catch((error) => { + this.destroy(error); + }); + }, + destroy(error: Error | null, cb: (error: Error | null) => void) { + // Send done event to notify upstream to release the connection. + cursor.emit('done'); + cb(error); + }, + // automatically destroy() the stream when it emits 'finish' or errors. Node > 10.16 + autoDestroy: true, + }); + return { + getColumns: () => + firstChunk.result.fields.map((field) => ({ + name: field.name, + type: mapFromPGTypeId(field.dataTypeID), + })), + getData: () => stream, + }; + } + + public async cursorRead(cursor: Cursor, maxRows: number) { + return new Promise<{ rows: any[]; result: QueryResult }>( + (resolve, reject) => { + cursor.read(maxRows, (err, rows, result) => { + if (err) { + return reject(err); + } + resolve({ rows, result }); + }); + } + ); + } +} diff --git a/packages/extension-driver-pg/src/lib/typeMapper.ts b/packages/extension-driver-pg/src/lib/typeMapper.ts new file mode 100644 index 00000000..d102d49a --- /dev/null +++ b/packages/extension-driver-pg/src/lib/typeMapper.ts @@ -0,0 +1,25 @@ +import { builtins, TypeId } from 'pg-types'; + +const typeMapping = new Map(); + +const register = (pgTypeId: TypeId, type: string) => { + typeMapping.set(pgTypeId, type); +}; + +// Reference +// https://github.com/brianc/node-pg-types/blob/master/lib/textParsers.js +// https://github.com/brianc/node-pg-types/blob/master/lib/binaryParsers.js + +register(builtins.INT8, 'number'); +register(builtins.INT4, 'number'); +register(builtins.INT2, 'number'); +register(builtins.OID, 'number'); +register(builtins.NUMERIC, 'number'); +register(builtins.FLOAT4, 'number'); // float4/real +register(builtins.FLOAT8, 'number'); // float8/double +register(builtins.BOOL, 'boolean'); + +export const mapFromPGTypeId = (pgTypeId: number) => { + if (typeMapping.has(pgTypeId)) return typeMapping.get(pgTypeId)!; + return 'string'; +}; diff --git a/packages/extension-driver-pg/test/pgDataSource.spec.ts b/packages/extension-driver-pg/test/pgDataSource.spec.ts new file mode 100644 index 00000000..ab6dbab9 --- /dev/null +++ b/packages/extension-driver-pg/test/pgDataSource.spec.ts @@ -0,0 +1,353 @@ +import { PGServer } from './pgServer'; +import { PGDataSource, PGOptions } from '../src'; +import { streamToArray } from '@vulcan-sql/core'; +import { Writable } from 'stream'; + +const pg = new PGServer(); +let dataSource: PGDataSource; + +beforeAll(async () => { + await pg.prepare(); +}, 5 * 60 * 1000); // it might take some time to pull images. + +afterEach(async () => { + try { + if (dataSource) await dataSource.destroy(); + } catch { + // ignore + } +}); + +afterAll(async () => { + await pg.destroy(); +}, 30000); + +it('Data source should be activate without any error when all profiles are valid', async () => { + // Arrange + dataSource = new PGDataSource({}, '', [ + { + name: 'profile1', + type: 'pg', + connection: { + host: pg.host, + user: pg.user, + password: pg.password, + database: pg.database, + port: pg.port, + } as PGOptions, + allow: '*', + }, + ]); + // Act, Assert + await expect(dataSource.activate()).resolves.not.toThrow(); +}); + +it('Data source should throw error when activating if any profile is invalid', async () => { + // Arrange + dataSource = new PGDataSource({}, '', [ + { + name: 'profile1', + type: 'pg', + connection: { + host: pg.host, + user: pg.user, + password: pg.password, + database: pg.database, + port: pg.port, + } as PGOptions, + allow: '*', + }, + { + name: 'wrong-password', + type: 'pg', + connection: { + host: pg.host, + user: pg.user, + password: pg.password + '123', + database: pg.database, + port: pg.port, + } as PGOptions, + allow: '*', + }, + ]); + // Act, Assert + await expect(dataSource.activate()).rejects.toThrow(); +}); + +it('Data source should return correct rows with 2 chunks', async () => { + // Arrange + dataSource = new PGDataSource({}, '', [ + { + name: 'profile1', + type: 'pg', + connection: { + host: pg.host, + user: pg.user, + password: pg.password, + database: pg.database, + port: pg.port, + } as PGOptions, + allow: '*', + }, + ]); + await dataSource.activate(); + // Act + const { getData } = await dataSource.execute({ + statement: 'select * from users limit 193', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const rows = await streamToArray(getData()); + // Assert + expect(rows.length).toBe(193); +}, 30000); + +it('Data source should return correct rows with 1 chunk', async () => { + // Arrange + dataSource = new PGDataSource({}, '', [ + { + name: 'profile1', + type: 'pg', + connection: { + host: pg.host, + user: pg.user, + password: pg.password, + database: pg.database, + port: pg.port, + } as PGOptions, + allow: '*', + }, + ]); + await dataSource.activate(); + // Act + const { getData } = await dataSource.execute({ + statement: 'select * from users limit 12', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const rows = await streamToArray(getData()); + // Assert + expect(rows.length).toBe(12); +}, 30000); + +it('Data source should return empty data with no row', async () => { + // Arrange + dataSource = new PGDataSource({}, '', [ + { + name: 'profile1', + type: 'pg', + connection: { + host: pg.host, + user: pg.user, + password: pg.password, + database: pg.database, + port: pg.port, + } as PGOptions, + allow: '*', + }, + ]); + await dataSource.activate(); + // Act + const { getData } = await dataSource.execute({ + statement: 'select * from users limit 0', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const rows = await streamToArray(getData()); + // Assert + expect(rows.length).toBe(0); +}, 30000); + +it('Data source should release the connection when finished no matter success or not', async () => { + // Arrange + dataSource = new PGDataSource({}, '', [ + { + name: 'profile1', + type: 'pg', + connection: { + host: pg.host, + user: pg.user, + password: pg.password, + database: pg.database, + port: pg.port, + max: 1, // Limit the pool size to 1, we'll get blocked with any leak. + min: 1, + } as PGOptions, + allow: '*', + }, + ]); + await dataSource.activate(); + + // Act + // send parallel queries to test pool leak + const result = await Promise.all( + [ + async () => { + const { getData } = await dataSource.execute({ + statement: 'select * from users limit 1', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + return await streamToArray(getData()); + }, + async () => { + try { + const { getData } = await dataSource.execute({ + statement: 'wrong sql', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + await streamToArray(getData()); + return [{}]; // fake data + } catch { + // ignore error + return []; + } + }, + async () => { + const { getData } = await dataSource.execute({ + statement: 'select * from users limit 1', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + return await streamToArray(getData()); + }, + ].map((task) => task()) + ); + + // Assert + expect(result[0].length).toBe(1); + expect(result[1].length).toBe(0); + expect(result[2].length).toBe(1); +}, 30000); + +it('Data source should work with prepare statements', async () => { + // Arrange + dataSource = new PGDataSource({}, '', [ + { + name: 'profile1', + type: 'pg', + connection: { + host: pg.host, + user: pg.user, + password: pg.password, + database: pg.database, + port: pg.port, + } as PGOptions, + allow: '*', + }, + ]); + await dataSource.activate(); + // Act + const bindParams = new Map(); + const var1Name = await dataSource.prepare({ + parameterIndex: 1, + value: '123', + profileName: 'profile1', + }); + bindParams.set(var1Name, '123'); + const var2Name = await dataSource.prepare({ + parameterIndex: 2, + value: '456', + profileName: 'profile1', + }); + bindParams.set(var2Name, '456'); + + const { getData } = await dataSource.execute({ + statement: `select ${var1Name} as v1, ${var2Name} as v2;`, + bindParams, + profileName: 'profile1', + operations: {} as any, + }); + const rows = await streamToArray(getData()); + // Assert + expect(rows[0].v1).toBe('123'); + expect(rows[0].v2).toBe('456'); +}, 30000); + +it('Data source should return correct column types', async () => { + // Arrange + dataSource = new PGDataSource({}, '', [ + { + name: 'profile1', + type: 'pg', + connection: { + host: pg.host, + user: pg.user, + password: pg.password, + database: pg.database, + port: pg.port, + } as PGOptions, + allow: '*', + }, + ]); + await dataSource.activate(); + // Act + const { getColumns, getData } = await dataSource.execute({ + statement: 'select * from users limit 0', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const column = getColumns(); + // We need to destroy the data stream or the driver waits for us + const data = getData(); + data.destroy(); + + // Assert + expect(column[0]).toEqual({ name: 'id', type: 'number' }); + expect(column[1]).toEqual({ name: 'name', type: 'string' }); + expect(column[2]).toEqual({ name: 'enabled', type: 'boolean' }); +}, 30000); + +it('Data source should release connection when readable stream is destroyed', async () => { + // Arrange + dataSource = new PGDataSource({}, '', [ + { + name: 'profile1', + type: 'pg', + connection: { + host: pg.host, + user: pg.user, + password: pg.password, + database: pg.database, + port: pg.port, + } as PGOptions, + allow: '*', + }, + ]); + await dataSource.activate(); + // Act + const { getData } = await dataSource.execute({ + statement: 'select * from users limit 100', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const readStream = getData(); + const rows: any[] = []; + let resolve: any; + const waitForStream = () => new Promise((res) => (resolve = res)); + const writeStream = new Writable({ + write(chunk, _, cb) { + rows.push(chunk); + // After read 5 records, destroy the upstream + if (rows.length === 5) { + readStream.destroy(); + resolve(); + } else cb(); + }, + objectMode: true, + }); + readStream.pipe(writeStream); + await waitForStream(); + // Assert + expect(rows.length).toBe(5); + // afterEach hook will timeout if any leak occurred. +}, 30000); diff --git a/packages/extension-driver-pg/test/pgServer.ts b/packages/extension-driver-pg/test/pgServer.ts new file mode 100644 index 00000000..632124ed --- /dev/null +++ b/packages/extension-driver-pg/test/pgServer.ts @@ -0,0 +1,81 @@ +/* istanbul ignore file */ +import * as Docker from 'dockerode'; +import faker from '@faker-js/faker'; +import { Client } from 'pg'; +import * as BPromise from 'bluebird'; + +const docker = new Docker(); + +/** + * PG Server in docker + * table: users (id INTEGER, name VARCHAR, enabled BOOLEAN) + * rows: 200 rows. + */ +export class PGServer { + public readonly password = '123'; + public readonly image = 'postgres:12.12'; + public readonly port = faker.datatype.number({ min: 20000, max: 30000 }); + public readonly host = 'localhost'; + public readonly user = 'postgres'; + public readonly database = 'postgres'; + private container?: Docker.Container; + + public async prepare() { + const pullStream = await docker.pull(this.image); + // https://github.com/apocas/dockerode/issues/647 + await new Promise((res) => docker.modem.followProgress(pullStream, res)); + this.container = await docker.createContainer({ + name: `vulcan-pg-test-${faker.random.word()}`, + Image: this.image, + ExposedPorts: { + '5432/tcp': {}, + }, + Env: [`POSTGRES_PASSWORD=${this.password}`], + HostConfig: { + PortBindings: { '5432/tcp': [{ HostPort: `${this.port}` }] }, + }, + }); + await this.container.start({}); + await this.waitPGReady(); + // Init data + const client = new Client({ + host: 'localhost', + port: this.port, + password: this.password, + user: 'postgres', + database: 'postgres', + }); + await client.connect(); + await client.query( + `create table if not exists users (id INTEGER, name VARCHAR, enabled BOOLEAN);` + ); + for (let i = 1; i <= 200; i++) { + await client.query( + `insert into users values(${i}, $1, ${faker.datatype.boolean()});`, + [faker.name.firstName()] + ); + } + await client.end(); + } + + public async destroy() { + await this.container?.remove({ force: true }); + } + + private async waitPGReady() { + const waitConnection = await this.container?.exec({ + Cmd: ['sh', '-c', 'until pg_isready; do sleep 5; echo "not ready"; done'], + }); + if (!waitConnection) return; + await waitConnection.start({}); + let wait = 20; + while (wait--) { + const { Running, ExitCode } = await waitConnection.inspect(); + if (!Running && ExitCode === 0) return; + else if (!Running && ExitCode && ExitCode > 0) + throw new Error(`PG wait commend return exit code ${ExitCode}`); + await BPromise.delay(1000); + } + throw new Error(`PG timeout`); + } +} diff --git a/packages/extension-driver-pg/tsconfig.json b/packages/extension-driver-pg/tsconfig.json new file mode 100644 index 00000000..f5b85657 --- /dev/null +++ b/packages/extension-driver-pg/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/packages/extension-driver-pg/tsconfig.lib.json b/packages/extension-driver-pg/tsconfig.lib.json new file mode 100644 index 00000000..1925baa1 --- /dev/null +++ b/packages/extension-driver-pg/tsconfig.lib.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "declaration": true, + "types": [] + }, + "include": ["**/*.ts", "../../types/*.d.ts"], + "exclude": ["jest.config.ts", "**/*.spec.ts", "**/*.test.ts"] +} diff --git a/packages/extension-driver-pg/tsconfig.spec.json b/packages/extension-driver-pg/tsconfig.spec.json new file mode 100644 index 00000000..eb72f635 --- /dev/null +++ b/packages/extension-driver-pg/tsconfig.spec.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["jest", "node"] + }, + "include": [ + "jest.config.ts", + "**/*.test.ts", + "**/*.spec.ts", + "**/*.d.ts", + "../../types/*.d.ts" + ] +} diff --git a/packages/serve/src/models/extensions/responseFormatter.ts b/packages/serve/src/models/extensions/responseFormatter.ts index a635d36c..2c7aafe6 100644 --- a/packages/serve/src/models/extensions/responseFormatter.ts +++ b/packages/serve/src/models/extensions/responseFormatter.ts @@ -37,6 +37,10 @@ export abstract class BaseResponseFormatter // if response has data and columns. const { data, columns } = ctx.response.body as BodyResponse; const formatted = this.format(data, columns); + // koa destroy the stream when connection close, we need to destroy our upstream too to notice them to release the resource. + formatted.on('close', () => { + data.destroy(); + }); // set formatted stream to response in context this.toResponse(formatted, ctx); return; diff --git a/packages/serve/test/response-formatter/json.spec.ts b/packages/serve/test/response-formatter/json.spec.ts index 430d8fee..ee394bc2 100644 --- a/packages/serve/test/response-formatter/json.spec.ts +++ b/packages/serve/test/response-formatter/json.spec.ts @@ -25,6 +25,37 @@ describe('Test to respond to json', () => { expect(ctx.response.body).toEqual(expected); }); + it('Test to destroy the source when downstream is destroyed', (done) => { + // Arrange + const stubResponse = sinon.stubInterface(); + const source = new Stream.Readable({ + read() { + this.push('some-data'); + // we don't push pull to simulate an endless stream + }, + destroy(err, cb) { + done(); + cb(err); + }, + }); + stubResponse.body = { + data: source, + columns: {}, + }; + const ctx = { + ...sinon.stubInterface(), + url: faker.internet.url(), + response: stubResponse, + }; + // Act + const formatter = new JsonFormatter({}, ''); + formatter.formatToResponse(ctx); + (ctx.response.body as Stream.Readable).destroy(); + // Assert + expect(true).toEqual(true); + // the done function should be call after destroyed. + }, 1000); + it.each([ { input: { diff --git a/tsconfig.base.json b/tsconfig.base.json index a405ca3a..0de8a26e 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -72,6 +72,9 @@ "@vulcan-sql/extension-driver-duckdb": [ "packages/extension-driver-duckdb/src/index.ts" ], + "@vulcan-sql/extension-driver-pg": [ + "packages/extension-driver-pg/src/index.ts" + ], "@vulcan-sql/integration-testing": [ "packages/integration-testing/src/index" ], @@ -80,6 +83,8 @@ "@vulcan-sql/serve/auth": ["packages/serve/src/lib/auth/index"], "@vulcan-sql/serve/auth/*": ["packages/serve/src/lib/auth/*"], "@vulcan-sql/serve/containers": ["packages/serve/src/containers/index"], + "@vulcan-sql/serve/evaluator": ["packages/serve/src/lib/evaluator/index"], + "@vulcan-sql/serve/evaluator/*": ["packages/serve/src/lib/evaluator/*"], "@vulcan-sql/serve/loader": ["packages/serve/src/lib/loader"], "@vulcan-sql/serve/middleware": [ "packages/serve/src/lib/middleware/index" @@ -102,8 +107,6 @@ "@vulcan-sql/serve/types": ["packages/serve/src/containers/types"], "@vulcan-sql/serve/utils": ["packages/serve/src/lib/utils/index"], "@vulcan-sql/serve/utils/*": ["packages/serve/src/lib/utils/*"], - "@vulcan-sql/serve/evaluator": ["packages/serve/src/lib/evaluator/index"], - "@vulcan-sql/serve/evaluator/*": ["packages/serve/src/lib/evaluator/*"], "@vulcan-sql/test-utility": ["packages/test-utility/src/index"] } }, diff --git a/workspace.json b/workspace.json index 6128e907..eab054aa 100644 --- a/workspace.json +++ b/workspace.json @@ -7,6 +7,7 @@ "extension-dbt": "packages/extension-dbt", "extension-debug-tools": "packages/extension-debug-tools", "extension-driver-duckdb": "packages/extension-driver-duckdb", + "extension-driver-pg": "packages/extension-driver-pg", "integration-testing": "packages/integration-testing", "serve": "packages/serve", "test-utility": "packages/test-utility" diff --git a/yarn.lock b/yarn.lock index 69e1284f..15027aec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -288,6 +288,11 @@ "@babel/helper-validator-identifier" "^7.16.7" to-fast-properties "^2.0.0" +"@balena/dockerignore@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@balena/dockerignore/-/dockerignore-1.0.2.tgz#9ffe4726915251e8eb69f44ef3547e0da2c03e0d" + integrity sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q== + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -1136,6 +1141,22 @@ "@types/keygrip" "*" "@types/node" "*" +"@types/docker-modem@*": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@types/docker-modem/-/docker-modem-3.0.2.tgz#c49c902e17364fc724e050db5c1d2b298c6379d4" + integrity sha512-qC7prjoEYR2QEe6SmCVfB1x3rfcQtUr1n4x89+3e0wSTMQ/KYCyf+/RAA9n2tllkkNc6//JMUZePdFRiGIWfaQ== + dependencies: + "@types/node" "*" + "@types/ssh2" "*" + +"@types/dockerode@^3.3.9": + version "3.3.9" + resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-3.3.9.tgz#8c6e519fd4d0d86717b2c6a864904f4e6aa8af40" + integrity sha512-SYRN5FF/qmwpxUT6snJP5D8k0wgoUKOGVs625XvpRJOOUi6s//UYI4F0tbyE3OmzpI70Fo1+aqpzX27zCrInww== + dependencies: + "@types/docker-modem" "*" + "@types/node" "*" + "@types/express-serve-static-core@^4.17.18": version "4.17.28" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz#c47def9f34ec81dc6328d0b1b5303d1ec98d86b8" @@ -1355,6 +1376,23 @@ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== +"@types/pg-cursor@^2.7.0": + version "2.7.0" + resolved "https://registry.yarnpkg.com/@types/pg-cursor/-/pg-cursor-2.7.0.tgz#9f11ffb2fa77d5c693429ba188ee55fce5928316" + integrity sha512-4Milg/OUqTO2VuPvvRwPxaQTaiVb+bXvSK+ZCwiHjwinbD4/lPqV9AREg8sJAT0cy5ruY38aaajBc2FbdPaKcA== + dependencies: + "@types/node" "*" + "@types/pg" "*" + +"@types/pg@*", "@types/pg@^8.6.5": + version "8.6.5" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.6.5.tgz#2dce9cb468a6a5e0f1296a59aea3ac75dd27b702" + integrity sha512-tOkGtAqRVkHa/PVZicq67zuujI4Oorfglsr2IbKofDwBSysnaqSx7W1mDqFqdkGE6Fbgh+PZAl0r/BWON/mozw== + dependencies: + "@types/node" "*" + pg-protocol "*" + pg-types "^2.2.0" + "@types/prettier@^2.1.5": version "2.6.0" resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.6.0.tgz#efcbd41937f9ae7434c714ab698604822d890759" @@ -1427,6 +1465,13 @@ resolved "https://registry.yarnpkg.com/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.2.tgz#bf2e02a3dbd4aecaf95942ecd99b7402e03fad5e" integrity sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA== +"@types/ssh2@*": + version "1.11.5" + resolved "https://registry.yarnpkg.com/@types/ssh2/-/ssh2-1.11.5.tgz#b669c97fa4f9dfd7193c4750e87eaabffad0ae9d" + integrity sha512-RaBsPKr+YP/slH8iR7XfC7chyomU+V57F/gJ5cMSP2n6/YWKVmeRLx7lrkgw4YYLpEW5lXLAdfZJqGo0PXboSA== + dependencies: + "@types/node" "*" + "@types/stack-utils@^2.0.0": version "2.0.1" resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c" @@ -1796,7 +1841,7 @@ asap@^2.0.0, asap@^2.0.3: resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= -asn1@~0.2.3: +asn1@^0.2.4, asn1@~0.2.3: version "0.2.6" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d" integrity sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ== @@ -1927,7 +1972,7 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" -bcrypt-pbkdf@^1.0.0: +bcrypt-pbkdf@^1.0.0, bcrypt-pbkdf@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" integrity sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w== @@ -1953,6 +1998,11 @@ bl@^4.0.3, bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" +bluebird@^3.7.2: + version "3.7.2" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" + integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -2046,6 +2096,11 @@ buffer@^5.5.0, buffer@^5.6.0: base64-js "^1.3.1" ieee754 "^1.1.13" +buildcheck@0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/buildcheck/-/buildcheck-0.0.3.tgz#70451897a95d80f7807e68fc412eb2e7e35ff4d5" + integrity sha512-pziaA+p/wdVImfcbsZLNF32EiWyujlQLwolMqUQE8xpKNOH7KmZQaY8sXN7DGOEzPAElo9QTaeNRfGnf3iOJbA== + bytes@3.1.2, bytes@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" @@ -2172,6 +2227,11 @@ chokidar@^3.5.1, chokidar@^3.5.3: optionalDependencies: fsevents "~2.3.2" +chownr@^1.1.1: + version "1.1.4" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" + integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== + chownr@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" @@ -2471,6 +2531,14 @@ cosmiconfig@^7, cosmiconfig@^7.0.0: path-type "^4.0.0" yaml "^1.10.0" +cpu-features@~0.0.4: + version "0.0.4" + resolved "https://registry.yarnpkg.com/cpu-features/-/cpu-features-0.0.4.tgz#0023475bb4f4c525869c162e4108099e35bf19d8" + integrity sha512-fKiZ/zp1mUwQbnzb9IghXtHtDoTMtNeb8oYGx6kX2SYfhnG0HNdBEBIzB9b5KlXu5DQPhfy3mInbBxFcgwAr3A== + dependencies: + buildcheck "0.0.3" + nan "^2.15.0" + create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" @@ -2712,6 +2780,25 @@ discontinuous-range@1.0.0: resolved "https://registry.yarnpkg.com/discontinuous-range/-/discontinuous-range-1.0.0.tgz#e38331f0844bba49b9a9cb71c771585aab1bc65a" integrity sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ== +docker-modem@^3.0.0: + version "3.0.6" + resolved "https://registry.yarnpkg.com/docker-modem/-/docker-modem-3.0.6.tgz#8c76338641679e28ec2323abb65b3276fb1ce597" + integrity sha512-h0Ow21gclbYsZ3mkHDfsYNDqtRhXS8fXr51bU0qr1dxgTMJj0XufbzX+jhNOvA8KuEEzn6JbvLVhXyv+fny9Uw== + dependencies: + debug "^4.1.1" + readable-stream "^3.5.0" + split-ca "^1.0.1" + ssh2 "^1.11.0" + +dockerode@^3.3.4: + version "3.3.4" + resolved "https://registry.yarnpkg.com/dockerode/-/dockerode-3.3.4.tgz#875de614a1be797279caa9fe27e5637cf0e40548" + integrity sha512-3EUwuXnCU+RUlQEheDjmBE0B7q66PV9Rw5NiH1sXwINq0M9c5ERP9fxgkw36ZHOtzf4AGEEYySnkx/sACC9EgQ== + dependencies: + "@balena/dockerignore" "^1.0.2" + docker-modem "^3.0.0" + tar-fs "~2.0.1" + doctrine@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961" @@ -2790,7 +2877,7 @@ encodeurl@^1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= -end-of-stream@^1.4.1: +end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== @@ -5175,6 +5262,11 @@ mixin-deep@^1.2.0: for-in "^1.0.2" is-extendable "^1.0.1" +mkdirp-classic@^0.5.2: + version "0.5.3" + resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" + integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== + mkdirp@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" @@ -5267,6 +5359,11 @@ mute-stream@0.0.8: resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d" integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA== +nan@^2.15.0, nan@^2.16.0: + version "2.16.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.16.0.tgz#664f43e45460fb98faf00edca0bb0d7b8dce7916" + integrity sha512-UdAqHyFngu7TfQKsCBgAA6pWDkT8MAO7d0jyOecVhN5354xbLqdn8mV9Tat9gepAupm0bt2DbeaSC8vS52MuFA== + nanomatch@^1.2.9: version "1.2.13" resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.13.tgz#b87a8aa4fc0de8fe6be88895b38983ff265bd119" @@ -5588,7 +5685,7 @@ on-finished@^2.3.0: dependencies: ee-first "1.1.1" -once@1.4.0, once@^1.3.0, once@^1.4.0: +once@1.4.0, once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= @@ -5819,6 +5916,11 @@ pg-connection-string@^2.5.0: resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== +pg-cursor@^2.7.4: + version "2.7.4" + resolved "https://registry.yarnpkg.com/pg-cursor/-/pg-cursor-2.7.4.tgz#e53ac24c7d227b1c3b5a7c5d4b4c64148c13d040" + integrity sha512-CNWwOzTTZ9QvphoOL+Wg/7pmVr9GnAWBjPbuK2FRclrB4A/WRO/ssCJ9BlkzIGmmofK2M/LyokNHgsLSn+fMHA== + pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" @@ -5844,12 +5946,17 @@ pg-pool@^3.3.0: resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.5.1.tgz#f499ce76f9bf5097488b3b83b19861f28e4ed905" integrity sha512-6iCR0wVrro6OOHFsyavV+i6KYL4lVNyYAB9RD18w66xSzN+d8b66HiwuP30Gp1SH5O9T82fckkzsRjlrhD0ioQ== -pg-protocol@^1.5.0: +pg-pool@^3.5.2: + version "3.5.2" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.5.2.tgz#ed1bed1fb8d79f1c6fd5fb1c99e990fbf9ddf178" + integrity sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w== + +pg-protocol@*, pg-protocol@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.5.0.tgz#b5dd452257314565e2d54ab3c132adc46565a6a0" integrity sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ== -pg-types@^2.1.0: +pg-types@^2.1.0, pg-types@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== @@ -5873,6 +5980,19 @@ pg@8.6.0: pg-types "^2.1.0" pgpass "1.x" +pg@^8.8.0: + version "8.8.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.8.0.tgz#a77f41f9d9ede7009abfca54667c775a240da686" + integrity sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw== + dependencies: + buffer-writer "2.0.0" + packet-reader "1.0.0" + pg-connection-string "^2.5.0" + pg-pool "^3.5.2" + pg-protocol "^1.5.0" + pg-types "^2.1.0" + pgpass "1.x" + pgpass@1.x: version "1.0.5" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" @@ -6010,6 +6130,14 @@ psl@^1.1.33: resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== +pump@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" + integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== + dependencies: + end-of-stream "^1.1.0" + once "^1.3.1" + punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" @@ -6091,7 +6219,7 @@ readable-stream@^2.0.0, readable-stream@^2.0.6: string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: +readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -6606,6 +6734,11 @@ sparse-bitfield@^3.0.3: dependencies: memory-pager "^1.0.2" +split-ca@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6" + integrity sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ== + split-string@^3.0.1, split-string@^3.0.2: version "3.1.0" resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2" @@ -6628,6 +6761,17 @@ sqlstring@2.3.2: resolved "https://registry.yarnpkg.com/sqlstring/-/sqlstring-2.3.2.tgz#cdae7169389a1375b18e885f2e60b3e460809514" integrity sha512-vF4ZbYdKS8OnoJAWBmMxCQDkiEBkGQYU7UZPtL8flbDRSNkhaXvRJ279ZtI6M+zDaQovVU4tuRgzK5fVhvFAhg== +ssh2@^1.11.0: + version "1.11.0" + resolved "https://registry.yarnpkg.com/ssh2/-/ssh2-1.11.0.tgz#ce60186216971e12f6deb553dcf82322498fe2e4" + integrity sha512-nfg0wZWGSsfUe/IBJkXVll3PEZ//YH2guww+mP88gTpuSU4FtZN7zu9JoeTGOyCNx2dTDtT9fOpWwlzyj4uOOw== + dependencies: + asn1 "^0.2.4" + bcrypt-pbkdf "^1.0.2" + optionalDependencies: + cpu-features "~0.0.4" + nan "^2.16.0" + sshpk@^1.7.0: version "1.17.0" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.17.0.tgz#578082d92d4fe612b13007496e543fa0fbcbe4c5" @@ -6833,7 +6977,17 @@ symbol-tree@^3.2.4: resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== -tar-stream@~2.2.0: +tar-fs@~2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2" + integrity sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA== + dependencies: + chownr "^1.1.1" + mkdirp-classic "^0.5.2" + pump "^3.0.0" + tar-stream "^2.0.0" + +tar-stream@^2.0.0, tar-stream@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==