Feature: New driver - pg

.circleci/config.yml

@@ -3,10 +3,12 @@ version: 2.1
 executors:
   node-executor:
     working_directory: ~/repo
-    docker:
-      # Jest has issues with node > 16.11.0, please stick node version until the issues are resolved
-      # https://github.com/facebook/jest/issues/11956
-      - image: cimg/node:16.10.0
+    machine:
+      # What software on the machine https://discuss.circleci.com/t/linux-machine-executor-update-2022-july-q3-update/44873
+      image: ubuntu-2204:2022.07.1
+      # About DLC https://circleci.com/docs/docker-layer-caching
+      docker_layer_caching: true
+    resource_class: large
 
 parameters:
   workflow_name:
@@ -134,9 +136,7 @@ jobs:
             - codecov-0.1.0_4653
       - run: rm -rf ./coverage
         # Clear coverage folder
-      - run: yarn nx affected --target=test --ci --coverage --maxWorkers=2 --coverageReporters=lcov ${AFFECTED_ARGS}
-        # --maxWorkers=2 is required because we'll run virtual machine with 32 cores CPU (with actually 4 CPI), jest spawns lots of workers if we don't fix the worker size.
-        # https://support.circleci.com/hc/en-us/articles/360005442714-Your-test-tools-are-smart-and-that-s-a-problem-Learn-about-when-optimization-goes-wrong-
+      - run: yarn nx affected --target=test --ci --coverage --coverageReporters=lcov ${AFFECTED_ARGS}
       - run:
           name: upload reports with flags
           command: |
@@ -163,9 +163,6 @@ jobs:
           <<: *set_env
       - restore_cache:
           <<: *yarn_cache
-      # Setup remote docker to build image
-      # https://circleci.com/docs/building-docker-images
-      - setup_remote_docker
       - run:
           name: NPM publish
           # running with yarn nx causes npm config issues

jest.setup.ts

@@ -2,3 +2,10 @@
  * Add reflect-metadata to collect Inversify IoC decorator information when running jest test.
  */
 import 'reflect-metadata';
+/**
+ * https://github.com/prisma/prisma/issues/8558
+ * in jest 27+, we'll have some issues with setImmediate function, this is a workaround
+ */
+global.setImmediate =
+  global.setImmediate ||
+  ((fn: any, ...args: any) => global.setTimeout(fn, 0, ...args));

package.json

@@ -9,6 +9,7 @@
   "dependencies": {
     "@koa/cors": "^3.3.0",
     "chokidar": "^3.5.3",
+    "bluebird": "^3.7.2",
     "bcryptjs": "^2.4.3",
     "bytes": "^3.1.2",
     "class-validator": "^0.13.2",
@@ -31,6 +32,8 @@
     "nunjucks": "^3.2.3",
     "openapi3-ts": "^2.0.2",
     "ora": "^5.4.1",
+    "pg": "^8.8.0",
+    "pg-cursor": "^2.7.4",
     "redoc": "2.0.0-rc.76",
     "reflect-metadata": "^0.1.13",
     "semver": "^7.3.7",
@@ -48,6 +51,7 @@
     "@nrwl/workspace": "14.0.3",
     "@types/bcryptjs": "^2.4.2",
     "@types/bytes": "^3.1.1",
+    "@types/dockerode": "^3.3.9",
     "@types/from2": "^2.3.1",
     "@types/glob": "^7.2.0",
     "@types/inquirer": "^8.0.0",
@@ -63,13 +67,16 @@
     "@types/lodash": "^4.14.182",
     "@types/md5": "^2.3.2",
     "@types/node": "16.11.7",
+    "@types/pg": "^8.6.5",
+    "@types/pg-cursor": "^2.7.0",
     "@types/semver": "^7.3.12",
     "@types/supertest": "^2.0.12",
     "@types/uuid": "^8.3.4",
     "@typescript-eslint/eslint-plugin": "~5.18.0",
     "@typescript-eslint/parser": "~5.18.0",
     "commitizen": "^4.2.5",
     "cz-conventional-changelog": "^3.3.0",
+    "dockerode": "^3.3.4",
     "eslint": "~8.12.0",
     "eslint-config-prettier": "8.1.0",
     "from2": "^2.3.0",

packages/build/test/builder/profile.yaml

@@ -1,3 +1,3 @@
 - name: test
-  type: pg
+  type: mock
   allow: '*'

packages/core/src/lib/data-source/index.ts

@@ -1,4 +1,4 @@
-import { PGDataSource } from './pg';
-export const builtInDataSource = [PGDataSource];
+import { MockDataSource } from './mock';
+export const builtInDataSource = [MockDataSource];
 
-export * from './pg';
+export * from './mock';

packages/core/src/lib/data-source/pg.ts → packages/core/src/lib/data-source/mock.ts

@@ -9,8 +9,8 @@ import {
 } from '../../models/extensions';
 
 @VulcanInternalExtension()
-@VulcanExtensionId('pg')
-export class PGDataSource extends DataSource {
+@VulcanExtensionId('mock')
+export class MockDataSource extends DataSource {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   public async execute(options: ExecuteOptions): Promise<DataResult> {
     return {

packages/extension-driver-duckdb/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vulcan-sql/extension-driver-duckdb",
   "description": "duckdb driver for Vulcan SQL",
-  "version": "0.1.0-alpha.1",
+  "version": "0.2.0",
   "type": "commonjs",
   "publishConfig": {
     "access": "public"
@@ -23,6 +23,6 @@
   },
   "license": "MIT",
   "peerDependencies": {
-    "@vulcan-sql/core": ">= 0.1.0 || >= 0.1.0-alpha.1"
+    "@vulcan-sql/core": "~0.2.0-0"
   }
 }

packages/extension-driver-duckdb/project.json

@@ -49,7 +49,7 @@
     "publish": {
       "executor": "@nrwl/workspace:run-commands",
       "options": {
-        "command": "node ../../../tools/scripts/publish.mjs {args.tag}",
+        "command": "node ../../../tools/scripts/publish.mjs {args.tag} {args.version}",
         "cwd": "dist/packages/extension-driver-duckdb"
       },
       "dependsOn": [

packages/extension-driver-pg/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

packages/extension-driver-pg/README.md

@@ -0,0 +1,58 @@
+# extension-driver-pg
+
+[node-postgres](https://node-postgres.com/) driver for Vulcan SQL.
+
+## Install
+
+1. Install package
+
+   ```sql
+   npm i @vulcan-sql/extension-driver-pg
+   ```
+
+2. Update `vulcan.yaml`, enable the extension.
+
+   ```yaml
+   extensions:
+     pg: '@vulcan-sql/extension-driver-pg'
+   ```
+
+3. Create a new profile in `profiles.yaml` or in your profiles' paths.
+
+   ```yaml
+   - name: pg # profile name
+     type: pg
+     connection:
+      # Optional: The max rows we should fetch once.
+      chunkSize: 100
+      # Optional: Maximum number of clients the pool should contain.
+      max: 10
+      # Optional: Number of milliseconds a client must sit idle in the pool and not be checked out before it is disconnected from the backend and discarded.
+      idleTimeoutMillis: 10000
+      # Optional: Number of milliseconds to wait before timing out when connecting a new client by default this is 0 which means no timeout
+      connectionTimeoutMillis: 0
+      # Optional: The user to connect to database. Default process.env.PGUSER || process.env.USER
+      user: string
+      # Optional: Password to connect to database. default process.env.PGPASSWORD
+      password: string
+      # Optional: Server host. default process.env.PGHOST
+      host: string
+      # Optional: Name of database. default process.env.PGDATABASE || user
+      database: string
+      # Optional: Server port. default process.env.PGPORT
+      port: 5432
+      # Optional: Connection string.
+      connectionString: postgres://user:password@host:5432/database
+      # Optional: Passed directly to node.TLSSocket, supports all tls.connect options
+      ssl: false
+      # Optional: Number of milliseconds before a statement in query will time out, default is no timeout
+      statement_timeout: 0
+      # Optional: Number of milliseconds before a query call will timeout, default is no timeout
+      query_timeout: 0
+      # Optional: The name of the application that created this Client instance
+      application_name: string
+      # Optional: Number of milliseconds to wait for connection, default is no timeout
+      connectionTimeoutMillis: 0
+      # Optional: Number of milliseconds before terminating any session with an open idle transaction, default is no timeout
+      idle_in_transaction_session_timeout: 0
+   ```

packages/extension-driver-pg/jest.config.ts

@@ -0,0 +1,14 @@
+module.exports = {
+  displayName: 'extension-driver-pg',
+  preset: '../../jest.preset.ts',
+  globals: {
+    'ts-jest': {
+      tsconfig: '<rootDir>/tsconfig.spec.json',
+    },
+  },
+  transform: {
+    '^.+\\.[tj]s$': 'ts-jest',
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+  coverageDirectory: '../../coverage/packages/extension-driver-pg',
+};

packages/extension-driver-pg/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@vulcan-sql/extension-driver-pg",
+  "description": "PG driver for Vulcan SQL",
+  "version": "0.2.0",
+  "type": "commonjs",
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "vulcan",
+    "vulcan-sql",
+    "data",
+    "sql",
+    "database",
+    "data-warehouse",
+    "data-lake",
+    "api-builder",
+    "postgres",
+    "pg"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Canner/vulcan.git"
+  },
+  "license": "MIT",
+  "peerDependencies": {
+    "@vulcan-sql/core": "~0.2.0-0"
+  }
+}

packages/extension-driver-pg/project.json

@@ -0,0 +1,64 @@
+{
+  "root": "packages/extension-driver-pg",
+  "sourceRoot": "packages/extension-driver-pg/src",
+  "targets": {
+    "build": {
+      "executor": "@nrwl/workspace:run-commands",
+      "options": {
+        "command": "yarn ts-node ./tools/scripts/replaceAlias.ts extension-driver-pg"
+      },
+      "dependsOn": [
+        {
+          "projects": "self",
+          "target": "tsc"
+        }
+      ]
+    },
+    "tsc": {
+      "executor": "@nrwl/js:tsc",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "outputPath": "dist/packages/extension-driver-pg",
+        "main": "packages/extension-driver-pg/src/index.ts",
+        "tsConfig": "packages/extension-driver-pg/tsconfig.lib.json",
+        "assets": ["packages/extension-driver-pg/*.md"],
+        "buildableProjectDepsInPackageJsonType": "dependencies"
+      },
+      "dependsOn": [
+        {
+          "projects": "dependencies",
+          "target": "build"
+        }
+      ]
+    },
+    "lint": {
+      "executor": "@nrwl/linter:eslint",
+      "outputs": ["{options.outputFile}"],
+      "options": {
+        "lintFilePatterns": ["packages/extension-driver-pg/**/*.ts"]
+      }
+    },
+    "test": {
+      "executor": "@nrwl/jest:jest",
+      "outputs": ["coverage/packages/extension-driver-pg"],
+      "options": {
+        "jestConfig": "packages/extension-driver-pg/jest.config.ts",
+        "passWithNoTests": true
+      }
+    },
+    "publish": {
+      "executor": "@nrwl/workspace:run-commands",
+      "options": {
+        "command": "node ../../../tools/scripts/publish.mjs {args.tag} {args.version}",
+        "cwd": "dist/packages/extension-driver-pg"
+      },
+      "dependsOn": [
+        {
+          "projects": "self",
+          "target": "build"
+        }
+      ]
+    }
+  },
+  "tags": []
+}

packages/extension-driver-pg/src/index.ts

@@ -0,0 +1,3 @@
+export * from './lib/pgDataSource';
+import { PGDataSource } from './lib/pgDataSource';
+export default [PGDataSource];