Switch to Postgres DB for UDMIF and other various fixes (#553)

cloud/gcp/api.tf

@@ -12,7 +12,8 @@ resource "google_project_service" "service" {
     "cloudfunctions.googleapis.com",
     "cloudbuild.googleapis.com",    
     "container.googleapis.com",
-    "containerregistry.googleapis.com"
+    "containerregistry.googleapis.com",
+    "servicenetworking.googleapis.com"
   ])
 
   service = each.key

cloud/gcp/postgresql.tf

@@ -0,0 +1,55 @@
+resource "google_sql_database_instance" "main" {
+  name             = var.instance_name 
+  database_version = var.database_version 
+  region           = var.region
+  deletion_protection = var.deletion_protection
+
+  settings {
+    # Second-generation instance tiers are based on the machine
+    # type. See argument reference below.
+    tier = var.tier 
+    user_labels = {
+      "environment" = var.environment
+    }
+    backup_configuration {
+      location = "us"
+      point_in_time_recovery_enabled = true
+      enabled = true
+      start_time = "00:00"
+    }
+    maintenance_window {
+      day = 7
+      hour = 0
+      update_track = "stable"
+    }
+  }
+}
+
+resource "google_compute_network" "private_network" {
+  name = "private-network"
+}
+
+resource "google_compute_global_address" "private_ip_address" {
+  name          = "private-ip-address"
+  purpose       = "VPC_PEERING"
+  address_type  = "INTERNAL"
+  prefix_length = 16
+  network       = google_compute_network.private_network.id
+}
+
+resource "google_service_networking_connection" "private_vpc_connection" {
+  network                 = google_compute_network.private_network.id
+  service                 = "servicenetworking.googleapis.com"
+  reserved_peering_ranges = [google_compute_global_address.private_ip_address.name]
+}
+
+resource "google_sql_database" "database" {
+    name = var.db_name
+    instance = google_sql_database_instance.main.name
+} 
+
+resource "google_sql_user" "users" {
+  name     = var.db_user 
+  instance = google_sql_database_instance.main.name
+  password = var.password  
+}

cloud/gcp/variables.tf

@@ -109,6 +109,45 @@ variable "create_vpc" {
   description = "Indicates if we use the default vpc or create a new one."
 }
 
+## Postgresql variables
+variable "instance_name" {
+  type = string
+  description = "The name of the DB instance"
+}
+variable "database_version" {
+  type = string
+  description = "DB Server version to use"
+}
+variable "region" {
+  type = string
+  description = "The region the instance will sit in."
+}
+variable "tier" {
+  type = string
+  description = "custom machine type to use"
+}
+variable "deletion_protection" {
+  type = bool
+  default = true
+  description = "Whether or not to allow Terraform to destroy the instance"
+}
+variable "environment" {
+  type = string
+  description = "user label of the instace"
+}
+variable "db_name" {
+  type = string
+  description = "database name in GCP"
+}
+variable "db_user" {
+  type = string
+  description = "Postgresql db user name in GCP"
+}
+variable "password" {
+  type = string
+  description = "Postgresql db password in GCP"
+}
+
 ## Mongodb variables
 variable "public_key" {
   type        = string

cloud/gcp/vpc.tf

@@ -13,3 +13,11 @@ resource "google_compute_subnetwork" "subnet" {
   network       = google_compute_network.vpc[count.index].name
   ip_cidr_range = var.ip_cidr_range
 }
+
+# serverless VPC
+resource "google_vpc_access_connector" "connector" {
+  name          = "udmi-cf-sql-vpc"
+  ip_cidr_range = "10.9.0.0/28"
+  network       = "private-network"
+  max_throughput = 300
+} 

udmif/api/.env.example

@@ -6,16 +6,6 @@ NODE_ENV=development
 DATASOURCE=STATIC
 # the google project id where this is running
 PROJECT_ID=
-# Mongo Protocol
-MONGO_PROTOCOL=mongodb
-# Mongo Connection String
-MONGO_HOST=127.0.0.1
-# Mongo Database
-MONGO_DATABASE=udmi
-# Mongo User Name
-MONGO_USERNAME=
-# Mongo Password
-MONGO_PASSWORD=
 # Authentication Client id
 AUTH_CLIENT_ID=
 # Client ids

udmif/api/.gitignore

@@ -18,7 +18,4 @@ globalConfig.json
 # environment variables
 .env
 
-# mongo output
-.mongodb
-
 /util/devices.json

udmif/api/Dockerfile

@@ -9,6 +9,7 @@ WORKDIR /usr/src/app
 COPY package*.json ./
 COPY tsconfig.json .
 COPY src src
+COPY knexfile.ts .
 
 RUN npm install
 

udmif/api/README.md

@@ -18,9 +18,7 @@ A Google account that can login to web application at https://web.staging.udmi.b
     ./buildDev.sh
     ```
 
-2.  If you want to point your API to a MongoDB, you can edit the values in .env as needed
-
-3.  Update the AUTH_CLIENT_ID and CLIENT_IDS with the values for Google identity provider.  These values should be provided by whomever manages your Auth Provider.
+2.  Update the AUTH_CLIENT_ID and CLIENT_IDS with the values for Google identity provider.  These values should be provided by whomever manages your Auth Provider.
 
 ## Run the Project
 ---
@@ -37,34 +35,14 @@ A Google account that can login to web application at https://web.staging.udmi.b
 
 **Assumptions:** 
 
-1. mongodb is installed and running
-2. udmi db has been created in mongo
-
-Install the required mongodb tools
-```
-brew tap mongodb/brew
-brew install mongodb-database-tools
-```
-
-Execute the data import
-```
-mongoimport --uri="mongodb://127.0.0.1:27017/udmi" -c=device --file=util/devices.json --mode=upsert --jsonArray
-```
-
-## Convert runtime to use MongoDB
----
-
-By using Mongo as the DB, the data will not be static or hard coded and can be changed without needing to restart the service or changing code.  Switching to Mongo entails:
-
-1. Change the DATASOURCE field to 'MONGO' in the .env file.
+1. PostgreSQL is installed and running
+2. udmi db has been created in PostgreSQL
 
 ## Testing
 ---
 
 ### Automated Tests
 
-Followed instructions at [Using with MongoDB](https://jestjs.io/docs/mongodb) and [jest-mongodb](https://github.com/shelfio/jest-mongodb) to configure and run in memory mongodb for testing
-
 - Run `npm test` to execute the unit tests via [Jest](https://jestjs.io).
 - Run `npm run testInteractive` to continuosly execute the unit tests via [Jest](https://jestjs.io).  The tests will be run every time a src file is saved.
 
@@ -75,10 +53,11 @@ You can send GraphQL requests to [http://localhost:4300/](http://localhost:4300/
 ## Notes
 ---
 - Run `npm build` to build the project. The build artifacts will be stored in the `dist/` directory.
-- Creating a MongoDB - Create a 'udmi' db with a 'device' collection by following the instructions here: [Creating a DB](https://www.mongodb.com/basics/create-database).  
 
 ## Common Problems and Solutions
 ---
 | Problem | Solution | 
 | --- | --- |
-| The API does not return data. | Fix - Double check that the client id's are filled in the .env file |
+| The API does not return data. | Fix - Double check that the client id's are filled in the .env file |
+
+

udmif/api/build.yml

@@ -14,7 +14,6 @@ stages:
   - static-scan
   - publish-docker
   - deploy-staging
-  - deploy-lab
 
 sonarqube-scan:
   rules:
@@ -42,20 +41,8 @@ deploy-staging:
     NAMESPACE: udmi
     CHART_NAME: udmi-api
     HELM_REPO: udmif/helm
-    CHART_VALUES: --set image.tag=$CI_PIPELINE_IID --set image.repository=registry.gitlab.com/buildings-iot/udmi/udmif-api --set dockerSecret.accessToken=$GITLAB_REGISTRY_TOKEN --create-namespace --set dockerSecret.registry=$GITLAB_REGISTRY_URL --set dockerSecret.username=gitlab --set dockerSecret.create=true --set env.AUTH_CLIENT_ID=$AUTH_CLIENT_ID --set env.CLIENT_IDS=$CLIENT_IDS --set env.MONGO_PROTOCOL=$MONGO_PROTOCOL --set env.MONGO_USER=$MONGO_USER --set env.MONGO_PWD=$MONGO_PWD --set env.MONGO_HOST=$MONGO_HOST --set env.MONGO_DATABASE=$MONGO_DATABASE --set env.DATASOURCE=$DATASOURCE
+    CHART_VALUES: --set image.tag=$CI_PIPELINE_IID --set image.repository=registry.gitlab.com/buildings-iot/udmi/udmif-api --set dockerSecret.accessToken=$GITLAB_REGISTRY_TOKEN --create-namespace --set dockerSecret.registry=$GITLAB_REGISTRY_URL --set dockerSecret.username=gitlab --set dockerSecret.create=true --set env.AUTH_CLIENT_ID=$AUTH_CLIENT_ID --set env.CLIENT_IDS=$CLIENT_IDS --set env.POSTGRESQL_INSTANCE_HOST=$POSTGRESQL_INSTANCE_HOST --set env.POSTGRESQL_PORT=$POSTGRESQL_PORT --set env.POSTGRESQL_USER=$POSTGRESQL_USER --set env.POSTGRESQL_PASSWORD=$POSTGRESQL_PASSWORD --set env.POSTGRESQL_DATABASE=$POSTGRESQL_DATABASE 
 
-deploy-lab:
-  extends: .gks-deploy
-  stage: deploy-lab
-  environment: lab
-  variables:
-    NAMESPACE: udmi
-    CHART_NAME: udmi-api
-    HELM_REPO: udmif/helm
-    CHART_VALUES: --set image.tag=$CI_PIPELINE_IID --set image.repository=registry.gitlab.com/buildings-iot/udmi/udmif-api --set dockerSecret.accessToken=$GITLAB_REGISTRY_TOKEN --create-namespace --set dockerSecret.registry=$GITLAB_REGISTRY_URL --set dockerSecret.username=gitlab --set dockerSecret.create=true --set env.AUTH_CLIENT_ID=$AUTH_CLIENT_ID --set env.CLIENT_IDS=$CLIENT_IDS --set env.MONGO_PROTOCOL=$MONGO_PROTOCOL --set env.MONGO_USER=$MONGO_USER --set env.MONGO_PWD=$MONGO_PWD --set env.MONGO_HOST=$MONGO_HOST --set env.MONGO_DATABASE=$MONGO_DATABASE --set env.DATASOURCE=$DATASOURCE
-  rules:
-      - if: $CI_COMMIT_BRANCH == "master"
-        when: manual  
 
 
 

udmif/api/jest.config.js

@@ -13,7 +13,6 @@ module.exports = {
     "cobertura",
     "lcov"
   ],
-  preset: "@shelf/jest-mongodb",
   // testEnvironment: "node",
   testRegex: '(/__tests__/spec.ts|(\\.|/)(test|spec))\\.tsx?$',
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],

udmif/api/knexfile.ts

@@ -0,0 +1,17 @@
+require('ts-node/register');
+
+module.exports = {
+    client: 'pg',
+    connection: {
+        host: process.env.POSTGRESQL_INSTANCE_HOST,
+        port: parseInt(process.env.POSTGRESQL_PORT || ""),
+        user: process.env.POSTGRESQL_USER,
+        password: process.env.POSTGRESQL_PASSWORD,
+        database: process.env.POSTGRESQL_DATABASE,
+    },
+    migrations: {
+        directory: './src/migrations'
+    },
+    debug: true
+};
+