From ec91547de90d1d1e947c46afda907f12e2ec4968 Mon Sep 17 00:00:00 2001 From: Chris Nurse Date: Sat, 4 Sep 2021 11:37:32 +0800 Subject: [PATCH] Fix bad commit and race conditions --- cli.js | 22 --- docker-compose.yml | 58 +++---- qik-trak-hasura.js | 46 +++++- qik-trak.js | 370 +++++++++++++-------------------------------- templates/read.api | 7 - 5 files changed, 172 insertions(+), 331 deletions(-) delete mode 100644 cli.js delete mode 100644 templates/read.api diff --git a/cli.js b/cli.js deleted file mode 100644 index ac3a9da..0000000 --- a/cli.js +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env node - -const env = require('dotenv').config(); -const QikTrak = require('./qik-trak'); - -const cfg = { - operations: - { - untrack: true, - trackTables: true, - trackRelationships: true - }, - - logOutput: true, - hasuraAdminSecret: process.env.HASURA_GRAPHQL_ADMIN_SECRET, - hasuraEndpoint: process.env.HASURA_GRAPHQL_ENDPOINT, - targetDatabase: process.env.TARGET_DATABASE, - targetSchema: process.env.TARGET_SCHEMA, - inContainer: process.env.IN_CONTAINER == 'true' || false - } - -new QikTrak(cfg).ExecuteQikTrack(); \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 461eb4d..7970527 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,60 +1,45 @@ version: "3.6" services: + qiktrak_db: - env_file: .env + env_file: .env - container_name: qiktrak_db - image: postgres + container_name: qiktrak_db + image: postgres - volumes: - - ./initdb:/docker-entrypoint-initdb.d/ + volumes: + - ./initdb:/docker-entrypoint-initdb.d/ - ports: - - "7001:5432" + ports: + - '7000:5432' - restart: always + restart: always qiktrak_gql: - env_file: .env - - image: hasura/graphql-engine:latest - container_name: qiktrak_gql + env_file: .env + + image: hasura/graphql-engine:latest + container_name: qiktrak_gql - links: - - qiktrak_db + links: + - qiktrak_db - ports: - - "7002:8080" + ports: + - '7001:8080' - command: - - graphql-engine - - serve + command: + - graphql-engine + - serve - restart: always + restart: always -<<<<<<< HEAD - qik-trak: - env_file: .env - environment: - - IN_CONTAINER=true -======= # Use this if you want to build the app into a container that you can run from the docker # qik-trak: # env_file: .env ->>>>>>> 5be04e37e07a4ff127d8bf97465e748a8c061761 # container_name: qik-trak -<<<<<<< HEAD - build: - context: . - dockerfile: ./Dockerfile - - links: - - qiktrak_db - - qiktrak_gql -======= # build: # context: . # dockerfile: ./Dockerfile @@ -62,7 +47,6 @@ services: # links: # - qiktrak_db # - qiktrak_gql ->>>>>>> 5be04e37e07a4ff127d8bf97465e748a8c061761 # command: # - node diff --git a/qik-trak-hasura.js b/qik-trak-hasura.js index 32ad520..3275201 100644 --- a/qik-trak-hasura.js +++ b/qik-trak-hasura.js @@ -12,6 +12,46 @@ class QikTrakHasura { this.Namer = new QikTrakNames(config); } + async UntrackTable(table_name) { + + var query = { + type: "pg_untrack_table", + args: { + table: { + schema: this.config.targetSchema, + name: table_name + }, + source: this.config.targetDatabase, + cascade: true + } + }; + + await this.runGraphQL_Query('/v1/metadata', query) + .catch(e => { + if (e.response.data.error.includes("already untracked")) { + return; + } + + this.Config.Logger.Log(""); + this.Config.Logger.Log(""); + this.Config.Logger.Log("--------------------------------------------------------------"); + this.Config.Logger.Log(""); + this.Config.Logger.Log("QIK-TRAK: ERROR"); + this.Config.Logger.Log(""); + this.Config.Logger.Log("GRAPHQL QUERY FAILED TO EXECUTE"); + this.Config.Logger.Log(""); + this.Config.Logger.Log("Error Message : " + e.response.data.internal.error.message); + this.Config.Logger.Log(e.response.request.data); + this.Config.Logger.Log(""); + this.Config.Logger.Log("Query:"); + this.Config.Logger.Log(""); + this.Config.Logger.Log(JSON.stringify(query)); + this.Config.Logger.Log(""); + this.Config.Logger.Log("Are Hasura and the database fully initialised?"); + this.Config.Logger.Log(""); + this.Config.Logger.Log("--------------------------------------------------------------"); + });; + } async createRelationships(relationship) { const array_rel_spec = { @@ -186,11 +226,13 @@ class QikTrakHasura { const root = JSON.parse(content); root.views.map(async (view) => { - await this.buildView(view); + await this.buildJsonView(view); }); } - async buildView(view) { + //-------------------------------------------------------------------------------------------------------------------------- + // Build a specific JSON view + async buildJsonView(view) { const view_header = ` diff --git a/qik-trak.js b/qik-trak.js index 8c6a939..c12890b 100644 --- a/qik-trak.js +++ b/qik-trak.js @@ -12,110 +12,23 @@ const QikTrakHasura = require("./qik-trak-hasura"); class QikTrack { constructor(cfg){ -<<<<<<< HEAD - this.config = cfg; - - if (!cfg.primaryKeySuffix) { - this.config.primaryKeySuffix = "_id"; - } - - // -------------------------------------------------------------------------------------------------------------------------- - // SQL to acquire table and foreign key metadata - - this.config.table_sql = -` -SELECT table_name FROM information_schema.tables WHERE table_schema = '${this.config.targetSchema}' -UNION -SELECT table_name FROM information_schema.views WHERE table_schema = '${this.config.targetSchema}' -ORDER BY table_name -;`; - - this.config.foreignKey_sql = -` -SELECT tc.table_name, kcu.column_name, ccu.table_name AS foreign_table_name, ccu.column_name AS foreign_column_name -FROM information_schema.table_constraints AS tc -JOIN information_schema.key_column_usage AS kcu ON tc.constraint_name = kcu.constraint_name AND kcu.constraint_schema = '${this.config.targetSchema}' -JOIN information_schema.constraint_column_usage AS ccu ON ccu.constraint_name = tc.constraint_name AND ccu.constraint_schema = '${this.config.targetSchema}' -WHERE constraint_type = 'FOREIGN KEY' -AND tc.table_schema = '${this.config.targetSchema}' -;`; - - } - - //--------------------------------------------------------------------------------------------------------------------------- - // Entry point - async ExecuteQikTrack() { - - this.tracker_log("--------------------------------------------------------------"); - this.tracker_log(""); - this.tracker_log(" qik-track : Rapid, intuitive Hasura tracking setup"); - this.tracker_log(""); - this.tracker_log(" IN CONTAINER : " + this.config.inContainer ? 'yes' : 'no'); - this.tracker_log(""); - this.tracker_log(" DATABASE : '" + this.config.targetDatabase + "'"); - this.tracker_log(" SCHEMA : '" + this.config.targetSchema + "'"); - this.tracker_log(" HASURA ENDPOINT : '" + this.config.hasuraEndpoint + "'"); - this.tracker_log(" PRIMARY KEY SUFFIX : '" + this.config.primaryKeySuffix + "'"); - this.tracker_log(""); - this.tracker_log("--------------------------------------------------------------"); - this.tracker_log(""); - - // In a docker build the database container needs time to - // startup and initialise, so we wait 10 seconds - const timeout = this.config.inContainer ? 10000 : 1; - - // The tracking operations are staggered - // This may be clunky, but seems to fix a race condition - // where table tracking possiblydoesn't complete before - // relationship tracking setup begins - setTimeout(() => {this.runUntrack()}, timeout); - setTimeout(() => {this.runTrackTables()}, timeout + 2000); - setTimeout(() => {this.runTrackRelationships()}, timeout + 5000); - } - - async runUntrack() { - - if (!this.config.operations.untrack) - return true; - - await this.runSQL_Query(this.config.table_sql) - .then(async (results) => { - - var tables = results - .map(t => t[0]) - .splice(1); -======= - this.config = - { + this.config = { ...cfg, JsonViews:[] - }; + }; + + if (!this.config.primaryKeySuffix) { + this.config.primaryKeySuffix = "_id"; + } this.Logger = new QikTrakLogger(this.config); this.Hasura = new QikTrakHasura(this.config); this.config.Logger = this.Logger; ->>>>>>> 5be04e37e07a4ff127d8bf97465e748a8c061761 // -------------------------------------------------------------------------------------------------------------------------- - // Drop tracking information for all tables / views, this will also untrack any relationships - await this.untrackTables(tables); - }); -} + // SQL to acquire metadata -<<<<<<< HEAD - async runTrackTables(){ - - if (!this.config.operations.trackTables) - return; - - await this.runSQL_Query(this.config.table_sql) - .then(async (results) => { - - var tables = results - .map(t => t[0]) - .splice(1); -======= - this.table_sql = + this.config.table_sql = ` SELECT table_name FROM information_schema.tables WHERE table_schema = '${this.config.targetSchema}' UNION @@ -123,7 +36,7 @@ AND tc.table_schema = '${this.config.targetSchema}' ORDER BY table_name; `; - this.foreignKey_sql = + this.config.foreignKey_sql = ` SELECT tc.table_name, kcu.column_name, ccu.table_name AS foreign_table_name, ccu.column_name AS foreign_column_name FROM information_schema.table_constraints AS tc @@ -137,9 +50,6 @@ AND tc.table_schema = '${this.config.targetSchema}' //--------------------------------------------------------------------------------------------------------------------------- // Entry point async ExecuteQikTrack() { - if (!this.config.primaryKeySuffix) { - this.config.primaryKeySuffix = "_id"; - } this.Logger.Log("--------------------------------------------------------------"); this.Logger.Log(""); @@ -147,125 +57,130 @@ AND tc.table_schema = '${this.config.targetSchema}' this.Logger.Log(""); this.Logger.Log("DATABASE : '" + this.config.targetDatabase + "'"); this.Logger.Log("SCHEMA : '" + this.config.targetSchema + "'"); + this.Logger.Log(""); this.Logger.Log("HASURA ENDPOINT : '" + this.config.hasuraEndpoint + "'"); this.Logger.Log("PRIMARY KEY SUFFIX : '" + this.config.primaryKeySuffix + "'"); this.Logger.Log(""); this.Logger.Log("--------------------------------------------------------------"); this.Logger.Log(""); + this.Logger.Log("Initial 10 second wait whilst database stabilises..."); + this.Logger.Log(""); + // When running in a container we wait 10 seconds so the database container can finish starting, initialising and be stable + const timeout = 10000; // milliseconds - if (this.config.operations.untrack) { + setTimeout(async () => { + await this.RunUntrack(); + }, timeout); - await this.Hasura.runSQL_Query(this.table_sql) - .then(async (results) => { + setTimeout(async () => { + await this.RunPreScripts(); + await this.RunViews(); + await this.RunPostScripts(); + }, timeout+2000); - var tables = results - .map(t => t[0]) - .splice(1); + setTimeout(async () => { + await this.RunTrackTables(); + }, timeout+5000); - // -------------------------------------------------------------------------------------------------------------------------- - // Drop tracking information for all tables / views, this will also untrack any relationships - await this.untrackTables(tables); - }); + setTimeout(async () => { + await this.RunTrackRelationships(); + }, timeout+10000); + } - this.Logger.Log(""); - } + async RunUntrack(){ + if (!this.config.operations.untrack) + return; - if (this.config.operations.executeSqlScripts) { - this.Logger.Log("EXECUTE SQL SCRIPTS BEFORE VIEW BUILDER"); - await this.executeScriptsBeforeViewBuilder(); - this.Logger.Log(""); - } + await this.Hasura.runSQL_Query(this.config.table_sql) + .then(async (results) => { - if (this.config.operations.createJsonViews) { - this.Logger.Log("GENERATE JSON VIEWS"); - await this.createJsonViews(); - this.Logger.Log(""); - } + var tables = results + .map(t => t[0]) + .splice(1); + + // -------------------------------------------------------------------------------------------------------------------------- + // Drop tracking information for all tables / views, this will also untrack any relationships + await this.untrackTables(tables); + }); - if (this.config.operations.executeSqlScripts) { - this.Logger.Log("EXECUTE SQL SCRIPTS AFTER VIEW BUILDER"); - await this.executeScriptsAfterViewBuilder(); this.Logger.Log(""); - } + } - if (this.config.operations.trackTables) { - await this.Hasura.runSQL_Query(this.table_sql) - .then(async (results) => { ->>>>>>> 5be04e37e07a4ff127d8bf97465e748a8c061761 + async RunPreScripts() { + if (!this.config.operations.executeSqlScripts) + return; - // -------------------------------------------------------------------------------------------------------------------------- - // Configure HASURA to track all TABLES and VIEWS - tables and views are added to the GraphQL schema automatically - await this.trackTables(tables); - }); + this.Logger.Log("EXECUTE SQL SCRIPTS BEFORE VIEW BUILDER"); + await this.executeScriptsBeforeViewBuilder(); + this.Logger.Log(""); } - async runTrackRelationships() { - - if (!this.config.operations.trackRelationships) - return; - - // Create the list of relationships required by foreign keys - await this.runSQL_Query(this.config.foreignKey_sql) - .then(async (results) => { - - var foreignKeys = results.splice(1) - .map(fk => { - return { - table1: fk[0], - key1: fk[1], - table2: fk[2], - key2: fk[3], - addArrayRelationship: true, - addObjectRelationship: true - }; - }); - -<<<<<<< HEAD - // -------------------------------------------------------------------------------------------------------------------------- - // Configure HASURA to track all FOREIGN KEY RELATIONSHIPS - enables GraphQL to fetch related (nested) entities - await this.trackRelationships(foreignKeys); - this.tracker_log(""); - }); + async RunViews() { + if (!this.config.operations.createJsonViews) + return; + this.Logger.Log("GENERATE JSON VIEWS"); + await this.createJsonViews(); + this.Logger.Log(""); } - // Setup of relationship naming should be done before execution of the auto tracker - //#region Relationship naming + async RunPostScripts(){ + if (!this.config.operations.executeSqlScripts) + return; + + this.Logger.Log("EXECUTE SQL SCRIPTS AFTER VIEW BUILDER"); + await this.executeScriptsAfterViewBuilder(); + this.Logger.Log(""); + } - //#endregion + async RunTrackTables(){ + if (!this.config.operations.trackTables) + return; + + await this.Hasura.runSQL_Query(this.config.table_sql) + .then(async (results) => { -======= - if (this.config.operations.trackRelationships) { - - // Create the list of relationships required by foreign keys - await this.Hasura.runSQL_Query(this.foreignKey_sql) - .then(async (results) => { - - var foreignKeys = results.splice(1) - .map(fk => { - return { - referencing_table: fk[0], - referencing_key: fk[1], - referenced_table: fk[2], - referenced_key: fk[3] - }; - }); - - // -------------------------------------------------------------------------------------------------------------------------- - // Configure HASURA to track all FOREIGN KEY RELATIONSHIPS - enables GraphQL to fetch related (nested) entities - await this.trackRelationships(foreignKeys); - this.Logger.Log(""); - }); - - // track relationships created by JSON views - this.config.JsonViews.map(async(relationship) =>{ - await this.createRelationships(relationship); + var tables = results + .map(t => t[0]) + .splice(1); + + // -------------------------------------------------------------------------------------------------------------------------- + // Configure HASURA to track all TABLES and VIEWS - tables and views are added to the GraphQL schema automatically + await this.trackTables(tables); + }); + } + + async RunTrackRelationships(){ + + if (!this.config.operations.trackRelationships) + return; + + // Create the list of relationships required by foreign keys + await this.Hasura.runSQL_Query(this.config.foreignKey_sql) + .then(async (results) => { + + var relationships = results.splice(1) + .map(fk => { + return { + referencing_table: fk[0], + referencing_key: fk[1], + referenced_table: fk[2], + referenced_key: fk[3] + }; + }); + + // Add relationships from the Json views + this.config.JsonViews.map((r) => {relationships.push(r)}); + + // -------------------------------------------------------------------------------------------------------------------------- + // Configure HASURA to track all FOREIGN KEY RELATIONSHIPS - enables GraphQL to fetch related (nested) entities + + relationships.map(async (r) => {await this.Hasura.createRelationships(r)}); + this.Logger.Log(""); }); - } } ->>>>>>> 5be04e37e07a4ff127d8bf97465e748a8c061761 //#region Table Tracking // -------------------------------------------------------------------------------------------------------------------------- @@ -275,44 +190,7 @@ AND tc.table_schema = '${this.config.targetSchema}' tables.map(async (table_name) => { this.Logger.Log(" UNTRACK TABLE - " + table_name); - - var query = { - type: "pg_untrack_table", - args: { - table: { - schema: this.config.targetSchema, - name: table_name - }, - source: this.config.targetDatabase, - cascade: true - } - }; - - await this.Hasura.runGraphQL_Query('/v1/metadata', query) - .catch(e => { - if (e.response.data.error.includes("already untracked")) { - return; - } - - this.Logger.Log(""); - this.Logger.Log(""); - this.Logger.Log("--------------------------------------------------------------"); - this.Logger.Log(""); - this.Logger.Log("QIK-TRAK: ERROR"); - this.Logger.Log(""); - this.Logger.Log("GRAPHQL QUERY FAILED TO EXECUTE"); - this.Logger.Log(""); - this.Logger.Log("Error Message : " + e.response.data.internal.error.message); - this.Logger.Log(e.response.request.data); - this.Logger.Log(""); - this.Logger.Log("Query:"); - this.Logger.Log(""); - this.Logger.Log(JSON.stringify(query)); - this.Logger.Log(""); - this.Logger.Log("Are Hasura and the database fully initialised?"); - this.Logger.Log(""); - this.Logger.Log("--------------------------------------------------------------"); - });; + await this.Hasura.UntrackTable(table_name); }); } @@ -338,28 +216,12 @@ AND tc.table_schema = '${this.config.targetSchema}' } }; -<<<<<<< HEAD - await this.runGraphQL_Query('/v1/metadata', query) - .catch(e => { -======= await this.Hasura.runGraphQL_Query('/v1/metadata', query).catch(e => { ->>>>>>> 5be04e37e07a4ff127d8bf97465e748a8c061761 if (e.response.data.error.includes("already tracked")) { return; } -<<<<<<< HEAD - this.tracker_log("GRAPHQL QUERY FAILED TO EXECUTE: "); - this.tracker_log(""); - this.tracker_log(JSON.stringify(query)); - this.tracker_log(""); - this.tracker_log("EXCEPTION DETAILS - creating " + currentRelationshipType + " - " + currentRelationshipName); - this.tracker_log(""); - this.tracker_log(e.response.request.data); - this.tracker_log(""); - }); -======= this.Logger.Log("GRAPHQL QUERY FAILED TO EXECUTE: "); this.Logger.Log(""); this.Logger.Log(JSON.stringify(query)); @@ -369,24 +231,6 @@ AND tc.table_schema = '${this.config.targetSchema}' this.Logger.Log(e.response.request.data); this.Logger.Log(""); });; ->>>>>>> 5be04e37e07a4ff127d8bf97465e748a8c061761 - }); - } - - //#endregion - - - //#region Relationship Tracking - - // -------------------------------------------------------------------------------------------------------------------------- - // Configure HASURA to track all relationships - // This requires an array relationship in one direction and an object relationship in the opposite direction - async trackRelationships(relationships) { - this.Logger.Log(""); - this.Logger.Log("Configure HASURA RELATIONSHIP TRACKING"); - - relationships.map(async (relationship) => { - await this.Hasura.createRelationships(relationship); }); } diff --git a/templates/read.api b/templates/read.api deleted file mode 100644 index 5064ab3..0000000 --- a/templates/read.api +++ /dev/null @@ -1,7 +0,0 @@ -path: {{entity}}/:offset/:limit/:order_by -query: -all_{{entity}}($offset: Int!,$limit: Int!, $order_by: [syndicates_order_by!] = {syndicate_id: asc}) { - {{entity}}(order_by: $order_by, limit: $limit, offset: $offset) { - {{data}} - } -}