From 4d6ecff898f860f92f753d144b8546dcf60c3e7d Mon Sep 17 00:00:00 2001 From: Elizabeth Phillips Date: Wed, 18 Mar 2020 16:40:20 +0000 Subject: [PATCH 1/7] finish tidy up --- app/routes.js | 273 +------------------------------------------ convertJSONScript.js | 42 +++++++ lib/constants.js | 42 +++++++ lib/createData.js | 227 +++++++++++++++++++++++++++++++++++ 4 files changed, 315 insertions(+), 269 deletions(-) create mode 100644 convertJSONScript.js create mode 100644 lib/constants.js create mode 100644 lib/createData.js diff --git a/app/routes.js b/app/routes.js index e864122..ad2d370 100644 --- a/app/routes.js +++ b/app/routes.js @@ -1,54 +1,11 @@ const express = require('express') const router = express.Router() - +const createData = require('../lib/createData'); // Add your routes here - above the module.exports line -var devices = [ - "Air Compressors / Pumps", - "Bellows", - "Self-inflating bags", - "Gas mixing valves", - "Pressure Regulators", - "Flow Control valves", - "Solenoid valves", - "Pressure relief valves", - "Check valves / one-way valves", - "Industrial Automation components (Safety Relays, PLCs)", - "Power Supplies", - "Electric Motors, and motor controllers", - "Linear actuators and controllers", - "Tubing and fittings", - "Pressure Sensors and Indicators", - "Oxygen Sensors and Indicators", - "Flow Sensors and Indicators", - "Manometers", - "Heat and moisture exchanging filters (HMEFs)", - "Air Filter, HEPA Filters" -]; - -var expertise = [ - "Design / specification", - "Rapid prototyping", - "Manufacturing (manual)", - "Manufacturing (automated)", - "Machine Shops/sheet metal/tool manufacture", - "Pneumatic part manufacturers/suppliers", - "Contract/Product Assembly", - "Certification/regulation/testing", - "Logistics", - "Medical Training" -]; - -var resources = [ - "Suitable space", - "Equipment", - "Trained personnel", - "Other", -]; +var {devices, expertise, resources} = require('../lib/constants'); const { Client } = require('pg'); - - // landing page router.get("/", function (req, res, next) { res.render("index", { @@ -65,229 +22,7 @@ router.get('/error', function(req, res, next) { router.get("/submit", function (req, res, next) { var data = req.session.data; - - // pullout specific BASIC vars - var companyName = data['organisation-name'] || ""; - var companyNumber = data['company-number'] || ""; - var contact = data['primary-contact'] || ""; - var role = data['primary-contact-role'] || ""; - var phone = data['phone'] || ""; - var email = data['email'] || ""; - - // SUPPLY CHAIN - var isClinical = data['is-clinical'] || "no"; - var isHumanUse= data['human-use'] || "no"; - var isVetUse= data['vet-use'] || "no"; - var isOtherUse= data['other-use'] || "no"; - // freetext - var ventilatorText = data['ventilator-detail'] || ""; - //console.log(isClinical, isHumanUse, isVetUse, isOtherUse, ventilatorText); - - // MEDICAL DEVICES - var med_devices = {}; - // loop thru design - if (data['design']) { - let len = data['design'].length; - for (var i = 0; i < len; i++) { - //convert name to string - var name = data['design'][i].split(" /")[0]; - name = name.split("(")[0]; - name = name.split(",")[0]; - name = name.split(" and")[0]; - name = name.split("-").join("_"); - name = name.split(" ").join("_") + "_design"; - name = name.split("__").join("_").toLowerCase(); - med_devices[name] = "yes"; - } - } - if (data['manufacture']) { - let len = data['manufacture'].length; - for (var i = 0; i < len; i++) { - //convert name to string - var name = data['manufacture'][i].split(" /")[0]; - name = name.split("(")[0]; - name = name.split(",")[0]; - name = name.split(" and")[0]; - name = name.split("-").join("_"); - name = name.split(" ").join("_") + "_manufacture"; - name = name.split("__").join("_").toLowerCase(); - med_devices[name] = "yes"; - } - } - if (data['supply']) { - let len = data['supply'].length; - for (var i = 0; i < len; i++) { - //convert name to string - var name = data['supply'][i].split(" /")[0]; - name = name.split("(")[0]; - name = name.split(",")[0]; - name = name.split(" and")[0]; - name = name.split("-").join("_"); - name = name.split(" ").join("_") + "_supply"; - name = name.split("__").join("_").toLowerCase(); - med_devices[name] = "yes"; - } - } - // get locations - for (var i = 1; i < 21; i++) { - var ref = 0; - if (data['location-' + i] !== "") { - var name = devices[i - 1].split("/")[0]; - name = name.split("(")[0]; - name = name.split(",")[0]; - name = name.split(" and")[0]; - name = name.split("-").join("_"); - name = name.split(" ").join("_") + "_location"; - name = name.split("__").join("_").toLowerCase(); - ref = parseInt(data['location-' + i]); - med_devices[name] = ref; - } - - } - //console.log(med_devices); - - - // Q5 - // freetext - var offerText = data['offer'] || ""; - - // SKILLS and SPECIALISM - var cats = {}; - // loop thru skillz - if (data['skills']) { - let len = data['skills'].length; - for (var i = 0; i < len; i++) { - //convert name to string - var name = data['skills'][i].split("/")[0]; - name = name.split("(").join(""); - name = name.split(")").join(""); - name = name.split(" ").join("_") + "_skills"; - name = name.split("__").join("_").toLowerCase(); - cats[name] = "yes"; - } - } - //var specialism = []; - if (data['specialism']) { - let len = data['specialism'].length; - for (var i = 0; i < len; i++) { - //convert name to string - var name = data['specialism'][i].split("/")[0]; - name = name.split("(").join(""); - name = name.split(")").join(""); - name = name.split(" ").join("_") + "_specialism"; - name = name.split("__").join("_").toLowerCase(); - cats[name] = "yes"; - } - } - // get locations - for (var i = 1; i < 11; i++) { - var ref = 0; - if (data['specialism-location-' + i] !== "") { - var name = expertise[i - 1].split("/")[0]; - name = name.split("(").join(""); - name = name.split(")").join(""); - name = name.split(" ").join("_") + "_location"; - name = name.split("__").join("_").toLowerCase(); - ref = parseInt(data['specialism-location-' + i]); - cats[name] = ref; - } - - } - //console.log(cats); - - - // Q7 - var resources = []; - var resources_space = ""; - var resources_equipment = ""; - var resources_personnel = ""; - var resources_other = ""; - if (data['resources']) { - console.log(data['resources']); - let len = data['resources'].length; - for (var i = 0; i < len; i++) { - console.log( data['resources'][i]); - if(data['resources'][i]==="Suitable space"){ - resources_space="yes"; - } - if(data['resources'][i]==="Equipment"){ - resources_equipment="yes"; - } - if(data['resources'][i]==="Trained personnel"){ - resources_personnel="yes"; - } - if(data['resources'][i]==="Other"){ - resources_other="yes"; - } - - } - } - // freetext - var resourcesText = data['resources-detail'] || ""; - //console.log(resources, resourcesText); - - var deviceFields = ""; - var devicesResults = ""; - for (item in med_devices) { - deviceFields += item + ", "; - } - for (item in med_devices) { - devicesResults += "'" + med_devices[item] + "', "; - } - - var json = JSON.stringify(req.session.data); - var sql_values = { - "info" : json, - "company_name" : companyName, - "company_number" : companyNumber, - "contact_name" : contact, - "contact_role" : role, - "contact_phone" : phone, - "contact_email" : email, - "ventilator_production" : isClinical, - "ventilator_parts_human" : isHumanUse, - "ventilator_parts_veterinary" : isVetUse, - "ventilator_parts_any" : isOtherUse, - "ventilator_parts_details" : ventilatorText, - "offer_organisation" : offerText, - "resources_space" : resources_space, - "resources_equipment" : resources_equipment, - "resources_personnel" : resources_personnel, - "resources_other" : resources_other, - "resource_details" : resourcesText - } - - // Add in fields from med_devices - var med_keys = Object.keys(med_devices) - for (var i = 0; i < med_keys.length; i++) { - key = med_keys[i] - sql_values[key] = med_devices[key] - } - - // Add in fields from cats - var cat_keys = Object.keys(cats) - for (var i = 0; i < cat_keys.length; i++) { - key = cat_keys[i] - sql_values[key] = cats[key] - } - - // Quick check - console.log("SQL values: " + sql_values) - - // Build arrays of entries for the SQL query - var fieldNames = []; - var valuePositions = []; - var values = []; - var sql_keys = Object.keys(sql_values) - for (var i = 0; i < sql_keys.length; i++) { - key = sql_keys[i] - fieldNames.push(key) - valuePositions.push("$"+(i+1)) - values.push(sql_values[key]) - } - - var fields = fieldNames.join(", ") - var positions = valuePositions.join(", ") + const {fields, positions,json,values} = createData(data); var sql = "INSERT INTO companies(" + fields + ") VALUES (" + positions + ");" // var catFields = ""; @@ -390,7 +125,7 @@ router.get("/submit", function (req, res, next) { connectionString: process.env.HEROKU_POSTGRESQL_RED_URL || process.env.DATABASE_URL, ssl: true, }); - + client.connect(); client.query(query, (err, res) => { diff --git a/convertJSONScript.js b/convertJSONScript.js new file mode 100644 index 0000000..c3ccbbb --- /dev/null +++ b/convertJSONScript.js @@ -0,0 +1,42 @@ +const { Client } = require('pg'); +const createData = require('./lib/createData'); +const parse = async (id,parsed)=> { + try { + let SQL = ''; + const { fieldNames, values } = createData(parsed); + for (let i = 0; i < fieldNames.length; i++) { + SQL = SQL + `UPDATE companies SET ${fieldNames[i]}='${values[i]}' WHERE id=${id};`; + } + const client = new Client({ + connectionString: process.env.HEROKU_POSTGRESQL_RED_URL || process.env.DATABASE_URL, + ssl: true, + }); + await client.query(SQL); + client.end(); + } + catch(err){ + console.log(err.toString()) + } +} +const getRows = async ()=>{ + // + try { + const client = new Client({ + connectionString: process.env.HEROKU_POSTGRESQL_RED_URL || process.env.DATABASE_URL, + ssl: true, + }); + + client.connect(); + const SQL = 'SELECT * from companies;' + let rows = await client.query(SQL); + console.log('Converting ',rows.rows.length,' rows'); + await Promise.all(rows.rows.slice(1,2).map(async (row)=>{ + parse(row.id,JSON.stringify(row.info),client) + })); + client.end(); + } + catch(err){ + console.log(err.toString()) + } +} +getRows(); diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000..d9ae4e2 --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,42 @@ +module.exports.devices = [ + "Air Compressors / Pumps", + "Bellows", + "Self-inflating bags", + "Gas mixing valves", + "Pressure Regulators", + "Flow Control valves", + "Solenoid valves", + "Pressure relief valves", + "Check valves / one-way valves", + "Industrial Automation components (Safety Relays, PLCs)", + "Power Supplies", + "Electric Motors, and motor controllers", + "Linear actuators and controllers", + "Tubing and fittings", + "Pressure Sensors and Indicators", + "Oxygen Sensors and Indicators", + "Flow Sensors and Indicators", + "Manometers", + "Heat and moisture exchanging filters (HMEFs)", + "Air Filter, HEPA Filters" +]; + +module.exports.expertise = [ + "Design / specification", + "Rapid prototyping", + "Manufacturing (manual)", + "Manufacturing (automated)", + "Machine Shops/sheet metal/tool manufacture", + "Pneumatic part manufacturers/suppliers", + "Contract/Product Assembly", + "Certification/regulation/testing", + "Logistics", + "Medical Training" +]; + +module.exports.resources = [ + "Suitable space", + "Equipment", + "Trained personnel", + "Other", +]; diff --git a/lib/createData.js b/lib/createData.js new file mode 100644 index 0000000..0a59e23 --- /dev/null +++ b/lib/createData.js @@ -0,0 +1,227 @@ +module.exports = (data)=>{ + var {devices, expertise} = require('../lib/constants'); + + // pullout specific BASIC vars + var companyName = data['organisation-name'] || ""; + var companyNumber = data['company-number'] || ""; + var contact = data['primary-contact'] || ""; + var role = data['primary-contact-role'] || ""; + var phone = data['phone'] || ""; + var email = data['email'] || ""; + + // SUPPLY CHAIN + var isClinical = data['is-clinical'] || "no"; + var isHumanUse= data['human-use'] || "no"; + var isVetUse= data['vet-use'] || "no"; + var isOtherUse= data['other-use'] || "no"; + // freetext + var ventilatorText = data['ventilator-detail'] || ""; + //console.log(isClinical, isHumanUse, isVetUse, isOtherUse, ventilatorText); + + // MEDICAL DEVICES + var med_devices = {}; + // loop thru design + if (data['design']) { + let len = data['design'].length; + for (var i = 0; i < len; i++) { + //convert name to string + var name = data['design'][i].split(" /")[0]; + name = name.split("(")[0]; + name = name.split(",")[0]; + name = name.split(" and")[0]; + name = name.split("-").join("_"); + name = name.split(" ").join("_") + "_design"; + name = name.split("__").join("_").toLowerCase(); + med_devices[name] = "yes"; + } + } + if (data['manufacture']) { + let len = data['manufacture'].length; + for (var i = 0; i < len; i++) { + //convert name to string + var name = data['manufacture'][i].split(" /")[0]; + name = name.split("(")[0]; + name = name.split(",")[0]; + name = name.split(" and")[0]; + name = name.split("-").join("_"); + name = name.split(" ").join("_") + "_manufacture"; + name = name.split("__").join("_").toLowerCase(); + med_devices[name] = "yes"; + } + } + if (data['supply']) { + let len = data['supply'].length; + for (var i = 0; i < len; i++) { + //convert name to string + var name = data['supply'][i].split(" /")[0]; + name = name.split("(")[0]; + name = name.split(",")[0]; + name = name.split(" and")[0]; + name = name.split("-").join("_"); + name = name.split(" ").join("_") + "_supply"; + name = name.split("__").join("_").toLowerCase(); + med_devices[name] = "yes"; + } + } + // get locations + for (var i = 1; i < 21; i++) { + var ref = 0; + if (data['location-' + i] !== "") { + var name = devices[i - 1].split("/")[0]; + name = name.split("(")[0]; + name = name.split(",")[0]; + name = name.split(" and")[0]; + name = name.split("-").join("_"); + name = name.split(" ").join("_") + "_location"; + name = name.split("__").join("_").toLowerCase(); + ref = parseInt(data['location-' + i]); + med_devices[name] = ref; + } + + } + //console.log(med_devices); + + + // Q5 + // freetext + var offerText = data['offer'] || ""; + + // SKILLS and SPECIALISM + var cats = {}; + // loop thru skillz + if (data['skills']) { + let len = data['skills'].length; + for (var i = 0; i < len; i++) { + //convert name to string + var name = data['skills'][i].split("/")[0]; + name = name.split("(").join(""); + name = name.split(")").join(""); + name = name.split(" ").join("_") + "_skills"; + name = name.split("__").join("_").toLowerCase(); + cats[name] = "yes"; + } + } + //var specialism = []; + if (data['specialism']) { + let len = data['specialism'].length; + for (var i = 0; i < len; i++) { + //convert name to string + var name = data['specialism'][i].split("/")[0]; + name = name.split("(").join(""); + name = name.split(")").join(""); + name = name.split(" ").join("_") + "_specialism"; + name = name.split("__").join("_").toLowerCase(); + cats[name] = "yes"; + } + } + // get locations + for (var i = 1; i < 11; i++) { + var ref = 0; + if (data['specialism-location-' + i] !== "") { + var name = expertise[i - 1].split("/")[0]; + name = name.split("(").join(""); + name = name.split(")").join(""); + name = name.split(" ").join("_") + "_location"; + name = name.split("__").join("_").toLowerCase(); + ref = parseInt(data['specialism-location-' + i]); + cats[name] = ref; + } + + } + //console.log(cats); + + + // Q7 + var resources = []; + var resources_space = ""; + var resources_equipment = ""; + var resources_personnel = ""; + var resources_other = ""; + if (data['resources']) { + console.log(data['resources']); + let len = data['resources'].length; + for (var i = 0; i < len; i++) { + console.log( data['resources'][i]); + if(data['resources'][i]==="Suitable space"){ + resources_space="yes"; + } + if(data['resources'][i]==="Equipment"){ + resources_equipment="yes"; + } + if(data['resources'][i]==="Trained personnel"){ + resources_personnel="yes"; + } + if(data['resources'][i]==="Other"){ + resources_other="yes"; + } + + } + } + // freetext + var resourcesText = data['resources-detail'] || ""; + //console.log(resources, resourcesText); + + var deviceFields = ""; + var devicesResults = ""; + for (item in med_devices) { + deviceFields += item + ", "; + } + for (item in med_devices) { + devicesResults += "'" + med_devices[item] + "', "; + } + + var json = JSON.stringify(data); + var sql_values = { + "info" : json, + "company_name" : 'asd', + "company_number" : companyNumber, + "contact_name" : contact, + "contact_role" : role, + "contact_phone" : phone, + "contact_email" : email, + "ventilator_production" : isClinical, + "ventilator_parts_human" : isHumanUse, + "ventilator_parts_veterinary" : isVetUse, + "ventilator_parts_any" : isOtherUse, + "ventilator_parts_details" : ventilatorText, + "offer_organisation" : offerText, + "resources_space" : resources_space, + "resources_equipment" : resources_equipment, + "resources_personnel" : resources_personnel, + "resources_other" : resources_other, + "resource_details" : resourcesText + } + + // Add in fields from med_devices + var med_keys = Object.keys(med_devices) + for (var i = 0; i < med_keys.length; i++) { + key = med_keys[i] + sql_values[key] = med_devices[key] + } + + // Add in fields from cats + var cat_keys = Object.keys(cats) + for (var i = 0; i < cat_keys.length; i++) { + key = cat_keys[i] + sql_values[key] = cats[key] + } + + // Quick check + console.log("SQL values: " + sql_values) + + // Build arrays of entries for the SQL query + var fieldNames = []; + var valuePositions = []; + var values = []; + var sql_keys = Object.keys(sql_values) + for (var i = 0; i < sql_keys.length; i++) { + key = sql_keys[i] + fieldNames.push(key) + valuePositions.push("$"+(i+1)) + values.push(sql_values[key]) + } + + var fields = fieldNames.join(", ") + var positions = valuePositions.join(", "); + return {fields,positions,values,fieldNames,json}; +} \ No newline at end of file From f409ea221c81bf115bef195ca4fe16700c20693b Mon Sep 17 00:00:00 2001 From: Elizabeth Phillips Date: Wed, 18 Mar 2020 17:03:51 +0000 Subject: [PATCH 2/7] fix --- lib/createData.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/createData.js b/lib/createData.js index 0a59e23..079b639 100644 --- a/lib/createData.js +++ b/lib/createData.js @@ -173,7 +173,7 @@ module.exports = (data)=>{ var json = JSON.stringify(data); var sql_values = { "info" : json, - "company_name" : 'asd', + "company_name" : companyName, "company_number" : companyNumber, "contact_name" : contact, "contact_role" : role, From 02da173b589a7eaf4c71fba7f96025caa81b7d4d Mon Sep 17 00:00:00 2001 From: Elizabeth Phillips Date: Thu, 19 Mar 2020 08:47:27 +0000 Subject: [PATCH 3/7] tidy up and handle bad entries in DB --- convertJSONScript.js | 37 ++++++++++++++++++++++--------------- lib/createData.js | 6 ++++-- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/convertJSONScript.js b/convertJSONScript.js index c3ccbbb..80003ad 100644 --- a/convertJSONScript.js +++ b/convertJSONScript.js @@ -1,25 +1,24 @@ const { Client } = require('pg'); const createData = require('./lib/createData'); -const parse = async (id,parsed)=> { + +const parse = async (id,parsed,client)=> { try { let SQL = ''; - const { fieldNames, values } = createData(parsed); - for (let i = 0; i < fieldNames.length; i++) { - SQL = SQL + `UPDATE companies SET ${fieldNames[i]}='${values[i]}' WHERE id=${id};`; - } - const client = new Client({ - connectionString: process.env.HEROKU_POSTGRESQL_RED_URL || process.env.DATABASE_URL, - ssl: true, + const { sql_values } = createData(JSON.parse(parsed)); + Object.keys(sql_values).map((key)=> { + if (key === 'info'|| key.includes('unchecked') || key.includes('power_supplies _')|| key.includes('m_specialism')) { + } + else { + SQL = SQL + `UPDATE companies SET ${key}='${fix_quote(sql_values[key])}' WHERE id=${id} ;`; + } }); await client.query(SQL); - client.end(); } catch(err){ - console.log(err.toString()) + console.log('Error parsing JSON into columns;', err.toString()); } } const getRows = async ()=>{ - // try { const client = new Client({ connectionString: process.env.HEROKU_POSTGRESQL_RED_URL || process.env.DATABASE_URL, @@ -29,14 +28,22 @@ const getRows = async ()=>{ client.connect(); const SQL = 'SELECT * from companies;' let rows = await client.query(SQL); - console.log('Converting ',rows.rows.length,' rows'); - await Promise.all(rows.rows.slice(1,2).map(async (row)=>{ - parse(row.id,JSON.stringify(row.info),client) + console.log('Converting ',rows.rows.length,' rows',process.env.DATABASE_URL); + await Promise.all(rows.rows.map(async (row)=>{ + await parse(row.id,JSON.stringify(row.info),client); })); client.end(); } catch(err){ - console.log(err.toString()) + console.log('Error retrieving rows from database;', err.toString()); + } +} +const fix_quote = (string) => { + if (typeof string === 'string'){ + return string.replace("'" , "''"); + } + else { + return string; } } getRows(); diff --git a/lib/createData.js b/lib/createData.js index 079b639..9abf9cf 100644 --- a/lib/createData.js +++ b/lib/createData.js @@ -2,7 +2,9 @@ module.exports = (data)=>{ var {devices, expertise} = require('../lib/constants'); // pullout specific BASIC vars + // console.log(data); var companyName = data['organisation-name'] || ""; + // console.log('cococo',companyName); var companyNumber = data['company-number'] || ""; var contact = data['primary-contact'] || ""; var role = data['primary-contact-role'] || ""; @@ -207,7 +209,7 @@ module.exports = (data)=>{ } // Quick check - console.log("SQL values: " + sql_values) + // console.log("SQL values: ",sql_values) // Build arrays of entries for the SQL query var fieldNames = []; @@ -223,5 +225,5 @@ module.exports = (data)=>{ var fields = fieldNames.join(", ") var positions = valuePositions.join(", "); - return {fields,positions,values,fieldNames,json}; + return {fields,positions,values,json,sql_values}; } \ No newline at end of file From 5ccc4895b72cb7224d327d1135431649a732fb65 Mon Sep 17 00:00:00 2001 From: Elizabeth Phillips Date: Thu, 19 Mar 2020 10:12:17 +0000 Subject: [PATCH 4/7] fix comments --- app/routes.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/routes.js b/app/routes.js index baa19ce..9977e92 100644 --- a/app/routes.js +++ b/app/routes.js @@ -21,7 +21,8 @@ router.get('/error', function(req, res, next) { router.post("/submit", function (req, res, next) { try{ - var data = req.session.data; + var data = req.body; + console.log(req.body) const { fields, positions, json, values } = createData(data); var sql = "INSERT INTO companies(" + fields + ") VALUES (" + positions + ");" @@ -49,7 +50,7 @@ router.post("/submit", function (req, res, next) { client.connect(); } - catch(errr){ + catch(err){ throw new err('Failed to connect to database') } console.log("connect"); From 61f3c0668dc06c52bb5d8a4cfe7d7971cfde2af6 Mon Sep 17 00:00:00 2001 From: Elizabeth Phillips Date: Thu, 19 Mar 2020 11:32:50 +0000 Subject: [PATCH 5/7] update so we have positional SQL| --- convertJSONScript.js | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/convertJSONScript.js b/convertJSONScript.js index 80003ad..cebfd4c 100644 --- a/convertJSONScript.js +++ b/convertJSONScript.js @@ -3,16 +3,21 @@ const createData = require('./lib/createData'); const parse = async (id,parsed,client)=> { try { - let SQL = ''; - const { sql_values } = createData(JSON.parse(parsed)); - Object.keys(sql_values).map((key)=> { - if (key === 'info'|| key.includes('unchecked') || key.includes('power_supplies _')|| key.includes('m_specialism')) { + // let SQL = ''; + const { fields,sql_values,values} = createData(JSON.parse(parsed)); + const fields_array = fields.split(', '); + await Promise.all(Object.keys(sql_values).map(async (key, index)=> { + if ( key.includes('unchecked') || key.includes('power_supplies _')|| key.includes('m_specialism')) { } else { - SQL = SQL + `UPDATE companies SET ${key}='${fix_quote(sql_values[key])}' WHERE id=${id} ;`; + const SQL = `UPDATE companies SET ${fields_array[index]}=$1 WHERE id=${id};`; + const query = { + text: SQL, + values: values.slice(index,index+1), + } + await client.query(query); } - }); - await client.query(SQL); + })); } catch(err){ console.log('Error parsing JSON into columns;', err.toString()); @@ -22,7 +27,7 @@ const getRows = async ()=>{ try { const client = new Client({ connectionString: process.env.HEROKU_POSTGRESQL_RED_URL || process.env.DATABASE_URL, - ssl: true, + ssl: false, }); client.connect(); @@ -38,12 +43,4 @@ const getRows = async ()=>{ console.log('Error retrieving rows from database;', err.toString()); } } -const fix_quote = (string) => { - if (typeof string === 'string'){ - return string.replace("'" , "''"); - } - else { - return string; - } -} getRows(); From c3222057eff158e518c52d529f6c86825bb66bed Mon Sep 17 00:00:00 2001 From: Elizabeth Phillips Date: Thu, 19 Mar 2020 16:26:03 +0000 Subject: [PATCH 6/7] fix to only parse non-empty json and to skip over entries with company name already --- convertJSONScript.js | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/convertJSONScript.js b/convertJSONScript.js index cebfd4c..e068fa5 100644 --- a/convertJSONScript.js +++ b/convertJSONScript.js @@ -1,23 +1,31 @@ const { Client } = require('pg'); const createData = require('./lib/createData'); -const parse = async (id,parsed,client)=> { +const parse = async (id,parsed,client,company_name)=> { try { // let SQL = ''; - const { fields,sql_values,values} = createData(JSON.parse(parsed)); - const fields_array = fields.split(', '); - await Promise.all(Object.keys(sql_values).map(async (key, index)=> { - if ( key.includes('unchecked') || key.includes('power_supplies _')|| key.includes('m_specialism')) { + + if(parsed.length>2) { + let json = JSON.parse(parsed); + const { fields, sql_values, values } = createData(json); + if(!company_name) { + const fields_array = fields.split(', '); + await Promise.all(Object.keys(sql_values).map(async (key, index) => { + if (key.includes('unchecked') || key.includes('power_supplies _') || key.includes('m_specialism')) { + } else { + const SQL = `UPDATE companies SET ${fields_array[index]}=$1 WHERE id=${id};`; + const query = { + text: SQL, + values: values.slice(index, index + 1), + } + await client.query(query); + } + })); } else { - const SQL = `UPDATE companies SET ${fields_array[index]}=$1 WHERE id=${id};`; - const query = { - text: SQL, - values: values.slice(index,index+1), - } - await client.query(query); + console.log('skipping row for id', id); } - })); + } } catch(err){ console.log('Error parsing JSON into columns;', err.toString()); @@ -27,7 +35,7 @@ const getRows = async ()=>{ try { const client = new Client({ connectionString: process.env.HEROKU_POSTGRESQL_RED_URL || process.env.DATABASE_URL, - ssl: false, + ssl: true, }); client.connect(); @@ -35,7 +43,7 @@ const getRows = async ()=>{ let rows = await client.query(SQL); console.log('Converting ',rows.rows.length,' rows',process.env.DATABASE_URL); await Promise.all(rows.rows.map(async (row)=>{ - await parse(row.id,JSON.stringify(row.info),client); + await parse(row.id,JSON.stringify(row.info),client,row.company_name); })); client.end(); } From 397e31296abb63c0d4b24424ffe190dbe2d406bc Mon Sep 17 00:00:00 2001 From: Elizabeth Phillips Date: Thu, 19 Mar 2020 16:57:58 +0000 Subject: [PATCH 7/7] restrict to valid fields --- convertJSONScript.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/convertJSONScript.js b/convertJSONScript.js index e068fa5..ae9200a 100644 --- a/convertJSONScript.js +++ b/convertJSONScript.js @@ -3,8 +3,6 @@ const createData = require('./lib/createData'); const parse = async (id,parsed,client,company_name)=> { try { - // let SQL = ''; - if(parsed.length>2) { let json = JSON.parse(parsed); const { fields, sql_values, values } = createData(json);