master: Some improvements and fixes on query generator and parsing

.vscode/launch.json

@@ -1,6 +1,6 @@
 // A launch configuration that compiles the extension and then opens it inside a new window
 {
-  "version": "0.1.0",
+  "version": "0.2.0",
   "configurations": [
     {
       "name": "Launch Extension",

CHANGELOG.md

@@ -1,8 +1,16 @@
 # Changelog
 
-
 ## v0.17
 
+### v0.17.11
+
+* **Enhancements**
+  * Improved query parser for better handling MSSQL queires
+  * Insert query generator includes column name and type on placeholders
+
+* **Fixes**
+  - Fixed history cutting some query parts on history explorer.
+
 ### v0.17.10
 
 * **Enhancements**

jest.config.js

@@ -1,7 +1,7 @@
 module.exports = {
   ...(require('./test/config/baseConfig')),
   preset: 'ts-jest',
-  collectCoverageFrom: ['**/*.ts', '**/*.tsx'],
+  collectCoverageFrom: ['<rootDir>/packages/**/*.ts', '<rootDir>/packages/**/*.tsx'],
   coverageDirectory: '<rootDir>/coverage',
   coverageThreshold: {
     global: {

package.json

@@ -20,6 +20,7 @@
   "scripts": {
     "clean": "rimraf -rf ../dist",
     "pretest": "rimraf -rf ./coverage",
+    "jest": "jest --config jest.config.js --passWithNoTests",
     "test": "cross-env CODE_DISABLE_EXTENSIONS=1 node ./node_modules/vscode/bin/test",
     "test:watch": "cross-env WATCH=1 yarn run test",
     "precompile": "yarn test && yarn run clean",

packages/core/dialect/mssql/index.ts

@@ -71,7 +71,7 @@ export default class MSSQL extends GenericDialect<MSSQLLib.ConnectionPool> imple
     const request = pool.request();
     request.multiple = true;
     const { recordsets = [], rowsAffected, error } = <IResult<any> & { error: any }>(await request.query(query.replace(/^[ \t]*GO;?[ \t]*$/gmi, '')).catch(error => Promise.resolve({ error, recordsets: [], rowsAffected: [] })));
-    const queries = Utils.query.parse(query, 'mssql', ';');
+    const queries = Utils.query.parse(query, 'mssql');
     return queries.map((q, i): DatabaseInterface.QueryResults => {
       const r = recordsets[i] || [];
       const messages = [];

packages/core/dialect/pgsql/index.ts

@@ -41,7 +41,7 @@ export default class PostgreSQL extends GenericDialect<Pool> implements Connecti
     return this.open()
       .then((conn) => conn.query(query))
       .then((results: any[] | any) => {
-        const queries = Utils.query.parse(query, 'pg', ';');
+        const queries = Utils.query.parse(query, 'pg');
         const messages = [];
         if (!Array.isArray(results)) {
           results = [results];

packages/core/interface/database.ts

@@ -19,7 +19,7 @@ export namespace DatabaseInterface {
     tableSchema?: string;
     tableDatabase?: string;
     tableCatalog?: string;
-    defaultValue: string;
+    defaultValue?: string;
     isNullable: boolean;
     isPk?: boolean;
     isFk?: boolean;

packages/core/utils/query.test.ts

@@ -0,0 +1,84 @@
+import { cleanUp, parse, generateInsert } from './query';
+
+describe('query cleanUp', () => {
+  it('returns empty string for empty inputs', () => {
+    expect(cleanUp('')).toEqual('');
+    expect(cleanUp()).toEqual('');
+    expect(cleanUp(null)).toEqual('');
+    expect(cleanUp(<any>false)).toEqual('');
+  })
+
+  it('removes line breaks from simple select query', () => {
+    const query = `
+select
+  *
+from
+  table
+    `;
+    expect(cleanUp(query)).toEqual('select * from table')
+  });
+
+  it('removes comments from queries', () => {
+    const query = `
+select
+  * -- here is a comment
+  /**
+   * multiline comment
+   */
+from
+  table
+    `;
+    expect(cleanUp(query)).toEqual('select * from table')
+  });
+
+  it('don`t change inline queries', () => {
+    const query = `udpate tablename set value = 2, value2 = 'string' where id = 1`;
+    expect(cleanUp(query)).toEqual(query);
+  });
+});
+
+describe('query parse', () => {
+  it('parses single query string to array', () => {
+    let query = `select
+  * -- here is a comment
+  /**
+   * multiline comment
+   */
+from
+  table`;
+    expect(parse(query)).toEqual([query]);
+    expect(parse(query, 'mysql')).toEqual([query]);
+    expect(parse(query, 'pg')).toEqual([query]);
+    const mssqlQuery = `${query};
+GO`
+    expect(parse(mssqlQuery, 'mssql')).toEqual([`${query};`]);
+  });
+
+  it('parses muliple query string to array of queries', () => {
+    let query = `select
+  *
+from
+  table;`;
+    expect(parse(`${query}\n${query}`)).toEqual([query, query]);
+    expect(parse(`${query}\n${query}`, 'mysql')).toEqual([query, query]);
+    expect(parse(`${query}\n${query}`, 'pg')).toEqual([query, query]);
+    expect(parse(`${query}\n${query}`, 'mssql')).toEqual([query, query]);
+    const mssqlQuery = `${query}
+GO
+
+${query}
+GO`
+    expect(parse(mssqlQuery, 'mssql')).toEqual([query, query]);
+  });
+});
+
+describe('generateInsert query', () => {
+  const generated = generateInsert('tablename', [
+    { type: 'integer', tableName: 'tablename', columnName: 'col1', isNullable: false },
+  ]);
+  const expected = `INSERT INTO
+  tablename (col1)
+VALUES
+  (\${1:col1:integer});$0`;
+  expect(generated).toBe(expected);
+});

packages/core/utils/query.ts

@@ -2,36 +2,57 @@ import { DatabaseInterface, Settings } from '../interface';
 import { format } from '@sqltools/plugins/formatter/utils';
 import multipleQueiesParse from './query/parse';
 
-export function parse(query: string, dialect: 'pg' | 'mysql' | 'mssql' = 'mysql', delimiter: string = ';'): string[] {
-  try {
-    return multipleQueiesParse(query.replace(/^[ \t]*GO;?[ \t]*$/gmi, ''), dialect, delimiter)
-  } catch (error) {
-    return query.split(/\s*;\s*(?=([^']*'[^']*')*[^']*$)/g).filter((v) => !!v && !!`${v}`.trim());
-  }
+/**
+ * Parse multiple queries to an array of queries
+ *
+ * @export
+ * @param {string} query
+ * @param {('pg' | 'mysql' | 'mssql')} [dialect='mysql']
+ * @param {string} [delimiter=';']
+ * @returns {string[]}
+ */
+export function parse(query: string, dialect: 'pg' | 'mysql' | 'mssql' = 'mysql'): string[] {
+  return multipleQueiesParse(query, dialect);
+  // return fixedQuery.split(/\s*;\s*(?=([^']*'[^']*')*[^']*$)/g).filter((v) => !!v && !!`${v}`.trim()).map(v => `${v};`);
 }
+/**
+ * Removes comments and line breaks from query
+ *
+ * @export
+ * @param {string} [query='']
+ * @returns
+ */
+export function cleanUp(query: string = '') {
+  if (!query) return '';
 
-// @todo add some tests for this new function
-export function cleanUp(query = '') {
-  return query.replace('\t', '  ')
-    .replace(/('(''|[^'])*')|(--[^\r\n]*)|(\/\*[\w\W]*?(?=\*\/)\*\/)/gmi, '')
+  return query.toString().replace('\t', '  ')
+    .replace(/(--.*)|(((\/\*)+?[\w\W]+?(\*\/)+))/gmi, '')
     .split(/\r\n|\n/gi)
     .map(v => v.trim())
     .filter(Boolean)
     .join(' ')
     .trim();
 }
 
+/**
+ * Generates insert queries based on table columns
+ *
+ * @export
+ * @param {string} table
+ * @param {Array<DatabaseInterface.TableColumn>} cols
+ * @param {Settings['format']} [formatOptions]
+ * @returns {string}
+ */
 export function generateInsert(
   table: string,
-  cols: Array<{ value: string, column: DatabaseInterface.TableColumn }>,
+  cols: Array<DatabaseInterface.TableColumn>,
   formatOptions?: Settings['format'],
 ): string {
-  // @todo: snippet should have variable name and type
-  let insertQuery = `INSERT INTO ${table} (${cols.map((col) => col.value).join(', ')}) VALUES (`;
+  let insertQuery = `INSERT INTO ${table} (${cols.map((col) => col.columnName).join(', ')}) VALUES (`;
   cols.forEach((col, index) => {
-    insertQuery = insertQuery.concat(`'\${${index + 1}:${col.column.type}}', `);
+    insertQuery = insertQuery.concat(`'\${${index + 1}:${col.columnName}:${col.type}}', `);
   });
   return format(`${insertQuery.substr(0, Math.max(0, insertQuery.length - 2))});`, formatOptions)
-  .replace(/'(\${\d+:(int|bool|num)[\w ]+})'/gi, '$1')
+  .replace(/'\${(\d+):([\w\s]+):((int|bool|num)[\w\s]+)}'/gi, (_, pos, colName, type) => `\${${pos}:${colName.trim()}:${type.trim()}}`)
   .concat('$0');
 }

packages/core/utils/query/parse.ts

@@ -4,14 +4,15 @@
  */
 
 class QueryParser {
-  static parse(query: string, dialect: 'pg' | 'mysql' | 'mssql' = 'mysql', delimiter: string = ';'): Array<string> {
+  static parse(query: string, dialect: 'pg' | 'mysql' | 'mssql' = 'mysql'): Array<string> {
+    const delimiter: string = ';';
     var queries: Array<string> = [];
     var flag = true;
     while (flag) {
       if (restOfQuery == null) {
         restOfQuery = query;
       }
-      var statementAndRest = this.getStatements(restOfQuery, dialect, delimiter);
+      var statementAndRest = QueryParser.getStatements(restOfQuery, dialect, delimiter);
 
       var statement = statementAndRest[0];
       if (statement != null && statement.trim() != '') {
@@ -27,7 +28,7 @@ class QueryParser {
     return queries;
   }
 
-  private static getStatements(query: string, dialect: string, delimiter: string): Array<string> {
+  static getStatements(query: string, dialect: string, delimiter: string): Array<string> {
     var charArray: Array<string> = Array.from(query);
     var previousChar: string = null;
     var nextChar: string = null;
@@ -84,56 +85,53 @@ class QueryParser {
       }
 
       if (char.toLowerCase() == 'd' && isInComment == false && isInString == false) {
-        var delimiterResult = this.getDelimiter(index, query, dialect);
+        var delimiterResult = QueryParser.getDelimiter(index, query, dialect);
         if (delimiterResult != null) {
           // it's delimiter
           var delimiterSymbol: string = delimiterResult[0];
           var delimiterEndIndex: number = delimiterResult[1];
           query = query.substring(delimiterEndIndex);
-          resultQueries = this.getStatements(query, dialect, delimiterSymbol);
+          resultQueries = QueryParser.getStatements(query, dialect, delimiterSymbol);
           break;
         }
       }
 
       if (char == '$' && isInComment == false && isInString == false) {
         var queryUntilTagSymbol = query.substring(index);
         if (isInTag == false) {
-          var tagSymbolResult = this.getTag(queryUntilTagSymbol, dialect);
+          var tagSymbolResult = QueryParser.getTag(queryUntilTagSymbol, dialect);
           if (tagSymbolResult != null) {
             isInTag = true;
             tagChar = tagSymbolResult[0];
           }
         } else {
-          var tagSymbolResult = this.getTag(queryUntilTagSymbol, dialect);
+          var tagSymbolResult = QueryParser.getTag(queryUntilTagSymbol, dialect);
           if (tagSymbolResult != null) {
             var tagSymbol = tagSymbolResult[0];
-            var tagSymbolIndex = tagSymbolResult[1];
             if (tagSymbol == tagChar) {
               isInTag = false;
             }
           }
         }
       }
-
-      if (delimiter.length > 1 && charArray[index + delimiter.length - 1] != undefined) {
-        for (var i = index + 1; i < index + delimiter.length; i++) {
-          char += charArray[i];
-        }
+      if (dialect === 'mssql' && char.toLowerCase() === 'g' && charArray[index + 1] && charArray[index + 1].toLowerCase() === 'o') {
+        char = `${char}${charArray[index + 1]}`;
       }
 
       // it's a query, continue until you get delimiter hit
       if (
-        char.toLowerCase() == delimiter.toLowerCase() &&
+        (char.toLowerCase() === delimiter.toLowerCase() || char.toLowerCase() === 'go') &&
         isInString == false &&
         isInComment == false &&
         isInTag == false
       ) {
-        if (this.isGoDelimiter(dialect, query, index) == false) {
-          continue;
+        var splittingIndex = index + 1;
+        if (dialect === 'mssql' && char.toLowerCase() === 'go') {
+          splittingIndex = index;
+          resultQueries = QueryParser.getQueryParts(query, splittingIndex, 2);
+          break;
         }
-        var splittingIndex = index;
-        // if (delimiter == ";") {     splittingIndex = index + 1 }
-        resultQueries = this.getQueryParts(query, splittingIndex, delimiter);
+        resultQueries = QueryParser.getQueryParts(query, splittingIndex);
         break;
       }
     }
@@ -147,9 +145,9 @@ class QueryParser {
     return resultQueries;
   }
 
-  private static getQueryParts(query: string, splittingIndex: number, delimiter: string): Array<string> {
+  static getQueryParts(query: string, splittingIndex: number, numChars: number = 1): Array<string> {
     var statement: string = query.substring(0, splittingIndex);
-    var restOfQuery: string = query.substring(splittingIndex + delimiter.length);
+    var restOfQuery: string = query.substring(splittingIndex + numChars);
     var result: Array<string> = [];
     if (statement != null) {
       statement = statement.trim();
@@ -159,7 +157,7 @@ class QueryParser {
     return result;
   }
 
-  private static getDelimiter(index: number, query: string, dialect: string): Array<any> {
+  static getDelimiter(index: number, query: string, dialect: string): Array<any> {
     if (dialect == 'mysql') {
       var delimiterKeyword = 'delimiter ';
       var delimiterLength = delimiterKeyword.length;
@@ -174,7 +172,7 @@ class QueryParser {
         parsedQueryAfterIndex = parsedQueryAfterIndex.substring(0, indexOfNewLine);
         parsedQueryAfterIndex = parsedQueryAfterIndex.substring(delimiterLength);
         var delimiterSymbol = parsedQueryAfterIndex.trim();
-        delimiterSymbol = this.clearTextUntilComment(delimiterSymbol, dialect);
+        delimiterSymbol = QueryParser.clearTextUntilComment(delimiterSymbol);
         if (delimiterSymbol != null) {
           delimiterSymbol = delimiterSymbol.trim();
           var delimiterSymbolEndIndex =
@@ -192,7 +190,7 @@ class QueryParser {
     }
   }
 
-  private static getTag(query: string, dialect: string): Array<any> {
+  static getTag(query: string, dialect: string): Array<any> {
     if (dialect == 'pg') {
       var matchTag = query.match(/^(\$[a-zA-Z]*\$)/i);
       if (matchTag != null && matchTag.length > 1) {
@@ -208,7 +206,7 @@ class QueryParser {
     }
   }
 
-  private static isGoDelimiter(dialect: string, query: string, index: number): boolean {
+  static isGoDelimiter(dialect: string, query: string, index: number): boolean {
     if (dialect == 'mssql') {
       var match = /(?:\bgo\b\s*)/i.exec(query);
       if (match != null && match.index == index) {
@@ -219,16 +217,16 @@ class QueryParser {
     }
   }
 
-  private static clearTextUntilComment(text: string, dialect: string): string {
-    var previousChar: string = null;
+  static clearTextUntilComment(text: string): string {
+    // var previousChar: string = null;
     var nextChar: string = null;
     var charArray: Array<string> = Array.from(text);
     var clearedText: string = null;
     for (var index = 0; index < charArray.length; index++) {
       var char = charArray[index];
-      if (index > 0) {
-        previousChar = charArray[index - 1];
-      }
+      // if (index > 0) {
+      //   previousChar = charArray[index - 1];
+      // }
 
       if (index < charArray.length) {
         nextChar = charArray[index + 1];