From 3b128b1b37ae3d99d53a6b2d204c46f0dbd45fe4 Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Fri, 26 Aug 2022 10:02:15 -0400 Subject: [PATCH 1/4] Fix identifying transaction statements --- src/parser.ts | 21 +++++++++++--- test/identifier/multiple-statement.spec.ts | 32 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 1b42f93..97c7532 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -12,7 +12,7 @@ import type { } from './defines'; interface StatementParser { - addToken: (token: Token) => void; + addToken: (token: Token, nextToken: Token) => void; getStatement: () => Statement; } @@ -81,6 +81,15 @@ function createInitialStatement(): Statement { }; } +function nextNonWhitespaceToken(state: State): Token { + let token: Token; + do { + state = initState({ prevState: state }); + token = scanToken(state); + } while (token.type === 'whitespace'); + return token; +} + /** * Parser */ @@ -115,6 +124,7 @@ export function parse(input: string, isStrict = true, dialect: Dialect = 'generi while (prevState.position < topLevelState.end) { const tokenState = initState({ prevState }); const token = scanToken(tokenState, dialect); + const nextToken = nextNonWhitespaceToken(tokenState); if (!statementParser) { // ignore blank tokens before the start of a CTE / not part of a statement @@ -193,7 +203,7 @@ export function parse(input: string, isStrict = true, dialect: Dialect = 'generi } } - statementParser.addToken(token); + statementParser.addToken(token, nextToken); topLevelStatement.tokens.push(token); prevState = tokenState; @@ -618,7 +628,7 @@ function stateMachineStatementParser( return statement; }, - addToken(token: Token) { + addToken(token: Token, nextToken: Token) { /* eslint no-param-reassign: 0 */ if (statement.endStatement) { throw new Error('This statement has already got to the end.'); @@ -650,7 +660,10 @@ function stateMachineStatementParser( if ( token.type === 'keyword' && blockOpeners[dialect].includes(token.value) && - prevPrevToken?.value.toUpperCase() !== 'END' + prevNonWhitespaceToken?.value.toUpperCase() !== 'END' && + (token.value.toUpperCase() !== 'BEGIN' || + (token.value.toUpperCase() === 'BEGIN' && + nextToken.value.toUpperCase() !== 'TRANSACTION')) ) { if ( ['oracle', 'bigquery'].includes(dialect) && diff --git a/test/identifier/multiple-statement.spec.ts b/test/identifier/multiple-statement.spec.ts index 0dbca16..c01cada 100644 --- a/test/identifier/multiple-statement.spec.ts +++ b/test/identifier/multiple-statement.spec.ts @@ -363,4 +363,36 @@ describe('identifier', () => { expect(sql.substring(actual[1].start, actual[1].end + 1)).to.eql(statement2); }); }); + + it('should identify transactions', () => { + const statements = ['BEGIN TRANSACTION;', 'SELECT 1;', 'COMMIT;']; + const actual = identify(statements.join('\n'), { strict: false }); + const expected = [ + { + start: 0, + end: 17, + text: statements[0], + type: 'UNKNOWN', + executionType: 'UNKNOWN', + parameters: [], + }, + { + start: 19, + end: 27, + text: statements[1], + type: 'SELECT', + executionType: 'LISTING', + parameters: [], + }, + { + start: 29, + end: 35, + text: statements[2], + type: 'UNKNOWN', + executionType: 'UNKNOWN', + parameters: [], + }, + ]; + expect(actual).to.eql(expected); + }); }); From a92ac851aede6261958c14177ebd2cacabe54738 Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Fri, 26 Aug 2022 10:19:38 -0400 Subject: [PATCH 2/4] fix sqlite transactions having keywords --- src/parser.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/parser.ts b/src/parser.ts index 97c7532..fc6f825 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -663,7 +663,10 @@ function stateMachineStatementParser( prevNonWhitespaceToken?.value.toUpperCase() !== 'END' && (token.value.toUpperCase() !== 'BEGIN' || (token.value.toUpperCase() === 'BEGIN' && - nextToken.value.toUpperCase() !== 'TRANSACTION')) + nextToken.value.toUpperCase() !== 'TRANSACTION' && + (dialect !== 'sqlite' || + (dialect === 'sqlite' && + !['DEFERRED', 'IMMEDIATE', 'EXCLUSIVE'].includes(nextToken.value.toUpperCase()))))) ) { if ( ['oracle', 'bigquery'].includes(dialect) && From e534eb283b48c4f7e6ff48dd1d62784654de9fad Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Fri, 26 Aug 2022 10:23:27 -0400 Subject: [PATCH 3/4] Update multiple-statement.spec.ts --- test/identifier/multiple-statement.spec.ts | 99 +++++++++++++++------- 1 file changed, 69 insertions(+), 30 deletions(-) diff --git a/test/identifier/multiple-statement.spec.ts b/test/identifier/multiple-statement.spec.ts index c01cada..fdb6fb2 100644 --- a/test/identifier/multiple-statement.spec.ts +++ b/test/identifier/multiple-statement.spec.ts @@ -364,35 +364,74 @@ describe('identifier', () => { }); }); - it('should identify transactions', () => { - const statements = ['BEGIN TRANSACTION;', 'SELECT 1;', 'COMMIT;']; - const actual = identify(statements.join('\n'), { strict: false }); - const expected = [ - { - start: 0, - end: 17, - text: statements[0], - type: 'UNKNOWN', - executionType: 'UNKNOWN', - parameters: [], - }, - { - start: 19, - end: 27, - text: statements[1], - type: 'SELECT', - executionType: 'LISTING', - parameters: [], - }, - { - start: 29, - end: 35, - text: statements[2], - type: 'UNKNOWN', - executionType: 'UNKNOWN', - parameters: [], - }, - ]; - expect(actual).to.eql(expected); + describe('identifying transactions', () => { + it('should identify transactions', () => { + const statements = ['BEGIN TRANSACTION;', 'SELECT 1;', 'COMMIT;']; + const actual = identify(statements.join('\n'), { strict: false }); + const expected = [ + { + start: 0, + end: 17, + text: statements[0], + type: 'UNKNOWN', + executionType: 'UNKNOWN', + parameters: [], + }, + { + start: 19, + end: 27, + text: statements[1], + type: 'SELECT', + executionType: 'LISTING', + parameters: [], + }, + { + start: 29, + end: 35, + text: statements[2], + type: 'UNKNOWN', + executionType: 'UNKNOWN', + parameters: [], + }, + ]; + expect(actual).to.eql(expected); + }); + + describe('identifying keywords for sqlite transactions', () => { + ['DEFERRED', 'IMMEDIATE', 'EXCLUSIVE'].forEach((type) => { + it(`identifies BEGIN ${type} TRANSACTION`, () => { + const statements = [`BEGIN ${type} TRANSACTION;`, 'SELECT 1;', 'COMMIT;']; + const actual = identify(statements.join('\n'), { dialect: 'sqlite', strict: false }); + const offset = type.length + 1; + const expected = [ + { + start: 0, + end: 17 + offset, + text: statements[0], + type: 'UNKNOWN', + executionType: 'UNKNOWN', + parameters: [], + }, + { + start: 19 + offset, + end: 27 + offset, + text: statements[1], + type: 'SELECT', + executionType: 'LISTING', + parameters: [], + }, + { + start: 29 + offset, + end: 35 + offset, + text: statements[2], + type: 'UNKNOWN', + executionType: 'UNKNOWN', + parameters: [], + }, + ]; + expect(actual).to.eql(expected); + }); + }); + }); }); }); From 22a994800190c984c01b52e4354f030ee6ed1295 Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Fri, 26 Aug 2022 10:27:23 -0400 Subject: [PATCH 4/4] Update parser.ts --- src/parser.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index fc6f825..06e9d61 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -591,7 +591,6 @@ function stateMachineStatementParser( ): StatementParser { let currentStepIndex = 0; let prevToken: Token | undefined; - let prevPrevToken: Token | undefined; let prevNonWhitespaceToken: Token | undefined; let lastBlockOpener: Token | undefined; @@ -616,7 +615,6 @@ function stateMachineStatementParser( }; const setPrevToken = (token: Token) => { - prevPrevToken = prevToken; prevToken = token; if (token.type !== 'whitespace') { prevNonWhitespaceToken = token;