From e38204b72b43eb5f380ff4624947e330a5aa79bb Mon Sep 17 00:00:00 2001 From: amochin Date: Fri, 12 Sep 2025 15:44:10 +0200 Subject: [PATCH 1/3] New keyword "Split SQL Script" --- src/DatabaseLibrary/query.py | 154 ++++++++++-------- .../script_file_tests/split_commands.sql | 2 + test/tests/common_tests/script_files.robot | 16 +- 3 files changed, 101 insertions(+), 71 deletions(-) create mode 100644 test/resources/script_file_tests/split_commands.sql diff --git a/src/DatabaseLibrary/query.py b/src/DatabaseLibrary/query.py index 2aaac35..d4b9939 100644 --- a/src/DatabaseLibrary/query.py +++ b/src/DatabaseLibrary/query.py @@ -293,84 +293,98 @@ def execute_sql_script( | Execute SQL Script | insert_data_in_person_table.sql | split=False | """ db_connection = self.connection_store.get_connection(alias) - with open(script_path, encoding="UTF-8") as sql_file: - cur = None - try: - cur = db_connection.client.cursor() - if not split: + cur = None + try: + cur = db_connection.client.cursor() + if not split: + with open(script_path, encoding="UTF-8") as sql_file: logger.info("Statements splitting disabled - pass entire script content to the database module") self._execute_sql( cur, sql_file.read(), omit_trailing_semicolon=db_connection.omit_trailing_semicolon, ) - else: - logger.info("Splitting script file into statements...") - statements_to_execute = [] - current_statement = "" - inside_statements_group = False - proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?") + else: + statements_to_execute = self.split_sql_script(script_path) + for statement in statements_to_execute: proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") - for line in sql_file: - line = line.strip() - if line.startswith("#") or line.startswith("--") or line == "/": - continue - - # check if the line matches the creating procedure regexp pattern - if proc_start_pattern.match(line.lower()): - inside_statements_group = True - elif line.lower().startswith("begin"): - inside_statements_group = True - - # semicolons inside the line? use them to separate statements - # ... but not if they are inside a begin/end block (aka. statements group) - sqlFragments = line.split(";") - # no semicolons - if len(sqlFragments) == 1: - current_statement += line + " " - continue + line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$") + omit_semicolon = not line_ends_with_proc_end.search(statement.lower()) + self._execute_sql(cur, statement, omit_semicolon) + self._commit_if_needed(db_connection, no_transaction) + except Exception as e: + self._rollback_and_raise(db_connection, no_transaction, e) + + def split_sql_script( + self, + script_path: str, + ): + """ + Splits the content of the SQL script file loaded from `script_path` into individual SQL commands + and returns them as a list of strings. + SQL commands are expected to be delimited by a semicolon (';'). + """ + with open(script_path, encoding="UTF-8") as sql_file: + logger.info("Splitting script file into statements...") + statements_to_execute = [] + current_statement = "" + inside_statements_group = False + proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?") + proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") + for line in sql_file: + line = line.strip() + if line.startswith("#") or line.startswith("--") or line == "/": + continue + + # check if the line matches the creating procedure regexp pattern + if proc_start_pattern.match(line.lower()): + inside_statements_group = True + elif line.lower().startswith("begin"): + inside_statements_group = True + + # semicolons inside the line? use them to separate statements + # ... but not if they are inside a begin/end block (aka. statements group) + sqlFragments = line.split(";") + # no semicolons + if len(sqlFragments) == 1: + current_statement += line + " " + continue + quotes = 0 + # "select * from person;" -> ["select..", ""] + for sqlFragment in sqlFragments: + if len(sqlFragment.strip()) == 0: + continue + + if inside_statements_group: + # if statements inside a begin/end block have semicolns, + # they must persist - even with oracle + sqlFragment += "; " + + if proc_end_pattern.match(sqlFragment.lower()): + inside_statements_group = False + elif proc_start_pattern.match(sqlFragment.lower()): + inside_statements_group = True + elif sqlFragment.lower().startswith("begin"): + inside_statements_group = True + + # check if the semicolon is a part of the value (quoted string) + quotes += sqlFragment.count("'") + quotes -= sqlFragment.count("\\'") + inside_quoted_string = quotes % 2 != 0 + if inside_quoted_string: + sqlFragment += ";" # restore the semicolon + + current_statement += sqlFragment + if not inside_statements_group and not inside_quoted_string: + statements_to_execute.append(current_statement.strip()) + current_statement = "" quotes = 0 - # "select * from person;" -> ["select..", ""] - for sqlFragment in sqlFragments: - if len(sqlFragment.strip()) == 0: - continue - - if inside_statements_group: - # if statements inside a begin/end block have semicolns, - # they must persist - even with oracle - sqlFragment += "; " - - if proc_end_pattern.match(sqlFragment.lower()): - inside_statements_group = False - elif proc_start_pattern.match(sqlFragment.lower()): - inside_statements_group = True - elif sqlFragment.lower().startswith("begin"): - inside_statements_group = True - - # check if the semicolon is a part of the value (quoted string) - quotes += sqlFragment.count("'") - quotes -= sqlFragment.count("\\'") - inside_quoted_string = quotes % 2 != 0 - if inside_quoted_string: - sqlFragment += ";" # restore the semicolon - - current_statement += sqlFragment - if not inside_statements_group and not inside_quoted_string: - statements_to_execute.append(current_statement.strip()) - current_statement = "" - quotes = 0 - - current_statement = current_statement.strip() - if len(current_statement) != 0: - statements_to_execute.append(current_statement) - - for statement in statements_to_execute: - line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$") - omit_semicolon = not line_ends_with_proc_end.search(statement.lower()) - self._execute_sql(cur, statement, omit_semicolon) - self._commit_if_needed(db_connection, no_transaction) - except Exception as e: - self._rollback_and_raise(db_connection, no_transaction, e) + + current_statement = current_statement.strip() + if len(current_statement) != 0: + statements_to_execute.append(current_statement) + + return statements_to_execute @renamed_args( mapping={ diff --git a/test/resources/script_file_tests/split_commands.sql b/test/resources/script_file_tests/split_commands.sql new file mode 100644 index 0000000..c52a2c0 --- /dev/null +++ b/test/resources/script_file_tests/split_commands.sql @@ -0,0 +1,2 @@ +SELECT * FROM person; +SELECT * FROM person WHERE id=1; diff --git a/test/tests/common_tests/script_files.robot b/test/tests/common_tests/script_files.robot index 09d3ae0..719ce90 100644 --- a/test/tests/common_tests/script_files.robot +++ b/test/tests/common_tests/script_files.robot @@ -6,6 +6,9 @@ Suite Teardown Disconnect From Database Test Setup Create Person Table Test Teardown Drop Tables Person And Foobar +*** Variables *** +${Script files dir} ${CURDIR}/../../resources/script_file_tests + *** Test Cases *** Semicolons As Statement Separators In One Line @@ -35,9 +38,20 @@ Semicolons And Quotes In Values Should Be Equal As Strings ${results}[0] (5, 'Miles', "O'Brian") Should Be Equal As Strings ${results}[1] (6, 'Keiko', "O'Brian") +Split Script Into Statements + Insert Data In Person Table Using SQL Script + @{Expected commands}= Create List + ... SELECT * FROM person + ... SELECT * FROM person WHERE id=1 + ${extracted commands}= Split Sql Script ${Script files dir}/split_commands.sql + Lists Should Be Equal ${Expected commands} ${extracted commands} + FOR ${command} IN @{extracted commands} + ${results}= Query ${command} + END + + *** Keywords *** Run SQL Script File [Arguments] ${File Name} - ${Script files dir}= Set Variable ${CURDIR}/../../resources/script_file_tests Execute Sql Script ${Script files dir}/${File Name}.sql From 3d8a45e2215ed491351c7eda016328e99c9e8414 Mon Sep 17 00:00:00 2001 From: amochin Date: Fri, 12 Sep 2025 16:02:25 +0200 Subject: [PATCH 2/3] Allow using sqlparse to split SQL script files (fix #205) --- pyproject.toml | 3 +- src/DatabaseLibrary/query.py | 114 +++++++++++---------- test/tests/common_tests/script_files.robot | 12 ++- 3 files changed, 75 insertions(+), 54 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f678224..89b6f1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,8 @@ readme = "README.md" requires-python = ">=3.8.1" dependencies = [ "robotframework>=5.0.1", - "robotframework-assertion-engine" + "robotframework-assertion-engine", + "sqlparse" ] classifiers = [ "Programming Language :: Python :: 3", diff --git a/src/DatabaseLibrary/query.py b/src/DatabaseLibrary/query.py index d4b9939..ec23d7d 100644 --- a/src/DatabaseLibrary/query.py +++ b/src/DatabaseLibrary/query.py @@ -18,6 +18,7 @@ import sys from typing import List, Optional, Tuple +import sqlparse from robot.api import logger from robot.utils.dotdict import DotDict @@ -263,6 +264,7 @@ def execute_sql_script( no_transaction: bool = False, alias: Optional[str] = None, split: bool = True, + external_parser=False, *, sqlScriptFileName: Optional[str] = None, sansTran: Optional[bool] = None, @@ -274,6 +276,8 @@ def execute_sql_script( Set ``split`` to _False_ to disable this behavior - in this case the entire script content will be passed to the database module for execution as a single command. + Set `external_parser` to _True_ to use the external `sqlparse` library for splitting the script. + Set ``no_transaction`` to _True_ to run command without explicit transaction commit or rollback in case of error. See `Commit behavior` for details. @@ -305,7 +309,7 @@ def execute_sql_script( omit_trailing_semicolon=db_connection.omit_trailing_semicolon, ) else: - statements_to_execute = self.split_sql_script(script_path) + statements_to_execute = self.split_sql_script(script_path, external_parser=external_parser) for statement in statements_to_execute: proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$") @@ -318,71 +322,77 @@ def execute_sql_script( def split_sql_script( self, script_path: str, + external_parser=False, ): """ Splits the content of the SQL script file loaded from `script_path` into individual SQL commands and returns them as a list of strings. SQL commands are expected to be delimited by a semicolon (';'). + + Set `external_parser` to _True_ to use the external `sqlparse` library. """ with open(script_path, encoding="UTF-8") as sql_file: logger.info("Splitting script file into statements...") statements_to_execute = [] - current_statement = "" - inside_statements_group = False - proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?") - proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") - for line in sql_file: - line = line.strip() - if line.startswith("#") or line.startswith("--") or line == "/": - continue - - # check if the line matches the creating procedure regexp pattern - if proc_start_pattern.match(line.lower()): - inside_statements_group = True - elif line.lower().startswith("begin"): - inside_statements_group = True - - # semicolons inside the line? use them to separate statements - # ... but not if they are inside a begin/end block (aka. statements group) - sqlFragments = line.split(";") - # no semicolons - if len(sqlFragments) == 1: - current_statement += line + " " - continue - quotes = 0 - # "select * from person;" -> ["select..", ""] - for sqlFragment in sqlFragments: - if len(sqlFragment.strip()) == 0: + if external_parser: + statements_to_execute = sqlparse.split(sql_file.read()) + else: + current_statement = "" + inside_statements_group = False + proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?") + proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?") + for line in sql_file: + line = line.strip() + if line.startswith("#") or line.startswith("--") or line == "/": continue - if inside_statements_group: - # if statements inside a begin/end block have semicolns, - # they must persist - even with oracle - sqlFragment += "; " - - if proc_end_pattern.match(sqlFragment.lower()): - inside_statements_group = False - elif proc_start_pattern.match(sqlFragment.lower()): + # check if the line matches the creating procedure regexp pattern + if proc_start_pattern.match(line.lower()): inside_statements_group = True - elif sqlFragment.lower().startswith("begin"): + elif line.lower().startswith("begin"): inside_statements_group = True - # check if the semicolon is a part of the value (quoted string) - quotes += sqlFragment.count("'") - quotes -= sqlFragment.count("\\'") - inside_quoted_string = quotes % 2 != 0 - if inside_quoted_string: - sqlFragment += ";" # restore the semicolon - - current_statement += sqlFragment - if not inside_statements_group and not inside_quoted_string: - statements_to_execute.append(current_statement.strip()) - current_statement = "" - quotes = 0 - - current_statement = current_statement.strip() - if len(current_statement) != 0: - statements_to_execute.append(current_statement) + # semicolons inside the line? use them to separate statements + # ... but not if they are inside a begin/end block (aka. statements group) + sqlFragments = line.split(";") + # no semicolons + if len(sqlFragments) == 1: + current_statement += line + " " + continue + quotes = 0 + # "select * from person;" -> ["select..", ""] + for sqlFragment in sqlFragments: + if len(sqlFragment.strip()) == 0: + continue + + if inside_statements_group: + # if statements inside a begin/end block have semicolns, + # they must persist - even with oracle + sqlFragment += "; " + + if proc_end_pattern.match(sqlFragment.lower()): + inside_statements_group = False + elif proc_start_pattern.match(sqlFragment.lower()): + inside_statements_group = True + elif sqlFragment.lower().startswith("begin"): + inside_statements_group = True + + # check if the semicolon is a part of the value (quoted string) + quotes += sqlFragment.count("'") + quotes -= sqlFragment.count("\\'") + inside_quoted_string = quotes % 2 != 0 + if inside_quoted_string: + sqlFragment += ";" # restore the semicolon + + current_statement += sqlFragment + if not inside_statements_group and not inside_quoted_string: + statements_to_execute.append(current_statement.strip()) + current_statement = "" + quotes = 0 + + current_statement = current_statement.strip() + if len(current_statement) != 0: + statements_to_execute.append(current_statement) return statements_to_execute diff --git a/test/tests/common_tests/script_files.robot b/test/tests/common_tests/script_files.robot index 719ce90..db94c88 100644 --- a/test/tests/common_tests/script_files.robot +++ b/test/tests/common_tests/script_files.robot @@ -38,7 +38,7 @@ Semicolons And Quotes In Values Should Be Equal As Strings ${results}[0] (5, 'Miles', "O'Brian") Should Be Equal As Strings ${results}[1] (6, 'Keiko', "O'Brian") -Split Script Into Statements +Split Script Into Statements - Internal Parser Insert Data In Person Table Using SQL Script @{Expected commands}= Create List ... SELECT * FROM person @@ -49,6 +49,16 @@ Split Script Into Statements ${results}= Query ${command} END +Split Script Into Statements - External Parser + Insert Data In Person Table Using SQL Script + @{Expected commands}= Create List + ... SELECT * FROM person; + ... SELECT * FROM person WHERE id=1; + ${extracted commands}= Split Sql Script ${Script files dir}/split_commands.sql external_parser=True + Lists Should Be Equal ${Expected commands} ${extracted commands} + FOR ${command} IN @{extracted commands} + ${results}= Query ${command} + END *** Keywords *** From 31123c939bfb6ce3f8fff7143736ed45b38fd41c Mon Sep 17 00:00:00 2001 From: amochin Date: Fri, 12 Sep 2025 16:06:27 +0200 Subject: [PATCH 3/3] Include sqlparse into dependencies --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 89b6f1f..ae6a7c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,8 @@ requires = [ "setuptools>=61.0", "robotframework>=5.0.1", - "robotframework-assertion-engine" + "robotframework-assertion-engine", + "sqlparse" ] build-backend = "setuptools.build_meta"