From 5227c4cc33c7df2d1cee92c995b0fabe6e1062d3 Mon Sep 17 00:00:00 2001 From: Antonio Sejas Date: Wed, 8 Oct 2025 10:40:05 +0100 Subject: [PATCH 1/9] Remove custom parser in favor of AST parser --- src/Import.php | 85 ++++++++------------------------------------------ 1 file changed, 13 insertions(+), 72 deletions(-) diff --git a/src/Import.php b/src/Import.php index d9aa644..fa4be43 100644 --- a/src/Import.php +++ b/src/Import.php @@ -50,8 +50,13 @@ public function run( $sql_file_path, $args ) { * @throws Exception */ protected function execute_statements( $import_file ) { - foreach ( $this->parse_statements( $import_file ) as $statement ) { - $result = $this->driver->query( $statement ); + $raw_queries = file_get_contents( $import_file ); + $queries_text = $this->remove_comments( $raw_queries ); + $parser = $this->driver->create_parser( $queries_text ); + while ( $parser->next_query() ) { + $ast = $parser->get_query_ast(); + $statement = substr( $queries_text, $ast->get_start(), $ast->get_length() ); + $result = $this->driver->query( $statement ); if ( false === $result ) { WP_CLI::warning( 'Could not execute statement: ' . $statement ); echo $this->driver->get_error_message(); @@ -60,77 +65,13 @@ protected function execute_statements( $import_file ) { } /** - * Parse SQL statements from an SQL dump file. - * @param string $sql_file_path The path to the SQL dump file. + * Remove comments from the input. + * + * @param string $input * - * @return Generator A generator that yields SQL statements. + * @return string */ - public function parse_statements( $sql_file_path ) { - - $handle = fopen( $sql_file_path, 'r' ); - - if ( ! $handle ) { - WP_CLI::error( "Unable to open file: $sql_file_path" ); - } - - $single_quotes = 0; - $double_quotes = 0; - $in_comment = false; - $buffer = ''; - - // phpcs:ignore - while ( ( $line = fgets( $handle ) ) !== false ) { - $line = trim( $line ); - - // Skip empty lines and comments - if ( empty( $line ) || strpos( $line, '--' ) === 0 || strpos( $line, '#' ) === 0 ) { - continue; - } - - // Handle multi-line comments - if ( ! $in_comment && strpos( $line, '/*' ) === 0 ) { - $in_comment = true; - } - if ( $in_comment ) { - if ( strpos( $line, '*/' ) !== false ) { - $in_comment = false; - } - continue; - } - - $strlen = strlen( $line ); - for ( $i = 0; $i < $strlen; $i++ ) { - $ch = $line[ $i ]; - - // Handle escaped characters - if ( $i > 0 && '\\' === $line[ $i - 1 ] ) { - $buffer .= $ch; - continue; - } - - // Handle quotes - if ( "'" === $ch && 0 === $double_quotes ) { - $single_quotes = 1 - $single_quotes; - } - if ( '"' === $ch && 0 === $single_quotes ) { - $double_quotes = 1 - $double_quotes; - } - - // Process statement end - if ( ';' === $ch && 0 === $single_quotes && 0 === $double_quotes ) { - yield trim( $buffer ); - $buffer = ''; - } else { - $buffer .= $ch; - } - } - } - - // Handle any remaining buffer content - if ( ! empty( $buffer ) ) { - yield trim( $buffer ); - } - - fclose( $handle ); + protected function remove_comments( $text ) { + return preg_replace( '/\/\*.*?\*\//s', '', $text ); } } From 84a3a8d6cdb129f3778e80d5c10c6a501a364e35 Mon Sep 17 00:00:00 2001 From: Antonio Sejas Date: Wed, 8 Oct 2025 11:53:59 +0100 Subject: [PATCH 2/9] Add try catch instead of checking $result false value --- src/Import.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Import.php b/src/Import.php index fa4be43..d3f4850 100644 --- a/src/Import.php +++ b/src/Import.php @@ -56,9 +56,10 @@ protected function execute_statements( $import_file ) { while ( $parser->next_query() ) { $ast = $parser->get_query_ast(); $statement = substr( $queries_text, $ast->get_start(), $ast->get_length() ); - $result = $this->driver->query( $statement ); - if ( false === $result ) { - WP_CLI::warning( 'Could not execute statement: ' . $statement ); + try { + $this->driver->query( $statement ); + } catch ( Exception $e ) { + WP_CLI::error( 'SQLite import could not execute statement: ' . $statement ); echo $this->driver->get_error_message(); } } From 6fe2f55b2ff3c87f332d2e19f598f9e2d59dc834 Mon Sep 17 00:00:00 2001 From: Antonio Sejas Date: Wed, 8 Oct 2025 11:54:31 +0100 Subject: [PATCH 3/9] Add semi colon to comments regex --- src/Import.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Import.php b/src/Import.php index d3f4850..f7cc34b 100644 --- a/src/Import.php +++ b/src/Import.php @@ -73,6 +73,6 @@ protected function execute_statements( $import_file ) { * @return string */ protected function remove_comments( $text ) { - return preg_replace( '/\/\*.*?\*\//s', '', $text ); + return preg_replace( '/\/\*.*?\*\/(;)?/s', '', $text ); } } From 79063a8a9187aa154be7d64d840c96c40a543089 Mon Sep 17 00:00:00 2001 From: Antonio Sejas Date: Wed, 8 Oct 2025 12:27:30 +0100 Subject: [PATCH 4/9] Add support for old driver WP_SQLite_Translator --- src/Import.php | 95 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/src/Import.php b/src/Import.php index f7cc34b..95d9a6b 100644 --- a/src/Import.php +++ b/src/Import.php @@ -35,12 +35,103 @@ public function run( $sql_file_path, $args ) { $is_stdin = '-' === $sql_file_path; $import_file = $is_stdin ? 'php://stdin' : $sql_file_path; - $this->execute_statements( $import_file ); + if ( method_exists( $this->driver, 'create_parser' ) ) { + $this->execute_statements_with_ast_parser( $import_file ); + } else { + $this->execute_statements( $import_file ); + } $imported_from = $is_stdin ? 'STDIN' : $sql_file_path; WP_CLI::success( sprintf( "Imported from '%s'.", $imported_from ) ); } + + + protected function execute_statements( $import_file ) { + foreach ( $this->parse_statements( $import_file ) as $statement ) { + $result = $this->driver->query( $statement ); + if ( false === $result ) { + WP_CLI::warning( 'Could not execute statement: ' . $statement ); + echo $this->driver->get_error_message(); + } + } + } + + /** + * Parse SQL statements from an SQL dump file. + * @param string $sql_file_path The path to the SQL dump file. + * + * @return Generator A generator that yields SQL statements. + */ + public function parse_statements( $sql_file_path ) { + + $handle = fopen( $sql_file_path, 'r' ); + + if ( ! $handle ) { + WP_CLI::error( "Unable to open file: $sql_file_path" ); + } + + $single_quotes = 0; + $double_quotes = 0; + $in_comment = false; + $buffer = ''; + + // phpcs:ignore + while ( ( $line = fgets( $handle ) ) !== false ) { + $line = trim( $line ); + + // Skip empty lines and comments + if ( empty( $line ) || strpos( $line, '--' ) === 0 || strpos( $line, '#' ) === 0 ) { + continue; + } + + // Handle multi-line comments + if ( ! $in_comment && strpos( $line, '/*' ) === 0 ) { + $in_comment = true; + } + if ( $in_comment ) { + if ( strpos( $line, '*/' ) !== false ) { + $in_comment = false; + } + continue; + } + + $strlen = strlen( $line ); + for ( $i = 0; $i < $strlen; $i++ ) { + $ch = $line[ $i ]; + + // Handle escaped characters + if ( $i > 0 && '\\' === $line[ $i - 1 ] ) { + $buffer .= $ch; + continue; + } + + // Handle quotes + if ( "'" === $ch && 0 === $double_quotes ) { + $single_quotes = 1 - $single_quotes; + } + if ( '"' === $ch && 0 === $single_quotes ) { + $double_quotes = 1 - $double_quotes; + } + + // Process statement end + if ( ';' === $ch && 0 === $single_quotes && 0 === $double_quotes ) { + yield trim( $buffer ); + $buffer = ''; + } else { + $buffer .= $ch; + } + } + } + + // Handle any remaining buffer content + if ( ! empty( $buffer ) ) { + yield trim( $buffer ); + } + + fclose( $handle ); + } + /** * Execute SQL statements from an SQL dump file. * @@ -49,7 +140,7 @@ public function run( $sql_file_path, $args ) { * @return void * @throws Exception */ - protected function execute_statements( $import_file ) { + protected function execute_statements_with_ast_parser( $import_file ) { $raw_queries = file_get_contents( $import_file ); $queries_text = $this->remove_comments( $raw_queries ); $parser = $this->driver->create_parser( $queries_text ); From e7e85705c76616a69863ce058d38d18d84296f75 Mon Sep 17 00:00:00 2001 From: Antonio Sejas Date: Wed, 8 Oct 2025 12:28:53 +0100 Subject: [PATCH 5/9] Update comments --- src/Import.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Import.php b/src/Import.php index 95d9a6b..b09bc9d 100644 --- a/src/Import.php +++ b/src/Import.php @@ -45,8 +45,14 @@ public function run( $sql_file_path, $args ) { WP_CLI::success( sprintf( "Imported from '%s'.", $imported_from ) ); } - - + /** + * Execute SQL statements from an SQL dump file. + * + * @param $import_file + * + * @return void + * @throws Exception + */ protected function execute_statements( $import_file ) { foreach ( $this->parse_statements( $import_file ) as $statement ) { $result = $this->driver->query( $statement ); @@ -133,7 +139,7 @@ public function parse_statements( $sql_file_path ) { } /** - * Execute SQL statements from an SQL dump file. + * Execute SQL statements from an SQL dump file using the AST parser. * * @param $import_file * From ca2f090aaff299eaadad4062d566e5d2ddfdb2f8 Mon Sep 17 00:00:00 2001 From: Antonio Sejas Date: Wed, 8 Oct 2025 12:47:02 +0100 Subject: [PATCH 6/9] Bypass SET comments instead of removing all the comments --- src/Import.php | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/src/Import.php b/src/Import.php index b09bc9d..0805455 100644 --- a/src/Import.php +++ b/src/Import.php @@ -139,37 +139,30 @@ public function parse_statements( $sql_file_path ) { } /** - * Execute SQL statements from an SQL dump file using the AST parser. - * - * @param $import_file - * - * @return void - * @throws Exception - */ + * Execute SQL statements from an SQL dump file using the AST parser. + * + * @param $import_file + * + * @return void + * @throws Exception + */ protected function execute_statements_with_ast_parser( $import_file ) { - $raw_queries = file_get_contents( $import_file ); - $queries_text = $this->remove_comments( $raw_queries ); - $parser = $this->driver->create_parser( $queries_text ); + $raw_queries = file_get_contents( $import_file ); + $parser = $this->driver->create_parser( $raw_queries ); while ( $parser->next_query() ) { $ast = $parser->get_query_ast(); - $statement = substr( $queries_text, $ast->get_start(), $ast->get_length() ); + $statement = substr( $raw_queries, $ast->get_start(), $ast->get_length() ); try { $this->driver->query( $statement ); } catch ( Exception $e ) { + // Skip errors when executing SET comment statements + if ( 0 === strpos( $statement, 'SET ' ) && false !== strpos( $statement, '*/' ) ) { + WP_CLI::warning( 'SQLite import SET comment statement: ' . $statement ); + continue; + } WP_CLI::error( 'SQLite import could not execute statement: ' . $statement ); echo $this->driver->get_error_message(); } } } - - /** - * Remove comments from the input. - * - * @param string $input - * - * @return string - */ - protected function remove_comments( $text ) { - return preg_replace( '/\/\*.*?\*\/(;)?/s', '', $text ); - } } From b58afacc8f921e28eb8476b11b3500c392ecdbd0 Mon Sep 17 00:00:00 2001 From: Antonio Sejas Date: Wed, 8 Oct 2025 13:53:43 +0100 Subject: [PATCH 7/9] Fix comment indentation --- src/Import.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Import.php b/src/Import.php index 0805455..2e23b62 100644 --- a/src/Import.php +++ b/src/Import.php @@ -139,13 +139,13 @@ public function parse_statements( $sql_file_path ) { } /** - * Execute SQL statements from an SQL dump file using the AST parser. - * - * @param $import_file - * - * @return void - * @throws Exception - */ + * Execute SQL statements from an SQL dump file using the AST parser. + * + * @param $import_file + * + * @return void + * @throws Exception + */ protected function execute_statements_with_ast_parser( $import_file ) { $raw_queries = file_get_contents( $import_file ); $parser = $this->driver->create_parser( $raw_queries ); From d897a9ef80c99a4298b0d11c2d42d08a200d1810 Mon Sep 17 00:00:00 2001 From: Antonio Sejas Date: Wed, 8 Oct 2025 13:57:44 +0100 Subject: [PATCH 8/9] Fix file encoding like: https://github.com/WordPress/sqlite-database-integration/issues/250 --- src/Import.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Import.php b/src/Import.php index 2e23b62..259586a 100644 --- a/src/Import.php +++ b/src/Import.php @@ -148,7 +148,14 @@ public function parse_statements( $sql_file_path ) { */ protected function execute_statements_with_ast_parser( $import_file ) { $raw_queries = file_get_contents( $import_file ); - $parser = $this->driver->create_parser( $raw_queries ); + + // Detect and convert encoding to UTF-8 + $detected_encoding = mb_detect_encoding( $raw_queries, mb_list_encodings(), true ); + if ( $detected_encoding && 'UTF-8' !== $detected_encoding ) { + $raw_queries = mb_convert_encoding( $raw_queries, 'UTF-8', $detected_encoding ); + } + + $parser = $this->driver->create_parser( $raw_queries ); while ( $parser->next_query() ) { $ast = $parser->get_query_ast(); $statement = substr( $raw_queries, $ast->get_start(), $ast->get_length() ); From fdf1b6335bf8989153c6baab6c586abc9e4eb64e Mon Sep 17 00:00:00 2001 From: Antonio Sejas Date: Wed, 8 Oct 2025 14:46:07 +0100 Subject: [PATCH 9/9] Echo the skipped comments instead of using a wp-cli warning to reduce noise in Studio alert error messages --- src/Import.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Import.php b/src/Import.php index 259586a..7d23723 100644 --- a/src/Import.php +++ b/src/Import.php @@ -164,7 +164,7 @@ protected function execute_statements_with_ast_parser( $import_file ) { } catch ( Exception $e ) { // Skip errors when executing SET comment statements if ( 0 === strpos( $statement, 'SET ' ) && false !== strpos( $statement, '*/' ) ) { - WP_CLI::warning( 'SQLite import SET comment statement: ' . $statement ); + echo 'Warning - SQLite import skipped SET comment statement: ' . $statement . PHP_EOL; continue; } WP_CLI::error( 'SQLite import could not execute statement: ' . $statement );