From 053f71783e9c4b14b41684058a7da8c995b2059b Mon Sep 17 00:00:00 2001 From: Jan Jakes Date: Thu, 9 Oct 2025 11:16:42 +0200 Subject: [PATCH 1/9] Fix character escaping to support escaped backslashes --- features/sqlite-import.feature | 24 ++++++++++++++++++++++++ src/Import.php | 12 ++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/features/sqlite-import.feature b/features/sqlite-import.feature index 547d1d0..21ea8e5 100644 --- a/features/sqlite-import.feature +++ b/features/sqlite-import.feature @@ -43,3 +43,27 @@ Feature: WP-CLI SQLite Import Command Success: Imported from 'STDIN'. """ And the SQLite database should contain the imported data + + @require-sqlite + Scenario: Import a file with escape sequences + Given a SQL dump file named "test_import.sql" with content: + """ + SET sql_mode='NO_BACKSLASH_ESCAPES'; + CREATE TABLE test_table (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT); + INSERT INTO test_table (name) VALUES ('Test that escaping a backslash \\ works'); + INSERT INTO test_table (name) VALUES ('Test that escaping multiple backslashes \\\\\\ works'); + INSERT INTO test_table (name) VALUES ('Test that escaping a character \a works'); + INSERT INTO test_table (name) VALUES ('Test that escaping a backslash followed by a character \\a works'); + INSERT INTO test_table (name) VALUES ('Test that escaping a backslash and a character \\\a works'); + """ + When I run `wp sqlite --enable-ast-driver import test_import.sql` + Then STDOUT should contain: + """ + Success: Imported from 'test_import.sql'. + """ + And the SQLite database should contain a table named "test_table" + And the "test_table" should contain a row with name "Test that escaping a backslash \\ works" + And the "test_table" should contain a row with name "Test that escaping multiple backslashes \\\\\\ works" + And the "test_table" should contain a row with name "Test that escaping a character \a works" + And the "test_table" should contain a row with name "Test that escaping a backslash followed by a character \\a works" + And the "test_table" should contain a row with name "Test that escaping a backslash and a character \\\a works" diff --git a/src/Import.php b/src/Import.php index d9aa644..2805955 100644 --- a/src/Import.php +++ b/src/Import.php @@ -102,8 +102,16 @@ public function parse_statements( $sql_file_path ) { for ( $i = 0; $i < $strlen; $i++ ) { $ch = $line[ $i ]; - // Handle escaped characters - if ( $i > 0 && '\\' === $line[ $i - 1 ] ) { + // Count preceding backslashes. + $slashes = 0; + while ( $slashes < $i && '\\' === $line[ $i - $slashes - 1 ] ) { + ++$slashes; + } + + // Handle escaped characters. + // A characters is escaped only when the number of preceding backslashes + // is odd - "\" is an escape sequence, "\\" is an escaped backslash. + if ( 1 === $slashes % 2 ) { $buffer .= $ch; continue; } From da3e71a2b14db440e29ff3bae052c693da2407e3 Mon Sep 17 00:00:00 2001 From: Jan Jakes Date: Thu, 9 Oct 2025 11:29:54 +0200 Subject: [PATCH 2/9] Detect and fix encoding of the database dump when importing --- src/Import.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Import.php b/src/Import.php index 2805955..680db79 100644 --- a/src/Import.php +++ b/src/Import.php @@ -80,6 +80,12 @@ public function parse_statements( $sql_file_path ) { // phpcs:ignore while ( ( $line = fgets( $handle ) ) !== false ) { + // Detect and convert encoding to UTF-8 + $detected_encoding = mb_detect_encoding( $line, mb_list_encodings(), true ); + if ( $detected_encoding && 'UTF-8' !== $detected_encoding ) { + $line = mb_convert_encoding( $line, 'UTF-8', $detected_encoding ); + } + $line = trim( $line ); // Skip empty lines and comments From 6c22b98a43aaf2f7f92e12336b2709eab21fd9fb Mon Sep 17 00:00:00 2001 From: Jan Jakes Date: Thu, 9 Oct 2025 14:59:34 +0200 Subject: [PATCH 3/9] Fix whitespace being trimmed in multi-line strings --- features/bootstrap/SQLiteFeatureContext.php | 1 + features/sqlite-import.feature | 20 ++++++++++++++++++++ src/Import.php | 2 -- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/features/bootstrap/SQLiteFeatureContext.php b/features/bootstrap/SQLiteFeatureContext.php index 87598ca..d44c3b2 100644 --- a/features/bootstrap/SQLiteFeatureContext.php +++ b/features/bootstrap/SQLiteFeatureContext.php @@ -34,6 +34,7 @@ public function theSqliteDatabaseShouldContainATableNamed( $table_name ) { /** * @Then /^the "([^"]*)" should contain a row with name "([^"]*)"$/ + * @Then /^the "([^"]*)" should contain a row with name:$/ */ public function theTableShouldContainARowWithName( $table_name, $name ) { $this->connectToDatabase(); diff --git a/features/sqlite-import.feature b/features/sqlite-import.feature index 21ea8e5..e9eb90c 100644 --- a/features/sqlite-import.feature +++ b/features/sqlite-import.feature @@ -67,3 +67,23 @@ Feature: WP-CLI SQLite Import Command And the "test_table" should contain a row with name "Test that escaping a character \a works" And the "test_table" should contain a row with name "Test that escaping a backslash followed by a character \\a works" And the "test_table" should contain a row with name "Test that escaping a backslash and a character \\\a works" + + @require-sqlite + Scenario: Import a file with newlines in strings + Given a SQL dump file named "test_import.sql" with content: + """ + CREATE TABLE test_table (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT); + INSERT INTO test_table (name) VALUES ('Test that a string containing + a newline character and some whitespace works'); + """ + When I run `wp sqlite --enable-ast-driver import test_import.sql` + Then STDOUT should contain: + """ + Success: Imported from 'test_import.sql'. + """ + And the SQLite database should contain a table named "test_table" + And the "test_table" should contain a row with name: + """ + Test that a string containing + a newline character and some whitespace works + """ diff --git a/src/Import.php b/src/Import.php index 680db79..0b8213f 100644 --- a/src/Import.php +++ b/src/Import.php @@ -86,8 +86,6 @@ public function parse_statements( $sql_file_path ) { $line = mb_convert_encoding( $line, 'UTF-8', $detected_encoding ); } - $line = trim( $line ); - // Skip empty lines and comments if ( empty( $line ) || strpos( $line, '--' ) === 0 || strpos( $line, '#' ) === 0 ) { continue; From c1a41cd75c3abeee0789ccde6569aef16f0143fa Mon Sep 17 00:00:00 2001 From: Jan Jakes Date: Thu, 9 Oct 2025 15:27:40 +0200 Subject: [PATCH 4/9] Fix comment handling - allow them to start anywhere, don't match them in strings --- features/sqlite-import.feature | 27 +++++++++++++++++++++++ src/Import.php | 39 ++++++++++++++++++++-------------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/features/sqlite-import.feature b/features/sqlite-import.feature index e9eb90c..3ff226b 100644 --- a/features/sqlite-import.feature +++ b/features/sqlite-import.feature @@ -87,3 +87,30 @@ Feature: WP-CLI SQLite Import Command Test that a string containing a newline character and some whitespace works """ + + @require-sqlite + Scenario: Import a file with comments + Given a SQL dump file named "test_import.sql" with content: + """ + CREATE TABLE test_table (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT); + -- This is an inline comment. + # This is an inline comment. + INSERT INTO test_table (name) VALUES ('one'); -- This is an inline comment. + /* This is a block comment */ + INSERT INTO test_table (name) VALUES ('two'); /* This + is a block comment + on multiple lines */ INSERT INTO test_table (name) VALUES ('three'); + INSERT INTO test_table (name) VALUES ('fo -- this looks like a comment ur'); + INSERT INTO test_table (name) VALUES ('fi/* this looks like a comment */ve'); + """ + When I run `wp sqlite --enable-ast-driver import test_import.sql` + Then STDOUT should contain: + """ + Success: Imported from 'test_import.sql'. + """ + And the SQLite database should contain a table named "test_table" + And the "test_table" should contain a row with name "one" + And the "test_table" should contain a row with name "two" + And the "test_table" should contain a row with name "three" + And the "test_table" should contain a row with name "fo -- this looks like a comment ur" + And the "test_table" should contain a row with name "fi/* this looks like a comment */ve" diff --git a/src/Import.php b/src/Import.php index 0b8213f..616986d 100644 --- a/src/Import.php +++ b/src/Import.php @@ -86,22 +86,6 @@ public function parse_statements( $sql_file_path ) { $line = mb_convert_encoding( $line, 'UTF-8', $detected_encoding ); } - // Skip empty lines and comments - if ( empty( $line ) || strpos( $line, '--' ) === 0 || strpos( $line, '#' ) === 0 ) { - continue; - } - - // Handle multi-line comments - if ( ! $in_comment && strpos( $line, '/*' ) === 0 ) { - $in_comment = true; - } - if ( $in_comment ) { - if ( strpos( $line, '*/' ) !== false ) { - $in_comment = false; - } - continue; - } - $strlen = strlen( $line ); for ( $i = 0; $i < $strlen; $i++ ) { $ch = $line[ $i ]; @@ -120,6 +104,29 @@ public function parse_statements( $sql_file_path ) { continue; } + // Handle comments. + if ( 0 === $single_quotes && 0 === $double_quotes ) { + $prev_ch = isset( $line[ $i - 1 ] ) ? $line[ $i - 1 ] : null; + $next_ch = isset( $line[ $i + 1 ] ) ? $line[ $i + 1 ] : null; + + // Skip inline comments. + if ( ( '-' === $ch && '-' === $next_ch ) || '#' === $ch ) { + break; // Stop for the current line. + } + + // Skip multi-line comments. + if ( ! $in_comment && '/' === $ch && '*' === $next_ch ) { + $in_comment = true; + continue; + } + if ( $in_comment ) { + if ( '*' === $prev_ch && '/' === $ch ) { + $in_comment = false; + } + continue; + } + } + // Handle quotes if ( "'" === $ch && 0 === $double_quotes ) { $single_quotes = 1 - $single_quotes; From a84c9543fcbc9217053a1394c52e211c1fbdff2c Mon Sep 17 00:00:00 2001 From: Jan Jakes Date: Thu, 9 Oct 2025 16:00:38 +0200 Subject: [PATCH 5/9] Simplify quote matching state, add tests --- features/bootstrap/SQLiteFeatureContext.php | 2 +- features/sqlite-import.feature | 23 +++++++++++++++++++++ src/Import.php | 20 ++++++++---------- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/features/bootstrap/SQLiteFeatureContext.php b/features/bootstrap/SQLiteFeatureContext.php index d44c3b2..87cf08c 100644 --- a/features/bootstrap/SQLiteFeatureContext.php +++ b/features/bootstrap/SQLiteFeatureContext.php @@ -38,7 +38,7 @@ public function theSqliteDatabaseShouldContainATableNamed( $table_name ) { */ public function theTableShouldContainARowWithName( $table_name, $name ) { $this->connectToDatabase(); - $result = $this->db->query( "SELECT * FROM $table_name WHERE name='$name'" ); + $result = $this->db->query( "SELECT * FROM $table_name WHERE name='" . $this->db->escapeString( $name ) . "'" ); $row = $result->fetchArray(); if ( ! $row ) { throw new Exception( "Row with name '$name' not found in table '$table_name'." ); diff --git a/features/sqlite-import.feature b/features/sqlite-import.feature index 3ff226b..334e426 100644 --- a/features/sqlite-import.feature +++ b/features/sqlite-import.feature @@ -114,3 +114,26 @@ Feature: WP-CLI SQLite Import Command And the "test_table" should contain a row with name "three" And the "test_table" should contain a row with name "fo -- this looks like a comment ur" And the "test_table" should contain a row with name "fi/* this looks like a comment */ve" + + @require-sqlite + Scenario: Import a file quoted strings + Given a SQL dump file named "test_import.sql" with content: + """ + CREATE TABLE test_table (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT); + INSERT INTO test_table (name) VALUES ('a single-quoted string with \' '' some " tricky ` chars'); + INSERT INTO test_table (name) VALUES ("a double-quoted string with ' some \" "" tricky ` chars"); + """ + When I run `wp sqlite --enable-ast-driver import test_import.sql` + Then STDOUT should contain: + """ + Success: Imported from 'test_import.sql'. + """ + And the SQLite database should contain a table named "test_table" + And the "test_table" should contain a row with name: + """ + a single-quoted string with ' ' some " tricky ` chars + """ + And the "test_table" should contain a row with name: + """ + a double-quoted string with ' some " " tricky ` chars + """ diff --git a/src/Import.php b/src/Import.php index 616986d..d54249c 100644 --- a/src/Import.php +++ b/src/Import.php @@ -73,10 +73,9 @@ public function parse_statements( $sql_file_path ) { WP_CLI::error( "Unable to open file: $sql_file_path" ); } - $single_quotes = 0; - $double_quotes = 0; - $in_comment = false; - $buffer = ''; + $starting_quote = null; + $in_comment = false; + $buffer = ''; // phpcs:ignore while ( ( $line = fgets( $handle ) ) !== false ) { @@ -105,7 +104,7 @@ public function parse_statements( $sql_file_path ) { } // Handle comments. - if ( 0 === $single_quotes && 0 === $double_quotes ) { + if ( null === $starting_quote ) { $prev_ch = isset( $line[ $i - 1 ] ) ? $line[ $i - 1 ] : null; $next_ch = isset( $line[ $i + 1 ] ) ? $line[ $i + 1 ] : null; @@ -128,15 +127,14 @@ public function parse_statements( $sql_file_path ) { } // Handle quotes - if ( "'" === $ch && 0 === $double_quotes ) { - $single_quotes = 1 - $single_quotes; - } - if ( '"' === $ch && 0 === $single_quotes ) { - $double_quotes = 1 - $double_quotes; + if ( null === $starting_quote && ( "'" === $ch || '"' === $ch ) ) { + $starting_quote = $ch; + } elseif ( null !== $starting_quote && $ch === $starting_quote ) { + $starting_quote = null; } // Process statement end - if ( ';' === $ch && 0 === $single_quotes && 0 === $double_quotes ) { + if ( ';' === $ch && null === $starting_quote ) { yield trim( $buffer ); $buffer = ''; } else { From 8e78cf771cf3ece3d912434b583a47b807181942 Mon Sep 17 00:00:00 2001 From: Jan Jakes Date: Thu, 9 Oct 2025 16:12:56 +0200 Subject: [PATCH 6/9] Correctly parse backtick-quoted strings --- features/bootstrap/SQLiteFeatureContext.php | 2 +- features/sqlite-import.feature | 14 ++++++++++++++ src/Import.php | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/features/bootstrap/SQLiteFeatureContext.php b/features/bootstrap/SQLiteFeatureContext.php index 87cf08c..9a07975 100644 --- a/features/bootstrap/SQLiteFeatureContext.php +++ b/features/bootstrap/SQLiteFeatureContext.php @@ -25,7 +25,7 @@ public function aSqlDumpFileNamedWithContent( $filename, PyStringNode $content ) */ public function theSqliteDatabaseShouldContainATableNamed( $table_name ) { $this->connectToDatabase(); - $result = $this->db->query( "SELECT name FROM sqlite_master WHERE type='table' AND name='$table_name'" ); + $result = $this->db->query( "SELECT name FROM sqlite_master WHERE type='table' AND name='" . $this->db->escapeString( $table_name ) . "'" ); $row = $result->fetchArray(); if ( ! $row ) { throw new Exception( "Table '$table_name' not found in the database." ); diff --git a/features/sqlite-import.feature b/features/sqlite-import.feature index 334e426..1711f13 100644 --- a/features/sqlite-import.feature +++ b/features/sqlite-import.feature @@ -137,3 +137,17 @@ Feature: WP-CLI SQLite Import Command """ a double-quoted string with ' some " " tricky ` chars """ + + @require-sqlite + Scenario: Import a file backtick-quoted identifiers + Given a SQL dump file named "test_import.sql" with content: + """ + CREATE TABLE `a'strange``identifier\\name` (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT); + """ + When I run `wp sqlite --enable-ast-driver import test_import.sql` + Then STDOUT should contain: + """ + Success: Imported from 'test_import.sql'. + """ + + And the SQLite database should contain a table named "a'strange`identifier\name" diff --git a/src/Import.php b/src/Import.php index d54249c..7d1f25a 100644 --- a/src/Import.php +++ b/src/Import.php @@ -127,7 +127,7 @@ public function parse_statements( $sql_file_path ) { } // Handle quotes - if ( null === $starting_quote && ( "'" === $ch || '"' === $ch ) ) { + if ( null === $starting_quote && ( "'" === $ch || '"' === $ch || '`' === $ch ) ) { $starting_quote = $ch; } elseif ( null !== $starting_quote && $ch === $starting_quote ) { $starting_quote = null; From 79f24f9577aa273ad55d5de2b34f406a987a8d1c Mon Sep 17 00:00:00 2001 From: Jan Jakes Date: Thu, 9 Oct 2025 16:15:52 +0200 Subject: [PATCH 7/9] Interpret escape sequences only in strings that support it --- src/Import.php | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/Import.php b/src/Import.php index 7d1f25a..d0db509 100644 --- a/src/Import.php +++ b/src/Import.php @@ -89,18 +89,22 @@ public function parse_statements( $sql_file_path ) { for ( $i = 0; $i < $strlen; $i++ ) { $ch = $line[ $i ]; - // Count preceding backslashes. - $slashes = 0; - while ( $slashes < $i && '\\' === $line[ $i - $slashes - 1 ] ) { - ++$slashes; - } + // Handle escape sequences in single and double quoted strings. + // TODO: Support NO_BACKSLASH_ESCAPES SQL mode. + if ( "'" === $ch || '"' === $ch ) { + // Count preceding backslashes. + $slashes = 0; + while ( $slashes < $i && '\\' === $line[ $i - $slashes - 1 ] ) { + ++$slashes; + } - // Handle escaped characters. - // A characters is escaped only when the number of preceding backslashes - // is odd - "\" is an escape sequence, "\\" is an escaped backslash. - if ( 1 === $slashes % 2 ) { - $buffer .= $ch; - continue; + // Handle escaped characters. + // A characters is escaped only when the number of preceding backslashes + // is odd - "\" is an escape sequence, "\\" is an escaped backslash. + if ( 1 === $slashes % 2 ) { + $buffer .= $ch; + continue; + } } // Handle comments. From 5d7507f9e72322d80a7607ed59beb8fad296e76a Mon Sep 17 00:00:00 2001 From: Jan Jakes Date: Fri, 10 Oct 2025 08:20:08 +0200 Subject: [PATCH 8/9] Fix handling of empty lines, add test --- features/sqlite-import.feature | 20 ++++++++++++++++++++ src/Import.php | 8 ++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/features/sqlite-import.feature b/features/sqlite-import.feature index 1711f13..75a4365 100644 --- a/features/sqlite-import.feature +++ b/features/sqlite-import.feature @@ -151,3 +151,23 @@ Feature: WP-CLI SQLite Import Command """ And the SQLite database should contain a table named "a'strange`identifier\name" + + @require-sqlite + Scenario: Import a file with whitespace and empty lines + Given a SQL dump file named "test_import.sql" with content: + """ + + CREATE TABLE test_table (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT); + + + INSERT INTO test_table (name) VALUES ('Test Name'); + + """ + When I run `wp sqlite --enable-ast-driver import test_import.sql` + Then STDOUT should contain: + """ + Success: Imported from 'test_import.sql'. + """ + + And the SQLite database should contain a table named "test_table" + And the "test_table" should contain a row with name "Test Name" diff --git a/src/Import.php b/src/Import.php index d0db509..ca848e0 100644 --- a/src/Import.php +++ b/src/Import.php @@ -139,7 +139,10 @@ public function parse_statements( $sql_file_path ) { // Process statement end if ( ';' === $ch && null === $starting_quote ) { - yield trim( $buffer ); + $buffer = trim( $buffer ); + if ( ! empty( $buffer ) ) { + yield $buffer; + } $buffer = ''; } else { $buffer .= $ch; @@ -148,8 +151,9 @@ public function parse_statements( $sql_file_path ) { } // Handle any remaining buffer content + $buffer = trim( $buffer ); if ( ! empty( $buffer ) ) { - yield trim( $buffer ); + yield $buffer; } fclose( $handle ); From cfdd79c91ea9101c0d7c27b4864a2e1e864aa21a Mon Sep 17 00:00:00 2001 From: Antonio Sejas Date: Fri, 10 Oct 2025 11:32:07 +0100 Subject: [PATCH 9/9] Move encoding check to execute_statements --- src/Import.php | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/Import.php b/src/Import.php index ca848e0..de13bb0 100644 --- a/src/Import.php +++ b/src/Import.php @@ -51,10 +51,21 @@ public function run( $sql_file_path, $args ) { */ protected function execute_statements( $import_file ) { foreach ( $this->parse_statements( $import_file ) as $statement ) { - $result = $this->driver->query( $statement ); - if ( false === $result ) { - WP_CLI::warning( 'Could not execute statement: ' . $statement ); - echo $this->driver->get_error_message(); + try { + $this->driver->query( $statement ); + } catch ( Exception $e ) { + try { + // Try converting encoding and retry + $detected_encoding = mb_detect_encoding( $statement, mb_list_encodings(), true ); + if ( $detected_encoding && 'UTF-8' !== $detected_encoding ) { + $converted_statement = mb_convert_encoding( $statement, 'UTF-8', $detected_encoding ); + echo 'Converted ecoding for statement: ' . $converted_statement . PHP_EOL; + $this->driver->query( $converted_statement ); + } + } catch ( Exception $e ) { + WP_CLI::warning( 'Could not execute statement: ' . $statement ); + echo $e->getMessage(); + } } } } @@ -79,12 +90,6 @@ public function parse_statements( $sql_file_path ) { // phpcs:ignore while ( ( $line = fgets( $handle ) ) !== false ) { - // Detect and convert encoding to UTF-8 - $detected_encoding = mb_detect_encoding( $line, mb_list_encodings(), true ); - if ( $detected_encoding && 'UTF-8' !== $detected_encoding ) { - $line = mb_convert_encoding( $line, 'UTF-8', $detected_encoding ); - } - $strlen = strlen( $line ); for ( $i = 0; $i < $strlen; $i++ ) { $ch = $line[ $i ];