Skip to content

Commit 83be865

Browse files
sejassejas
authored
Fix import query parsing #14
* Adds multiple fixes and improvements that make the query recognition in the input dump almost complete. * The only known edge cases that aren't supported yet are the usage of NO_BACKSLASH_ESCAPES SQL mode, and the MySQL DELIMITER ... statement. We likely won't implement those in the current logic—it's better to make the new query tokenizer fully support streaming to handle all edge cases correctly.
2 parents 92c3558 + 8556960 commit 83be865

File tree

3 files changed

+199
-40
lines changed

3 files changed

+199
-40
lines changed

‎features/bootstrap/SQLiteFeatureContext.php‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ public function aSqlDumpFileNamedWithContent( $filename, PyStringNode $content )
2525
*/
2626
public function theSqliteDatabaseShouldContainATableNamed( $table_name ) {
2727
$this->connectToDatabase();
28-
$result = $this->db->query( "SELECT name FROM sqlite_master WHERE type='table' AND name='$table_name'" );
28+
$result = $this->db->query( "SELECT name FROM sqlite_master WHERE type='table' AND name='" . $this->db->escapeString( $table_name ) . "'" );
2929
$row = $result->fetchArray();
3030
if ( ! $row ) {
3131
throw new Exception( "Table '$table_name' not found in the database." );
@@ -34,10 +34,11 @@ public function theSqliteDatabaseShouldContainATableNamed( $table_name ) {
3434

3535
/**
3636
* @Then /^the "([^"]*)" should contain a row with name "([^"]*)"$/
37+
* @Then /^the "([^"]*)" should contain a row with name:$/
3738
*/
3839
public function theTableShouldContainARowWithName( $table_name, $name ) {
3940
$this->connectToDatabase();
40-
$result = $this->db->query( "SELECT * FROM $table_name WHERE name='$name'" );
41+
$result = $this->db->query( "SELECT * FROM $table_name WHERE name='" . $this->db->escapeString( $name ) . "'" );
4142
$row = $result->fetchArray();
4243
if ( ! $row ) {
4344
throw new Exception( "Row with name '$name' not found in table '$table_name'." );

‎features/sqlite-import.feature‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,131 @@ Feature: WP-CLI SQLite Import Command
4343
Success: Imported from 'STDIN'.
4444
"""
4545
And the SQLite database should contain the imported data
46+
47+
@require-sqlite
48+
Scenario: Import a file with escape sequences
49+
Given a SQL dump file named "test_import.sql" with content:
50+
"""
51+
SET sql_mode='NO_BACKSLASH_ESCAPES';
52+
CREATE TABLE test_table (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT);
53+
INSERT INTO test_table (name) VALUES ('Test that escaping a backslash \\ works');
54+
INSERT INTO test_table (name) VALUES ('Test that escaping multiple backslashes \\\\\\ works');
55+
INSERT INTO test_table (name) VALUES ('Test that escaping a character \a works');
56+
INSERT INTO test_table (name) VALUES ('Test that escaping a backslash followed by a character \\a works');
57+
INSERT INTO test_table (name) VALUES ('Test that escaping a backslash and a character \\\a works');
58+
"""
59+
When I run `wp sqlite --enable-ast-driver import test_import.sql`
60+
Then STDOUT should contain:
61+
"""
62+
Success: Imported from 'test_import.sql'.
63+
"""
64+
And the SQLite database should contain a table named "test_table"
65+
And the "test_table" should contain a row with name "Test that escaping a backslash \\ works"
66+
And the "test_table" should contain a row with name "Test that escaping multiple backslashes \\\\\\ works"
67+
And the "test_table" should contain a row with name "Test that escaping a character \a works"
68+
And the "test_table" should contain a row with name "Test that escaping a backslash followed by a character \\a works"
69+
And the "test_table" should contain a row with name "Test that escaping a backslash and a character \\\a works"
70+
71+
@require-sqlite
72+
Scenario: Import a file with newlines in strings
73+
Given a SQL dump file named "test_import.sql" with content:
74+
"""
75+
CREATE TABLE test_table (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT);
76+
INSERT INTO test_table (name) VALUES ('Test that a string containing
77+
a newline character and some whitespace works');
78+
"""
79+
When I run `wp sqlite --enable-ast-driver import test_import.sql`
80+
Then STDOUT should contain:
81+
"""
82+
Success: Imported from 'test_import.sql'.
83+
"""
84+
And the SQLite database should contain a table named "test_table"
85+
And the "test_table" should contain a row with name:
86+
"""
87+
Test that a string containing
88+
a newline character and some whitespace works
89+
"""
90+
91+
@require-sqlite
92+
Scenario: Import a file with comments
93+
Given a SQL dump file named "test_import.sql" with content:
94+
"""
95+
CREATE TABLE test_table (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT);
96+
-- This is an inline comment.
97+
# This is an inline comment.
98+
INSERT INTO test_table (name) VALUES ('one'); -- This is an inline comment.
99+
/* This is a block comment */
100+
INSERT INTO test_table (name) VALUES ('two'); /* This
101+
is a block comment
102+
on multiple lines */ INSERT INTO test_table (name) VALUES ('three');
103+
INSERT INTO test_table (name) VALUES ('fo -- this looks like a comment ur');
104+
INSERT INTO test_table (name) VALUES ('fi/* this looks like a comment */ve');
105+
"""
106+
When I run `wp sqlite --enable-ast-driver import test_import.sql`
107+
Then STDOUT should contain:
108+
"""
109+
Success: Imported from 'test_import.sql'.
110+
"""
111+
And the SQLite database should contain a table named "test_table"
112+
And the "test_table" should contain a row with name "one"
113+
And the "test_table" should contain a row with name "two"
114+
And the "test_table" should contain a row with name "three"
115+
And the "test_table" should contain a row with name "fo -- this looks like a comment ur"
116+
And the "test_table" should contain a row with name "fi/* this looks like a comment */ve"
117+
118+
@require-sqlite
119+
Scenario: Import a file quoted strings
120+
Given a SQL dump file named "test_import.sql" with content:
121+
"""
122+
CREATE TABLE test_table (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT);
123+
INSERT INTO test_table (name) VALUES ('a single-quoted string with \' '' some " tricky ` chars');
124+
INSERT INTO test_table (name) VALUES ("a double-quoted string with ' some \" "" tricky ` chars");
125+
"""
126+
When I run `wp sqlite --enable-ast-driver import test_import.sql`
127+
Then STDOUT should contain:
128+
"""
129+
Success: Imported from 'test_import.sql'.
130+
"""
131+
And the SQLite database should contain a table named "test_table"
132+
And the "test_table" should contain a row with name:
133+
"""
134+
a single-quoted string with ' ' some " tricky ` chars
135+
"""
136+
And the "test_table" should contain a row with name:
137+
"""
138+
a double-quoted string with ' some " " tricky ` chars
139+
"""
140+
141+
@require-sqlite
142+
Scenario: Import a file backtick-quoted identifiers
143+
Given a SQL dump file named "test_import.sql" with content:
144+
"""
145+
CREATE TABLE `a'strange``identifier\\name` (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT);
146+
"""
147+
When I run `wp sqlite --enable-ast-driver import test_import.sql`
148+
Then STDOUT should contain:
149+
"""
150+
Success: Imported from 'test_import.sql'.
151+
"""
152+
153+
And the SQLite database should contain a table named "a'strange`identifier\name"
154+
155+
@require-sqlite
156+
Scenario: Import a file with whitespace and empty lines
157+
Given a SQL dump file named "test_import.sql" with content:
158+
"""
159+
160+
CREATE TABLE test_table (id INTEGER PRIMARY KEY AUTO_INCREMENT, name TEXT);
161+
162+
163+
INSERT INTO test_table (name) VALUES ('Test Name');
164+
165+
"""
166+
When I run `wp sqlite --enable-ast-driver import test_import.sql`
167+
Then STDOUT should contain:
168+
"""
169+
Success: Imported from 'test_import.sql'.
170+
"""
171+
172+
And the SQLite database should contain a table named "test_table"
173+
And the "test_table" should contain a row with name "Test Name"

‎src/Import.php‎

Lines changed: 68 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,21 @@ public function run( $sql_file_path, $args ) {
5151
*/
5252
protected function execute_statements( $import_file ) {
5353
foreach ( $this->parse_statements( $import_file ) as $statement ) {
54-
$result = $this->driver->query( $statement );
55-
if ( false === $result ) {
56-
WP_CLI::warning( 'Could not execute statement: ' . $statement );
57-
echo $this->driver->get_error_message();
54+
try {
55+
$this->driver->query( $statement );
56+
} catch ( Exception $e ) {
57+
try {
58+
// Try converting encoding and retry
59+
$detected_encoding = mb_detect_encoding( $statement, mb_list_encodings(), true );
60+
if ( $detected_encoding && 'UTF-8' !== $detected_encoding ) {
61+
$converted_statement = mb_convert_encoding( $statement, 'UTF-8', $detected_encoding );
62+
echo 'Converted ecoding for statement: ' . $converted_statement . PHP_EOL;
63+
$this->driver->query( $converted_statement );
64+
}
65+
} catch ( Exception $e ) {
66+
WP_CLI::warning( 'Could not execute statement: ' . $statement );
67+
echo $e->getMessage();
68+
}
5869
}
5970
}
6071
}
@@ -73,52 +84,70 @@ public function parse_statements( $sql_file_path ) {
7384
WP_CLI::error( "Unable to open file: $sql_file_path" );
7485
}
7586

76-
$single_quotes = 0;
77-
$double_quotes = 0;
78-
$in_comment = false;
79-
$buffer = '';
87+
$starting_quote = null;
88+
$in_comment = false;
89+
$buffer = '';
8090

8191
// phpcs:ignore
8292
while ( ( $line = fgets( $handle ) ) !== false ) {
83-
$line = trim( $line );
84-
85-
// Skip empty lines and comments
86-
if ( empty( $line ) || strpos( $line, '--' ) === 0 || strpos( $line, '#' ) === 0 ) {
87-
continue;
88-
}
89-
90-
// Handle multi-line comments
91-
if ( ! $in_comment && strpos( $line, '/*' ) === 0 ) {
92-
$in_comment = true;
93-
}
94-
if ( $in_comment ) {
95-
if ( strpos( $line, '*/' ) !== false ) {
96-
$in_comment = false;
97-
}
98-
continue;
99-
}
100-
10193
$strlen = strlen( $line );
10294
for ( $i = 0; $i < $strlen; $i++ ) {
10395
$ch = $line[ $i ];
10496

105-
// Handle escaped characters
106-
if ( $i > 0 && '\\' === $line[ $i - 1 ] ) {
107-
$buffer .= $ch;
108-
continue;
97+
// Handle escape sequences in single and double quoted strings.
98+
// TODO: Support NO_BACKSLASH_ESCAPES SQL mode.
99+
if ( "'" === $ch || '"' === $ch ) {
100+
// Count preceding backslashes.
101+
$slashes = 0;
102+
while ( $slashes < $i && '\\' === $line[ $i - $slashes - 1 ] ) {
103+
++$slashes;
104+
}
105+
106+
// Handle escaped characters.
107+
// A characters is escaped only when the number of preceding backslashes
108+
// is odd - "\" is an escape sequence, "\\" is an escaped backslash.
109+
if ( 1 === $slashes % 2 ) {
110+
$buffer .= $ch;
111+
continue;
112+
}
109113
}
110114

111-
// Handle quotes
112-
if ( "'" === $ch && 0 === $double_quotes ) {
113-
$single_quotes = 1 - $single_quotes;
115+
// Handle comments.
116+
if ( null === $starting_quote ) {
117+
$prev_ch = isset( $line[ $i - 1 ] ) ? $line[ $i - 1 ] : null;
118+
$next_ch = isset( $line[ $i + 1 ] ) ? $line[ $i + 1 ] : null;
119+
120+
// Skip inline comments.
121+
if ( ( '-' === $ch && '-' === $next_ch ) || '#' === $ch ) {
122+
break; // Stop for the current line.
123+
}
124+
125+
// Skip multi-line comments.
126+
if ( ! $in_comment && '/' === $ch && '*' === $next_ch ) {
127+
$in_comment = true;
128+
continue;
129+
}
130+
if ( $in_comment ) {
131+
if ( '*' === $prev_ch && '/' === $ch ) {
132+
$in_comment = false;
133+
}
134+
continue;
135+
}
114136
}
115-
if ( '"' === $ch && 0 === $single_quotes ) {
116-
$double_quotes = 1 - $double_quotes;
137+
138+
// Handle quotes
139+
if ( null === $starting_quote && ( "'" === $ch || '"' === $ch || '`' === $ch ) ) {
140+
$starting_quote = $ch;
141+
} elseif ( null !== $starting_quote && $ch === $starting_quote ) {
142+
$starting_quote = null;
117143
}
118144

119145
// Process statement end
120-
if ( ';' === $ch && 0 === $single_quotes && 0 === $double_quotes ) {
121-
yield trim( $buffer );
146+
if ( ';' === $ch && null === $starting_quote ) {
147+
$buffer = trim( $buffer );
148+
if ( ! empty( $buffer ) ) {
149+
yield $buffer;
150+
}
122151
$buffer = '';
123152
} else {
124153
$buffer .= $ch;
@@ -127,8 +156,9 @@ public function parse_statements( $sql_file_path ) {
127156
}
128157

129158
// Handle any remaining buffer content
159+
$buffer = trim( $buffer );
130160
if ( ! empty( $buffer ) ) {
131-
yield trim( $buffer );
161+
yield $buffer;
132162
}
133163

134164
fclose( $handle );

0 commit comments

Comments
 (0)