Permalink
Browse files

Added new SqlTokenizer class to support parsing of non-trivial SELECT…

… statements. Modified SqlString.Parse() to ignore parameter references in SQL comments.
  • Loading branch information...
ggeurts authored and hazzik committed Jan 23, 2012
1 parent 3fb22c9 commit 082fe6169f3a05a90b55edc880b30ade01a1abbd
@@ -1091,6 +1091,7 @@
<Compile Include="ReadOnly\StudentDto.cs" />
<Compile Include="ReadOnly\TextHolder.cs" />
<Compile Include="ReadOnly\VersionedNode.cs" />
+ <Compile Include="SqlCommandTest\SqlTokenizerFixture.cs" />
<Compile Include="Stateless\Contact.cs" />
<Compile Include="Stateless\Country.cs" />
<Compile Include="Stateless\FetchingLazyCollections\LazyCollectionFetchTests.cs" />
@@ -0,0 +1,195 @@
+using NUnit.Framework;
+using NHibernate.SqlCommand;
+using NHibernate.SqlCommand.Parser;
+
+namespace NHibernate.Test.SqlCommandTest
+{
+ [TestFixture]
+ public class SqlTokenizerFixture
+ {
+ [Test]
+ public void TokenizeSimpleSelectStatement()
+ {
+ VerifyTokenizer("SELECT * FROM some_table WHERE key = ? ORDER BY some_field DESC",
+ Text("SELECT"), Whitespace(" "), Text("*"),
+ Whitespace(" "),
+ Text("FROM"), Whitespace(" "), Text("some_table"),
+ Whitespace(" "),
+ Text("WHERE"), Whitespace(" "), Text("key"), Whitespace(" "), Text("="), Whitespace(" "), Parameter(),
+ Whitespace(" "),
+ Text("ORDER"), Whitespace(" "), Text("BY"), Whitespace(" "), Text("some_field"), Whitespace(" "), Text("DESC"));
+ }
+
+ [Test]
+ public void TokenizeLineComments()
+ {
+ VerifyTokenizer("--", Comment("--"));
+ VerifyTokenizer("---", Comment("---"));
+ VerifyTokenizer("--\r", Comment("--"), Whitespace("\r"));
+ VerifyTokenizer("-- Any comment will do",
+ Comment("-- Any comment will do"));
+ VerifyTokenizer("-- Two comments\n--will do too",
+ Comment("-- Two comments"), Whitespace("\n"), Comment("--will do too"));
+ }
+
+ [Test]
+ public void TokenizeBlockComments()
+ {
+ VerifyTokenizer("/**/", Comment("/**/"));
+ VerifyTokenizer("/***/", Comment("/***/"));
+ VerifyTokenizer("/*/*/", Comment("/*/*/"));
+ VerifyTokenizer("/****/", Comment("/****/"));
+ VerifyTokenizer("//**//", Text("/"), Comment("/**/"), Text("/"));
+ VerifyTokenizer("/*\n*/", Comment("/*\n*/"));
+ VerifyTokenizer("/**/\n", Comment("/**/"), Whitespace("\n"));
+ VerifyTokenizer("/**/*", Comment("/**/"), Text("*"));
+ VerifyTokenizer("/**/*/", Comment("/**/"), Text("*/"));
+ VerifyTokenizer("*//**/", Text("*/"), Comment("/**/"));
+ VerifyTokenizer("SELECT/**/*", Text("SELECT"), Comment("/**/"), Text("*"));
+ }
+
+ [Test]
+ public void TokenizeBrackets()
+ {
+ VerifyTokenizer("()", BracketOpen(), BracketClose());
+ VerifyTokenizer("(())", BracketOpen(), BracketOpen(), BracketClose(), BracketClose());
+ VerifyTokenizer("()()", BracketOpen(), BracketClose(), BracketOpen(), BracketClose());
+ VerifyTokenizer("(\n)", BracketOpen(), Whitespace("\n"), BracketClose());
+ VerifyTokenizer("()--()", BracketOpen(), BracketClose(), Comment("--()"));
+ VerifyTokenizer("(--\n)", BracketOpen(), Comment("--"), Whitespace("\n"), BracketClose());
+ VerifyTokenizer("(SELECT)", BracketOpen(), Text("SELECT"), BracketClose());
+
+ VerifyTokenizer("SELECT (SELECT COUNT(*) FROM table), ?",
+ Text("SELECT"), Whitespace(" "),
+ BracketOpen(),
+ Text("SELECT"), Whitespace(" "), Text("COUNT"),
+ BracketOpen(),
+ Text("*"),
+ BracketClose(),
+ Whitespace(" "), Text("FROM"), Whitespace(" "), Text("table"),
+ BracketClose(),
+ Comma(), Whitespace(" "), Parameter());
+ }
+
+ [Test]
+ public void TokenizeQuotedString()
+ {
+ VerifyTokenizer("''", DelimitedText("''"));
+ VerifyTokenizer("''''", DelimitedText("''''"));
+ VerifyTokenizer("'string literal'", DelimitedText("'string literal'"));
+ VerifyTokenizer("'x''s value'", DelimitedText("'x''s value'"));
+ }
+
+ [Test]
+ public void TokenizeQuotedIdentifier()
+ {
+ VerifyTokenizer(@"""Identifier""", DelimitedText(@"""Identifier"""));
+ VerifyTokenizer(@"""""""Identifier""""""", DelimitedText(@"""""""Identifier"""""""));
+ VerifyTokenizer("[Identifier]", DelimitedText("[Identifier]"));
+ VerifyTokenizer("[[Identifier]]]", DelimitedText("[[Identifier]]]"));
+ }
+
+ [Test]
+ public void TokenizeParameters()
+ {
+ VerifyTokenizer("?", Parameter());
+ VerifyTokenizer("'?'", DelimitedText("'?'"));
+ VerifyTokenizer(@"""?""", DelimitedText(@"""?"""));
+ VerifyTokenizer("[?]", DelimitedText("[?]"));
+ VerifyTokenizer("--?", Comment("--?"));
+ VerifyTokenizer("/*?*/", Comment("/*?*/"));
+ VerifyTokenizer("(?)", BracketOpen(), Parameter(), BracketClose());
+ VerifyTokenizer("EXEC InsertSomething ?, ?",
+ Text("EXEC"), Whitespace(" "), Text("InsertSomething"),
+ Whitespace(" "), Parameter(), Comma(),
+ Whitespace(" "), Parameter());
+ }
+
+ private static void VerifyTokenizer(string sql, params ExpectedToken[] expectedTokens)
+ {
+ var sqlString = SqlString.Parse(sql);
+
+ int tokenIndex = 0;
+ int sqlIndex = 0;
+ foreach (var token in new SqlTokenizer(sqlString) { IgnoreComments = false, IgnoreWhitespace = false })
+ {
+ if (tokenIndex >= expectedTokens.Length)
+ {
+ Assert.Fail("Tokenizer returns more than expected '{0}' tokens. \nSQL: {1}\nLast Token: {2}({3})",
+ expectedTokens.Length, sql, token.TokenType, token.Value);
+ }
+
+ var expectedToken = expectedTokens[tokenIndex];
+ Assert.That(token.TokenType, Is.EqualTo(expectedToken.TokenType), "[Token #{0} in '{1}']TokenType", tokenIndex, sql);
+ Assert.That(token.Value, Is.EqualTo(expectedToken.Value), "[Token #{0} in {1}]Value", tokenIndex, sql);
+ Assert.That(token.SqlIndex, Is.EqualTo(sqlIndex), "[Token #{0} in {1}]SqlIndex", tokenIndex, sql);
+ Assert.That(token.Length, Is.EqualTo(expectedToken.Length), "[Token #{0} in {1}]Length", tokenIndex, sql);
+
+ tokenIndex++;
+ sqlIndex += expectedToken.Length;
+ }
+
+ if (tokenIndex < expectedTokens.Length)
+ {
+ Assert.Fail("Tokenizer returns less than expected '{0}' tokens.\nSQL: {1}", expectedTokens.Length, sql);
+ }
+ }
+
+ private static ExpectedToken Comma()
+ {
+ return new ExpectedToken(SqlTokenType.Comma, ",");
+ }
+
+ private static ExpectedToken Parameter()
+ {
+ return new ExpectedToken(SqlTokenType.Parameter, "?");
+ }
+
+ private static ExpectedToken BracketOpen()
+ {
+ return new ExpectedToken(SqlTokenType.BracketOpen, "(");
+ }
+
+ private static ExpectedToken BracketClose()
+ {
+ return new ExpectedToken(SqlTokenType.BracketClose, ")");
+ }
+
+ private static ExpectedToken DelimitedText(string text)
+ {
+ return new ExpectedToken(SqlTokenType.DelimitedText, text);
+ }
+
+ private static ExpectedToken Text(string text)
+ {
+ return new ExpectedToken(SqlTokenType.Text, text);
+ }
+
+ private static ExpectedToken Comment(string text)
+ {
+ return new ExpectedToken(SqlTokenType.Comment, text);
+ }
+
+ private static ExpectedToken Whitespace(string text)
+ {
+ return new ExpectedToken(SqlTokenType.Whitespace, text);
+ }
+
+ private class ExpectedToken
+ {
+ public SqlTokenType TokenType { get; private set; }
+ public string Value { get; private set; }
+
+ public ExpectedToken(SqlTokenType tokenType, string value)
+ {
+ this.TokenType = tokenType;
+ this.Value = value;
+ }
+
+ public int Length
+ {
+ get { return this.Value != null ? this.Value.Length : 0; }
+ }
+ }
+ }
+}
@@ -607,17 +607,21 @@
<Compile Include="SqlCommand\JoinFragment.cs" />
<Compile Include="SqlCommand\OracleJoinFragment.cs" />
<Compile Include="SqlCommand\Parameter.cs" />
+ <Compile Include="SqlCommand\Parser\SqlToken.cs" />
+ <Compile Include="SqlCommand\Parser\SqlTokenType.cs" />
<Compile Include="SqlCommand\QueryJoinFragment.cs" />
<Compile Include="SqlCommand\QuerySelect.cs" />
<Compile Include="SqlCommand\SelectFragment.cs" />
<Compile Include="SqlCommand\SqlBaseBuilder.cs" />
<Compile Include="SqlCommand\SqlDeleteBuilder.cs" />
<Compile Include="SqlCommand\SqlInsertBuilder.cs" />
<Compile Include="SqlCommand\SqlCommandImpl.cs" />
+ <Compile Include="SqlCommand\Parser\SqlParserUtils.cs" />
<Compile Include="SqlCommand\SqlSelectBuilder.cs" />
<Compile Include="SqlCommand\SqlSimpleSelectBuilder.cs" />
<Compile Include="SqlCommand\SqlString.cs" />
<Compile Include="SqlCommand\SqlStringBuilder.cs" />
+ <Compile Include="SqlCommand\Parser\SqlTokenizer.cs" />
<Compile Include="SqlCommand\SqlUpdateBuilder.cs" />
<Compile Include="SqlCommand\SqlStringHelper.cs" />
<Compile Include="SqlCommand\Template.cs" />
@@ -0,0 +1,99 @@
+using NHibernate.Exceptions;
+
+namespace NHibernate.SqlCommand.Parser
+{
+ internal static class SqlParserUtils
+ {
+ public static int ReadDelimitedText(string text, int maxOffset, int offset)
+ {
+ var startOffset = offset;
+ char quoteEndChar;
+
+ // Determine end delimiter
+ var quoteChar = text[offset++];
+ switch (quoteChar)
+ {
+ case '\'':
+ case '"':
+ quoteEndChar = quoteChar;
+ break;
+ case '[':
+ quoteEndChar = ']';
+ break;
+ default:
+ throw new SqlParseException(string.Format("Quoted text cannot start with '{0}' character", text[offset]));
+ }
+
+ // Find end delimiter, but ignore escaped end delimiters
+ while (offset < maxOffset)
+ {
+ if (text[offset++] == quoteEndChar)
+ {
+ if (offset >= maxOffset || text[offset] != quoteEndChar)
+ {
+ // Non-escaped delimiter char
+ return offset - startOffset;
+ }
+
+ // Escaped delimiter char
+ offset++;
+ }
+ }
+
+ throw new SqlParseException(string.Format("Cannot find terminating '{0}' character for quoted text.", quoteEndChar));
+ }
+
+ public static int ReadLineComment(string text, int maxOffset, int offset)
+ {
+ var startOffset = offset;
+
+ offset += 2;
+ for (; offset < maxOffset; offset++)
+ {
+ switch (text[offset])
+ {
+ case '\r':
+ case '\n':
+ return offset - startOffset;
+ }
+ }
+
+ return offset - startOffset;
+ }
+
+ public static int ReadMultilineComment(string text, int maxOffset, int offset)
+ {
+ var startOffset = offset;
+ offset += 2;
+
+ var prevChar = '\0';
+ for (; offset < maxOffset; offset++)
+ {
+ var ch = text[offset];
+ if (ch == '/' && prevChar == '*')
+ {
+ return offset + 1 - startOffset;
+ }
+
+ prevChar = ch;
+ }
+
+ throw new SqlParseException(string.Format("Cannot find terminating '*/' string for multiline comment."));
+ }
+
+ public static int ReadWhitespace(string text, int maxOffset, int offset)
+ {
+ var startOffset = offset;
+
+ offset++;
+ while (offset < maxOffset)
+ {
+ if (!char.IsWhiteSpace(text[offset])) break;
+ offset++;
+ }
+
+ var result = offset - startOffset;
+ return result;
+ }
+ }
+}
Oops, something went wrong.

0 comments on commit 082fe61

Please sign in to comment.