Fixed regression, caused by fix for CORE-4811 - error parsing pagesiz…

…e=N without spaces

src/common/Tokens.cpp

@@ -29,11 +29,34 @@
 
 #include "../common/StatusArg.h"
 
+namespace {
+
+Firebird::Tokens::Comment sqlComments[3] = {
+	{ "/*", "*/" },
+	{ "--", "\n" },
+	{ NULL, NULL }
+};
+const char* sqlSpaces = " \t\r\n";
+const char* sqlSeps = "!\"#%&'()*+,-./:;<=>?@[\\]^`{|}~";
+const char* sqlQuotes = "\"'";
+
+} // anonymous namespace
+
 namespace Firebird {
 
-Tokens::Tokens(FB_SIZE_T length, const char* toParse, const char* spaces, const char* quotes, const Comment* comments)
-	: tokens(getPool()), str(getPool())
+Tokens::Tokens()
+	: tokens(getPool()),
+	  str(getPool()),
+	  wsps(sqlSpaces),
+	  qs(sqlQuotes),
+	  comms(sqlComments),
+	  seps(sqlSeps)
+{ }
+
+void Tokens::parse(FB_SIZE_T length, const char* toParse)
 {
+	tokens.clear();
+
 	if (!length)
 		length = strlen(toParse);
 	str.assign(toParse, length);
@@ -46,9 +69,9 @@ Tokens::Tokens(FB_SIZE_T length, const char* toParse, const char* spaces, const
 	FB_SIZE_T p = 0;
 	while (p < str.length())
 	{
-		if (comments && !inStr)
+		if (comms && !inStr)
 		{
-			for (const Comment* comm = comments; comm->start; ++comm)
+			for (const Comment* comm = comms; comm->start; ++comm)
 			{
 				if (strncmp(comm->start, &str[p], strlen(comm->start)) == 0)
 				{
@@ -85,8 +108,7 @@ Tokens::Tokens(FB_SIZE_T length, const char* toParse, const char* spaces, const
 			continue;
 		}
 
-		bool space = spaces && strchr(spaces, c);
-		if (space)
+		if (wsps && strchr(wsps, c))
 		{
 			if (inToken)
 			{
@@ -98,7 +120,7 @@ Tokens::Tokens(FB_SIZE_T length, const char* toParse, const char* spaces, const
 			continue;
 		}
 
-		bool quote = quotes && strchr(quotes, c);
+		bool quote = qs && strchr(qs, c);
 		if (quote)
 		{
 			if (inToken)
@@ -110,16 +132,27 @@ Tokens::Tokens(FB_SIZE_T length, const char* toParse, const char* spaces, const
 			inStr = c;
 		}
 
-		if (!inToken)
+		if ((!quote) && seps && strchr(seps, c))
+		{
+			// close current token
+			if (inToken)
+			{
+				inToken->length = p - startp;
+				inToken = NULL;
+			}
+			// and create new one for one symbol
+			inToken = createToken(p, origin);
+			inToken->length = 1;
+			inToken = NULL;
+		}
+		else if (!inToken)
 		{
 			// start token
 			startp = p;
-			tokens.grow(tokens.getCount() + 1);
-			inToken = &tokens[tokens.getCount() - 1];
-			inToken->text = &str[p];
-			inToken->origin = origin;
+			inToken = createToken(p, origin);
 		}
 
+
 		// done with char
 		++p;
 		++origin;
@@ -130,6 +163,20 @@ Tokens::Tokens(FB_SIZE_T length, const char* toParse, const char* spaces, const
 
 	if (inToken)
 		inToken->length = p - startp;
+//#define DEBUG_TOKENS
+#ifdef DEBUG_TOKENS
+	for (unsigned dbg = 0; dbg < getCount(); ++dbg)
+		printf("%2d %.*s\n", dbg, tokens[dbg].length, tokens[dbg].text);
+#endif
+}
+
+Tokens::Tok* Tokens::createToken(FB_SIZE_T p, FB_SIZE_T origin)
+{
+	tokens.grow(tokens.getCount() + 1);
+	Tok* tok = &tokens[tokens.getCount() - 1];
+	tok->text = &str[p];
+	tok->origin = origin;
+	return tok;
 }
 
 void Tokens::error(const char* fmt, ...)
@@ -157,12 +204,4 @@ string Tokens::Tok::stripped() const
 	return rc;
 }
 
-Tokens::Comment sqlComments[3] = {
-	{ "/*", "*/" },
-	{ "--", "\n" },
-	{ NULL, NULL }
-};
-
-const char* sqlSpaces = " \t\r\n";
-
 } // namespace Firebird

src/common/Tokens.h

@@ -44,15 +44,37 @@ class Tokens : public AutoStorage
 		const char* stop;
 	};
 
-	Tokens(FB_SIZE_T length, const char* string, const char* spaces, const char* quotes, const Comment* comments);
-
 	struct Tok
 	{
 		const char* text;
 		FB_SIZE_T length, origin;
 		string stripped() const;
 	};
 
+	Tokens();
+
+	void spaces(const char* s)
+	{
+		wsps = s;
+	}
+
+	void quotes(const char* s)
+	{
+		qs = s;
+	}
+
+	void comments(const Comment* ptr)
+	{
+		comms = ptr;
+	}
+
+	void separators(const char* s)
+	{
+		seps = s;
+	}
+
+	void parse(FB_SIZE_T length, const char* string);
+
 	const Tok& operator[](FB_SIZE_T pos) const
 	{
 		return tokens[pos];
@@ -65,13 +87,16 @@ class Tokens : public AutoStorage
 
 private:
 	static void error(const char* fmt, ...);
+	Tok* createToken(FB_SIZE_T p, FB_SIZE_T origin);
 
 	HalfStaticArray<Tok, 16> tokens;
 	string str;
-};
 
-extern Tokens::Comment sqlComments[3];
-extern const char* sqlSpaces;
+	const char* wsps;
+	const char* qs;
+	const Comment* comms;
+	const char* seps;
+};
 
 } // namespace Firebird
 

src/isql/isql.epp

@@ -3613,10 +3613,12 @@ static processing_state create_db(const TEXT* statement, TEXT* d_name)
 	ISQL_disconnect_database(false);
 
 	// Parse statement to tokens
-	const char* quotes = isqlGlob.SQL_dialect == 1 ? "\"'" : "'";
+	const char* quotes = "\"'";
 	Firebird::string nlStatement(statement);
 	nlStatement += '\n';
-	Firebird::Tokens toks(0, nlStatement.c_str(), Firebird::sqlSpaces, quotes, Firebird::sqlComments);
+	Firebird::Tokens toks;
+	toks.quotes(quotes);
+	toks.parse(0, nlStatement.c_str());
 
 	const unsigned KEY_USER = 0;
 	const unsigned KEY_PASS = 1;

src/yvalve/preparse.cpp

@@ -170,7 +170,9 @@ bool PREPARSE_execute(CheckStatusWrapper* status, Why::YAttachment** ptrAtt,
 			Arg::Gds(isc_command_end_err).raise();
 		}
 
-		Tokens tks(stmt_length, stmt, sqlSpaces, quotes, sqlComments);
+		Tokens tks;
+		tks.quotes(quotes);
+		tks.parse(stmt_length, stmt);
 		unsigned pos = 0;
 
 		if (getToken(pos, tks) != pp_symbols[PP_CREATE].symbol)