Formula engine: Add string escapes mechanism

This builds on the string substitution syntax:
- [(] means an open bracket
- [)] means a close bracket
- ['] means a single quote / apostrophe

The tokenizer is now aware of string substitutions, making this possible
as well as the nesting of strings within string substitutions.

src/formula.cpp

@@ -569,20 +569,44 @@ class string_expression : public formula_expression {
 		str_(),
 		subs_()
 	{
-		std::string::iterator i;
-		while((i = std::find(str.begin(), str.end(), '[')) != str.end()) {
-			std::string::iterator j = std::find(i, str.end(), ']');
+		std::string::iterator i = str.begin();
+		while((i = std::find(i, str.end(), '[')) != str.end()) {
+			int bracket_depth = 0;
+			std::string::iterator j = i + 1;
+			while(j != str.end() && (bracket_depth > 0 || *j != ']')) {
+				if(*j == '[') {
+					bracket_depth++;
+				} else if(*j == ']' && bracket_depth > 0) {
+					bracket_depth--;
+				}
+				++j;
+			}
 			if(j == str.end()) {
 				break;
 			}
 
 			const std::string formula_str(i+1, j);
 			const int pos = i - str.begin();
-			str.erase(i, j+1);
+			if(j - i == 2 && (i[1] == '(' || i[1] == '\'' || i[1] == ')')) {
+				// Bracket contained nothing but a quote or parenthesis.
+				// This means it was intended as a literal quote or square bracket.
+				i = str.erase(i);
+				if(*i == '(') *i = '[';
+				else if(*i == ')') *i = ']';
+				i = str.erase(i + 1);
+				continue;
+			} else {
+				i = str.erase(i, j+1);
+			}
 
 			substitution sub;
 			sub.pos = pos;
-			sub.calculation.reset(new formula(formula_str));
+			try {
+				sub.calculation.reset(new formula(formula_str));
+			} catch(formula_error& e) {
+				e.filename += " - string substitution";
+				throw e;
+			}
 			subs_.push_back(sub);
 		}
 

src/formula_tokenizer.cpp

@@ -35,7 +35,7 @@ void raise_exception(iterator& i1, iterator i2, std::string str) {
 
 }
 
-token get_token(iterator& i1, iterator i2) {
+token get_token(iterator& i1, const iterator i2) {
 
 	iterator it = i1;
 	if( *i1 >= 'A' ) {
@@ -109,6 +109,7 @@ token get_token(iterator& i1, iterator i2) {
 
 			//unused characters in this range:
 			// \ ` { | }
+			// Note: {} should never be used since they play poorly with WML preprocessor
 		}
 	} else {
 		//limit search to the lower-half of the ASCII table
@@ -189,6 +190,8 @@ token get_token(iterator& i1, iterator i2) {
 		// , . .+ .- .* ./ .. ( ) ' # + - -> * / % !=
 		//unused characters:
 		// ! " $ &
+		// ! is used only as part of !=
+		// Note: " should never be used since it plays poorly with WML
 		} else if ( *i1 == ',' ) {
 			return token( it, ++i1, TOKEN_COMMA);
 
@@ -211,9 +214,18 @@ token get_token(iterator& i1, iterator i2) {
 			return token( it, ++i1, TOKEN_RPARENS);
 
 		} else if ( *i1 == '\'' ) {
+			int bracket_depth = 0;
 			++i1;
-			while( i1 != i2 && *i1 != '\'' )
+			while (i1 != i2) {
+				if (*i1 == '[') {
+					bracket_depth++;
+				} else if(bracket_depth > 0 && *i1 == ']') {
+					bracket_depth--;
+				} else if(bracket_depth == 0 && *i1 == '\'') {
+					break;
+				}
 				++i1;
+			}
 
 			if( i1 != i2 ) {
 				return token( it, ++i1, TOKEN_STRING_LITERAL );

src/variant.cpp

@@ -1084,7 +1084,22 @@ void variant::serialize_to_string(std::string& str) const
 	}
 	case TYPE_STRING:
 		str += "'";
-		str += string_->str;
+		for(std::string::iterator it = string_->str.begin(); it < string_->str.end(); ++it) {
+			switch(*it) {
+			case '\'':
+				str += "[']";
+				break;
+			case '[':
+				str += "[(]";
+				break;
+			case ']':
+				str += "[)]";
+				break;
+			default:
+				str += *it;
+				break;
+			}
+		}
 		str += "'";
 		break;
 	default: