Permalink
Browse files

Fix #720 (String length for strings like this "\x61" is calculated w…

  • Loading branch information...
aggro80 committed Sep 27, 2009
1 parent 7e2208b commit d1f3953ccebdcd0ce77a4ca55e643de9e7bba60a
Showing with 105 additions and 2 deletions.
  1. +8 −1 src/mathlib.cpp
  2. +7 −0 src/mathlib.h
  3. +59 −1 src/tokenize.cpp
  4. +8 −0 src/tokenize.h
  5. +23 −0 test/testsimplifytokens.cpp
View
@@ -136,7 +136,7 @@ bool MathLib::isInt(const std::string & s)
// check octal notation
else if (Mode == eOctal)
{
- while (s[n] == '0' || s[n] == '1' || s[n] == '2' || s[n] == '3' || s[n] == '4' || s[n] == '5' || s[n] == '6' || s[n] == '7')
+ while (isOctalDigit(s[n]))
++n;
}
else if (Mode == eDefault)
@@ -254,4 +254,11 @@ bool MathLib::isGreater(const std::string &first, const std::string &second)
return toDoubleNumber(first) > toDoubleNumber(second);
}
+bool MathLib::isOctalDigit(char c)
+{
+ if (c == '0' || c == '1' || c == '2' || c == '3' || c == '4' || c == '5' || c == '6' || c == '7')
+ return true;
+
+ return false;
+}
View
@@ -48,6 +48,13 @@ class MathLib
static std::string tan(const std::string & tok);
static std::string abs(const std::string & tok);
static bool isGreater(const std::string & first, const std::string & second);
+
+ /**
+ * Return true if given character is 0,1,2,3,4,5,6 or 7.
+ * @param c The character to check
+ * @return true if given character is octal digit.
+ */
+ static bool isOctalDigit(char c);
};
/// @}
View
@@ -1534,8 +1534,14 @@ void Tokenizer::simplifyTokenList()
// Combine strings
for (Token *tok = _tokens; tok; tok = tok->next())
{
- while (tok->str()[0] == '"' && tok->next() && tok->next()->str()[0] == '"')
+ if (tok->str()[0] != '"')
+ continue;
+
+ tok->str(simplifyString(tok->str()));
+ while (tok->next() && tok->next()->str()[0] == '"')
{
+ tok->next()->str(simplifyString(tok->next()->str()));
+
// Two strings after each other, combine them
tok->concatStr(tok->next()->str());
tok->deleteNext();
@@ -4024,3 +4030,55 @@ bool Tokenizer::validate() const
return true;
}
+
+std::string Tokenizer::simplifyString(const std::string &source)
+{
+ std::string str = source;
+ bool escaped = false;
+ for (std::string::size_type i = 0; i + 2 < str.size(); i++)
+ {
+ if (!escaped)
+ {
+ if (str[i] == '\\')
+ escaped = true;
+
+ continue;
+ }
+
+ if (str[i] == 'x')
+ {
+ // Hex value
+ if (str[i+1] == '0' && str[i+2] == '0')
+ str.replace(i, 3, "0");
+ else
+ {
+ // We will replace all other character as 'a'
+ // If that causes problems in the future, this can
+ // be improved. But for now, this should be OK.
+ --i;
+ str.replace(i, 4, "a");
+ }
+ }
+ else if (MathLib::isOctalDigit(str[i]))
+ {
+ if (MathLib::isOctalDigit(str[i+1]) &&
+ MathLib::isOctalDigit(str[i+2]))
+ {
+ if (str[i+1] == '0' && str[i+2] == '0')
+ str.replace(i, 3, "0");
+ else
+ {
+ // We will replace all other character as 'a'
+ // If that causes problems in the future, this can
+ // be improved. But for now, this should be OK.
+ --i;
+ str.replace(i, 4, "a");
+ }
+ }
+ }
+
+ escaped = false;
+ }
+
+ return str;
+}
View
@@ -284,6 +284,14 @@ class Tokenizer
*/
void simplifyMathFunctions();
+ /**
+ * Modify strings in the token list by replacing hex and oct
+ * values. E.g. "\x61" -> "a" and "\000" -> "\0"
+ * @param source The string to be modified, e.g. "\x61"
+ * @return Modified string, e.g. "a"
+ */
+ std::string simplifyString(const std::string &source);
+
void insertTokens(Token *dest, const Token *src, unsigned int n);
/**
@@ -118,6 +118,7 @@ class TestSimplifyTokens : public TestFixture
TEST_CASE(argumentsWithSameName)
TEST_CASE(simplifyAtol)
+ TEST_CASE(simplifyHexInString)
}
std::string tok(const char code[])
@@ -1555,6 +1556,28 @@ class TestSimplifyTokens : public TestFixture
ASSERT_EQUALS("a = 0 ;", tok("a = std::atol(\"0\");"));
ASSERT_EQUALS("a = 10 ;", tok("a = atol(\"0xa\");"));
}
+
+ void simplifyHexInString()
+ {
+ ASSERT_EQUALS("\"a\"", tok("\"\\x61\""));
+ ASSERT_EQUALS("\"a\"", tok("\"\\141\""));
+
+ ASSERT_EQUALS("\"\\0\"", tok("\"\\x00\""));
+ ASSERT_EQUALS("\"\\0\"", tok("\"\\000\""));
+
+ ASSERT_EQUALS("\"\\nhello\"", tok("\"\\nhello\""));
+
+ ASSERT_EQUALS("\"aaa\"", tok("\"\\x61\\x61\\x61\""));
+ ASSERT_EQUALS("\"aaa\"", tok("\"\\141\\141\\141\""));
+
+ ASSERT_EQUALS("\"\\\\x61\"", tok("\"\\\\x61\""));
+
+ // These tests can fail, if other characters are handled
+ // more correctly. But fow now all non null characters should
+ // become 'a'
+ ASSERT_EQUALS("\"a\"", tok("\"\\x62\""));
+ ASSERT_EQUALS("\"a\"", tok("\"\\177\""));
+ }
};
REGISTER_TEST(TestSimplifyTokens)

0 comments on commit d1f3953

Please sign in to comment.