@@ -131,9 +131,6 @@ PK_EXPORT const char *PkTokenizerGetTokenCString(PkTokenizer *tokenizer); PK_EXPORT -bool PkTokenizerNewlineSeenWhileReadingToken(PkTokenizer *tokenizer); - -PK_EXPORT PkString *PkTokenizerGetSourceName(PkTokenizer *tokenizer); PK_EXPORT include/pekoe/tokenizer.h @@ -216,9 +216,6 @@ struct _PkTokenizer { /** end of script */ bool eos; - /** newline seen while reading token */ - bool newlineSeen; - /** lineno, columnno */ size_t lineno, columnno; /** the source position of last read token */ @@ -455,8 +452,6 @@ int PkTokenizerNextToken(PkTokenizer *it) { int c = it->eos ? EOF : nextc(it); int token = kPkTokenUnrecognized; - it->newlineSeen = false; - RETRY: /* @@ -669,7 +664,6 @@ NUMBER_END: /* Skip trailing whitespaces and newlines until newline seen */ while (isWhitespaceOrNewlineCharacter(c)) { if (c == '\n' || c == '\r') { - it->newlineSeen = true; return token; } c = nextc(it); @@ -682,10 +676,6 @@ const char *PkTokenizerGetTokenCString(PkTokenizer *tokenizer) { return (const char *) tokenizer->tokenBegin; } -bool PkTokenizerNewlineSeenWhileReadingToken(PkTokenizer *tokenizer) { - return tokenizer->newlineSeen; -} - PkString *PkTokenizerGetSourceName(PkTokenizer *tokenizer) { return tokenizer->name; } @@ -733,7 +723,6 @@ static PkRef tokenizer_init(PkRef object) { tokenizer->previousRead = EOF; tokenizer->eos = false; - tokenizer->newlineSeen = false; tokenizer->lineno = 0; tokenizer->columnno = 0; src/tokenizer.c @@ -257,24 +257,6 @@ static void test_token_cstring(void) { PkRelease(tokenizer); } -static void test_newline_read(void) { - PkTokenizer *tokenizer = PkTokenizerCreateWithCStringNoCopy("1 + \n2 -\n 3"); - - assert( !PkTokenizerNewlineSeenWhileReadingToken(tokenizer) ); - tokenizer = assertNextToken(tokenizer, kPkTokenNumber, "1"); - assert( !PkTokenizerNewlineSeenWhileReadingToken(tokenizer) ); - tokenizer = assertNextToken(tokenizer, '+', NULL); - assert( PkTokenizerNewlineSeenWhileReadingToken(tokenizer) ); - tokenizer = assertNextToken(tokenizer, kPkTokenNumber, "2"); - assert( !PkTokenizerNewlineSeenWhileReadingToken(tokenizer) ); - tokenizer = assertNextToken(tokenizer, '-', NULL); - assert( PkTokenizerNewlineSeenWhileReadingToken(tokenizer) ); - tokenizer = assertNextToken(tokenizer, kPkTokenNumber, "3"); - assert( !PkTokenizerNewlineSeenWhileReadingToken(tokenizer) ); - - PkRelease(tokenizer); -} - static void test_line_comment(void) { assertTokens("// comment \nx;", kPkTokenIdentifier, @@ -413,7 +395,6 @@ void PkTokenizerUnitTest(void) { test_single_token_characters(); test_simple_expr(); test_token_cstring(); - test_newline_read(); test_token_name(); test_lineno_columnno(); test_source_name(); test/tokenizer.c 1c7c007c351978eba12befcf446c7f82ba8b07fa Takanori Ishikawa takanori.ishikawa@gmail.com http://github.com/ishikawa/pekoe/commit/65d59a0fcc96e58cda0ad32e18402766bee7eb5f 65d59a0fcc96e58cda0ad32e18402766bee7eb5f 2009-11-11T05:54:09-08:00 2009-11-11T05:54:09-08:00 Removed unused function: PkTokenizerNewlineSeenWhileReadingToken() 1fdcfc39d94a55e94ae6dd48977d8c3ffe506a30 Takanori Ishikawa takanori.ishikawa@gmail.com