Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Added tests for token location.

Review URL: https://codereview.appspot.com/6118062

git-svn-id: http://angleproject.googlecode.com/svn/trunk@1058 736b8ea6-26fd-11df-bfd4-992fa37f6226
  • Loading branch information...
commit af4eb44a0749d33fcbbb01c230c0e52ff4ea6484 1 parent ce73a87
alokp@chromium.org authored
View
40 src/compiler/preprocessor/new/pp.l
@@ -27,18 +27,40 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
#include "Lexer.h"
#include "Token.h"
+// Token location (file and line) is encoded into a single int so that
+// we can use yylineno macro.
+
+const unsigned int kLocationLineSize = 16; // in bits.
+const unsigned int kLocationLineMask = (1 << kLocationLineSize) - 1;
+
+static int location(int file, int line)
+{
+ return (file << kLocationLineSize) | (line & kLocationLineMask);
+}
+
+static int locationFile(int loc)
+{
+ return loc >> kLocationLineSize;
+}
+
+static int locationLine(int loc)
+{
+ return loc & kLocationLineMask;
+}
+
typedef std::string YYSTYPE;
typedef pp::Token::Location YYLTYPE;
-#define YY_USER_ACTION \
- do { \
- yylloc->file = 0; \
- yylloc->line = yylineno; \
+#define YY_USER_ACTION \
+ do { \
+ yylloc->file = locationFile(yylineno); \
+ yylloc->line = locationLine(yylineno); \
} while(0);
// Suppress the default implementation of YY_INPUT which generated
// compiler warnings.
#define YY_INPUT
+
%}
%option nounput never-interactive
@@ -139,10 +161,15 @@ int ppwrap(yyscan_t scanner)
{
pp::Input* input = yyget_extra(scanner);
+ int file = -1;
// Delete the current buffer before switching to the next one.
YY_BUFFER_STATE buffer = static_cast<YY_BUFFER_STATE>(input->buffer);
if (buffer != NULL)
{
+ // Save the current token location before they get lost when the
+ // current input buffer is deleted.
+ file = locationFile(yyget_lineno(scanner));
+
yy_delete_buffer(buffer, scanner);
input->buffer = NULL;
}
@@ -157,7 +184,10 @@ int ppwrap(yyscan_t scanner)
else
buffer = yy_scan_bytes(input->string[index], length, scanner);
- // TODO(alokp): Increment token location.
+ // Increment file number and reset line number.
+ // This can be done only after we have created a new input buffer.
+ yyset_lineno(location(file + 1, 1), scanner);
+
input->index = index;
input->buffer = buffer;
return 0;
View
39 src/compiler/preprocessor/new/pp_lex.cpp
@@ -527,13 +527,34 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
#include "Lexer.h"
#include "Token.h"
+// Token location (file and line) is encoded into a single int so that
+// we can use yylineno macro.
+
+const unsigned int kLocationLineSize = 16; // in bits.
+const unsigned int kLocationLineMask = (1 << kLocationLineSize) - 1;
+
+static int location(int file, int line)
+{
+ return (file << kLocationLineSize) | (line & kLocationLineMask);
+}
+
+static int locationFile(int loc)
+{
+ return loc >> kLocationLineSize;
+}
+
+static int locationLine(int loc)
+{
+ return loc & kLocationLineMask;
+}
+
typedef std::string YYSTYPE;
typedef pp::Token::Location YYLTYPE;
-#define YY_USER_ACTION \
- do { \
- yylloc->file = 0; \
- yylloc->line = yylineno; \
+#define YY_USER_ACTION \
+ do { \
+ yylloc->file = locationFile(yylineno); \
+ yylloc->line = locationLine(yylineno); \
} while(0);
// Suppress the default implementation of YY_INPUT which generated
@@ -2170,10 +2191,15 @@ int ppwrap(yyscan_t scanner)
{
pp::Input* input = ppget_extra(scanner);
+ int file = -1;
// Delete the current buffer before switching to the next one.
YY_BUFFER_STATE buffer = static_cast<YY_BUFFER_STATE>(input->buffer);
if (buffer != NULL)
{
+ // Save the current token location before they get lost when the
+ // current input buffer is deleted.
+ file = locationFile(ppget_lineno(scanner));
+
pp_delete_buffer(buffer,scanner);
input->buffer = NULL;
}
@@ -2188,7 +2214,10 @@ int ppwrap(yyscan_t scanner)
else
buffer = pp_scan_bytes(input->string[index],length,scanner);
- // TODO(alokp): Increment token location.
+ // Increment file number and reset line number.
+ // This can be done only after we have created a new input buffer.
+ ppset_lineno(location(file + 1, 1),scanner);
+
input->index = index;
input->buffer = buffer;
return 0;
View
1  tests/build_tests.gyp
@@ -30,6 +30,7 @@
'../third_party/googletest/src/gtest_main.cc',
'preprocessor_tests/char_test.cpp',
'preprocessor_tests/identifier_test.cpp',
+ 'preprocessor_tests/location_test.cpp',
'preprocessor_tests/number_test.cpp',
'preprocessor_tests/token_test.cpp',
'preprocessor_tests/space_test.cpp',
View
107 tests/preprocessor_tests/location_test.cpp
@@ -0,0 +1,107 @@
+//
+// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+//
+
+#include "gtest/gtest.h"
+#include "Preprocessor.h"
+#include "Token.h"
+
+static void PreprocessAndVerifyLocation(int count,
+ const char* const string[],
+ const int length[],
+ pp::Token::Location location)
+{
+ pp::Token token;
+ pp::Preprocessor preprocessor;
+ ASSERT_TRUE(preprocessor.init(count, string, length));
+ EXPECT_EQ(pp::Token::IDENTIFIER, preprocessor.lex(&token));
+ EXPECT_EQ(pp::Token::IDENTIFIER, token.type);
+ EXPECT_STREQ("foo", token.value.c_str());
+
+ EXPECT_EQ(location.file, token.location.file);
+ EXPECT_EQ(location.line, token.location.line);
+}
+
+TEST(LocationTest, String0_Line1)
+{
+ const char* str = "foo";
+ pp::Token::Location loc;
+ loc.file = 0;
+ loc.line = 1;
+
+ SCOPED_TRACE("String0_Line1");
+ PreprocessAndVerifyLocation(1, &str, 0, loc);
+}
+
+TEST(LocationTest, String0_Line2)
+{
+ const char* str = "\nfoo";
+ pp::Token::Location loc;
+ loc.file = 0;
+ loc.line = 2;
+
+ SCOPED_TRACE("String0_Line2");
+ PreprocessAndVerifyLocation(1, &str, 0, loc);
+}
+
+TEST(LocationTest, String1_Line1)
+{
+ const char* const str[] = {"\n\n", "foo"};
+ pp::Token::Location loc;
+ loc.file = 1;
+ loc.line = 1;
+
+ SCOPED_TRACE("String1_Line1");
+ PreprocessAndVerifyLocation(2, str, 0, loc);
+}
+
+TEST(LocationTest, String1_Line2)
+{
+ const char* const str[] = {"\n\n", "\nfoo"};
+ pp::Token::Location loc;
+ loc.file = 1;
+ loc.line = 2;
+
+ SCOPED_TRACE("String1_Line2");
+ PreprocessAndVerifyLocation(2, str, 0, loc);
+}
+
+TEST(LocationTest, NewlineInsideCommentCounted)
+{
+ const char* str = "/*\n\n*/foo";
+ pp::Token::Location loc;
+ loc.file = 0;
+ loc.line = 3;
+
+ SCOPED_TRACE("NewlineInsideCommentCounted");
+ PreprocessAndVerifyLocation(1, &str, 0, loc);
+}
+
+// The location of a token straddling two or more strings is that of the
+// first character of the token.
+
+TEST(LocationTest, TokenStraddlingTwoStrings)
+{
+ const char* const str[] = {"f", "oo"};
+ pp::Token::Location loc;
+ loc.file = 0;
+ loc.line = 1;
+
+ SCOPED_TRACE("TokenStraddlingTwoStrings");
+ PreprocessAndVerifyLocation(2, str, 0, loc);
+}
+
+TEST(LocationTest, TokenStraddlingThreeStrings)
+{
+ const char* const str[] = {"f", "o", "o"};
+ pp::Token::Location loc;
+ loc.file = 0;
+ loc.line = 1;
+
+ SCOPED_TRACE("TokenStraddlingThreeStrings");
+ PreprocessAndVerifyLocation(3, str, 0, loc);
+}
+
+// TODO(alokp): Add tests for #line directives.
Please sign in to comment.
Something went wrong with that request. Please try again.