Find file
Fetching contributors…
Cannot retrieve contributors at this time
914 lines (822 sloc) 28.9 KB
/*
* OpenBOR - http://www.LavaLit.com
* -----------------------------------------------------------------------
* Licensed under the BSD license, see LICENSE in OpenBOR root for details.
*
* Copyright (c) 2004 - 2010 OpenBOR Team
*/
/**
* This is derived from Lexer.c in scriptlib, but is modified to be used by the
* script preprocessor. Although the two script lexers share much of their codebase,
* there are some significant differences - for example, the preprocessor lexer
* can detect preprocessor tokens (#include, #define) and doesn't eat whitespace.
*
* View this file as UTF-8 in order to read utunnels' original comments in Chinese,
* which look like random garbage otherwise. Above each Chinese comment, though,
* is a translation into English using Google Translate. The original Lexer.c
* was encoded with the GB18030 (Simplified Chinese) encoding.
*
* @author Plombo
* @date 15 October 2010
*/
#include "pp_lexer.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
//MACROS
/******************************************************************************
* CONSUMECHARACTER -- This macro inserts code to remove a character from the
* input stream and add it to the current token buffer.
******************************************************************************/
#define CONSUMECHARACTER \
plexer->theTokenSource[strlen(plexer->theTokenSource)] = *(plexer->pcurChar);\
plexer->theTokenSource[strlen(plexer->theTokenSource)] = '\0';\
plexer->pcurChar++; \
plexer->theTextPosition.col++; \
plexer->offset++;
/******************************************************************************
* MAKETOKEN(x) -- This macro inserts code to create a new CToken object of
* type x, using the current token position, and source.
******************************************************************************/
#define MAKETOKEN(x) \
pp_token_Init(theNextToken, x, plexer->theTokenSource, plexer->theTokenPosition, \
plexer->tokOffset);
/******************************************************************************
* SKIPCHARACTER -- Skip a character, not to plexer->theTokenSource in.
*
* Original comment: 跳过一个字符,不加入到plexer->theTokenSource中。
* 2007-1-22
******************************************************************************/
#define SKIPCHARACTER \
plexer->pcurChar++; \
plexer->theTextPosition.col++; \
plexer->offset++;
/******************************************************************************
* CONSUMEESCAPE -- Read the next escape character, and modify on plexer->theTokenSource.
* The escape character represents the character, reference CONSUMECHARACTER macro.
* Currently supported escape sequences are: \r \n \t \s \" \' \\
*
* Original comment: 读取下一个转义字符,并且修改上一个plexer->theTokenSource为该转义
* 字符代表的字符,参考CONSUMECHARACTER宏。
* 目前支持的转义字符有\r\n\t\s\"\'\\
* 2007-1-22
******************************************************************************/
#define CONSUMEESCAPE \
{\
CONSUMECHARACTER;\
MAKETOKEN( PP_TOKEN_ERROR );\
}
//Constructor
void pp_token_Init(pp_token* ptoken, PP_TOKEN_TYPE theType, LPCSTR theSource, TEXTPOS theTextPosition, ULONG charOffset)
{
ptoken->theType = theType;
ptoken->theTextPosition = theTextPosition;
ptoken->charOffset = charOffset;
strcpy(ptoken->theSource, theSource );
}
void pp_lexer_Init(pp_lexer* plexer, LPCSTR theSource, TEXTPOS theStartingPosition)
{
plexer->ptheSource = theSource;
plexer->theTextPosition = theStartingPosition;
plexer->pcurChar = (CHAR*)plexer->ptheSource;
plexer->offset = 0;
plexer->tokOffset = 0;
/*pl = plexer;*/
}
void pp_lexer_Clear(pp_lexer* plexer)
{
memset(plexer, 0, sizeof(pp_lexer));
}
/******************************************************************************
* getNextToken -- Thie method searches the input stream and returns the next
* token found within that stream, using the principle of maximal munch. It
* embodies the start state of the FSA.
*
* Parameters: theNextToken -- address of the next CToken found in the stream
* Returns: S_OK
* E_FAIL
******************************************************************************/
HRESULT pp_lexer_GetNextToken (pp_lexer* plexer, pp_token* theNextToken)
{
for(;;){
memset(plexer->theTokenSource, 0, MAX_PP_TOKEN_LENGTH * sizeof(CHAR));
plexer->theTokenPosition = plexer->theTextPosition;
plexer->tokOffset = plexer->offset;
//Whenever we get a new token, we need to watch out for the end-of-input.
//Otherwise, we could walk right off the end of the stream.
if ( !strncmp( plexer->pcurChar, "\0", 1)){ //A null character marks the end of the stream
MAKETOKEN( PP_TOKEN_EOF );
return S_OK;
}
//Windows line break (\r\n)
else if ( !strncmp( plexer->pcurChar, "\r\n", 2)){
//interpret as a newline
plexer->theTokenSource[strlen(plexer->theTokenSource)] = '\n';
plexer->theTokenSource[strlen(plexer->theTokenSource)] = '\0';
plexer->theTextPosition.col = 0;
plexer->theTextPosition.row++;
plexer->pcurChar += 2;
plexer->offset += 2;
MAKETOKEN( PP_TOKEN_NEWLINE );
return S_OK;
}
//newline (\n), carriage return (\r), or form feed (\f)
else if ( !strncmp( plexer->pcurChar, "\n", 1) || !strncmp( plexer->pcurChar, "\r", 1) ||
!strncmp( plexer->pcurChar, "\f", 1)){
//interpret as a newline
plexer->theTokenSource[strlen(plexer->theTokenSource)] = '\n';
plexer->theTokenSource[strlen(plexer->theTokenSource)] = '\0';
plexer->theTextPosition.col = 0;
plexer->theTextPosition.row++;
plexer->pcurChar++;
plexer->offset++;
MAKETOKEN( PP_TOKEN_NEWLINE );
return S_OK;
}
//tab
else if ( !strncmp( plexer->pcurChar, "\t", 1)){
//increment the offset counter by TABSIZE
plexer->theTokenSource[strlen(plexer->theTokenSource)] = *(plexer->pcurChar);
plexer->theTokenSource[strlen(plexer->theTokenSource)] = '\0';
plexer->theTextPosition.col += TABSIZE;
plexer->pcurChar++;
plexer->offset++;
MAKETOKEN( PP_TOKEN_WHITESPACE );
return S_OK;
}
//space
else if ( !strncmp(plexer->pcurChar, " ", 1)){
//increment the offset counter
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_WHITESPACE );
return S_OK;
}
//an Identifier starts with an alphabetical character or underscore
else if ( *plexer->pcurChar=='_' || (*plexer->pcurChar>= 'a' && *plexer->pcurChar <= 'z') ||
(*plexer->pcurChar >= 'A' && *plexer->pcurChar <= 'Z')){
return pp_lexer_GetTokenIdentifier(plexer, theNextToken );
}
//a Number starts with a numerical character
else if ((*plexer->pcurChar >= '0' && *plexer->pcurChar <= '9') ){
return pp_lexer_GetTokenNumber(plexer, theNextToken );
}
//string
else if (!strncmp( plexer->pcurChar, "\"", 1)){
return pp_lexer_GetTokenStringLiteral(plexer, theNextToken );
}
//character
else if (!strncmp( plexer->pcurChar, "'", 1)){
CONSUMECHARACTER;
//escape characters
if (!strncmp( plexer->pcurChar, "\\", 1)){
CONSUMECHARACTER;
CONSUMECHARACTER;
}
//must not be an empty character
else if(strncmp( plexer->pcurChar, "'", 1))
{
CONSUMECHARACTER;
}
else
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_ERROR );
return S_OK;
}
if (!strncmp( plexer->pcurChar, "'", 1)){
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_STRING_LITERAL );
return S_OK;
}
else{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_ERROR );
return S_OK;
}
}
//Before checking for comments
else if ( !strncmp( plexer->pcurChar, "/", 1)){
CONSUMECHARACTER;
if ( !strncmp( plexer->pcurChar, "/", 1)){
pp_lexer_SkipComment(plexer, COMMENT_SLASH);
//CONSUMECHARACTER;
//MAKETOKEN( PP_TOKEN_COMMENT_SLASH );
//return S_OK;
}
else if ( !strncmp( plexer->pcurChar, "*", 1)){
pp_lexer_SkipComment(plexer, COMMENT_STAR);
//CONSUMECHARACTER;
//MAKETOKEN( PP_TOKEN_COMMENT_STAR_BEGIN );
//return S_OK;
}
//Now complete the symbol scan for regular symbols.
else if ( !strncmp( plexer->pcurChar, "=", 1)){
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_DIV_ASSIGN );
return S_OK;
}
else{
MAKETOKEN( PP_TOKEN_DIV );
return S_OK;
}
}
//Check for the end of a star comment
else if ( !strncmp( plexer->pcurChar, "*/", 2)){
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_COMMENT_STAR_END );
return S_OK;
}
//Preprocessor directive
else if ( !strncmp( plexer->pcurChar, "#", 1)){
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_DIRECTIVE );
return S_OK;
}
//a Symbol starts with one of these characters
else if (( !strncmp( plexer->pcurChar, ">", 1)) || ( !strncmp( plexer->pcurChar, "<", 1))
|| ( !strncmp( plexer->pcurChar, "+", 1)) || ( !strncmp( plexer->pcurChar, "-", 1))
|| ( !strncmp( plexer->pcurChar, "*", 1)) || ( !strncmp( plexer->pcurChar, "/", 1))
|| ( !strncmp( plexer->pcurChar, "%", 1)) || ( !strncmp( plexer->pcurChar, "&", 1))
|| ( !strncmp( plexer->pcurChar, "^", 1)) || ( !strncmp( plexer->pcurChar, "|", 1))
|| ( !strncmp( plexer->pcurChar, "=", 1)) || ( !strncmp( plexer->pcurChar, "!", 1))
|| ( !strncmp( plexer->pcurChar, ";", 1)) || ( !strncmp( plexer->pcurChar, "{", 1))
|| ( !strncmp( plexer->pcurChar, "}", 1)) || ( !strncmp( plexer->pcurChar, ",", 1))
|| ( !strncmp( plexer->pcurChar, ":", 1)) || ( !strncmp( plexer->pcurChar, "(", 1))
|| ( !strncmp( plexer->pcurChar, ")", 1)) || ( !strncmp( plexer->pcurChar, "[", 1))
|| ( !strncmp( plexer->pcurChar, "]", 1)) || ( !strncmp( plexer->pcurChar, ".", 1))
|| ( !strncmp( plexer->pcurChar, "~", 1)) || ( !strncmp( plexer->pcurChar, "?", 1)))
{
return pp_lexer_GetTokenSymbol(plexer, theNextToken );
}
//If we get here, we've hit a character we don't recognize
else{
//Consume the character
CONSUMECHARACTER;
/* Create an "error" token, but continue normally since unrecognized
* characters are none of the preprocessor's business. Scriptlib can
* deal with them if necessary. */
MAKETOKEN( PP_TOKEN_ERROR );
//HandleCompileError( *theNextToken, UNRECOGNIZED_CHARACTER );
return S_OK;
}
}
}
/******************************************************************************
* Identifier -- This method extracts an identifier from the stream, once it's
* recognized as an identifier. After it is extracted, this method determines
* if the identifier is a keyword.
* Parameters: theNextToken -- address of the next CToken found in the stream
* Returns: S_OK
* E_FAIL
******************************************************************************/
HRESULT pp_lexer_GetTokenIdentifier(pp_lexer* plexer, pp_token* theNextToken)
{
//copy the source that makes up this token
//an identifier is a string of letters, digits and/or underscores
do{
CONSUMECHARACTER;
}while ((*plexer->pcurChar >= '0' && *plexer->pcurChar <= '9') ||
(*plexer->pcurChar >= 'a' && *plexer->pcurChar <= 'z') ||
(*plexer->pcurChar >= 'A' && *plexer->pcurChar <= 'Z') ||
( !strncmp( plexer->pcurChar, "_", 1)));
//Check the Identifier against current keywords
if (!strcmp( plexer->theTokenSource, "auto")){
MAKETOKEN( PP_TOKEN_AUTO );}
else if (!strcmp( plexer->theTokenSource, "break")){
MAKETOKEN( PP_TOKEN_BREAK );}
else if (!strcmp( plexer->theTokenSource, "case")){
MAKETOKEN( PP_TOKEN_CASE );}
else if (!strcmp( plexer->theTokenSource, "char")){
MAKETOKEN( PP_TOKEN_CHAR );}
else if (!strcmp( plexer->theTokenSource, "const")){
MAKETOKEN( PP_TOKEN_CONST );}
else if (!strcmp( plexer->theTokenSource, "continue")){
MAKETOKEN( PP_TOKEN_CONTINUE );}
else if (!strcmp( plexer->theTokenSource, "default")){
MAKETOKEN( PP_TOKEN_DEFAULT );}
else if (!strcmp( plexer->theTokenSource, "do")){
MAKETOKEN( PP_TOKEN_DO );}
else if (!strcmp( plexer->theTokenSource, "double")){
MAKETOKEN( PP_TOKEN_DOUBLE );}
else if (!strcmp( plexer->theTokenSource, "else")){
MAKETOKEN( PP_TOKEN_ELSE );}
else if (!strcmp( plexer->theTokenSource, "enum")){
MAKETOKEN( PP_TOKEN_ENUM );}
else if (!strcmp( plexer->theTokenSource, "extern")){
MAKETOKEN( PP_TOKEN_EXTERN );}
else if (!strcmp( plexer->theTokenSource, "float")){
MAKETOKEN( PP_TOKEN_FLOAT );}
else if (!strcmp( plexer->theTokenSource, "for")){
MAKETOKEN( PP_TOKEN_FOR );}
else if (!strcmp( plexer->theTokenSource, "goto")){
MAKETOKEN( PP_TOKEN_GOTO );}
else if (!strcmp( plexer->theTokenSource, "if")){
MAKETOKEN( PP_TOKEN_IF );}
else if (!strcmp( plexer->theTokenSource, "int")){
MAKETOKEN( PP_TOKEN_INT );}
else if (!strcmp( plexer->theTokenSource, "long")){
MAKETOKEN( PP_TOKEN_LONG );}
else if (!strcmp( plexer->theTokenSource, "register")){
MAKETOKEN( PP_TOKEN_REGISTER );}
else if (!strcmp( plexer->theTokenSource, "return")){
MAKETOKEN( PP_TOKEN_RETURN );}
else if (!strcmp( plexer->theTokenSource, "short")){
MAKETOKEN( PP_TOKEN_SHORT );}
else if (!strcmp( plexer->theTokenSource, "signed")){
MAKETOKEN( PP_TOKEN_SIGNED );}
else if (!strcmp( plexer->theTokenSource, "sizeof")){
MAKETOKEN( PP_TOKEN_SIZEOF );}
else if (!strcmp( plexer->theTokenSource, "static")){
MAKETOKEN( PP_TOKEN_STATIC );}
else if (!strcmp( plexer->theTokenSource, "struct")){
MAKETOKEN( PP_TOKEN_STRUCT );}
else if (!strcmp( plexer->theTokenSource, "switch")){
MAKETOKEN( PP_TOKEN_SWITCH );}
else if (!strcmp( plexer->theTokenSource, "typedef")){
MAKETOKEN( PP_TOKEN_TYPEDEF );}
else if (!strcmp( plexer->theTokenSource, "union")){
MAKETOKEN( PP_TOKEN_UNION );}
else if (!strcmp( plexer->theTokenSource, "unsigned")){
MAKETOKEN( PP_TOKEN_UNSIGNED );}
else if (!strcmp( plexer->theTokenSource, "void")){
MAKETOKEN( PP_TOKEN_VOID );}
else if (!strcmp( plexer->theTokenSource, "volatile")){
MAKETOKEN( PP_TOKEN_VOLATILE );}
else if (!strcmp( plexer->theTokenSource, "while")){
MAKETOKEN( PP_TOKEN_WHILE );}
else if (!strcmp( plexer->theTokenSource, "include")){
MAKETOKEN( PP_TOKEN_INCLUDE );}
else if (!strcmp( plexer->theTokenSource, "define")){
MAKETOKEN( PP_TOKEN_DEFINE );}
else if (!strcmp( plexer->theTokenSource, "undef")){
MAKETOKEN( PP_TOKEN_UNDEF );}
else if (!strcmp( plexer->theTokenSource, "pragma")){
MAKETOKEN( PP_TOKEN_PRAGMA );}
else if (!strcmp( plexer->theTokenSource, "ifdef")){
MAKETOKEN( PP_TOKEN_IFDEF );}
else if (!strcmp( plexer->theTokenSource, "ifndef")){
MAKETOKEN( PP_TOKEN_IFNDEF );}
else if (!strcmp( plexer->theTokenSource, "elif")){
MAKETOKEN( PP_TOKEN_ELIF );}
else if (!strcmp( plexer->theTokenSource, "endif")){
MAKETOKEN( PP_TOKEN_ENDIF );}
else if (!strcmp( plexer->theTokenSource, "warning")){
MAKETOKEN( PP_TOKEN_WARNING );}
else if (!strcmp( plexer->theTokenSource, "error")){
MAKETOKEN( PP_TOKEN_ERROR_TEXT );} // this is completely different from PP_TOKEN_ERROR!
else{
MAKETOKEN( PP_TOKEN_IDENTIFIER );}
return S_OK;
}
/******************************************************************************
* Number -- This method extracts a numerical constant from the stream. It
* only extracts the digits that make up the number. No conversion from string
* to numeral is performed here.
* Parameters: theNextToken -- address of the next CToken found in the stream
* Returns: S_OK
* E_FAIL
******************************************************************************/
HRESULT pp_lexer_GetTokenNumber(pp_lexer* plexer, pp_token* theNextToken)
{
//copy the source that makes up this token
//a constant is one of these:
//0[xX][a-fA-F0-9]+{u|U|l|L}
//0{D}+{u|U|l|L}
if (( !strncmp( plexer->pcurChar, "0X", 2)) || ( !strncmp( plexer->pcurChar, "0x", 2))){
CONSUMECHARACTER;
CONSUMECHARACTER;
while ((*plexer->pcurChar >= '0' && *plexer->pcurChar <= '9') ||
(*plexer->pcurChar >= 'a' && *plexer->pcurChar <= 'f') ||
(*plexer->pcurChar >= 'A' && *plexer->pcurChar <= 'F'))
{
CONSUMECHARACTER;
}
if (( !strncmp( plexer->pcurChar, "u", 1)) || ( !strncmp( plexer->pcurChar, "U", 1)) ||
( !strncmp( plexer->pcurChar, "l", 1)) || ( !strncmp( plexer->pcurChar, "L", 1)))
{
CONSUMECHARACTER;
}
MAKETOKEN( PP_TOKEN_HEXCONSTANT );
}
else{
while (*plexer->pcurChar >= '0' && *plexer->pcurChar <= '9')
{
CONSUMECHARACTER;
}
if (( !strncmp( plexer->pcurChar, "E", 1)) || ( !strncmp( plexer->pcurChar, "e", 1)))
{
CONSUMECHARACTER;
while (*plexer->pcurChar >= '0' && *plexer->pcurChar <= '9')
{
CONSUMECHARACTER;
}
if (( !strncmp( plexer->pcurChar, "f", 1)) || ( !strncmp( plexer->pcurChar, "F", 1)) ||
( !strncmp( plexer->pcurChar, "l", 1)) || ( !strncmp( plexer->pcurChar, "L", 1)))
{
CONSUMECHARACTER;
}
MAKETOKEN( PP_TOKEN_FLOATCONSTANT );
}
else if ( !strncmp( plexer->pcurChar, ".", 1))
{
CONSUMECHARACTER;
while (*plexer->pcurChar >= '0' && *plexer->pcurChar <= '9')
{
CONSUMECHARACTER;
}
if (( !strncmp( plexer->pcurChar, "E", 1)) || ( !strncmp( plexer->pcurChar, "e", 1)))
{
CONSUMECHARACTER;
while (*plexer->pcurChar >= '0' && *plexer->pcurChar <= '9')
{
CONSUMECHARACTER;
}
if (( !strncmp( plexer->pcurChar, "f", 1)) ||
( !strncmp( plexer->pcurChar, "F", 1)) ||
( !strncmp( plexer->pcurChar, "l", 1)) ||
( !strncmp( plexer->pcurChar, "L", 1)))
{
CONSUMECHARACTER;
}
}
MAKETOKEN( PP_TOKEN_FLOATCONSTANT );
}
else{
MAKETOKEN( PP_TOKEN_INTCONSTANT );
}
}
return S_OK;
}
/******************************************************************************
* StringLiteral -- This method extracts a string literal from the character
* stream.
* Parameters: theNextToken -- address of the next CToken found in the stream
* Returns: S_OK
* E_FAIL
******************************************************************************/
HRESULT pp_lexer_GetTokenStringLiteral(pp_lexer* plexer, pp_token* theNextToken)
{
//copy the source that makes up this token
//an identifier is a string of letters, digits and/or underscores
//consume that first quote mark
int esc = 0;
CONSUMECHARACTER;
while ( strncmp( plexer->pcurChar, "\"", 1))
{
if(!strncmp( plexer->pcurChar, "\\", 1))
{
esc = 1;
}
CONSUMECHARACTER;
if(esc)
{
CONSUMECHARACTER;
esc = 0;
}
}
//consume that last quote mark
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_STRING_LITERAL );
return S_OK;
}
/******************************************************************************
* Symbol -- This method extracts a symbol from the character stream. For the
* purposes of lexing, comments are considered symbols.
* Parameters: theNextToken -- address of the next CToken found in the stream
* Returns: S_OK
* E_FAIL
******************************************************************************/
HRESULT pp_lexer_GetTokenSymbol(pp_lexer* plexer, pp_token* theNextToken)
{
//">>="
if ( !strncmp( plexer->pcurChar, ">>=", 3))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_RIGHT_ASSIGN );
}
//">>"
else if ( !strncmp( plexer->pcurChar, ">>", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_RIGHT_OP );
}
//">="
else if ( !strncmp( plexer->pcurChar, ">=", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_GE_OP );
}
//">"
else if ( !strncmp( plexer->pcurChar, ">", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_GT );
}
//"<<="
else if ( !strncmp( plexer->pcurChar, "<<=", 3))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_LEFT_ASSIGN );
}
//"<<"
else if ( !strncmp( plexer->pcurChar, "<<", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_LEFT_OP );
}
//"<="
else if ( !strncmp( plexer->pcurChar, "<=", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_LE_OP );
}
//"<"
else if ( !strncmp( plexer->pcurChar, "<", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_LT );
}
//"++"
else if ( !strncmp( plexer->pcurChar, "++", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_INC_OP );
}
//"+="
else if ( !strncmp( plexer->pcurChar, "+=", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_ADD_ASSIGN );
}
//"+"
else if ( !strncmp( plexer->pcurChar, "+", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_ADD );
}
//"--"
else if ( !strncmp( plexer->pcurChar, "--", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_DEC_OP );
}
//"-="
else if ( !strncmp( plexer->pcurChar, "-=", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_SUB_ASSIGN );
}
//"-"
else if ( !strncmp( plexer->pcurChar, "-", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_SUB );
}
//"*="
else if ( !strncmp( plexer->pcurChar, "*=", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_MUL_ASSIGN );
}
//"*"
else if ( !strncmp( plexer->pcurChar, "*", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_MUL );
}
//"%="
else if ( !strncmp( plexer->pcurChar, "%=", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_MOD_ASSIGN );
}
//"%"
else if ( !strncmp( plexer->pcurChar, "%", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_MOD );
}
//"&&"
else if ( !strncmp( plexer->pcurChar, "&&", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_AND_OP );
}
//"&="
else if ( !strncmp( plexer->pcurChar, "&=", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_AND_ASSIGN );
}
//"&"
else if ( !strncmp( plexer->pcurChar, "&", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_BITWISE_AND );
}
//"^="
else if ( !strncmp( plexer->pcurChar, "^=", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_XOR_ASSIGN );
}
//"^"
else if ( !strncmp( plexer->pcurChar, "^", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_XOR );
}
//"||"
else if ( !strncmp( plexer->pcurChar, "||", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_OR_OP );
}
//"|="
else if ( !strncmp( plexer->pcurChar, "|=", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_OR_ASSIGN );
}
//"|"
else if ( !strncmp( plexer->pcurChar, "|", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_BITWISE_OR );
}
//"=="
else if ( !strncmp( plexer->pcurChar, "==", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_EQ_OP );
}
//"="
else if ( !strncmp( plexer->pcurChar, "=", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_ASSIGN );
}
//"!="
else if ( !strncmp( plexer->pcurChar, "!=", 2))
{
CONSUMECHARACTER;
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_NE_OP );
}
//"!"
else if ( !strncmp( plexer->pcurChar, "!", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_BOOLEAN_NOT );
}
//";"
else if ( !strncmp( plexer->pcurChar, ";", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_SEMICOLON);
}
//"{"
else if ( !strncmp( plexer->pcurChar, "{", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_LCURLY);
}
//"}"
else if ( !strncmp( plexer->pcurChar, "}", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_RCURLY);
}
//","
else if ( !strncmp( plexer->pcurChar, ",", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_COMMA);
}
//":"
else if ( !strncmp( plexer->pcurChar, ":", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_COLON);
}
//"("
else if ( !strncmp( plexer->pcurChar, "(", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_LPAREN);
}
//")"
else if ( !strncmp( plexer->pcurChar, ")", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_RPAREN);
}
//"["
else if ( !strncmp( plexer->pcurChar, "[", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_LBRACKET);
}
//"]"
else if ( !strncmp( plexer->pcurChar, "]", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_RBRACKET);
}
//"."
else if ( !strncmp( plexer->pcurChar, ".", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_FIELD);
}
//"~"
else if ( !strncmp( plexer->pcurChar, "~", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_BITWISE_NOT);
}
//"?"
else if ( !strncmp( plexer->pcurChar, "?", 1))
{
CONSUMECHARACTER;
MAKETOKEN( PP_TOKEN_CONDITIONAL);
}
return S_OK;
}
/******************************************************************************
* Comment -- This method extracts a symbol from the character stream.
* Parameters: theNextToken -- address of the next CToken found in the stream
* Returns: S_OK
* E_FAIL
******************************************************************************/
HRESULT pp_lexer_SkipComment(pp_lexer* plexer, COMMENT_TYPE theType)
{
if (theType == COMMENT_SLASH){
do{
SKIPCHARACTER;
//break out if we hit a new line
if (!strncmp( plexer->pcurChar, "\n", 1)){
plexer->theTextPosition.col = 0;
plexer->theTextPosition.row++;
break;
}
else if (!strncmp( plexer->pcurChar, "\r", 1)){
plexer->theTextPosition.col = 0;
//plexer->theTextPosition.row++;
break;
}
else if (!strncmp( plexer->pcurChar, "\f", 1)){
plexer->theTextPosition.row++;
break;
}
}while (strncmp( plexer->pcurChar, "\0", 1));
}
else if (theType == COMMENT_STAR){
//consume the '*' that gets this comment started
SKIPCHARACTER;
//loop through the characters till we hit '*/'
while (strncmp( plexer->pcurChar, "\0", 1)){
if (0==strncmp( plexer->pcurChar, "*/", 2)){
SKIPCHARACTER;
SKIPCHARACTER;
break;
}
else if (!strncmp( plexer->pcurChar, "\n", 1)){
plexer->theTextPosition.col = 0;
plexer->theTextPosition.row++;
}
else if (!strncmp( plexer->pcurChar, "\r", 1)){
plexer->theTextPosition.col = 0;
//plexer->theTextPosition.row++;
}
else if (!strncmp( plexer->pcurChar, "\f", 1)){
plexer->theTextPosition.row++;
}
SKIPCHARACTER;
};
}
return S_OK;
}