From 200f4d026940e26aff2d34040f490330c8ece442 Mon Sep 17 00:00:00 2001 From: Josef Reidinger Date: Wed, 26 Jun 2013 08:46:54 +0200 Subject: [PATCH] Add to tokens comments and whitespaces if Y2EXPORTCOMMENTS is set it is needed for exporting comments from ycp code to ruby one. It is enclosed in env variable to avoid performance regression in common run. --- libycp/src/Scanner.cc | 12 ++ libycp/src/include/ycp/Scanner.h | 9 + libycp/src/parser.yy | 10 +- libycp/src/scanner.ll | 343 +++++++++++++++++-------------- 4 files changed, 221 insertions(+), 153 deletions(-) diff --git a/libycp/src/Scanner.cc b/libycp/src/Scanner.cc index cf8258fbe..6b56a7e38 100644 --- a/libycp/src/Scanner.cc +++ b/libycp/src/Scanner.cc @@ -324,6 +324,18 @@ Scanner::scannedType () const return m_scannedType; } +void +Scanner::setCommentBefore (const string & comment_before) +{ + m_commentBefore = comment_before; +} + +std::string +Scanner::commentBefore () const +{ + return m_commentBefore; +} + int Scanner::lineNumber () const diff --git a/libycp/src/include/ycp/Scanner.h b/libycp/src/include/ycp/Scanner.h index 0091e6f31..a3b1473b6 100644 --- a/libycp/src/include/ycp/Scanner.h +++ b/libycp/src/include/ycp/Scanner.h @@ -120,6 +120,11 @@ class Scanner : public yyFlexLexer, public Logger */ constTypePtr m_scannedType; + /** + * Holds the comment before the most recent token + */ + std::string m_commentBefore; + /** * Holds the line number of scanned_value */ @@ -263,6 +268,8 @@ class Scanner : public yyFlexLexer, public Logger */ constTypePtr scannedType() const; + std::string commentBefore () const; + /** * Gets the line number of the latest scanned token. */ @@ -309,6 +316,8 @@ class Scanner : public yyFlexLexer, public Logger */ void setScannedToken (const tokenValue & value, constTypePtr type); + void setCommentBefore (const string & comment_before); + /** * Internal helper function that deals with strings of arbitrary * length diff --git a/libycp/src/parser.yy b/libycp/src/parser.yy index f64bcd6f7..ff91d7888 100644 --- a/libycp/src/parser.yy +++ b/libycp/src/parser.yy @@ -19,12 +19,13 @@ Implementation rules - yystype is a struct with four elements + yystype is a struct with five elements YCodePtr c pointer to code (where applicable) tokenValue v value of token (where applicable) constTypePtr t type of current syntactic element int l line number of syntactic element + string com comment preceding the token c and v somehow represent a similar kind of information and are mostly valid alternating. @@ -33,6 +34,9 @@ identifiers, etc. c is used for the 'high level' (parser) syntax like expressions, statements, blocks, etc. + NOTE that the distinction is mostly between terminals and nonterminals, + but notably the 'type' nonterminal noes not have YCode and holds a Type + tree instead. t is valid everywhere since every syntactic element has a type. @@ -40,6 +44,9 @@ l is valid everywhere since every syntactic element appears at a distinctive line number in the source file. + + com is valid for scanner tokens. As soon as there is an YCode, we + move it off com to c. /-*/ %{ #include @@ -85,6 +92,7 @@ struct yystype_type { tokenValue v; // token (for scanner level syntax) constTypePtr t; // type (NULL for error) int l; + std::string com; // comment before the token }; #define YYSTYPE yystype_type diff --git a/libycp/src/scanner.ll b/libycp/src/scanner.ll index 175b6be93..fb72e9a1b 100644 --- a/libycp/src/scanner.ll +++ b/libycp/src/scanner.ll @@ -19,6 +19,7 @@ %{ +#include #include #include #include @@ -71,11 +72,27 @@ static SymbolTable *namespaceTable = NULL; // the namespace prefix when BEGIN(namespace) is called. static char *namespace_prefix = 0; -static char *scanner_token; - static tokenValue token_value; -#define RESULT(type,token) setScannedToken(token_value, type); scanner_token = yytext; return token + static std::string saved_comment; + + void save_comment(const char *text) { + if (getenv("Y2PARSECOMMENTS")) + saved_comment += text; + } + + // TODO: detect that we've given the parser a comment + // that was ignored by the parser +#define TOKEN(token) do { \ + setCommentBefore(saved_comment); \ + saved_comment = ""; \ + return (token); \ + } while(0) + +#define VALUE_TOKEN(type,token) do { \ + setScannedToken(token_value, type); \ + TOKEN(token); \ + } while (0) %} @@ -100,18 +117,41 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) /* Whitespace and Comments */ /* ----------------------------------- */ + /* Unix style Comments */ +^[\t ]*#.*$ { + save_comment(yytext); + } +\/\* { + BEGIN (comment); + save_comment(yytext); + } +\n { + save_comment(yytext); + INC_LINE; + } +\*\/ { + save_comment(yytext); + BEGIN (INITIAL); + } +\* { + save_comment(yytext); + } +[^*]* { + save_comment(yytext); + } -^[\t ]*#.*$ { /* Ignore Unix style Comments */ } -\/\* { BEGIN (comment); } -\n { INC_LINE; } -\*\/ { BEGIN (INITIAL); } -. { /* skip */ } + /* C++ style comments */ +\/\/.*$ { + save_comment(yytext); + } -\/\/.*$ { /* Ignore C++ style comments */ } -\n { /* Ignore newlines */ +\n { INC_LINE; + save_comment(yytext); } -[\f\t\r\v ]+ { /* Ignore Whitespaces according to isspace(3) */ } +[\f\t\r\v ]+ { + save_comment(yytext); +} /* ----------------------------------- */ @@ -126,7 +166,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) std::istringstream is(input); is.imbue(std::locale::classic()); /* ensure that we use C locale for float parsing */ is >> token_value.fval; - RESULT (Type::ConstFloat, C_FLOAT); + VALUE_TOKEN (Type::ConstFloat, C_FLOAT); } /* integer constant */ @@ -134,7 +174,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) [1-9][0-9]* { debug_scanner(""); token_value.ival = atoll (yytext); - RESULT (Type::ConstInteger, C_INTEGER); + VALUE_TOKEN (Type::ConstInteger, C_INTEGER); } /* octal constant */ @@ -142,7 +182,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) 0[0-7]* { debug_scanner(""); sscanf (yytext, "%Lo", &(token_value.ival)); - RESULT (Type::ConstInteger, C_INTEGER); + VALUE_TOKEN (Type::ConstInteger, C_INTEGER); } /* wrong octal constant */ @@ -156,7 +196,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) 0x[0-9A-Fa-f]+ { debug_scanner(""); sscanf (yytext, "0x%Lx", &(token_value.ival)); - RESULT (Type::ConstInteger, C_INTEGER); + VALUE_TOKEN (Type::ConstInteger, C_INTEGER); } /* wrong hexadecimal constant */ @@ -173,7 +213,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) \#\[ { if (m_scandataBuffer == 0) m_scandataBuffer = extend_scanbuffer (sizeof (long)); - if (m_scandataBuffer == 0) return SCANNER_ERROR; + if (m_scandataBuffer == 0) TOKEN(SCANNER_ERROR); m_scandataBufferPtr = m_scandataBuffer + sizeof (long); BEGIN (bybl); } @@ -187,7 +227,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) long length = (long)(m_scandataBufferPtr - bytes); memcpy (m_scandataBuffer, &length, sizeof (long)); token_value.cval = (unsigned char *)m_scandataBuffer; - RESULT (Type::ConstByteblock, C_BYTEBLOCK); + VALUE_TOKEN (Type::ConstByteblock, C_BYTEBLOCK); } /* byteblock data */ @@ -199,7 +239,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) if (used + needed/2 >= m_scandataBufferSize) { if (extend_scanbuffer (needed) == 0) - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); m_scandataBufferPtr = m_scandataBuffer + used; // might be realloced } for (int i=0; i < needed/2; i++) @@ -223,7 +263,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) . { logError("bad character in byteblock constant", LINE_VAR); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } /* ----------------------------------- */ @@ -237,7 +277,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) if (m_scandataBuffer == 0) m_scandataBuffer = extend_scanbuffer (1); if (m_scandataBuffer == 0) - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); m_scandataBufferPtr = m_scandataBuffer; BEGIN (str); @@ -251,7 +291,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) BEGIN (INITIAL); *m_scandataBufferPtr = '\0'; token_value.sval = Scanner::doStrdup (m_scandataBuffer); - RESULT (Type::ConstString, STRING); + VALUE_TOKEN (Type::ConstString, STRING); } /* string octal character */ @@ -266,7 +306,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) if (result >= m_scandataBufferSize) { if (extend_scanbuffer (1) == 0) - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); m_scandataBufferPtr = m_scandataBuffer + result; } (void) sscanf (yytext + 1, "%o", &result); @@ -289,7 +329,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) if (offset >= m_scandataBufferSize) { if (extend_scanbuffer (1) == 0) - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); m_scandataBufferPtr = m_scandataBuffer + offset; } switch (yytext[1]) @@ -315,7 +355,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) if (used + needed >= m_scandataBufferSize) { if (extend_scanbuffer (needed) == 0) - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); m_scandataBufferPtr = m_scandataBuffer + used; // might be realloced } strcpy (m_scandataBufferPtr, yptr); @@ -329,7 +369,7 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) if (offset >= m_scandataBufferSize) { if (extend_scanbuffer (1) == 0) - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); m_scandataBufferPtr = m_scandataBuffer + offset; } *m_scandataBufferPtr++ = '\n'; @@ -346,19 +386,19 @@ SYMBOL ([[:alpha:]_][[:alnum:]_]+|[[:alpha:]][[:alnum:]_]*) nil { debug_scanner(""); - RESULT (Type::ConstVoid, C_VOID); + VALUE_TOKEN (Type::ConstVoid, C_VOID); } true { debug_scanner(""); token_value.bval = true; - RESULT (Type::ConstBoolean, C_BOOLEAN); + VALUE_TOKEN (Type::ConstBoolean, C_BOOLEAN); } false { debug_scanner(""); token_value.bval = false; - RESULT (Type::ConstBoolean, C_BOOLEAN); + VALUE_TOKEN (Type::ConstBoolean, C_BOOLEAN); } /* -- path */ @@ -369,90 +409,89 @@ false { if (yytext[yyleng-1] == '-') { logError ("dash behind path constant not allowed", LINE_VAR); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } - RESULT (Type::ConstPath, C_PATH); + VALUE_TOKEN (Type::ConstPath, C_PATH); } /* ----------------------------------- */ /* Statement Keywords */ /* ----------------------------------- */ - -return { return RETURN; }; -Wiederkehr { return RETURN; }; -define { return DEFINE; }; -definieren { return DEFINE; }; -undefine { return UNDEFINE; }; -continue { return CONTINUE; }; -weiterleben { return CONTINUE; }; -break { return BREAK; }; -Ruhepause { return BREAK; }; -if { return IF; }; -falls { return IF; }; -is { return IS; }; -ist { return IS; }; -else { return ELSE; }; -sonstwas { return ELSE; }; -do { return DO; }; -mach { return DO; }; -while { return WHILE; }; -solange { return WHILE; }; -repeat { return REPEAT; }; -Wiederholungsaufforderung { return REPEAT; }; -until { return UNTIL; }; -bis { return UNTIL; }; -empty { return EMPTY; }; -list { return LIST; }; -Verzeichnis { return LIST; }; -map { return MAP; }; -Karte { return MAP; }; -struct { return STRUCT; }; -block { return BLOCK; }; -Klotz { return BLOCK; }; -include { return INCLUDE; }; -enthalten { return INCLUDE; }; -import { return IMPORT; }; -Einfuhrgenehmigung { return IMPORT; }; -export { return EXPORT; }; -global { return GLOBAL; }; -weltumspannend { return GLOBAL; }; -static { return STATIC; }; -extern { return EXTERN; }; -module { return MODULE; }; -Baustein { return MODULE; }; -textdomain { return TEXTDOMAIN; }; -Textzielbereich { return TEXTDOMAIN; }; -const { return CONST; }; -typedef { return TYPEDEF; }; -lookup { return LOOKUP; }; -betrachten { return LOOKUP; }; -select { return SELECT; }; -switch { return SWITCH; }; -case { return CASE; }; -default { return DEFAULT; }; -selektieren { return SELECT; }; +return { TOKEN(RETURN); }; +Wiederkehr { TOKEN(RETURN); }; +define { TOKEN(DEFINE); }; +definieren { TOKEN(DEFINE); }; +undefine { TOKEN(UNDEFINE); }; +continue { TOKEN(CONTINUE); }; +weiterleben { TOKEN(CONTINUE); }; +break { TOKEN(BREAK); }; +Ruhepause { TOKEN(BREAK); }; +if { TOKEN(IF); }; +falls { TOKEN(IF); }; +is { TOKEN(IS); }; +ist { TOKEN(IS); }; +else { TOKEN(ELSE); }; +sonstwas { TOKEN(ELSE); }; +do { TOKEN(DO); }; +mach { TOKEN(DO); }; +while { TOKEN(WHILE); }; +solange { TOKEN(WHILE); }; +repeat { TOKEN(REPEAT); }; +Wiederholungsaufforderung { TOKEN(REPEAT); }; +until { TOKEN(UNTIL); }; +bis { TOKEN(UNTIL); }; +empty { TOKEN(EMPTY); }; +list { TOKEN(LIST); }; +Verzeichnis { TOKEN(LIST); }; +map { TOKEN(MAP); }; +Karte { TOKEN(MAP); }; +struct { TOKEN(STRUCT); }; +block { TOKEN(BLOCK); }; +Klotz { TOKEN(BLOCK); }; +include { TOKEN(INCLUDE); }; +enthalten { TOKEN(INCLUDE); }; +import { TOKEN(IMPORT); }; +Einfuhrgenehmigung { TOKEN(IMPORT); }; +export { TOKEN(EXPORT); }; +global { TOKEN(GLOBAL); }; +weltumspannend { TOKEN(GLOBAL); }; +static { TOKEN(STATIC); }; +extern { TOKEN(EXTERN); }; +module { TOKEN(MODULE); }; +Baustein { TOKEN(MODULE); }; +textdomain { TOKEN(TEXTDOMAIN); }; +Textzielbereich { TOKEN(TEXTDOMAIN); }; +const { TOKEN(CONST); }; +typedef { TOKEN(TYPEDEF); }; +lookup { TOKEN(LOOKUP); }; +betrachten { TOKEN(LOOKUP); }; +select { TOKEN(SELECT); }; +switch { TOKEN(SWITCH); }; +case { TOKEN(CASE); }; +default { TOKEN(DEFAULT); }; +selektieren { TOKEN(SELECT); }; /* ----------------------------------- */ /* Type Keywords */ /* ----------------------------------- */ -any { RESULT (Type::Any, C_TYPE); }; -void { RESULT (Type::Void, C_TYPE); }; -boolean { RESULT (Type::Boolean, C_TYPE); }; -integer { RESULT (Type::Integer, C_TYPE); }; -float { RESULT (Type::Float, C_TYPE); }; -string { RESULT (Type::String, C_TYPE); }; -byteblock { RESULT (Type::Byteblock, C_TYPE); }; +any { VALUE_TOKEN (Type::Any, C_TYPE); }; +void { VALUE_TOKEN (Type::Void, C_TYPE); }; +boolean { VALUE_TOKEN (Type::Boolean, C_TYPE); }; +integer { VALUE_TOKEN (Type::Integer, C_TYPE); }; +float { VALUE_TOKEN (Type::Float, C_TYPE); }; +string { VALUE_TOKEN (Type::String, C_TYPE); }; +byteblock { VALUE_TOKEN (Type::Byteblock, C_TYPE); }; /* list, map are keywords and done by parser.yy */ -locale { RESULT (Type::Locale, C_TYPE); }; -term { RESULT (Type::Term, C_TYPE); }; -path { RESULT (Type::Path, C_TYPE); }; -symbol { RESULT (Type::Symbol, C_TYPE); }; +locale { VALUE_TOKEN (Type::Locale, C_TYPE); }; +term { VALUE_TOKEN (Type::Term, C_TYPE); }; +path { VALUE_TOKEN (Type::Path, C_TYPE); }; +symbol { VALUE_TOKEN (Type::Symbol, C_TYPE); }; /* common mistyped names */ -int { logError ("Seen 'int', use 'integer' instead", LINE_VAR); return SCANNER_ERROR; }; -char { logError ("Seen 'char', use 'string' instead", LINE_VAR); return SCANNER_ERROR; }; -bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER_ERROR; }; +int { logError ("Seen 'int', use 'integer' instead", LINE_VAR); TOKEN(SCANNER_ERROR); }; +char { logError ("Seen 'char', use 'string' instead", LINE_VAR); TOKEN(SCANNER_ERROR); }; +bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); TOKEN(SCANNER_ERROR); }; /* ----------------------------------- */ /* Quotes */ @@ -463,7 +502,7 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER \`{SYMBOL} { debug_scanner("<`symbol>"); token_value.yval = Scanner::doStrdup (yytext + 1); - RESULT (Type::ConstSymbol, C_SYMBOL); + VALUE_TOKEN (Type::ConstSymbol, C_SYMBOL); } /* -- double quoted symbol */ @@ -471,7 +510,7 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER \`\`{SYMBOL} { debug_scanner("<``symbol>"); logError ("*** Don't use ``symbol... but ``(symbol ... ***", LINE_VAR); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } /* -- symbols */ @@ -513,7 +552,7 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER logError ("Auto-Import '%s' failed", LINE_VAR, yytext); } logError ("Prefix '%s' is not a namespace", LINE_VAR, yytext); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } if (getenv (XREFDEBUG) != 0) y2milestone ("Autoimported (%s), table %p", yytext, namespaceTable); else y2debug ("builtin namespace (%s) -> table %p", yytext, namespaceTable); @@ -536,7 +575,7 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER if (tentry == 0) { logError ("Unknown namespace '%s'. Missing 'import'?", LINE_VAR, yytext); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } else if (tentry->sentry()->category() == SymbolEntry::c_self) // self reference { @@ -551,7 +590,7 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER if (! tentry->sentry()->isModule()) { logError ("Not a module '%s'", LINE_VAR, yytext); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } const Y2Namespace *name_space = ((YSymbolEntryPtr)tentry->sentry())->payloadNamespace(); @@ -562,7 +601,7 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER if (namespaceTable == 0) { logError ("Module table is empty", LINE_VAR); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } y2debug ("imported namespace (%s) -> table %p", yytext, namespaceTable); @@ -624,7 +663,7 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER { logError ("Not a module '%s'", LINE_VAR, colon); delete[] namespace_prefix; - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } const Y2Namespace *name_space = tentry->sentry()->nameSpace(); @@ -635,7 +674,7 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER { logError ("Module table is empty", LINE_VAR); delete[] namespace_prefix; - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } } @@ -651,7 +690,7 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER logError ("Unknown identifier '%s::%s'", LINE_VAR, namespace_prefix, yytext); } delete[] namespace_prefix; - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } if (next == 0) @@ -675,7 +714,7 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER case SymbolEntry::c_predefined: { logError ("Module/Namespace name '%s' after '::' ?!", LINE_VAR, yytext); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } break; case SymbolEntry::c_variable: @@ -684,30 +723,30 @@ bool { logError ("Seen 'bool', use 'boolean' instead", LINE_VAR); return SCANNER case SymbolEntry::c_builtin: { token_value.tval = tentry; - RESULT (tentry->sentry()->type(), IDENTIFIER); + VALUE_TOKEN (tentry->sentry()->type(), IDENTIFIER); } break; case SymbolEntry::c_typedef: { - RESULT (tentry->sentry()->type(), C_TYPE); + VALUE_TOKEN (tentry->sentry()->type(), C_TYPE); } break; case SymbolEntry::c_const: { // FIXME token_value.sval = Scanner::doStrdup (yytext); - RESULT (Type::ConstString, STRING); + VALUE_TOKEN (Type::ConstString, STRING); } break; } logError ("Global identifier of unknown category '%s'", LINE_VAR, yytext); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } . { debug_scanner("?%s?", yytext); logError ("Unexpected char '%s'", LINE_VAR, yytext); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } /* plain symbol */ @@ -729,7 +768,7 @@ if (tentry!=0) y2debug ("'%s' is global", yytext); { debug_scanner("", yytext); token_value.nval = Scanner::doStrdup (yytext); - RESULT (Type::Unspec, SYMBOL); // symbol of unknown type + VALUE_TOKEN (Type::Unspec, SYMBOL); // symbol of unknown type } token_value.tval = tentry; switch (tentry->sentry()->category()) @@ -740,13 +779,13 @@ if (tentry!=0) y2debug ("'%s' is global", yytext); { y2debug ("", yytext, tentry); token_value.nval = Scanner::doStrdup (yytext); - RESULT (Type::Unspec, SYMBOL); // symbol of unknown type + VALUE_TOKEN (Type::Unspec, SYMBOL); // symbol of unknown type } break; case SymbolEntry::c_typedef: { y2debug ("", yytext, tentry); - RESULT (tentry->sentry()->type(), C_TYPE); + VALUE_TOKEN (tentry->sentry()->type(), C_TYPE); } break; case SymbolEntry::c_const: @@ -754,14 +793,14 @@ if (tentry!=0) y2debug ("'%s' is global", yytext); y2debug ("", yytext, tentry); // FIXME token_value.sval = Scanner::doStrdup (yytext); - RESULT (Type::ConstString, STRING); + VALUE_TOKEN (Type::ConstString, STRING); } break; case SymbolEntry::c_namespace: { y2debug ("", yytext, tentry); token_value.tval = tentry; - RESULT (tentry->sentry()->type(), SYM_NAMESPACE); + VALUE_TOKEN (tentry->sentry()->type(), SYM_NAMESPACE); } break; case SymbolEntry::c_unspec: @@ -774,14 +813,14 @@ if (tentry!=0) y2debug ("'%s' is global", yytext); { y2debug ("", yytext, tentry); token_value.tval = tentry; - RESULT (tentry->sentry()->type(), IDENTIFIER); + VALUE_TOKEN (tentry->sentry()->type(), IDENTIFIER); } break; case SymbolEntry::c_filename: break; } logError ("Identifier of unknown category '%s'", LINE_VAR, yytext); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } /* global symbol */ @@ -790,14 +829,14 @@ if (tentry!=0) y2debug ("'%s' is global", yytext); if (builtinTable->find (name) != 0) { logError ("Keyword used as variable '%s'", LINE_VAR, name); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } TableEntry *tentry = m_globalTable->find (name, SymbolEntry::c_unspec); if (tentry == 0) { y2debug ("'%s' not found in %p", name, m_globalTable); logError ("Unknown global variable '%s'", LINE_VAR, name); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } switch (tentry->sentry()->category()) { @@ -810,7 +849,7 @@ if (tentry!=0) y2debug ("'%s' is global", yytext); case SymbolEntry::c_module: { logError ("Module name '%s' after '::' ?!", LINE_VAR, name); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } break; case SymbolEntry::c_variable: @@ -819,26 +858,26 @@ if (tentry!=0) y2debug ("'%s' is global", yytext); case SymbolEntry::c_builtin: { token_value.tval = tentry; - RESULT (tentry->sentry()->type(), IDENTIFIER); + VALUE_TOKEN (tentry->sentry()->type(), IDENTIFIER); } break; case SymbolEntry::c_typedef: { - RESULT (tentry->sentry()->type(), C_TYPE); + VALUE_TOKEN (tentry->sentry()->type(), C_TYPE); } break; case SymbolEntry::c_const: { // FIXME token_value.sval = Scanner::doStrdup (name); - RESULT (Type::ConstString, STRING); + VALUE_TOKEN (Type::ConstString, STRING); } break; case SymbolEntry::c_filename: break; } logError ("Global identifier of unknown category '%s'", LINE_VAR, name); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); } @@ -847,60 +886,60 @@ if (tentry!=0) y2debug ("'%s' is global", yytext); /* ----------------------------------- */ /* -- locale */ -_\( { return I18N; } +_\( { TOKEN(I18N); } /* -- map expression */ -\$\[ { return MAPEXPR; } +\$\[ { TOKEN(MAPEXPR); } /* -- comparison operators */ -== { return EQUALS; } -\< { return yytext[0]; } -\> { return yytext[0]; } -\<= { return LE; } -\>= { return GE; } -\!= { return NEQ; } +== { TOKEN(EQUALS); } +\< { TOKEN(yytext[0]); } +\> { TOKEN(yytext[0]); } +\<= { TOKEN(LE); } +\>= { TOKEN(GE); } +\!= { TOKEN(NEQ); } /* -- boolean operators */ -&& { return AND; } -\|\| { return OR; } -\! { return yytext[0]; } -\<\< { return LEFT; } -\>\> { return RIGHT; } +&& { TOKEN(AND); } +\|\| { TOKEN(OR); } +\! { TOKEN(yytext[0]); } +\<\< { TOKEN(LEFT); } +\>\> { TOKEN(RIGHT); } /* -- quotes */ -::\`\`\{ { return DCQUOTED_BLOCK; } -::\{ { return DCQUOTED_BLOCK; } -\`\`\{ { return QUOTED_BLOCK; } -\`\`\( { return QUOTED_EXPRESSION; } -\`\` { logError ("Lonely doubleqoute", LINE_VAR); return SCANNER_ERROR; } -\` { logError ("Lonely qoute", LINE_VAR); return SCANNER_ERROR; } +::\`\`\{ { TOKEN(DCQUOTED_BLOCK); } +::\{ { TOKEN(DCQUOTED_BLOCK); } +\`\`\{ { TOKEN(QUOTED_BLOCK); } +\`\`\( { TOKEN(QUOTED_EXPRESSION); } +\`\` { logError ("Lonely doubleqoute", LINE_VAR); TOKEN(SCANNER_ERROR); } +\` { logError ("Lonely qoute", LINE_VAR); TOKEN(SCANNER_ERROR); } /* -- bit operators */ \||\^|\&|~ { - return yytext[0]; + TOKEN(yytext[0]); } /* -- math operators */ \+|\-|\*|\/|\% { - return yytext[0]; + TOKEN(yytext[0]); } \]\: { - return CLOSEBRACKET; + TOKEN(CLOSEBRACKET); } \[|\]|\(|\)|,|\{|\}|:|\;|\=|\? { - return yytext[0]; + TOKEN(yytext[0]); } <> { - return END_OF_FILE; + TOKEN(END_OF_FILE); } . { debug_scanner("?%s?", yytext); logError ("Unexpected char '%s'", LINE_VAR, yytext); - return SCANNER_ERROR; + TOKEN(SCANNER_ERROR); }