diff --git a/Doxyfile b/Doxyfile index 44eb5b524be..100347becb7 100644 --- a/Doxyfile +++ b/Doxyfile @@ -106,10 +106,12 @@ WARN_LOGFILE = warnings.log # Configuration options related to the input files #--------------------------------------------------------------------------- INPUT = src \ - vhdlparser + vhdlparser \ + xml INPUT_ENCODING = UTF-8 FILE_PATTERNS = *.h \ *.cpp \ + *.l \ *.md RECURSIVE = NO EXCLUDE = diff --git a/doc/features.doc b/doc/features.doc index e746ac39f17..9c42d5bc4f0 100644 --- a/doc/features.doc +++ b/doc/features.doc @@ -28,7 +28,7 @@ output even from undocumented code.
  • Generates structured XML output for parsed sources, which can be used by external tools. -
  • Supports C/C++, Java, (Corba and Microsoft) Java, Python, VHDL, PHP +
  • Supports C/C++, Lex, Java, (Corba and Microsoft) Java, Python, VHDL, PHP IDL, C#, Fortran, Objective-C 2.0, and to some extent D sources.
  • Supports documentation of files, namespaces, packages, classes, structs, unions, templates, variables, functions, typedefs, enums and diff --git a/doc/starting.doc b/doc/starting.doc index ad34af3aa1b..44f2a92a925 100644 --- a/doc/starting.doc +++ b/doc/starting.doc @@ -37,7 +37,7 @@ tries to be complete): \section step0 Step 0: Check if doxygen supports your programming language First, assure that your programming language has a reasonable chance of being -recognized by doxygen. These languages are supported by default: C, C++, C#, +recognized by doxygen. These languages are supported by default: C, C++, Lex, C#, Objective-C, IDL, Java, VHDL, PHP, Python, Fortran and D. It is possible to configure certain file type extensions to use certain parsers: see the \ref cfg_extension_mapping "Configuration/ExtensionMappings" for details. @@ -120,7 +120,7 @@ Extension | Language | Extension | Language | Extension | Language .ixx |C / C++ | .php5 |PHP | .vhdl |VHDL .ipp |C / C++ | .inc |PHP | .ucf |VHDL .i++ |C / C++ | .phtml |PHP | .qsf |VHDL -.inl |C / C++ | .m |Objective-C |   |  +.inl |C / C++ | .m |Objective-C | .l |Lex .h |C / C++ | .M |Objective-C | .md |Markdown .H |C / C++ | .py |Python | .markdown |Markdown .hh |C / C++ | .pyw |Python | .ice |Slice diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c6af8131196..e85d8f2ef6b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -117,6 +117,8 @@ set(LEX_FILES scanner constexp xmlcode sqlcode + lexcode + lexscanner configimpl) # unfortunately ${LEX_FILES_H} and ${LEX_FILES_CPP} don't work in older versions of CMake (like 3.6.2) for add_library @@ -184,6 +186,8 @@ add_library(doxymain STATIC ${GENERATED_SRC}/doctokenizer.l.h ${GENERATED_SRC}/fortrancode.l.h ${GENERATED_SRC}/fortranscanner.l.h + ${GENERATED_SRC}/lexcode.l.h + ${GENERATED_SRC}/lexscanner.l.h ${GENERATED_SRC}/pre.l.h ${GENERATED_SRC}/pycode.l.h ${GENERATED_SRC}/pyscanner.l.h @@ -199,6 +203,8 @@ add_library(doxymain STATIC ${GENERATED_SRC}/doctokenizer.cpp ${GENERATED_SRC}/fortrancode.cpp ${GENERATED_SRC}/fortranscanner.cpp + ${GENERATED_SRC}/lexcode.cpp + ${GENERATED_SRC}/lexscanner.cpp ${GENERATED_SRC}/pre.cpp ${GENERATED_SRC}/pycode.cpp ${GENERATED_SRC}/pyscanner.cpp diff --git a/src/code.h b/src/code.h index 42265ad506c..ba242423808 100644 --- a/src/code.h +++ b/src/code.h @@ -46,6 +46,7 @@ class CCodeParser : public CodeParserInterface bool collectXRefs=TRUE ); void resetCodeParserState(); + void setStartCodeLine(const bool inp); private: struct Private; std::unique_ptr p; diff --git a/src/code.l b/src/code.l index 233a4f0741c..083660b8cc4 100644 --- a/src/code.l +++ b/src/code.l @@ -98,6 +98,11 @@ struct codeYY_state QCString parmType; QCString parmName; + bool beginCodeLine = true; //!< signals whether or not we should with the first line + //!< write a start line code or not. Essential + //!< when this code parser is called from another + //!< code parser. + const char * inputString = 0; //!< the code fragment as text yy_size_t inputPosition = 0; //!< read offset during parsing int inputLines = 0; //!< number of line in the code fragment @@ -3788,6 +3793,12 @@ void CCodeParser::resetCodeParserState() yyextra->anchorCount = 0; } +void CCodeParser::setStartCodeLine(const bool inp) +{ + struct yyguts_t *yyg = (struct yyguts_t*)p->yyscanner; + yyextra->beginCodeLine = inp; +} + void CCodeParser::parseCode(CodeOutputInterface &od,const char *className,const QCString &s, SrcLangExt lang,bool exBlock, const char *exName,FileDef *fd, int startLine,int endLine,bool inlineFragment, @@ -3862,7 +3873,8 @@ void CCodeParser::parseCode(CodeOutputInterface &od,const char *className,const } yyextra->includeCodeFragment = inlineFragment; //printf("** exBlock=%d exName=%s include=%d\n",exBlock,exName,inlineFragment); - startCodeLine(yyscanner); + + if (yyextra->beginCodeLine) startCodeLine(yyscanner); yyextra->type.resize(0); yyextra->name.resize(0); yyextra->args.resize(0); diff --git a/src/config.xml b/src/config.xml index d9b29aa23ae..8526a2bb090 100644 --- a/src/config.xml +++ b/src/config.xml @@ -647,7 +647,7 @@ Go to the next section or return to the With this tag you can assign which parser to use for a given extension. Doxygen has a built-in mapping, but you can override or extend it using this tag. The format is ext=language, where \c ext is a file extension, and language is one of - the parsers supported by doxygen: IDL, Java, JavaScript, Csharp (C#), C, C++, D, PHP, + the parsers supported by doxygen: IDL, Java, JavaScript, Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: Fortran. In the later case the parser tries to guess whether the code is fixed or free @@ -1434,6 +1434,7 @@ FILE_VERSION_FILTER = "cleartool desc -fmt \%Vn" + diff --git a/src/context.cpp b/src/context.cpp index d2cfa9ee7b6..e0017fbab5a 100644 --- a/src/context.cpp +++ b/src/context.cpp @@ -1540,6 +1540,7 @@ class DefinitionContext case SrcLangExt_SQL: result="sql"; break; case SrcLangExt_Markdown: result="markdown"; break; case SrcLangExt_Slice: result="slice"; break; + case SrcLangExt_Lex: result="lex"; break; } return result; } diff --git a/src/defargs.l b/src/defargs.l index e25c1fe081b..76d07ec425f 100644 --- a/src/defargs.l +++ b/src/defargs.l @@ -27,12 +27,12 @@ * The Argument list as a whole can be pure, constant or volatile. * * Examples of input strings are: - * \code + * \verbatim * "(int a,int b) const" * "(const char *s="hello world",int=5) = 0" * "" * "(char c,const char)" - * \endcode + * \endverbatim * * Note: It is not always possible to distinguish between the name and * type of an argument. In case of doubt the name is added to the diff --git a/src/docsets.cpp b/src/docsets.cpp index 4f04623c54e..9c29818f953 100644 --- a/src/docsets.cpp +++ b/src/docsets.cpp @@ -328,6 +328,7 @@ void DocSets::addIndexItem(const Definition *context,const MemberDef *md, case SrcLangExt_SQL: lang="sql"; break; // Sql case SrcLangExt_Markdown:lang="markdown"; break; // Markdown case SrcLangExt_Slice: lang="slice"; break; // Slice + case SrcLangExt_Lex: lang="lex"; break; // Lex case SrcLangExt_Unknown: lang="unknown"; break; // should not happen! } diff --git a/src/doxygen.cpp b/src/doxygen.cpp index f5a280be6e7..c268f2667e1 100644 --- a/src/doxygen.cpp +++ b/src/doxygen.cpp @@ -77,6 +77,8 @@ #include "fortranscanner.h" #include "xmlcode.h" #include "sqlcode.h" +#include "lexcode.h" +#include "lexscanner.h" #include "code.h" #include "portable.h" #include "vhdljjparser.h" @@ -10058,6 +10060,8 @@ void initDoxygen() make_parser_factory()); Doxygen::parserManager->registerParser("md", make_parser_factory(), make_parser_factory()); + Doxygen::parserManager->registerParser("lex", make_parser_factory(), + make_parser_factory()); // register any additional parsers here... diff --git a/src/lexcode.h b/src/lexcode.h new file mode 100644 index 00000000000..38aec508e0c --- /dev/null +++ b/src/lexcode.h @@ -0,0 +1,57 @@ +/****************************************************************************** + * + * Copyright (C) 1997-2021 by Dimitri van Heesch. + * + * Permission to use, copy, modify, and distribute this software and its + * documentation under the terms of the GNU General Public License is hereby + * granted. No representations are made about the suitability of this software + * for any purpose. It is provided "as is" without express or implied warranty. + * See the GNU General Public License for more details. + * + * Documents produced by Doxygen are derivative works derived from the + * input used in their production; they are not affected by this license. + * + */ + + +#ifndef LEXCODE_H +#define LEXCODE_H + +#include "parserintf.h" + +class CodeOutputInterface; +class FileDef; +class MemberDef; +class QCString; +class Definition; + +/** LEX code scanner. + */ +class LexCodeParser : public CodeParserInterface +{ + public: + LexCodeParser(); + virtual ~LexCodeParser(); + void parseCode(CodeOutputInterface &codeOutIntf, + const char *scopeName, + const QCString &input, + SrcLangExt, + bool isExampleBlock, + const char *exampleName=0, + FileDef *fileDef=0, + int startLine=-1, + int endLine=-1, + bool inlineFragment=FALSE, + const MemberDef *memberDef=0, + bool showLineNumbers=TRUE, + const Definition *searchCtx=0, + bool collectXRefs=TRUE + ); + void resetCodeParserState(); + private: + struct Private; + std::unique_ptr p; +}; + + +#endif diff --git a/src/lexcode.l b/src/lexcode.l new file mode 100644 index 00000000000..d6b41b02833 --- /dev/null +++ b/src/lexcode.l @@ -0,0 +1,1223 @@ +/****************************************************************************** + * + * Copyright (C) 1997-2021 by Dimitri van Heesch. + * + * Permission to use, copy, modify, and distribute this software and its + * documentation under the terms of the GNU General Public License is hereby + * granted. No representations are made about the suitability of this software + * for any purpose. It is provided "as is" without express or implied warranty. + * See the GNU General Public License for more details. + * + * Documents produced by Doxygen are derivative works derived from the + * input used in their production; they are not affected by this license. + * + */ + +%option never-interactive +%option prefix="lexcodeYY" +%option noyywrap +%option reentrant +%option extra-type="struct lexcodeYY_state *" +%top{ +#include +} + +%{ + +#include + +#include "config.h" +#include "doxygen.h" +#include "outputgen.h" +#include "code.h" +#include "lexcode.h" +#include "filedef.h" +#include "message.h" + +#define YY_NEVER_INTERACTIVE 1 +#define YY_NO_INPUT 1 +#define YY_NO_UNISTD_H 1 + +#define USE_STATE2STRING 0 + +struct lexcodeYY_state +{ + CodeOutputInterface * code; + CCodeParser ccodeParser; + const char *inputString; //!< the code fragment as text + yy_size_t inputPosition; //!< read offset during parsing + int inputLines; //!< number of line in the code fragment + int yyLineNr; //!< current line number + bool needsTermination; + + bool lineNumbers = FALSE; + const Definition *searchCtx; + bool collectXRefs = FALSE; + + int lastContext = 0; + int lastCContext = 0; + int lastStringContext = 0; + int docBlockContext = 0; + int lastPreLineCtrlContext = 0; + int lastRawStringContext = 0; + int curlyCount = 0; + + QCString rulesPatternBuffer; + QCString CCodeBuffer; + int startCCodeLine = -1; + int roundCount = 0; + int squareCount = 0; + bool insideCode = FALSE; + QCString delimiter; + QCString docBlockName; + uint fencedSize = 0; + bool nestedComment = false; + + bool exampleBlock; + QCString exampleName; + QCString classScope; + + FileDef *sourceFileDef; + const Definition *currentDefinition; + const MemberDef *currentMemberDef; + bool includeCodeFragment; + const char *currentFontClass; +}; + +#if USE_STATE2STRING +static const char *stateToString(int state); +#endif + +static void setCurrentDoc(yyscan_t yyscanner,const QCString &anchor); +static void startCodeLine(yyscan_t yyscanner); +static void endFontClass(yyscan_t yyscanner); +static void endCodeLine(yyscan_t yyscanner); +static void nextCodeLine(yyscan_t yyscanner); +static void codifyLines(yyscan_t yyscanner,const char *text); +static void startFontClass(yyscan_t yyscanner,const char *s); +static int countLines(yyscan_t yyscanner); +static yy_size_t yyread(yyscan_t yyscanner,char *buf,yy_size_t max_size); +static void lineCount(yyscan_t yyscanner); +static void handleCCode(yyscan_t yyscanner); + +#undef YY_INPUT +#define YY_INPUT(buf,result,max_size) result=yyread(yyscanner,buf,max_size); + +%} + +nl (\r\n|\r|\n) +ws [ \t] +nws [^ \t\n] +TopStart "%top{"{nl} +TopEnd "}"{nl} +LiteralStart "%{"{nl} +LiteralEnd "%}"{nl} +RulesStart "%%"{nl} +RulesEnd "%%"{nl} +RulesSharp "<"[^>]*">" +RulesCurly "{"[^{}\n]*"}" +StartSquare "[" +StartDouble "\"" +StartRound "(" +EscapeRulesCharOpen "\\["|"\\<"|"\\{"|"\\("|"\\\""|"\\{"|"\\ " +EscapeRulesCharClose "\\]"|"\\>"|"\\}"|"\\)" +EscapeRulesChar {EscapeRulesCharOpen}|{EscapeRulesCharClose} + +CMD ("\\"|"@") +BN [ \t\n\r] +BL [ \t\r]*"\n" +B [ \t] +ID "$"?[a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF]* +PRE [pP][rR][eE] +CODE [cC][oO][dD][eE] +RAWBEGIN (u|U|L|u8)?R\"[^ \t\(\)\\]{0,16}"(" +RAWEND ")"[^ \t\(\)\\]{0,16}\" +CHARLIT (("'"\\[0-7]{1,3}"'")|("'"\\."'")|("'"[^'\\\n]{1,4}"'")) + +%x DefSection +%x DefSectionLine +%x RulesSectionInit +%x RulesPattern +%x RulesDouble +%x RulesRoundDouble +%x RulesSquare +%x RulesRoundSquare +%x RulesRound +%x UserSection + +%x TopSection +%x LiteralSection + +%x COMMENT + +%x SkipCurly +%x SkipCurlyEndDoc +%x PreLineCtrl +%x DocLine +%x DocBlock +%x DocCopyBlock +%x SkipString +%x RawString +%x SkipComment +%x SkipCxxComment +%x Comment + +%% + +<*>\x0d +^{TopStart} { + handleCCode(yyscanner); + codifyLines(yyscanner,yytext); + yyextra->lastContext = YY_START; + yyextra->startCCodeLine=yyextra->yyLineNr; + BEGIN (TopSection); + } +^{LiteralStart} { + handleCCode(yyscanner); + codifyLines(yyscanner,yytext); + yyextra->lastContext = YY_START; + yyextra->startCCodeLine=yyextra->yyLineNr; + BEGIN (LiteralSection); + } +^{TopEnd} { + handleCCode(yyscanner); + codifyLines(yyscanner,yytext); + BEGIN( yyextra->lastContext ) ; + } +.*{nl} { + yyextra->CCodeBuffer += yytext; + yyextra->yyLineNr++; + } +^{LiteralEnd} { + handleCCode(yyscanner); + codifyLines(yyscanner,yytext); + BEGIN( yyextra->lastContext ) ; + } +.*{nl} { + yyextra->CCodeBuffer += yytext; + yyextra->yyLineNr++; + } +"//".*{nl} { + yyextra->CCodeBuffer += yytext; + yyextra->yyLineNr++; + } +^{ws}*"/*" { + yyextra->CCodeBuffer += yytext; + yyextra->lastContext = YY_START; + BEGIN(COMMENT); + } +"*/"{ws}*{nl} { + yyextra->CCodeBuffer+=yytext; + yyextra->yyLineNr++; + handleCCode(yyscanner); + BEGIN(yyextra->lastContext); + } +"*/" { + yyextra->CCodeBuffer+=yytext; + handleCCode(yyscanner); + BEGIN(yyextra->lastContext); + } +[^*\n]+ { + yyextra->CCodeBuffer += yytext; + } +"//"|"/*" { + yyextra->CCodeBuffer += yytext; + } +{nl} { + yyextra->CCodeBuffer += yytext; + yyextra->yyLineNr++; + } +. { + yyextra->CCodeBuffer += yytext; + } +^{nl} { + handleCCode(yyscanner); + codifyLines(yyscanner,yytext); + yyextra->startCCodeLine=yyextra->yyLineNr; + } +^{ws}.*{nl} { + yyextra->CCodeBuffer += yytext; + yyextra->yyLineNr++; + } +^{RulesStart} { + handleCCode(yyscanner); + codifyLines(yyscanner,yytext); + yyextra->startCCodeLine=yyextra->yyLineNr; + BEGIN (RulesSectionInit); + } +^{nws} { + handleCCode(yyscanner); + codifyLines(yyscanner,yytext); + BEGIN(DefSectionLine); + } +.*{nl} { + codifyLines(yyscanner,yytext); + yyextra->startCCodeLine=yyextra->yyLineNr; + BEGIN(DefSection); + } +^{RulesEnd} { + handleCCode(yyscanner); + codifyLines(yyscanner,yytext); + yyextra->startCCodeLine=yyextra->yyLineNr; + BEGIN (UserSection); + } +^{nws} { + handleCCode(yyscanner); + unput(*yytext); + BEGIN(RulesPattern); + } +{nl} { + yyextra->CCodeBuffer += yytext; + yyextra->yyLineNr++; + } +^{ws}.*{nl} { + yyextra->CCodeBuffer += yytext; + yyextra->yyLineNr++; + } +"<>" { + yyextra->rulesPatternBuffer += yytext; + } +{EscapeRulesChar} { + yyextra->rulesPatternBuffer += yytext; + } +{RulesSharp} { + yyextra->rulesPatternBuffer += yytext; + } +{RulesCurly} { + yyextra->rulesPatternBuffer += yytext; + } +{StartDouble} { + yyextra->rulesPatternBuffer += yytext; + yyextra->lastContext = YY_START; + BEGIN(RulesDouble); + } +"\\\\" { + yyextra->rulesPatternBuffer += yytext; + } +"\\\"" { + yyextra->rulesPatternBuffer += yytext; + } +"\"" { + yyextra->rulesPatternBuffer += yytext; + BEGIN( yyextra->lastContext ) ; + } +"\"" { + yyextra->rulesPatternBuffer += yytext; + BEGIN(RulesRound) ; + } +. { + yyextra->rulesPatternBuffer += yytext; + } +{StartSquare} { + yyextra->squareCount++; + yyextra->rulesPatternBuffer += yytext; + yyextra->lastContext = YY_START; + BEGIN(RulesSquare); + } +"\\[" | +"\\]" { + yyextra->rulesPatternBuffer += yytext; + } +"[" { + yyextra->squareCount++; + yyextra->rulesPatternBuffer += yytext; + } +"]" { + yyextra->squareCount--; + yyextra->rulesPatternBuffer += yytext; + if (!yyextra->squareCount) BEGIN(RulesPattern) ; + } +"]" { + yyextra->squareCount--; + yyextra->rulesPatternBuffer += yytext; + if (!yyextra->squareCount) BEGIN(RulesRound) ; + } +"\\\\" { + yyextra->rulesPatternBuffer += yytext; + } +. { + yyextra->rulesPatternBuffer += yytext; + } +{StartRound} { + yyextra->roundCount++; + yyextra->rulesPatternBuffer += yytext; + yyextra->lastContext = YY_START; + BEGIN(RulesRound); + } +{RulesCurly} { + yyextra->rulesPatternBuffer += yytext; + } +{StartSquare} { + yyextra->squareCount++; + yyextra->rulesPatternBuffer += yytext; + BEGIN(RulesRoundSquare); + } +{StartDouble} { + yyextra->rulesPatternBuffer += yytext; + BEGIN(RulesRoundDouble); + } +"\\(" | +"\\)" { + yyextra->rulesPatternBuffer += yytext; + } +"(" { + yyextra->roundCount++; + yyextra->rulesPatternBuffer += yytext; + } +")" { + yyextra->roundCount--; + yyextra->rulesPatternBuffer += yytext; + if (!yyextra->roundCount) BEGIN( yyextra->lastContext ) ; + } +. { + yyextra->rulesPatternBuffer += yytext; + } +{ws}+"|" { + if (!yyextra->rulesPatternBuffer.isEmpty()) + { + startFontClass(yyscanner,"stringliteral"); + codifyLines(yyscanner,yyextra->rulesPatternBuffer.data()); + yyextra->rulesPatternBuffer.resize(0); + endFontClass(yyscanner); + } + codifyLines(yyscanner,yytext); + yyextra->startCCodeLine=yyextra->yyLineNr; + yyextra->curlyCount = 0; + BEGIN(SkipCurly); + } +^{ws}*{nl} { + codifyLines(yyscanner,"\n"); + } +^{ws}+ { + codifyLines(yyscanner,yytext); + } +({ws}|{nl}) { + unput(*yytext); + if (!yyextra->rulesPatternBuffer.isEmpty()) + { + startFontClass(yyscanner,"stringliteral"); + codifyLines(yyscanner,yyextra->rulesPatternBuffer.data()); + yyextra->rulesPatternBuffer.resize(0); + endFontClass(yyscanner); + } + yyextra->startCCodeLine=yyextra->yyLineNr; + yyextra->curlyCount = 0; + BEGIN(SkipCurly); + } +"\\\\" { + yyextra->rulesPatternBuffer += yytext; + } +"/*" { + if (!yyextra->rulesPatternBuffer.isEmpty()) + { + startFontClass(yyscanner,"stringliteral"); + codifyLines(yyscanner,yyextra->rulesPatternBuffer.data()); + yyextra->rulesPatternBuffer.resize(0); + endFontClass(yyscanner); + } + yyextra->CCodeBuffer += yytext; + yyextra->lastContext = YY_START; + BEGIN(COMMENT); + } +. { + yyextra->rulesPatternBuffer += yytext; + } +{B}*"#"{B}+[0-9]+{B}+/"\"" { /* line control directive */ + yyextra->CCodeBuffer += yytext; + yyextra->lastPreLineCtrlContext = YY_START; + BEGIN( PreLineCtrl ); + } +"\""[^\n\"]*"\"" { + yyextra->CCodeBuffer += yytext; + } +. { + yyextra->CCodeBuffer += yytext; + } +\n { + yyextra->CCodeBuffer += yytext; + yyextra->yyLineNr++; + BEGIN( yyextra->lastPreLineCtrlContext ); + } +"{" { + yyextra->CCodeBuffer += yytext; + ++yyextra->curlyCount ; + } +"}"/{BN}*("/*!"|"/**"|"//!"|"///")"