Consistent way to show scanner state

Create a consistent way to display the state mnemonics of the different scanners (analogous to the fortranscanner.l)
Use an automatic procedure to generate the routine with the translation of the states to a string.

addon/doxywizard/CMakeLists.txt

@@ -72,7 +72,17 @@ OUTPUT ${GENERATED_SRC_WIZARD}/configdoc.cpp
 )
 set_source_files_properties(${GENERATED_SRC_WIZARD}/configdoc.cpp PROPERTIES GENERATED 1)
 
-FLEX_TARGET(config_doxyw config_doxyw.l ${GENERATED_SRC_WIZARD}/config_doxyw.cpp COMPILE_FLAGS "${LEX_FLAGS}")
+set(LEX_FILES config_doxyw) 
+foreach(lex_file ${LEX_FILES})
+    add_custom_command(
+	    COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/src/scan_states.py ${CMAKE_SOURCE_DIR}/addon/doxywizard/${lex_file}.l > ${GENERATED_SRC_WIZARD}/${lex_file}.l.h
+        DEPENDS ${CMAKE_SOURCE_DIR}/src/scan_states.py ${CMAKE_SOURCE_DIR}/addon/doxywizard/${lex_file}.l
+	OUTPUT  ${GENERATED_SRC_WIZARD}/${lex_file}.l.h
+    )
+    set_source_files_properties(${GENERATED_SRC_WIZARD}/${lex_file}.l.h PROPERTIES GENERATED 1)
+
+    FLEX_TARGET(${lex_file}        ${lex_file}.l        ${GENERATED_SRC_WIZARD}/${lex_file}.cpp        COMPILE_FLAGS "${LEX_FLAGS}")
+endforeach()
 
 qt_wrap_cpp(doxywizard_MOC
 doxywizard.h
@@ -97,6 +107,7 @@ inputint.cpp
 inputstrlist.cpp
 ${GENERATED_SRC_WIZARD}/settings.h
 ${GENERATED_SRC_WIZARD}/config_doxyw.cpp
+${GENERATED_SRC_WIZARD}/config_doxyw.l.h
 ${GENERATED_SRC_WIZARD}/configdoc.cpp
 ${doxywizard_MOC}
 ${doxywizard_RESOURCES_RCC}

addon/doxywizard/config_doxyw.l

@@ -64,6 +64,8 @@ static QString                        g_codecName = QString::fromLatin1("UTF-8")
 static int                            g_lastState;
 static QByteArray                     g_tmpString;
 
+static const char *stateToString(int state);
+
 /* -----------------------------------------------------------------
  */
 #undef	YY_INPUT
@@ -585,4 +587,4 @@ void writeStringValue(QTextStream &t,QTextCodec *codec,const QString &s)
     }
   }
 }
-
+#include "config_doxyw.l.h"

libmscgen/CMakeLists.txt

@@ -4,6 +4,18 @@ include_directories(
     ${GENERATED_SRC}
 )
 
+set(LEX_FILES mscgen_lexer) 
+foreach(lex_file ${LEX_FILES})
+    add_custom_command(
+        COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/src/scan_states.py ${CMAKE_SOURCE_DIR}/libmscgen/${lex_file}.l > ${GENERATED_SRC}/${lex_file}.l.h
+        DEPENDS ${CMAKE_SOURCE_DIR}/src/scan_states.py ${CMAKE_SOURCE_DIR}/libmscgen/${lex_file}.l
+        OUTPUT  ${GENERATED_SRC}/${lex_file}.l.h
+    )
+    set_source_files_properties(${GENERATED_SRC}/${lex_file}.l.h PROPERTIES GENERATED 1)
+
+    FLEX_TARGET(${lex_file}        ${lex_file}.l        ${GENERATED_SRC}/${lex_file}.cpp        COMPILE_FLAGS "${LEX_FLAGS}")
+endforeach()
+
 add_library(mscgen
 gd.c
 gd_security.c
@@ -19,6 +31,7 @@ mscgen_ps_out.c
 mscgen_null_out.c
 ${GENERATED_SRC}/mscgen_language.cpp
 ${GENERATED_SRC}/mscgen_lexer.cpp
+${GENERATED_SRC}/mscgen_lexer.l.h
 mscgen_api.c
 mscgen_msc.c
 mscgen_safe.c
@@ -28,10 +41,6 @@ mscgen_utf8.c
 )
 
 
-FLEX_TARGET(mscgen_lexer
-            mscgen_lexer.l
-            ${GENERATED_SRC}/mscgen_lexer.cpp
-            COMPILE_FLAGS "${LEX_FLAGS}")
 BISON_TARGET(mscgen_language
              mscgen_language.y
              ${GENERATED_SRC}/mscgen_language.cpp

libmscgen/mscgen_lexer.l

@@ -41,15 +41,16 @@ static Boolean        lex_utf8 = FALSE;
 /* Local function prototypes */
 static void newline(const char *text, unsigned int n);
 static char *trimQstring(char *s);
-
+static const char *stateToString(int state);
 %}
 
 /* Not used, so prevent compiler warning */
 %option never-interactive
 %option noinput
 %option noyywrap
 
-%x IN_COMMENT BODY
+%x IN_COMMENT
+%x BODY
 %%
 
 <INITIAL>{
@@ -233,4 +234,5 @@ Boolean lex_getutf8(void)
     return lex_utf8;
 }
 
+#include "mscgen_lexer.l.h"
 /* END OF FILE */

src/CMakeLists.txt

@@ -100,25 +100,39 @@ add_custom_command(
 )
 set_source_files_properties(${GENERATED_SRC}/layout_default.xml.h PROPERTIES GENERATED 1)
 
-# Targets for flex/bison   generated files
-FLEX_TARGET(scanner        scanner.l        ${GENERATED_SRC}/scanner.cpp        COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(code           code.l           ${GENERATED_SRC}/code.cpp           COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(pyscanner      pyscanner.l      ${GENERATED_SRC}/pyscanner.cpp      COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(pycode         pycode.l         ${GENERATED_SRC}/pycode.cpp         COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(fortranscanner fortranscanner.l ${GENERATED_SRC}/fortranscanner.cpp COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(fortrancode    fortrancode.l    ${GENERATED_SRC}/fortrancode.cpp    COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(vhdlcode       vhdlcode.l       ${GENERATED_SRC}/vhdlcode.cpp       COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(tclscanner     tclscanner.l     ${GENERATED_SRC}/tclscanner.cpp     COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(pre            pre.l            ${GENERATED_SRC}/pre.cpp            COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(declinfo       declinfo.l       ${GENERATED_SRC}/declinfo.cpp       COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(defargs        defargs.l        ${GENERATED_SRC}/defargs.cpp        COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(doctokenizer   doctokenizer.l   ${GENERATED_SRC}/doctokenizer.cpp   COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(commentcnv     commentcnv.l     ${GENERATED_SRC}/commentcnv.cpp     COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(commentscan    commentscan.l    ${GENERATED_SRC}/commentscan.cpp    COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(constexp       constexp.l       ${GENERATED_SRC}/constexp.cpp       COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(xmlcode        xmlcode.l        ${GENERATED_SRC}/xmlcode.cpp        COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(sqlcode        sqlcode.l        ${GENERATED_SRC}/sqlcode.cpp        COMPILE_FLAGS "${LEX_FLAGS}")
-FLEX_TARGET(configimpl     configimpl.l     ${GENERATED_SRC}/configimpl.cpp     COMPILE_FLAGS "${LEX_FLAGS}")
+set(LEX_FILES scanner 
+    code
+    pyscanner
+    pycode
+    fortranscanner
+    fortrancode
+    vhdlcode
+    tclscanner
+    pre
+    declinfo
+    defargs
+    doctokenizer
+    commentcnv
+    commentscan
+    constexp
+    xmlcode
+    sqlcode
+    configimpl)
+
+# unfortunately ${LEX_FILES_H} and ${LEX_FILES_CPP} don't work in older versions of CMake (like 3.6.2) for add_library
+foreach(lex_file ${LEX_FILES})
+    set(LEX_FILES_H ${LEX_FILES_H} " " ${GENERATED_SRC}/${lex_file}.l.h CACHE INTERNAL "Stores generated files")
+    set(LEX_FILES_CPP ${LEX_FILES_CPP} " " ${GENERATED_SRC}/${lex_file}.cpp CACHE INTERNAL "Stores generated files")
+    add_custom_command(
+        COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/src/scan_states.py ${CMAKE_SOURCE_DIR}/src/${lex_file}.l > ${GENERATED_SRC}/${lex_file}.l.h
+        DEPENDS ${CMAKE_SOURCE_DIR}/src/scan_states.py ${CMAKE_SOURCE_DIR}/src/${lex_file}.l
+        OUTPUT  ${GENERATED_SRC}/${lex_file}.l.h
+    )
+    set_source_files_properties(${GENERATED_SRC}/${lex_file}.l.h PROPERTIES GENERATED 1)
+
+    FLEX_TARGET(${lex_file}        ${lex_file}.l        ${GENERATED_SRC}/${lex_file}.cpp        COMPILE_FLAGS "${LEX_FLAGS}")
+endforeach()
+
 
 BISON_TARGET(constexp      constexp.y       ${GENERATED_SRC}/ce_parse.cpp       COMPILE_FLAGS "${YACC_FLAGS}")
 
@@ -140,24 +154,45 @@ add_library(_doxygen STATIC
     ${GENERATED_SRC}/ce_parse.h
     ${GENERATED_SRC}/configvalues.h
     ${GENERATED_SRC}/resources.cpp
-    # generated by flex/bison
-    ${GENERATED_SRC}/scanner.cpp
+    # generated for/by flex/bison
+    #${LEX_FILES_H} #unfortunately doesn't work in older versions of CMake (like 3.6.2)
+    #${LEX_FILES_CPP} #unfortunately doesn't work in older versions of CMake (like 3.6.2)
+    ${GENERATED_SRC}/code.l.h
+    ${GENERATED_SRC}/commentcnv.l.h
+    ${GENERATED_SRC}/commentscan.l.h
+    ${GENERATED_SRC}/configimpl.l.h
+    ${GENERATED_SRC}/constexp.l.h
+    ${GENERATED_SRC}/declinfo.l.h
+    ${GENERATED_SRC}/defargs.l.h
+    ${GENERATED_SRC}/doctokenizer.l.h
+    ${GENERATED_SRC}/fortrancode.l.h
+    ${GENERATED_SRC}/fortranscanner.l.h
+    ${GENERATED_SRC}/pre.l.h
+    ${GENERATED_SRC}/pycode.l.h
+    ${GENERATED_SRC}/pyscanner.l.h
+    ${GENERATED_SRC}/scanner.l.h
+    ${GENERATED_SRC}/sqlcode.l.h
+    ${GENERATED_SRC}/tclscanner.l.h
+    ${GENERATED_SRC}/vhdlcode.l.h
+    ${GENERATED_SRC}/xmlcode.l.h
     ${GENERATED_SRC}/code.cpp
-    ${GENERATED_SRC}/pyscanner.cpp
-    ${GENERATED_SRC}/pycode.cpp
-    ${GENERATED_SRC}/fortranscanner.cpp
-    ${GENERATED_SRC}/fortrancode.cpp
-    ${GENERATED_SRC}/vhdlcode.cpp
-    ${GENERATED_SRC}/tclscanner.cpp
-    ${GENERATED_SRC}/pre.cpp
-    ${GENERATED_SRC}/declinfo.cpp
-    ${GENERATED_SRC}/defargs.cpp
-    ${GENERATED_SRC}/doctokenizer.cpp
     ${GENERATED_SRC}/commentcnv.cpp
     ${GENERATED_SRC}/commentscan.cpp
+    ${GENERATED_SRC}/configimpl.cpp
     ${GENERATED_SRC}/constexp.cpp
-    ${GENERATED_SRC}/xmlcode.cpp
+    ${GENERATED_SRC}/declinfo.cpp
+    ${GENERATED_SRC}/defargs.cpp
+    ${GENERATED_SRC}/doctokenizer.cpp
+    ${GENERATED_SRC}/fortrancode.cpp
+    ${GENERATED_SRC}/fortranscanner.cpp
+    ${GENERATED_SRC}/pre.cpp
+    ${GENERATED_SRC}/pycode.cpp
+    ${GENERATED_SRC}/pyscanner.cpp
+    ${GENERATED_SRC}/scanner.cpp
     ${GENERATED_SRC}/sqlcode.cpp
+    ${GENERATED_SRC}/tclscanner.cpp
+    ${GENERATED_SRC}/vhdlcode.cpp
+    ${GENERATED_SRC}/xmlcode.cpp
     #
     ${GENERATED_SRC}/ce_parse.cpp
     #
@@ -251,6 +286,10 @@ add_library(_doxygen STATIC
     docgroup.cpp
 )
 
+##foreach(lex_file ${LEX_FILES})
+##add_library(_doxygen STATIC ${GENERATED_SRC}/${lex_file}.l.h)
+##endforeach()
+
 add_executable(doxygen main.cpp)
 
 if (use_libclang)

src/code.l

@@ -165,6 +165,8 @@ static int g_braceCount=0;
 static void saveObjCContext();
 static void restoreObjCContext();
 
+static const char *stateToString(int state);
+
 static QCString g_forceTagReference;
 
 
@@ -3875,3 +3877,4 @@ extern "C" { // some bogus code to keep the compiler happy
 #error "You seem to be using a version of flex newer than 2.5.4. These are currently incompatible with 2.5.4, and do NOT work with doxygen! Please use version 2.5.4 or expect things to be parsed wrongly! A bug report has been submitted (#732132)."
 #endif
 
+#include "code.l.h"

src/commentcnv.l

@@ -94,6 +94,8 @@ static bool     g_vhdl; // for VHDL old style --! comment
 static SrcLangExt g_lang;
 static bool       isFixedForm; // For Fortran
 
+static const char *stateToString(int state);
+
 static void replaceCommentMarker(const char *s,int len)
 {
   const char *p=s;
@@ -1145,3 +1147,4 @@ extern "C" { // some bogus code to keep the compiler happy
 }
 #endif
 
+#include "commentcnv.l.h"

src/commentscan.l

@@ -132,6 +132,8 @@ static bool handleEndParBlock(const QCString &, const QCStringList &);
 static bool handleParam(const QCString &, const QCStringList &);
 static bool handleRetval(const QCString &, const QCStringList &);
 
+static const char *stateToString(int state);
+
 typedef bool (*DocCmdFunc)(const QCString &name, const QCStringList &optList);
 
 struct DocCmdMap
@@ -3278,3 +3280,4 @@ extern "C" { // some bogus code to keep the compiler happy
 }
 #endif
 
+#include "commentscan.l.h"

src/configimpl.l

@@ -43,6 +43,8 @@
 #define YY_NO_INPUT 1
 #define YY_NO_UNISTD_H 1
 
+static const char *stateToString(int state);
+
 static const char *warning_str = "warning: ";
 static const char *error_str = "error: ";
 
@@ -1882,3 +1884,4 @@ void Config::deinit()
   ConfigImpl::instance()->deleteInstance();
 }
 
+#include "configimpl.l.h"

src/constexp.l

@@ -32,6 +32,7 @@
 #define YY_NO_UNISTD_H 1
 
 
+static const char *stateToString(int state);
 static int yyread(char *buf,int max_size,yyscan_t yyscanner);
 
 #undef  YY_INPUT
@@ -139,3 +140,4 @@ bool parseconstexp(const char *fileName,int lineNr,const QCString &s)
 extern "C" {
   int constexpYYwrap(yyscan_t yyscanner) { return 1; }
 }
+#include "constexp.l.h"

src/declinfo.l

@@ -63,6 +63,7 @@ struct declinfoYY_state
      bool         insidePHP;
 };
 
+static const char *stateToString(int state);
 static void addType(yyscan_t yyscanner);
 static void addTypeName(yyscan_t yyscanner);
 static int yyread(char *buf,int max_size, yyscan_t yyscanner);
@@ -391,3 +392,4 @@ extern "C" { // some bogus code to keep the compiler happy
 }
 #endif
 
+#include "declinfo.l.h"

src/defargs.l

@@ -87,6 +87,7 @@ static int              g_lastDocChar;
 static int              g_lastExtendsContext;
 static QCString         g_delimiter;
 
+static const char *stateToString(int state);
 /* -----------------------------------------------------------------
  */
 #undef	YY_INPUT
@@ -637,3 +638,4 @@ extern "C" { // some bogus code to keep the compiler happy
 }
 #endif
 
+#include "defargs.l.h"

src/doctokenizer.l

@@ -76,6 +76,7 @@ struct DocLexerContext
 
 static QStack<DocLexerContext> g_lexerStack;
 
+static const char *stateToString(int state);
 //--------------------------------------------------------------------------
 
 void doctokenizerYYpushContext()
@@ -1615,3 +1616,4 @@ extern "C" { // some bogus code to keep the compiler happy
 }
 #endif
 
+#include "doctokenizer.l.h"

src/fortrancode.l

@@ -166,6 +166,8 @@ static int           inTypeDecl = 0;
 
 static bool      g_endComment;
 
+static const char *stateToString(int state);
+
 static void endFontClass()
 {
   if (g_currentFontClass)
@@ -1405,3 +1407,4 @@ extern "C" { // some bogus code to keep the compiler happy
 }
 #endif
 
+#include "fortrancode.l.h"