From 1bf6b7fbc8904735f5c348772f4dd00b21113e14 Mon Sep 17 00:00:00 2001 From: albert-github Date: Tue, 19 Dec 2023 10:36:10 +0100 Subject: [PATCH] Incorrect handling of `\include{doc}` for Python Analogous to the PR 10494 "Incorrect handling of `\include{doc}` for Fortran" this patch gives the handling for `\include{doc}` within Python. Furthermore the `'''` (triple single quote) was not handled in the commentcnv.l, this has been corrected. --- src/commentcnv.l | 33 +++++++++++++++++++-- src/pyscanner.l | 76 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/src/commentcnv.l b/src/commentcnv.l index 806d52cc687..df1db0bfff7 100644 --- a/src/commentcnv.l +++ b/src/commentcnv.l @@ -74,6 +74,7 @@ struct commentcnv_FileState YY_BUFFER_STATE bufState = 0; QCString fileName; QCString oldFileName; + QCString oldIncPrefix; int oldState = 0; BufStr fileBuf; const BufStr *oldFileBuf = nullptr; @@ -107,12 +108,14 @@ struct commentcnvYY_state int charContext = 0; int javaBlock = 0; bool specialComment = FALSE; + QCString incPrefix; QCString aliasString; int blockCount = 0; bool lastEscaped = FALSE; int lastBlockContext= 0; bool pythonDocString = FALSE; + char pythonDocStringChar = '\0'; int nestingCount= 0; bool vhdl = FALSE; // for VHDL old style --! comment @@ -235,6 +238,7 @@ SLASHopt [/]* [,= ;\t] { /* eat , so we have a nice separator in long initialization lines */ copyToOutput(yyscanner,yytext,(int)yyleng); } +"'''"! | "\"\"\""! { /* start of python long comment */ if (yyextra->lang!=SrcLangExt_Python) { @@ -243,13 +247,16 @@ SLASHopt [/]* else { yyextra->pythonDocString = TRUE; + yyextra->pythonDocStringChar = yytext[0]; yyextra->nestingCount=1; clearCommentStack(yyscanner); /* to be on the save side */ copyToOutput(yyscanner,yytext,(int)yyleng); BEGIN(CComment); + yyextra->incPrefix = ""; yyextra->commentStack.push(yyextra->lineNr); } } +"'''" | "\"\"\"" { /* start of python long comment */ if (yyextra->lang!=SrcLangExt_Python) { @@ -262,10 +269,12 @@ SLASHopt [/]* else { /* handle as if """! */ yyextra->pythonDocString = TRUE; + yyextra->pythonDocStringChar = yytext[0]; yyextra->nestingCount=1; clearCommentStack(yyscanner); /* to be on the save side */ copyToOutput(yyscanner,yytext,(int)yyleng); BEGIN(CComment); + yyextra->incPrefix = ""; yyextra->commentStack.push(yyextra->lineNr); } } @@ -705,21 +714,38 @@ SLASHopt [/]* copyToOutput(yyscanner,yytext,(int)yyleng); } -[^ `~<\\!@*\n{\"\/]* { /* anything that is not a '*' or command */ +{B}*"#"("#")? { + if (yyextra->lang!=SrcLangExt_Python) + { + REJECT; + } + else + { + yyextra->incPrefix = yytext; + copyToOutput(yyscanner,yytext,(int)yyleng); + } + } +[^ `~<\\!@*\n{\"'\/]* { /* anything that is not a '*' or command */ copyToOutput(yyscanner,yytext,(int)yyleng); } "*"+[^*\/<\\@\n{\"]* { /* stars without slashes */ copyToOutput(yyscanner,yytext,(int)yyleng); } +"'''" | "\"\"\"" { /* end of Python docstring */ if (yyextra->lang!=SrcLangExt_Python) { REJECT; } + else if (yyextra->pythonDocStringChar != yytext[0]) + { + copyToOutput(yyscanner,yytext,(int)yyleng); + } else { yyextra->nestingCount--; yyextra->pythonDocString = FALSE; + yyextra->pythonDocStringChar = '\0'; copyToOutput(yyscanner,yytext,(int)yyleng); BEGIN(Scan); } @@ -1110,7 +1136,9 @@ SLASHopt [/]* yyextra->inBuf = fs->oldFileBuf; yyextra->inBufPos = fs->oldFileBufPos; yyextra->includeCtx = fs->oldIncludeCtx; - QCString lineStr=" \\ilinebr\\ifile \""+yyextra->fileName+"\" \\iline "+QCString().setNum(yyextra->lineNr)+" "; + yyextra->incPrefix = fs->oldIncPrefix; + QCString lineStr=yyextra->incPrefix + " \\ilinebr\\ifile \""+yyextra->fileName+"\" \\iline "+QCString().setNum(yyextra->lineNr)+" "; + yyextra->incPrefix = ""; copyToOutput(yyscanner,lineStr.data(),lineStr.length()); yyextra->includeStack.pop_back(); //printf("<> switch back to %s line %d inbufPos=%d outbufPos=%d\n", @@ -1411,6 +1439,7 @@ static bool readIncludeFile(yyscan_t yyscanner,const QCString &inc,const QCStrin fs->oldFileBuf = yyextra->inBuf; fs->oldFileBufPos = yyextra->inBufPos; fs->oldIncludeCtx = yyextra->includeCtx; + fs->oldIncPrefix = yyextra->incPrefix; yy_switch_to_buffer(yy_create_buffer(0, YY_BUF_SIZE, yyscanner),yyscanner); yyextra->fileName = absFileName; yyextra->lineNr = lineNr; diff --git a/src/pyscanner.l b/src/pyscanner.l index bb96d56fad8..6bbc57af1e5 100644 --- a/src/pyscanner.l +++ b/src/pyscanner.l @@ -160,8 +160,7 @@ static inline const char *getLexerFILE() {return __FILE__;} /* start command character */ - - +CMD ("\\"|"@") BB [ \t]+ B [ \t]* NEWLINE \n @@ -205,6 +204,17 @@ SCRIPTCOMMENT "#!".* STARTDOCSYMS "##" +LINENR {B}*[1-9][0-9]* +FILEICHAR [a-z_A-Z0-9\\:\\\/\-\+=&#@] +FILEECHAR [a-z_A-Z0-9\-\+=&#@] +FILECHARS {FILEICHAR}*{FILEECHAR}+ +HFILEMASK {FILEICHAR}*("."{FILEICHAR}+)+{FILECHARS}* +VFILEMASK {FILECHARS}("."{FILECHARS})* +FILEMASK {VFILEMASK}|{HFILEMASK} + +IDSYM [a-z_A-Z0-9] +ID [a-z_A-Z%]+{IDSYM}* + %option noyywrap /* Main start state */ @@ -1419,14 +1429,42 @@ STARTDOCSYMS "##" else yyextra->docBlock += yytext; } - [^"'\n \t\\]+ { + [^"'\n \t\\@]+ { yyextra->docBlock += yytext; } \n { incLineNr(yyscanner); yyextra->docBlock += yytext; } - \\. { // escaped char + {CMD}"ifile"{B}+"\""[^\n\"]+"\"" { + yyextra->fileName = &yytext[6]; + yyextra->fileName = yyextra->fileName.stripWhiteSpace(); + yyextra->fileName = yyextra->fileName.mid(1,yyextra->fileName.length()-2); + yyextra->docBlock+=yytext; + } + {CMD}"ifile"{B}+{FILEMASK} { + yyextra->fileName = &yytext[6]; + yyextra->fileName = yyextra->fileName.stripWhiteSpace(); + yyextra->docBlock+=yytext; + } + {CMD}"iline"{LINENR}/[\n\.] | + {CMD}"iline"{LINENR}{B} { + bool ok = false; + int nr = QCString(&yytext[6]).toInt(&ok); + if (!ok) + { + warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '%s' for iline command",yytext); + } + else + { + yyextra->yyLineNr = nr; + } + yyextra->docBlock+=yytext; + } + ({CMD}{CMD}){ID}/[^a-z_A-Z0-9] { // escaped command + yyextra->docBlock+=yytext; + } + \\. { // escaped char TO be extended yyextra->docBlock += yytext; } . { @@ -1442,7 +1480,35 @@ STARTDOCSYMS "##" yyextra->docBrief = FALSE; incLineNr(yyscanner); } - [^#\n]+ { // any other stuff + {CMD}"ifile"{B}+"\""[^\n\"]+"\"" { + yyextra->fileName = &yytext[6]; + yyextra->fileName = yyextra->fileName.stripWhiteSpace(); + yyextra->fileName = yyextra->fileName.mid(1,yyextra->fileName.length()-2); + yyextra->docBlock+=yytext; + } + {CMD}"ifile"{B}+{FILEMASK} { + yyextra->fileName = &yytext[6]; + yyextra->fileName = yyextra->fileName.stripWhiteSpace(); + yyextra->docBlock+=yytext; + } + {CMD}"iline"{LINENR}/[\n\.] | + {CMD}"iline"{LINENR}{B} { + bool ok = false; + int nr = QCString(&yytext[6]).toInt(&ok); + if (!ok) + { + warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '%s' for iline command",yytext); + } + else + { + yyextra->yyLineNr = nr; + } + yyextra->docBlock+=yytext; + } + ({CMD}{CMD}){ID}/[^a-z_A-Z0-9] { // escaped command + yyextra->docBlock+=yytext; + } + [^#\\@\n]+ { // any other stuff yyextra->docBlock+=yytext; } \n { // new line that ends the comment