Skip to content

Commit

Permalink
Incorrect handling of \include{doc} for Python
Browse files Browse the repository at this point in the history 
Analogous to the PR 10494 "Incorrect handling of `\include{doc}` for Fortran"  this patch gives the handling for `\include{doc}` within Python.
Furthermore the `'''` (triple single quote) was not handled in the commentcnv.l, this has been corrected.
  • Loading branch information
@albert-github
albert-github committed Dec 19, 2023
1 parent 351dba6 commit 1bf6b7f
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 7 deletions.
33 changes: 31 additions & 2 deletions src/commentcnv.l
View file
Expand Up @@ -74,6 +74,7 @@ struct commentcnv_FileState
YY_BUFFER_STATE bufState = 0;
QCString fileName;
QCString oldFileName;
QCString oldIncPrefix;
int oldState = 0;
BufStr fileBuf;
const BufStr *oldFileBuf = nullptr;
Expand Down Expand Up @@ -107,12 +108,14 @@ struct commentcnvYY_state
int charContext = 0;
int javaBlock = 0;
bool specialComment = FALSE;
QCString incPrefix;

QCString aliasString;
int blockCount = 0;
bool lastEscaped = FALSE;
int lastBlockContext= 0;
bool pythonDocString = FALSE;
char pythonDocStringChar = '\0';
int nestingCount= 0;

bool vhdl = FALSE; // for VHDL old style --! comment
Expand Down Expand Up @@ -235,6 +238,7 @@ SLASHopt [/]*
<Scan>[,= ;\t] { /* eat , so we have a nice separator in long initialization lines */
copyToOutput(yyscanner,yytext,(int)yyleng);
}
<Scan>"'''"! |
<Scan>"\"\"\""! { /* start of python long comment */
if (yyextra->lang!=SrcLangExt_Python)
{
Expand All @@ -243,13 +247,16 @@ SLASHopt [/]*
else
{
yyextra->pythonDocString = TRUE;
yyextra->pythonDocStringChar = yytext[0];
yyextra->nestingCount=1;
clearCommentStack(yyscanner); /* to be on the save side */
copyToOutput(yyscanner,yytext,(int)yyleng);
BEGIN(CComment);
yyextra->incPrefix = "";
yyextra->commentStack.push(yyextra->lineNr);
}
}
<Scan>"'''" |
<Scan>"\"\"\"" { /* start of python long comment */
if (yyextra->lang!=SrcLangExt_Python)
{
Expand All @@ -262,10 +269,12 @@ SLASHopt [/]*
else
{ /* handle as if """! */
yyextra->pythonDocString = TRUE;
yyextra->pythonDocStringChar = yytext[0];
yyextra->nestingCount=1;
clearCommentStack(yyscanner); /* to be on the save side */
copyToOutput(yyscanner,yytext,(int)yyleng);
BEGIN(CComment);
yyextra->incPrefix = "";
yyextra->commentStack.push(yyextra->lineNr);
}
}
Expand Down Expand Up @@ -705,21 +714,38 @@ SLASHopt [/]*
copyToOutput(yyscanner,yytext,(int)yyleng);
}
<CComment,CNComment>[^ `~<\\!@*\n{\"\/]* { /* anything that is not a '*' or command */
<CComment>{B}*"#"("#")? {
if (yyextra->lang!=SrcLangExt_Python)
{
REJECT;
}
else
{
yyextra->incPrefix = yytext;
copyToOutput(yyscanner,yytext,(int)yyleng);
}
}
<CComment,CNComment>[^ `~<\\!@*\n{\"'\/]* { /* anything that is not a '*' or command */
copyToOutput(yyscanner,yytext,(int)yyleng);
}
<CComment,CNComment>"*"+[^*\/<\\@\n{\"]* { /* stars without slashes */
copyToOutput(yyscanner,yytext,(int)yyleng);
}
<CComment>"'''" |
<CComment>"\"\"\"" { /* end of Python docstring */
if (yyextra->lang!=SrcLangExt_Python)
{
REJECT;
}
else if (yyextra->pythonDocStringChar != yytext[0])
{
copyToOutput(yyscanner,yytext,(int)yyleng);
}
else
{
yyextra->nestingCount--;
yyextra->pythonDocString = FALSE;
yyextra->pythonDocStringChar = '\0';
copyToOutput(yyscanner,yytext,(int)yyleng);
BEGIN(Scan);
}
Expand Down Expand Up @@ -1110,7 +1136,9 @@ SLASHopt [/]*
yyextra->inBuf = fs->oldFileBuf;
yyextra->inBufPos = fs->oldFileBufPos;
yyextra->includeCtx = fs->oldIncludeCtx;
QCString lineStr=" \\ilinebr\\ifile \""+yyextra->fileName+"\" \\iline "+QCString().setNum(yyextra->lineNr)+" ";
yyextra->incPrefix = fs->oldIncPrefix;
QCString lineStr=yyextra->incPrefix + " \\ilinebr\\ifile \""+yyextra->fileName+"\" \\iline "+QCString().setNum(yyextra->lineNr)+" ";
yyextra->incPrefix = "";
copyToOutput(yyscanner,lineStr.data(),lineStr.length());
yyextra->includeStack.pop_back();
//printf("<<EOF>> switch back to %s line %d inbufPos=%d outbufPos=%d\n",
Expand Down Expand Up @@ -1411,6 +1439,7 @@ static bool readIncludeFile(yyscan_t yyscanner,const QCString &inc,const QCStrin
fs->oldFileBuf = yyextra->inBuf;
fs->oldFileBufPos = yyextra->inBufPos;
fs->oldIncludeCtx = yyextra->includeCtx;
fs->oldIncPrefix = yyextra->incPrefix;
yy_switch_to_buffer(yy_create_buffer(0, YY_BUF_SIZE, yyscanner),yyscanner);
yyextra->fileName = absFileName;
yyextra->lineNr = lineNr;
Expand Down
76 changes: 71 additions & 5 deletions src/pyscanner.l
View file
Expand Up @@ -160,8 +160,7 @@ static inline const char *getLexerFILE() {return __FILE__;}

/* start command character */



CMD ("\\"|"@")
BB [ \t]+
B [ \t]*
NEWLINE \n
Expand Down Expand Up @@ -205,6 +204,17 @@ SCRIPTCOMMENT "#!".*
STARTDOCSYMS "##"
LINENR {B}*[1-9][0-9]*
FILEICHAR [a-z_A-Z0-9\\:\\\/\-\+=&#@]
FILEECHAR [a-z_A-Z0-9\-\+=&#@]
FILECHARS {FILEICHAR}*{FILEECHAR}+
HFILEMASK {FILEICHAR}*("."{FILEICHAR}+)+{FILECHARS}*
VFILEMASK {FILECHARS}("."{FILECHARS})*
FILEMASK {VFILEMASK}|{HFILEMASK}
IDSYM [a-z_A-Z0-9]
ID [a-z_A-Z%]+{IDSYM}*
%option noyywrap
/* Main start state */
Expand Down Expand Up @@ -1419,14 +1429,42 @@ STARTDOCSYMS "##"
else
yyextra->docBlock += yytext;
}
[^"'\n \t\\]+ {
[^"'\n \t\\@]+ {
yyextra->docBlock += yytext;
}
\n {
incLineNr(yyscanner);
yyextra->docBlock += yytext;
}
\\. { // escaped char
{CMD}"ifile"{B}+"\""[^\n\"]+"\"" {
yyextra->fileName = &yytext[6];
yyextra->fileName = yyextra->fileName.stripWhiteSpace();
yyextra->fileName = yyextra->fileName.mid(1,yyextra->fileName.length()-2);
yyextra->docBlock+=yytext;
}
{CMD}"ifile"{B}+{FILEMASK} {
yyextra->fileName = &yytext[6];
yyextra->fileName = yyextra->fileName.stripWhiteSpace();
yyextra->docBlock+=yytext;
}
{CMD}"iline"{LINENR}/[\n\.] |
{CMD}"iline"{LINENR}{B} {
bool ok = false;
int nr = QCString(&yytext[6]).toInt(&ok);
if (!ok)
{
warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '%s' for iline command",yytext);
}
else
{
yyextra->yyLineNr = nr;
}
yyextra->docBlock+=yytext;
}
({CMD}{CMD}){ID}/[^a-z_A-Z0-9] { // escaped command
yyextra->docBlock+=yytext;
}
\\. { // escaped char TO be extended
yyextra->docBlock += yytext;
}
. {
Expand All @@ -1442,7 +1480,35 @@ STARTDOCSYMS "##"
yyextra->docBrief = FALSE;
incLineNr(yyscanner);
}
[^#\n]+ { // any other stuff
{CMD}"ifile"{B}+"\""[^\n\"]+"\"" {
yyextra->fileName = &yytext[6];
yyextra->fileName = yyextra->fileName.stripWhiteSpace();
yyextra->fileName = yyextra->fileName.mid(1,yyextra->fileName.length()-2);
yyextra->docBlock+=yytext;
}
{CMD}"ifile"{B}+{FILEMASK} {
yyextra->fileName = &yytext[6];
yyextra->fileName = yyextra->fileName.stripWhiteSpace();
yyextra->docBlock+=yytext;
}
{CMD}"iline"{LINENR}/[\n\.] |
{CMD}"iline"{LINENR}{B} {
bool ok = false;
int nr = QCString(&yytext[6]).toInt(&ok);
if (!ok)
{
warn(yyextra->fileName,yyextra->yyLineNr,"Invalid line number '%s' for iline command",yytext);
}
else
{
yyextra->yyLineNr = nr;
}
yyextra->docBlock+=yytext;
}
({CMD}{CMD}){ID}/[^a-z_A-Z0-9] { // escaped command
yyextra->docBlock+=yytext;
}
[^#\\@\n]+ { // any other stuff
yyextra->docBlock+=yytext;
}
\n { // new line that ends the comment
Expand Down

0 comments on commit 1bf6b7f

Please sign in to comment.