issue #10567 Incorrectly closed markdown code block causes doxygen to forget about consecutive macro definitions

albert-github · albert-github · commit 98242a3e3721 · 2024-01-19T13:04:13.000+01:00
An exact match of the strings for starting and ending fenced code blocks was required, this has been relaxed to that the number of fence characters should match.

Also protecting that the block starts and ends with the same type of fence character.
diff --git a/src/pre.l b/src/pre.l
@@ -293,6 +293,7 @@ struct preYY_state
   bool               isSource       = false;
 
   yy_size_t          fenceSize      = 0;
+  char               fenceChar      = ' ';
   bool               ccomment       = false;
   QCString           delimiter;
   bool               isSpecialComment = false;
@@ -340,6 +341,7 @@ static void         setFileName(yyscan_t yyscanner,const QCString &name);
 static int               yyread(yyscan_t yyscanner,char *buf,int max_size);
 static Define *       isDefined(yyscan_t yyscanner,const QCString &name);
 static void  determineBlockName(yyscan_t yyscanner);
+static yy_size_t   getFenceSize(char *txt, yy_size_t leng);
 
 /* ----------------------------------------------------------------- */
 
@@ -1422,7 +1424,8 @@ WSopt [ \t\r]*
                                           }
                                           else
                                           {
-                                            yyextra->fenceSize=(int)yyleng;
+                                            yyextra->fenceChar='~';
+                                            yyextra->fenceSize=(int)getFenceSize(yytext,yyleng);
                                             BEGIN(SkipCondVerbatim);
                                           }
                                         }
@@ -1434,7 +1437,8 @@ WSopt [ \t\r]*
                                           }
                                           else
                                           {
-                                            yyextra->fenceSize=(int)yyleng;
+                                            yyextra->fenceChar='`';
+                                            yyextra->fenceSize=(int)getFenceSize(yytext,yyleng);
                                             BEGIN(SkipCondVerbatim);
                                           }
                                         }
@@ -1447,7 +1451,8 @@ WSopt [ \t\r]*
                                           else
                                           {
                                             outputArray(yyscanner,yytext,yyleng);
-                                            yyextra->fenceSize=(int)yyleng;
+                                            yyextra->fenceChar='~';
+                                            yyextra->fenceSize=(int)getFenceSize(yytext,yyleng);
                                             BEGIN(SkipVerbatim);
                                           }
                                         }
@@ -1460,7 +1465,8 @@ WSopt [ \t\r]*
                                           else
                                           {
                                             outputArray(yyscanner,yytext,yyleng);
-                                            yyextra->fenceSize=(int)yyleng;
+                                            yyextra->fenceChar='`';
+                                            yyextra->fenceSize=(int)getFenceSize(yytext,yyleng);
                                             BEGIN(SkipVerbatim);
                                           }
                                         }
@@ -1626,27 +1632,27 @@ WSopt [ \t\r]*
                                           }
                                         }
 <SkipCondVerbatim>^({B}*"*"+)?{B}{0,3}"~~~"[~]*                 {
-                                          if (yyextra->fenceSize==(yy_size_t)yyleng)
+                                          if (yyextra->fenceSize==getFenceSize(yytext,yyleng) &&  yyextra->fenceChar=='~') 
                                           {
                                             BEGIN(SkipCond);
                                           }
                                         }
 <SkipCondVerbatim>^({B}*"*"+)?{B}{0,3}"```"[`]*                 {
-                                          if (yyextra->fenceSize==(yy_size_t)yyleng)
+                                          if (yyextra->fenceSize==getFenceSize(yytext,yyleng) &&  yyextra->fenceChar=='`')
                                           {
                                             BEGIN(SkipCond);
                                           }
                                         }
 <SkipVerbatim>^({B}*"*"+)?{B}{0,3}"~~~"[~]*                 {
                                           outputArray(yyscanner,yytext,yyleng);
-                                          if (yyextra->fenceSize==(yy_size_t)yyleng)
+                                          if (yyextra->fenceSize==getFenceSize(yytext,yyleng) &&  yyextra->fenceChar=='~')
                                           {
                                             BEGIN(SkipCComment);
                                           }
                                         }
 <SkipVerbatim>^({B}*"*"+)?{B}{0,3}"```"[`]*                 {
                                           outputArray(yyscanner,yytext,yyleng);
-                                          if (yyextra->fenceSize==(yy_size_t)yyleng)
+                                          if (yyextra->fenceSize==getFenceSize(yytext,yyleng) &&  yyextra->fenceChar=='`')
                                           {
                                             BEGIN(SkipCComment);
                                           }
@@ -2083,6 +2089,17 @@ static int yyread(yyscan_t yyscanner,char *buf,int max_size)
   return bytesToCopy;
 }
 
+static yy_size_t getFenceSize(char *txt, yy_size_t leng)
+{
+  yy_size_t fenceSize = 0;
+  for (size_t i = 0; i < leng; i++)
+  {
+     if (txt[i] != ' ' && txt[i] != '*' && txt[i] != '\t') break;
+     fenceSize++;
+  }
+  return leng-fenceSize;
+}
+
 static void setFileName(yyscan_t yyscanner,const QCString &name)
 {
   YY_EXTRA_TYPE state = preYYget_extra(yyscanner);
diff --git a/src/scanner.l b/src/scanner.l
@@ -7114,15 +7114,15 @@ NONLopt [^\n]*
 <DocCopyBlock>^({B}*"*"+)?{B}{0,3}"~~~"[~]* {
                                           QCString pat = substitute(yytext,"*"," ");
                                           yyextra->docBlock << pat;
-                                          if (yyextra->fencedSize==pat.stripWhiteSpace().length())
+                                          if (yyextra->docBlockName == "~~~" && yyextra->fencedSize==pat.stripWhiteSpace().length())
                                           {
                                             BEGIN(DocBlock);
                                           }
                                         }
 <DocCopyBlock>^({B}*"*"+)?{B}{0,3}"```"[`]*                 {
                                           QCString pat = substitute(yytext,"*"," ");
                                           yyextra->docBlock << pat;
-                                          if (yyextra->fencedSize==pat.stripWhiteSpace().length())
+                                          if (yyextra->docBlockName == "```" && yyextra->fencedSize==pat.stripWhiteSpace().length())
                                           {
                                             BEGIN(DocBlock);
                                           }
@@ -7448,6 +7448,7 @@ static inline int computeIndent(const char *s,int startIndent)
   }
   return col;
 }
+
 static inline void initMethodProtection(yyscan_t yyscanner,Protection prot)
 {
   struct yyguts_t *yyg = (struct yyguts_t*)yyscanner;

Original file line number	Diff line number	Diff line change
`@@ -293,6 +293,7 @@ struct preYY_state`
`293`	`293`	`bool isSource = false;`
`294`	`294`
`295`	`295`	`yy_size_t fenceSize = 0;`
	`296`	`+ char fenceChar = ' ';`
`296`	`297`	`bool ccomment = false;`
`297`	`298`	`QCString delimiter;`
`298`	`299`	`bool isSpecialComment = false;`
`@@ -340,6 +341,7 @@ static void setFileName(yyscan_t yyscanner,const QCString &name);`
`340`	`341`	`static int yyread(yyscan_t yyscanner,char *buf,int max_size);`
`341`	`342`	`static Define * isDefined(yyscan_t yyscanner,const QCString &name);`
`342`	`343`	`static void determineBlockName(yyscan_t yyscanner);`
	`344`	`+static yy_size_t getFenceSize(char *txt, yy_size_t leng);`
`343`	`345`
`344`	`346`	`/* ----------------------------------------------------------------- */`
`345`	`347`
`@@ -1422,7 +1424,8 @@ WSopt [ \t\r]*`
`1422`	`1424`	`}`
`1423`	`1425`	`else`
`1424`	`1426`	`{`
`1425`		`- yyextra->fenceSize=(int)yyleng;`
	`1427`	`+ yyextra->fenceChar='~';`
	`1428`	`+ yyextra->fenceSize=(int)getFenceSize(yytext,yyleng);`
`1426`	`1429`	`BEGIN(SkipCondVerbatim);`
`1427`	`1430`	`}`
`1428`	`1431`	`}`
`@@ -1434,7 +1437,8 @@ WSopt [ \t\r]*`
`1434`	`1437`	`}`
`1435`	`1438`	`else`
`1436`	`1439`	`{`
`1437`		`- yyextra->fenceSize=(int)yyleng;`
	`1440`	+ yyextra->fenceChar='`';
	`1441`	`+ yyextra->fenceSize=(int)getFenceSize(yytext,yyleng);`
`1438`	`1442`	`BEGIN(SkipCondVerbatim);`
`1439`	`1443`	`}`
`1440`	`1444`	`}`
`@@ -1447,7 +1451,8 @@ WSopt [ \t\r]*`
`1447`	`1451`	`else`
`1448`	`1452`	`{`
`1449`	`1453`	`outputArray(yyscanner,yytext,yyleng);`
`1450`		`- yyextra->fenceSize=(int)yyleng;`
	`1454`	`+ yyextra->fenceChar='~';`
	`1455`	`+ yyextra->fenceSize=(int)getFenceSize(yytext,yyleng);`
`1451`	`1456`	`BEGIN(SkipVerbatim);`
`1452`	`1457`	`}`
`1453`	`1458`	`}`
`@@ -1460,7 +1465,8 @@ WSopt [ \t\r]*`
`1460`	`1465`	`else`
`1461`	`1466`	`{`
`1462`	`1467`	`outputArray(yyscanner,yytext,yyleng);`
`1463`		`- yyextra->fenceSize=(int)yyleng;`
	`1468`	+ yyextra->fenceChar='`';
	`1469`	`+ yyextra->fenceSize=(int)getFenceSize(yytext,yyleng);`
`1464`	`1470`	`BEGIN(SkipVerbatim);`
`1465`	`1471`	`}`
`1466`	`1472`	`}`
`@@ -1626,27 +1632,27 @@ WSopt [ \t\r]*`
`1626`	`1632`	`}`
`1627`	`1633`	`}`
`1628`	`1634`	`<SkipCondVerbatim>^({B}""+)?{B}{0,3}"~~~"[~]* {`
`1629`		`- if (yyextra->fenceSize==(yy_size_t)yyleng)`
	`1635`	`+ if (yyextra->fenceSize==getFenceSize(yytext,yyleng) && yyextra->fenceChar=='~')`
`1630`	`1636`	`{`
`1631`	`1637`	`BEGIN(SkipCond);`
`1632`	`1638`	`}`
`1633`	`1639`	`}`
`1634`	`1640`	<SkipCondVerbatim>^({B}""+)?{B}{0,3}"```"[`]* {
`1635`		`- if (yyextra->fenceSize==(yy_size_t)yyleng)`
	`1641`	+ if (yyextra->fenceSize==getFenceSize(yytext,yyleng) && yyextra->fenceChar=='`')
`1636`	`1642`	`{`
`1637`	`1643`	`BEGIN(SkipCond);`
`1638`	`1644`	`}`
`1639`	`1645`	`}`
`1640`	`1646`	`<SkipVerbatim>^({B}""+)?{B}{0,3}"~~~"[~]* {`
`1641`	`1647`	`outputArray(yyscanner,yytext,yyleng);`
`1642`		`- if (yyextra->fenceSize==(yy_size_t)yyleng)`
	`1648`	`+ if (yyextra->fenceSize==getFenceSize(yytext,yyleng) && yyextra->fenceChar=='~')`
`1643`	`1649`	`{`
`1644`	`1650`	`BEGIN(SkipCComment);`
`1645`	`1651`	`}`
`1646`	`1652`	`}`
`1647`	`1653`	<SkipVerbatim>^({B}""+)?{B}{0,3}"```"[`]* {
`1648`	`1654`	`outputArray(yyscanner,yytext,yyleng);`
`1649`		`- if (yyextra->fenceSize==(yy_size_t)yyleng)`
	`1655`	+ if (yyextra->fenceSize==getFenceSize(yytext,yyleng) && yyextra->fenceChar=='`')
`1650`	`1656`	`{`
`1651`	`1657`	`BEGIN(SkipCComment);`
`1652`	`1658`	`}`
`@@ -2083,6 +2089,17 @@ static int yyread(yyscan_t yyscanner,char *buf,int max_size)`
`2083`	`2089`	`return bytesToCopy;`
`2084`	`2090`	`}`
`2085`	`2091`
	`2092`	`+static yy_size_t getFenceSize(char *txt, yy_size_t leng)`
	`2093`	`+{`
	`2094`	`+ yy_size_t fenceSize = 0;`
	`2095`	`+ for (size_t i = 0; i < leng; i++)`
	`2096`	`+ {`
	`2097`	`+ if (txt[i] != ' ' && txt[i] != '*' && txt[i] != '\t') break;`
	`2098`	`+ fenceSize++;`
	`2099`	`+ }`
	`2100`	`+ return leng-fenceSize;`
	`2101`	`+}`
	`2102`	`+`
`2086`	`2103`	`static void setFileName(yyscan_t yyscanner,const QCString &name)`
`2087`	`2104`	`{`
`2088`	`2105`	`YY_EXTRA_TYPE state = preYYget_extra(yyscanner);`
Original file line number	Diff line number	Diff line change
`@@ -7114,15 +7114,15 @@ NONLopt [^\n]*`
`7114`	`7114`	`<DocCopyBlock>^({B}""+)?{B}{0,3}"~~~"[~]* {`
`7115`	`7115`	`QCString pat = substitute(yytext,"*"," ");`
`7116`	`7116`	`yyextra->docBlock << pat;`
`7117`		`- if (yyextra->fencedSize==pat.stripWhiteSpace().length())`
	`7117`	`+ if (yyextra->docBlockName == "~~~" && yyextra->fencedSize==pat.stripWhiteSpace().length())`
`7118`	`7118`	`{`
`7119`	`7119`	`BEGIN(DocBlock);`
`7120`	`7120`	`}`
`7121`	`7121`	`}`
`7122`	`7122`	<DocCopyBlock>^({B}""+)?{B}{0,3}"```"[`]* {
`7123`	`7123`	`QCString pat = substitute(yytext,"*"," ");`
`7124`	`7124`	`yyextra->docBlock << pat;`
`7125`		`- if (yyextra->fencedSize==pat.stripWhiteSpace().length())`
	`7125`	+ if (yyextra->docBlockName == "```" && yyextra->fencedSize==pat.stripWhiteSpace().length())
`7126`	`7126`	`{`
`7127`	`7127`	`BEGIN(DocBlock);`
`7128`	`7128`	`}`
`@@ -7448,6 +7448,7 @@ static inline int computeIndent(const char *s,int startIndent)`
`7448`	`7448`	`}`
`7449`	`7449`	`return col;`
`7450`	`7450`	`}`
	`7451`	`+`
`7451`	`7452`	`static inline void initMethodProtection(yyscan_t yyscanner,Protection prot)`
`7452`	`7453`	`{`
`7453`	`7454`	`struct yyguts_t yyg = (struct yyguts_t)yyscanner;`