From fdefd3091128296d8f572f7daba6d838de24bf4b Mon Sep 17 00:00:00 2001 From: Dimitri van Heesch Date: Mon, 27 Jul 2020 14:15:58 +0200 Subject: [PATCH] Additional tweaks to get markdown tables inside ALIASES work - Changed \_linebr to \ilinebr - \ilinebr is now also passed to doctokenizer - Also fixes issue #7493 regarding \snippet inside markdown tables and dealing with wrong line on issues detected by docparser after a markdown table. - Added function tracing to markdown (enabled with -d markdown in a debug build) --- doc/custcmd.doc | 26 +++---- src/commentscan.l | 62 ++++++++--------- src/doctokenizer.l | 61 +++++++++++------ src/doxygen.cpp | 4 +- src/markdown.cpp | 164 +++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 235 insertions(+), 82 deletions(-) diff --git a/doc/custcmd.doc b/doc/custcmd.doc index 02805da8e38..c6be4c41440 100644 --- a/doc/custcmd.doc +++ b/doc/custcmd.doc @@ -1,12 +1,12 @@ /****************************************************************************** * - * + * * * Copyright (C) 1997-2015 by Dimitri van Heesch. * * Permission to use, copy, modify, and distribute this software and its - * documentation under the terms of the GNU General Public License is hereby - * granted. No representations are made about the suitability of this software + * documentation under the terms of the GNU General Public License is hereby + * granted. No representations are made about the suitability of this software * for any purpose. It is provided "as is" without express or implied warranty. * See the GNU General Public License for more details. * @@ -18,11 +18,11 @@ \tableofcontents{html,latex} -Doxygen provides a large number of \ref commands "special commands", +Doxygen provides a large number of \ref commands "special commands", \ref xmlcmds "XML commands", and \ref htmlcmds "HTML commands". -that can be used to enhance or structure the documentation inside a comment block. +that can be used to enhance or structure the documentation inside a comment block. If you for some reason have a need to define new commands you can do -so by means of an \e alias definition. +so by means of an \e alias definition. The definition of an alias should be specified in the configuration file using the \ref cfg_aliases "ALIASES" configuration tag. @@ -32,15 +32,15 @@ The simplest form of an alias is a simple substitution of the form \verbatim name=value \endverbatim - For example defining the following alias: + For example defining the following alias: \verbatim - ALIASES += sideeffect="\par Side Effects:\n" + ALIASES += sideeffect="\par Side Effects:\n" \endverbatim will allow you to - put the command `\sideeffect` (or `@sideeffect`) in the documentation, which + put the command `\sideeffect` (or `@sideeffect`) in the documentation, which will result in a user-defined paragraph with heading Side Effects:. -Note that you can put `\n`'s in the value part of an alias to insert newlines +Note that you cannot put `\n`'s in the value part of an alias to insert newlines (in the resulting output). You can put `^^` in the value part of an alias to insert a newline as if a physical newline was in the original file. @@ -52,7 +52,7 @@ use a double escape (\c \\\\{ and \c \\\\}) Also note that you can redefine existing special commands if you wish. Some commands, such as \ref cmdxrefitem "\\xrefitem" are designed to be used in -combination with aliases. +combination with aliases. \section custcmd_complex Aliases with arguments Aliases can also have one or more arguments. In the alias definition you then need @@ -101,9 +101,9 @@ ALIASES += reminder="\xreflist{reminders,Reminder,Reminders}" Note that if for aliases with more than one argument a comma is used as a separator, if you want to put a comma inside the command, you will need to escape it with a backslash, -i.e. +i.e. \verbatim -\l{SomeClass,Some text\, with an escaped comma} +\l{SomeClass,Some text\, with an escaped comma} \endverbatim given the alias definition of `\l` in the example above. diff --git a/src/commentscan.l b/src/commentscan.l index 1923e4d3dd6..da8e5779d2d 100644 --- a/src/commentscan.l +++ b/src/commentscan.l @@ -163,7 +163,7 @@ struct DocCmdMap static const std::map< std::string, DocCmdMap > docCmdMap = { // command name handler function command spacing - { "_linebr", { &handleLineBr, CommandSpacing::Inline }}, + { "ilinebr", { &handleLineBr, CommandSpacing::Inline }}, { "addindex", { &handleAddIndex, CommandSpacing::Invisible }}, { "addtogroup", { &handleAddToGroup, CommandSpacing::Invisible }}, { "anchor", { &handleAnchor, CommandSpacing::Invisible }}, @@ -472,7 +472,7 @@ BL [ \t\r]*"\n" B [ \t] BS ^(({B}*"//")?)(({B}*"*"+)?){B}* ATTR ({B}+[^>\n]*)? -DOCNL "\n"|"\\_linebr" +DOCNL "\n"|"\\ilinebr" LC "\\"{B}*"\n" NW [^a-z_A-Z0-9] FILESCHAR [a-z_A-Z0-9\x80-\xFF\\:\\\/\-\+@&#] @@ -505,34 +505,34 @@ RCSTAG "$"{ID}":"[^\n$]+"$" %x XRefItemParam3 %x FileDocArg1 %x ParamArg1 -%x EnumDocArg1 -%x NameSpaceDocArg1 -%x PackageDocArg1 -%x GroupDocArg1 -%x GroupDocArg2 -%x SectionLabel -%x SectionTitle -%x SubpageLabel -%x SubpageTitle -%x FormatBlock -%x LineParam -%x GuardParam -%x GuardParamEnd -%x SkipGuardedSection -%x SkipInternal +%x EnumDocArg1 +%x NameSpaceDocArg1 +%x PackageDocArg1 +%x GroupDocArg1 +%x GroupDocArg2 +%x SectionLabel +%x SectionTitle +%x SubpageLabel +%x SubpageTitle +%x FormatBlock +%x LineParam +%x GuardParam +%x GuardParamEnd +%x SkipGuardedSection +%x SkipInternal %x NameParam -%x InGroupParam -%x FnParam -%x OverloadParam -%x InheritParam -%x ExtendsParam +%x InGroupParam +%x FnParam +%x OverloadParam +%x InheritParam +%x ExtendsParam %x ReadFormulaShort -%x ReadFormulaLong -%x AnchorLabel +%x ReadFormulaLong +%x AnchorLabel %x HtmlComment %x SkipLang -%x CiteLabel -%x CopyDoc +%x CiteLabel +%x CopyDoc %x GuardExpr %x CdataSection %x Noop @@ -833,7 +833,7 @@ RCSTAG "$"{ID}":"[^\n$]+"$" ".."[\.]?/[^ \t\n] { // internal ellipsis addOutput(yyscanner,yytext); } -(\n|\\_linebr)({B}*(\n|\\_linebr))+ { // at least one blank line (or blank line command) +(\n|\\ilinebr)({B}*(\n|\\ilinebr))+ { // at least one blank line (or blank line command) if (yyextra->inContext==OutputXRef) { // see bug 613024, we need to put the newlines after ending the XRef section. @@ -842,7 +842,7 @@ RCSTAG "$"{ID}":"[^\n$]+"$" for (i=0;i<(yy_size_t)yyleng;) { if (yytext[i]=='\n') addOutput(yyscanner,'\n'),i++; - else if (strcmp(yytext+i,"\\_linebr")==0) addOutput(yyscanner,'\n'),i+=8; + else if (strcmp(yytext+i,"\\ilinebr")==0) addOutput(yyscanner,"\\ilinebr"),i+=8; else i++; } } @@ -852,7 +852,7 @@ RCSTAG "$"{ID}":"[^\n$]+"$" for (i=0;i<(yy_size_t)yyleng;) { if (yytext[i]=='\n') addOutput(yyscanner,'\n'),i++; - else if (strcmp(yytext+i,"\\_linebr")==0) addOutput(yyscanner,'\n'),i+=8; + else if (strcmp(yytext+i,"\\ilinebr")==0) addOutput(yyscanner,"\\ilinebr"),i+=8; else i++; } setOutput(yyscanner,OutputDoc); @@ -1342,7 +1342,7 @@ RCSTAG "$"{ID}":"[^\n$]+"$" addOutput(yyscanner,yytext); BEGIN( Comment ); } -[^\n@\\]*/"\\_linebr" { // end of section title +[^\n@\\]*/"\\ilinebr" { // end of section title addSection(yyscanner); addOutput(yyscanner,yytext); BEGIN( Comment ); @@ -2540,7 +2540,7 @@ static bool handleInternal(yyscan_t yyscanner,const QCString &, const QCStringLi static bool handleLineBr(yyscan_t yyscanner,const QCString &, const QCStringList &) { - addOutput(yyscanner,'\n'); + addOutput(yyscanner,"\\ilinebr"); return FALSE; } diff --git a/src/doctokenizer.l b/src/doctokenizer.l index b1aa82fcf41..6ea39d99db1 100644 --- a/src/doctokenizer.l +++ b/src/doctokenizer.l @@ -114,6 +114,20 @@ bool doctokenizerYYpopContext() return TRUE; } +QCString extractPartAfterNewLine(const QCString &text) +{ + int nl1 = text.findRev('\n'); + if (nl1!=-1) + { + return text.mid(nl1+1); + } + int nl2 = text.findRev("\\ilinebr"); + if (nl2!=-1) + { + return text.mid(nl2+8); + } + return text; +} //-------------------------------------------------------------------------- @@ -520,25 +534,23 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} return TK_LISTITEM; } } -{BLANK}*\n{LISTITEM} { /* list item on next line */ - QCString text=yytext; - text=text.right(text.length()-text.find('\n')-1); +{BLANK}*(\n|"\\ilinebr"){LISTITEM} { /* list item on next line */ + QCString text=extractPartAfterNewLine(yytext); int dashPos = text.findRev('-'); g_token->isEnumList = text.at(dashPos+1)=='#'; g_token->id = -1; g_token->indent = computeIndent(text,dashPos); return TK_LISTITEM; } -{BLANK}*\n{MLISTITEM} { /* list item on next line */ +{BLANK}*(\n|"\\ilinebr"){MLISTITEM} { /* list item on next line */ if (!g_markdownSupport || g_insidePre) { REJECT; } else { - QCString text=yytext; + QCString text=extractPartAfterNewLine(yytext); static QRegExp re("[*+]"); - text=text.right(text.length()-text.find('\n')-1); int markPos = text.findRev(re); g_token->isEnumList = FALSE; g_token->id = -1; @@ -546,17 +558,14 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} return TK_LISTITEM; } } -{BLANK}*\n{OLISTITEM} { /* list item on next line */ +{BLANK}*(\n|"\\ilinebr"){OLISTITEM} { /* list item on next line */ if (!g_markdownSupport || g_insidePre) { REJECT; } else { - QCString text=yytext; - int nl=text.findRev('\n'); - int len=text.length(); - text=text.right(len-nl-1); + QCString text=extractPartAfterNewLine(yytext); static QRegExp re("[1-9]"); int digitPos = text.find(re); int dotPos = text.find('.',digitPos); @@ -571,11 +580,10 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->indent = computeIndent(yytext,dotPos); return TK_ENDLIST; } -{BLANK}*\n{ENDLIST} { /* end list on next line */ - QCString text=yytext; - text=text.right(text.length()-text.find('\n')-1); +{BLANK}*(\n|"\\ilinebr"){ENDLIST} { /* end list on next line */ + QCString text=extractPartAfterNewLine(yytext); int dotPos = text.findRev('.'); - g_token->indent = computeIndent(text,dotPos); + g_token->indent = computeIndent(text,dotPos); return TK_ENDLIST; } "{"{BLANK}*"@link"/{BLANK}+ { @@ -586,9 +594,9 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->name = "inheritdoc"; return TK_COMMAND_AT; } -"@_fakenl" { // artificial new line - yylineno++; - } +"@_fakenl" { // artificial new line + yylineno++; + } {SPCMD3} { g_token->name = "_form"; bool ok; @@ -603,6 +611,8 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} g_token->paramDir=TokenInfo::Unspecified; return TK_COMMAND_SEL(); } +"\\ilinebr" { + } {SPCMD1} | {SPCMD2} | {SPCMD5} | @@ -1367,17 +1377,23 @@ REFWORD_NOCV {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV} warn(g_fileName,yylineno,"Unexpected character '%s' while looking for section label or title",yytext); } -[^\n]+ | -[^\n]*\n { - g_token->name = yytext; +[^\\\n]+ { + g_token->name += yytext; + } +"\\" { + g_token->name += yytext; + } +(\n|"\\ilinebr") { g_token->name = g_token->name.stripWhiteSpace(); - return TK_WORD; + return TK_WORD; } /* Generic rules that work for all states */ <*>\n { warn(g_fileName,yylineno,"Unexpected new line character"); } +<*>"\\ilinebr" { + } <*>[\\@<>&$#%~"=] { /* unescaped special character */ //warn(g_fileName,yylineno,"Unexpected character '%s', assuming command \\%s was meant.",yytext,yytext); g_token->name = yytext; @@ -1571,6 +1587,7 @@ void doctokenizerYYsetStateAnchor() void doctokenizerYYsetStateSnippet() { + g_token->name=""; BEGIN(St_Snippet); } diff --git a/src/doxygen.cpp b/src/doxygen.cpp index f073051ade4..bc8eed7ac1e 100644 --- a/src/doxygen.cpp +++ b/src/doxygen.cpp @@ -9628,7 +9628,7 @@ static void escapeAliases() value.mid(in,14)!="\\nosubgrouping" ) { - newValue+="\\_linebr "; + newValue+="\\ilinebr "; } else { @@ -9643,7 +9643,7 @@ static void escapeAliases() while ((in=value.find("^^",p))!=-1) { newValue+=value.mid(p,in-p); - newValue+="\\\\_linebr "; + newValue+="\\ilinebr "; p=in+2; } newValue+=value.mid(p,value.length()-p); diff --git a/src/markdown.cpp b/src/markdown.cpp index 34a99357835..2add54c9b40 100644 --- a/src/markdown.cpp +++ b/src/markdown.cpp @@ -52,6 +52,90 @@ #include "message.h" #include "portable.h" +#if !defined(NDEBUG) +#define ENABLE_TRACING +#endif + +#ifdef ENABLE_TRACING +#define IOSTREAM stdout +#define DATA_BUFSIZE 20 +#if defined(_WIN32) && !defined(CYGWIN) +#define PRETTY_FUNC __FUNCSIG__ +#else +#define PRETTY_FUNC __PRETTY_FUNCTION__ +#endif + +class Trace +{ + public: + Trace(const char *func) : m_func(func) + { + if (Debug::isFlagSet(Debug::Markdown)) + { + fprintf(IOSTREAM,"> %s\n",func); + s_indent++; + } + } + Trace(const char *func,const char *data) : m_func(func) + { + if (Debug::isFlagSet(Debug::Markdown)) + { + indent(); + char data_s[DATA_BUFSIZE*2+1] = ""; // worst case each input char outputs 2 chars + 0 terminator. + int j=0; + if (data) + { + for (int i=0;i %s data=[%s…]\n",func,data_s); + s_indent++; + } + } + ~Trace() + { + if (Debug::isFlagSet(Debug::Markdown)) + { + s_indent--; + indent(); + fprintf(IOSTREAM,"< %s\n",m_func); + } + } + void trace(const char *fmt,...) + { + if (Debug::isFlagSet(Debug::Markdown)) + { + indent(); + fprintf(IOSTREAM,": %s: ",m_func); + va_list args; + va_start(args,fmt); + vfprintf(IOSTREAM, fmt, args); + va_end(args); + } + } + private: + void indent() { for (int i=0;i0 && data[-1]=='{'; bool isEscaped = offset>0 && (data[-1]=='\\' || data[-1]=='@'); if (isEscaped) return QCString(); @@ -260,6 +349,7 @@ QCString Markdown::isBlockCommand(const char *data,int offset,int size) */ int Markdown::findEmphasisChar(const char *data, int size, char c, int c_size) { + TRACE(data); int i = 1; while (i0 && data[-1]=='\\') return 0; // escaped < // find the end of the html tag @@ -616,11 +712,13 @@ int Markdown::processHtmlTagWrite(const char *data,int offset,int size,bool doWr int Markdown::processHtmlTag(const char *data,int offset,int size) { + TRACE(data); return processHtmlTagWrite(data,offset,size,true); } int Markdown::processEmphasis(const char *data,int offset,int size) { + TRACE(data); if ((offset>0 && !isOpenEmphChar(-1)) || // invalid char before * or _ (size>1 && data[0]!=data[1] && !(isIdChar(1) || extraChar(1) || data[1]=='[')) || // invalid char after * or _ (size>2 && data[0]==data[1] && !(isIdChar(2) || extraChar(2) || data[2]=='['))) // invalid char after ** or __ @@ -686,6 +784,7 @@ void Markdown::writeMarkdownImage(const char *fmt, bool explicitTitle, int Markdown::processLink(const char *data,int,int size) { + TRACE(data); QCString content; QCString link; QCString title; @@ -985,6 +1084,7 @@ int Markdown::processLink(const char *data,int,int size) /** '`' parsing a code span (assuming codespan != 0) */ int Markdown::processCodeSpan(const char *data, int /*offset*/, int size) { + TRACE(data); int end, nb = 0, i, f_begin, f_end; /* counting the number of backticks in the delimiter */ @@ -1059,6 +1159,7 @@ int Markdown::processCodeSpan(const char *data, int /*offset*/, int size) void Markdown::addStrEscapeUtf8Nbsp(const char *s,int len) { + TRACE(s); if (Portable::strnstr(s,g_doxy_nsbp,len)==0) // no escape needed -> fast { m_out.addStr(s,len); @@ -1071,10 +1172,12 @@ void Markdown::addStrEscapeUtf8Nbsp(const char *s,int len) int Markdown::processSpecialCommand(const char *data, int offset, int size) { + TRACE(data); int i=1; QCString endBlockName = isBlockCommand(data,offset,size); if (!endBlockName.isEmpty()) { + TRACE_MORE("endBlockName=%s\n",qPrint(endBlockName)); int l = endBlockName.length(); while (i3 && data[2]=='-' && data[3]=='-') // \--- { m_out.addStr(&data[1],3); + TRACE_MORE("result=4\n"); return 4; } else if (c=='-' && size>2 && data[2]=='-') // \-- { m_out.addStr(&data[1],2); + TRACE_MORE("result=3\n"); return 3; } } + TRACE_MORE("result=0\n"); return 0; } void Markdown::processInline(const char *data,int size) { + TRACE(data); int i=0, end=0; Action_t action; while (i0 && data[size-1]=='\n') size--; // ignore newline character while (i static int computeIndentExcludingListMarkers(const char *data,int size) { + TRACE(data); int i=0; int indent=0; bool isDigit=FALSE; @@ -1487,6 +1604,7 @@ static int computeIndentExcludingListMarkers(const char *data,int size) static bool isFencedCodeBlock(const char *data,int size,int refIndent, QCString &lang,int &start,int &end,int &offset) { + TRACE(data); // rules: at least 3 ~~~, end of the block same amount of ~~~'s, otherwise // return FALSE int i=0; @@ -1528,6 +1646,7 @@ static bool isFencedCodeBlock(const char *data,int size,int refIndent, static bool isCodeBlock(const char *data,int offset,int size,int &indent) { + TRACE(data); //printf(" " + cellText + " "); + m_out.addStr("> " + cellText + "\\ilinebr "); } cellTag = "td"; cellClass = "class=\"markdownTableBody"; @@ -1871,6 +1995,7 @@ int Markdown::writeTableBlock(const char *data,int size) static int hasLineBreak(const char *data,int size) { + TRACE(data); int i=0; int j=0; // search for end of line and also check if it is not a completely blank @@ -1887,6 +2012,7 @@ static int hasLineBreak(const char *data,int size) void Markdown::writeOneLineHeaderOrRuler(const char *data,int size) { + TRACE(data); int level; QCString header; QCString id; @@ -1939,6 +2065,7 @@ void Markdown::writeOneLineHeaderOrRuler(const char *data,int size) int Markdown::writeBlockQuote(const char *data,int size) { + TRACE(data); int l; int i=0; int curLevel=0; @@ -1995,6 +2122,7 @@ int Markdown::writeBlockQuote(const char *data,int size) int Markdown::writeCodeBlock(const char *data,int size,int refIndent) { + TRACE(data); int i=0,end; //printf("writeCodeBlock: data={%s}\n",QCString(data).left(size).data()); m_out.addStr("@verbatim\n"); @@ -2047,12 +2175,13 @@ int Markdown::writeCodeBlock(const char *data,int size,int refIndent) void Markdown::findEndOfLine(const char *data,int size, int &pi,int&i,int &end) { + TRACE(data); // find end of the line int nb=0; end=i+1; //while (end<=size && data[end-1]!='\n') - int j = 0; - while (end<=size && !(j = isNewline(data+end-1))) + int j=0; + while (end<=size && (j=isNewline(data+end-1))==0) { // while looking for the end of the line we might encounter a block // that needs to be passed unprocessed. @@ -2113,13 +2242,14 @@ void Markdown::findEndOfLine(const char *data,int size, end++; } } - if (j) end += j-1; + if (j>0) end+=j-1; //printf("findEndOfLine pi=%d i=%d end=%d {%s}\n",pi,i,end,QCString(data+i).left(end-i).data()); } void Markdown::writeFencedCodeBlock(const char *data,const char *lng, int blockStart,int blockEnd) { + TRACE(data); QCString lang = lng; if (!lang.isEmpty() && lang.at(0)=='.') lang=lang.mid(1); m_out.addStr("@code"); @@ -2134,6 +2264,7 @@ void Markdown::writeFencedCodeBlock(const char *data,const char *lng, QCString Markdown::processQuotations(const QCString &s,int refIndent) { + TRACE(s.data()); m_out.clear(); const char *data = s.data(); int size = s.length(); @@ -2192,6 +2323,7 @@ QCString Markdown::processQuotations(const QCString &s,int refIndent) QCString Markdown::processBlocks(const QCString &s,int indent) { + TRACE(s.data()); m_out.clear(); const char *data = s.data(); int size = s.length(); @@ -2333,6 +2465,7 @@ QCString Markdown::processBlocks(const QCString &s,int indent) /** returns TRUE if input string docs starts with \@page or \@mainpage command */ static bool isExplicitPage(const QCString &docs) { + TRACE(docs.data()); int i=0; const char *data = docs.data(); if (data) @@ -2355,6 +2488,7 @@ static bool isExplicitPage(const QCString &docs) QCString Markdown::extractPageTitle(QCString &docs,QCString &id) { + TRACE(docs.data()); int ln=0; // first first non-empty line QCString title; @@ -2402,6 +2536,7 @@ QCString Markdown::extractPageTitle(QCString &docs,QCString &id) QCString Markdown::detab(const QCString &s,int &refIndent) { + TRACE(s.data()); int tabSize = Config_getInt(TAB_SIZE); int size = s.length(); m_out.clear(); @@ -2517,6 +2652,7 @@ QCString Markdown::process(const QCString &input) QCString markdownFileNameToId(const QCString &fileName) { + TRACE(fileName.data()); QCString baseFn = stripFromPath(QFileInfo(fileName).absFilePath().utf8()); int i = baseFn.findRev('.'); if (i!=-1) baseFn = baseFn.left(i);