Skip to content

Commit

Permalink
Refactor: make preprocessor run in parallel
Browse files Browse the repository at this point in the history 
And at the same time make sure it gives the same results as when
processed using a single thread.
  • Loading branch information
@doxygen
doxygen committed Jun 28, 2020
1 parent d0f24b7 commit ff5ebce
Show file tree
Hide file tree
Showing 10 changed files with 408 additions and 481 deletions.
5 changes: 1 addition & 4 deletions src/context.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1774,10 +1774,7 @@ class IncludeInfoListContext::Private : public GenericNodeListContext
IncludeInfo *ii;
for (li.toFirst();(ii=li.current());++li)
{
if (!ii->indirect)
{
append(IncludeInfoContext::alloc(ii,lang));
}
append(IncludeInfoContext::alloc(ii,lang));
}
}
};
Expand Down
7 changes: 5 additions & 2 deletions src/define.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@
#ifndef DEFINE_H
#define DEFINE_H

#include <map>
#include <string>
#include <vector>
#include <memory>
#include <string>
#include <unordered_map>

#include <qcstring.h>
#include "containers.h"

class FileDef;

Expand All @@ -44,5 +46,6 @@ class Define

/** List of all macro definitions */
using DefineList = std::vector< std::unique_ptr<Define> >;
using DefinesPerFileList = std::unordered_map< std::string, DefineList >;

#endif
45 changes: 26 additions & 19 deletions src/doxygen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ bool Doxygen::userComments = FALSE;
QCString Doxygen::spaces;
bool Doxygen::generatingXmlOutput = FALSE;
GenericsSDict *Doxygen::genericsDict;
DefineList Doxygen::macroDefinitions;
DefinesPerFileList Doxygen::macroDefinitions;

// locally accessible globals
static std::unordered_map< std::string, const Entry* > g_classEntries;
Expand Down Expand Up @@ -7701,28 +7701,35 @@ static void addSourceReferences()
// add the macro definitions found during preprocessing as file members
static void buildDefineList()
{
for (const auto &def : Doxygen::macroDefinitions)
for (const auto &s : g_inputFiles)
{
std::unique_ptr<MemberDef> md { createMemberDef(
def->fileName,def->lineNr,def->columnNr,
"#define",def->name,def->args,0,
Public,Normal,FALSE,Member,MemberType_Define,
ArgumentList(),ArgumentList(),"") };

if (!def->args.isEmpty())
auto it = Doxygen::macroDefinitions.find(s);
if (it!=Doxygen::macroDefinitions.end())
{
md->moveArgumentList(stringToArgumentList(SrcLangExt_Cpp, def->args));
}
md->setInitializer(def->definition);
md->setFileDef(def->fileDef);
md->setDefinition("#define "+def->name);
for (const auto &def : it->second)
{
std::unique_ptr<MemberDef> md { createMemberDef(
def->fileName,def->lineNr,def->columnNr,
"#define",def->name,def->args,0,
Public,Normal,FALSE,Member,MemberType_Define,
ArgumentList(),ArgumentList(),"") };

MemberName *mn=Doxygen::functionNameLinkedMap->add(def->name);
if (def->fileDef)
{
def->fileDef->insertMember(md.get());
if (!def->args.isEmpty())
{
md->moveArgumentList(stringToArgumentList(SrcLangExt_Cpp, def->args));
}
md->setInitializer(def->definition);
md->setFileDef(def->fileDef);
md->setDefinition("#define "+def->name);

MemberName *mn=Doxygen::functionNameLinkedMap->add(def->name);
if (def->fileDef)
{
def->fileDef->insertMember(md.get());
}
mn->push_back(std::move(md));
}
}
mn->push_back(std::move(md));
}
}

Expand Down
4 changes: 3 additions & 1 deletion src/doxygen.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@
#include "memberlist.h"
#include "define.h"

#ifndef MULTITHREADED_INPUT
#define MULTITHREADED_INPUT 0
#endif

#if MULTITHREADED_INPUT
#define THREAD_LOCAL thread_local
Expand Down Expand Up @@ -149,7 +151,7 @@ class Doxygen
static QCString spaces;
static bool generatingXmlOutput;
static GenericsSDict *genericsDict;
static DefineList macroDefinitions;
static DefinesPerFileList macroDefinitions;
};

void initDoxygen();
Expand Down
141 changes: 66 additions & 75 deletions src/filedef.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ class FileDefImpl : public DefinitionImpl, public FileDef
virtual void combineUsingRelations();
virtual bool generateSourceFile() const;
virtual void sortMemberLists();
virtual void addIncludeDependency(FileDef *fd,const char *incName,bool local,bool imported,bool indirect);
virtual void addIncludeDependency(FileDef *fd,const char *incName,bool local,bool imported);
virtual void addIncludedByDependency(FileDef *fd,const char *incName,bool local,bool imported);
virtual void addMembersToMemberGroup();
virtual void distributeMemberGroupDocumentation();
Expand Down Expand Up @@ -357,25 +357,22 @@ void FileDefImpl::writeTagFile(FTextStream &tagFile)
IncludeInfo *ii;
for (;(ii=ili.current());++ili)
{
if (!ii->indirect)
FileDef *fd=ii->fileDef;
if (fd && fd->isLinkable() && !fd->isReference())
{
FileDef *fd=ii->fileDef;
if (fd && fd->isLinkable() && !fd->isReference())
{
bool isIDLorJava = FALSE;
SrcLangExt lang = fd->getLanguage();
isIDLorJava = lang==SrcLangExt_IDL || lang==SrcLangExt_Java;
const char *locStr = (ii->local || isIDLorJava) ? "yes" : "no";
const char *impStr = (ii->imported || isIDLorJava) ? "yes" : "no";
tagFile << " <includes id=\""
<< convertToXML(fd->getOutputFileBase()) << "\" "
<< "name=\"" << convertToXML(fd->name()) << "\" "
<< "local=\"" << locStr << "\" "
<< "imported=\"" << impStr << "\">"
<< convertToXML(ii->includeName)
<< "</includes>"
<< endl;
}
bool isIDLorJava = FALSE;
SrcLangExt lang = fd->getLanguage();
isIDLorJava = lang==SrcLangExt_IDL || lang==SrcLangExt_Java;
const char *locStr = (ii->local || isIDLorJava) ? "yes" : "no";
const char *impStr = (ii->imported || isIDLorJava) ? "yes" : "no";
tagFile << " <includes id=\""
<< convertToXML(fd->getOutputFileBase()) << "\" "
<< "name=\"" << convertToXML(fd->name()) << "\" "
<< "local=\"" << locStr << "\" "
<< "imported=\"" << impStr << "\">"
<< convertToXML(ii->includeName)
<< "</includes>"
<< endl;
}
}
}
Expand Down Expand Up @@ -600,61 +597,58 @@ void FileDefImpl::writeIncludeFiles(OutputList &ol)
IncludeInfo *ii;
for (;(ii=ili.current());++ili)
{
if (!ii->indirect)
FileDef *fd=ii->fileDef;
bool isIDLorJava = FALSE;
if (fd)
{
FileDef *fd=ii->fileDef;
bool isIDLorJava = FALSE;
if (fd)
{
SrcLangExt lang = fd->getLanguage();
isIDLorJava = lang==SrcLangExt_IDL || lang==SrcLangExt_Java;
}
ol.startTypewriter();
if (isIDLorJava) // IDL/Java include
{
ol.docify("import ");
}
else if (ii->imported) // Objective-C include
{
ol.docify("#import ");
}
else // C/C++ include
{
ol.docify("#include ");
}
if (ii->local || isIDLorJava)
ol.docify("\"");
else
ol.docify("<");
ol.disable(OutputGenerator::Html);
ol.docify(ii->includeName);
ol.enableAll();
ol.disableAllBut(OutputGenerator::Html);

// Here we use the include file name as it appears in the file.
// we could also we the name as it is used within doxygen,
// then we should have used fd->docName() instead of ii->includeName
if (fd && fd->isLinkable())
{
ol.writeObjectLink(fd->getReference(),
fd->generateSourceFile() ? fd->includeName() : fd->getOutputFileBase(),
0,ii->includeName);
}
else
{
ol.docify(ii->includeName);
}
SrcLangExt lang = fd->getLanguage();
isIDLorJava = lang==SrcLangExt_IDL || lang==SrcLangExt_Java;
}
ol.startTypewriter();
if (isIDLorJava) // IDL/Java include
{
ol.docify("import ");
}
else if (ii->imported) // Objective-C include
{
ol.docify("#import ");
}
else // C/C++ include
{
ol.docify("#include ");
}
if (ii->local || isIDLorJava)
ol.docify("\"");
else
ol.docify("<");
ol.disable(OutputGenerator::Html);
ol.docify(ii->includeName);
ol.enableAll();
ol.disableAllBut(OutputGenerator::Html);

ol.enableAll();
if (ii->local || isIDLorJava)
ol.docify("\"");
else
ol.docify(">");
if (isIDLorJava)
ol.docify(";");
ol.endTypewriter();
ol.lineBreak();
// Here we use the include file name as it appears in the file.
// we could also we the name as it is used within doxygen,
// then we should have used fd->docName() instead of ii->includeName
if (fd && fd->isLinkable())
{
ol.writeObjectLink(fd->getReference(),
fd->generateSourceFile() ? fd->includeName() : fd->getOutputFileBase(),
0,ii->includeName);
}
else
{
ol.docify(ii->includeName);
}

ol.enableAll();
if (ii->local || isIDLorJava)
ol.docify("\"");
else
ol.docify(">");
if (isIDLorJava)
ol.docify(";");
ol.endTypewriter();
ol.lineBreak();
}
ol.endTextBlock();
}
Expand Down Expand Up @@ -1544,8 +1538,7 @@ void FileDefImpl::addUsingDeclaration(Definition *d)
}
}

void FileDefImpl::addIncludeDependency(FileDef *fd,const char *incName,bool local,
bool imported,bool indirect)
void FileDefImpl::addIncludeDependency(FileDef *fd,const char *incName,bool local,bool imported)
{
//printf("FileDefImpl::addIncludeDependency(%p,%s,%d)\n",fd,incName,local);
QCString iName = fd ? fd->absFilePath().data() : incName;
Expand All @@ -1562,7 +1555,6 @@ void FileDefImpl::addIncludeDependency(FileDef *fd,const char *incName,bool loca
ii->includeName = incName;
ii->local = local;
ii->imported = imported;
ii->indirect = indirect;
m_includeList->append(ii);
m_includeDict->insert(iName,ii);
}
Expand Down Expand Up @@ -1659,7 +1651,6 @@ void FileDefImpl::addIncludedByDependency(FileDef *fd,const char *incName,
ii->includeName = incName;
ii->local = local;
ii->imported = imported;
ii->indirect = FALSE;
m_includedByList->append(ii);
m_includedByDict->insert(iName,ii);
}
Expand Down
5 changes: 2 additions & 3 deletions src/filedef.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,12 @@ class FTextStream;
/** Class representing the data associated with a \#include statement. */
struct IncludeInfo
{
IncludeInfo() : fileDef(0), local(FALSE), imported(FALSE), indirect(FALSE) {}
IncludeInfo() : fileDef(0), local(FALSE), imported(FALSE) {}
~IncludeInfo() {}
FileDef *fileDef;
QCString includeName;
bool local;
bool imported;
bool indirect;
};

/** A model of a file symbol.
Expand Down Expand Up @@ -170,7 +169,7 @@ class FileDef : virtual public Definition
virtual bool generateSourceFile() const = 0;
virtual void sortMemberLists() = 0;

virtual void addIncludeDependency(FileDef *fd,const char *incName,bool local,bool imported,bool indirect) = 0;
virtual void addIncludeDependency(FileDef *fd,const char *incName,bool local,bool imported) = 0;
virtual void addIncludedByDependency(FileDef *fd,const char *incName,bool local,bool imported) = 0;

virtual void addMembersToMemberGroup() = 0;
Expand Down
Loading

0 comments on commit ff5ebce

Please sign in to comment.