Prevent recursive lockup when instantiating complex recursive templates

Problematic file can be found in the open source project folly, file: folly/functional/traits.h
doxygen · Aug 30, 2023 · 09f9ff7 · 09f9ff7
1 parent a88a25c
commit 09f9ff7
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 14 deletions.
diff --git a/src/doxygen.cpp b/src/doxygen.cpp
@@ -4242,7 +4242,7 @@ static void findBaseClassesForClass(
     }
     BaseInfo tbi = bi;
     tbi.name = substituteTemplateArgumentsInString(bi.name,formalArgs,actualArgs);
-    //printf("bi->name=%s tbi.name=%s\n",qPrint(bi->name),qPrint(tbi.name));
+    //printf("masterCd=%p instanceCd=%p bi->name=%s tbi.name=%s\n",(void*)masterCd,(void*)instanceCd,qPrint(bi.name),qPrint(tbi.name));
 
     if (mode==DocumentedOnly)
     {
@@ -4277,10 +4277,9 @@ static void findTemplateInstanceRelation(const Entry *root,
   AUTO_TRACE("Derived from template '{}' with parameters '{}' isArtificial={}",
          templateClass->name(),templSpec,isArtificial);
 
-  bool existingClass = (templSpec ==
-                        tempArgListToString(templateClass->templateArguments(),root->lang,false)
-                       );
-  if (existingClass) return;
+  QCString tempArgsStr = tempArgListToString(templateClass->templateArguments(),root->lang,false);
+  bool existingClass = templSpec==tempArgsStr;
+  if (existingClass) return; // avoid recursion
 
   bool freshInstance=FALSE;
   ClassDefMutable *instanceClass = toClassDefMutable(
@@ -4615,7 +4614,7 @@ static bool findClassRelation(
             // doxygen
             if (baseClassTypeDef==0)
             {
-              //printf("       => findTemplateInstanceRelation: %p\n",baseClassTypeDef);
+              //printf("       => findTemplateInstanceRelation: %s\n",qPrint(baseClass->name()));
               findTemplateInstanceRelation(root,context,baseClass,templSpec,templateNames,baseClass->isArtificial());
             }
           }

diff --git a/src/util.cpp b/src/util.cpp
@@ -4567,35 +4567,47 @@ QCString substituteTemplateArgumentsInString(
         formArg.name = formArg.type.mid(9);
         formArg.type = "typename";
       }
+      //printf("n=%s formArg->type='%s' formArg->name='%s' formArg->defval='%s' actArg->type='%s' actArg->name='%s' \n",
+      //  qPrint(n),qPrint(formArg.type),qPrint(formArg.name),qPrint(formArg.defval),qPrint(actArg.type),qPrint(actArg.name));
       if (formArg.type=="class" || formArg.type=="typename" || formArg.type.startsWith("template"))
       {
-        //printf("n=%s formArg->type='%s' formArg->name='%s' formArg->defval='%s'\n",
-        //  qPrint(n),qPrint(formArg->type),qPrint(formArg->name),qPrint(formArg->defval));
-        //printf(">> n='%s' formArg->name='%s' actArg->type='%s' actArg->name='%s'\n",
-        //    qPrint(n),qPrint(formArg.name),actIt!=actualArgs.end() ? qPrint(actIt->type) : "",actIt!=actualArgs.end() ? qPrint(actIt->name) : ""
-        //    );
         if (formArg.name==n && actualArgs && actIt!=actualArgs->end() && !actArg.type.isEmpty()) // base class is a template argument
         {
+          static constexpr auto hasRecursion = [](const QCString &name,const QCString &subst) -> bool
+          {
+            int i;
+            int p=0;
+            while ((i=subst.find(name,p))!=-1)
+            {
+              bool beforeNonWord = i==0 || !isId(subst.at(i-1));
+              bool afterNonWord  = subst.length()==i+name.length() || !isId(subst.at(i+name.length()));
+              if (beforeNonWord && afterNonWord) return true; // if name=='A' then subst=='A::Z' or 'S<A>' or 'Z::A' should return true, but 'AA::ZZ' or 'BAH' should not match
+              p=i+name.length();
+            }
+            return false;
+          };
           // replace formal argument with the actual argument of the instance
-          if (!leftScopeMatch(actArg.type,n))
+          if (!hasRecursion(n,actArg.type))
             // the scope guard is to prevent recursive lockup for
             // template<class A> class C : public<A::T>,
             // where A::T would become A::T::T here,
             // since n==A and actArg->type==A::T
             // see bug595833 for an example
+            //
+            // Also prevent recursive subtitution if n is part of actArg.type, i.e.
+            // n='A' in argType='S< A >' would produce 'S< S< A > >'
           {
             if (actArg.name.isEmpty())
             {
               result += actArg.type+" ";
-              found=TRUE;
             }
             else
               // for case where the actual arg is something like "unsigned int"
               // the "int" part is in actArg->name.
             {
               result += actArg.type+" "+actArg.name+" ";
-              found=TRUE;
             }
+            found=TRUE;
           }
         }
         else if (formArg.name==n &&