Fix various issues of internal procptr mangling

- bydesc array parameter dimensions weren't being encoded (e.g. array()
  and array(any), or array(any) and array(any, any) were seen as the same)

- Function result CONSTness wasn't encoded, and will be encoded now. This
  means two function pointers that differ only in function result CONSTness
  are seen as different now. This matches the overloading rules and the C++
  mangling.

- BYVAL parameter CONSTness was encoded, and will no longer be encoded now.
  Two function pointers that differ only in a BYVAL parameter that's CONST
  in one and non-CONST in the other, are now seen as the same type, as in
  C++. Overloading isn't allowed based on BYVAL parameter CONSTness either,
  and the C++ mangling ignores BYVAL parameter CONSTness too.

changelog.txt

@@ -16,6 +16,7 @@ Version 0.91.0:
 - #699: The default global New/New[]/Delete/Delete[] operators now simply use allocate()/deallocate() internally, instead of the functions from GCC's libsupc++. This avoids platform-dependant behaviour of the GCC functions (exception throwing, aborting) in case of allocation failure (instead, NULL will be returned as with allocate()).
 - FB no longer uses libsupc++ (from GCC's libstdc++) by default
 - The implicitly generated copy constructors and LET overloads will now have "BYREF AS CONST MyUdt" parameters (previously they didn't use CONST), to allow copying from CONST objects to work at least for the built-in data types or with nested UDTs. For backwards compatibility, if there is a user-defined "UDT.let(byref as UDT)" LET overload, FB will still add the non-const "UDT.constructor(byref as UDT)" copy-constructor. It's still the programmer's responsibility to write proper "byref as const UDT" versions of the copy-constructor or LET overload if the UDT needs deep-copying and CONST instances of the UDT are being used.
+- Function pointer types that differ only in function result CONSTness will now be seen as different, as in C++. Function pointer types that differ only in BYVAL parameter CONSTness will now be seen as being the same, as in C++.
 
 [added]
 - 64bit support (-arch x86_64) based on -gen gcc: INTEGER/POINTER = 64bit, LONG = 32bit DWORD. Compiler #define: __FB_64BIT__

src/compiler/symb-mangling.bas

@@ -485,7 +485,14 @@ sub symbMangleType _
 
 	'' const?
 	if( typeGetConstMask( dtype ) ) then
-		'' note: nothing is added (as in C++) because it's not a 'const ptr'
+		'' The type has some CONST bits. For C++ mangling we remove the
+		'' toplevel one and recursively mangle the rest of the type.
+		''
+		'' It could be a BYVAL x as CONST foo type. In this case the
+		'' CONST is not encoded in the C++ mangling, because it makes no
+		'' difference. It's not allowed to have overloads that differ
+		'' only in BYVAL CONSTness. The CONST only matters if it's a
+		'' pointer or BYREF type.
 		symbMangleType( mangled, typeUnsetIsConst( dtype ), subtype )
 
 		hAbbrevAdd( dtype, subtype )
@@ -534,6 +541,8 @@ sub symbMangleType _
 		end if
 
 		'' const?
+		'' (for function results, even BYVAL CONST is encoded into the
+		'' C++ mangling, unlike for parameters)
 		if( typeIsConst( symbGetFullType( subtype ) ) ) then
 			mangled += "K"
 		end if

src/compiler/symb-proc.bas

@@ -11,15 +11,6 @@
 #include once "list.bi"
 #include once "ast.bi"
 
-declare function hMangleFunctionPtr	_
-	( _
-		byval proc as FBSYMBOL ptr, _
-		byval dtype as integer, _
-		byval subtype as FBSYMBOL ptr, _
-		byval attrib as integer, _
-		byval mode as integer _
-	) as zstring ptr
-
 ''::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 '' init
 ''::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
@@ -1064,10 +1055,10 @@ function symbAddProcPtr _
 		byval mode as integer _
 	) as FBSYMBOL ptr
 
-	dim as zstring ptr id = any
 	dim as FBSYMBOL ptr sym = any, parent = any
 	dim as FBSYMBOLTB ptr symtb = any
 	dim as FBHASHTB ptr hashtb = any
+	dim as string id
 
 	''
 	'' The procptr prototypes are mangled, allowing them to be re-used.
@@ -1090,7 +1081,38 @@ function symbAddProcPtr _
 	'' also requires them to be scoped locally.
 	''
 
-	id = hMangleFunctionPtr( proc, dtype, subtype, attrib, mode )
+	'' Add information to the prototype so we can pass the PROC subtype to
+	'' C++ mangling functions directly (instead of having to encode
+	'' parameters & function result manually).
+	proc->attrib or= attrib
+	assert( proc->typ = FB_DATATYPE_INVALID )
+	proc->typ = dtype
+	proc->subtype = subtype
+
+	id = "{fbfp}"
+
+	'' Must encode the parameter/function result mode & dtype uniquely
+	'' - Cannot just encoded hex( param->typ ) because a BYVAL parameter
+	''   that is CONST is the same type as a non-CONST BYVAL parameter; to
+	''   achieve this they must be encoded the same.
+	'' - Cannot just encode hex( param->subtype ) because if it's
+	''   a forward reference symbol it may be removed, the encoded
+	''   address may then be that of another symbol.
+	'' - UDT namespace must be encoded, because UDT name itself is
+	''   not enough (same UDT may exist in separate namespaces)
+	'' - Bydesc dimensions must be encoded
+	'' - BYVAL/BYREF function result and its CONSTness (even if BYVAL, as
+	''   with C++ mangling)
+	symbMangleInitAbbrev( )
+	symbMangleType( id, FB_DATATYPE_FUNCTION, proc )
+	symbMangleEndAbbrev( )
+
+	'' Calling convention, must be encoded manually as it's not encoded in
+	'' the C++ mangling. We want function pointers with different calling
+	'' conventions to be seen as different types though, even though that's
+	'' not encoded in the C++ mangling, as with g++/clang++.
+	id += "$"
+	id += hex( mode )
 
 	sym = symbLookupInternallyMangledSubtype( id, attrib, parent, symtb, hashtb )
 	if( sym ) then
@@ -2750,73 +2772,6 @@ function symbProcHasFwdRefInSignature( byval proc as FBSYMBOL ptr ) as integer
 	function = FALSE
 end function
 
-'':::::
-private function hMangleFunctionPtr _
-	( _
-		byval proc as FBSYMBOL ptr, _
-		byval dtype as integer, _
-		byval subtype as FBSYMBOL ptr, _
-		byval attrib as integer, _
-		byval mode as integer _
-	) as zstring ptr
-
-    static as string id
-    dim as integer i = any
-    dim as FBSYMBOL ptr param = any
-
-    '' cheapo and fast internal mangling..
-    id = "{fbfp}("
-
-    symbMangleInitAbbrev( )
-
-    '' for each param..
-    param = symbGetProcHeadParam( proc )
-    for i = 0 to symbGetProcParams( proc )-1
-    	if( i > 0 ) then
-    		id += ","
-    	end if
-
-    	'' not an UDT?
-    	if( param->subtype = NULL ) then
-    		id += hex( param->typ ) + "M" + hex( cint(param->param.mode) )
-    	else
-    		'' notes:
-    		'' - can't use hex( param->subtype ), because slots can be
-    		''   reused if fwd types were resolved and removed
-    		'' - can't use only the param->id.name because UDT's with the same
-    		''   name declared inside different namespaces
-			symbMangleParam( id, param )
-    	end if
-
-    	param = symbGetParamNext( param )
-    next
-
-    '' return type
-    id += ")"
-	if( subtype = NULL ) then
-		id += hex( dtype )
-	else
-		'' see the notes above
-		symbMangleType( id, dtype, subtype )
-	end if
-
-    symbMangleEndAbbrev( )
-
-	'' return BYREF? - must be mangled explicitly, to distinguish it from
-	'' other function pointers with same types & parameters, that are not
-	'' returning BYREF though.
-	if( attrib and FB_SYMBATTRIB_RETURNSBYREF ) then
-		id += "$"  '' prevent the R from looking like part of the previous type id (if any)
-		id += "R"  '' R for reference, as in C++ mangling
-	end if
-
-    '' calling convention
-    id += "$"
-    id += hex( mode )
-
-	function = strptr( id )
-end function
-
 private sub hSubOrFuncToStr( byref s as string, byval proc as FBSYMBOL ptr )
 	if( symbGetType( proc ) = FB_DATATYPE_VOID ) then
 		s += "sub"

src/compiler/symb-var.bas

@@ -88,11 +88,19 @@ function symbGetDescTypeDimensions( byval desctype as FBSYMBOL ptr ) as integer
 
 	assert( symbIsStruct( desctype) and symbIsDescriptor( desctype ) )
 
+	'' If the dimensions were unknown, return -1 again, not FB_MAXARRAYDIMS,
+	'' so that if this is used in a call to symbAddArrayDescriptorType(), it
+	'' would produce/re-use the same descriptor type as this one.
+	if( *desctype->id.alias = "FBARRAY" ) then
+		return -1
+	end if
+
 	'' dimensions = sizeof(dimTB) \ sizeof(FBARRAYDIM)
 	dimtbsize = symbGetLen( desctype ) - (env.pointersize * 5)
 	dimensions = dimtbsize \ (env.pointersize * 3)
 
 	assert( (dimensions > 0) and (dimensions <= FB_MAXARRAYDIMS) )
+	assert( *desctype->id.alias = "FBARRAY" & dimensions )
 	function = dimensions
 end function
 
@@ -128,23 +136,32 @@ function symbAddArrayDescriptorType _
 	end if
 	assert( typeGetDtOnly( arraydtype ) <> FB_DATATYPE_FIXSTR )
 
-	'' If dimensions are unknown, the descriptor type must have room for
-	'' FB_MAXARRAYDIMS
-	if( dimensions = -1 ) then
-		dimensions = FB_MAXARRAYDIMS
+	''
+	'' The ALIAS for the C++ mangling (hMangleUdtId() will add the array
+	'' dtype as template parameter later, because it may have to use
+	'' abbreviations according to the Itanium C++ ABI mangling rules).
+	'' Arrays becoming FBARRAY matches how strings become FBSTRING.
+	''   array()         = "FBARRAY"
+	''   array(any)      = "FBARRAY1"
+	''   array(any, any) = "FBARRAY2"
+	''   array(any * 8)  = "FBARRAY8"
+	''
+	'' array() and array(any * FB_MAXARRAYDIMS) must be mangled different so
+	'' that we get different C++ mangling for these two procedures:
+	''     sub f( byval p as sub( array() as integer ) )
+	''     sub f( byval p as sub( array(any, any, any, any, any, any, any, any) as integer ) )
+	'' otherwise they'd collide in the generated .asm.
+	''
+	aliasid = "FBARRAY"
+	if( dimensions > 0 ) then
+		aliasid &= dimensions
 	end if
 
-	assert( (dimensions >= 1) and (dimensions <= FB_MAXARRAYDIMS) )
-
 	'' Some unique internal id that allows this descriptor type to be looked
 	'' up later when we need one with the same dimensions & array dtype
-	'' again.
-	''
-	'' Using a simplified ALIAS, because the internal mangling should never
-	'' be exposed, it's not proper GCC C++ mangling anyways. The mangling
-	'' done by symbGetMangledName()/hMangleUdtId() is based on this ALIAS.
-	aliasid = "__FBARRAY" & dimensions
-	id = aliasid
+	'' again. '$' prefix ensures that there are no collisions with user's
+	'' ids.
+	id = "$" + aliasid
 	id += "<"
 	symbMangleInitAbbrev( )
 	symbMangleType( id, arraydtype, arraysubtype )
@@ -180,6 +197,12 @@ function symbAddArrayDescriptorType _
 	symbAddField( sym, "size", 0, dTB(), FB_DATATYPE_INTEGER, NULL, 0, 0, 0 )
 	symbAddField( sym, "element_len", 0, dTB(), FB_DATATYPE_INTEGER, NULL, 0, 0, 0 )
 	symbAddField( sym, "dimensions", 0, dTB(), FB_DATATYPE_INTEGER, NULL, 0, 0, 0 )
+	'' If dimensions are unknown, the descriptor type must have room for
+	'' FB_MAXARRAYDIMS
+	if( dimensions = -1 ) then
+		dimensions = FB_MAXARRAYDIMS
+	end if
+	assert( (dimensions >= 1) and (dimensions <= FB_MAXARRAYDIMS) )
 	dTB(0).lower = 0
 	dTB(0).upper = dimensions-1
 	symbAddField( sym, "dimTB", 1, dTB(), FB_DATATYPE_STRUCT, symb.fbarraydim, 0, 0, 0 )

tests/functions/mangling-procptr-byvalconst-1-1.bas

@@ -0,0 +1,5 @@
+' TEST_MODE : COMPILE_ONLY_FAIL
+
+'' Overloading based on BYVAL CONSTness shouldn't be allowed, as in g++/clang++.
+sub f overload( byval p as sub( byval as integer       ) ) : end sub
+sub f overload( byval p as sub( byval as const integer ) ) : end sub

tests/functions/mangling-procptr-byvalconst-1-2.bas

@@ -0,0 +1,7 @@
+' TEST_MODE : COMPILE_ONLY_FAIL
+
+'' Overloading based on BYVAL CONSTness shouldn't be allowed, as in g++/clang++.
+'' Using ALIAS'es to ensure we test the compiler's duplicated definition error,
+'' not the assembler's one.
+sub f overload alias "f1"( byval p as sub( byval as integer       ) ) : end sub
+sub f overload alias "f2"( byval p as sub( byval as const integer ) ) : end sub

tests/functions/mangling-procptr-byvalconst-2-1.bas

@@ -0,0 +1,4 @@
+' TEST_MODE : COMPILE_ONLY_FAIL
+
+sub f overload( byval p as sub( byval as integer ptr       ) ) : end sub
+sub f overload( byval p as sub( byval as integer const ptr ) ) : end sub

tests/functions/mangling-procptr-byvalconst-2-2.bas

@@ -0,0 +1,4 @@
+' TEST_MODE : COMPILE_ONLY_FAIL
+
+sub f overload alias "f1"( byval p as sub( byval as integer ptr       ) ) : end sub
+sub f overload alias "f2"( byval p as sub( byval as integer const ptr ) ) : end sub

tests/functions/mangling-procptr-byvalconst-3-1.bas

@@ -0,0 +1,4 @@
+' TEST_MODE : COMPILE_ONLY_FAIL
+
+sub f overload( byval p as sub( byval as const integer ptr       ) ) : end sub
+sub f overload( byval p as sub( byval as const integer const ptr ) ) : end sub

tests/functions/mangling-procptr-byvalconst-3-2.bas

@@ -0,0 +1,4 @@
+' TEST_MODE : COMPILE_ONLY_FAIL
+
+sub f overload alias "f1"( byval p as sub( byval as const integer ptr       ) ) : end sub
+sub f overload alias "f2"( byval p as sub( byval as const integer const ptr ) ) : end sub

tests/functions/mangling-procptr-callconv-1.bas

@@ -0,0 +1,4 @@
+' TEST_MODE : COMPILE_ONLY_FAIL
+
+sub f overload( byval p as sub stdcall( ) ) : end sub
+sub f overload( byval p as sub cdecl  ( ) ) : end sub

tests/functions/mangling-procptr-callconv-2.bas

@@ -0,0 +1,4 @@
+' TEST_MODE : COMPILE_ONLY_FAIL
+
+sub f overload( byval p as sub cdecl  ( ) ) : end sub
+sub f overload( byval p as sub pascal ( ) ) : end sub

tests/functions/mangling-procptr-callconv-3.bas

@@ -0,0 +1,4 @@
+' TEST_MODE : COMPILE_ONLY_FAIL
+
+sub f overload( byval p as sub stdcall( ) ) : end sub
+sub f overload( byval p as sub pascal ( ) ) : end sub