update dmd and include the API needed for semantic analysis (#66)

* update dmd and include the API needed for semantic analysis * update libparse + initial implementation for properly documented public functions * test * refactor * update workflows * delete unused code
Dlang-UPB · May 19, 2023 · 016d6c3 · 016d6c3
1 parent bd60179
commit 016d6c3
Show file tree

Hide file tree

Showing 33 changed files with 533 additions and 729 deletions.
diff --git a/.github/workflows/default.yml b/.github/workflows/default.yml
@@ -92,6 +92,10 @@ jobs:
           sudo apt-get install gdc-12 -y
           gdc-12 --version
 
+      # - name: Setup upterm session
+      #   if: ${{ matrix.build.type == 'make' && matrix.host == 'macos-latest'}}
+      #   uses: lhotari/action-upterm@v1
+
       # Compile D-Scanner and execute all tests without dub
       - name: Build and test without dub
         if: ${{ matrix.build.type == 'make' }}
@@ -104,7 +108,8 @@ jobs:
             ./build.bat
             ./build.bat test
           else
-            make "-j$(nproc)" all test
+            NUM_PROC=$(nproc || getconf _NPROCESSORS_ONLN || 1)
+            make "-j$((NUM_PROC / 2))" all test
           fi
 
       # Compile D-Scanner and execute all tests using a specific dependency version

diff --git a/DCD b/DCD
diff --git a/build.bat b/build.bat
@@ -18,8 +18,8 @@ if %githashsize% == 0 (
 	move /y bin\githash_.txt bin\githash.txt
 )
 
-set DFLAGS=-O -release -version=StdLoggerDisableWarning -version=CallbackAPI -version=DMDLIB -version=MARS -Jbin -Jdmd %MFLAGS%
-set TESTFLAGS=-g -w -version=StdLoggerDisableWarning -version=CallbackAPI -version=DMDLIB -version=MARS -Jbin -Jdmd
+set DFLAGS=-O -release -version=StdLoggerDisableWarning -version=CallbackAPI -version=DMDLIB -version=MARS -Jbin -Jdmd -Jdmd\compiler\src\dmd\res %MFLAGS%
+set TESTFLAGS=-g -w -version=StdLoggerDisableWarning -version=CallbackAPI -version=DMDLIB -version=MARS -Jbin -Jdmd -Jdmd\compiler\src\dmd\res
 set CORE=
 set LIBDPARSE=
 set STD=
@@ -29,6 +29,8 @@ set DSYMBOL=
 set CONTAINERS=
 set LIBDDOC=
 
+SET DMD_FRONTEND_SRC=objc_glue.obj clone.obj transitivevisitor.obj iasm.obj iasmdmd.obj canthrow.obj tokens.obj optimize.obj func.obj semantic2.obj dvarstats.obj ph2.obj code.obj cdef.obj xmm.obj out.obj elfobj.obj glocal.obj dvec.obj code_x86.obj iasm2.obj string2.obj file2.obj obj.obj go.obj inliner.obj cc.obj bcomplex.obj mscoffobj.obj ptrntab.obj dlist.obj pdata.obj fp.obj cod3.obj os.obj cgelem.obj dcode.obj disasm86.obj exh.obj blockopt.obj aarray.obj cg.obj newman.obj dwarfdbginf.obj codebuilder.obj var.obj cod2.obj machobj.obj cgobj.obj cod4.obj dtype.obj cv4.obj backend.obj el.obj cgcod.obj cv8.obj dwarf.obj evalu8.obj ty.obj mem.obj cgxmm.obj gdag.obj gother.obj goh.obj cgcv.obj debugprint.obj cgsched.obj dwarfeh.obj cgreg.obj backconfig.obj gloop.obj divcoeff.obj cod5.obj dwarf2.obj cg87.obj nteh.obj dcgcv.obj util2.obj compress.obj type.obj elpicpie.obj gsroa.obj cgcs.obj ee.obj symbol.obj barray.obj melf.obj oper.obj cgcse.obj rtlsym.obj mscoff.obj drtlsym.obj symtab.obj dt.obj mach.obj cod1.obj global.obj filespec.obj gflow.obj elem.obj cgen.obj md5.obj chkformat.obj argtypes_sysv_x64.obj sideeffect.obj denum.obj apply.obj e2ir.obj typinf.obj statement.obj arraytypes.obj blockexit.obj init.obj scanomf.obj utils.obj parsetimevisitor.obj errorsink.obj scanmscoff.obj initsem.obj arrayop.obj nogc.obj dsymbol.obj hdrgen.obj dmangle.obj astenums.obj libmscoff.obj compiler.obj foreachvar.obj scanmach.obj dcast.obj tocsym.obj tocvdebug.obj semantic3.obj builtin.obj sapply.obj printast.obj dtemplate.obj importc.obj file_manager.obj dclass.obj argtypes_x86.obj glue.obj statement_rewrite_walker.obj target.obj aggregate.obj stringtable.obj ctfloat.obj response.obj strtold.obj port.obj aav.obj env.obj optional.obj filename.obj man.obj rootobject.obj complex.obj hash.obj region.obj utf.obj speller.obj rmem.obj array.obj longdouble.obj bitarray.obj eh.obj strictvisitor.obj permissivevisitor.obj lambdacomp.obj ctfeexpr.obj cparse.obj imphint.obj delegatize.obj access.obj identifier.obj todt.obj dmsc.obj entity.obj impcnvtab.obj dimport.obj lexer.obj dinifile.obj libomf.obj vsoptions.obj dstruct.obj aliasthis.obj ctorflow.obj errors.obj astcodegen.obj mtype.obj dtoh.obj argtypes_aarch64.obj cpreprocess.obj dmdparams.obj lib.obj id.obj parse.obj doc.obj scanelf.obj iasmgcc.obj cppmanglewin.obj stmtstate.obj ob.obj expression.obj declaration.obj location.obj dinterpret.obj inline.obj bitfields.obj string.obj int128.obj file.obj outbuffer.obj nspace.obj gluelayer.obj json.obj toir.obj intrange.obj cond.obj constfold.obj dversion.obj staticassert.obj dmodule.obj traits.obj opover.obj link.obj toctype.obj staticcond.obj statementsem.obj globals.obj libmach.obj toobj.obj s2ir.obj inlinecost.obj objc.obj visitor.obj asttypename.obj mustuse.obj dsymbolsem.obj frontend.obj safe.obj dscope.obj attrib.obj ast_node.obj escape.obj cli.obj templateparamsem.obj libelf.obj console.obj cppmangle.obj astbase.obj dmacro.obj typesem.obj expressionsem.obj
+
 set DMD_ROOT_SRC=
 for %%x in (dmd\compiler\src\dmd\common\*.d) do set DMD_ROOT_SRC=!DMD_ROOT_SRC! %%x
 for %%x in (dmd\compiler\src\dmd\root\*.d) do set DMD_ROOT_SRC=!DMD_ROOT_SRC! %%x
@@ -67,9 +69,21 @@ for %%x in (DCD\dsymbol\src\dsymbol\conversion\*.d) do set DSYMBOL=!DSYMBOL! %%x
 for %%x in (containers\src\containers\*.d) do set CONTAINERS=!CONTAINERS! %%x
 for %%x in (containers\src\containers\internal\*.d) do set CONTAINERS=!CONTAINERS! %%x
 
+for %%x in (dmd\compiler\src\dmd\common\*.d) do %DC% %DFLAGS% -c %%x -od. -I"dmd\compiler\src"
+for %%x in (dmd\compiler\src\dmd\root\*.d) do %DC% %DFLAGS% -c %%x -od. -I"dmd\compiler\src"
+for %%x in (dmd\compiler\src\dmd\backend\*.d) do %DC% %DFLAGS% -c %%x -od. -I"dmd\compiler\src"
+for %%x in (dmd\compiler\src\dmd\*.d) do %DC% %DFLAGS% -c %%x -od. -I"dmd\compiler\src"
+
+%DC% %DFLAGS% -c dmd\compiler\src\dmd\backend\iasm.d -od. -ofiasm2.obj -I"dmd\compiler\src"
+%DC% %DFLAGS% -c dmd\compiler\src\dmd\common\string.d -od. -ofstring2.obj -I"dmd\compiler\src"
+%DC% %DFLAGS% -c dmd\compiler\src\dmd\common\file.d -od. -offile2.obj -I"dmd\compiler\src"
+
 if "%1" == "test" goto test_cmd
 
 @echo on
+dir
+echo %DMD_FRONTEND_SRC%
+
 %DC% %MFLAGS%^
 	%CORE%^
 	%STD%^
@@ -79,9 +93,7 @@ if "%1" == "test" goto test_cmd
 	%INIFILED%^
 	%DSYMBOL%^
 	%CONTAINERS%^
-	%DMD_ROOT_SRC%^
-	%DMD_LEXER_SRC%^
-	%DMD_PARSER_SRC%^
+	%DMD_FRONTEND_SRC%^
 	%DFLAGS%^
 	-I"libdparse\src"^
 	-I"DCD\dsymbol\src"^
@@ -102,14 +114,13 @@ set TESTNAME="bin\dscanner-unittest"
 	%INIFILED%^
 	%DSYMBOL%^
 	%CONTAINERS%^
-	%DMD_ROOT_SRC%^
-	%DMD_LEXER_SRC%^
-	%DMD_PARSER_SRC%^
+	%DMD_FRONTEND_SRC%^
 	-I"libdparse\src"^
 	-I"DCD\dsymbol\src"^
 	-I"containers\src"^
 	-I"libddoc\src"^
 	-I"dmd\compiler\src"^
+	-I"dmd\compiler\src\dmd\res"^
 	-lib %TESTFLAGS%^
 	-of%TESTNAME%.lib
 if exist %TESTNAME%.lib %DC% %MFLAGS%^
@@ -124,6 +135,7 @@ if exist %TESTNAME%.lib %DC% %MFLAGS%^
 	-I"libddoc\src"^
 	-I"libddoc\common\source"^
 	-I"dmd\compiler\src"^
+	-I"dmd\compiler\src\dmd\res"^
 	-unittest^
 	%TESTFLAGS%^
 	-of%TESTNAME%.exe

diff --git a/containers b/containers
diff --git a/dmd b/dmd
diff --git a/dub.json b/dub.json
@@ -17,9 +17,10 @@
     "inifiled" : "~>1.3.1",
     "emsi_containers" : "~>0.9.0",
     "libddoc" : "~>0.8.0",
-    "dmd:root": "~master",
-    "dmd:lexer": "~master",
-    "dmd:parser": "~master"
+    "dmd": {
+      "repository": "git+https://github.com/dlang/dmd.git",
+      "version": "a4220358ecfcffe7ea38ab4a1996ffc5a5331f22"
+    }
   },
   "targetPath" : "bin",
   "stringImportPaths" : [

diff --git a/inifiled b/inifiled
diff --git a/libdparse b/libdparse
diff --git a/makefile b/makefile
@@ -1,24 +1,35 @@
 .PHONY: all test clean
 
+.DEFAULT_GOAL := all
+
 DC ?= dmd
 GIT ?= git
 DMD := $(DC)
 GDC := gdc
 LDC := ldc2
 DMD_ROOT_SRC := \
 	$(shell find dmd/compiler/src/dmd/common -name "*.d")\
-	$(shell find dmd/compiler/src/dmd/root -name "*.d")
+	$(shell find dmd/compiler/src/dmd/root -name "*.d")\
+
+DMD_FRONTEND_SRC := \
+	$(shell find dmd/compiler/src/dmd/common -name "*.d")\
+	$(shell find dmd/compiler/src/dmd/root -name "*.d")\
+	$(shell find dmd/compiler/src/dmd/backend -name "*.d")\
+	$(shell find dmd/compiler/src/dmd -maxdepth 1 -name "*.d" ! -name "mars.d" )
+
 DMD_LEXER_SRC := \
 	dmd/compiler/src/dmd/console.d \
 	dmd/compiler/src/dmd/entity.d \
 	dmd/compiler/src/dmd/errors.d \
+	dmd/compiler/src/dmd/errorsink.d \
 	dmd/compiler/src/dmd/file_manager.d \
 	dmd/compiler/src/dmd/globals.d \
 	dmd/compiler/src/dmd/id.d \
 	dmd/compiler/src/dmd/identifier.d \
 	dmd/compiler/src/dmd/lexer.d \
 	dmd/compiler/src/dmd/tokens.d \
 	dmd/compiler/src/dmd/utils.d \
+	dmd/compiler/src/dmd/location.d \
 	$(DMD_ROOT_SRC)
 
 DMD_PARSER_SRC := \
@@ -39,7 +50,8 @@ LIB_SRC := \
 	$(shell find libdparse/src/dparse/ -name "*.d")\
 	$(shell find libddoc/src -name "*.d") \
 	$(shell find libddoc/common/source -name "*.d") \
-	$(DMD_PARSER_SRC)
+	$(DMD_FRONTEND_SRC)
+
 PROJECT_SRC := $(shell find src/ -name "*.d")
 SRC := $(LIB_SRC) $(PROJECT_SRC)
 
@@ -78,17 +90,17 @@ LDC_DEBUG_VERSIONS = -d-version=dparse_verbose
 GDC_VERSIONS = -fversion=StdLoggerDisableWarning -fversion=CallbackAPI -fversion=DMDLIB -fversion=MARS
 GDC_DEBUG_VERSIONS = -fversion=dparse_verbose
 
-DC_FLAGS += -Jbin -Jdmd
+DC_FLAGS += -Jbin -Jdmd -Jdmd/compiler/src/dmd/res
 override DMD_FLAGS += $(DFLAGS) -w -release -O -od${OBJ_DIR}
 override LDC_FLAGS += $(DFLAGS) -O5 -release -oq
 override GDC_FLAGS += $(DFLAGS) -O3 -frelease -fall-instantiations
 
 override GDC_TEST_FLAGS += -fall-instantiations
 
-DC_TEST_FLAGS += -g -Jbin -Jdmd
+DC_TEST_FLAGS += -g -Jbin -Jdmd -Jdmd/compiler/src/dmd/res
 override DMD_TEST_FLAGS += -w
 
-DC_DEBUG_FLAGS := -g -Jbin -Jdmd
+DC_DEBUG_FLAGS := -g -Jbin -Jdmd -Jdmd/compiler/src/dmd/res
 
 ifeq ($(DC), $(filter $(DC), dmd ldmd2 gdmd))
 	VERSIONS := $(DMD_VERSIONS)
@@ -113,7 +125,13 @@ SHELL:=/usr/bin/env bash
 
 GITHASH = bin/githash.txt
 
+FIRST_RUN_FLAG := $(OBJ_DIR)/$(DC)/first_run.flag
+
 $(OBJ_DIR)/$(DC)/%.o: %.d
+	if [ ! -f $(FIRST_RUN_FLAG) ]; then \
+		${DC} -run dmd/config.d bin VERSION /etc; \
+		touch $(FIRST_RUN_FLAG); \
+	fi
 	${DC} ${DC_FLAGS} ${VERSIONS} ${INCLUDE_PATHS} -c $< ${WRITE_TO_TARGET_NAME}
 
 $(UT_OBJ_DIR)/$(DC)/%.o: %.d

diff --git a/src/dscanner/analysis/alias_syntax_check.d b/src/dscanner/analysis/alias_syntax_check.d
@@ -8,15 +8,15 @@ module dscanner.analysis.alias_syntax_check;
 import dscanner.analysis.base;
 import dmd.tokens;
 import dmd.lexer : Lexer;
-import dmd.globals : Loc;
+import dmd.location : Loc;
 
 /**
  * Checks for uses of the old alias syntax.
  */
-extern(C++) class AliasSyntaxCheck(AST) : BaseAnalyzerDmd!AST
+extern(C++) class AliasSyntaxCheck(AST) : BaseAnalyzerDmd
 {
 	mixin AnalyzerInfo!"alias_syntax_check";
-	alias visit = BaseAnalyzerDmd!AST.visit;
+	alias visit = BaseAnalyzerDmd.visit;
 
 	extern(D) this(string fileName)
 	{
@@ -26,15 +26,21 @@ extern(C++) class AliasSyntaxCheck(AST) : BaseAnalyzerDmd!AST
 	override void visit(AST.AliasDeclaration ad)
 	{
 		import dscanner.utils: readFile;
+		import dmd.errorsink : ErrorSinkNull;
+		import dmd.globals : global;
+
+		__gshared ErrorSinkNull errorSinkNull;
+		if (!errorSinkNull)
+			errorSinkNull = new ErrorSinkNull;
 
 		auto bytes = readFile(fileName);
 		bool foundEq = false;
 		Loc idLoc;
 
-		bytes ~= "\0";
+		bytes ~= '\0';
 		bytes = bytes[ad.loc.fileOffset .. $];
 
-		scope lexer = new Lexer(null, cast(char*) bytes, 0, bytes.length, 0, 0);
+		scope lexer = new Lexer(null, cast(char*) bytes, 0, bytes.length, 0, 0, errorSinkNull, &global.compileEnv);
 		TOK nextTok;
 		lexer.nextToken();
 

diff --git a/src/dscanner/analysis/assert_without_msg.d b/src/dscanner/analysis/assert_without_msg.d
@@ -10,10 +10,10 @@ import std.stdio;
 /**
  * Check that all asserts have an explanatory message.
  */
-extern(C++) class AssertWithoutMessageCheck(AST) : BaseAnalyzerDmd!AST
+extern(C++) class AssertWithoutMessageCheck(AST) : BaseAnalyzerDmd
 {
 	mixin AnalyzerInfo!"assert_without_msg";
-	alias visit = BaseAnalyzerDmd!AST.visit;
+	alias visit = BaseAnalyzerDmd.visit;
 
 	///
 	extern(D) this(string fileName, bool skipTests = false)
@@ -38,7 +38,7 @@ extern(C++) class AssertWithoutMessageCheck(AST) : BaseAnalyzerDmd!AST
 
 	override void visit(AST.StaticAssert ae)
 	{
-		if (!ae.msg)
+		if (!ae.msgs)
 			addErrorMessage(ae.loc.linnum, ae.loc.charnum, KEY, MESSAGE);
 	}
 

diff --git a/src/dscanner/analysis/auto_ref_assignment.d b/src/dscanner/analysis/auto_ref_assignment.d
@@ -10,10 +10,10 @@ import dscanner.analysis.base;
 /**
  * Checks for assignment to auto-ref function parameters.
  */
-extern(C++) class AutoRefAssignmentCheck(AST) : BaseAnalyzerDmd!AST
+extern(C++) class AutoRefAssignmentCheck(AST) : BaseAnalyzerDmd
 {
 	mixin AnalyzerInfo!"auto_ref_assignment_check";
-	alias visit = BaseAnalyzerDmd!AST.visit;
+	alias visit = BaseAnalyzerDmd.visit;
 
 	mixin ScopedVisit!(AST.ClassDeclaration);
 	mixin ScopedVisit!(AST.StructDeclaration);

diff --git a/src/dscanner/analysis/base.d b/src/dscanner/analysis/base.d
@@ -9,6 +9,9 @@ import dmd.transitivevisitor;
 import core.stdc.string;
 import std.conv : to;
 
+import dmd.visitor;
+import dmd.func;
+
 struct Message
 {
 	/// Name of the file where the warning was triggered
@@ -112,9 +115,9 @@ protected:
  * Visitor that implements the AST traversal logic.
  * Supports collecting error messages
  */
-extern(C++) class BaseAnalyzerDmd(AST) : ParseTimeTransitiveVisitor!AST
+extern(C++) class BaseAnalyzerDmd : SemanticTimeTransitiveVisitor
 {
-	alias visit = ParseTimeTransitiveVisitor!AST.visit;
+	alias visit = SemanticTimeTransitiveVisitor.visit;
 
 	extern(D) this(string fileName, bool skipTests = false)
 	{
@@ -137,7 +140,7 @@ extern(C++) class BaseAnalyzerDmd(AST) : ParseTimeTransitiveVisitor!AST
 		return _messages[].array;
 	}
 
-	override void visit(AST.UnitTestDeclaration ud)
+	override void visit(UnitTestDeclaration ud)
 	{
 		if (!skipTests)
 			super.visit(ud);

diff --git a/src/dscanner/analysis/builtin_property_names.d b/src/dscanner/analysis/builtin_property_names.d
@@ -21,9 +21,9 @@ import dscanner.analysis.base;
  * ---
  */
 
-extern(C++) class BuiltinPropertyNameCheck(AST) : BaseAnalyzerDmd!AST
+extern(C++) class BuiltinPropertyNameCheck(AST) : BaseAnalyzerDmd
 {
-	alias visit = BaseAnalyzerDmd!AST.visit;
+	alias visit = BaseAnalyzerDmd.visit;
 	mixin AnalyzerInfo!"builtin_property_names_check";
 
 	extern(D) this(string fileName)

diff --git a/src/dscanner/analysis/constructors.d b/src/dscanner/analysis/constructors.d
@@ -4,9 +4,9 @@ import std.stdio;
 import dscanner.analysis.base;
 import dscanner.analysis.helpers;
 
-extern(C++) class ConstructorCheck(AST) : BaseAnalyzerDmd!AST
+extern(C++) class ConstructorCheck(AST) : BaseAnalyzerDmd
 {
-	alias visit = BaseAnalyzerDmd!AST.visit;
+	alias visit = BaseAnalyzerDmd.visit;
 	mixin AnalyzerInfo!"constructor_check";
 
 	extern(D) this(string fileName)

diff --git a/src/dscanner/analysis/del.d b/src/dscanner/analysis/del.d
@@ -13,10 +13,10 @@ import dsymbol.scope_;
 /**
  * Checks for use of the deprecated 'delete' keyword
  */
-extern(C++) class DeleteCheck(AST) : BaseAnalyzerDmd!AST
+extern(C++) class DeleteCheck(AST) : BaseAnalyzerDmd
 {
-	alias visit = BaseAnalyzerDmd!AST.visit;
-
+	// alias visit = BaseAnalyzerDmd!AST.visit;
+	alias visit = BaseAnalyzerDmd.visit;
 	mixin AnalyzerInfo!"delete_check";
 
 	extern(D) this(string fileName)

diff --git a/src/dscanner/analysis/enumarrayliteral.d b/src/dscanner/analysis/enumarrayliteral.d
@@ -7,10 +7,10 @@ module dscanner.analysis.enumarrayliteral;
 
 import dscanner.analysis.base;
 
-extern(C++) class EnumArrayVisitor(AST) : BaseAnalyzerDmd!AST
+extern(C++) class EnumArrayVisitor(AST) : BaseAnalyzerDmd
 {
 	mixin AnalyzerInfo!"enum_array_literal_check";
-	alias visit = BaseAnalyzerDmd!AST.visit;
+	alias visit = BaseAnalyzerDmd.visit;
 
 	extern(D) this(string fileName)
 	{
@@ -34,4 +34,4 @@ extern(C++) class EnumArrayVisitor(AST) : BaseAnalyzerDmd!AST
 	}
 
 	private enum KEY = "dscanner.performance.enum_array_literal";
-}
+}
diff --git a/src/dscanner/analysis/explicitly_annotated_unittests.d b/src/dscanner/analysis/explicitly_annotated_unittests.d
@@ -10,10 +10,10 @@ import dscanner.analysis.helpers;
 /**
  * Requires unittests to be explicitly annotated with either @safe or @system
  */
-extern(C++) class ExplicitlyAnnotatedUnittestCheck(AST) : BaseAnalyzerDmd!AST
+extern(C++) class ExplicitlyAnnotatedUnittestCheck(AST) : BaseAnalyzerDmd
 {
     mixin AnalyzerInfo!"explicitly_annotated_unittests";
-	alias visit = BaseAnalyzerDmd!AST.visit;
+	alias visit = BaseAnalyzerDmd.visit;
 
 	extern(D) this(string fileName)
 	{
@@ -23,7 +23,7 @@ extern(C++) class ExplicitlyAnnotatedUnittestCheck(AST) : BaseAnalyzerDmd!AST
 	override void visit(AST.UnitTestDeclaration d)
 	{
 		import dmd.astenums : STC;
-	
+
 		if (!(d.storage_class & STC.safe || d.storage_class & STC.system))
 			addErrorMessage(cast(ulong) d.loc.linnum, cast(ulong) d.loc.charnum,
 					KEY, MESSAGE);