diff --git a/csharp/extractor/.gitignore b/csharp/extractor/.gitignore new file mode 100644 index 000000000000..f81ecc73dffa --- /dev/null +++ b/csharp/extractor/.gitignore @@ -0,0 +1,13 @@ +obj/ +TestResults/ +*.manifest +*.pdb +*.suo +*.mdb +*.vsmdi +csharp.log +**/bin/Debug +**/bin/Release +*.tlog +.vs +*.user \ No newline at end of file diff --git a/csharp/extractor/CSharpExtractor.sln b/csharp/extractor/CSharpExtractor.sln new file mode 100644 index 000000000000..9f0d615ed387 --- /dev/null +++ b/csharp/extractor/CSharpExtractor.sln @@ -0,0 +1,83 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.27130.2036 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Util", "Semmle.Util\Semmle.Util.csproj", "{CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction", "Semmle.Extraction\Semmle.Extraction.csproj", "{81EAAD75-4BE1-44E4-91DF-20778216DB64}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp", "Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj", "{C4D62DA0-B64B-440B-86DC-AB52318CB8BF}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL", "Semmle.Extraction.CIL\Semmle.Extraction.CIL.csproj", "{399A1579-68F0-40F4-9A23-F241BA697F9C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Autobuild", "Semmle.Autobuild\Semmle.Autobuild.csproj", "{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Standalone", "Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj", "{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL.Driver", "Semmle.Extraction.CIL.Driver\Semmle.Extraction.CIL.Driver.csproj", "{EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Driver", "Semmle.Extraction.CSharp.Driver\Semmle.Extraction.CSharp.Driver.csproj", "{C36453BF-0C82-448A-B15D-26947503A2D3}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.Tests", "Semmle.Extraction.Tests\Semmle.Extraction.Tests.csproj", "{CD8D3F90-AD2E-4BB5-8E82-B94AA293864A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Util.Tests", "Semmle.Util.Tests\Semmle.Util.Tests.csproj", "{55A620F0-23F6-440D-A5BA-0567613B3C0F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Autobuild.Tests", "Semmle.Autobuild.Tests\Semmle.Autobuild.Tests.csproj", "{CE267461-D762-4F53-A275-685A0A4EC48D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CDD7AD69-0FD8-40F0-A9DA-F1077A2A85D6}.Release|Any CPU.Build.0 = Release|Any CPU + {81EAAD75-4BE1-44E4-91DF-20778216DB64}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {81EAAD75-4BE1-44E4-91DF-20778216DB64}.Debug|Any CPU.Build.0 = Debug|Any CPU + {81EAAD75-4BE1-44E4-91DF-20778216DB64}.Release|Any CPU.ActiveCfg = Release|Any CPU + {81EAAD75-4BE1-44E4-91DF-20778216DB64}.Release|Any CPU.Build.0 = Release|Any CPU + {C4D62DA0-B64B-440B-86DC-AB52318CB8BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C4D62DA0-B64B-440B-86DC-AB52318CB8BF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4D62DA0-B64B-440B-86DC-AB52318CB8BF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C4D62DA0-B64B-440B-86DC-AB52318CB8BF}.Release|Any CPU.Build.0 = Release|Any CPU + {399A1579-68F0-40F4-9A23-F241BA697F9C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {399A1579-68F0-40F4-9A23-F241BA697F9C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {399A1579-68F0-40F4-9A23-F241BA697F9C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {399A1579-68F0-40F4-9A23-F241BA697F9C}.Release|Any CPU.Build.0 = Release|Any CPU + {5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Release|Any CPU.Build.0 = Release|Any CPU + {D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Release|Any CPU.Build.0 = Release|Any CPU + {EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}.Release|Any CPU.Build.0 = Release|Any CPU + {C36453BF-0C82-448A-B15D-26947503A2D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C36453BF-0C82-448A-B15D-26947503A2D3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C36453BF-0C82-448A-B15D-26947503A2D3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C36453BF-0C82-448A-B15D-26947503A2D3}.Release|Any CPU.Build.0 = Release|Any CPU + {CD8D3F90-AD2E-4BB5-8E82-B94AA293864A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CD8D3F90-AD2E-4BB5-8E82-B94AA293864A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CD8D3F90-AD2E-4BB5-8E82-B94AA293864A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {55A620F0-23F6-440D-A5BA-0567613B3C0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {55A620F0-23F6-440D-A5BA-0567613B3C0F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {55A620F0-23F6-440D-A5BA-0567613B3C0F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE267461-D762-4F53-A275-685A0A4EC48D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CE267461-D762-4F53-A275-685A0A4EC48D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CE267461-D762-4F53-A275-685A0A4EC48D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE267461-D762-4F53-A275-685A0A4EC48D}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {E2B2BAC0-D55C-45DB-8CB3-8CEBA86FB547} + EndGlobalSection +EndGlobal diff --git a/csharp/extractor/Semmle.Autobuild.Tests/BuildScripts.cs b/csharp/extractor/Semmle.Autobuild.Tests/BuildScripts.cs new file mode 100644 index 000000000000..d49493c55480 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild.Tests/BuildScripts.cs @@ -0,0 +1,883 @@ +using Xunit; +using Semmle.Autobuild; +using System.Collections.Generic; +using System; +using System.Linq; +using Microsoft.Build.Construction; + +namespace Semmle.Extraction.Tests +{ + /// + /// Test class to script Autobuilder scenarios. + /// For most methods, it uses two fields: + /// - an IList to capture the the arguments passed to it + /// - an IDictionary of possible return values. + /// + class TestActions : IBuildActions + { + /// + /// List of strings passed to FileDelete. + /// + public IList FileDeleteIn = new List(); + + void IBuildActions.FileDelete(string file) + { + FileDeleteIn.Add(file); + } + + public IList FileExistsIn = new List(); + public IDictionary FileExists = new Dictionary(); + + bool IBuildActions.FileExists(string file) + { + FileExistsIn.Add(file); + if (FileExists.TryGetValue(file, out var ret)) + return ret; + throw new ArgumentException("Missing FileExists " + file); + } + + public IList RunProcessIn = new List(); + public IDictionary RunProcess = new Dictionary(); + public IDictionary RunProcessOut = new Dictionary(); + public IDictionary RunProcessWorkingDirectory = new Dictionary(); + + int IBuildActions.RunProcess(string cmd, string args, string workingDirectory, IDictionary env, out IList stdOut) + { + var pattern = cmd + " " + args; + RunProcessIn.Add(pattern); + if (RunProcessOut.TryGetValue(pattern, out var str)) + stdOut = str.Split("\n"); + else + throw new ArgumentException("Missing RunProcessOut " + pattern); + RunProcessWorkingDirectory.TryGetValue(pattern, out var wd); + if (wd != workingDirectory) + throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern); + if (RunProcess.TryGetValue(pattern, out var ret)) + return ret; + throw new ArgumentException("Missing RunProcess " + pattern); + } + + int IBuildActions.RunProcess(string cmd, string args, string workingDirectory, IDictionary env) + { + var pattern = cmd + " " + args; + RunProcessIn.Add(pattern); + RunProcessWorkingDirectory.TryGetValue(pattern, out var wd); + if (wd != workingDirectory) + throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern); + if (RunProcess.TryGetValue(pattern, out var ret)) + return ret; + throw new ArgumentException("Missing RunProcess " + pattern); + } + + public IList DirectoryDeleteIn = new List(); + + void IBuildActions.DirectoryDelete(string dir, bool recursive) + { + DirectoryDeleteIn.Add(dir); + } + + public IDictionary DirectoryExists = new Dictionary(); + public IList DirectoryExistsIn = new List(); + + bool IBuildActions.DirectoryExists(string dir) + { + DirectoryExistsIn.Add(dir); + if (DirectoryExists.TryGetValue(dir, out var ret)) + return ret; + throw new ArgumentException("Missing DirectoryExists " + dir); + } + + public IDictionary GetEnvironmentVariable = new Dictionary(); + + string IBuildActions.GetEnvironmentVariable(string name) + { + if (GetEnvironmentVariable.TryGetValue(name, out var ret)) + return ret; + throw new ArgumentException("Missing GetEnvironmentVariable " + name); + } + + public string GetCurrentDirectory; + + string IBuildActions.GetCurrentDirectory() + { + return GetCurrentDirectory; + } + + public IDictionary EnumerateFiles = new Dictionary(); + + IEnumerable IBuildActions.EnumerateFiles(string dir) + { + if (EnumerateFiles.TryGetValue(dir, out var str)) + return str.Split("\n"); + throw new ArgumentException("Missing EnumerateFiles " + dir); + } + + public IDictionary EnumerateDirectories = new Dictionary(); + + IEnumerable IBuildActions.EnumerateDirectories(string dir) + { + if (EnumerateDirectories.TryGetValue(dir, out var str)) + return string.IsNullOrEmpty(str) ? Enumerable.Empty() : str.Split("\n"); + throw new ArgumentException("Missing EnumerateDirectories " + dir); + } + + public bool IsWindows; + + bool IBuildActions.IsWindows() => IsWindows; + + string IBuildActions.PathCombine(params string[] parts) + { + return string.Join('\\', parts); + } + + void IBuildActions.WriteAllText(string filename, string contents) + { + } + } + + /// + /// A fake solution to build. + /// + class TestSolution : ISolution + { + public IEnumerable Projects => throw new NotImplementedException(); + + public IEnumerable Configurations => throw new NotImplementedException(); + + public string DefaultConfigurationName => "Release"; + + public string DefaultPlatformName => "x86"; + + public string Path { get; set; } + + public int ProjectCount => throw new NotImplementedException(); + + public Version ToolsVersion => new Version("14.0"); + + public TestSolution(string path) + { + Path = path; + } + } + + public class BuildScriptTests + { + TestActions Actions = new TestActions(); + + // Records the arguments passed to StartCallback. + IList StartCallbackIn = new List(); + + void StartCallback(string s) + { + StartCallbackIn.Add(s); + } + + // Records the arguments passed to EndCallback + IList EndCallbackIn = new List(); + IList EndCallbackReturn = new List(); + + void EndCallback(int ret, string s) + { + EndCallbackReturn.Add(ret); + EndCallbackIn.Add(s); + } + + [Fact] + public void TestBuildCommand() + { + var cmd = BuildScript.Create("abc", "def ghi", null, null); + + Actions.RunProcess["abc def ghi"] = 1; + cmd.Run(Actions, StartCallback, EndCallback); + Assert.Equal("abc def ghi", Actions.RunProcessIn[0]); + Assert.Equal("abc def ghi", StartCallbackIn[0]); + Assert.Equal("", EndCallbackIn[0]); + Assert.Equal(1, EndCallbackReturn[0]); + } + + [Fact] + public void TestAnd1() + { + var cmd = BuildScript.Create("abc", "def ghi", null, null) & BuildScript.Create("odasa", null, null, null); + + Actions.RunProcess["abc def ghi"] = 1; + cmd.Run(Actions, StartCallback, EndCallback); + + Assert.Equal("abc def ghi", Actions.RunProcessIn[0]); + Assert.Equal("abc def ghi", StartCallbackIn[0]); + Assert.Equal("", EndCallbackIn[0]); + Assert.Equal(1, EndCallbackReturn[0]); + } + + [Fact] + public void TestAnd2() + { + var cmd = BuildScript.Create("odasa", null, null, null) & BuildScript.Create("abc", "def ghi", null, null); + + Actions.RunProcess["abc def ghi"] = 1; + Actions.RunProcess["odasa "] = 0; + cmd.Run(Actions, StartCallback, EndCallback); + + Assert.Equal("odasa ", Actions.RunProcessIn[0]); + Assert.Equal("odasa ", StartCallbackIn[0]); + Assert.Equal("", EndCallbackIn[0]); + Assert.Equal(0, EndCallbackReturn[0]); + + Assert.Equal("abc def ghi", Actions.RunProcessIn[1]); + Assert.Equal("abc def ghi", StartCallbackIn[1]); + Assert.Equal("", EndCallbackIn[1]); + Assert.Equal(1, EndCallbackReturn[1]); + } + + [Fact] + public void TestOr1() + { + var cmd = BuildScript.Create("odasa", null, null, null) | BuildScript.Create("abc", "def ghi", null, null); + + Actions.RunProcess["abc def ghi"] = 1; + Actions.RunProcess["odasa "] = 0; + cmd.Run(Actions, StartCallback, EndCallback); + + Assert.Equal("odasa ", Actions.RunProcessIn[0]); + Assert.Equal("odasa ", StartCallbackIn[0]); + Assert.Equal("", EndCallbackIn[0]); + Assert.Equal(0, EndCallbackReturn[0]); + Assert.Equal(1, EndCallbackReturn.Count); + } + + [Fact] + public void TestOr2() + { + var cmd = BuildScript.Create("abc", "def ghi", null, null) | BuildScript.Create("odasa", null, null, null); + + Actions.RunProcess["abc def ghi"] = 1; + Actions.RunProcess["odasa "] = 0; + cmd.Run(Actions, StartCallback, EndCallback); + + Assert.Equal("abc def ghi", Actions.RunProcessIn[0]); + Assert.Equal("abc def ghi", StartCallbackIn[0]); + Assert.Equal("", EndCallbackIn[0]); + Assert.Equal(1, EndCallbackReturn[0]); + + Assert.Equal("odasa ", Actions.RunProcessIn[1]); + Assert.Equal("odasa ", StartCallbackIn[1]); + Assert.Equal("", EndCallbackIn[1]); + Assert.Equal(0, EndCallbackReturn[1]); + } + + [Fact] + public void TestSuccess() + { + Assert.Equal(0, BuildScript.Success.Run(Actions, StartCallback, EndCallback)); + } + + [Fact] + public void TestFailure() + { + Assert.NotEqual(0, BuildScript.Failure.Run(Actions, StartCallback, EndCallback)); + } + + [Fact] + public void TestDeleteDirectorySuccess() + { + Actions.DirectoryExists["trap"] = true; + Assert.Equal(0, BuildScript.DeleteDirectory("trap").Run(Actions, StartCallback, EndCallback)); + Assert.Equal("trap", Actions.DirectoryDeleteIn[0]); + } + + [Fact] + public void TestDeleteDirectoryFailure() + { + Actions.DirectoryExists["trap"] = false; + Assert.NotEqual(0, BuildScript.DeleteDirectory("trap").Run(Actions, StartCallback, EndCallback)); + } + + [Fact] + public void TestDeleteFileSuccess() + { + Actions.FileExists["csharp.log"] = true; + Assert.Equal(0, BuildScript.DeleteFile("csharp.log").Run(Actions, StartCallback, EndCallback)); + Assert.Equal("csharp.log", Actions.FileExistsIn[0]); + Assert.Equal("csharp.log", Actions.FileDeleteIn[0]); + } + + [Fact] + public void TestDeleteFileFailure() + { + Actions.FileExists["csharp.log"] = false; + Assert.NotEqual(0, BuildScript.DeleteFile("csharp.log").Run(Actions, StartCallback, EndCallback)); + Assert.Equal("csharp.log", Actions.FileExistsIn[0]); + } + + [Fact] + public void TestTry() + { + Assert.Equal(0, BuildScript.Try(BuildScript.Failure).Run(Actions, StartCallback, EndCallback)); + } + + Autobuilder CreateAutoBuilder(string lgtmLanguage, bool isWindows, + string buildless=null, string solution=null, string buildCommand=null, string ignoreErrors=null, + string msBuildArguments=null, string msBuildPlatform=null, string msBuildConfiguration=null, string msBuildTarget=null, + string dotnetArguments=null, string dotnetVersion=null, string vsToolsVersion=null, + string nugetRestore=null, string allSolutions=null, + string cwd=@"C:\Project") + { + Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa"; + Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java"; + Actions.GetEnvironmentVariable["LGTM_PROJECT_LANGUAGE"] = lgtmLanguage; + Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools"; + Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion; + Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments; + Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform; + Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration; + Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget; + Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments; + Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion; + Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand; + Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution; + Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors; + Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless; + Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions; + Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore; + Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null; + Actions.GetCurrentDirectory = cwd; + Actions.IsWindows = isWindows; + + var options = new AutobuildOptions(); + options.ReadEnvironment(Actions); + return new Autobuilder(Actions, options); + } + + [Fact] + public void TestDefaultCSharpAutoBuilder() + { + Actions.RunProcess["cmd.exe /C dotnet --info"] = 0; + Actions.RunProcess["cmd.exe /C dotnet clean"] = 0; + Actions.RunProcess["cmd.exe /C dotnet restore"] = 0; + Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false"] = 0; + Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; + Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.FileExists["csharp.log"] = true; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\nbar.cs"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", true); + TestAutobuilderScript(autobuilder, 0, 6); + } + + [Fact] + public void TestLinuxCSharpAutoBuilder() + { + Actions.RunProcess["dotnet --info"] = 0; + Actions.RunProcess["dotnet clean"] = 0; + Actions.RunProcess["dotnet restore"] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false"] = 0; + Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.FileExists["csharp.log"] = true; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.cs"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", false); + TestAutobuilderScript(autobuilder, 0, 6); + } + + [Fact] + public void TestLinuxCSharpAutoBuilderExtractorFailed() + { + Actions.RunProcess["dotnet --info"] = 0; + Actions.RunProcess["dotnet clean"] = 0; + Actions.RunProcess["dotnet restore"] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false"] = 0; + Actions.FileExists["csharp.log"] = false; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.cs"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", false); + TestAutobuilderScript(autobuilder, 1, 4); + } + + + [Fact] + public void TestDefaultCppAutobuilder() + { + Actions.EnumerateFiles[@"C:\Project"] = ""; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("cpp", true); + var script = autobuilder.GetBuildScript(); + + // Fails due to no solutions present. + Assert.NotEqual(0, script.Run(Actions, StartCallback, EndCallback)); + } + + [Fact] + public void TestCppAutobuilderSuccess() + { + Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore C:\Project\test.sln"] = 1; + Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0; + Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = ""; + Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1; + Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0; + Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = ""; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("cpp", true); + var solution = new TestSolution(@"C:\Project\test.sln"); + autobuilder.SolutionsToBuild.Add(solution); + TestAutobuilderScript(autobuilder, 0, 2); + } + + [Fact] + public void TestVsWhereSucceeded() + { + Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)"; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true; + Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 0; + Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "C:\\VS1\nC:\\VS2"; + Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = "10.0\n11.0"; + Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0; + + var candidates = BuildTools.GetCandidateVcVarsFiles(Actions).ToArray(); + Assert.Equal("C:\\VS1\\VC\\vcvarsall.bat", candidates[0].Path); + Assert.Equal(10, candidates[0].ToolsVersion); + Assert.Equal("C:\\VS2\\VC\\vcvarsall.bat", candidates[1].Path); + Assert.Equal(11, candidates[1].ToolsVersion); + } + + [Fact] + public void TestVsWhereNotExist() + { + Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)"; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false; + + var candidates = BuildTools.GetCandidateVcVarsFiles(Actions).ToArray(); + Assert.Equal(4, candidates.Length); + } + + [Fact] + public void TestVcVarsAllBatFiles() + { + Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)"; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = false; + + var vcvarsfiles = BuildTools.VcVarsAllBatFiles(Actions).ToArray(); + Assert.Equal(2, vcvarsfiles.Length); + } + + [Fact] + public void TestLinuxBuildlessExtractionSuccess() + { + Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone --references:."] = 0; + Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.FileExists["csharp.log"] = true; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", false, buildless:"true"); + TestAutobuilderScript(autobuilder, 0, 3); + } + + [Fact] + public void TestLinuxBuildlessExtractionFailed() + { + Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone --references:."] = 10; + Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.FileExists["csharp.log"] = true; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true"); + TestAutobuilderScript(autobuilder, 10, 1); + } + + [Fact] + public void TestLinuxBuildlessExtractionSolution() + { + Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone foo.sln --references:."] = 0; + Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.FileExists["csharp.log"] = true; + Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; + Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; + Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln"; + Actions.EnumerateDirectories[@"C:\Project"] = ""; + + var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true", solution: "foo.sln"); + TestAutobuilderScript(autobuilder, 0, 3); + } + + void SkipVsWhere() + { + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false; + Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = false; + } + + void TestAutobuilderScript(Autobuilder autobuilder, int expectedOutput, int commandsRun) + { + Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(Actions, StartCallback, EndCallback)); + + // Check expected commands actually ran + Assert.Equal(commandsRun, StartCallbackIn.Count); + Assert.Equal(commandsRun, EndCallbackIn.Count); + Assert.Equal(commandsRun, EndCallbackReturn.Count); + + var action = Actions.RunProcess.GetEnumerator(); + for(int cmd=0; cmd + + + Exe + netcoreapp2.0 + false + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Autobuild/AspBuildRule.cs b/csharp/extractor/Semmle.Autobuild/AspBuildRule.cs new file mode 100644 index 000000000000..7765e94f5e16 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/AspBuildRule.cs @@ -0,0 +1,20 @@ +using System.IO; + +namespace Semmle.Autobuild +{ + /// + /// ASP extraction. + /// + class AspBuildRule : IBuildRule + { + public BuildScript Analyse(Autobuilder builder) + { + var command = new CommandBuilder(builder.Actions). + RunCommand(builder.Actions.PathCombine(builder.SemmleJavaHome, "bin", "java")). + Argument("-jar"). + QuoteArgument(builder.Actions.PathCombine(builder.SemmleDist, "tools", "extractor-asp.jar")). + Argument("."); + return command.Script; + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/AutobuildOptions.cs b/csharp/extractor/Semmle.Autobuild/AutobuildOptions.cs new file mode 100644 index 000000000000..53ecf697721e --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/AutobuildOptions.cs @@ -0,0 +1,103 @@ +using System; +using System.Linq; + +namespace Semmle.Autobuild +{ + /// + /// Encapsulates build options. + /// + public class AutobuildOptions + { + public readonly int SearchDepth = 3; + public string RootDirectory = null; + static readonly string prefix = "LGTM_INDEX_"; + + public string VsToolsVersion; + public string MsBuildArguments; + public string MsBuildPlatform; + public string MsBuildConfiguration; + public string MsBuildTarget; + public string DotNetArguments; + public string DotNetVersion; + public string BuildCommand; + public string[] Solution; + + public bool IgnoreErrors; + public bool Buildless; + public bool AllSolutions; + public bool NugetRestore; + + public Language Language; + + /// + /// Reads options from environment variables. + /// Throws ArgumentOutOfRangeException for invalid arguments. + /// + public void ReadEnvironment(IBuildActions actions) + { + RootDirectory = actions.GetCurrentDirectory(); + VsToolsVersion = actions.GetEnvironmentVariable(prefix + "VSTOOLS_VERSION"); + MsBuildArguments = actions.GetEnvironmentVariable(prefix + "MSBUILD_ARGUMENTS"); + MsBuildPlatform = actions.GetEnvironmentVariable(prefix + "MSBUILD_PLATFORM"); + MsBuildConfiguration = actions.GetEnvironmentVariable(prefix + "MSBUILD_CONFIGURATION"); + MsBuildTarget = actions.GetEnvironmentVariable(prefix + "MSBUILD_TARGET"); + DotNetArguments = actions.GetEnvironmentVariable(prefix + "DOTNET_ARGUMENTS"); + DotNetVersion = actions.GetEnvironmentVariable(prefix + "DOTNET_VERSION"); + BuildCommand = actions.GetEnvironmentVariable(prefix + "BUILD_COMMAND"); + Solution = actions.GetEnvironmentVariable(prefix + "SOLUTION").AsList(new string[0]); + + IgnoreErrors = actions.GetEnvironmentVariable(prefix + "IGNORE_ERRORS").AsBool("ignore_errors", false); + Buildless = actions.GetEnvironmentVariable(prefix + "BUILDLESS").AsBool("buildless", false); + AllSolutions = actions.GetEnvironmentVariable(prefix + "ALL_SOLUTIONS").AsBool("all_solutions", false); + NugetRestore = actions.GetEnvironmentVariable(prefix + "NUGET_RESTORE").AsBool("nuget_restore", true); + + Language = actions.GetEnvironmentVariable("LGTM_PROJECT_LANGUAGE").AsLanguage(); + } + } + + public static class OptionsExtensions + { + public static bool AsBool(this string value, string param, bool defaultValue) + { + if (value == null) return defaultValue; + switch (value.ToLower()) + { + case "on": + case "yes": + case "true": + case "enabled": + return true; + case "off": + case "no": + case "false": + case "disabled": + return false; + default: + throw new ArgumentOutOfRangeException(param, value, "The Boolean value is invalid."); + } + } + + public static Language AsLanguage(this string key) + { + switch (key) + { + case null: + throw new ArgumentException("Environment variable required: LGTM_PROJECT_LANGUAGE"); + case "csharp": + return Language.CSharp; + case "cpp": + return Language.Cpp; + default: + throw new ArgumentException("Language key not understood: '" + key + "'"); + } + } + + public static string[] AsList(this string value, string[] defaultValue) + { + if (value == null) + return defaultValue; + + return value.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).ToArray(); + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/Autobuilder.cs b/csharp/extractor/Semmle.Autobuild/Autobuilder.cs new file mode 100644 index 000000000000..3edcb802ffed --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Autobuilder.cs @@ -0,0 +1,352 @@ +using Semmle.Extraction.CSharp; +using Semmle.Util.Logging; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Semmle.Autobuild +{ + /// + /// A build rule analyses the files in "builder" and outputs a build script. + /// + interface IBuildRule + { + /// + /// Analyse the files and produce a build script. + /// + /// The files and options relating to the build. + BuildScript Analyse(Autobuilder builder); + } + + /// + /// Main application logic, containing all data + /// gathered from the project and filesystem. + /// + /// The overall design is intended to be extensible so that in theory, + /// it should be possible to add new build rules without touching this code. + /// + public class Autobuilder + { + /// + /// Full file paths of files found in the project directory. + /// + public IEnumerable Paths => pathsLazy.Value; + readonly Lazy> pathsLazy; + + /// + /// Gets a list of paths matching a set of extensions + /// (including the "."). + /// + /// The extensions to find. + /// The files matching the extension. + public IEnumerable GetExtensions(params string[] extensions) => + Paths.Where(p => extensions.Contains(Path.GetExtension(p))); + + /// + /// Gets all paths matching a particular filename. + /// + /// The filename to find. + /// Possibly empty sequence of paths with the given filename. + public IEnumerable GetFilename(string name) => Paths.Where(p => Path.GetFileName(p) == name); + + /// + /// Holds if a given path, relative to the root of the source directory + /// was found. + /// + /// The relative path. + /// True iff the path was found. + public bool HasRelativePath(string path) => HasPath(Actions.PathCombine(RootDirectory, path)); + + /// + /// List of solution files to build. + /// + public IList SolutionsToBuild => solutionsToBuildLazy.Value; + readonly Lazy> solutionsToBuildLazy; + + /// + /// Holds if a given path was found. + /// + /// The path of the file. + /// True iff the path was found. + public bool HasPath(string path) => Paths.Any(p => path == p); + + void FindFiles(string dir, int depth, IList results) + { + foreach (var f in Actions.EnumerateFiles(dir)) + { + results.Add(f); + } + + if (depth > 1) + { + foreach (var d in Actions.EnumerateDirectories(dir)) + { + FindFiles(d, depth - 1, results); + } + } + } + + /// + /// The root of the source directory. + /// + string RootDirectory => Options.RootDirectory; + + /// + /// Gets the supplied build configuration. + /// + public AutobuildOptions Options { get; } + + /// + /// The set of build actions used during the autobuilder. + /// Could be real system operations, or a stub for testing. + /// + public IBuildActions Actions { get; } + + /// + /// Find all the relevant files and picks the best + /// solution file and tools. + /// + /// The command line options. + public Autobuilder(IBuildActions actions, AutobuildOptions options) + { + Actions = actions; + Options = options; + + pathsLazy = new Lazy>(() => + { + var files = new List(); + FindFiles(options.RootDirectory, options.SearchDepth, files); + return files. + OrderBy(s => s.Count(c => c == Path.DirectorySeparatorChar)). + ThenBy(s => Path.GetFileName(s).Length). + ToArray(); + }); + + solutionsToBuildLazy = new Lazy>(() => + { + if (options.Solution.Any()) + { + var ret = new List(); + foreach (var solution in options.Solution) + { + if (actions.FileExists(solution)) + ret.Add(new Solution(this, solution)); + else + Log(Severity.Error, "The specified solution file {0} was not found", solution); + } + return ret; + } + + var solutions = GetExtensions(".sln"). + Select(s => new Solution(this, s)). + Where(s => s.ProjectCount > 0). + OrderByDescending(s => s.ProjectCount). + ThenBy(s => s.Path.Length). + ToArray(); + + foreach (var sln in solutions) + { + Log(Severity.Info, $"Found {sln.Path} with {sln.ProjectCount} {this.Options.Language} projects, version {sln.ToolsVersion}, config {string.Join(" ", sln.Configurations.Select(c => c.FullName))}"); + } + + return new List(options.AllSolutions ? + solutions : + solutions.Take(1)); + }); + + SemmleDist = Actions.GetEnvironmentVariable("SEMMLE_DIST"); + + SemmleJavaHome = Actions.GetEnvironmentVariable("SEMMLE_JAVA_HOME"); + + SemmlePlatformTools = Actions.GetEnvironmentVariable("SEMMLE_PLATFORM_TOOLS"); + + if (SemmleDist == null) + Log(Severity.Error, "The environment variable SEMMLE_DIST has not been set."); + } + + readonly ILogger logger = new ConsoleLogger(Verbosity.Info); + + /// + /// Log a given build event to the console. + /// + /// The format string. + /// Inserts to the format string. + public void Log(Severity severity, string format, params object[] args) + { + logger.Log(severity, format, args); + } + + /// + /// Attempt to build this project. + /// + /// The exit code, 0 for success and non-zero for failures. + public int AttemptBuild() + { + Log(Severity.Info, $"Working directory: {Options.RootDirectory}"); + + var script = GetBuildScript(); + if (Options.IgnoreErrors) + script |= BuildScript.Success; + + void startCallback(string s) => Log(Severity.Info, $"\nRunning {s}"); + void exitCallback(int ret, string msg) => Log(Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}"); + return script.Run(Actions, startCallback, exitCallback); + } + + /// + /// Returns the build script to use for this project. + /// + public BuildScript GetBuildScript() + { + var isCSharp = Options.Language == Language.CSharp; + return isCSharp ? GetCSharpBuildScript() : GetCppBuildScript(); + } + + BuildScript GetCSharpBuildScript() + { + /// + /// A script that checks that the C# extractor has been executed. + /// + BuildScript CheckExtractorRun(bool warnOnFailure) => + BuildScript.Create(actions => + { + if (actions.FileExists(Extractor.GetCSharpLogPath())) + return 0; + + if (warnOnFailure) + Log(Severity.Error, "No C# code detected during build."); + + return 1; + }); + + var attempt = BuildScript.Failure; + switch (GetCSharpBuildStrategy()) + { + case CSharpBuildStrategy.CustomBuildCommand: + attempt = new BuildCommandRule().Analyse(this) & CheckExtractorRun(true); + break; + case CSharpBuildStrategy.Buildless: + // No need to check that the extractor has been executed in buildless mode + attempt = new StandaloneBuildRule().Analyse(this); + break; + case CSharpBuildStrategy.MSBuild: + attempt = new MsBuildRule().Analyse(this) & CheckExtractorRun(true); + break; + case CSharpBuildStrategy.DotNet: + attempt = new DotNetRule().Analyse(this) & CheckExtractorRun(true); + break; + case CSharpBuildStrategy.Auto: + var cleanTrapFolder = + BuildScript.DeleteDirectory(Actions.GetEnvironmentVariable("TRAP_FOLDER")); + var cleanSourceArchive = + BuildScript.DeleteDirectory(Actions.GetEnvironmentVariable("SOURCE_ARCHIVE")); + var cleanExtractorLog = + BuildScript.DeleteFile(Extractor.GetCSharpLogPath()); + var attemptExtractorCleanup = + BuildScript.Try(cleanTrapFolder) & + BuildScript.Try(cleanSourceArchive) & + BuildScript.Try(cleanExtractorLog); + + /// + /// Execute script `s` and check that the C# extractor has been executed. + /// If either fails, attempt to cleanup any artifacts produced by the extractor, + /// and exit with code 1, in order to proceed to the next attempt. + /// + BuildScript IntermediateAttempt(BuildScript s) => + (s & CheckExtractorRun(false)) | + (attemptExtractorCleanup & BuildScript.Failure); + + attempt = + // First try .NET Core + IntermediateAttempt(new DotNetRule().Analyse(this)) | + // Then MSBuild + (() => IntermediateAttempt(new MsBuildRule().Analyse(this))) | + // And finally look for a script that might be a build script + (() => new BuildCommandAutoRule().Analyse(this) & CheckExtractorRun(true)) | + // All attempts failed: print message + AutobuildFailure(); + break; + } + + return + attempt & + (() => new AspBuildRule().Analyse(this)) & + (() => new XmlBuildRule().Analyse(this)); + } + + /// + /// Gets the build strategy that the autobuilder should apply, based on the + /// options in the `lgtm.yml` file. + /// + CSharpBuildStrategy GetCSharpBuildStrategy() + { + if (Options.BuildCommand != null) + return CSharpBuildStrategy.CustomBuildCommand; + + if (Options.Buildless) + return CSharpBuildStrategy.Buildless; + + if (Options.MsBuildArguments != null + || Options.MsBuildConfiguration != null + || Options.MsBuildPlatform != null + || Options.MsBuildTarget != null) + return CSharpBuildStrategy.MSBuild; + + if (Options.DotNetArguments != null || Options.DotNetVersion != null) + return CSharpBuildStrategy.DotNet; + + return CSharpBuildStrategy.Auto; + } + + enum CSharpBuildStrategy + { + CustomBuildCommand, + Buildless, + MSBuild, + DotNet, + Auto + } + + BuildScript GetCppBuildScript() + { + if (Options.BuildCommand != null) + return new BuildCommandRule().Analyse(this); + + return + // First try MSBuild + new MsBuildRule().Analyse(this) | + // Then look for a script that might be a build script + (() => new BuildCommandAutoRule().Analyse(this)) | + // All attempts failed: print message + AutobuildFailure(); + } + + BuildScript AutobuildFailure() => + BuildScript.Create(actions => + { + Log(Severity.Error, "Could not auto-detect a suitable build method"); + return 1; + }); + + /// + /// Value of SEMMLE_DIST environment variable. + /// + public string SemmleDist { get; private set; } + + /// + /// Value of SEMMLE_JAVA_HOME environment variable. + /// + public string SemmleJavaHome { get; private set; } + + /// + /// Value of SEMMLE_PLATFORM_TOOLS environment variable. + /// + public string SemmlePlatformTools { get; private set; } + + /// + /// The absolute path of the odasa executable. + /// + public string Odasa => SemmleDist == null ? null : Actions.PathCombine(SemmleDist, "tools", "odasa"); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/BuildActions.cs b/csharp/extractor/Semmle.Autobuild/BuildActions.cs new file mode 100644 index 000000000000..906d6701c1a8 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/BuildActions.cs @@ -0,0 +1,176 @@ +using Semmle.Util; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; + +namespace Semmle.Autobuild +{ + /// + /// Wrapper around system calls so that the build scripts can be unit-tested. + /// + public interface IBuildActions + { + /// + /// Runs a process and captures its output. + /// + /// The exe to run. + /// The other command line arguments. + /// The working directory (null for current directory). + /// Additional environment variables. + /// The lines of stdout. + /// The process exit code. + int RunProcess(string exe, string args, string workingDirectory, IDictionary env, out IList stdOut); + + /// + /// Runs a process but does not capture its output. + /// + /// The exe to run. + /// The other command line arguments. + /// The working directory (null for current directory). + /// Additional environment variables. + /// The process exit code. + int RunProcess(string exe, string args, string workingDirectory, IDictionary env); + + /// + /// Tests whether a file exists, File.Exists(). + /// + /// The filename. + /// True iff the file exists. + bool FileExists(string file); + + /// + /// Tests whether a directory exists, Directory.Exists(). + /// + /// The directory name. + /// True iff the directory exists. + bool DirectoryExists(string dir); + + /// + /// Deletes a file, File.Delete(). + /// + /// The filename. + void FileDelete(string file); + + /// + /// Deletes a directory, Directory.Delete(). + /// + void DirectoryDelete(string dir, bool recursive); + + /// + /// Gets an environment variable, Environment.GetEnvironmentVariable(). + /// + /// The name of the variable. + /// The string value, or null if the variable is not defined. + string GetEnvironmentVariable(string name); + + /// + /// Gets the current directory, Directory.GetCurrentDirectory(). + /// + /// The current directory. + string GetCurrentDirectory(); + + /// + /// Enumerates files in a directory, Directory.EnumerateFiles(). + /// + /// The directory to enumerate. + /// A list of filenames, or an empty list. + IEnumerable EnumerateFiles(string dir); + + /// + /// Enumerates the directories in a directory, Directory.EnumerateDirectories(). + /// + /// The directory to enumerate. + /// List of subdirectories, or empty list. + IEnumerable EnumerateDirectories(string dir); + + /// + /// True if we are running on Windows. + /// + bool IsWindows(); + + /// + /// Combine path segments, Path.Combine(). + /// + /// The parts of the path. + /// The combined path. + string PathCombine(params string[] parts); + + /// + /// Writes contents to file, File.WriteAllText(). + /// + /// The filename. + /// The text. + void WriteAllText(string filename, string contents); + } + + /// + /// An implementation of IBuildActions that actually performs the requested operations. + /// + class SystemBuildActions : IBuildActions + { + void IBuildActions.FileDelete(string file) => File.Delete(file); + + bool IBuildActions.FileExists(string file) => File.Exists(file); + + ProcessStartInfo GetProcessStartInfo(string exe, string arguments, string workingDirectory, IDictionary environment, bool redirectStandardOutput) + { + var pi = new ProcessStartInfo(exe, arguments) + { + UseShellExecute = false, + RedirectStandardOutput = redirectStandardOutput + }; + if (workingDirectory != null) + pi.WorkingDirectory = workingDirectory; + + // Environment variables can only be used when not redirecting stdout + if (!redirectStandardOutput) + { + pi.Environment["UseSharedCompilation"] = "false"; + if (environment != null) + environment.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value); + } + return pi; + } + + int IBuildActions.RunProcess(string cmd, string args, string workingDirectory, IDictionary environment) + { + var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, false); + using (var p = Process.Start(pi)) + { + p.WaitForExit(); + return p.ExitCode; + } + } + + int IBuildActions.RunProcess(string cmd, string args, string workingDirectory, IDictionary environment, out IList stdOut) + { + var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, true); + return pi.ReadOutput(out stdOut); + } + + void IBuildActions.DirectoryDelete(string dir, bool recursive) => Directory.Delete(dir, recursive); + + bool IBuildActions.DirectoryExists(string dir) => Directory.Exists(dir); + + string IBuildActions.GetEnvironmentVariable(string name) => Environment.GetEnvironmentVariable(name); + + string IBuildActions.GetCurrentDirectory() => Directory.GetCurrentDirectory(); + + IEnumerable IBuildActions.EnumerateFiles(string dir) => Directory.EnumerateFiles(dir); + + IEnumerable IBuildActions.EnumerateDirectories(string dir) => Directory.EnumerateDirectories(dir); + + bool IBuildActions.IsWindows() => Win32.IsWindows(); + + string IBuildActions.PathCombine(params string[] parts) => Path.Combine(parts); + + void IBuildActions.WriteAllText(string filename, string contents) => File.WriteAllText(filename, contents); + + private SystemBuildActions() + { + } + + public static readonly IBuildActions Instance = new SystemBuildActions(); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/BuildCommandAutoRule.cs b/csharp/extractor/Semmle.Autobuild/BuildCommandAutoRule.cs new file mode 100644 index 000000000000..627651eaefe9 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/BuildCommandAutoRule.cs @@ -0,0 +1,61 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Semmle.Util; +using Semmle.Util.Logging; + +namespace Semmle.Autobuild +{ + /// + /// Auto-detection of build scripts. + /// + class BuildCommandAutoRule : IBuildRule + { + readonly IEnumerable winExtensions = new List { + ".bat", + ".cmd", + ".exe" + }; + + readonly IEnumerable linuxExtensions = new List { + "", + ".sh" + }; + + readonly IEnumerable buildScripts = new List { + "build" + }; + + public BuildScript Analyse(Autobuilder builder) + { + builder.Log(Severity.Info, "Attempting to locate build script"); + + var extensions = builder.Actions.IsWindows() ? winExtensions : linuxExtensions; + var scripts = buildScripts.SelectMany(s => extensions.Select(e => s + e)); + var scriptPath = builder.Paths.Where(p => scripts.Any(p.ToLower().EndsWith)).OrderBy(p => p.Length).FirstOrDefault(); + + if (scriptPath == null) + return BuildScript.Failure; + + var chmod = new CommandBuilder(builder.Actions); + chmod.RunCommand("/bin/chmod", $"u+x {scriptPath}"); + var chmodScript = builder.Actions.IsWindows() ? BuildScript.Success : BuildScript.Try(chmod.Script); + + var path = Path.GetDirectoryName(scriptPath); + + // A specific .NET Core version may be required + return chmodScript & DotNetRule.WithDotNet(builder, dotNet => + { + var command = new CommandBuilder(builder.Actions, path, dotNet?.Environment); + + // A specific Visual Studio version may be required + var vsTools = MsBuildRule.GetVcVarsBatFile(builder); + if (vsTools != null) + command.CallBatFile(vsTools.Path); + + command.IndexCommand(builder.Odasa, scriptPath); + return command.Script; + }); + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/BuildCommandRule.cs b/csharp/extractor/Semmle.Autobuild/BuildCommandRule.cs new file mode 100644 index 000000000000..2dd3a1442223 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/BuildCommandRule.cs @@ -0,0 +1,28 @@ +namespace Semmle.Autobuild +{ + /// + /// Execute the build_command rule. + /// + class BuildCommandRule : IBuildRule + { + public BuildScript Analyse(Autobuilder builder) + { + if (builder.Options.BuildCommand == null) + return BuildScript.Failure; + + // Custom build commands may require a specific .NET Core version + return DotNetRule.WithDotNet(builder, dotNet => + { + var command = new CommandBuilder(builder.Actions, null, dotNet?.Environment); + + // Custom build commands may require a specific Visual Studio version + var vsTools = MsBuildRule.GetVcVarsBatFile(builder); + if (vsTools != null) + command.CallBatFile(vsTools.Path); + command.IndexCommand(builder.Odasa, builder.Options.BuildCommand); + + return command.Script; + }); + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/BuildScript.cs b/csharp/extractor/Semmle.Autobuild/BuildScript.cs new file mode 100644 index 000000000000..d36e98a0db53 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/BuildScript.cs @@ -0,0 +1,274 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using Semmle.Util; + +namespace Semmle.Autobuild +{ + /// + /// A build script. + /// + public abstract class BuildScript + { + /// + /// Run this build script. + /// + /// + /// The interface used to implement the build actions. + /// + /// + /// A call back that is called every time a new process is started. The + /// argument to the call back is a textual representation of the process. + /// + /// + /// A call back that is called every time a new process exits. The first + /// argument to the call back is the exit code, and the second argument is + /// an exit message. + /// + /// The exit code from this build script. + public abstract int Run(IBuildActions actions, Action startCallback, Action exitCallBack); + + /// + /// Run this build command. + /// + /// + /// The interface used to implement the build actions. + /// + /// + /// A call back that is called every time a new process is started. The + /// argument to the call back is a textual representation of the process. + /// + /// + /// A call back that is called every time a new process exits. The first + /// argument to the call back is the exit code, and the second argument is + /// an exit message. + /// + /// Contents of standard out. + /// The exit code from this build script. + public abstract int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout); + + class BuildCommand : BuildScript + { + readonly string exe, arguments, workingDirectory; + readonly IDictionary environment; + + /// + /// Create a simple build command. + /// + /// The executable to run. + /// The arguments to the executable, or null. + /// The working directory (null for current directory). + /// Additional environment variables. + public BuildCommand(string exe, string argumentsOpt, string workingDirectory = null, IDictionary environment = null) + { + this.exe = exe; + this.arguments = argumentsOpt ?? ""; + this.workingDirectory = workingDirectory; + this.environment = environment; + } + + public override string ToString() => exe + " " + arguments; + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) + { + startCallback(this.ToString()); + var ret = 1; + var retMessage = ""; + try + { + ret = actions.RunProcess(exe, arguments, workingDirectory, environment); + } + catch (Exception ex) + when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException) + { + retMessage = ex.Message; + } + + exitCallBack(ret, retMessage); + return ret; + } + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) + { + startCallback(this.ToString()); + var ret = 1; + var retMessage = ""; + try + { + ret = actions.RunProcess(exe, arguments, workingDirectory, environment, out stdout); + } + catch (Exception ex) + when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException) + { + retMessage = ex.Message; + stdout = new string[0]; + } + exitCallBack(ret, retMessage); + return ret; + } + + } + + class ReturnBuildCommand : BuildScript + { + readonly Func func; + public ReturnBuildCommand(Func func) + { + this.func = func; + } + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) => func(actions); + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) + { + stdout = new string[0]; + return func(actions); + } + } + + class BindBuildScript : BuildScript + { + readonly BuildScript s1; + readonly Func, int, BuildScript> s2a; + readonly Func s2b; + public BindBuildScript(BuildScript s1, Func, int, BuildScript> s2) + { + this.s1 = s1; + this.s2a = s2; + } + + public BindBuildScript(BuildScript s1, Func s2) + { + this.s1 = s1; + this.s2b = s2; + } + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) + { + int ret1; + if (s2a != null) + { + ret1 = s1.Run(actions, startCallback, exitCallBack, out var stdout1); + return s2a(stdout1, ret1).Run(actions, startCallback, exitCallBack); + } + + ret1 = s1.Run(actions, startCallback, exitCallBack); + return s2b(ret1).Run(actions, startCallback, exitCallBack); + } + + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) + { + var ret1 = s1.Run(actions, startCallback, exitCallBack, out var stdout1); + var ret2 = (s2a != null ? s2a(stdout1, ret1) : s2b(ret1)).Run(actions, startCallback, exitCallBack, out var stdout2); + var @out = new List(); + @out.AddRange(stdout1); + @out.AddRange(stdout2); + stdout = @out; + return ret2; + } + } + + /// + /// Creates a simple build script that runs the specified exe. + /// + /// The arguments to the executable, or null. + /// The working directory (null for current directory). + /// Additional environment variables. + public static BuildScript Create(string exe, string argumentsOpt, string workingDirectory, IDictionary environment) => + new BuildCommand(exe, argumentsOpt, workingDirectory, environment); + + /// + /// Creates a simple build script that runs the specified function. + /// + public static BuildScript Create(Func func) => + new ReturnBuildCommand(func); + + /// + /// Creates a build script that runs , followed by running the script + /// produced by on the exit code from . + /// + public static BuildScript Bind(BuildScript s1, Func s2) => + new BindBuildScript(s1, s2); + + /// + /// Creates a build script that runs , followed by running the script + /// produced by on the exit code and standard output from + /// . + /// + public static BuildScript Bind(BuildScript s1, Func, int, BuildScript> s2) => + new BindBuildScript(s1, s2); + + const int SuccessCode = 0; + /// + /// The empty build script that always returns exit code 0. + /// + public static readonly BuildScript Success = Create(actions => SuccessCode); + + const int FailureCode = 1; + /// + /// The empty build script that always returns exit code 1. + /// + public static readonly BuildScript Failure = Create(actions => FailureCode); + + static bool Succeeded(int i) => i == SuccessCode; + + public static BuildScript operator &(BuildScript s1, BuildScript s2) => + new BindBuildScript(s1, ret1 => Succeeded(ret1) ? s2 : Create(actions => ret1)); + + public static BuildScript operator &(BuildScript s1, Func s2) => + new BindBuildScript(s1, ret1 => Succeeded(ret1) ? s2() : Create(actions => ret1)); + + public static BuildScript operator |(BuildScript s1, BuildScript s2) => + new BindBuildScript(s1, ret1 => Succeeded(ret1) ? Success : s2); + + public static BuildScript operator |(BuildScript s1, Func s2) => + new BindBuildScript(s1, ret1 => Succeeded(ret1) ? Success : s2()); + + /// + /// Creates a build script that attempts to run the build script , + /// always returning a successful exit code. + /// + public static BuildScript Try(BuildScript s) => s | Success; + + /// + /// Creates a build script that deletes the given directory. + /// + public static BuildScript DeleteDirectory(string dir) => + Create(actions => + { + if (string.IsNullOrEmpty(dir) || !actions.DirectoryExists(dir)) + return FailureCode; + + try + { + actions.DirectoryDelete(dir, true); + } + catch + { + return FailureCode; + } + return SuccessCode; + }); + + /// + /// Creates a build script that deletes the given file. + /// + public static BuildScript DeleteFile(string file) => + Create(actions => + { + if (string.IsNullOrEmpty(file) || !actions.FileExists(file)) + return FailureCode; + + try + { + actions.FileDelete(file); + } + catch + { + return FailureCode; + } + return SuccessCode; + }); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/BuildTools.cs b/csharp/extractor/Semmle.Autobuild/BuildTools.cs new file mode 100644 index 000000000000..ca9285d2c9c1 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/BuildTools.cs @@ -0,0 +1,99 @@ +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Autobuild +{ + /// + /// A BAT file used to initialise the appropriate + /// Visual Studio version/platform. + /// + public class VcVarsBatFile + { + public readonly int ToolsVersion; + public readonly string Path; + public readonly string[] Platform; + + public VcVarsBatFile(string path, int version, params string[] platform) + { + Path = path; + ToolsVersion = version; + Platform = platform; + } + }; + + /// + /// Collection of available Visual Studio build tools. + /// + public static class BuildTools + { + public static IEnumerable GetCandidateVcVarsFiles(IBuildActions actions) + { + var programFilesx86 = actions.GetEnvironmentVariable("ProgramFiles(x86)"); + if (programFilesx86 == null) + yield break; + + // Attempt to use vswhere to find installations of Visual Studio + string vswhere = actions.PathCombine(programFilesx86, "Microsoft Visual Studio", "Installer", "vswhere.exe"); + + if (actions.FileExists(vswhere)) + { + int exitCode1 = actions.RunProcess(vswhere, "-prerelease -legacy -property installationPath", null, null, out var installationList); + int exitCode2 = actions.RunProcess(vswhere, "-prerelease -legacy -property installationVersion", null, null, out var versionList); + + if (exitCode1 == 0 && exitCode2 == 0 && versionList.Count == installationList.Count) + { + // vswhere ran successfully and produced the expected output + foreach (var vsInstallation in versionList.Zip(installationList, (v, i) => (Version: v, InstallationPath: i))) + { + var dot = vsInstallation.Version.IndexOf('.'); + var majorVersionString = dot == -1 ? vsInstallation.Version : vsInstallation.Version.Substring(0, dot); + if (int.TryParse(majorVersionString, out int majorVersion)) + { + if (majorVersion < 15) + { + yield return new VcVarsBatFile(actions.PathCombine(vsInstallation.InstallationPath, @"VC\vcvarsall.bat"), majorVersion, "x86"); + } + else + { + yield return new VcVarsBatFile(actions.PathCombine(vsInstallation.InstallationPath, @"VC\Auxiliary\Build\vcvars32.bat"), majorVersion, "x86"); + yield return new VcVarsBatFile(actions.PathCombine(vsInstallation.InstallationPath, @"VC\Auxiliary\Build\vcvars64.bat"), majorVersion, "x64"); + } + } + // else: Skip installation without a version + } + yield break; + } + } + + // vswhere not installed or didn't run correctly - return legacy Visual Studio versions + yield return new VcVarsBatFile(actions.PathCombine(programFilesx86, @"Microsoft Visual Studio 14.0\VC\vcvarsall.bat"), 14, "x86"); + yield return new VcVarsBatFile(actions.PathCombine(programFilesx86, @"Microsoft Visual Studio 12.0\VC\vcvarsall.bat"), 12, "x86"); + yield return new VcVarsBatFile(actions.PathCombine(programFilesx86, @"Microsoft Visual Studio 11.0\VC\vcvarsall.bat"), 11, "x86"); + yield return new VcVarsBatFile(actions.PathCombine(programFilesx86, @"Microsoft Visual Studio 10.0\VC\vcvarsall.bat"), 10, "x86"); + } + + /// + /// Enumerates all available tools. + /// + public static IEnumerable VcVarsAllBatFiles(IBuildActions actions) => + GetCandidateVcVarsFiles(actions).Where(v => actions.FileExists(v.Path)); + + /// + /// Finds a VcVars file that provides a compatible environment for the given solution. + /// + /// The solution file. + /// A compatible file, or throws an exception. + public static VcVarsBatFile FindCompatibleVcVars(IBuildActions actions, ISolution sln) => + FindCompatibleVcVars(actions, sln.ToolsVersion.Major); + + /// + /// Finds a VcVars that provides a compatible environment for the given tools version. + /// + /// The tools version. + /// A compatible file, or null. + public static VcVarsBatFile FindCompatibleVcVars(IBuildActions actions, int targetVersion) => + targetVersion < 10 ? + VcVarsAllBatFiles(actions).OrderByDescending(b => b.ToolsVersion).FirstOrDefault() : + VcVarsAllBatFiles(actions).Where(b => b.ToolsVersion >= targetVersion).OrderBy(b => b.ToolsVersion).FirstOrDefault(); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/CommandBuilder.cs b/csharp/extractor/Semmle.Autobuild/CommandBuilder.cs new file mode 100644 index 000000000000..273cc52b0362 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/CommandBuilder.cs @@ -0,0 +1,195 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Semmle.Autobuild +{ + /// + /// Utility to construct a build command. + /// + class CommandBuilder + { + enum EscapeMode { Process, Cmd }; + + readonly StringBuilder arguments; + bool firstCommand; + string executable; + readonly EscapeMode escapingMode; + readonly string workingDirectory; + readonly IDictionary environment; + + /// + /// Initializes a new instance of the class. + /// + /// The working directory (null for current directory). + /// Additional environment variables. + public CommandBuilder(IBuildActions actions, string workingDirectory = null, IDictionary environment = null) + { + arguments = new StringBuilder(); + if (actions.IsWindows()) + { + executable = "cmd.exe"; + arguments.Append("/C"); + escapingMode = EscapeMode.Cmd; + } + else + { + escapingMode = EscapeMode.Process; + } + + firstCommand = true; + this.workingDirectory = workingDirectory; + this.environment = environment; + } + + void OdasaIndex(string odasa) + { + RunCommand(odasa, "index --auto"); + } + + public CommandBuilder CallBatFile(string batFile, string argumentsOpt = null) + { + NextCommand(); + arguments.Append(" CALL"); + QuoteArgument(batFile); + Argument(argumentsOpt); + return this; + } + + /// + /// Perform odasa index on a given command or BAT file. + /// + /// The odasa executable. + /// The command to run. + /// Additional arguments. + /// this for chaining calls. + public CommandBuilder IndexCommand(string odasa, string command, string argumentsOpt = null) + { + OdasaIndex(odasa); + QuoteArgument(command); + Argument(argumentsOpt); + return this; + } + + static readonly char[] specialChars = { ' ', '\t', '\n', '\v', '\"' }; + static readonly char[] cmdMetacharacter = { '(', ')', '%', '!', '^', '\"', '<', '>', '&', '|' }; + + /// + /// Appends the given argument to the command line. + /// + /// The argument to append. + /// Whether to always quote the argument. + /// Whether to escape for cmd.exe + /// + /// + /// This implementation is copied from + /// https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ + /// + void ArgvQuote(string argument, bool force) + { + bool cmd = escapingMode == EscapeMode.Cmd; + if (!force && + !string.IsNullOrEmpty(argument) && + argument.IndexOfAny(specialChars) == -1) + { + arguments.Append(argument); + } + else + { + if (cmd) arguments.Append('^'); + arguments.Append('\"'); + for (int it = 0; ; ++it) + { + var numBackslashes = 0; + while (it != argument.Length && argument[it] == '\\') + { + ++it; + ++numBackslashes; + } + + if (it == argument.Length) + { + arguments.Append('\\', numBackslashes * 2); + break; + } + else if (argument[it] == '\"') + { + arguments.Append('\\', numBackslashes * 2 + 1); + if (cmd) arguments.Append('^'); + arguments.Append(arguments[it]); + } + else + { + arguments.Append('\\', numBackslashes); + if (cmd && cmdMetacharacter.Any(c => c == argument[it])) + arguments.Append('^'); + + arguments.Append(argument[it]); + } + } + if (cmd) arguments.Append('^'); + arguments.Append('\"'); + } + } + + public CommandBuilder QuoteArgument(string argumentsOpt) + { + if (argumentsOpt != null) + { + NextArgument(); + ArgvQuote(argumentsOpt, false); + } + return this; + } + + void NextArgument() + { + if (arguments.Length > 0) + arguments.Append(' '); + } + + public CommandBuilder Argument(string argumentsOpt) + { + if (argumentsOpt != null) + { + NextArgument(); + arguments.Append(argumentsOpt); + } + return this; + } + + void NextCommand() + { + if (firstCommand) + firstCommand = false; + else + arguments.Append(" &&"); + } + + public CommandBuilder RunCommand(string exe, string argumentsOpt = null) + { + var (exe0, arg0) = + escapingMode == EscapeMode.Process && exe.EndsWith(".exe", System.StringComparison.Ordinal) + ? ("mono", exe) // Linux + : (exe, null); + + NextCommand(); + if (executable == null) + { + executable = exe0; + } + else + { + QuoteArgument(exe0); + } + Argument(arg0); + Argument(argumentsOpt); + return this; + } + + /// + /// Returns a build script that contains just this command. + /// + public BuildScript Script => BuildScript.Create(executable, arguments.ToString(), workingDirectory, environment); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/DotNetRule.cs b/csharp/extractor/Semmle.Autobuild/DotNetRule.cs new file mode 100644 index 000000000000..a862a4f1a1b7 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/DotNetRule.cs @@ -0,0 +1,268 @@ +using System; +using Semmle.Util.Logging; +using System.Linq; +using Newtonsoft.Json.Linq; +using System.Collections.Generic; +using System.IO; + +namespace Semmle.Autobuild +{ + /// + /// A build rule where the build command is of the form "dotnet build". + /// Currently unused because the tracer does not work with dotnet. + /// + class DotNetRule : IBuildRule + { + public BuildScript Analyse(Autobuilder builder) + { + builder.Log(Severity.Info, "Attempting to build using .NET Core"); + + var projects = builder.SolutionsToBuild.Any() + ? builder.SolutionsToBuild.SelectMany(s => s.Projects).ToArray() + : builder.GetExtensions(Language.CSharp.ProjectExtension).Select(p => new Project(builder, p)).ToArray(); + + var notDotNetProject = projects.FirstOrDefault(p => !p.DotNetProject); + if (notDotNetProject != null) + { + builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject); + return BuildScript.Failure; + } + + if (!builder.SolutionsToBuild.Any()) + // Attempt dotnet build in root folder + return WithDotNet(builder, dotNet => + { + var info = GetInfoCommand(builder.Actions, dotNet); + var clean = GetCleanCommand(builder.Actions, dotNet).Script; + var restore = GetRestoreCommand(builder.Actions, dotNet).Script; + var build = GetBuildCommand(builder, dotNet).Script; + return info & clean & BuildScript.Try(restore) & build; + }); + + // Attempt dotnet build on each solution + return WithDotNet(builder, dotNet => + { + var ret = GetInfoCommand(builder.Actions, dotNet); + foreach (var solution in builder.SolutionsToBuild) + { + var cleanCommand = GetCleanCommand(builder.Actions, dotNet); + cleanCommand.QuoteArgument(solution.Path); + var clean = cleanCommand.Script; + + var restoreCommand = GetRestoreCommand(builder.Actions, dotNet); + restoreCommand.QuoteArgument(solution.Path); + var restore = restoreCommand.Script; + + var buildCommand = GetBuildCommand(builder, dotNet); + buildCommand.QuoteArgument(solution.Path); + var build = buildCommand.Script; + + ret &= clean & BuildScript.Try(restore) & build; + } + return ret; + }); + } + + /// + /// Returns a script that attempts to download relevant version(s) of the + /// .NET Core SDK, followed by running the script generated by . + /// + /// The first element DotNetPath of the argument to + /// is the path where .NET Core was installed, and the second element Environment + /// is any additional required environment variables. The tuple argument is null + /// when the installation failed. + /// + public static BuildScript WithDotNet(Autobuilder builder, Func<(string DotNetPath, IDictionary Environment)?, BuildScript> f) + { + var installDir = builder.Actions.PathCombine(builder.Options.RootDirectory, ".dotnet"); + var installScript = DownloadDotNet(builder, installDir); + return BuildScript.Bind(installScript, installed => + { + if (installed == 0) + { + // The installation succeeded, so use the newly installed .NET Core + var path = builder.Actions.GetEnvironmentVariable("PATH"); + var delim = builder.Actions.IsWindows() ? ";" : ":"; + var env = new Dictionary{ + { "DOTNET_MULTILEVEL_LOOKUP", "false" }, // prevent look up of other .NET Core SDKs + { "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" }, + { "PATH", installDir + delim + path } + }; + return f((installDir, env)); + } + + return f(null); + }); + } + + /// + /// Returns a script for downloading relevant versions of the + /// .NET Core SDK. The SDK(s) will be installed at installDir + /// (provided that the script succeeds). + /// + static BuildScript DownloadDotNet(Autobuilder builder, string installDir) + { + if (!string.IsNullOrEmpty(builder.Options.DotNetVersion)) + // Specific version supplied in configuration: always use that + return DownloadDotNetVersion(builder, installDir, builder.Options.DotNetVersion); + + // Download versions mentioned in `global.json` files + // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json + var installScript = BuildScript.Success; + var validGlobalJson = false; + foreach (var path in builder.Paths.Where(p => p.EndsWith("global.json", StringComparison.Ordinal))) + { + string version; + try + { + var o = JObject.Parse(File.ReadAllText(path)); + version = (string)o["sdk"]["version"]; + } + catch + { + // not a valid global.json file + continue; + } + + installScript &= DownloadDotNetVersion(builder, installDir, version); + validGlobalJson = true; + } + + return validGlobalJson ? installScript : BuildScript.Failure; + } + + /// + /// Returns a script for downloading a specific .NET Core SDK version, if the + /// version is not already installed. + /// + /// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script. + /// + static BuildScript DownloadDotNetVersion(Autobuilder builder, string path, string version) + { + return BuildScript.Bind(GetInstalledSdksScript(builder.Actions), (sdks, sdksRet) => + { + if (sdksRet == 0 && sdks.Count() == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal)) + // The requested SDK is already installed (and no other SDKs are installed), so + // no need to reinstall + return BuildScript.Failure; + + builder.Log(Severity.Info, "Attempting to download .NET Core {0}", version); + + if (builder.Actions.IsWindows()) + { + var psScript = @"param([string]$Version, [string]$InstallDir) + +add-type @"" +using System.Net; +using System.Security.Cryptography.X509Certificates; +public class TrustAllCertsPolicy : ICertificatePolicy +{ + public bool CheckValidationResult(ServicePoint srvPoint, X509Certificate certificate, WebRequest request, int certificateProblem) + { + return true; + } +} +""@ +$AllProtocols = [System.Net.SecurityProtocolType]'Ssl3,Tls,Tls11,Tls12' +[System.Net.ServicePointManager]::SecurityProtocol = $AllProtocols +[System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy +$Script = Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1' + +$arguments = @{ + Channel = 'release' + Version = $Version + InstallDir = $InstallDir +} + +$ScriptBlock = [scriptblock]::create("".{$($Script)} $(&{$args} @arguments)"") + +Invoke-Command -ScriptBlock $ScriptBlock"; + var psScriptFile = builder.Actions.PathCombine(builder.Options.RootDirectory, "install-dotnet.ps1"); + builder.Actions.WriteAllText(psScriptFile, psScript); + + var install = new CommandBuilder(builder.Actions). + RunCommand("powershell"). + Argument("-NoProfile"). + Argument("-ExecutionPolicy"). + Argument("unrestricted"). + Argument("-file"). + Argument(psScriptFile). + Argument("-Version"). + Argument(version). + Argument("-InstallDir"). + Argument(path); + return install.Script; + } + else + { + var curl = new CommandBuilder(builder.Actions). + RunCommand("curl"). + Argument("-sO"). + Argument("https://dot.net/v1/dotnet-install.sh"); + + var chmod = new CommandBuilder(builder.Actions). + RunCommand("chmod"). + Argument("u+x"). + Argument("dotnet-install.sh"); + + var install = new CommandBuilder(builder.Actions). + RunCommand("./dotnet-install.sh"). + Argument("--channel"). + Argument("release"). + Argument("--version"). + Argument(version). + Argument("--install-dir"). + Argument(path); + + return curl.Script & chmod.Script & install.Script; + } + }); + } + + static BuildScript GetInstalledSdksScript(IBuildActions actions) + { + var listSdks = new CommandBuilder(actions). + RunCommand("dotnet"). + Argument("--list-sdks"); + return listSdks.Script; + } + + static string DotNetCommand(IBuildActions actions, string dotNetPath) => + dotNetPath != null ? actions.PathCombine(dotNetPath, "dotnet") : "dotnet"; + + BuildScript GetInfoCommand(IBuildActions actions, (string DotNetPath, IDictionary Environment)? arg) + { + var info = new CommandBuilder(actions, null, arg?.Environment). + RunCommand(DotNetCommand(actions, arg?.DotNetPath)). + Argument("--info"); + return info.Script; + } + + CommandBuilder GetCleanCommand(IBuildActions actions, (string DotNetPath, IDictionary Environment)? arg) + { + var clean = new CommandBuilder(actions, null, arg?.Environment). + RunCommand(DotNetCommand(actions, arg?.DotNetPath)). + Argument("clean"); + return clean; + } + + CommandBuilder GetRestoreCommand(IBuildActions actions, (string DotNetPath, IDictionary Environment)? arg) + { + var restore = new CommandBuilder(actions, null, arg?.Environment). + RunCommand(DotNetCommand(actions, arg?.DotNetPath)). + Argument("restore"); + return restore; + } + + CommandBuilder GetBuildCommand(Autobuilder builder, (string DotNetPath, IDictionary Environment)? arg) + { + var build = new CommandBuilder(builder.Actions, null, arg?.Environment). + IndexCommand(builder.Odasa, DotNetCommand(builder.Actions, arg?.DotNetPath)). + Argument("build"). + Argument("--no-incremental"). + Argument("/p:UseSharedCompilation=false"). + Argument(builder.Options.DotNetArguments); + return build; + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/Language.cs b/csharp/extractor/Semmle.Autobuild/Language.cs new file mode 100644 index 000000000000..6f8d73b2a471 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Language.cs @@ -0,0 +1,24 @@ +namespace Semmle.Autobuild +{ + public sealed class Language + { + public static readonly Language Cpp = new Language(".vcxproj"); + public static readonly Language CSharp = new Language(".csproj"); + + public bool ProjectFileHasThisLanguage(string path) => + System.IO.Path.GetExtension(path) == ProjectExtension; + + public static bool IsProjectFileForAnySupportedLanguage(string path) => + Cpp.ProjectFileHasThisLanguage(path) || CSharp.ProjectFileHasThisLanguage(path); + + public readonly string ProjectExtension; + + private Language(string extension) + { + ProjectExtension = extension; + } + + public override string ToString() => + ProjectExtension == Cpp.ProjectExtension ? "C/C++" : "C#"; + } +} diff --git a/csharp/extractor/Semmle.Autobuild/MsBuildRule.cs b/csharp/extractor/Semmle.Autobuild/MsBuildRule.cs new file mode 100644 index 000000000000..cb7eae2c23b1 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/MsBuildRule.cs @@ -0,0 +1,118 @@ +using Semmle.Util.Logging; +using System.IO; +using System.Linq; + +namespace Semmle.Autobuild +{ + /// + /// A build rule using msbuild. + /// + class MsBuildRule : IBuildRule + { + /// + /// The name of the msbuild command. + /// + const string MsBuild = "msbuild"; + + public BuildScript Analyse(Autobuilder builder) + { + builder.Log(Severity.Info, "Attempting to build using MSBuild"); + + if (!builder.SolutionsToBuild.Any()) + { + builder.Log(Severity.Info, "Could not find a suitable solution file to build"); + return BuildScript.Failure; + } + + var vsTools = GetVcVarsBatFile(builder); + + if (vsTools == null && builder.SolutionsToBuild.Any()) + { + vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, builder.SolutionsToBuild.First()); + } + + if (vsTools == null && builder.Actions.IsWindows()) + { + builder.Log(Severity.Warning, "Could not find a suitable version of vcvarsall.bat"); + } + + var nuget = builder.Actions.PathCombine(builder.SemmlePlatformTools, "csharp", "nuget", "nuget.exe"); + + var ret = BuildScript.Success; + + foreach (var solution in builder.SolutionsToBuild) + { + if (builder.Options.NugetRestore) + { + var nugetCommand = new CommandBuilder(builder.Actions). + RunCommand(nuget). + Argument("restore"). + QuoteArgument(solution.Path); + ret &= BuildScript.Try(nugetCommand.Script); + } + + var command = new CommandBuilder(builder.Actions); + + if (vsTools != null) + { + command.CallBatFile(vsTools.Path); + } + + command.IndexCommand(builder.Odasa, MsBuild); + command.QuoteArgument(solution.Path); + + command.Argument("/p:UseSharedCompilation=false"); + + string target = builder.Options.MsBuildTarget != null ? builder.Options.MsBuildTarget : "rebuild"; + string platform = builder.Options.MsBuildPlatform != null ? builder.Options.MsBuildPlatform : solution.DefaultPlatformName; + string configuration = builder.Options.MsBuildConfiguration != null ? builder.Options.MsBuildConfiguration : solution.DefaultConfigurationName; + + command.Argument("/t:" + target); + command.Argument(string.Format("/p:Platform=\"{0}\"", platform)); + command.Argument(string.Format("/p:Configuration=\"{0}\"", configuration)); + command.Argument("/p:MvcBuildViews=true"); + + command.Argument(builder.Options.MsBuildArguments); + + ret &= command.Script; + } + + return ret; + } + + /// + /// Gets the BAT file used to initialize the appropriate Visual Studio + /// version/platform, as specified by the `vstools_version` property in + /// lgtm.yml. + /// + /// Returns null when no version is specified. + /// + public static VcVarsBatFile GetVcVarsBatFile(Autobuilder builder) + { + VcVarsBatFile vsTools = null; + + if (builder.Options.VsToolsVersion != null) + { + if (int.TryParse(builder.Options.VsToolsVersion, out var msToolsVersion)) + { + foreach (var b in BuildTools.VcVarsAllBatFiles(builder.Actions)) + { + builder.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion); + } + + vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, msToolsVersion); + if (vsTools == null) + builder.Log(Severity.Warning, "Could not find build tools matching version {0}", msToolsVersion); + else + builder.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path); + } + else + { + builder.Log(Severity.Error, "The format of vstools_version is incorrect. Please specify an integer."); + } + } + + return vsTools; + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/Program.cs b/csharp/extractor/Semmle.Autobuild/Program.cs new file mode 100644 index 000000000000..c4542864a091 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Program.cs @@ -0,0 +1,28 @@ +using System; + +namespace Semmle.Autobuild +{ + class Program + { + static int Main() + { + var options = new AutobuildOptions(); + var actions = SystemBuildActions.Instance; + + try + { + options.ReadEnvironment(actions); + } + catch (ArgumentOutOfRangeException ex) + { + Console.WriteLine("The value \"{0}\" for parameter \"{1}\" is invalid", ex.ActualValue, ex.ParamName); + } + + var builder = new Autobuilder(actions, options); + + Console.WriteLine($"Semmle autobuilder for {options.Language}"); + + return builder.AttemptBuild(); + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/Project.cs b/csharp/extractor/Semmle.Autobuild/Project.cs new file mode 100644 index 000000000000..e0a1ee8386dc --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Project.cs @@ -0,0 +1,69 @@ +using System; +using System.IO; +using System.Xml; +using Semmle.Util.Logging; + +namespace Semmle.Autobuild +{ + /// + /// Representation of a .csproj (C#) or .vcxproj (C++) file. + /// C# project files come in 2 flavours, .Net core and msbuild, but they + /// have the same file extension. + /// + public class Project + { + /// + /// Holds if this project is for .Net core. + /// + public bool DotNetProject { get; private set; } + + public bool ValidToolsVersion { get; private set; } + + public Version ToolsVersion { get; private set; } + + readonly string filename; + + public Project(Autobuilder builder, string filename) + { + this.filename = filename; + ToolsVersion = new Version(); + + if (!File.Exists(filename)) + return; + + var projFile = new XmlDocument(); + projFile.Load(filename); + var root = projFile.DocumentElement; + + if (root.Name == "Project") + { + if (root.HasAttribute("Sdk")) + { + DotNetProject = true; + } + else + { + var toolsVersion = root.GetAttribute("ToolsVersion"); + if (string.IsNullOrEmpty(toolsVersion)) + { + builder.Log(Severity.Warning, "Project {0} is missing a tools version", filename); + } + else + { + try + { + ToolsVersion = new Version(toolsVersion); + ValidToolsVersion = true; + } + catch // Generic catch clause - Version constructor throws about 5 different exceptions. + { + builder.Log(Severity.Warning, "Project {0} has invalid tools version {1}", filename, toolsVersion); + } + } + } + } + } + + public override string ToString() => filename; + } +} diff --git a/csharp/extractor/Semmle.Autobuild/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Autobuild/Properties/AssemblyInfo.cs new file mode 100644 index 000000000000..e3da7ca22e9e --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Autobuild")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Semmle")] +[assembly: AssemblyProduct("Semmle Visual Studio Autobuild")] +[assembly: AssemblyCopyright("Copyright © Semmle 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("1d9920ad-7b00-4df1-8b01-9ff5b687828e")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Autobuild/Semmle.Autobuild.csproj b/csharp/extractor/Semmle.Autobuild/Semmle.Autobuild.csproj new file mode 100644 index 000000000000..8a9d272169c5 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Semmle.Autobuild.csproj @@ -0,0 +1,27 @@ + + + + netcoreapp2.0 + Semmle.Autobuild + Semmle.Autobuild + + Exe + + false + + + + + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Autobuild/Solution.cs b/csharp/extractor/Semmle.Autobuild/Solution.cs new file mode 100644 index 000000000000..66bbff777ac8 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/Solution.cs @@ -0,0 +1,107 @@ +using Microsoft.Build.Construction; +using Microsoft.Build.Exceptions; +using System; +using System.Collections.Generic; +using System.Linq; +using Semmle.Util; + +namespace Semmle.Autobuild +{ + /// + /// A solution file, extension .sln. + /// + public interface ISolution + { + /// + /// List of C# or C++ projects contained in the solution. + /// (There could be other project types as well - these are ignored.) + /// + + IEnumerable Projects { get; } + + /// + /// Solution configurations. + /// + IEnumerable Configurations { get; } + + /// + /// The default configuration name, e.g. "Release" + /// + string DefaultConfigurationName { get; } + + /// + /// The default platform name, e.g. "x86" + /// + string DefaultPlatformName { get; } + + /// + /// The path of the solution file. + /// + string Path { get; } + + /// + /// The number of C# or C++ projects. + /// + int ProjectCount { get; } + + /// + /// Gets the "best" tools version for this solution. + /// If there are several versions, because the project files + /// are inconsistent, then pick the highest/latest version. + /// If no tools versions are present, return 0.0.0.0. + /// + Version ToolsVersion { get; } + } + + /// + /// A solution file on the filesystem, read using Microsoft.Build. + /// + class Solution : ISolution + { + readonly SolutionFile solution; + + public IEnumerable Projects { get; private set; } + + public IEnumerable Configurations => + solution == null ? Enumerable.Empty() : solution.SolutionConfigurations; + + public string DefaultConfigurationName => + solution == null ? "" : solution.GetDefaultConfigurationName(); + + public string DefaultPlatformName => + solution == null ? "" : solution.GetDefaultPlatformName(); + + public Solution(Autobuilder builder, string path) + { + Path = System.IO.Path.GetFullPath(path); + try + { + solution = SolutionFile.Parse(Path); + + Projects = + solution.ProjectsInOrder. + Where(p => p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat). + Select(p => System.IO.Path.GetFullPath(FileUtils.ConvertToNative(p.AbsolutePath))). + Where(p => builder.Options.Language.ProjectFileHasThisLanguage(p)). + Select(p => new Project(builder, p)). + ToArray(); + } + catch (InvalidProjectFileException) + { + // We allow specifying projects as solutions in lgtm.yml, so model + // that scenario as a solution with just that one project + Projects = Language.IsProjectFileForAnySupportedLanguage(Path) + ? new[] { new Project(builder, Path) } + : new Project[0]; + } + } + + public string Path { get; private set; } + + public int ProjectCount => Projects.Count(); + + IEnumerable ToolsVersions => Projects.Where(p => p.ValidToolsVersion).Select(p => p.ToolsVersion); + + public Version ToolsVersion => ToolsVersions.Any() ? ToolsVersions.Max() : new Version(); + } +} diff --git a/csharp/extractor/Semmle.Autobuild/StandaloneBuildRule.cs b/csharp/extractor/Semmle.Autobuild/StandaloneBuildRule.cs new file mode 100644 index 000000000000..3fafa84e454b --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/StandaloneBuildRule.cs @@ -0,0 +1,46 @@ +using System.IO; + +namespace Semmle.Autobuild +{ + /// + /// Build using standalone extraction. + /// + class StandaloneBuildRule : IBuildRule + { + public BuildScript Analyse(Autobuilder builder) + { + BuildScript GetCommand(string solution) + { + var standalone = builder.Actions.PathCombine(builder.SemmlePlatformTools, "csharp", "Semmle.Extraction.CSharp.Standalone"); + var cmd = new CommandBuilder(builder.Actions); + cmd.RunCommand(standalone); + + if (solution != null) + cmd.QuoteArgument(solution); + + cmd.Argument("--references:."); + + if (!builder.Options.NugetRestore) + { + cmd.Argument("--skip-nuget"); + } + + return cmd.Script; + } + + if (!builder.Options.Buildless) + return BuildScript.Failure; + + var solutions = builder.Options.Solution.Length; + + if (solutions == 0) + return GetCommand(null); + + var script = BuildScript.Success; + foreach (var solution in builder.Options.Solution) + script &= GetCommand(solution); + + return script; + } + } +} diff --git a/csharp/extractor/Semmle.Autobuild/XmlBuildRule.cs b/csharp/extractor/Semmle.Autobuild/XmlBuildRule.cs new file mode 100644 index 000000000000..6b0a5b1cbb19 --- /dev/null +++ b/csharp/extractor/Semmle.Autobuild/XmlBuildRule.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace Semmle.Autobuild +{ + /// + /// XML extraction. + /// + class XmlBuildRule : IBuildRule + { + public BuildScript Analyse(Autobuilder builder) + { + var command = new CommandBuilder(builder.Actions). + RunCommand(builder.Odasa). + Argument("index --xml --extensions config"); + return command.Script; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/ExtractorOptions.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/ExtractorOptions.cs new file mode 100644 index 000000000000..0e9a8f9e116f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/ExtractorOptions.cs @@ -0,0 +1,267 @@ +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System.Reflection.PortableExecutable; +using System.Reflection.Metadata; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Globalization; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CIL.Driver +{ + /// + /// Information about a single assembly. + /// In particular, provides references between assemblies. + /// + class AssemblyInfo + { + public override string ToString() => filename; + + static AssemblyName CreateAssemblyName(MetadataReader mdReader, StringHandle name, System.Version version, StringHandle culture) + { + var cultureString = mdReader.GetString(culture); + + var assemblyName = new AssemblyName() + { + Name = mdReader.GetString(name), + Version = version + }; + + if (cultureString != "neutral") + assemblyName.CultureInfo = CultureInfo.GetCultureInfo(cultureString); + + return assemblyName; + } + + static AssemblyName CreateAssemblyName(MetadataReader mdReader, AssemblyReference ar) + { + var an = CreateAssemblyName(mdReader, ar.Name, ar.Version, ar.Culture); + if (!ar.PublicKeyOrToken.IsNil) + an.SetPublicKeyToken(mdReader.GetBlobBytes(ar.PublicKeyOrToken)); + return an; + } + + static AssemblyName CreateAssemblyName(MetadataReader mdReader, AssemblyDefinition ad) + { + var an = CreateAssemblyName(mdReader, ad.Name, ad.Version, ad.Culture); + if (!ad.PublicKey.IsNil) + an.SetPublicKey(mdReader.GetBlobBytes(ad.PublicKey)); + return an; + } + + public AssemblyInfo(string path) + { + filename = path; + + // Attempt to open the file and see if it's a valid assembly. + using (var stream = File.OpenRead(path)) + using (var peReader = new PEReader(stream)) + { + try + { + isAssembly = peReader.HasMetadata; + if (!isAssembly) return; + + var mdReader = peReader.GetMetadataReader(); + + isAssembly = mdReader.IsAssembly; + if (!mdReader.IsAssembly) return; + + // Get our own assembly name + name = CreateAssemblyName(mdReader, mdReader.GetAssemblyDefinition()); + + references = mdReader.AssemblyReferences. + Select(r => mdReader.GetAssemblyReference(r)). + Select(ar => CreateAssemblyName(mdReader, ar)). + ToArray(); + } + catch (System.BadImageFormatException) + { + // This failed on one of the Roslyn tests that includes + // a deliberately malformed assembly. + // In this case, we just skip the extraction of this assembly. + isAssembly = false; + } + } + } + + public readonly AssemblyName name; + public readonly string filename; + public bool extract; + public readonly bool isAssembly; + public readonly AssemblyName[] references; + } + + /// + /// Helper to manage a collection of assemblies. + /// Resolves references between assemblies and determines which + /// additional assemblies need to be extracted. + /// + class AssemblyList + { + class AssemblyNameComparer : IEqualityComparer + { + bool IEqualityComparer.Equals(AssemblyName x, AssemblyName y) => + x.Name == y.Name && x.Version == y.Version; + + int IEqualityComparer.GetHashCode(AssemblyName obj) => + obj.Name.GetHashCode() + 7 * obj.Version.GetHashCode(); + } + + readonly Dictionary assembliesRead = new Dictionary(new AssemblyNameComparer()); + + public void AddFile(string assemblyPath, bool extractAll) + { + if (!filesAnalyzed.Contains(assemblyPath)) + { + filesAnalyzed.Add(assemblyPath); + var info = new AssemblyInfo(assemblyPath); + if (info.isAssembly) + { + info.extract = extractAll; + if (!assembliesRead.ContainsKey(info.name)) + assembliesRead.Add(info.name, info); + } + } + } + + public IEnumerable AssembliesToExtract => assembliesRead.Values.Where(info => info.extract); + + IEnumerable AssembliesToReference => AssembliesToExtract.SelectMany(info => info.references); + + public void ResolveReferences() + { + var assembliesToReference = new Stack(AssembliesToReference); + + while (assembliesToReference.Any()) + { + var item = assembliesToReference.Pop(); + AssemblyInfo info; + if (assembliesRead.TryGetValue(item, out info)) + { + if (!info.extract) + { + info.extract = true; + foreach (var reference in info.references) + assembliesToReference.Push(reference); + } + } + else + { + missingReferences.Add(item); + } + } + } + + readonly HashSet filesAnalyzed = new HashSet(); + public readonly HashSet missingReferences = new HashSet(); + } + + /// + /// Parses the command line and collates a list of DLLs/EXEs to extract. + /// + class ExtractorOptions + { + readonly AssemblyList assemblyList = new AssemblyList(); + + public void AddDirectory(string directory, bool extractAll) + { + foreach (var file in + Directory.EnumerateFiles(directory, "*.dll", SearchOption.AllDirectories). + Concat(Directory.EnumerateFiles(directory, "*.exe", SearchOption.AllDirectories))) + { + assemblyList.AddFile(file, extractAll); + } + } + + void AddFrameworkDirectories(bool extractAll) + { + AddDirectory(RuntimeEnvironment.GetRuntimeDirectory(), extractAll); + } + + public Verbosity Verbosity { get; private set; } + public bool NoCache { get; private set; } + public int Threads { get; private set; } + public bool PDB { get; private set; } + + void AddFileOrDirectory(string path) + { + path = Path.GetFullPath(path); + if (File.Exists(path)) + { + assemblyList.AddFile(path, true); + AddDirectory(Path.GetDirectoryName(path), false); + } + else if (Directory.Exists(path)) + { + AddDirectory(path, true); + } + } + + void ResolveReferences() + { + assemblyList.ResolveReferences(); + AssembliesToExtract = assemblyList.AssembliesToExtract.ToArray(); + } + + public IEnumerable AssembliesToExtract { get; private set; } + + /// + /// Gets the assemblies that were referenced but were not available to be + /// extracted. This is not an error, it just means that the database is not + /// as complete as it could be. + /// + public IEnumerable MissingReferences => assemblyList.missingReferences; + + public static ExtractorOptions ParseCommandLine(string[] args) + { + var options = new ExtractorOptions(); + options.Verbosity = Verbosity.Info; + options.Threads = System.Environment.ProcessorCount; + options.PDB = true; + + foreach (var arg in args) + { + if (arg == "--verbose") + { + options.Verbosity = Verbosity.All; + } + else if (arg == "--silent") + { + options.Verbosity = Verbosity.Off; + } + else if (arg.StartsWith("--verbosity:")) + { + options.Verbosity = (Verbosity)int.Parse(arg.Substring(12)); + } + else if (arg == "--dotnet") + { + options.AddFrameworkDirectories(true); + } + else if (arg == "--nocache") + { + options.NoCache = true; + } + else if (arg.StartsWith("--threads:")) + { + options.Threads = int.Parse(arg.Substring(10)); + } + else if (arg == "--no-pdb") + { + options.PDB = false; + } + else + { + options.AddFileOrDirectory(arg); + } + } + + options.AddFrameworkDirectories(false); + options.ResolveReferences(); + + return options; + } + + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs new file mode 100644 index 000000000000..df17f5a187ef --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs @@ -0,0 +1,68 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using System.IO; +using Semmle.Util.Logging; +using System.Diagnostics; + +namespace Semmle.Extraction.CIL.Driver +{ + class Program + { + static void DisplayHelp() + { + Console.WriteLine("CIL command line extractor"); + Console.WriteLine(); + Console.WriteLine("Usage: Semmle.Extraction.CIL.Driver.exe [options] path ..."); + Console.WriteLine(" --verbose Turn on verbose output"); + Console.WriteLine(" --dotnet Extract the .Net Framework"); + Console.WriteLine(" --nocache Overwrite existing trap files"); + Console.WriteLine(" --no-pdb Do not extract PDB files"); + Console.WriteLine(" path A directory/dll/exe to analyze"); + } + + static void ExtractAssembly(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs) + { + string trapFile; + bool extracted; + var sw = new Stopwatch(); + sw.Start(); + Entities.Assembly.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, out trapFile, out extracted); + sw.Stop(); + logger.Log(Severity.Info, " {0} ({1})", assemblyPath, sw.Elapsed); + } + + static void Main(string[] args) + { + if (args.Length == 0) + { + DisplayHelp(); + return; + } + + var options = ExtractorOptions.ParseCommandLine(args); + var layout = new Layout(); + var logger = new ConsoleLogger(options.Verbosity); + + var actions = options. + AssembliesToExtract.Select(asm => asm.filename). + Select(filename => () => ExtractAssembly(layout, filename, logger, options.NoCache, options.PDB)). + ToArray(); + + foreach (var missingRef in options.MissingReferences) + logger.Log(Severity.Info, " Missing assembly " + missingRef); + + var sw = new Stopwatch(); + sw.Start(); + var piOptions = new ParallelOptions + { + MaxDegreeOfParallelism = options.Threads + }; + + Parallel.Invoke(piOptions, actions); + + sw.Stop(); + logger.Log(Severity.Info, "Extraction completed in {0}", sw.Elapsed); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/Properties/AssemblyInfo.cs new file mode 100644 index 000000000000..56f7f94c1437 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.CIL.Driver")] +[assembly: AssemblyDescription("Semmle CIL extractor")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Semmle Ltd")] +[assembly: AssemblyProduct("Semmle.Extraction.CIL.Driver")] +[assembly: AssemblyCopyright("Copyright © Semmle 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("5642ae68-9c26-43c9-bd3c-49923dddf02d")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Semmle.Extraction.CIL.Driver.csproj b/csharp/extractor/Semmle.Extraction.CIL.Driver/Semmle.Extraction.CIL.Driver.csproj new file mode 100644 index 000000000000..b0e1a525490e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Semmle.Extraction.CIL.Driver.csproj @@ -0,0 +1,21 @@ + + + + Exe + netcoreapp2.0 + Semmle.Extraction.CIL.Driver + Semmle.Extraction.CIL.Driver + false + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.CIL/CachedFunction.cs b/csharp/extractor/Semmle.Extraction.CIL/CachedFunction.cs new file mode 100644 index 000000000000..3bbc386a691e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/CachedFunction.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL +{ + /// + /// A factory and a cache for mapping source entities to target entities. + /// Could be considered as a memoizer. + /// + /// The type of the source. + /// The type of the generated object. + public class CachedFunction + { + readonly Func generator; + readonly Dictionary cache; + + /// + /// Initializes the factory with a given mapping. + /// + /// The mapping. + public CachedFunction(Func g) + { + generator = g; + cache = new Dictionary(); + } + + /// + /// Gets the target for a given source. + /// Create it if it does not exist. + /// + /// The source object. + /// The created object. + public TargetType this[SrcType src] + { + get + { + TargetType result; + if (!cache.TryGetValue(src, out result)) + { + result = generator(src); + cache[src] = result; + } + return result; + } + } + } + + /// + /// A factory for mapping a pair of source entities to a target entity. + /// + /// Source entity type 1. + /// Source entity type 2. + /// The target type. + public class CachedFunction + { + readonly CachedFunction<(Src1, Src2), Target> factory; + + /// + /// Initializes the factory with a given mapping. + /// + /// The mapping. + public CachedFunction(Func g) + { + factory = new CachedFunction<(Src1, Src2), Target>(p => g(p.Item1, p.Item2)); + } + + public Target this[Src1 s1, Src2 s2] => factory[(s1, s2)]; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.cs new file mode 100644 index 000000000000..dc2801e5bc72 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.cs @@ -0,0 +1,173 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +namespace Semmle.Extraction.CIL +{ + /// + /// Extraction context for CIL extraction. + /// Adds additional context that is specific for CIL extraction. + /// One context = one DLL/EXE. + /// + partial class Context : IDisposable + { + public Extraction.Context cx; + readonly FileStream stream; + public readonly MetadataReader mdReader; + public readonly PEReader peReader; + public readonly string assemblyPath; + public Entities.Assembly assembly; + public PDB.IPdb pdb; + + public Context(Extraction.Context cx, string assemblyPath, bool extractPdbs) + { + this.cx = cx; + this.assemblyPath = assemblyPath; + stream = File.OpenRead(assemblyPath); + peReader = new PEReader(stream, PEStreamOptions.PrefetchEntireImage); + mdReader = peReader.GetMetadataReader(); + TypeSignatureDecoder = new Entities.TypeSignatureDecoder(this); + + globalNamespace = new Lazy(() => Populate(new Entities.Namespace(this, GetId(""), null))); + systemNamespace = new Lazy(() => Populate(new Entities.Namespace(this, "System"))); + genericHandleFactory = new CachedFunction(CreateGenericHandle); + namespaceFactory = new CachedFunction(n => CreateNamespace(mdReader.GetString(n))); + namespaceDefinitionFactory = new CachedFunction(CreateNamespace); + sourceFiles = new CachedFunction(path => new Entities.PdbSourceFile(this, path)); + folders = new CachedFunction(path => new Entities.Folder(this, path)); + sourceLocations = new CachedFunction(location => new Entities.PdbSourceLocation(this, location)); + + defaultGenericContext = new EmptyContext(this); + + var def = mdReader.GetAssemblyDefinition(); + AssemblyPrefix = GetId(def.Name) + "_" + def.Version.ToString() + "::"; + + if (extractPdbs) + { + pdb = PDB.PdbReader.Create(assemblyPath, peReader); + if (pdb != null) + { + cx.Extractor.Logger.Log(Util.Logging.Severity.Info, string.Format("Found PDB information for {0}", assemblyPath)); + } + } + } + + void IDisposable.Dispose() + { + if (pdb != null) + pdb.Dispose(); + peReader.Dispose(); + stream.Dispose(); + } + + /// + /// Extract the contents of a given entity. + /// + /// The entity to extract. + public void Extract(IExtractedEntity entity) + { + foreach (var content in entity.Contents) + { + content.Extract(this); + } + } + + public readonly Id AssemblyPrefix; + + public readonly Entities.TypeSignatureDecoder TypeSignatureDecoder; + + /// + /// A type used to signify something we can't handle yet. + /// Specifically, function pointers (used in C++). + /// + public Entities.Type ErrorType + { + get + { + var errorType = new Entities.ErrorType(this); + Populate(errorType); + return errorType; + } + } + + /// + /// Attempt to locate debugging information for a particular method. + /// + /// Returns null on failure, for example if there was no PDB information found for the + /// DLL, or if the particular method is compiler generated or doesn't come from source code. + /// + /// The handle of the method. + /// The debugging information, or null if the information could not be located. + public PDB.IMethod GetMethodDebugInformation(MethodDefinitionHandle handle) + { + return pdb == null ? null : pdb.GetMethod(handle.ToDebugInformationHandle()); + } + } + + /// + /// When we decode a type/method signature, we need access to + /// generic parameters. + /// + public abstract class GenericContext + { + public Context cx; + + public GenericContext(Context cx) + { + this.cx = cx; + } + + /// + /// The list of generic type parameters. + /// + public abstract IEnumerable TypeParameters { get; } + + /// + /// The list of generic method parameters. + /// + public abstract IEnumerable MethodParameters { get; } + + /// + /// Gets the `p`th type parameter. + /// + /// The index of the parameter. + /// + /// For constructed types, the supplied type. + /// For unbound types, the type parameter. + /// + public Entities.Type GetGenericTypeParameter(int p) + { + return TypeParameters.ElementAt(p); + } + + /// + /// Gets the `p`th method type parameter. + /// + /// The index of the parameter. + /// + /// For constructed types, the supplied type. + /// For unbound types, the type parameter. + /// + public Entities.Type GetGenericMethodParameter(int p) + { + return MethodParameters.ElementAt(p); + } + } + + /// + /// A generic context which does not contain any type parameters. + /// + public class EmptyContext : GenericContext + { + public EmptyContext(Context cx) : base(cx) + { + } + + public override IEnumerable TypeParameters { get { yield break; } } + + public override IEnumerable MethodParameters { get { yield break; } } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs new file mode 100644 index 000000000000..31745a434f67 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs @@ -0,0 +1,152 @@ +using System.Reflection; +using System.Globalization; +using System.Collections.Generic; +using Semmle.Util.Logging; +using System; + +namespace Semmle.Extraction.CIL.Entities +{ + public interface ILocation : IEntity + { + } + + interface IAssembly : ILocation + { + } + + /// + /// An assembly to extract. + /// + public class Assembly : LabelledEntity, IAssembly + { + public override Id IdSuffix => suffix; + + readonly File file; + readonly AssemblyName assemblyName; + + public Assembly(Context cx) : base(cx) + { + cx.assembly = this; + var def = cx.mdReader.GetAssemblyDefinition(); + + assemblyName = new AssemblyName(); + assemblyName.Name = cx.mdReader.GetString(def.Name); + assemblyName.Version = def.Version; + assemblyName.CultureInfo = new CultureInfo(cx.mdReader.GetString(def.Culture)); + + if (!def.PublicKey.IsNil) + assemblyName.SetPublicKey(cx.mdReader.GetBlobBytes(def.PublicKey)); + + ShortId = cx.GetId(assemblyName.FullName) + "#file:///" + cx.assemblyPath.Replace("\\", "/"); + + file = new File(cx, cx.assemblyPath); + } + + static readonly Id suffix = new StringId(";assembly"); + + public override IEnumerable Contents + { + get + { + yield return file; + yield return Tuples.assemblies(this, file, assemblyName.FullName, assemblyName.Name, assemblyName.Version.ToString()); + + if (cx.pdb != null) + { + foreach (var f in cx.pdb.SourceFiles) + { + yield return cx.CreateSourceFile(f); + } + } + + foreach (var handle in cx.mdReader.TypeDefinitions) + { + IExtractionProduct product = null; + try + { + product = cx.Create(handle); + } + catch (InternalError e) + { + cx.cx.Extractor.Message(new Message + { + exception = e, + message = "Error processing type definition", + severity = Semmle.Util.Logging.Severity.Error + }); + } + + // Limitation of C#: Cannot yield return inside a try-catch. + if (product != null) + yield return product; + } + + foreach (var handle in cx.mdReader.MethodDefinitions) + { + IExtractionProduct product = null; + try + { + product = cx.Create(handle); + } + catch (InternalError e) + { + cx.cx.Extractor.Message(new Message + { + exception = e, + message = "Error processing bytecode", + severity = Semmle.Util.Logging.Severity.Error + }); + } + + if (product != null) + yield return product; + } + } + } + + static void ExtractCIL(Extraction.Context cx, string assemblyPath, bool extractPdbs) + { + using (var cilContext = new Context(cx, assemblyPath, extractPdbs)) + { + cilContext.Populate(new Assembly(cilContext)); + cilContext.cx.PopulateAll(); + } + } + + /// + /// Main entry point to the CIL extractor. + /// Call this to extract a given assembly. + /// + /// The trap layout. + /// The full path of the assembly to extract. + /// The logger. + /// True to overwrite existing trap file. + /// Whether to extract PDBs. + /// The path of the trap file. + /// Whether the file was extracted (false=cached). + public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, out string trapFile, out bool extracted) + { + trapFile = ""; + extracted = false; + try + { + var extractor = new Extractor(false, assemblyPath, logger); + var project = layout.LookupProjectOrDefault(assemblyPath); + using (var trapWriter = project.CreateTrapWriter(logger, assemblyPath + ".cil", true)) + { + trapFile = trapWriter.TrapFile; + if (nocache || !System.IO.File.Exists(trapFile)) + { + var cx = new Extraction.Context(extractor, null, trapWriter, null); + ExtractCIL(cx, assemblyPath, extractPdbs); + extracted = true; + } + } + } + catch (Exception ex) + { + logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs new file mode 100644 index 000000000000..774de6cf145a --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.Reflection.Metadata; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A CIL attribute. + /// + interface IAttribute : IExtractedEntity + { + } + + /// + /// Entity representing a CIL attribute. + /// + class Attribute : UnlabelledEntity, IAttribute + { + readonly CustomAttribute attrib; + readonly IEntity @object; + + public Attribute(Context cx, IEntity @object, CustomAttributeHandle handle) : base(cx) + { + attrib = cx.mdReader.GetCustomAttribute(handle); + this.@object = @object; + } + + public override IEnumerable Contents + { + get + { + var constructor = (Method)cx.Create(attrib.Constructor); + yield return constructor; + + yield return Tuples.cil_attribute(this, @object, constructor); + + CustomAttributeValue decoded; + + try + { + decoded = attrib.DecodeValue(new CustomAttributeDecoder(cx)); + } + catch (NotImplementedException) + { + // Attribute decoding is only partial at this stage. + yield break; + } + + for (int index = 0; index < decoded.FixedArguments.Length; ++index) + { + object value = decoded.FixedArguments[index].Value; + yield return Tuples.cil_attribute_positional_argument(this, index, value == null ? "null" : value.ToString()); + } + + foreach (var p in decoded.NamedArguments) + { + object value = p.Value; + yield return Tuples.cil_attribute_named_argument(this, p.Name, value == null ? "null" : value.ToString()); + } + } + } + + public static IEnumerable Populate(Context cx, IEntity @object, CustomAttributeHandleCollection attributes) + { + foreach (var attrib in attributes) + { + yield return new Attribute(cx, @object, attrib); + } + } + } + + /// + /// Helper class to decode the attribute structure. + /// Note that there are some unhandled cases that should be fixed in due course. + /// + class CustomAttributeDecoder : ICustomAttributeTypeProvider + { + readonly Context cx; + public CustomAttributeDecoder(Context cx) { this.cx = cx; } + + public Type GetPrimitiveType(PrimitiveTypeCode typeCode) => cx.Populate(new PrimitiveType(cx, typeCode)); + + public Type GetSystemType() => throw new NotImplementedException(); + + public Type GetSZArrayType(Type elementType) => + cx.Populate(new ArrayType(cx, elementType)); + + public Type GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => + (Type)cx.Create(handle); + + public Type GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => + (Type)cx.Create(handle); + + public Type GetTypeFromSerializedName(string name) => throw new NotImplementedException(); + + public PrimitiveTypeCode GetUnderlyingEnumType(Type type) => throw new NotImplementedException(); + + public bool IsSystemType(Type type) => type is PrimitiveType; // ?? + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs new file mode 100644 index 000000000000..009afc1b1e9f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs @@ -0,0 +1,67 @@ +using System.Collections.Generic; +using System.Reflection.Metadata; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// An event. + /// + interface IEvent : ILabelledEntity + { + } + + /// + /// An event entity. + /// + class Event : LabelledEntity, IEvent + { + readonly Type parent; + readonly EventDefinition ed; + static readonly Id suffix = CIL.Id.Create(";cil-event"); + + public Event(Context cx, Type parent, EventDefinitionHandle handle) : base(cx) + { + this.parent = parent; + ed = cx.mdReader.GetEventDefinition(handle); + ShortId = parent.ShortId + cx.Dot + cx.ShortName(ed.Name) + suffix; + } + + public override IEnumerable Contents + { + get + { + var signature = (Type)cx.CreateGeneric(parent, ed.Type); + yield return signature; + + yield return Tuples.cil_event(this, parent, cx.ShortName(ed.Name), signature); + + var accessors = ed.GetAccessors(); + if (!accessors.Adder.IsNil) + { + var adder = (Method)cx.CreateGeneric(parent, accessors.Adder); + yield return adder; + yield return Tuples.cil_adder(this, adder); + } + + if (!accessors.Remover.IsNil) + { + var remover = (Method)cx.CreateGeneric(parent, accessors.Remover); + yield return remover; + yield return Tuples.cil_remover(this, remover); + } + + if (!accessors.Raiser.IsNil) + { + var raiser = (Method)cx.CreateGeneric(parent, accessors.Raiser); + yield return raiser; + yield return Tuples.cil_raiser(this, raiser); + } + + foreach (var c in Attribute.Populate(cx, this, ed.GetCustomAttributes())) + yield return c; + } + } + + public override Id IdSuffix => suffix; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/ExceptionRegion.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/ExceptionRegion.cs new file mode 100644 index 000000000000..57035d993af0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/ExceptionRegion.cs @@ -0,0 +1,63 @@ +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL.Entities +{ + interface IExceptionRegion : IExtractedEntity + { + } + + /// + /// An exception region entity. + /// + class ExceptionRegion : UnlabelledEntity, IExceptionRegion + { + readonly GenericContext gc; + readonly MethodImplementation method; + readonly int index; + readonly System.Reflection.Metadata.ExceptionRegion r; + readonly Dictionary jump_table; + + public ExceptionRegion(GenericContext gc, MethodImplementation method, int index, System.Reflection.Metadata.ExceptionRegion r, Dictionary jump_table) : base(gc.cx) + { + this.gc = gc; + this.method = method; + this.index = index; + this.r = r; + this.jump_table = jump_table; + } + + public override IEnumerable Contents + { + get + { + IInstruction try_start, try_end, handler_start; + + if (!jump_table.TryGetValue(r.TryOffset, out try_start)) + throw new InternalError("Failed to retrieve handler"); + if (!jump_table.TryGetValue(r.TryOffset + r.TryLength, out try_end)) + throw new InternalError("Failed to retrieve handler"); + if (!jump_table.TryGetValue(r.HandlerOffset, out handler_start)) + throw new InternalError("Failed to retrieve handler"); + + + yield return Tuples.cil_handler(this, method, index, (int)r.Kind, try_start, try_end, handler_start); + + if (r.FilterOffset != -1) + { + IInstruction filter_start; + if (!jump_table.TryGetValue(r.FilterOffset, out filter_start)) + throw new InternalError("ExceptionRegion filter clause"); + + yield return Tuples.cil_handler_filter(this, filter_start); + } + + if (!r.CatchType.IsNil) + { + var catchType = (Type)cx.CreateGeneric(gc, r.CatchType); + yield return catchType; + yield return Tuples.cil_handler_type(this, catchType); + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs new file mode 100644 index 000000000000..64367ca9ae0c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs @@ -0,0 +1,147 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.CodeAnalysis; +using System.Reflection.Metadata; +using System.Reflection; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// An entity represting a member. + /// Used to type tuples correctly. + /// + interface IMember : ILabelledEntity + { + } + + /// + /// An entity representing a field. + /// + interface IField : IMember + { + } + + /// + /// An entity representing a field. + /// + abstract class Field : GenericContext, IField + { + protected Field(Context cx) : base(cx) + { + } + + public bool NeedsPopulation { get { return true; } } + + public Label Label { get; set; } + + public IId Id => ShortId + IdSuffix; + + public Id IdSuffix => fieldSuffix; + + static readonly StringId fieldSuffix = new StringId(";cil-field"); + + public Id ShortId + { + get; set; + } + + public abstract Id Name { get; } + + public abstract Type DeclaringType { get; } + + public Location ReportingLocation => throw new NotImplementedException(); + + abstract public Type Type { get; } + + public virtual IEnumerable Contents + { + get + { + yield return Tuples.cil_field(this, DeclaringType, Name.Value, Type); + } + } + + public void Extract(Context cx) + { + cx.Populate(this); + } + + TrapStackBehaviour IEntity.TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } + + sealed class DefinitionField : Field + { + readonly FieldDefinition fd; + readonly GenericContext gc; + + public DefinitionField(GenericContext gc, FieldDefinitionHandle handle) : base(gc.cx) + { + this.gc = gc; + fd = cx.mdReader.GetFieldDefinition(handle); + ShortId = DeclaringType.ShortId + cx.Dot + Name; + } + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) + yield return c; + + if (fd.Attributes.HasFlag(FieldAttributes.Private)) + yield return Tuples.cil_private(this); + + if (fd.Attributes.HasFlag(FieldAttributes.Public)) + yield return Tuples.cil_public(this); + + if (fd.Attributes.HasFlag(FieldAttributes.Family)) + yield return Tuples.cil_protected(this); + + if (fd.Attributes.HasFlag(FieldAttributes.Static)) + yield return Tuples.cil_static(this); + + if (fd.Attributes.HasFlag(FieldAttributes.Assembly)) + yield return Tuples.cil_internal(this); + + foreach (var c in Attribute.Populate(cx, this, fd.GetCustomAttributes())) + yield return c; + } + } + + public override Id Name => cx.GetId(fd.Name); + + public override Type DeclaringType => (Type)cx.Create(fd.GetDeclaringType()); + + public override Type Type => fd.DecodeSignature(cx.TypeSignatureDecoder, DeclaringType); + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + } + + sealed class MemberReferenceField : Field + { + readonly MemberReference mr; + readonly GenericContext gc; + readonly Type declType; + + public MemberReferenceField(GenericContext gc, MemberReferenceHandle handle) : base(gc.cx) + { + this.gc = gc; + mr = cx.mdReader.GetMemberReference(handle); + declType = (Type)cx.CreateGeneric(gc, mr.Parent); + ShortId = declType.ShortId + cx.Dot + Name; + } + + public override Id Name => cx.GetId(mr.Name); + + public override Type DeclaringType => declType; + + public override Type Type => mr.DecodeFieldSignature(cx.TypeSignatureDecoder, this); + + public override IEnumerable TypeParameters => gc.TypeParameters.Concat(declType.TypeParameters); + + public override IEnumerable MethodParameters => gc.MethodParameters.Concat(declType.MethodParameters); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs new file mode 100644 index 000000000000..de8f890312a1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs @@ -0,0 +1,66 @@ +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL.Entities +{ + interface IFileOrFolder : IEntity + { + } + + interface IFile : IFileOrFolder + { + } + + public class File : LabelledEntity, IFile + { + protected readonly string path; + + public File(Context cx, string path) : base(cx) + { + this.path = path.Replace("\\", "/"); + ShortId = new StringId(path.Replace(":", "_")); + } + + public override IEnumerable Contents + { + get + { + var parent = cx.CreateFolder(System.IO.Path.GetDirectoryName(path)); + yield return parent; + yield return Tuples.containerparent(parent, this); + yield return Tuples.files(this, path, System.IO.Path.GetFileNameWithoutExtension(path), System.IO.Path.GetExtension(path).Substring(1)); + } + } + + public override Id IdSuffix => suffix; + + static readonly Id suffix = new StringId(";sourcefile"); + } + + public class PdbSourceFile : File + { + readonly PDB.ISourceFile file; + + public PdbSourceFile(Context cx, PDB.ISourceFile file) : base(cx, file.Path) + { + this.file = file; + } + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) + yield return c; + + var text = file.Contents; + + if (text == null) + cx.cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, string.Format("PDB source file {0} could not be found", path)); + else + cx.cx.TrapWriter.Archive(path, text); + + yield return Tuples.file_extraction_mode(this, 2); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs new file mode 100644 index 000000000000..0ed1b51dd995 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs @@ -0,0 +1,42 @@ +using System.Collections.Generic; +using System.IO; + +namespace Semmle.Extraction.CIL.Entities +{ + interface IFolder : IFileOrFolder + { + } + + public class Folder : LabelledEntity, IFolder + { + readonly string path; + + public Folder(Context cx, string path) : base(cx) + { + this.path = path; + ShortId = new StringId(path.Replace("\\", "/").Replace(":", "_")); + } + + static readonly Id suffix = new StringId(";folder"); + + public override IEnumerable Contents + { + get + { + // On Posix, we could get a Windows directory of the form "C:" + bool windowsDriveLetter = path.Length == 2 && char.IsLetter(path[0]) && path[1] == ':'; + + var parent = Path.GetDirectoryName(path); + if (parent != null && !windowsDriveLetter) + { + var parentFolder = cx.CreateFolder(parent); + yield return parentFolder; + yield return Tuples.containerparent(parentFolder, this); + } + yield return Tuples.folders(this, path, Path.GetFileName(path)); + } + } + + public override Id IdSuffix => suffix; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs new file mode 100644 index 000000000000..de6640235f32 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs @@ -0,0 +1,486 @@ +using System; +using System.Collections.Generic; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A CIL instruction. + /// + interface IInstruction : IExtractedEntity + { + /// + /// Gets the extraction products for branches. + /// + /// The map from offset to instruction. + /// The extraction products. + IEnumerable JumpContents(Dictionary jump_table); + } + + /// + /// A CIL instruction. + /// + class Instruction : UnlabelledEntity, IInstruction + { + /// + /// The additional data following the opcode, if any. + /// + public enum Payload + { + None, TypeTok, Field, Target8, Class, + Method, Arg8, Local8, Target32, Int8, + Int16, Int32, Int64, Float32, Float64, + CallSiteDesc, Switch, String, Constructor, ValueType, + Type, Arg16, Ignore8, Token, Local16, MethodRef + } + + /// + /// For each Payload, how many additional bytes in the bytestream need to be read. + /// + internal static readonly int[] payloadSizes = { + 0, 4, 4, 1, 4, + 4, 1, 1, 4, 1, + 2, 4, 8, 4, 8, + 4, -1, 4, 4, 4, + 4, 2, 1, 4, 2, 4 }; + + // Maps opcodes to payloads for each instruction. + public static readonly Dictionary opPayload = new Dictionary() + { + { ILOpCode.Nop, Payload.None }, + { ILOpCode.Break, Payload.None }, + { ILOpCode.Ldarg_0, Payload.None }, + { ILOpCode.Ldarg_1, Payload.None }, + { ILOpCode.Ldarg_2, Payload.None }, + { ILOpCode.Ldarg_3, Payload.None }, + { ILOpCode.Ldloc_0, Payload.None }, + { ILOpCode.Ldloc_1, Payload.None }, + { ILOpCode.Ldloc_2, Payload.None }, + { ILOpCode.Ldloc_3, Payload.None }, + { ILOpCode.Stloc_0, Payload.None }, + { ILOpCode.Stloc_1, Payload.None }, + { ILOpCode.Stloc_2, Payload.None }, + { ILOpCode.Stloc_3, Payload.None }, + { ILOpCode.Ldarg_s, Payload.Arg8 }, + { ILOpCode.Ldarga_s, Payload.Arg8 }, + { ILOpCode.Starg_s, Payload.Arg8 }, + { ILOpCode.Ldloc_s, Payload.Local8 }, + { ILOpCode.Ldloca_s, Payload.Local8 }, + { ILOpCode.Stloc_s, Payload.Local8 }, + { ILOpCode.Ldnull, Payload.None }, + { ILOpCode.Ldc_i4_m1, Payload.None }, + { ILOpCode.Ldc_i4_0, Payload.None }, + { ILOpCode.Ldc_i4_1, Payload.None }, + { ILOpCode.Ldc_i4_2, Payload.None }, + { ILOpCode.Ldc_i4_3, Payload.None }, + { ILOpCode.Ldc_i4_4, Payload.None }, + { ILOpCode.Ldc_i4_5, Payload.None }, + { ILOpCode.Ldc_i4_6, Payload.None }, + { ILOpCode.Ldc_i4_7, Payload.None }, + { ILOpCode.Ldc_i4_8, Payload.None }, + { ILOpCode.Ldc_i4_s, Payload.Int8 }, + { ILOpCode.Ldc_i4, Payload.Int32 }, + { ILOpCode.Ldc_i8, Payload.Int64 }, + { ILOpCode.Ldc_r4, Payload.Float32 }, + { ILOpCode.Ldc_r8, Payload.Float64 }, + { ILOpCode.Dup, Payload.None }, + { ILOpCode.Pop, Payload.None }, + { ILOpCode.Jmp, Payload.Method }, + { ILOpCode.Call, Payload.Method }, + { ILOpCode.Calli, Payload.CallSiteDesc }, + { ILOpCode.Ret, Payload.None }, + { ILOpCode.Br_s, Payload.Target8 }, + { ILOpCode.Brfalse_s, Payload.Target8 }, + { ILOpCode.Brtrue_s, Payload.Target8 }, + { ILOpCode.Beq_s, Payload.Target8 }, + { ILOpCode.Bge_s, Payload.Target8 }, + { ILOpCode.Bgt_s, Payload.Target8 }, + { ILOpCode.Ble_s, Payload.Target8 }, + { ILOpCode.Blt_s, Payload.Target8 }, + { ILOpCode.Bne_un_s, Payload.Target8 }, + { ILOpCode.Bge_un_s, Payload.Target8 }, + { ILOpCode.Bgt_un_s, Payload.Target8 }, + { ILOpCode.Ble_un_s, Payload.Target8 }, + { ILOpCode.Blt_un_s, Payload.Target8 }, + { ILOpCode.Br, Payload.Target32 }, + { ILOpCode.Brfalse, Payload.Target32 }, + { ILOpCode.Brtrue, Payload.Target32 }, + { ILOpCode.Beq, Payload.Target32 }, + { ILOpCode.Bge, Payload.Target32 }, + { ILOpCode.Bgt, Payload.Target32 }, + { ILOpCode.Ble, Payload.Target32 }, + { ILOpCode.Blt, Payload.Target32 }, + { ILOpCode.Bne_un, Payload.Target32 }, + { ILOpCode.Bge_un, Payload.Target32 }, + { ILOpCode.Bgt_un, Payload.Target32 }, + { ILOpCode.Ble_un, Payload.Target32 }, + { ILOpCode.Blt_un, Payload.Target32 }, + { ILOpCode.Switch, Payload.Switch }, + { ILOpCode.Ldind_i1, Payload.None }, + { ILOpCode.Ldind_u1, Payload.None }, + { ILOpCode.Ldind_i2, Payload.None }, + { ILOpCode.Ldind_u2, Payload.None }, + { ILOpCode.Ldind_i4, Payload.None }, + { ILOpCode.Ldind_u4, Payload.None }, + { ILOpCode.Ldind_i8, Payload.None }, + { ILOpCode.Ldind_i, Payload.None }, + { ILOpCode.Ldind_r4, Payload.None }, + { ILOpCode.Ldind_r8, Payload.None }, + { ILOpCode.Ldind_ref, Payload.None }, + { ILOpCode.Stind_ref, Payload.None }, + { ILOpCode.Stind_i1, Payload.None }, + { ILOpCode.Stind_i2, Payload.None }, + { ILOpCode.Stind_i4, Payload.None }, + { ILOpCode.Stind_i8, Payload.None }, + { ILOpCode.Stind_r4, Payload.None }, + { ILOpCode.Stind_r8, Payload.None }, + { ILOpCode.Add, Payload.None }, + { ILOpCode.Sub, Payload.None }, + { ILOpCode.Mul, Payload.None }, + { ILOpCode.Div, Payload.None }, + { ILOpCode.Div_un, Payload.None }, + { ILOpCode.Rem, Payload.None }, + { ILOpCode.Rem_un, Payload.None }, + { ILOpCode.And, Payload.None }, + { ILOpCode.Or, Payload.None }, + { ILOpCode.Xor, Payload.None }, + { ILOpCode.Shl, Payload.None }, + { ILOpCode.Shr, Payload.None }, + { ILOpCode.Shr_un, Payload.None }, + { ILOpCode.Neg, Payload.None }, + { ILOpCode.Not, Payload.None }, + { ILOpCode.Conv_i1, Payload.None }, + { ILOpCode.Conv_i2, Payload.None }, + { ILOpCode.Conv_i4, Payload.None }, + { ILOpCode.Conv_i8, Payload.None }, + { ILOpCode.Conv_r4, Payload.None }, + { ILOpCode.Conv_r8, Payload.None }, + { ILOpCode.Conv_u4, Payload.None }, + { ILOpCode.Conv_u8, Payload.None }, + { ILOpCode.Callvirt, Payload.MethodRef }, + { ILOpCode.Cpobj, Payload.TypeTok }, + { ILOpCode.Ldobj, Payload.TypeTok }, + { ILOpCode.Ldstr, Payload.String }, + { ILOpCode.Newobj, Payload.Constructor }, + { ILOpCode.Castclass, Payload.Class }, + { ILOpCode.Isinst, Payload.Class }, + { ILOpCode.Conv_r_un, Payload.None }, + { ILOpCode.Unbox, Payload.ValueType }, + { ILOpCode.Throw, Payload.None }, + { ILOpCode.Ldfld, Payload.Field }, + { ILOpCode.Ldflda, Payload.Field }, + { ILOpCode.Stfld, Payload.Field }, + { ILOpCode.Ldsfld, Payload.Field }, + { ILOpCode.Ldsflda, Payload.Field }, + { ILOpCode.Stsfld, Payload.Field }, + { ILOpCode.Stobj, Payload.Field }, + { ILOpCode.Conv_ovf_i1_un, Payload.None }, + { ILOpCode.Conv_ovf_i2_un, Payload.None }, + { ILOpCode.Conv_ovf_i4_un, Payload.None }, + { ILOpCode.Conv_ovf_i8_un, Payload.None }, + { ILOpCode.Conv_ovf_u1_un, Payload.None }, + { ILOpCode.Conv_ovf_u2_un, Payload.None }, + { ILOpCode.Conv_ovf_u4_un, Payload.None }, + { ILOpCode.Conv_ovf_u8_un, Payload.None }, + { ILOpCode.Conv_ovf_i_un, Payload.None }, + { ILOpCode.Conv_ovf_u_un, Payload.None }, + { ILOpCode.Box, Payload.TypeTok }, + { ILOpCode.Newarr, Payload.TypeTok }, + { ILOpCode.Ldlen, Payload.None }, + { ILOpCode.Ldelema, Payload.Class }, + { ILOpCode.Ldelem_i1, Payload.None }, + { ILOpCode.Ldelem_u1, Payload.None }, + { ILOpCode.Ldelem_i2, Payload.None }, + { ILOpCode.Ldelem_u2, Payload.None }, + { ILOpCode.Ldelem_i4, Payload.None }, + { ILOpCode.Ldelem_u4, Payload.None }, + { ILOpCode.Ldelem_i8, Payload.None }, + { ILOpCode.Ldelem_i, Payload.None }, + { ILOpCode.Ldelem_r4, Payload.None }, + { ILOpCode.Ldelem_r8, Payload.None }, + { ILOpCode.Ldelem_ref, Payload.None }, + { ILOpCode.Stelem_i, Payload.None }, + { ILOpCode.Stelem_i1, Payload.None }, + { ILOpCode.Stelem_i2, Payload.None }, + { ILOpCode.Stelem_i4, Payload.None }, + { ILOpCode.Stelem_i8, Payload.None }, + { ILOpCode.Stelem_r4, Payload.None }, + { ILOpCode.Stelem_r8, Payload.None }, + { ILOpCode.Stelem_ref, Payload.None }, + { ILOpCode.Ldelem, Payload.TypeTok }, + { ILOpCode.Stelem, Payload.TypeTok }, + { ILOpCode.Unbox_any, Payload.TypeTok }, + { ILOpCode.Conv_ovf_i1, Payload.None }, + { ILOpCode.Conv_ovf_u1, Payload.None }, + { ILOpCode.Conv_ovf_i2, Payload.None }, + { ILOpCode.Conv_ovf_u2, Payload.None }, + { ILOpCode.Conv_ovf_i4, Payload.None }, + { ILOpCode.Conv_ovf_u4, Payload.None }, + { ILOpCode.Conv_ovf_i8, Payload.None }, + { ILOpCode.Conv_ovf_u8, Payload.None }, + { ILOpCode.Refanyval, Payload.Type }, + { ILOpCode.Ckfinite, Payload.None }, + { ILOpCode.Mkrefany, Payload.Class }, + { ILOpCode.Ldtoken, Payload.Token }, + { ILOpCode.Conv_u2, Payload.None }, + { ILOpCode.Conv_u1, Payload.None }, + { ILOpCode.Conv_i, Payload.None }, + { ILOpCode.Conv_ovf_i, Payload.None }, + { ILOpCode.Conv_ovf_u, Payload.None }, + { ILOpCode.Add_ovf, Payload.None }, + { ILOpCode.Add_ovf_un, Payload.None }, + { ILOpCode.Mul_ovf, Payload.None }, + { ILOpCode.Mul_ovf_un, Payload.None }, + { ILOpCode.Sub_ovf, Payload.None }, + { ILOpCode.Sub_ovf_un, Payload.None }, + { ILOpCode.Endfinally, Payload.None }, + { ILOpCode.Leave, Payload.Target32 }, + { ILOpCode.Leave_s, Payload.Target8 }, + { ILOpCode.Stind_i, Payload.None }, + { ILOpCode.Conv_u, Payload.None }, + { ILOpCode.Arglist, Payload.None }, + { ILOpCode.Ceq, Payload.None }, + { ILOpCode.Cgt, Payload.None }, + { ILOpCode.Cgt_un, Payload.None }, + { ILOpCode.Clt, Payload.None }, + { ILOpCode.Clt_un, Payload.None }, + { ILOpCode.Ldftn, Payload.Method }, + { ILOpCode.Ldvirtftn, Payload.Method }, + { ILOpCode.Ldarg, Payload.Arg16 }, + { ILOpCode.Ldarga, Payload.Arg16 }, + { ILOpCode.Starg, Payload.Arg16 }, + { ILOpCode.Ldloc, Payload.Local16 }, + { ILOpCode.Ldloca, Payload.Local16 }, + { ILOpCode.Stloc, Payload.Local16 }, + { ILOpCode.Localloc, Payload.None }, + { ILOpCode.Endfilter, Payload.None }, + { ILOpCode.Unaligned, Payload.Ignore8 }, + { ILOpCode.Volatile, Payload.None }, + { ILOpCode.Tail, Payload.None }, + { ILOpCode.Initobj, Payload.TypeTok }, + { ILOpCode.Constrained, Payload.Type }, + { ILOpCode.Cpblk, Payload.None }, + { ILOpCode.Initblk, Payload.None }, + { ILOpCode.Rethrow, Payload.None }, + { ILOpCode.Sizeof, Payload.TypeTok }, + { ILOpCode.Refanytype, Payload.None }, + { ILOpCode.Readonly, Payload.None } + }; + + public readonly DefinitionMethod Method; + public readonly ILOpCode OpCode; + public readonly int Offset; + public readonly int Index; + readonly int PayloadValue; + readonly uint UnsignedPayloadValue; + + public Payload PayloadType + { + get + { + Payload result; + if (!opPayload.TryGetValue(OpCode, out result)) + throw new InternalError("Unknown op code " + OpCode); + return result; + } + } + + public override string ToString() => Index + ": " + OpCode; + + /// + /// The number of bytes of this instruction, + /// including the payload (if any). + /// + public int Width + { + get + { + if (OpCode == ILOpCode.Switch) return 5 + 4 * PayloadValue; + + return ((int)OpCode > 255 ? 2 : 1) + PayloadSize; + } + } + + Label IEntity.Label + { + get; set; + } + + + readonly byte[] data; + + int PayloadSize => payloadSizes[(int)PayloadType]; + + /// + /// Reads the instruction from a byte stream. + /// + /// The byte stream. + /// The offset of the instruction. + /// The index of this instruction in the callable. + public Instruction(Context cx, DefinitionMethod method, byte[] data, int offset, int index) : base(cx) + { + Method = method; + Offset = offset; + Index = index; + this.data = data; + int opcode = data[offset]; + ++offset; + + /* + * An opcode is either 1 or 2 bytes, followed by an optional payload depending on the instruction. + * Instructions where the first byte is 0xfe are 2-byte instructions. + */ + if (opcode == 0xfe) + opcode = opcode << 8 | data[offset++]; + OpCode = (ILOpCode)opcode; + + switch (PayloadSize) + { + case 0: + PayloadValue = 0; + break; + case 1: + PayloadValue = (sbyte)data[offset]; + UnsignedPayloadValue = data[offset]; + break; + case 2: + PayloadValue = BitConverter.ToInt16(data, offset); + UnsignedPayloadValue = BitConverter.ToUInt16(data, offset); + break; + case -1: // Switch + case 4: + PayloadValue = BitConverter.ToInt32(data, offset); + break; + case 8: // Not handled here. + break; + default: + throw new InternalError("Unhandled CIL instruction Payload"); + } + } + + public override IEnumerable Contents + { + get + { + int offset = Offset; + + yield return Tuples.cil_instruction(this, (int)OpCode, Index, Method.Implementation); + + switch (PayloadType) + { + case Payload.String: + yield return Tuples.cil_value(this, cx.mdReader.GetUserString(MetadataTokens.UserStringHandle(PayloadValue))); + break; + case Payload.Float32: + yield return Tuples.cil_value(this, BitConverter.ToSingle(data, offset).ToString()); + break; + case Payload.Float64: + yield return Tuples.cil_value(this, BitConverter.ToDouble(data, offset).ToString()); + break; + case Payload.Int8: + yield return Tuples.cil_value(this, data[offset].ToString()); + break; + case Payload.Int16: + yield return Tuples.cil_value(this, BitConverter.ToInt16(data, offset).ToString()); + break; + case Payload.Int32: + yield return Tuples.cil_value(this, BitConverter.ToInt32(data, offset).ToString()); + break; + case Payload.Int64: + yield return Tuples.cil_value(this, BitConverter.ToInt64(data, offset).ToString()); + break; + case Payload.Constructor: + case Payload.Method: + case Payload.MethodRef: + case Payload.Class: + case Payload.TypeTok: + case Payload.Token: + case Payload.Type: + case Payload.Field: + case Payload.ValueType: + // A generic EntityHandle. + var handle = MetadataTokens.EntityHandle(PayloadValue); + var target = cx.CreateGeneric(Method, handle); + yield return target; + if (target != null) + { + yield return Tuples.cil_access(this, target); + } + else + { + throw new InternalError("Unable to create payload type {0} for opcode {1}", PayloadType, OpCode); + } + break; + case Payload.Arg8: + case Payload.Arg16: + yield return Tuples.cil_access(this, Method.Parameters[(int)UnsignedPayloadValue]); + break; + case Payload.Local8: + case Payload.Local16: + yield return Tuples.cil_access(this, Method.LocalVariables[(int)UnsignedPayloadValue]); + break; + case Payload.None: + case Payload.Target8: + case Payload.Target32: + case Payload.Switch: + case Payload.Ignore8: + case Payload.CallSiteDesc: + // These are not handled here. + // Some of these are handled by JumpContents(). + break; + default: + throw new InternalError("Unhandled payload type {0}", PayloadType); + } + } + } + + // Called to populate the jumps in each instruction. + public IEnumerable JumpContents(Dictionary jump_table) + { + int target; + IInstruction inst; + + switch (PayloadType) + { + case Payload.Target8: + target = Offset + PayloadValue + 2; + break; + case Payload.Target32: + target = Offset + PayloadValue + 5; + break; + case Payload.Switch: + int end = Offset + Width; + + int offset = Offset + 5; + + for (int b = 0; b < PayloadValue; ++b, offset += 4) + { + target = BitConverter.ToInt32(data, offset) + end; + if (!jump_table.TryGetValue(target, out inst)) + throw new InternalError("Invalid jump target"); + yield return Tuples.cil_switch(this, b, inst); + } + + yield break; + default: + // Not a jump + yield break; + } + + + if (jump_table.TryGetValue(target, out inst)) + { + yield return Tuples.cil_jump(this, inst); + } + else + { + // Sometimes instructions can jump outside the current method. + // TODO: Find a solution to this. + + // For now, just log the error + cx.cx.Extractor.Message(new Message { message = "A CIL instruction jumps outside the current method", severity = Util.Logging.Severity.Warning }); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs new file mode 100644 index 000000000000..4e3fbf43662d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL.Entities +{ + interface ILocal : ILabelledEntity + { + } + + class LocalVariable : LabelledEntity, ILocal + { + readonly MethodImplementation method; + readonly int index; + readonly Type type; + + public LocalVariable(Context cx, MethodImplementation m, int i, Type t) : base(cx) + { + method = m; + index = i; + type = t; + ShortId = CIL.Id.Create(method.Label) + underscore + index; + } + + static readonly Id underscore = CIL.Id.Create("_"); + static readonly Id suffix = CIL.Id.Create(";cil-local"); + public override Id IdSuffix => suffix; + + public override IEnumerable Contents + { + get + { + yield return type; + yield return Tuples.cil_local_variable(this, method, index, type); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs new file mode 100644 index 000000000000..306e2205c0bd --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs @@ -0,0 +1,515 @@ +using System; +using System.Collections.Immutable; +using System.Reflection.Metadata; +using Microsoft.CodeAnalysis; +using System.Collections.Generic; +using System.Reflection; +using System.Linq; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A method entity. + /// + interface IMethod : IMember + { + } + + /// + /// A method entity. + /// + abstract class Method : TypeContainer, IMethod + { + protected Method(GenericContext gc) : base(gc.cx) + { + this.gc = gc; + } + + public override IEnumerable TypeParameters => gc.TypeParameters.Concat(declaringType.TypeParameters); + + public override IEnumerable MethodParameters => genericParams == null ? Enumerable.Empty() : genericParams; + + public int GenericParameterCount => signature.GenericParameterCount; + + public virtual Method SourceDeclaration => this; + + public abstract Type DeclaringType { get; } + public abstract string Name { get; } + + public virtual IList LocalVariables => throw new NotImplementedException(); + public IList Parameters { get; private set; } + + static readonly Id tick = CIL.Id.Create("`"); + static readonly Id space = CIL.Id.Create(" "); + static readonly Id dot = CIL.Id.Create("."); + static readonly Id open = CIL.Id.Create("("); + static readonly Id close = CIL.Id.Create(")"); + + internal protected Id MakeMethodId(Type parent, Id methodName) + { + var id = signature.ReturnType.MakeId(gc) + space + parent.ShortId + dot + methodName; + + if (signature.GenericParameterCount > 0) + { + id += tick + signature.GenericParameterCount; + } + + id += open + CIL.Id.CommaSeparatedList(signature.ParameterTypes.Select(p => p.MakeId(gc))) + close; + return id; + } + + protected MethodTypeParameter[] genericParams; + protected Type declaringType; + protected GenericContext gc; + protected MethodSignature signature; + protected Id name; + + static readonly StringId methodSuffix = new StringId(";cil-method"); + + public override Id IdSuffix => methodSuffix; + + protected void PopulateParameters(IEnumerable parameterTypes) + { + Parameters = MakeParameters(parameterTypes).ToArray(); + } + + protected IEnumerable PopulateFlags + { + get + { + if (IsStatic) + yield return Tuples.cil_static(this); + } + } + + public abstract bool IsStatic { get; } + + private IEnumerable MakeParameters(IEnumerable parameterTypes) + { + int i = 0; + + if (!IsStatic) + { + yield return cx.Populate(new Parameter(cx, this, i++, DeclaringType)); + } + + foreach (var p in parameterTypes) + yield return cx.Populate(new Parameter(cx, this, i++, p)); + } + } + + /// + /// A method implementation entity. + /// + interface IMethodImplementation : IExtractedEntity + { + } + + /// + /// A method implementation entity. + /// In the database, the same method could in principle have multiple implementations. + /// + class MethodImplementation : UnlabelledEntity, IMethodImplementation + { + readonly Method m; + + public MethodImplementation(Method m) : base(m.cx) + { + this.m = m; + } + + public override IEnumerable Contents + { + get + { + yield return Tuples.cil_method_implementation(this, m, cx.assembly); + } + } + } + + + /// + /// A definition method - a method defined in the current assembly. + /// + class DefinitionMethod : Method, IMember + { + readonly MethodDefinition md; + readonly PDB.IMethod methodDebugInformation; + + LocalVariable[] locals; + + public MethodImplementation Implementation { get; private set; } + + public override IList LocalVariables => locals; + + public DefinitionMethod(GenericContext gc, MethodDefinitionHandle handle) : base(gc) + { + md = cx.mdReader.GetMethodDefinition(handle); + this.gc = gc; + name = cx.GetId(md.Name); + + declaringType = (Type)cx.CreateGeneric(this, md.GetDeclaringType()); + + signature = md.DecodeSignature(new SignatureDecoder(), this); + ShortId = MakeMethodId(declaringType, name); + + methodDebugInformation = cx.GetMethodDebugInformation(handle); + } + + public override bool IsStatic => !signature.Header.IsInstance; + + public override Type DeclaringType => declaringType; + + public override string Name => cx.ShortName(md.Name); + + /// + /// Holds if this method has bytecode. + /// + public bool HasBytecode => md.ImplAttributes == MethodImplAttributes.IL && md.RelativeVirtualAddress != 0; + + public override IEnumerable Contents + { + get + { + if (md.GetGenericParameters().Any()) + { + // We need to perform a 2-phase population because some type parameters can + // depend on other type parameters (as a constraint). + genericParams = new MethodTypeParameter[md.GetGenericParameters().Count]; + for (int i = 0; i < genericParams.Length; ++i) + genericParams[i] = cx.Populate(new MethodTypeParameter(this, this, i)); + for (int i = 0; i < genericParams.Length; ++i) + genericParams[i].PopulateHandle(this, md.GetGenericParameters()[i]); + foreach (var p in genericParams) + yield return p; + } + + var typeSignature = md.DecodeSignature(cx.TypeSignatureDecoder, this); + + PopulateParameters(typeSignature.ParameterTypes); + + foreach (var c in Parameters) + yield return c; + + foreach (var c in PopulateFlags) + yield return c; + + foreach (var p in md.GetParameters().Select(h => cx.mdReader.GetParameter(h)).Where(p => p.SequenceNumber > 0)) + { + var pe = Parameters[IsStatic ? p.SequenceNumber - 1 : p.SequenceNumber]; + if (p.Attributes.HasFlag(ParameterAttributes.Out)) + yield return Tuples.cil_parameter_out(pe); + if (p.Attributes.HasFlag(ParameterAttributes.In)) + yield return Tuples.cil_parameter_in(pe); + Attribute.Populate(cx, pe, p.GetCustomAttributes()); + } + + yield return Tuples.cil_method(this, Name, declaringType, typeSignature.ReturnType); + yield return Tuples.cil_method_source_declaration(this, this); + yield return Tuples.cil_method_location(this, cx.assembly); + + if (HasBytecode) + { + Implementation = new MethodImplementation(this); + yield return Implementation; + + var body = cx.peReader.GetMethodBody(md.RelativeVirtualAddress); + + if (!body.LocalSignature.IsNil) + { + var locals = cx.mdReader.GetStandaloneSignature(body.LocalSignature); + var localVariableTypes = locals.DecodeLocalSignature(cx.TypeSignatureDecoder, this); + + this.locals = new LocalVariable[localVariableTypes.Length]; + + for (int l = 0; l < this.locals.Length; ++l) + { + this.locals[l] = cx.Populate(new LocalVariable(cx, Implementation, l, localVariableTypes[l])); + yield return this.locals[l]; + } + } + + var jump_table = new Dictionary(); + + foreach (var c in Decode(body.GetILBytes(), jump_table)) + yield return c; + + int filter_index = 0; + foreach (var region in body.ExceptionRegions) + { + yield return new ExceptionRegion(this, Implementation, filter_index++, region, jump_table); + } + + yield return Tuples.cil_method_stack_size(Implementation, body.MaxStack); + + if (methodDebugInformation != null) + { + var sourceLocation = cx.CreateSourceLocation(methodDebugInformation.Location); + yield return sourceLocation; + yield return Tuples.cil_method_location(this, sourceLocation); + } + } + + // Flags + + if (md.Attributes.HasFlag(MethodAttributes.Private)) + yield return Tuples.cil_private(this); + + if (md.Attributes.HasFlag(MethodAttributes.Public)) + yield return Tuples.cil_public(this); + + if (md.Attributes.HasFlag(MethodAttributes.Family)) + yield return Tuples.cil_protected(this); + + if (md.Attributes.HasFlag(MethodAttributes.Final)) + yield return Tuples.cil_sealed(this); + + if (md.Attributes.HasFlag(MethodAttributes.Virtual)) + yield return Tuples.cil_virtual(this); + + if (md.Attributes.HasFlag(MethodAttributes.Abstract)) + yield return Tuples.cil_abstract(this); + + if (md.Attributes.HasFlag(MethodAttributes.HasSecurity)) + yield return Tuples.cil_security(this); + + if (md.Attributes.HasFlag(MethodAttributes.RequireSecObject)) + yield return Tuples.cil_requiresecobject(this); + + if (md.Attributes.HasFlag(MethodAttributes.SpecialName)) + yield return Tuples.cil_specialname(this); + + if (md.Attributes.HasFlag(MethodAttributes.NewSlot)) + yield return Tuples.cil_newslot(this); + + // Populate attributes + Attribute.Populate(cx, this, md.GetCustomAttributes()); + } + } + + IEnumerable Decode(byte[] ilbytes, Dictionary jump_table) + { + // Sequence points are stored in order of offset. + // We use an enumerator to locate the correct sequence point for each instruction. + // The sequence point gives the location of each instruction. + // The location of an instruction is given by the sequence point *after* the + // instruction. + IEnumerator nextSequencePoint = null; + PdbSourceLocation instructionLocation = null; + + if (methodDebugInformation != null) + { + nextSequencePoint = methodDebugInformation.SequencePoints.GetEnumerator(); + if (nextSequencePoint.MoveNext()) + { + instructionLocation = cx.CreateSourceLocation(nextSequencePoint.Current.Location); + yield return instructionLocation; + } + else + { + nextSequencePoint = null; + } + } + + int child = 0; + for (int offset = 0; offset < ilbytes.Length;) + { + var instruction = new Instruction(cx, this, ilbytes, offset, child++); + yield return instruction; + + if (nextSequencePoint != null && offset >= nextSequencePoint.Current.Offset) + { + instructionLocation = cx.CreateSourceLocation(nextSequencePoint.Current.Location); + yield return instructionLocation; + if (!nextSequencePoint.MoveNext()) + nextSequencePoint = null; + } + + if (instructionLocation != null) + yield return Tuples.cil_instruction_location(instruction, instructionLocation); + + jump_table.Add(instruction.Offset, instruction); + offset += instruction.Width; + } + + foreach (var i in jump_table) + { + foreach (var t in i.Value.JumpContents(jump_table)) + yield return t; + } + } + + /// + /// Display the instructions in the method in the debugger. + /// This is only used for debugging, not in the code itself. + /// + public IEnumerable DebugInstructions + { + get + { + if (md.ImplAttributes == MethodImplAttributes.IL && md.RelativeVirtualAddress != 0) + { + var body = cx.peReader.GetMethodBody(md.RelativeVirtualAddress); + + var ilbytes = body.GetILBytes(); + + int child = 0; + for (int offset = 0; offset < ilbytes.Length;) + { + Instruction decoded; + try + { + decoded = new Instruction(cx, this, ilbytes, offset, child++); + offset += decoded.Width; + } + catch + { + yield break; + } + yield return decoded; + } + } + } + } + } + + /// + /// This is a late-bound reference to a method. + /// + class MemberReferenceMethod : Method + { + readonly MemberReference mr; + readonly Type declType; + readonly GenericContext parent; + readonly Method sourceDeclaration; + + public MemberReferenceMethod(GenericContext gc, MemberReferenceHandle handle) : base(gc) + { + this.gc = gc; + mr = cx.mdReader.GetMemberReference(handle); + + signature = mr.DecodeMethodSignature(new SignatureDecoder(), gc); + + parent = (GenericContext)cx.CreateGeneric(gc, mr.Parent); + + var parentMethod = parent as Method; + var nameLabel = cx.GetId(mr.Name); + + declType = parentMethod == null ? parent as Type : parentMethod.DeclaringType; + + ShortId = MakeMethodId(declType, nameLabel); + + var typeSourceDeclaration = declType.SourceDeclaration; + sourceDeclaration = typeSourceDeclaration == declType ? (Method)this : typeSourceDeclaration.LookupMethod(mr.Name, mr.Signature); + } + + public override Method SourceDeclaration => sourceDeclaration; + + public override bool IsStatic => !signature.Header.IsInstance; + + public override Type DeclaringType => declType; + + public override string Name => cx.ShortName(mr.Name); + + public override IEnumerable TypeParameters => parent.TypeParameters.Concat(gc.TypeParameters); + + public override IEnumerable Contents + { + get + { + genericParams = new MethodTypeParameter[signature.GenericParameterCount]; + for (int p = 0; p < genericParams.Length; ++p) + genericParams[p] = cx.Populate(new MethodTypeParameter(this, this, p)); + + foreach (var p in genericParams) + yield return p; + + var typeSignature = mr.DecodeMethodSignature(cx.TypeSignatureDecoder, this); + + PopulateParameters(typeSignature.ParameterTypes); + foreach (var p in Parameters) yield return p; + + foreach (var f in PopulateFlags) yield return f; + + yield return Tuples.cil_method(this, Name, DeclaringType, typeSignature.ReturnType); + + if (SourceDeclaration != null) + yield return Tuples.cil_method_source_declaration(this, SourceDeclaration); + } + } + } + + /// + /// A constructed method. + /// + class MethodSpecificationMethod : Method + { + readonly MethodSpecification ms; + readonly Method unboundMethod; + readonly ImmutableArray typeParams; + + public MethodSpecificationMethod(GenericContext gc, MethodSpecificationHandle handle) : base(gc) + { + ms = cx.mdReader.GetMethodSpecification(handle); + + typeParams = ms.DecodeSignature(cx.TypeSignatureDecoder, gc); + + unboundMethod = (Method)cx.CreateGeneric(gc, ms.Method); + declaringType = unboundMethod.DeclaringType; + + ShortId = unboundMethod.ShortId + openAngle + CIL.Id.CommaSeparatedList(typeParams.Select(p => p.ShortId)) + closeAngle; + } + + static readonly Id openAngle = CIL.Id.Create("<"); + static readonly Id closeAngle = CIL.Id.Create(">"); + + public override Method SourceDeclaration => unboundMethod; + + public override Type DeclaringType => unboundMethod.DeclaringType; + + public override string Name => unboundMethod.Name; + + public override bool IsStatic => unboundMethod.IsStatic; + + public override IEnumerable MethodParameters => typeParams; + + public override IEnumerable Contents + { + get + { + MethodSignature constructedTypeSignature; + switch (ms.Method.Kind) + { + case HandleKind.MemberReference: + var mr = cx.mdReader.GetMemberReference((MemberReferenceHandle)ms.Method); + constructedTypeSignature = mr.DecodeMethodSignature(cx.TypeSignatureDecoder, this); + break; + case HandleKind.MethodDefinition: + var md = cx.mdReader.GetMethodDefinition((MethodDefinitionHandle)ms.Method); + constructedTypeSignature = md.DecodeSignature(cx.TypeSignatureDecoder, this); + break; + default: + throw new InternalError("Unexpected constructed method handle kind {0}", ms.Method.Kind); + } + + PopulateParameters(constructedTypeSignature.ParameterTypes); + foreach (var p in Parameters) + yield return p; + + foreach (var f in PopulateFlags) + yield return f; + + yield return Tuples.cil_method(this, Name, DeclaringType, constructedTypeSignature.ReturnType); + yield return Tuples.cil_method_source_declaration(this, SourceDeclaration); + + if (typeParams.Count() != unboundMethod.GenericParameterCount) + throw new InternalError("Method type parameter mismatch"); + + for (int p = 0; p < typeParams.Length; ++p) + { + yield return Tuples.cil_type_argument(this, p, typeParams[p]); + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs new file mode 100644 index 000000000000..e34d68554e71 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A namespace. + /// + interface INamespace : ITypeContainer + { + } + + /// + /// A namespace. + /// + public class Namespace : TypeContainer, INamespace + { + public Namespace ParentNamespace; + public readonly StringId Name; + + public bool IsGlobalNamespace => ParentNamespace == null; + + static readonly Id suffix = CIL.Id.Create(";namespace"); + + public Id CreateId + { + get + { + if (ParentNamespace != null && !ParentNamespace.IsGlobalNamespace) + { + return ParentNamespace.ShortId + cx.Dot + Name; + } + return Name; + } + } + + public override Id IdSuffix => suffix; + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + static string parseNamespaceName(string fqn) + { + var i = fqn.LastIndexOf('.'); + return i == -1 ? fqn : fqn.Substring(i + 1); + } + + static Namespace createParentNamespace(Context cx, string fqn) + { + if (fqn == "") return null; + var i = fqn.LastIndexOf('.'); + return i == -1 ? cx.GlobalNamespace : cx.Populate(new Namespace(cx, fqn.Substring(0, i))); + } + + public Namespace(Context cx, string fqn) : this(cx, cx.GetId(parseNamespaceName(fqn)), createParentNamespace(cx, fqn)) + { + } + + public Namespace(Context cx, StringId name, Namespace parent) : base(cx) + { + Name = name; + ParentNamespace = parent; + ShortId = CreateId; + } + + public override IEnumerable Contents + { + get + { + yield return Tuples.namespaces(this, Name.Value); + if (!IsGlobalNamespace) + yield return Tuples.parent_namespace(this, ParentNamespace); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs new file mode 100644 index 000000000000..b008ccc25109 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs @@ -0,0 +1,43 @@ +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A parameter entity. + /// + interface IParameter : ILabelledEntity + { + } + + /// + /// A parameter entity. + /// + class Parameter : LabelledEntity, IParameter + { + readonly Method method; + readonly int index; + readonly Type type; + + public Parameter(Context cx, Method m, int i, Type t) : base(cx) + { + method = m; + index = i; + type = t; + ShortId = openCurly + method.Label.Value + closeCurly + index; + } + + static readonly Id parameterSuffix = CIL.Id.Create(";cil-parameter"); + static readonly Id openCurly = CIL.Id.Create("{#"); + static readonly Id closeCurly = CIL.Id.Create("}_"); + + public override Id IdSuffix => parameterSuffix; + + public override IEnumerable Contents + { + get + { + yield return Tuples.cil_parameter(this, method, index, type); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs new file mode 100644 index 000000000000..2adacd2e8046 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs @@ -0,0 +1,64 @@ +using System.Collections.Generic; +using System.Reflection.Metadata; +using System.Linq; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A property. + /// + interface IProperty : ILabelledEntity + { + } + + /// + /// A property. + /// + class Property : LabelledEntity, IProperty + { + readonly Type type; + readonly PropertyDefinition pd; + static readonly Id suffix = CIL.Id.Create(";cil-property"); + + public Property(GenericContext gc, Type type, PropertyDefinitionHandle handle) : base(gc.cx) + { + pd = cx.mdReader.GetPropertyDefinition(handle); + this.type = type; + + var id = type.ShortId + gc.cx.Dot + cx.ShortName(pd.Name); + var signature = pd.DecodeSignature(new SignatureDecoder(), gc); + id += "(" + CIL.Id.CommaSeparatedList(signature.ParameterTypes.Select(p => p.MakeId(gc))) + ")"; + ShortId = id; + } + + public override IEnumerable Contents + { + get + { + var sig = pd.DecodeSignature(cx.TypeSignatureDecoder, type); + + yield return Tuples.cil_property(this, type, cx.ShortName(pd.Name), sig.ReturnType); + + var accessors = pd.GetAccessors(); + if (!accessors.Getter.IsNil) + { + var getter = (Method)cx.CreateGeneric(type, accessors.Getter); + yield return getter; + yield return Tuples.cil_getter(this, getter); + } + + if (!accessors.Setter.IsNil) + { + var setter = (Method)cx.CreateGeneric(type, accessors.Setter); + yield return setter; + yield return Tuples.cil_setter(this, setter); + } + + foreach (var c in Attribute.Populate(cx, this, pd.GetCustomAttributes())) + yield return c; + } + } + + public override Id IdSuffix => suffix; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs new file mode 100644 index 000000000000..885927890995 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; +using Semmle.Extraction.PDB; + +namespace Semmle.Extraction.CIL.Entities +{ + public interface ISourceLocation : ILocation + { + } + + public sealed class PdbSourceLocation : LabelledEntity, ISourceLocation + { + readonly Location location; + readonly PdbSourceFile file; + + public PdbSourceLocation(Context cx, PDB.Location location) : base(cx) + { + this.location = location; + file = cx.CreateSourceFile(location.File); + + ShortId = file.ShortId + separator + new IntId(location.StartLine) + separator + new IntId(location.StartColumn) + separator + new IntId(location.EndLine) + separator + new IntId(location.EndColumn); + } + + static readonly Id suffix = new StringId(";sourcelocation"); + static readonly Id separator = new StringId(","); + + public override IEnumerable Contents + { + get + { + yield return file; + yield return Tuples.locations_default(this, file, location.StartLine, location.StartColumn, location.EndLine, location.EndColumn); + } + } + + public override Id IdSuffix => suffix; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs new file mode 100644 index 000000000000..ac5c436770f5 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs @@ -0,0 +1,1285 @@ +using System; +using Microsoft.CodeAnalysis; +using System.Reflection.Metadata; +using System.Collections.Immutable; +using System.Linq; +using System.Collections.Generic; +using System.Reflection; +using Semmle.Util; + +namespace Semmle.Extraction.CIL.Entities +{ + /// + /// A type. + /// + interface IType : IEntity + { + } + + /// + /// An array type. + /// + interface IArrayType : IType + { + } + + /// + /// The CIL database type-kind. + /// + public enum CilTypeKind + { + ValueOrRefType, + TypeParameter, + Array, + Pointer + } + + /// + /// A type container (namespace/types/method). + /// + interface ITypeContainer : ILabelledEntity + { + } + + /// + /// Base class for all type containers (namespaces, types, methods). + /// + abstract public class TypeContainer : GenericContext, ITypeContainer + { + protected TypeContainer(Context cx) : base(cx) + { + this.cx = cx; + } + + public virtual Label Label { get; set; } + + public virtual IId Id { get { return ShortId + IdSuffix; } } + + public Id ShortId { get; set; } + public abstract Id IdSuffix { get; } + + Location IEntity.ReportingLocation => throw new NotImplementedException(); + + public void Extract(Context cx) { cx.Populate(this); } + + public abstract IEnumerable Contents { get; } + + public override string ToString() + { + return Id.ToString(); + } + + TrapStackBehaviour IEntity.TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } + + /// + /// A type. + /// + public abstract class Type : TypeContainer, IMember, IType + { + static readonly Id suffix = CIL.Id.Create(";cil-type"); + public override Id IdSuffix => suffix; + + /// + /// Find the method in this type matching the name and signature. + /// + /// The handle to the name. + /// + /// The handle to the signature. Note that comparing handles is a valid + /// shortcut to comparing the signature bytes since handles are unique. + /// + /// The method, or 'null' if not found or not supported. + internal virtual Method LookupMethod(StringHandle Name, BlobHandle signature) + { + return null; + } + + public IEnumerable TypeArguments + { + get + { + if (ContainingType != null) + foreach (var t in ContainingType.TypeArguments) + yield return t; + + foreach (var t in ThisTypeArguments) + yield return t; + + } + } + + public virtual IEnumerable ThisTypeArguments + { + get + { + yield break; + } + } + + /// + /// Gets the assembly identifier of this type. + /// + public abstract Id AssemblyPrefix { get; } + + /// + /// Gets the ID part to be used in a method. + /// + /// + /// Whether we should output the context prefix of type parameters. + /// (This is to avoid infinite recursion generating a method ID that returns a + /// type parameter.) + /// + public abstract Id MakeId(bool inContext); + + public Id GetId(bool inContext) + { + return inContext ? MakeId(true) : ShortId; + } + + protected Type(Context cx) : base(cx) { } + + public abstract CilTypeKind Kind + { + get; + } + + public virtual TypeContainer Parent => (TypeContainer)ContainingType ?? Namespace; + + public override IEnumerable Contents + { + get + { + yield return Tuples.cil_type(this, Name.Value, Kind, Parent, SourceDeclaration); + } + } + + /// + /// Whether this type is visible outside the current assembly. + /// + public virtual bool IsVisible => true; + + public abstract Id Name { get; } + + public abstract Namespace Namespace { get; } + + public abstract Type ContainingType { get; } + + public abstract Type Construct(IEnumerable typeArguments); + + /// + /// The number of type arguments, or 0 if this isn't generic. + /// The containing type may also have type arguments. + /// + public abstract int ThisTypeParameters { get; } + + /// + /// The total number of type parameters (including parent types). + /// This is used for internal consistency checking only. + /// + public int TotalTypeParametersCheck => + ContainingType == null ? ThisTypeParameters : ThisTypeParameters + ContainingType.TotalTypeParametersCheck; + + /// + /// Returns all bound/unbound generic arguments + /// of a constructed/unbound generic type. + /// + public virtual IEnumerable ThisGenericArguments + { + get + { + yield break; + } + } + + public virtual IEnumerable GenericArguments + { + get + { + if (ContainingType != null) + foreach (var t in ContainingType.GenericArguments) + yield return t; + foreach (var t in ThisGenericArguments) + yield return t; + } + } + + public virtual Type SourceDeclaration => this; + + protected static readonly Id builtin = CIL.Id.Create("builtin:"); + + public Id PrimitiveTypeId => builtin + Name; + + public bool IsPrimitiveType + { + get + { + if (ContainingType == null && Namespace.ShortId == cx.SystemNamespace.ShortId) + { + switch (Name.Value) + { + case "Boolean": + case "Object": + case "Byte": + case "SByte": + case "Int16": + case "UInt16": + case "Int32": + case "UInt32": + case "Int64": + case "UInt64": + case "Single": + case "Double": + case "String": + case "Void": + case "IntPtr": + case "UIntPtr": + case "Char": + case "TypedReference": + return true; + } + } + return false; + } + } + } + + /// + /// A type defined in the current assembly. + /// + public sealed class TypeDefinitionType : Type + { + readonly TypeDefinition td; + + public TypeDefinitionType(Context cx, TypeDefinitionHandle handle) : base(cx) + { + td = cx.mdReader.GetTypeDefinition(handle); + + declType = + td.GetDeclaringType().IsNil ? null : + (Type)cx.Create(td.GetDeclaringType()); + + ShortId = MakeId(false); + + // Lazy because should happen during population. + typeParams = new Lazy>(MakeTypeParameters); + } + + public override Id MakeId(bool inContext) + { + if (IsPrimitiveType) return PrimitiveTypeId; + + var name = cx.GetId(td.Name); + + Id l; + if (ContainingType != null) + { + l = ContainingType.GetId(inContext) + cx.Dot; + } + else + { + l = AssemblyPrefix; + + var ns = Namespace; + if (!ns.IsGlobalNamespace) + { + l = l + ns.ShortId + cx.Dot; + } + } + + return l + name; + } + + public override Id Name + { + get + { + var name = cx.GetId(td.Name); + var tick = name.Value.IndexOf('`'); + return tick == -1 ? name : cx.GetId(name.Value.Substring(0, tick)); + } + } + + public override Namespace Namespace => cx.Create(td.NamespaceDefinition); + + readonly Type declType; + + public override Type ContainingType => declType; + + public override int ThisTypeParameters + { + get + { + var containingType = td.GetDeclaringType(); + var parentTypeParameters = containingType.IsNil ? 0 : + cx.mdReader.GetTypeDefinition(containingType).GetGenericParameters().Count; + + return td.GetGenericParameters().Count - parentTypeParameters; + } + } + + public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; + + public override Type Construct(IEnumerable typeArguments) + { + return cx.Populate(new ConstructedType(cx, this, typeArguments)); + } + + public override Id AssemblyPrefix + { + get + { + var ct = ContainingType; + return ct != null ? ct.AssemblyPrefix : IsPrimitiveType ? builtin : cx.AssemblyPrefix; + } + } + + IEnumerable MakeTypeParameters() + { + if (ThisTypeParameters == 0) + return Enumerable.Empty(); + + var typeParams = new TypeTypeParameter[ThisTypeParameters]; + var genericParams = td.GetGenericParameters(); + int toSkip = genericParams.Count - typeParams.Length; + + // Two-phase population because type parameters can be mutually dependent + for (int i = 0; i < typeParams.Length; ++i) + typeParams[i] = cx.Populate(new TypeTypeParameter(this, this, i)); + for (int i = 0; i < typeParams.Length; ++i) + typeParams[i].PopulateHandle(this, genericParams[i + toSkip]); + return typeParams; + } + + readonly Lazy> typeParams; + + public override IEnumerable MethodParameters => Enumerable.Empty(); + + public override IEnumerable TypeParameters + { + get + { + if (declType != null) + { + foreach (var t in declType.TypeParameters) + yield return t; + } + + foreach (var t in typeParams.Value) + yield return t; + } + } + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) yield return c; + + MakeTypeParameters(); + + foreach (var f in td.GetFields()) + { + // Populate field if needed + yield return cx.CreateGeneric(this, f); + } + + foreach (var prop in td.GetProperties()) + { + yield return new Property(this, this, prop); + } + + foreach (var @event in td.GetEvents()) + { + yield return new Event(cx, this, @event); + } + + foreach (var a in Attribute.Populate(cx, this, td.GetCustomAttributes())) + yield return a; + + foreach (var impl in td.GetMethodImplementations().Select(i => cx.mdReader.GetMethodImplementation(i))) + { + var m = (Method)cx.CreateGeneric(this, impl.MethodBody); + var decl = (Method)cx.CreateGeneric(this, impl.MethodDeclaration); + + yield return m; + yield return decl; + yield return Tuples.cil_implements(m, decl); + } + + if (td.Attributes.HasFlag(TypeAttributes.Abstract)) + yield return Tuples.cil_abstract(this); + + if (td.Attributes.HasFlag(TypeAttributes.Class)) + yield return Tuples.cil_class(this); + + if (td.Attributes.HasFlag(TypeAttributes.Interface)) + yield return Tuples.cil_interface(this); + + if (td.Attributes.HasFlag(TypeAttributes.Public)) + yield return Tuples.cil_public(this); + + if (td.Attributes.HasFlag(TypeAttributes.Sealed)) + yield return Tuples.cil_sealed(this); + + if (td.Attributes.HasFlag(TypeAttributes.HasSecurity)) + yield return Tuples.cil_security(this); + + // Base types + + if (!td.BaseType.IsNil) + { + var @base = (Type)cx.CreateGeneric(this, td.BaseType); + yield return @base; + yield return Tuples.cil_base_class(this, @base); + } + + foreach (var @interface in td.GetInterfaceImplementations().Select(i => cx.mdReader.GetInterfaceImplementation(i))) + { + var t = (Type)cx.CreateGeneric(this, @interface.Interface); + yield return t; + yield return Tuples.cil_base_interface(this, t); + } + + // Only type definitions have locations. + yield return Tuples.cil_type_location(this, cx.assembly); + } + } + + internal override Method LookupMethod(StringHandle name, BlobHandle signature) + { + foreach (var h in td.GetMethods()) + { + var md = cx.mdReader.GetMethodDefinition(h); + + if (md.Name == name && md.Signature == signature) + { + return (Method)cx.Create(h); + } + } + + throw new InternalError("Couldn't locate method in type"); + } + } + + /// + /// A type reference, to a type in a referenced assembly. + /// + public sealed class TypeReferenceType : Type + { + readonly TypeReference tr; + readonly Lazy typeParams; + + public TypeReferenceType(Context cx, TypeReferenceHandle handle) : this(cx, cx.mdReader.GetTypeReference(handle)) + { + ShortId = MakeId(false); + typeParams = new Lazy(MakeTypeParameters); + } + + public TypeReferenceType(Context cx, TypeReference tr) : base(cx) + { + this.tr = tr; + } + + TypeTypeParameter[] MakeTypeParameters() + { + var typeParams = new TypeTypeParameter[ThisTypeParameters]; + for (int i = 0; i < typeParams.Length; ++i) + { + typeParams[i] = new TypeTypeParameter(this, this, i); + } + return typeParams; + } + + public override IEnumerable Contents + { + get + { + foreach (var tp in typeParams.Value) + yield return tp; + + foreach (var c in base.Contents) + yield return c; + } + } + + public override Id Name + { + get + { + var name = cx.GetId(tr.Name); + var tick = name.Value.IndexOf('`'); + return tick == -1 ? name : cx.GetId(name.Value.Substring(0, tick)); + } + } + + public override Namespace Namespace => cx.CreateNamespace(tr.Namespace); + + public override int ThisTypeParameters + { + get + { + // Parse the name + var name = cx.GetId(tr.Name); + var tick = name.Value.IndexOf('`'); + return tick == -1 ? 0 : int.Parse(name.Value.Substring(tick + 1)); + } + } + + public override IEnumerable ThisGenericArguments + { + get + { + foreach (var t in typeParams.Value) + yield return t; + } + } + + public override Type ContainingType + { + get + { + if (tr.ResolutionScope.Kind == HandleKind.TypeReference) + return (Type)cx.Create((TypeReferenceHandle)tr.ResolutionScope); + return null; + } + } + + public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; + + public override Id AssemblyPrefix + { + get + { + switch (tr.ResolutionScope.Kind) + { + case HandleKind.TypeReference: + return ContainingType.AssemblyPrefix; + case HandleKind.AssemblyReference: + var assemblyDef = cx.mdReader.GetAssemblyReference((AssemblyReferenceHandle)tr.ResolutionScope); + return cx.GetId(assemblyDef.Name) + "_" + cx.GetId(assemblyDef.Version.ToString()) + "::"; + default: + return cx.AssemblyPrefix; + } + } + } + + public override IEnumerable TypeParameters => typeParams.Value; + + public override IEnumerable MethodParameters => throw new InternalError("This type does not have method parameters"); + + public override Id MakeId(bool inContext) + { + if (IsPrimitiveType) return PrimitiveTypeId; + + var ct = ContainingType; + Id l = null; + if (ct != null) + { + l = ContainingType.GetId(inContext); + } + else + { + if (tr.ResolutionScope.Kind == HandleKind.AssemblyReference) + { + l = AssemblyPrefix; + } + + if (!Namespace.IsGlobalNamespace) + { + l += Namespace.ShortId; + } + } + + return l + cx.Dot + cx.GetId(tr.Name); + } + + public override Type Construct(IEnumerable typeArguments) + { + if (TotalTypeParametersCheck != typeArguments.Count()) + throw new InternalError("Mismatched type arguments"); + + return cx.Populate(new ConstructedType(cx, this, typeArguments)); + } + } + + + /// + /// A constructed type. + /// + public sealed class ConstructedType : Type + { + readonly Type unboundGenericType; + readonly Type[] thisTypeArguments; + + public override IEnumerable ThisTypeArguments => thisTypeArguments; + + public override IEnumerable ThisGenericArguments => thisTypeArguments.EnumerateNull(); + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) + yield return c; + + int i = 0; + foreach (var type in ThisGenericArguments) + { + yield return type; + yield return Tuples.cil_type_argument(this, i++, type); + } + } + } + + public override Type SourceDeclaration => unboundGenericType; + + public ConstructedType(Context cx, Type unboundType, IEnumerable typeArguments) : base(cx) + { + var suppliedArgs = typeArguments.Count(); + if (suppliedArgs != unboundType.TotalTypeParametersCheck) + throw new InternalError("Unexpected number of type arguments in ConstructedType"); + + unboundGenericType = unboundType; + var thisParams = unboundType.ThisTypeParameters; + var parentParams = suppliedArgs - thisParams; + + if (typeArguments.Count() == thisParams) + { + containingType = unboundType.ContainingType; + thisTypeArguments = typeArguments.ToArray(); + } + else if (thisParams == 0) + { + containingType = unboundType.ContainingType.Construct(typeArguments); + } + else + { + containingType = unboundType.ContainingType.Construct(typeArguments.Take(parentParams)); + thisTypeArguments = typeArguments.Skip(parentParams).ToArray(); + } + + ShortId = MakeId(false); + } + + readonly Type containingType; + public override Type ContainingType => containingType; + + public override Id Name => unboundGenericType.Name; + + public override Namespace Namespace => unboundGenericType.Namespace; + + public override int ThisTypeParameters => thisTypeArguments == null ? 0 : thisTypeArguments.Length; + + public override CilTypeKind Kind => unboundGenericType.Kind; + + public override Type Construct(IEnumerable typeArguments) + { + throw new NotImplementedException(); + } + + public override Id MakeId(bool inContext) + { + Id l; + if (ContainingType != null) + { + l = ContainingType.GetId(inContext) + cx.Dot; + } + else + { + l = AssemblyPrefix; + + if (!Namespace.IsGlobalNamespace) + { + l += Namespace.ShortId + cx.Dot; + } + } + l += unboundGenericType.Name; + + if (thisTypeArguments != null && thisTypeArguments.Any()) + { + l += open; + bool first = true; + foreach (var t in thisTypeArguments) + { + if (first) first = false; else l += comma; + l += t.ShortId; + } + l += close; + } + return l; + } + + static readonly StringId open = new StringId("<"); + static readonly StringId close = new StringId(">"); + static readonly StringId comma = new StringId(","); + + public override Id AssemblyPrefix => unboundGenericType.AssemblyPrefix; + + public override IEnumerable TypeParameters => GenericArguments; + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + } + + public sealed class PrimitiveType : Type + { + readonly PrimitiveTypeCode typeCode; + public PrimitiveType(Context cx, PrimitiveTypeCode tc) : base(cx) + { + typeCode = tc; + ShortId = MakeId(false); + } + + public override Id MakeId(bool inContext) + { + return builtin + Name; + } + + public override Id Name => typeCode.Id(); + + public override Namespace Namespace => cx.SystemNamespace; + + public override Type ContainingType => null; + + public override int ThisTypeParameters => 0; + + public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; + + static readonly Id empty = new StringId(""); + + public override Id AssemblyPrefix => empty; + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + public override Type Construct(IEnumerable typeArguments) => throw new NotImplementedException(); + } + + /// + /// An array type. + /// + sealed class ArrayType : Type, IArrayType + { + readonly Type elementType; + readonly int rank; + + public ArrayType(Context cx, Type element, ArrayShape shape) : base(cx) + { + rank = shape.Rank; + elementType = element; + ShortId = MakeId(false); + } + + public ArrayType(Context cx, Type element) : base(cx) + { + rank = 1; + elementType = element; + ShortId = MakeId(false); + } + + public override Id MakeId(bool inContext) => elementType.GetId(inContext) + openBracket + rank + closeBracket; + + static readonly StringId openBracket = new StringId("[]"); + static readonly StringId closeBracket = new StringId("[]"); + + public override Id Name => elementType.Name + openBracket + closeBracket; + + public override Namespace Namespace => cx.SystemNamespace; + + public override Type ContainingType => null; + + public override int ThisTypeParameters => elementType.ThisTypeParameters; + + public override CilTypeKind Kind => CilTypeKind.Array; + + public override Type Construct(IEnumerable typeArguments) => cx.Populate(new ArrayType(cx, elementType.Construct(typeArguments))); + + public override Type SourceDeclaration => cx.Populate(new ArrayType(cx, elementType.SourceDeclaration)); + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) + yield return c; + + yield return Tuples.cil_array_type(this, elementType, rank); + } + } + + public override Id AssemblyPrefix + { + get + { + return elementType.AssemblyPrefix; + } + } + + public override IEnumerable GenericArguments => elementType.GenericArguments; + + public override IEnumerable TypeParameters => elementType.TypeParameters; + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + } + + interface ITypeParameter : IType + { + } + + abstract class TypeParameter : Type, ITypeParameter + { + protected readonly GenericContext gc; + + public TypeParameter(GenericContext gc) : base(gc.cx) + { + this.gc = gc; + } + + public override Namespace Namespace => null; + + public override Type ContainingType => null; + + public override int ThisTypeParameters => 0; + + public override CilTypeKind Kind => CilTypeKind.TypeParameter; + + public override Id AssemblyPrefix => throw new NotImplementedException(); + + public override Type Construct(IEnumerable typeArguments) => throw new InternalError("Attempt to construct a type parameter"); + + public IEnumerable PopulateHandle(GenericContext gc, GenericParameterHandle parameterHandle) + { + if (!parameterHandle.IsNil) + { + var tp = cx.mdReader.GetGenericParameter(parameterHandle); + + if (tp.Attributes.HasFlag(GenericParameterAttributes.Contravariant)) + yield return Tuples.cil_typeparam_contravariant(this); + if (tp.Attributes.HasFlag(GenericParameterAttributes.Covariant)) + yield return Tuples.cil_typeparam_covariant(this); + if (tp.Attributes.HasFlag(GenericParameterAttributes.DefaultConstructorConstraint)) + yield return Tuples.cil_typeparam_new(this); + if (tp.Attributes.HasFlag(GenericParameterAttributes.ReferenceTypeConstraint)) + yield return Tuples.cil_typeparam_class(this); + if (tp.Attributes.HasFlag(GenericParameterAttributes.NotNullableValueTypeConstraint)) + yield return Tuples.cil_typeparam_struct(this); + + foreach (var constraint in tp.GetConstraints().Select(h => cx.mdReader.GetGenericParameterConstraint(h))) + { + var t = (Type)cx.CreateGeneric(this.gc, constraint.Type); + yield return t; + yield return Tuples.cil_typeparam_constraint(this, t); + } + } + } + } + + sealed class MethodTypeParameter : TypeParameter + { + readonly Method method; + readonly int index; + + public override Id MakeId(bool inContext) => inContext && method == gc ? Name : method.ShortId + Name; + + static readonly Id excl = new StringId("!"); + + public override Id Name => excl + index.ToString(); + + public MethodTypeParameter(GenericContext gc, Method m, int index) : base(gc) + { + method = m; + this.index = index; + ShortId = MakeId(false); + } + + public override TypeContainer Parent => method; + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + public override IEnumerable Contents + { + get + { + yield return Tuples.cil_type(this, Name.Value, Kind, method, SourceDeclaration); + yield return Tuples.cil_type_parameter(method, index, this); + } + } + } + + + sealed class TypeTypeParameter : TypeParameter + { + readonly Type type; + readonly int index; + + public TypeTypeParameter(GenericContext cx, Type t, int i) : base(cx) + { + index = i; + type = t; + ShortId = t.ShortId + Name; + } + + public override Id MakeId(bool inContext) => type.MakeId(inContext) + Name; + + public override TypeContainer Parent => type ?? gc as TypeContainer; + + static readonly Id excl = new StringId("!"); + public override Id Name => excl + index.ToString(); + + public override IEnumerable TypeParameters => Enumerable.Empty(); + + public override IEnumerable MethodParameters => Enumerable.Empty(); + + public override IEnumerable Contents + { + get + { + yield return Tuples.cil_type(this, Name.Value, Kind, type, SourceDeclaration); + yield return Tuples.cil_type_parameter(type, index, this); + } + } + } + + interface IPointerType : IType + { + } + + sealed class PointerType : Type, IPointerType + { + readonly Type pointee; + + public PointerType(Context cx, Type pointee) : base(cx) + { + this.pointee = pointee; + ShortId = MakeId(false); + } + + public override Id MakeId(bool inContext) => pointee.MakeId(inContext) + star; + + static readonly StringId star = new StringId("*"); + + public override Id Name => pointee.Name + star; + + public override Namespace Namespace => pointee.Namespace; + + public override Type ContainingType => pointee.ContainingType; + + public override TypeContainer Parent => pointee.Parent; + + public override int ThisTypeParameters => 0; + + public override CilTypeKind Kind => CilTypeKind.Pointer; + + public override Id AssemblyPrefix => pointee.AssemblyPrefix; + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + public override Type Construct(IEnumerable typeArguments) => throw new NotImplementedException(); + + public override IEnumerable Contents + { + get + { + foreach (var c in base.Contents) yield return c; + yield return Tuples.cil_pointer_type(this, pointee); + } + } + } + + sealed class ErrorType : Type + { + public ErrorType(Context cx) : base(cx) + { + ShortId = MakeId(false); + } + + public override Id MakeId(bool inContext) => CIL.Id.Create(""); + + public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; + + public override Id Name => new StringId("!error"); + + public override Namespace Namespace => cx.GlobalNamespace; + + public override Type ContainingType => null; + + public override int ThisTypeParameters => 0; + + public override Id AssemblyPrefix => throw new NotImplementedException(); + + public override IEnumerable TypeParameters => throw new NotImplementedException(); + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + public override Type Construct(IEnumerable typeArguments) => throw new NotImplementedException(); + } + + public sealed class TypeSpecificationType : Type + { + readonly TypeSpecification ts; + readonly Type decodedType; + + public TypeSpecificationType(GenericContext gc, TypeSpecificationHandle handle) : base(gc.cx) + { + ts = cx.mdReader.GetTypeSpecification(handle); + decodedType = ts.DecodeSignature(cx.TypeSignatureDecoder, gc); + ShortId = decodedType.ShortId; + } + + public override Id MakeId(bool inContext) => decodedType.MakeId(inContext); + + public override IEnumerable Contents + { + get + { + yield return decodedType; + } + } + + public override Id AssemblyPrefix => throw new NotImplementedException(); + + public override CilTypeKind Kind => throw new NotImplementedException(); + + public override Id Name => decodedType.Name; + + public override Namespace Namespace => throw new NotImplementedException(); + + public override Type ContainingType => decodedType.ContainingType; + + public override int ThisTypeParameters => throw new NotImplementedException(); + + public override IEnumerable TypeParameters => decodedType.TypeParameters; + + public override IEnumerable MethodParameters => throw new NotImplementedException(); + + public override Type Construct(IEnumerable typeArguments) => throw new NotImplementedException(); + + public override Type SourceDeclaration => decodedType.SourceDeclaration; + } + + interface ITypeSignature + { + Id MakeId(GenericContext gc); + } + + public class SignatureDecoder : ISignatureTypeProvider + { + struct Array : ITypeSignature + { + public ITypeSignature elementType; + public ArrayShape shape; + public Id MakeId(GenericContext gc) => elementType.MakeId(gc) + "[]"; // Make these static + } + + struct ByRef : ITypeSignature + { + public ITypeSignature elementType; + public Id MakeId(GenericContext gc) => "ref " + elementType.MakeId(gc); + } + + struct FnPtr : ITypeSignature + { + public MethodSignature signature; + public Id MakeId(GenericContext gc) => Id.Create(""); // !! + } + + ITypeSignature IConstructedTypeProvider.GetArrayType(ITypeSignature elementType, ArrayShape shape) => + new Array { elementType = elementType, shape = shape }; + + ITypeSignature IConstructedTypeProvider.GetByReferenceType(ITypeSignature elementType) => + new ByRef { elementType = elementType }; + + ITypeSignature ISignatureTypeProvider.GetFunctionPointerType(MethodSignature signature) => + new FnPtr { signature = signature }; + + class Instantiation : ITypeSignature + { + public ITypeSignature genericType; + public ImmutableArray typeArguments; + public Id MakeId(GenericContext gc) => + genericType.MakeId(gc) + "<" + Id.CommaSeparatedList(typeArguments.Select(arg => arg.MakeId(gc))) + ">"; + } + + ITypeSignature IConstructedTypeProvider.GetGenericInstantiation(ITypeSignature genericType, ImmutableArray typeArguments) => + new Instantiation { genericType = genericType, typeArguments = typeArguments }; + + class GenericMethodParameter : ITypeSignature + { + public int index; + static readonly Id excl = Id.Create("M!"); + public Id MakeId(GenericContext gc) => excl + index; + } + + class GenericTypeParameter : ITypeSignature + { + public int index; + static readonly Id excl = Id.Create("T!"); + public Id MakeId(GenericContext gc) => excl + index; + } + + ITypeSignature ISignatureTypeProvider.GetGenericMethodParameter(object genericContext, int index) => + new GenericMethodParameter { index = index }; + + ITypeSignature ISignatureTypeProvider.GetGenericTypeParameter(object genericContext, int index) => + new GenericTypeParameter { index = index }; + + class Modified : ITypeSignature + { + public ITypeSignature modifier; + public ITypeSignature unmodifiedType; + public bool isRequired; + + public Id MakeId(GenericContext gc) => unmodifiedType.MakeId(gc); + } + + ITypeSignature ISignatureTypeProvider.GetModifiedType(ITypeSignature modifier, ITypeSignature unmodifiedType, bool isRequired) + { + return new Modified { modifier = modifier, unmodifiedType = unmodifiedType, isRequired = isRequired }; + } + + class Pinned : ITypeSignature + { + public ITypeSignature elementType; + public Id MakeId(GenericContext gc) => "pinned " + elementType.MakeId(gc); + } + + ITypeSignature ISignatureTypeProvider.GetPinnedType(ITypeSignature elementType) + { + return new Pinned { elementType = elementType }; + } + + class PointerType : ITypeSignature + { + public ITypeSignature elementType; + public Id MakeId(GenericContext gc) => elementType.MakeId(gc) + "*"; + } + + ITypeSignature IConstructedTypeProvider.GetPointerType(ITypeSignature elementType) + { + return new PointerType { elementType = elementType }; + } + + class Primitive : ITypeSignature + { + public PrimitiveTypeCode typeCode; + public Id MakeId(GenericContext gc) => typeCode.Id(); + } + + ITypeSignature ISimpleTypeProvider.GetPrimitiveType(PrimitiveTypeCode typeCode) + { + return new Primitive { typeCode = typeCode }; + } + + class SzArrayType : ITypeSignature + { + public ITypeSignature elementType; + public Id MakeId(GenericContext gc) => elementType.MakeId(gc) + "[]"; + } + + ITypeSignature ISZArrayTypeProvider.GetSZArrayType(ITypeSignature elementType) + { + return new SzArrayType { elementType = elementType }; + } + + class TypeDefinition : ITypeSignature + { + public TypeDefinitionHandle handle; + public byte rawTypeKind; + Type type; + public Id MakeId(GenericContext gc) + { + type = (Type)gc.cx.Create(handle); + return type.ShortId; + } + } + + ITypeSignature ISimpleTypeProvider.GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) + { + return new TypeDefinition { handle = handle, rawTypeKind = rawTypeKind }; + } + + class TypeReference : ITypeSignature + { + public TypeReferenceHandle handle; + public byte rawTypeKind; // struct/class (not used) + Type type; + public Id MakeId(GenericContext gc) + { + type = (Type)gc.cx.Create(handle); + return type.ShortId; + } + } + + ITypeSignature ISimpleTypeProvider.GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) + { + return new TypeReference { handle = handle, rawTypeKind = rawTypeKind }; + } + + ITypeSignature ISignatureTypeProvider.GetTypeFromSpecification(MetadataReader reader, object genericContext, TypeSpecificationHandle handle, byte rawTypeKind) + { + var ts = reader.GetTypeSpecification(handle); + return ts.DecodeSignature(this, genericContext); + } + } + + + /// + /// Decodes a type signature and produces a Type, for use by DecodeSignature() and friends. + /// + public class TypeSignatureDecoder : ISignatureTypeProvider + { + readonly Context cx; + + public TypeSignatureDecoder(Context cx) + { + this.cx = cx; + } + + Type IConstructedTypeProvider.GetArrayType(Type elementType, ArrayShape shape) => + cx.Populate(new ArrayType(cx, elementType, shape)); + + Type IConstructedTypeProvider.GetByReferenceType(Type elementType) => + elementType; // ?? + + Type ISignatureTypeProvider.GetFunctionPointerType(MethodSignature signature) => + cx.ErrorType; // Don't know what to do !! + + Type IConstructedTypeProvider.GetGenericInstantiation(Type genericType, ImmutableArray typeArguments) => + genericType.Construct(typeArguments); + + Type ISignatureTypeProvider.GetGenericMethodParameter(GenericContext genericContext, int index) => + genericContext.GetGenericMethodParameter(index); + + Type ISignatureTypeProvider.GetGenericTypeParameter(GenericContext genericContext, int index) => + genericContext.GetGenericTypeParameter(index); + + Type ISignatureTypeProvider.GetModifiedType(Type modifier, Type unmodifiedType, bool isRequired) => + // !! Not implemented properly + unmodifiedType; + + Type ISignatureTypeProvider.GetPinnedType(Type elementType) => + cx.Populate(new PointerType(cx, elementType)); + + Type IConstructedTypeProvider.GetPointerType(Type elementType) => + cx.Populate(new PointerType(cx, elementType)); + + Type ISimpleTypeProvider.GetPrimitiveType(PrimitiveTypeCode typeCode) => + cx.Populate(new PrimitiveType(cx, typeCode)); + + Type ISZArrayTypeProvider.GetSZArrayType(Type elementType) => + cx.Populate(new ArrayType(cx, elementType)); + + Type ISimpleTypeProvider.GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => + (Type)cx.Create(handle); + + Type ISimpleTypeProvider.GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => + (Type)cx.Create(handle); + + Type ISignatureTypeProvider.GetTypeFromSpecification(MetadataReader reader, GenericContext genericContext, TypeSpecificationHandle handle, byte rawTypeKind) => + throw new NotImplementedException(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs b/csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs new file mode 100644 index 000000000000..16181d9b32e8 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Generic; + +namespace Semmle.Extraction.CIL +{ + /// + /// Something that is extracted from an entity. + /// + /// + /// + /// The extraction algorithm proceeds as follows: + /// - Construct entity + /// - Call Extract() + /// - ILabelledEntity check if already extracted + /// - Enumerate Contents to produce more extraction products + /// - Extract these until there is nothing left to extract + /// + public interface IExtractionProduct + { + /// + /// Perform further extraction/population of this item as necessary. + /// + /// + /// The extraction context. + void Extract(Context cx); + } + + /// + /// An entity which has been extracted. + /// + public interface IExtractedEntity : IEntity, IExtractionProduct + { + /// + /// The contents of the entity. + /// + IEnumerable Contents { get; } + } + + /// + /// An entity that has contents to extract. There is no need to populate + /// a key as it's done in the contructor. + /// + public abstract class UnlabelledEntity : IExtractedEntity + { + public abstract IEnumerable Contents { get; } + public Label Label { get; set; } + + public Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException(); + + public virtual IId Id => FreshId.Instance; + + public virtual void Extract(Context cx) + { + cx.Extract(this); + } + + public readonly Context cx; + + protected UnlabelledEntity(Context cx) + { + this.cx = cx; + cx.cx.AddFreshLabel(this); + } + + TrapStackBehaviour IEntity.TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } + + /// + /// An entity that needs to be populated during extraction. + /// This assigns a key and optionally extracts its contents. + /// + public abstract class LabelledEntity : ILabelledEntity + { + public abstract IEnumerable Contents { get; } + public Label Label { get; set; } + public Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException(); + + public Id ShortId { get; set; } + public abstract Id IdSuffix { get; } + public IId Id => ShortId + IdSuffix; + + public void Extract(Context cx) + { + cx.Populate(this); + } + + public readonly Context cx; + + protected LabelledEntity(Context cx) + { + this.cx = cx; + } + + public override string ToString() => Id.ToString(); + + TrapStackBehaviour IEntity.TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } + + /// + /// An entity with a defined ID. + /// + public interface ILabelledEntity : IExtractedEntity + { + Id ShortId { get; set; } + Id IdSuffix { get; } + } + + /// + /// A tuple that is an extraction product. + /// + class Tuple : IExtractionProduct + { + readonly Extraction.Tuple tuple; + + public Tuple(string name, params object[] args) + { + tuple = new Extraction.Tuple(name, args); + } + + public void Extract(Context cx) + { + cx.cx.Emit(tuple); + } + + public override string ToString() => tuple.ToString(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Factories.cs b/csharp/extractor/Semmle.Extraction.CIL/Factories.cs new file mode 100644 index 000000000000..b8909f2b8334 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Factories.cs @@ -0,0 +1,233 @@ +using System; +using System.Collections.Generic; +using System.Reflection.Metadata; + +namespace Semmle.Extraction.CIL +{ + /// + /// Provides methods for creating and caching various entities. + /// + public partial class Context + { + readonly Dictionary ids = new Dictionary(); + + public T Populate(T e) where T : ILabelledEntity + { + Id id = e.ShortId; + + if (ids.TryGetValue(id, out var existing)) + { + // It exists already + e.Label = existing.Item1; + e.ShortId = existing.Item2; // Reuse ID for efficiency + } + else + { + cx.DefineLabel(e); + ids.Add(id, (e.Label, id)); + cx.PopulateLater(() => + { + foreach (var c in e.Contents) + c.Extract(this); + }); + } + return e; + } + + public IExtractedEntity Create(Handle h) + { + var entity = CreateGeneric(defaultGenericContext, h); + return entity; + } + + /// + /// Creates an entity from a Handle in a GenericContext. + /// The type of the returned entity depends on the type of the handle. + /// The GenericContext is needed because some handles are generics which + /// need to be expanded in terms of the current instantiation. If this sounds + /// complex, you are right. + /// + /// The pair (h,genericContext) is cached in case it is needed again. + /// + /// The handle of the entity. + /// The generic context. + /// + public ILabelledEntity CreateGeneric(GenericContext genericContext, Handle h) => genericHandleFactory[genericContext, h]; + + readonly GenericContext defaultGenericContext; + + ILabelledEntity CreateGenericHandle(GenericContext gc, Handle handle) + { + ILabelledEntity entity; + switch (handle.Kind) + { + case HandleKind.MethodDefinition: + entity = new Entities.DefinitionMethod(gc, (MethodDefinitionHandle)handle); + break; + case HandleKind.MemberReference: + entity = Create(gc, (MemberReferenceHandle)handle); + break; + case HandleKind.MethodSpecification: + entity = new Entities.MethodSpecificationMethod(gc, (MethodSpecificationHandle)handle); + break; + case HandleKind.FieldDefinition: + entity = new Entities.DefinitionField(gc, (FieldDefinitionHandle)handle); + break; + case HandleKind.TypeReference: + entity = new Entities.TypeReferenceType(this, (TypeReferenceHandle)handle); + break; + case HandleKind.TypeSpecification: + entity = new Entities.TypeSpecificationType(gc, (TypeSpecificationHandle)handle); + break; + case HandleKind.TypeDefinition: + entity = new Entities.TypeDefinitionType(this, (TypeDefinitionHandle)handle); + break; + default: + throw new InternalError("Unhandled handle kind " + handle.Kind); + } + + Populate(entity); + return entity; + } + + ILabelledEntity Create(GenericContext gc, MemberReferenceHandle handle) + { + var mr = mdReader.GetMemberReference(handle); + switch (mr.GetKind()) + { + case MemberReferenceKind.Method: + return new Entities.MemberReferenceMethod(gc, handle); + case MemberReferenceKind.Field: + return new Entities.MemberReferenceField(gc, handle); + default: + throw new InternalError("Unhandled member reference handle"); + } + } + + #region Strings + readonly Dictionary stringHandleIds = new Dictionary(); + readonly Dictionary stringIds = new Dictionary(); + + /// + /// Return an ID containing the given string. + /// + /// The string handle. + /// An ID. + public StringId GetId(StringHandle h) + { + StringId result; + if (!stringHandleIds.TryGetValue(h, out result)) + { + result = new StringId(mdReader.GetString(h)); + stringHandleIds.Add(h, result); + } + return result; + } + + public readonly StringId Dot = new StringId("."); + + /// + /// Gets an ID containing the given string. + /// Caches existing IDs for more compact storage. + /// + /// The string. + /// An ID containing the string. + public StringId GetId(string str) + { + StringId result; + if (!stringIds.TryGetValue(str, out result)) + { + result = new StringId(str); + stringIds.Add(str, result); + } + return result; + } + #endregion + + #region Namespaces + + readonly CachedFunction namespaceFactory; + + public Entities.Namespace CreateNamespace(StringHandle fqn) => namespaceFactory[fqn]; + + readonly Lazy globalNamespace, systemNamespace; + + /// + /// The entity representing the global namespace. + /// + public Entities.Namespace GlobalNamespace => globalNamespace.Value; + + /// + /// The entity representing the System namespace. + /// + public Entities.Namespace SystemNamespace => systemNamespace.Value; + + /// + /// Creates a namespace from a fully-qualified name. + /// + /// The fully-qualified namespace name. + /// The namespace entity. + Entities.Namespace CreateNamespace(string fqn) => Populate(new Entities.Namespace(this, fqn)); + + readonly CachedFunction namespaceDefinitionFactory; + + /// + /// Creates a namespace from a namespace handle. + /// + /// The handle of the namespace. + /// The namespace entity. + public Entities.Namespace Create(NamespaceDefinitionHandle handle) => namespaceDefinitionFactory[handle]; + + Entities.Namespace CreateNamespace(NamespaceDefinitionHandle handle) + { + if (handle.IsNil) return GlobalNamespace; + NamespaceDefinition nd = mdReader.GetNamespaceDefinition(handle); + return Populate(new Entities.Namespace(this, GetId(nd.Name), Create(nd.Parent))); + } + #endregion + + #region Locations + readonly CachedFunction sourceFiles; + readonly CachedFunction folders; + readonly CachedFunction sourceLocations; + + /// + /// Creates a source file entity from a PDB source file. + /// + /// The PDB source file. + /// A source file entity. + public Entities.PdbSourceFile CreateSourceFile(PDB.ISourceFile file) => sourceFiles[file]; + + /// + /// Creates a folder entitiy with the given path. + /// + /// The path of the folder. + /// A folder entity. + public Entities.Folder CreateFolder(string path) => folders[path]; + + /// + /// Creates a source location. + /// + /// The source location from PDB. + /// A source location entity. + public Entities.PdbSourceLocation CreateSourceLocation(PDB.Location loc) => sourceLocations[loc]; + + #endregion + + readonly CachedFunction genericHandleFactory; + + /// + /// Gets the short name of a member, without the preceding interface qualifier. + /// + /// The handle of the name. + /// The short name. + public string ShortName(StringHandle handle) + { + string str = mdReader.GetString(handle); + if (str.EndsWith(".ctor")) return ".ctor"; + if (str.EndsWith(".cctor")) return ".cctor"; + var dot = str.LastIndexOf('.'); + return dot == -1 ? str : str.Substring(dot + 1); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Id.cs b/csharp/extractor/Semmle.Extraction.CIL/Id.cs new file mode 100644 index 000000000000..f55d8acc9cd6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Id.cs @@ -0,0 +1,204 @@ +using System.Collections.Generic; +using System.Reflection.Metadata; + +namespace Semmle.Extraction.CIL +{ + /// + /// An ID fragment which is designed to be shared, reused + /// and composed using the + operator. + /// + public abstract class Id : IId + { + public void AppendTo(ITrapBuilder tb) + { + tb.Append("@\""); + BuildParts(tb); + tb.Append("\""); + } + + public abstract void BuildParts(ITrapBuilder tb); + + public static Id operator +(Id l1, Id l2) => Create(l1, l2); + + public static Id operator +(Id l1, string l2) => Create(l1, Create(l2)); + + public static Id operator +(Id l1, int l2) => Create(l1, Create(l2)); + + public static Id operator +(string l1, Id l2) => Create(Create(l1), l2); + + public static Id operator +(int l1, Id l2) => Create(Create(l1), l2); + + public static Id Create(string s) => s == null ? null : new StringId(s); + + public static Id Create(int i) => new IntId(i); + + static readonly Id openCurly = Create("{#"); + static readonly Id closeCurly = Create("}"); + + public static Id Create(Label l) => openCurly + l.Value + closeCurly; + + static readonly Id comma = Id.Create(","); + + public static Id CommaSeparatedList(IEnumerable items) + { + Id result = null; + bool first = true; + foreach (var i in items) + { + if (first) first = false; else result += comma; + result += i; + } + return result; + } + + public static Id Create(Id l1, Id l2) + { + return l1 == null ? l2 : l2 == null ? l1 : new ConsId(l1, l2); + } + + public abstract string Value { get; } + + public override string ToString() => Value; + } + + /// + /// An ID concatenating two other IDs. + /// + public sealed class ConsId : Id + { + readonly Id left, right; + readonly int hash; + + public ConsId(Id l1, Id l2) + { + left = l1; + right = l2; + hash = unchecked(12 + 3 * (left.GetHashCode() + 51 * right.GetHashCode())); + } + + public override void BuildParts(ITrapBuilder tb) + { + left.BuildParts(tb); + right.BuildParts(tb); + } + + public override bool Equals(object other) + { + return other is ConsId && Equals((ConsId)other); + } + + public bool Equals(ConsId other) + { + return this == other || + (hash == other.hash && left.Equals(other.left) && right.Equals(other.right)); + } + + public override int GetHashCode() => hash; + + public override string Value => left.Value + right.Value; + + } + + /// + /// A leaf ID storing a string. + /// + public sealed class StringId : Id + { + readonly string value; + public override string Value => value; + + public StringId(string s) + { + value = s; + } + + public override void BuildParts(ITrapBuilder tb) + { + tb.Append(value); + } + + public override bool Equals(object obj) + { + return obj is StringId && ((StringId)obj).value == value; + } + + public override int GetHashCode() => Value.GetHashCode() * 31 + 9; + } + + /// + /// A leaf ID storing an integer. + /// + public sealed class IntId : Id + { + readonly int value; + public override string Value => value.ToString(); + + public IntId(int i) + { + value = i; + } + + public override void BuildParts(ITrapBuilder tb) + { + tb.Append(value); + } + + public override bool Equals(object obj) + { + return obj is IntId && ((IntId)obj).value == value; + } + + public override int GetHashCode() => unchecked(12 + value * 17); + } + + /// + /// Some predefined IDs. + /// + public static class IdUtils + { + public static StringId boolId = new StringId("Boolean"); + public static StringId byteId = new StringId("Byte"); + public static StringId charId = new StringId("Char"); + public static StringId doubleId = new StringId("Double"); + public static StringId shortId = new StringId("Int16"); + public static StringId intId = new StringId("Int32"); + public static StringId longId = new StringId("Int64"); + public static StringId intptrId = new StringId("IntPtr"); + public static StringId objectId = new StringId("Object"); + public static StringId sbyteId = new StringId("SByte"); + public static StringId floatId = new StringId("Single"); + public static StringId stringId = new StringId("String"); + public static StringId ushortId = new StringId("UInt16"); + public static StringId uintId = new StringId("UInt32"); + public static StringId ulongId = new StringId("UInt64"); + public static StringId uintptrId = new StringId("UIntPtr"); + public static StringId voidId = new StringId("Void"); + public static StringId typedReferenceId = new StringId("TypedReference"); + + public static StringId Id(this PrimitiveTypeCode typeCode) + { + switch (typeCode) + { + case PrimitiveTypeCode.Boolean: return boolId; + case PrimitiveTypeCode.Byte: return byteId; + case PrimitiveTypeCode.Char: return charId; + case PrimitiveTypeCode.Double: return doubleId; + case PrimitiveTypeCode.Int16: return shortId; + case PrimitiveTypeCode.Int32: return intId; + case PrimitiveTypeCode.Int64: return longId; + case PrimitiveTypeCode.IntPtr: return intptrId; + case PrimitiveTypeCode.Object: return objectId; + case PrimitiveTypeCode.SByte: return sbyteId; + case PrimitiveTypeCode.Single: return floatId; + case PrimitiveTypeCode.String: return stringId; + case PrimitiveTypeCode.UInt16: return ushortId; + case PrimitiveTypeCode.UInt32: return uintId; + case PrimitiveTypeCode.UInt64: return ulongId; + case PrimitiveTypeCode.UIntPtr: return uintptrId; + case PrimitiveTypeCode.Void: return voidId; + case PrimitiveTypeCode.TypedReference: return typedReferenceId; + default: throw new InternalError("Unhandled type code {0}", typeCode); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/PDB/MetadataPdbReader.cs b/csharp/extractor/Semmle.Extraction.CIL/PDB/MetadataPdbReader.cs new file mode 100644 index 000000000000..33e8460090d8 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/PDB/MetadataPdbReader.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +namespace Semmle.Extraction.PDB +{ + /// + /// A reader of PDB information using System.Reflection.Metadata. + /// This is cross platform, and the future of PDB. + /// + /// PDB information can be in a separate PDB file, or embedded in the DLL. + /// + class MetadataPdbReader : IPdb + { + class SourceFile : ISourceFile + { + public SourceFile(MetadataReader reader, DocumentHandle handle) + { + var doc = reader.GetDocument(handle); + Path = reader.GetString(doc.Name); + } + + public string Path { get; private set; } + + public string Contents => File.Exists(Path) ? File.ReadAllText(Path, System.Text.Encoding.Default) : null; + } + + // Turns out to be very important to keep the MetadataReaderProvider live + // or the reader will crash. + readonly MetadataReaderProvider provider; + readonly MetadataReader reader; + + public MetadataPdbReader(MetadataReaderProvider provider) + { + this.provider = provider; + reader = provider.GetMetadataReader(); + } + + public IEnumerable SourceFiles => reader.Documents.Select(handle => new SourceFile(reader, handle)); + + public IMethod GetMethod(MethodDebugInformationHandle handle) + { + var debugInfo = reader.GetMethodDebugInformation(handle); + + var sequencePoints = debugInfo.GetSequencePoints(). + Where(p => !p.Document.IsNil && !p.IsHidden). + Select(p => new SequencePoint(p.Offset, new Location(new SourceFile(reader, p.Document), p.StartLine, p.StartColumn, p.EndLine, p.EndColumn))). + Where(p => p.Location.File.Path != null). + ToArray(); + + return sequencePoints.Any() ? new Method() { SequencePoints = sequencePoints } : null; + } + + public static MetadataPdbReader CreateFromAssembly(string assemblyPath, PEReader peReader) + { + foreach (var provider in peReader. + ReadDebugDirectory(). + Where(d => d.Type == DebugDirectoryEntryType.EmbeddedPortablePdb). + Select(dirEntry => peReader.ReadEmbeddedPortablePdbDebugDirectoryData(dirEntry))) + { + return new MetadataPdbReader(provider); + } + + try + { + MetadataReaderProvider provider; + string pdbPath; + if (peReader.TryOpenAssociatedPortablePdb(assemblyPath, s => new FileStream(s, FileMode.Open, FileAccess.Read, FileShare.Read), out provider, out pdbPath)) + { + return new MetadataPdbReader(provider); + } + } + + catch (BadImageFormatException) + { + // Something is wrong with the file. + } + catch (FileNotFoundException) + { + // The PDB file was not found. + } + return null; + } + + public void Dispose() + { + provider.Dispose(); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/PDB/NativePdbReader.cs b/csharp/extractor/Semmle.Extraction.CIL/PDB/NativePdbReader.cs new file mode 100644 index 000000000000..0f25b281aab4 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/PDB/NativePdbReader.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection.PortableExecutable; +using Microsoft.DiaSymReader; +using System.Reflection.Metadata.Ecma335; +using System.Reflection.Metadata; +using System.IO; +using System.Reflection; + +namespace Semmle.Extraction.PDB +{ + /// + /// A PDB reader using Microsoft.DiaSymReader.Native. + /// This is an unmanaged Windows DLL, which therefore only works on Windows. + /// + class NativePdbReader : IPdb + { + sealed class Document : ISourceFile + { + readonly ISymUnmanagedDocument document; + + public Document(ISymUnmanagedDocument doc) + { + document = doc; + contents = new Lazy(() => + { + bool isEmbedded; + if (document.HasEmbeddedSource(out isEmbedded) == 0 && isEmbedded) + { + var rawContents = document.GetEmbeddedSource().ToArray(); + return System.Text.Encoding.Default.GetString(rawContents); + } + else + { + return File.Exists(Path) ? File.ReadAllText(Path) : null; + } + }); + } + + public override bool Equals(object obj) + { + var otherDoc = obj as Document; + return otherDoc != null && Path.Equals(otherDoc.Path); + } + + public override int GetHashCode() => Path.GetHashCode(); + + public string Path => document.GetName(); + + public override string ToString() => Path; + + readonly Lazy contents; + + public string Contents => contents.Value; + } + + public IEnumerable SourceFiles => reader.GetDocuments().Select(d => new Document(d)); + + public IMethod GetMethod(MethodDebugInformationHandle h) + { + int methodToken = MetadataTokens.GetToken(h.ToDefinitionHandle()); + var method = reader.GetMethod(methodToken); + if (method != null) + { + int count; + if (method.GetSequencePointCount(out count) != 0 || count == 0) + return null; + + var s = method.GetSequencePoints(). + Where(sp => !sp.IsHidden). + Select(sp => new SequencePoint(sp.Offset, new Location(new Document(sp.Document), sp.StartLine, sp.StartColumn, sp.EndLine, sp.EndColumn))). + ToArray(); + + return s.Any() ? new Method { SequencePoints = s } : null; + } + return null; + } + + NativePdbReader(string path) + { + pdbStream = new FileStream(path, FileMode.Open); + var metadataProvider = new MdProvider(); + reader = SymUnmanagedReaderFactory.CreateReader(pdbStream, metadataProvider); + } + + readonly ISymUnmanagedReader5 reader; + readonly FileStream pdbStream; + + public static NativePdbReader CreateFromAssembly(string assemblyPath, PEReader peReader) + { + // The Native PDB reader uses an unmanaged Windows DLL + // so only works on Windows. + if (!Semmle.Util.Win32.IsWindows()) + return null; + + var debugDirectory = peReader.ReadDebugDirectory(); + + foreach (var path in debugDirectory. + Where(d => d.Type == DebugDirectoryEntryType.CodeView). + Select(peReader.ReadCodeViewDebugDirectoryData). + Select(cv => cv.Path). + Where(path => File.Exists(path))) + { + return new NativePdbReader(path); + } + + return null; + } + + public void Dispose() + { + pdbStream.Dispose(); + } + } + + /// + /// This is not used but is seemingly needed in order to use DiaSymReader. + /// + class MdProvider : ISymReaderMetadataProvider + { + public MdProvider() + { + } + + public object GetMetadataImport() => null; + + public unsafe bool TryGetStandaloneSignature(int standaloneSignatureToken, out byte* signature, out int length) => + throw new NotImplementedException(); + + public bool TryGetTypeDefinitionInfo(int typeDefinitionToken, out string namespaceName, out string typeName, out TypeAttributes attributes, out int baseTypeToken) => + throw new NotImplementedException(); + + public bool TryGetTypeDefinitionInfo(int typeDefinitionToken, out string namespaceName, out string typeName, out TypeAttributes attributes) => + throw new NotImplementedException(); + + public bool TryGetTypeReferenceInfo(int typeReferenceToken, out string namespaceName, out string typeName, out int resolutionScopeToken) => + throw new NotImplementedException(); + + public bool TryGetTypeReferenceInfo(int typeReferenceToken, out string namespaceName, out string typeName) => + throw new NotImplementedException(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/PDB/PdbReader.cs b/csharp/extractor/Semmle.Extraction.CIL/PDB/PdbReader.cs new file mode 100644 index 000000000000..c2f4f94f59f1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/PDB/PdbReader.cs @@ -0,0 +1,151 @@ +using Microsoft.DiaSymReader; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; + +namespace Semmle.Extraction.PDB +{ + /// + /// A sequencepoint is a marker in the source code where you can put a breakpoint, and + /// maps instructions to source code. + /// + public struct SequencePoint + { + /// + /// The byte-offset of the instruction. + /// + public readonly int Offset; + + /// + /// The source location of the instruction. + /// + public readonly Location Location; + + public override string ToString() + { + return string.Format("{0} = {1}", Offset, Location); + } + + public SequencePoint(int offset, Location location) + { + Offset = offset; + Location = location; + } + } + + /// + /// A location in source code. + /// + public sealed class Location + { + /// + /// The file containing the code. + /// + public readonly ISourceFile File; + + /// + /// The span of text within the text file. + /// + public readonly int StartLine, StartColumn, EndLine, EndColumn; + + public override string ToString() + { + return string.Format("({0},{1})-({2},{3})", StartLine, StartColumn, EndLine, EndColumn); + } + + public override bool Equals(object obj) + { + var otherLocation = obj as Location; + + return otherLocation != null && + File.Equals(otherLocation.File) && + StartLine == otherLocation.StartLine && + StartColumn == otherLocation.StartColumn && + EndLine == otherLocation.EndLine && + EndColumn == otherLocation.EndColumn; + } + + public override int GetHashCode() + { + var h1 = StartLine + 37 * (StartColumn + 51 * (EndLine + 97 * EndColumn)); + return File.GetHashCode() + 17 * h1; + } + + public Location(ISourceFile file, int startLine, int startCol, int endLine, int endCol) + { + File = file; + StartLine = startLine; + StartColumn = startCol; + EndLine = endLine; + EndColumn = endCol; + } + } + + public interface IMethod + { + IEnumerable SequencePoints { get; } + Location Location { get; } + } + + class Method : IMethod + { + public IEnumerable SequencePoints { get; set; } + + public Location Location => SequencePoints.First().Location; + } + + /// + /// A source file reference in a PDB file. + /// + public interface ISourceFile + { + string Path { get; } + + /// + /// The contents of the file. + /// This property is needed in case the contents + /// of the file are embedded in the PDB instead of being on the filesystem. + /// + /// null if the contents are unavailable. + /// E.g. if the PDB file exists but the corresponding source files are missing. + /// + string Contents { get; } + } + + /// + /// Wrapper for reading PDB files. + /// This is needed because there are different libraries for dealing with + /// different types of PDB file, even though they share the same file extension. + /// + public interface IPdb : IDisposable + { + /// + /// Gets all source files in this PDB. + /// + IEnumerable SourceFiles { get; } + + /// + /// Look up a method from a given handle. + /// + /// The handle to query. + /// The method information, or null if the method does not have debug information. + IMethod GetMethod(MethodDebugInformationHandle methodHandle); + } + + class PdbReader + { + /// + /// Returns the PDB information associated with an assembly. + /// + /// The path to the assembly. + /// The PE reader for the assembky. + /// A PdbReader, or null if no PDB information is available. + public static IPdb Create(string assemblyPath, PEReader peReader) + { + return (IPdb)MetadataPdbReader.CreateFromAssembly(assemblyPath, peReader) ?? + NativePdbReader.CreateFromAssembly(assemblyPath, peReader); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CIL/Properties/AssemblyInfo.cs new file mode 100644 index 000000000000..110feb8de0d7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.CIL")] +[assembly: AssemblyDescription("Semme CIL extractor.")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Semmle.Extraction.CIL")] +[assembly: AssemblyCopyright("Copyright © Semmle 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("a23d9ec2-8aae-43da-97cb-579f640b89cd")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj b/csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj new file mode 100644 index 000000000000..80808686b4bd --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj @@ -0,0 +1,26 @@ + + + + netcoreapp2.0 + Semmle.Extraction.CIL + Semmle.Extraction.CIL + false + true + + + + + + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.CIL/Tuples.cs b/csharp/extractor/Semmle.Extraction.CIL/Tuples.cs new file mode 100644 index 000000000000..f75adc2963cc --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Tuples.cs @@ -0,0 +1,209 @@ +using Semmle.Extraction.CIL.Entities; + +namespace Semmle.Extraction.CIL +{ + + internal static class Tuples + { + internal static Tuple assemblies(Assembly assembly, File file, string identifier, string name, string version) => + new Tuple("assemblies", assembly, file, identifier, name, version); + + internal static Tuple cil_abstract(IMember method) => + new Tuple("cil_abstract", method); + + internal static Tuple cil_adder(IEvent member, IMethod method) => + new Tuple("cil_adder", member, method); + + internal static Tuple cil_access(IInstruction i, IEntity m) => + new Tuple("cil_access", i, m); + + internal static Tuple cil_attribute(IAttribute attribute, IEntity @object, IMethod constructor) => + new Tuple("cil_attribute", attribute, @object, constructor); + + internal static Tuple cil_attribute_named_argument(IAttribute attribute, string name, string value) => + new Tuple("cil_attribute_named_argument", attribute, name, value); + + internal static Tuple cil_attribute_positional_argument(IAttribute attribute, int index, string value) => + new Tuple("cil_attribute_positional_argument", attribute, index, value); + + internal static Tuple cil_array_type(IArrayType array, IType element, int rank) => + new Tuple("cil_array_type", array, element, rank); + + internal static Tuple cil_base_class(IType t, IType @base) => + new Tuple("cil_base_class", t, @base); + + internal static Tuple cil_base_interface(IType t, IType @base) => + new Tuple("cil_base_interface", t, @base); + + internal static Tuple cil_class(IMember method) => + new Tuple("cil_class", method); + + internal static Tuple cil_event(IEvent e, IType parent, string name, IType type) => + new Tuple("cil_event", e, parent, name, type); + + internal static Tuple cil_field(IField field, IType parent, string name, IType fieldType) => + new Tuple("cil_field", field, parent, name, fieldType); + + internal static Tuple cil_getter(IProperty member, IMethod method) => + new Tuple("cil_getter", member, method); + + internal static Tuple cil_handler(IExceptionRegion region, IMethodImplementation method, int index, int kind, + IInstruction region_start, + IInstruction region_end, + IInstruction handler_start) => + new Tuple("cil_handler", region, method, index, kind, region_start, region_end, handler_start); + + internal static Tuple cil_handler_filter(IExceptionRegion region, IInstruction filter_start) => + new Tuple("cil_handler_filter", region, filter_start); + + internal static Tuple cil_handler_type(IExceptionRegion region, Type t) => + new Tuple("cil_handler_type", region, t); + + internal static Tuple cil_implements(IMethod derived, IMethod declaration) => + new Tuple("cil_implements", derived, declaration); + + internal static Tuple cil_instruction(IInstruction instruction, int opcode, int index, IMethodImplementation parent) => + new Tuple("cil_instruction", instruction, opcode, index, parent); + + internal static Tuple cil_instruction_location(Instruction i, ILocation loc) => + new Tuple("cil_instruction_location", i, loc); + + internal static Tuple cil_interface(IMember method) => + new Tuple("cil_interface", method); + + internal static Tuple cil_internal(IMember modifiable) => + new Tuple("cil_internal", modifiable); + + internal static Tuple cil_jump(IInstruction from, IInstruction to) => + new Tuple("cil_jump", from, to); + + internal static Tuple cil_local_variable(ILocal l, IMethodImplementation m, int i, Type t) => + new Tuple("cil_local_variable", l, m, i, t); + + internal static Tuple cil_method(IMethod method, string name, IType declType, IType returnType) => + new Tuple("cil_method", method, name, declType, returnType); + + internal static Tuple cil_method_implementation(IMethodImplementation impl, IMethod method, IAssembly assembly) => + new Tuple("cil_method_implementation", impl, method, assembly); + + internal static Tuple cil_method_location(IMethod m, ILocation a) => + new Tuple("cil_method_location", m, a); + + internal static Tuple cil_method_source_declaration(IMethod method, IMethod sourceDecl) => + new Tuple("cil_method_source_declaration", method, sourceDecl); + + internal static Tuple cil_method_stack_size(IMethodImplementation method, int stackSize) => + new Tuple("cil_method_stack_size", method, stackSize); + + internal static Tuple cil_newslot(IMethod method) => + new Tuple("cil_newslot", method); + + internal static Tuple cil_parameter(IParameter p, IMethod m, int i, IType t) => + new Tuple("cil_parameter", p, m, i, t); + + internal static Tuple cil_parameter_in(IParameter p) => + new Tuple("cil_parameter_in", p); + + internal static Tuple cil_parameter_out(IParameter p) => + new Tuple("cil_parameter_out", p); + + internal static Tuple cil_pointer_type(IPointerType t, IType pointee) => + new Tuple("cil_pointer_type", t, pointee); + + internal static Tuple cil_private(IMember modifiable) => + new Tuple("cil_private", modifiable); + + internal static Tuple cil_protected(IMember modifiable) => + new Tuple("cil_protected", modifiable); + + internal static Tuple cil_property(IProperty p, IType parent, string name, IType propType) => + new Tuple("cil_property", p, parent, name, propType); + + internal static Tuple cil_public(IMember modifiable) => + new Tuple("cil_public", modifiable); + + internal static Tuple cil_raiser(IEvent member, IMethod method) => + new Tuple("cil_raiser", member, method); + + internal static Tuple cil_requiresecobject(IMethod method) => + new Tuple("cil_requiresecobject", method); + + internal static Tuple cil_remover(IEvent member, IMethod method) => + new Tuple("cil_remover", member, method); + + internal static Tuple cil_sealed(IMember modifiable) => + new Tuple("cil_sealed", modifiable); + + internal static Tuple cil_security(IMember method) => + new Tuple("cil_security", method); + + internal static Tuple cil_setter(IProperty member, IMethod method) => + new Tuple("cil_setter", member, method); + + internal static Tuple cil_specialname(IMethod method) => + new Tuple("cil_specialname", method); + + internal static Tuple cil_static(IMember modifiable) => + new Tuple("cil_static", modifiable); + + internal static Tuple cil_switch(IInstruction from, int index, IInstruction to) => + new Tuple("cil_switch", from, index, to); + + internal static Tuple cil_type(IType t, string name, CilTypeKind kind, ITypeContainer parent, IType sourceDecl) => + new Tuple("cil_type", t, name, (int)kind, parent, sourceDecl); + + internal static Tuple cil_type_argument(ITypeContainer constructedTypeOrMethod, int index, IType argument) => + new Tuple("cil_type_argument", constructedTypeOrMethod, index, argument); + + internal static Tuple cil_type_location(IType t, IAssembly a) => + new Tuple("cil_type_location", t, a); + + internal static Tuple cil_type_parameter(ITypeContainer unboundTypeOrMethod, int index, ITypeParameter parameter) => + new Tuple("cil_type_parameter", unboundTypeOrMethod, index, parameter); + + internal static Tuple cil_typeparam_covariant(ITypeParameter p) => + new Tuple("cil_typeparam_covariant", p); + + internal static Tuple cil_typeparam_contravariant(ITypeParameter p) => + new Tuple("cil_typeparam_contravariant", p); + + internal static Tuple cil_typeparam_class(ITypeParameter p) => + new Tuple("cil_typeparam_class", p); + + internal static Tuple cil_typeparam_constraint(ITypeParameter p, IType constraint) => + new Tuple("cil_typeparam_constraint", p, constraint); + + internal static Tuple cil_typeparam_new(ITypeParameter p) => + new Tuple("cil_typeparam_new", p); + + internal static Tuple cil_typeparam_struct(ITypeParameter p) => + new Tuple("cil_typeparam_struct", p); + + internal static Tuple cil_value(IInstruction i, string value) => + new Tuple("cil_value", i, value); + + internal static Tuple cil_virtual(IMethod method) => + new Tuple("cil_virtual", method); + + internal static Tuple containerparent(IFolder parent, IFileOrFolder child) => + new Tuple("containerparent", parent, child); + + internal static Tuple files(IFile file, string fullName, string name, string extension) => + new Tuple("files", file, fullName, name, extension, 0); + + internal static Tuple file_extraction_mode(IFile file, int mode) => + new Tuple("file_extraction_mode", file, mode); + + internal static Tuple folders(IFolder folder, string path, string name) => + new Tuple("folders", folder, path, name); + + internal static Tuple locations_default(ISourceLocation label, IFile file, int startLine, int startCol, int endLine, int endCol) => + new Tuple("locations_default", label, file, startLine, startCol, endLine, endCol); + + internal static Tuple namespaces(INamespace ns, string name) => + new Tuple("namespaces", ns, name); + + internal static Tuple parent_namespace(ITypeContainer child, INamespace parent) => + new Tuple("parent_namespace", child, parent); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Driver/Driver.cs b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Driver.cs new file mode 100644 index 000000000000..e2892678041f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Driver.cs @@ -0,0 +1,13 @@ +namespace Semmle.Extraction.CSharp +{ + /// + /// A command-line driver for the extractor. + /// + public class Driver + { + static int Main(string[] args) + { + return (int)Extractor.Run(args); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/AssemblyInfo.cs new file mode 100644 index 000000000000..4ef49f38f261 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.CSharp.Driver")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Semmle.Extraction.CSharp.Driver")] +[assembly: AssemblyCopyright("Copyright © 2015")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("c02a2b0e-8884-4b82-8275-ea282403a775")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Driver/Semmle.Extraction.CSharp.Driver.csproj b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Semmle.Extraction.CSharp.Driver.csproj new file mode 100644 index 000000000000..dcaccd3179dd --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Driver/Semmle.Extraction.CSharp.Driver.csproj @@ -0,0 +1,15 @@ + + + + Exe + netcoreapp2.0 + Semmle.Extraction.CSharp.Driver + Semmle.Extraction.CSharp.Driver + false + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs new file mode 100644 index 000000000000..db2664bf4c94 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs @@ -0,0 +1,184 @@ +using System.Collections.Generic; +using System.Linq; +using System.IO; +using System; + +namespace Semmle.BuildAnalyser +{ + /// + /// Manages the set of assemblies. + /// Searches for assembly DLLs, indexes them and provides + /// a lookup facility from assembly ID to filename. + /// + class AssemblyCache + { + /// + /// Locate all reference files and index them. + /// + /// Directories to search. + /// Callback for progress. + public AssemblyCache(IEnumerable dirs, IProgressMonitor progress) + { + foreach (var dir in dirs) + { + progress.FindingFiles(dir); + AddReferenceDirectory(dir); + } + IndexReferences(); + } + + /// + /// Finds all assemblies nested within a directory + /// and adds them to its index. + /// (Indexing is performed at a later stage by IndexReferences()). + /// + /// The directory to index. + /// The number of DLLs within this directory. + int AddReferenceDirectory(string dir) + { + int count = 0; + foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories)) + { + dlls.Add(dll.FullName); + ++count; + } + return count; + } + + /// + /// Indexes all DLLs we have located. + /// Because this is a potentially time-consuming operation, it is put into a separate stage. + /// + void IndexReferences() + { + // Read all of the files + foreach (var filename in dlls) + { + var info = AssemblyInfo.ReadFromFile(filename); + + if (info.Valid) + { + assemblyInfo[filename] = info; + } + else + { + failedDlls.Add(filename); + } + } + + // Index "assemblyInfo" by version string + // The OrderBy is used to ensure that we by default select the highest version number. + foreach (var info in assemblyInfo.Values.OrderBy(info => info.Id)) + { + foreach (var index in info.IndexStrings) + references[index] = info; + } + } + + /// + /// The number of DLLs which are assemblies. + /// + public int AssemblyCount => assemblyInfo.Count; + + /// + /// The number of DLLs which weren't assemblies. (E.g. C++). + /// + public int NonAssemblyCount => failedDlls.Count; + + /// + /// Given an assembly id, determine its full info. + /// + /// The given assembly id. + /// The information about the assembly. + public AssemblyInfo ResolveReference(string id) + { + // Fast path if we've already seen this before. + if (failedReferences.Contains(id)) + return AssemblyInfo.Invalid; + + var query = AssemblyInfo.MakeFromId(id); + id = query.Id; // Sanitise the id. + + // Look up the id in our references map. + AssemblyInfo result; + if (references.TryGetValue(id, out result)) + { + // The string is in the references map. + return result; + } + else + { + // Attempt to load the reference from the GAC. + try + { + var loadedAssembly = System.Reflection.Assembly.ReflectionOnlyLoad(id); + + if (loadedAssembly != null) + { + // The assembly was somewhere we haven't indexed before. + // Add this assembly to our index so that subsequent lookups are faster. + + result = AssemblyInfo.MakeFromAssembly(loadedAssembly); + references[id] = result; + assemblyInfo[loadedAssembly.Location] = result; + return result; + } + } + catch (FileNotFoundException) + { + // A suitable assembly could not be found + } + catch (FileLoadException) + { + // The assembly cannot be loaded for some reason + // e.g. The name is malformed. + } + catch (PlatformNotSupportedException) + { + // .NET Core does not have a GAC. + } + + // Fallback position - locate the assembly by its lower-case name only. + var asmName = query.Name.ToLowerInvariant(); + + if (references.TryGetValue(asmName, out result)) + { + references[asmName] = result; // Speed up the next time the same string is resolved + return result; + } + + failedReferences.Add(id); // Fail early next time + + return AssemblyInfo.Invalid; + } + } + + /// + /// All the assemblies we have indexed. + /// + public IEnumerable AllAssemblies => assemblyInfo.Select(a => a.Value); + + /// + /// Retrieve the assembly info of a pre-cached assembly. + /// + /// The filename to query. + /// The assembly info. + public AssemblyInfo GetAssemblyInfo(string filepath) => assemblyInfo[filepath]; + + // List of pending DLLs to index. + readonly List dlls = new List(); + + // Map from filename to assembly info. + readonly Dictionary assemblyInfo = new Dictionary(); + + // List of DLLs which are not assemblies. + // We probably don't need to keep this + readonly List failedDlls = new List(); + + // Map from assembly id (in various formats) to the full info. + readonly Dictionary references = new Dictionary(); + + // Set of failed assembly ids. + readonly HashSet failedReferences = new HashSet(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyInfo.cs new file mode 100644 index 000000000000..cacc2bca9459 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyInfo.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; + +namespace Semmle.BuildAnalyser +{ + /// + /// Stores information about an assembly file (DLL). + /// + sealed class AssemblyInfo + { + /// + /// The file containing the assembly. + /// + public string Filename { get; private set; } + + /// + /// Was the information correctly determined? + /// + public bool Valid { get; private set; } + + /// + /// The short name of this assembly. + /// + public string Name { get; private set; } + + /// + /// The version number of this assembly. + /// + public System.Version Version { get; private set; } + + /// + /// The public key token of the assembly. + /// + public string PublicKeyToken { get; private set; } + + /// + /// The culture. + /// + public string Culture { get; private set; } + + /// + /// Get/parse a canonical ID of this assembly. + /// + public string Id + { + get + { + var result = Name; + if (Version != null) + result = string.Format("{0}, Version={1}", result, Version); + if (Culture != null) + result = string.Format("{0}, Culture={1}", result, Culture); + if (PublicKeyToken != null) + result = string.Format("{0}, PublicKeyToken={1}", result, PublicKeyToken); + return result; + } + + private set + { + var sections = value.Split(new string[] { ", " }, StringSplitOptions.None); + + Name = sections.First(); + + foreach (var section in sections.Skip(1)) + { + if (section.StartsWith("Version=")) + Version = new Version(section.Substring(8)); + else if (section.StartsWith("Culture=")) + Culture = section.Substring(8); + else if (section.StartsWith("PublicKeyToken=")) + PublicKeyToken = section.Substring(15); + // else: Some other field like processorArchitecture - ignore. + } + + } + } + + public override string ToString() => Id; + + /// + /// Gets a list of canonical search strings for this assembly. + /// + public IEnumerable IndexStrings + { + get + { + yield return Id; + if (Version != null) + { + if (Culture != null) yield return string.Format("{0}, Version={1}, Culture={2}", Name, Version, Culture); + yield return string.Format("{0}, Version={1}", Name, Version); + } + yield return Name; + yield return Name.ToLowerInvariant(); + } + } + + /// + /// Get an invalid assembly info (Valid==false). + /// + public static AssemblyInfo Invalid { get; } = new AssemblyInfo(); + + private AssemblyInfo() { } + + /// + /// Get AssemblyInfo from a loaded Assembly. + /// + /// The assembly. + /// Info about the assembly. + public static AssemblyInfo MakeFromAssembly(Assembly assembly) => new AssemblyInfo() { Valid = true, Filename = assembly.Location, Id = assembly.FullName }; + + /// + /// Parse an assembly name/Id into an AssemblyInfo, + /// populating the available fields and leaving the others null. + /// + /// The assembly name/Id. + /// The deconstructed assembly info. + public static AssemblyInfo MakeFromId(string id) => new AssemblyInfo() { Valid = true, Id = id }; + + /// + /// Reads the assembly info from a file. + /// This uses System.Reflection.Metadata, which is a very performant and low-level + /// library. This is very convenient when scanning hundreds of DLLs at a time. + /// + /// The full filename of the assembly. + /// The information about the assembly. + public static AssemblyInfo ReadFromFile(string filename) + { + var result = new AssemblyInfo() { Filename = filename }; + try + { + /* This method is significantly faster and more lightweight than using + * System.Reflection.Assembly.ReflectionOnlyLoadFrom. It also allows + * loading the same assembly from different locations. + */ + using (var pereader = new System.Reflection.PortableExecutable.PEReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))) + { + var metadata = pereader.GetMetadata(); + unsafe + { + var reader = new System.Reflection.Metadata.MetadataReader(metadata.Pointer, metadata.Length); + var def = reader.GetAssemblyDefinition(); + + // This is how you compute the public key token from the full public key. + // The last 8 bytes of the SHA1 of the public key. + var publicKey = reader.GetBlobBytes(def.PublicKey); + var publicKeyToken = sha1.ComputeHash(publicKey); + var publicKeyString = new StringBuilder(); + foreach (var b in publicKeyToken.Skip(12).Reverse()) + publicKeyString.AppendFormat("{0:x2}", b); + + result.Name = reader.GetString(def.Name); + result.Version = def.Version; + result.Culture = def.Culture.IsNil ? "neutral" : reader.GetString(def.Culture); + result.PublicKeyToken = publicKeyString.ToString(); + result.Valid = true; + } + } + } + catch (BadImageFormatException) + { + // The DLL wasn't an assembly -> result.Valid = false. + } + catch (InvalidOperationException) + { + // Some other failure -> result.Valid = false. + } + + return result; + } + + static readonly SHA1CryptoServiceProvider sha1 = new SHA1CryptoServiceProvider(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs new file mode 100644 index 000000000000..864316ac60b1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs @@ -0,0 +1,312 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Semmle.Util; +using Semmle.Extraction.CSharp.Standalone; + +namespace Semmle.BuildAnalyser +{ + /// + /// The output of a build analysis. + /// + interface IBuildAnalysis + { + /// + /// Full filepaths of external references. + /// + IEnumerable ReferenceFiles { get; } + + /// + /// Full filepaths of C# source files from project files. + /// + IEnumerable ProjectSourceFiles { get; } + + /// + /// Full filepaths of C# source files in the filesystem. + /// + IEnumerable AllSourceFiles { get; } + + /// + /// The assembly IDs which could not be resolved. + /// + IEnumerable UnresolvedReferences { get; } + + /// + /// List of source files referenced by projects but + /// which were not found in the filesystem. + /// + IEnumerable MissingSourceFiles { get; } + } + + /// + /// Main implementation of the build analysis. + /// + class BuildAnalysis : IBuildAnalysis + { + readonly AssemblyCache assemblyCache; + readonly NugetPackages nuget; + readonly IProgressMonitor progressMonitor; + HashSet usedReferences = new HashSet(); + readonly HashSet usedSources = new HashSet(); + readonly HashSet missingSources = new HashSet(); + readonly Dictionary unresolvedReferences = new Dictionary(); + readonly DirectoryInfo sourceDir; + int failedProjects, succeededProjects; + readonly string[] allSources; + int conflictedReferences = 0; + + /// + /// Performs a C# build analysis. + /// + /// Analysis options from the command line. + /// Display of analysis progress. + public BuildAnalysis(Options options, IProgressMonitor progress) + { + progressMonitor = progress; + sourceDir = new DirectoryInfo(options.SrcDir); + + progressMonitor.FindingFiles(options.SrcDir); + + allSources = sourceDir.GetFiles("*.cs", SearchOption.AllDirectories). + Select(d => d.FullName). + Where(d => !options.ExcludesFile(d)). + ToArray(); + + var dllDirNames = options.DllDirs.Select(Path.GetFullPath); + + if (options.UseNuGet) + { + nuget = new NugetPackages(sourceDir.FullName); + ReadNugetFiles(); + dllDirNames = dllDirNames.Concat(Enumerators.Singleton(nuget.PackageDirectory)); + } + + // Find DLLs in the .Net Framework + if (options.ScanNetFrameworkDlls) + { + dllDirNames = dllDirNames.Concat(Runtime.Runtimes.Take(1)); + } + + assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); + + // Analyse all .csproj files in the source tree. + if (options.SolutionFile != null) + { + AnalyseSolution(options.SolutionFile); + } + else if (options.AnalyseCsProjFiles) + { + AnalyseProjectFiles(); + } + + if (!options.AnalyseCsProjFiles) + { + usedReferences = new HashSet(assemblyCache.AllAssemblies.Select(a => a.Filename)); + } + + ResolveConflicts(); + + if (options.UseMscorlib) + { + UseReference(typeof(object).Assembly.Location); + } + + // Output the findings + foreach (var r in usedReferences) + { + progressMonitor.ResolvedReference(r); + } + + foreach (var r in unresolvedReferences) + { + progressMonitor.UnresolvedReference(r.Key, r.Value); + } + + progressMonitor.Summary( + AllSourceFiles.Count(), + ProjectSourceFiles.Count(), + MissingSourceFiles.Count(), + ReferenceFiles.Count(), + UnresolvedReferences.Count(), + conflictedReferences, + succeededProjects + failedProjects, + failedProjects); + } + + /// + /// Resolves conflicts between all of the resolved references. + /// If the same assembly name is duplicated with different versions, + /// resolve to the higher version number. + /// + void ResolveConflicts() + { + var sortedReferences = usedReferences. + Select(r => assemblyCache.GetAssemblyInfo(r)). + OrderBy(r => r.Version). + ToArray(); + + Dictionary finalAssemblyList = new Dictionary(); + + // Pick the highest version for each assembly name + foreach (var r in sortedReferences) + finalAssemblyList[r.Name] = r; + + // Update the used references list + usedReferences = new HashSet(finalAssemblyList.Select(r => r.Value.Filename)); + + // Report the results + foreach (var r in sortedReferences) + { + var resolvedInfo = finalAssemblyList[r.Name]; + if (resolvedInfo.Version != r.Version) + { + progressMonitor.ResolvedConflict(r.Id, resolvedInfo.Id); + ++conflictedReferences; + } + } + } + + /// + /// Find and restore NuGet packages. + /// + void ReadNugetFiles() + { + nuget.FindPackages(); + nuget.InstallPackages(progressMonitor); + } + + /// + /// Store that a particular reference file is used. + /// + /// The filename of the reference. + void UseReference(string reference) + { + usedReferences.Add(reference); + } + + /// + /// Store that a particular source file is used (by a project file). + /// + /// The source file. + void UseSource(FileInfo sourceFile) + { + if (sourceFile.Exists) + { + usedSources.Add(sourceFile.FullName); + } + else + { + missingSources.Add(sourceFile.FullName); + } + } + + /// + /// The list of resolved reference files. + /// + public IEnumerable ReferenceFiles => this.usedReferences; + + /// + /// The list of source files used in projects. + /// + public IEnumerable ProjectSourceFiles => usedSources; + + /// + /// All of the source files in the source directory. + /// + public IEnumerable AllSourceFiles => allSources; + + /// + /// List of assembly IDs which couldn't be resolved. + /// + public IEnumerable UnresolvedReferences => this.unresolvedReferences.Select(r => r.Key); + + /// + /// List of source files which were mentioned in project files but + /// do not exist on the file system. + /// + public IEnumerable MissingSourceFiles => missingSources; + + /// + /// Record that a particular reference couldn't be resolved. + /// Note that this records at most one project file per missing reference. + /// + /// The assembly ID. + /// The project file making the reference. + void UnresolvedReference(string id, string projectFile) + { + unresolvedReferences[id] = projectFile; + } + + /// + /// Performs an analysis of all .csproj files. + /// + void AnalyseProjectFiles() + { + AnalyseProjectFiles(sourceDir.GetFiles("*.csproj", SearchOption.AllDirectories)); + } + + /// + /// Reads all the source files and references from the given list of projects. + /// + /// The list of projects to analyse. + void AnalyseProjectFiles(FileInfo[] projectFiles) + { + progressMonitor.AnalysingProjectFiles(projectFiles.Count()); + + foreach (var proj in projectFiles) + { + try + { + var csProj = new CsProjFile(proj); + + foreach (var @ref in csProj.References) + { + AssemblyInfo resolved = assemblyCache.ResolveReference(@ref); + if (!resolved.Valid) + { + UnresolvedReference(@ref, proj.FullName); + } + else + { + UseReference(resolved.Filename); + } + } + + foreach (var src in csProj.Sources) + { + // Make a note of which source files the projects use. + // This information doesn't affect the build but is dumped + // as diagnostic output. + UseSource(new FileInfo(src)); + } + ++succeededProjects; + } + catch (Exception ex) + { + ++failedProjects; + progressMonitor.FailedProjectFile(proj.FullName, ex.Message); + } + } + } + + /// + /// Delete packages directory. + /// + public void Cleanup() + { + if (nuget != null) nuget.Cleanup(progressMonitor); + } + + /// + /// Analyse all project files in a given solution only. + /// + /// The filename of the solution. + public void AnalyseSolution(string solutionFile) + { + var sln = new SolutionFile(solutionFile); + AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).ToArray()); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs new file mode 100644 index 000000000000..f866a435180c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/CsProjFile.cs @@ -0,0 +1,120 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml; + +namespace Semmle.BuildAnalyser +{ + /// + /// Represents a .csproj file and reads information from it. + /// + class CsProjFile + { + /// + /// Reads the .csproj file. + /// + /// The .csproj file. + public CsProjFile(FileInfo filename) + { + try + { + // This can fail if the .csproj is invalid or has + // unrecognised content or is the wrong version. + // This currently always fails on Linux because + // Microsoft.Build is not cross platform. + ReadMsBuildProject(filename); + } + catch + { + // There was some reason why the project couldn't be loaded. + // Fall back to reading the Xml document directly. + // This method however doesn't handle variable expansion. + ReadProjectFileAsXml(filename); + } + } + + /// + /// Read the .csproj file using Microsoft Build. + /// This occasionally fails if the project file is incompatible for some reason, + /// and there seems to be no way to make it succeed. Fails on Linux. + /// + /// The file to read. + public void ReadMsBuildProject(FileInfo filename) + { + var msbuildProject = new Microsoft.Build.Execution.ProjectInstance(filename.FullName); + + references = msbuildProject. + Items. + Where(item => item.ItemType == "Reference"). + Select(item => item.EvaluatedInclude). + ToArray(); + + csFiles = msbuildProject.Items + .Where(item => item.ItemType == "Compile") + .Select(item => item.GetMetadataValue("FullPath")) + .Where(fn => fn.EndsWith(".cs")) + .ToArray(); + } + + /// + /// Reads the .csproj file directly as XML. + /// This doesn't handle variables etc, and should only used as a + /// fallback if ReadMsBuildProject() fails. + /// + /// The .csproj file. + public void ReadProjectFileAsXml(FileInfo filename) + { + var projFile = new XmlDocument(); + var mgr = new XmlNamespaceManager(projFile.NameTable); + mgr.AddNamespace("msbuild", "http://schemas.microsoft.com/developer/msbuild/2003"); + projFile.Load(filename.FullName); + var projDir = filename.Directory; + var root = projFile.DocumentElement; + + references = + root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Reference/@Include", mgr). + NodeList(). + Select(node => node.Value). + ToArray(); + + var relativeCsIncludes = + root.SelectNodes("/msbuild:Project/msbuild:ItemGroup/msbuild:Compile/@Include", mgr). + NodeList(). + Select(node => node.Value). + ToArray(); + + csFiles = relativeCsIncludes. + Select(cs => Path.DirectorySeparatorChar == '/' ? cs.Replace("\\", "/") : cs). + Select(f => Path.GetFullPath(Path.Combine(projDir.FullName, f))). + ToArray(); + } + + string[] references; + string[] csFiles; + + /// + /// The list of references as a list of assembly IDs. + /// + public IEnumerable References => references; + + /// + /// The list of C# source files in full path format. + /// + public IEnumerable Sources => csFiles; + } + + static class XmlNodeHelper + { + /// + /// Helper to convert an XmlNodeList into an IEnumerable. + /// This allows it to be used with Linq. + /// + /// The list to convert. + /// A more useful data type. + public static IEnumerable NodeList(this XmlNodeList list) + { + foreach (var i in list) + yield return i as XmlNode; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs new file mode 100644 index 000000000000..067f0a84cfe1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; + +namespace Semmle.BuildAnalyser +{ + /// + /// Manage the downloading of NuGet packages. + /// Locates packages in a source tree and downloads all of the + /// referenced assemblies to a temp folder. + /// + class NugetPackages + { + /// + /// Create the package manager for a specified source tree. + /// + /// The source directory. + public NugetPackages(string sourceDir) + { + SourceDirectory = sourceDir; + PackageDirectory = computeTempDirectory(sourceDir); + + // Expect nuget.exe to be in a `nuget` directory under the directory containing this exe. + var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location; + nugetExe = Path.Combine(Path.GetDirectoryName(currentAssembly), "nuget", "nuget.exe"); + + if (!File.Exists(nugetExe)) + throw new FileNotFoundException(string.Format("NuGet could not be found at {0}", nugetExe)); + } + + /// + /// Locate all NuGet packages but don't download them yet. + /// + public void FindPackages() + { + packages = new DirectoryInfo(SourceDirectory). + EnumerateFiles("packages.config", SearchOption.AllDirectories). + ToArray(); + } + + // List of package files to download. + FileInfo[] packages; + + /// + /// The list of package files. + /// + public IEnumerable PackageFiles => packages; + + // Whether to delete the packages directory prior to each run. + // Makes each build more reproducible. + const bool cleanupPackages = true; + + public void Cleanup(IProgressMonitor pm) + { + var packagesDirectory = new DirectoryInfo(PackageDirectory); + + if (packagesDirectory.Exists) + { + try + { + packagesDirectory.Delete(true); + } + catch (System.IO.IOException ex) + { + pm.Warning(string.Format("Couldn't delete package directory - it's probably held open by something else: {0}", ex.Message)); + } + } + } + + /// + /// Download the packages to the temp folder. + /// + /// The progress monitor used for reporting errors etc. + public void InstallPackages(IProgressMonitor pm) + { + if (cleanupPackages) + { + Cleanup(pm); + } + + var packagesDirectory = new DirectoryInfo(PackageDirectory); + + if (!Directory.Exists(PackageDirectory)) + { + packagesDirectory.Create(); + } + + foreach (var package in packages) + { + RestoreNugetPackage(package.FullName, pm); + } + } + + /// + /// The source directory used. + /// + public string SourceDirectory + { + get; + private set; + } + + /// + /// The computed packages directory. + /// This will be in the Temp location + /// so as to not trample the source tree. + /// + public string PackageDirectory + { + get; + private set; + } + + readonly SHA1CryptoServiceProvider sha1 = new SHA1CryptoServiceProvider(); + + /// + /// Computes a unique temp directory for the packages associated + /// with this source tree. Use a SHA1 of the directory name. + /// + /// + /// The full path of the temp directory. + string computeTempDirectory(string srcDir) + { + var bytes = Encoding.Unicode.GetBytes(srcDir); + + var sha = sha1.ComputeHash(bytes); + var sb = new StringBuilder(); + foreach (var b in sha.Take(8)) + sb.AppendFormat("{0:x2}", b); + + return Path.Combine(Path.GetTempPath(), "Semmle", "packages", sb.ToString()); + } + + /// + /// Restore all files in a specified package. + /// + /// The package file. + /// Where to log progress/errors. + void RestoreNugetPackage(string package, IProgressMonitor pm) + { + pm.NugetInstall(package); + + /* Use nuget.exe to install a package. + * Note that there is a clutch of NuGet assemblies which could be used to + * invoke this directly, which would arguably be nicer. However they are + * really unwieldy and this solution works for now. + */ + + string exe, args; + if (Util.Win32.IsWindows()) + { + exe = nugetExe; + args = string.Format("install -OutputDirectory {0} {1}", PackageDirectory, package); + } + else + { + exe = "mono"; + args = string.Format("{0} install -OutputDirectory {1} {2}", nugetExe, PackageDirectory, package); + } + + var pi = new ProcessStartInfo(exe, args) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + try + { + using (var p = Process.Start(pi)) + { + string output = p.StandardOutput.ReadToEnd(); + string error = p.StandardError.ReadToEnd(); + + p.WaitForExit(); + if (p.ExitCode != 0) + { + pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error); + } + } + } + catch (Exception e) + when (e is System.ComponentModel.Win32Exception || e is FileNotFoundException) + { + pm.FailedNugetCommand(pi.FileName, pi.Arguments, e.Message); + } + } + + readonly string nugetExe; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs new file mode 100644 index 000000000000..89cc18007f0e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs @@ -0,0 +1,178 @@ +using Semmle.Util.Logging; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp.Standalone +{ + /// + /// The options controlling standalone extraction. + /// + public sealed class Options : CommonOptions + { + public override bool handleFlag(string key, bool value) + { + switch(key) + { + case "silent": + Verbosity = value ? Verbosity.Off : Verbosity.Info; + return true; + case "help": + Help = true; + return true; + case "dry-run": + SkipExtraction = value; + return true; + case "skip-nuget": + UseNuGet = !value; + return true; + case "all-references": + AnalyseCsProjFiles = !value; + return true; + case "stdlib": + UseMscorlib = value; + return true; + case "skip-dotnet": + ScanNetFrameworkDlls = !value; + return true; + default: + return base.handleFlag(key, value); + } + } + + public override bool handleOption(string key, string value) + { + switch(key) + { + case "exclude": + Excludes.Add(value); + return true; + case "references": + DllDirs.Add(value); + return true; + default: + return base.handleOption(key, value); + } + } + + public override bool handleArgument(string arg) + { + SolutionFile = arg; + var fi = new FileInfo(SolutionFile); + if (!fi.Exists) + { + System.Console.WriteLine("Error: The solution {0} does not exist", fi.FullName); + Errors = true; + } + return true; + } + + public override void invalidArgument(string argument) + { + System.Console.WriteLine($"Error: Invalid argument {argument}"); + Errors = true; + } + + /// + /// Files/patterns to exclude. + /// + public IList Excludes = new List(); + + /// + /// The number of concurrent threads to use. + /// + public int NumberOfThreads = Semmle.Extraction.Extractor.DefaultNumberOfThreads; + + /// + /// The directory containing the source code; + /// + public readonly string SrcDir = System.IO.Directory.GetCurrentDirectory(); + + /// + /// Whether to analyse NuGet packages. + /// + public bool UseNuGet = true; + + /// + /// Directories to search DLLs in. + /// + public IList DllDirs = new List(); + + /// + /// Whether to search the .Net framework directory. + /// + public bool ScanNetFrameworkDlls = true; + + /// + /// Whether to use mscorlib as a reference. + /// + public bool UseMscorlib = true; + + /// + /// Whether to search .csproj files. + /// + public bool AnalyseCsProjFiles = true; + + /// + /// The solution file to analyse, or null if not specified. + /// + public string SolutionFile; + + /// + /// Whether the extraction phase should be skipped (dry-run). + /// + public bool SkipExtraction = false; + + /// + /// Whether errors were encountered parsing the arguments. + /// + public bool Errors = false; + + /// + /// Whether to show help. + /// + public bool Help = false; + + /// + /// Determine whether the given path should be excluded. + /// + /// The path to query. + /// True iff the path matches an exclusion. + public bool ExcludesFile(string path) + { + return Excludes.Any(ex => path.Contains(ex)); + } + + /// + /// Outputs the command line options to the console. + /// + public void ShowHelp(System.IO.TextWriter output) + { + output.WriteLine("C# standalone extractor\n\nExtracts a C# project in the current directory without performing a build.\n"); + output.WriteLine("Additional options:\n"); + output.WriteLine(" xxx.sln Restrict sources to given solution"); + output.WriteLine(" --exclude:xxx Exclude a file or directory (can be specified multiple times)"); + output.WriteLine(" --references:xxx Scan additional files or directories for assemblies (can be specified multiple times)"); + output.WriteLine(" --skip-dotnet Do not reference the .Net Framework"); + output.WriteLine(" --dry-run Stop before extraction"); + output.WriteLine(" --skip-nuget Do not download nuget packages"); + output.WriteLine(" --all-references Use all references (default is to only use references in .csproj files)"); + output.WriteLine(" --nostdlib Do not link mscorlib.dll (use only for extracting mscorlib itself)"); + output.WriteLine(" --threads:nnn Specify number of threads (default=CPU cores)"); + output.WriteLine(" --verbose Produce more output"); + output.WriteLine(" --pdb Cross-reference information from PDBs where available"); + } + + private Options() + { + } + + public static Options Create(string[] args) + { + var options = new Options(); + options.ParseArguments(args); + return options; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs new file mode 100644 index 000000000000..e0367fa63c13 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs @@ -0,0 +1,158 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Semmle.BuildAnalyser; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CSharp.Standalone +{ + /// + /// One independent run of the extractor. + /// + class Extraction + { + public Extraction(string directory) + { + this.directory = directory; + } + + public readonly string directory; + public readonly List Sources = new List(); + }; + + /// + /// Searches for source/references and creates separate extractions. + /// + class Analysis + { + readonly ILogger logger; + + public Analysis(ILogger logger) + { + this.logger = logger; + } + + // The extraction configuration for the entire project. + Extraction projectExtraction; + + public IEnumerable References + { + get; private set; + } + + /// + /// The extraction configuration. + /// + public Extraction Extraction => projectExtraction; + + /// + /// Creates an extraction for the current directory + /// and adds it to the list of all extractions. + /// + /// The directory of the extraction. + /// The extraction. + void CreateExtraction(string dir) + { + projectExtraction = new Extraction(dir); + } + + BuildAnalysis buildAnalysis; + + /// + /// Analyse projects/solution and resolves references. + /// + /// The build analysis options. + public void AnalyseProjects(Options options) + { + CreateExtraction(options.SrcDir); + var progressMonitor = new ProgressMonitor(logger); + buildAnalysis = new BuildAnalysis(options, progressMonitor); + References = buildAnalysis.ReferenceFiles; + projectExtraction.Sources.AddRange(options.SolutionFile == null ? buildAnalysis.AllSourceFiles : buildAnalysis.ProjectSourceFiles); + } + + /// + /// Delete any Nuget assemblies. + /// + public void Cleanup() + { + buildAnalysis.Cleanup(); + } + }; + + public class Program + { + static int Main(string[] args) + { + var options = Options.Create(args); + var output = new ConsoleLogger(options.Verbosity); + var a = new Analysis(output); + + if (options.Help) + { + options.ShowHelp(System.Console.Out); + return 0; + } + + if (options.Errors) + return 1; + + output.Log(Severity.Info, "Running C# standalone extractor"); + a.AnalyseProjects(options); + int sourceFiles = a.Extraction.Sources.Count(); + + if (sourceFiles == 0) + { + output.Log(Severity.Error, "No source files found"); + return 1; + } + + if (!options.SkipExtraction) + { + output.Log(Severity.Info, ""); + output.Log(Severity.Info, "Extracting..."); + Extractor.ExtractStandalone( + a.Extraction.Sources, + a.References, + new ExtractionProgress(output), + new FileLogger(options.Verbosity, Extractor.GetCSharpLogPath()), + options); + output.Log(Severity.Info, "Extraction complete"); + } + + a.Cleanup(); + return 0; + } + + class ExtractionProgress : IProgressMonitor + { + public ExtractionProgress(ILogger output) + { + logger = output; + } + + readonly ILogger logger; + + public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action) + { + logger.Log(Severity.Info, "[{0}/{1}] {2} ({3})", item, total, source, + action == AnalysisAction.Extracted ? time.ToString() : action == AnalysisAction.Excluded ? "excluded" : "up to date"); + } + + public void MissingType(string type) + { + logger.Log(Severity.Debug, "Missing type {0}", type); + } + + public void MissingNamespace(string @namespace) + { + logger.Log(Severity.Info, "Missing namespace {0}", @namespace); + } + + public void MissingSummary(int missingTypes, int missingNamespaces) + { + logger.Log(Severity.Info, "Failed to resolve {0} types and {1} namespaces", missingTypes, missingNamespaces); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs new file mode 100644 index 000000000000..f4bde55ec557 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs @@ -0,0 +1,105 @@ +using Semmle.Util.Logging; + +namespace Semmle.BuildAnalyser +{ + /// + /// Callback for various events that may happen during the build analysis. + /// + interface IProgressMonitor + { + void FindingFiles(string dir); + void UnresolvedReference(string id, string project); + void AnalysingProjectFiles(int count); + void FailedProjectFile(string filename, string reason); + void FailedNugetCommand(string exe, string args, string message); + void NugetInstall(string package); + void ResolvedReference(string filename); + void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects); + void Warning(string message); + void ResolvedConflict(string asm1, string asm2); + void MissingProject(string projectFile); + } + + class ProgressMonitor : IProgressMonitor + { + readonly ILogger logger; + + public ProgressMonitor(ILogger logger) + { + this.logger = logger; + } + + public void FindingFiles(string dir) + { + logger.Log(Severity.Info, "Finding files in {0}...", dir); + } + + public void IndexingReferences(int count) + { + logger.Log(Severity.Info, "Indexing..."); + logger.Log(Severity.Debug, "Indexing {0} DLLs...", count); + } + + public void UnresolvedReference(string id, string project) + { + logger.Log(Severity.Info, "Unresolved reference {0}", id); + logger.Log(Severity.Debug, "Unresolved {0} referenced by {1}", id, project); + } + + public void AnalysingProjectFiles(int count) + { + logger.Log(Severity.Info, "Analyzing project files..."); + } + + public void FailedProjectFile(string filename, string reason) + { + logger.Log(Severity.Info, "Couldn't read project file {0}: {1}", filename, reason); + } + + public void FailedNugetCommand(string exe, string args, string message) + { + logger.Log(Severity.Info, "Command failed: {0} {1}", exe, args); + logger.Log(Severity.Info, " {0}", message); + } + + public void NugetInstall(string package) + { + logger.Log(Severity.Info, "Restoring {0}...", package); + } + + public void ResolvedReference(string filename) + { + logger.Log(Severity.Info, "Resolved {0}", filename); + } + + public void Summary(int existingSources, int usedSources, int missingSources, + int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects) + { + logger.Log(Severity.Info, ""); + logger.Log(Severity.Info, "Build analysis summary:"); + logger.Log(Severity.Info, "{0, 6} source files in the filesystem", existingSources); + logger.Log(Severity.Info, "{0, 6} source files in project files", usedSources); + logger.Log(Severity.Info, "{0, 6} sources missing from project files", missingSources); + logger.Log(Severity.Info, "{0, 6} resolved references", references); + logger.Log(Severity.Info, "{0, 6} unresolved references", unresolvedReferences); + logger.Log(Severity.Info, "{0, 6} resolved assembly conflicts", resolvedConflicts); + logger.Log(Severity.Info, "{0, 6} projects", totalProjects); + logger.Log(Severity.Info, "{0, 6} missing/failed projects", failedProjects); + } + + public void Warning(string message) + { + logger.Log(Severity.Warning, message); + } + + public void ResolvedConflict(string asm1, string asm2) + { + logger.Log(Severity.Info, "Resolved {0} as {1}", asm1, asm2); + } + + public void MissingProject(string projectFile) + { + logger.Log(Severity.Info, "Solution is missing {0}", projectFile); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Properties/AssemblyInfo.cs new file mode 100644 index 000000000000..4c5502fcde5d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.CSharp.Standalone")] +[assembly: AssemblyDescription("Standalone extractor for C#")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Semmle Ltd.")] +[assembly: AssemblyProduct("Semmle.Extraction.CSharp.Standalone")] +[assembly: AssemblyCopyright("Copyright © Semmle 2016")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("bb71e9da-7e0a-43e8-989c-c8e87c828e7c")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs new file mode 100644 index 000000000000..c226f8dd6cbf --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Runtime.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.IO; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Standalone +{ + /// + /// Locates .NET Runtimes. + /// + static class Runtime + { + static string ExecutingRuntime => RuntimeEnvironment.GetRuntimeDirectory(); + + /// + /// Locates .NET Core Runtimes. + /// + public static IEnumerable CoreRuntimes + { + get + { + string[] dotnetDirs = { "/usr/share/dotnet", @"C:\Program Files\dotnet" }; + + foreach (var dir in dotnetDirs.Where(Directory.Exists)) + return Directory.EnumerateDirectories(Path.Combine(dir, "shared", "Microsoft.NETCore.App")). + OrderByDescending(d => Path.GetFileName(d)); + return Enumerable.Empty(); + } + } + + /// + /// Locates .NET Desktop Runtimes. + /// This includes Mono and Microsoft.NET. + /// + public static IEnumerable DesktopRuntimes + { + get + { + string[] monoDirs = { "/usr/lib/mono", @"C:\Program Files\Mono\lib\mono" }; + + if (Directory.Exists(@"C:\Windows\Microsoft.NET\Framework64")) + { + return System.IO.Directory.EnumerateDirectories(@"C:\Windows\Microsoft.NET\Framework64", "v*"). + OrderByDescending(d => Path.GetFileName(d)); + } + + foreach (var dir in monoDirs.Where(Directory.Exists)) + { + return System.IO.Directory.EnumerateDirectories(dir). + Where(d => Char.IsDigit(Path.GetFileName(d)[0])). + OrderByDescending(d => Path.GetFileName(d)); + } + + return Enumerable.Empty(); + } + } + + public static IEnumerable Runtimes + { + get + { + foreach (var r in CoreRuntimes) + yield return r; + + foreach (var r in DesktopRuntimes) + yield return r; + + // A bad choice if it's the self-contained runtime distributed in odasa dist. + yield return ExecutingRuntime; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj new file mode 100644 index 000000000000..7b08c92c7db0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Semmle.Extraction.CSharp.Standalone.csproj @@ -0,0 +1,30 @@ + + + + Exe + netcoreapp2.0 + Semmle.Extraction.CSharp.Standalone + Semmle.Extraction.CSharp.Standalone + false + true + false + + + + + + + + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs new file mode 100644 index 000000000000..b1a3edd4cf6d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/SolutionFile.cs @@ -0,0 +1,68 @@ +using Microsoft.Build.Construction; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Semmle.BuildAnalyser +{ + /// + /// Access data in a .sln file. + /// + class SolutionFile + { + readonly Microsoft.Build.Construction.SolutionFile solutionFile; + + /// + /// Read the file. + /// + /// The filename of the .sln. + public SolutionFile(string filename) + { + // SolutionFile.Parse() expects a rooted path. + var fullPath = Path.GetFullPath(filename); + solutionFile = Microsoft.Build.Construction.SolutionFile.Parse(fullPath); + } + + /// + /// Projects directly included in the .sln file. + /// + public IEnumerable MsBuildProjects + { + get + { + return solutionFile.ProjectsInOrder. + Where(p => p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat). + Select(p => p.AbsolutePath). + Select(p => Path.DirectorySeparatorChar == '/' ? p.Replace("\\", "/") : p); + } + } + + /// + /// Projects included transitively via a subdirectory. + /// + public IEnumerable NestedProjects + { + get + { + return solutionFile.ProjectsInOrder. + Where(p => p.ProjectType == SolutionProjectType.SolutionFolder). + Where(p => Directory.Exists(p.AbsolutePath)). + SelectMany(p => new DirectoryInfo(p.AbsolutePath).EnumerateFiles("*.csproj", SearchOption.AllDirectories)). + Select(f => f.FullName); + } + } + + /// + /// List of projects which were mentioned but don't exist on disk. + /// + public IEnumerable MissingProjects => + // Only projects in the solution file can be missing. + // (NestedProjects are located on disk so always exist.) + MsBuildProjects.Where(p => !File.Exists(p)); + + /// + /// The list of project files. + /// + public IEnumerable Projects => MsBuildProjects.Concat(NestedProjects); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs new file mode 100644 index 000000000000..7b05a7bca0d0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs @@ -0,0 +1,505 @@ +using System; +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Diagnostics; +using System.IO; +using System.Linq; +using Semmle.Extraction.CSharp.Populators; +using System.Runtime.InteropServices; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using System.Diagnostics; +using Semmle.Util.Logging; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp +{ + /// + /// Encapsulates a C# analysis task. + /// + public class Analyser : IDisposable + { + IExtractor extractor; + + readonly Stopwatch stopWatch = new Stopwatch(); + + readonly IProgressMonitor progressMonitor; + + public readonly ILogger Logger; + + public Analyser(IProgressMonitor pm, ILogger logger) + { + Logger = logger; + Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now); + stopWatch.Start(); + progressMonitor = pm; + } + + CSharpCompilation compilation; + Layout layout; + + /// + /// Initialize the analyser. + /// + /// Arguments passed to csc. + /// The Roslyn compilation. + /// Extractor options. + public void Initialize( + CSharpCommandLineArguments commandLineArguments, + CSharpCompilation compilationIn, + Options options) + { + compilation = compilationIn; + + layout = new Layout(); + this.options = options; + + extractor = new Extraction.Extractor(false, GetOutputName(compilation, commandLineArguments), Logger); + + LogDiagnostics(); + SetReferencePaths(); + + CompilationErrors += FilteredDiagnostics.Count(); + } + + /// + /// Constructs the map from assembly string to its filename. + /// + /// Roslyn doesn't record the relationship between a filename and its assembly + /// information, so we need to retrieve this information manually. + /// + void SetReferencePaths() + { + foreach (var reference in compilation.References.OfType()) + { + try + { + var refPath = reference.FilePath; + + /* This method is significantly faster and more lightweight than using + * System.Reflection.Assembly.ReflectionOnlyLoadFrom. It is also allows + * loading the same assembly from different locations. + */ + using (var pereader = new System.Reflection.PortableExecutable.PEReader(new FileStream(refPath, FileMode.Open, FileAccess.Read, FileShare.Read))) + { + var metadata = pereader.GetMetadata(); + string assemblyIdentity; + unsafe + { + var reader = new System.Reflection.Metadata.MetadataReader(metadata.Pointer, metadata.Length); + var def = reader.GetAssemblyDefinition(); + assemblyIdentity = reader.GetString(def.Name) + " " + def.Version; + } + extractor.SetAssemblyFile(assemblyIdentity, refPath); + } + } + catch (Exception ex) + { + extractor.Message(new Message + { + exception = ex, + message = string.Format("Exception reading reference file {0}: {1}", + reference.FilePath, ex) + }); + } + } + } + + public void InitializeStandalone(CSharpCompilation compilationIn, CommonOptions options) + { + compilation = compilationIn; + layout = new Layout(); + extractor = new Extraction.Extractor(true, null, Logger); + this.options = options; + LogDiagnostics(); + SetReferencePaths(); + } + + readonly HashSet errorsToIgnore = new HashSet + { + "CS7027", // Code signing failure + "CS1589", // XML referencing not supported + "CS1569" // Error writing XML documentation + }; + + IEnumerable FilteredDiagnostics + { + get + { + return extractor == null || extractor.Standalone || compilation == null ? Enumerable.Empty() : + compilation. + GetDiagnostics(). + Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id)); + } + } + + public IEnumerable MissingTypes => extractor.MissingTypes; + + public IEnumerable MissingNamespaces => extractor.MissingNamespaces; + + /// + /// Determine the path of the output dll/exe. + /// + /// Information about the compilation. + /// Cancellation token required. + /// The filename. + static string GetOutputName(CSharpCompilation compilation, + CSharpCommandLineArguments commandLineArguments) + { + // There's no apparent way to access the output filename from the compilation, + // so we need to re-parse the command line arguments. + + if (commandLineArguments.OutputFileName == null) + { + // No output specified: Use name based on first filename + var entry = compilation.GetEntryPoint(System.Threading.CancellationToken.None); + if (entry == null) + { + if (compilation.SyntaxTrees.Length == 0) + throw new ArgumentNullException("No source files seen"); + + // Probably invalid, but have a go anyway. + var entryPointFile = compilation.SyntaxTrees.First().FilePath; + return Path.ChangeExtension(entryPointFile, ".exe"); + } + else + { + var entryPointFilename = entry.Locations.First().SourceTree.FilePath; + return Path.ChangeExtension(entryPointFilename, ".exe"); + } + } + else + { + return Path.Combine(commandLineArguments.OutputDirectory, commandLineArguments.OutputFileName); + } + } + + /// + /// Perform an analysis on a source file/syntax tree. + /// + /// Syntax tree to analyse. + public void AnalyseTree(SyntaxTree tree) + { + extractionTasks.Add(() => DoExtractTree(tree)); + } + + /// + /// Perform an analysis on an assembly. + /// + /// Assembly to analyse. + void AnalyseAssembly(PortableExecutableReference assembly) + { + // CIL first - it takes longer. + if (options.CIL) + extractionTasks.Add(() => DoExtractCIL(assembly)); + extractionTasks.Add(() => DoAnalyseAssembly(assembly)); + } + + readonly object progressMutex = new object(); + int taskCount = 0; + + CommonOptions options; + + static bool FileIsUpToDate(string src, string dest) + { + return File.Exists(dest) && + File.GetLastWriteTime(dest) >= File.GetLastWriteTime(src); + } + + bool FileIsCached(string src, string dest) + { + return options.Cache && FileIsUpToDate(src, dest); + } + + /// + /// Extract an assembly to a new trap file. + /// If the trap file exists, skip extraction to avoid duplicating + /// extraction within the snapshot. + /// + /// The assembly to extract. + void DoAnalyseAssembly(PortableExecutableReference r) + { + try + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + var assemblyPath = r.FilePath; + var projectLayout = layout.LookupProjectOrDefault(assemblyPath); + using (var trapWriter = projectLayout.CreateTrapWriter(Logger, assemblyPath, true)) + { + var skipExtraction = FileIsCached(assemblyPath, trapWriter.TrapFile); + + if (!skipExtraction) + { + /* Note on parallel builds: + * + * The trap writer and source archiver both perform atomic moves + * of the file to the final destination. + * + * If the same source file or trap file are generated concurrently + * (by different parallel invocations of the extractor), then + * last one wins. + * + * Specifically, if two assemblies are analysed concurrently in a build, + * then there is a small amount of duplicated work but the output should + * still be correct. + */ + + // compilation.Clone() reduces memory footprint by allowing the symbols + // in c to be garbage collected. + Compilation c = compilation.Clone(); + + var assembly = c.GetAssemblyOrModuleSymbol(r) as IAssemblySymbol; + + if (assembly != null) + { + var cx = new Context(extractor, c, trapWriter, new AssemblyScope(assembly, assemblyPath)); + + foreach (var module in assembly.Modules) + { + AnalyseNamespace(cx, module.GlobalNamespace); + } + + cx.PopulateAll(); + } + } + + ReportProgress(assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, skipExtraction ? AnalysisAction.UpToDate : AnalysisAction.Extracted); + } + } + catch (Exception ex) + { + Logger.Log(Severity.Error, " Unhandled exception analyzing {0}: {1}", r.FilePath, ex); + } + } + + void DoExtractCIL(PortableExecutableReference r) + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + string trapFile; + bool extracted; + CIL.Entities.Assembly.ExtractCIL(layout, r.FilePath, Logger, !options.Cache, options.PDB, out trapFile, out extracted); + stopwatch.Stop(); + ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate); + } + + void AnalyseNamespace(Context cx, INamespaceSymbol ns) + { + foreach (var memberNamespace in ns.GetNamespaceMembers()) + { + AnalyseNamespace(cx, memberNamespace); + } + + foreach (var memberType in ns.GetTypeMembers()) + { + Entities.Type.Create(cx, memberType).ExtractRecursive(); + } + } + + /// + /// Enqueue all reference analysis tasks. + /// + public void AnalyseReferences() + { + foreach (var r in compilation.References.OfType()) + { + AnalyseAssembly(r); + } + } + + // The bulk of the extraction work, potentially executed in parallel. + readonly List extractionTasks = new List(); + + void ReportProgress(string src, string output, TimeSpan time, AnalysisAction action) + { + lock (progressMutex) + progressMonitor.Analysed(++taskCount, extractionTasks.Count, src, output, time, action); + } + + void DoExtractTree(SyntaxTree tree) + { + try + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + var sourcePath = tree.FilePath; + + var projectLayout = layout.LookupProjectOrNull(sourcePath); + bool excluded = projectLayout == null; + string trapPath = excluded ? "" : projectLayout.GetTrapPath(Logger, sourcePath); + bool upToDate = false; + + if (!excluded) + { + // compilation.Clone() is used to allow symbols to be garbage collected. + using (var trapWriter = projectLayout.CreateTrapWriter(Logger, sourcePath, false)) + { + upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile); + + if (!upToDate) + { + Context cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree)); + Populators.CompilationUnit.Extract(cx, tree.GetRoot()); + cx.PopulateAll(); + cx.ExtractComments(cx.CommentGenerator); + } + } + } + + ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, excluded ? AnalysisAction.Excluded : upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted); + } + catch (Exception ex) + { + extractor.Message(new Message { exception = ex, message = string.Format("Unhandled exception processing {0}: {1}", tree.FilePath, ex), severity = Severity.Error }); + } + } + + /// + /// Run all extraction tasks. + /// + /// The number of threads to use. + public void PerformExtraction(int numberOfThreads) + { + Parallel.Invoke( + new ParallelOptions { MaxDegreeOfParallelism = numberOfThreads }, + extractionTasks.ToArray()); + } + + public void Dispose() + { + stopWatch.Stop(); + Logger.Log(Severity.Info, " Peak working set = {0} MB", Process.GetCurrentProcess().PeakWorkingSet64 / (1024 * 1024)); + + if (TotalErrors > 0) + Logger.Log(Severity.Info, "EXTRACTION FAILED with {0} error{1} in {2}", TotalErrors, TotalErrors == 1 ? "" : "s", stopWatch.Elapsed); + else + Logger.Log(Severity.Info, "EXTRACTION SUCCEEDED in {0}", stopWatch.Elapsed); + + Logger.Dispose(); + } + + /// + /// Number of errors encountered during extraction. + /// + public int ExtractorErrors => extractor == null ? 0 : extractor.Errors; + + /// + /// Number of errors encountered by the compiler. + /// + public int CompilationErrors { get; set; } + + /// + /// Total number of errors reported. + /// + public int TotalErrors => CompilationErrors + ExtractorErrors; + + void AppendQuoted(StringBuilder sb, string s) + { + if (s.IndexOf(' ') != -1) + sb.Append('\"').Append(s).Append('\"'); + else + sb.Append(s); + } + + /// + /// Logs detailed information about this invocation, + /// in the event that errors were detected. + /// + public void LogDiagnostics() + { + Logger.Log(Severity.Info, " Current working directory: {0}", Directory.GetCurrentDirectory()); + Logger.Log(Severity.Info, " Extractor: {0}", Environment.GetCommandLineArgs().First()); + if (extractor != null) + Logger.Log(Severity.Info, " Extractor version: {0}", extractor.Version); + var sb = new StringBuilder(); + sb.Append(" Expanded command line: "); + bool first = true; + foreach (var arg in Environment.GetCommandLineArgs().Skip(1)) + { + if (arg[0] == '@') + { + foreach (var line in File.ReadAllLines(arg.Substring(1))) + { + if (first) first = false; + else sb.Append(" "); + sb.Append(line); + } + } + else + { + if (first) first = false; + else sb.Append(" "); + AppendQuoted(sb, arg); + } + } + Logger.Log(Severity.Info, sb.ToString()); + + foreach (var error in FilteredDiagnostics) + { + Logger.Log(Severity.Error, " Compilation error: {0}", error); + } + + if (FilteredDiagnostics.Any()) + { + foreach (var reference in compilation.References) + { + Logger.Log(Severity.Info, " Resolved reference {0}", reference.Display); + } + } + } + } + + /// + /// What action was performed when extracting a file. + /// + public enum AnalysisAction + { + Extracted, + UpToDate, + Excluded + } + + /// + /// Callback for various extraction events. + /// (Used for display of progress). + /// + public interface IProgressMonitor + { + /// + /// Callback that a particular item has been analysed. + /// + /// The item number being processed. + /// The total number of items to process. + /// The name of the item, e.g. a source file. + /// The name of the item being output, e.g. a trap file. + /// The time to extract the item. + /// What action was taken for the file. + void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action); + + /// + /// A "using namespace" directive was seen but the given + /// namespace could not be found. + /// Only called once for each @namespace. + /// + /// + void MissingNamespace(string @namespace); + + /// + /// An ErrorType was found. + /// Called once for each type name. + /// + /// The full/partial name of the type. + void MissingType(string type); + + /// + /// Report a summary of missing entities. + /// + /// The number of missing types. + /// The number of missing using namespace declarations. + void MissingSummary(int types, int namespaces); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/CompilerVersion.cs b/csharp/extractor/Semmle.Extraction.CSharp/CompilerVersion.cs new file mode 100644 index 000000000000..ee1fca5bc2d3 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/CompilerVersion.cs @@ -0,0 +1,107 @@ +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; + +namespace Semmle.Extraction.CSharp +{ + /// + /// Identifies the compiler and framework from the command line arguments. + /// --compiler specifies the compiler + /// --framework specifies the .net framework + /// + public class CompilerVersion + { + const string csc_rsp = "csc.rsp"; + readonly string specifiedFramework = null; + + /// + /// The value specified by --compiler, or null. + /// + public string SpecifiedCompiler + { + get; + private set; + } + + /// + /// Why was the candidate exe rejected as a compiler? + /// + public string SkipReason + { + get; + private set; + } + + /// + /// Probes the compiler (if specified). + /// + /// The command line arguments. + public CompilerVersion(Options options) + { + SpecifiedCompiler = options.CompilerName; + specifiedFramework = options.Framework; + + if (SpecifiedCompiler != null) + { + if (!File.Exists(SpecifiedCompiler)) + { + SkipExtractionBecause("the specified file does not exist"); + return; + } + + // Reads the file details from the .exe + var versionInfo = FileVersionInfo.GetVersionInfo(SpecifiedCompiler); + + var compilerDir = Path.GetDirectoryName(SpecifiedCompiler); + bool known_compiler_name = versionInfo.OriginalFilename == "csc.exe" || versionInfo.OriginalFilename == "csc2.exe"; + bool copyright_microsoft = versionInfo.LegalCopyright != null && versionInfo.LegalCopyright.Contains("Microsoft"); + bool mscorlib_exists = File.Exists(Path.Combine(compilerDir, "mscorlib.dll")); + + if (specifiedFramework == null && mscorlib_exists) + { + specifiedFramework = compilerDir; + } + + if (!known_compiler_name) + { + SkipExtractionBecause("the exe name is not recognised"); + } + else if (!copyright_microsoft) + { + SkipExtractionBecause("the exe isn't copyright Microsoft"); + } + } + } + + void SkipExtractionBecause(string reason) + { + SkipExtraction = true; + SkipReason = reason; + } + + /// + /// The directory containing the .Net Framework. + /// + public string FrameworkPath => specifiedFramework ?? RuntimeEnvironment.GetRuntimeDirectory(); + + /// + /// The file csc.rsp. + /// + public string CscRsp => Path.Combine(FrameworkPath, csc_rsp); + + /// + /// Should we skip extraction? + /// Only if csc.exe was specified but it wasn't a compiler. + /// + public bool SkipExtraction + { + get; + private set; + } + + /// + /// Gets additional reference directories - the compiler directory. + /// + public string AdditionalReferenceDirectories => SpecifiedCompiler != null ? Path.GetDirectoryName(SpecifiedCompiler) : null; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs new file mode 100644 index 000000000000..b70ab6ce0862 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs @@ -0,0 +1,88 @@ +using Microsoft.CodeAnalysis; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Accessor : Method + { + protected Accessor(Context cx, IMethodSymbol init) + : base(cx, init) { } + + /// + /// Gets the property symbol associated with this accessor. + /// + IPropertySymbol PropertySymbol + { + get + { + // Usually, the property/indexer can be fetched from the associated symbol + var prop = symbol.AssociatedSymbol as IPropertySymbol; + if (prop != null) + return prop; + + // But for properties/indexers that implement explicit interfaces, Roslyn + // does not properly populate `AssociatedSymbol` + var props = symbol.ContainingType.GetMembers().OfType(); + props = props.Where(p => symbol.Equals(p.GetMethod) || symbol.Equals(p.SetMethod)); + return props.SingleOrDefault(); + } + } + + public new Accessor OriginalDefinition => Create(Context, symbol.OriginalDefinition); + + public override void Populate() + { + PopulateMethod(); + ExtractModifiers(); + ContainingType.ExtractGenerics(); + + var prop = PropertySymbol; + if (prop == null) + { + Context.ModelError(symbol, "Unhandled accessor associated symbol"); + return; + } + + var parent = Property.Create(Context, prop); + int kind; + Accessor unboundAccessor; + if (symbol.Equals(prop.GetMethod)) + { + kind = 1; + unboundAccessor = Create(Context, prop.OriginalDefinition.GetMethod); + } + else if (symbol.Equals(prop.SetMethod)) + { + kind = 2; + unboundAccessor = Create(Context, prop.OriginalDefinition.SetMethod); + } + else + { + Context.ModelError(symbol, "Undhandled accessor kind"); + return; + } + + Context.Emit(Tuples.accessors(this, kind, symbol.Name, parent, unboundAccessor)); + + foreach (var l in Locations) + Context.Emit(Tuples.accessor_location(this, l)); + + Overrides(); + + if (symbol.FromSource() && Block == null) + { + Context.Emit(Tuples.compiler_generated(this)); + } + } + + public new static Accessor Create(Context cx, IMethodSymbol symbol) => + AccessorFactory.Instance.CreateEntity(cx, symbol); + + class AccessorFactory : ICachedEntityFactory + { + public static readonly AccessorFactory Instance = new AccessorFactory(); + + public Accessor Create(Context cx, IMethodSymbol init) => new Accessor(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs new file mode 100644 index 000000000000..41deadc7fe97 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs @@ -0,0 +1,79 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Attribute : FreshEntity, IExpressionParentEntity + { + bool IExpressionParentEntity.IsTopLevelParent => true; + + public Attribute(Context cx, AttributeData attribute, IEntity entity) + : base(cx) + { + if (attribute.ApplicationSyntaxReference != null) + { + // !! Extract attributes from assemblies. + // This is harder because the "expression" entities presume the + // existence of a syntax tree. This is not the case for compiled + // attributes. + var syntax = attribute.ApplicationSyntaxReference.GetSyntax() as AttributeSyntax; + ExtractAttribute(cx, syntax, attribute.AttributeClass, entity); + } + } + + public Attribute(Context cx, AttributeSyntax attribute, IEntity entity) + : base(cx) + { + var info = cx.GetSymbolInfo(attribute); + ExtractAttribute(cx, attribute, info.Symbol.ContainingType, entity); + } + + void ExtractAttribute(Context cx, AttributeSyntax syntax, ITypeSymbol attributeClass, IEntity entity) + { + var type = Type.Create(cx, attributeClass); + cx.Emit(Tuples.attributes(this, type.TypeRef, entity)); + + cx.Emit(Tuples.attribute_location(this, cx.Create(syntax.Name.GetLocation()))); + + if (cx.Extractor.OutputPath != null) + cx.Emit(Tuples.attribute_location(this, Assembly.CreateOutputAssembly(cx))); + + TypeMention.Create(cx, syntax.Name, this, type); + + if (syntax.ArgumentList != null) + { + cx.PopulateLater(() => + { + int child = 0; + foreach (var arg in syntax.ArgumentList.Arguments) + { + Expression.Create(cx, arg.Expression, this, child++); + } + // !! Handle named arguments + }); + } + } + + public static void ExtractAttributes(Context cx, ISymbol symbol, IEntity entity) + { + foreach (var attribute in symbol.GetAttributes()) + { + new Attribute(cx, attribute, entity); + } + } + + public static void ExtractAttributes(Context cx, IEnumerable attributes, IEntity entity) + { + foreach (var attributeSyntax in attributes.SelectMany(l => l.Attributes)) + { + new Attribute(cx, attributeSyntax, entity); + } + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs new file mode 100644 index 000000000000..990b287cd70e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs @@ -0,0 +1,51 @@ +using Semmle.Extraction.CommentProcessing; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + class CommentBlock : CachedEntity + { + CommentBlock(Context cx, ICommentBlock init) + : base(cx, init) { } + + public override void Populate() + { + Context.Emit(Tuples.commentblock(this)); + int child = 0; + Context.Emit(Tuples.commentblock_location(this, Context.Create(symbol.Location))); + foreach (var l in symbol.CommentLines) + { + Context.Emit(Tuples.commentblock_child(this, (CommentLine)l, child++)); + } + } + + public override bool NeedsPopulation => true; + + public override IId Id + { + get + { + var loc = Context.Create(symbol.Location); + return new Key(loc, ";commentblock"); + } + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => symbol.Location; + + public void BindTo(Label entity, Binding binding) + { + Context.Emit(Tuples.commentblock_binding(this, entity, binding)); + } + + public static CommentBlock Create(Context cx, ICommentBlock block) => CommentBlockFactory.Instance.CreateEntity(cx, block); + + class CommentBlockFactory : ICachedEntityFactory + { + public static readonly CommentBlockFactory Instance = new CommentBlockFactory(); + + public CommentBlock Create(Context cx, ICommentBlock init) => new CommentBlock(cx, init); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs new file mode 100644 index 000000000000..234ff96935ef --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs @@ -0,0 +1,147 @@ +using Semmle.Extraction.CommentProcessing; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Entities; +using System; + +namespace Semmle.Extraction.CSharp.Entities +{ + class CommentLine : CachedEntity<(Microsoft.CodeAnalysis.Location, string)>, ICommentLine + { + CommentLine(Context cx, Microsoft.CodeAnalysis.Location loc, CommentType type, string text, string raw) + : base(cx, (loc, text)) + { + Type = type; + RawText = raw; + } + + public Microsoft.CodeAnalysis.Location Location => symbol.Item1; + public CommentType Type { get; private set; } + + public string Text { get { return symbol.Item2; } } + public string RawText { get; private set; } + + public static void Extract(Context cx, SyntaxTrivia trivia) + { + switch (trivia.Kind()) + { + case SyntaxKind.SingleLineDocumentationCommentTrivia: + /* + This is actually a multi-line comment consisting of /// lines. + So split it up. + */ + + var text = trivia.ToFullString(); + + var split = text.Split('\n'); + var currentLocation = trivia.GetLocation().SourceSpan.Start - 3; + + for (int line = 0; line < split.Length - 1; ++line) + { + string fullLine = split[line]; + var nextLineLocation = currentLocation + fullLine.Length + 1; + fullLine = fullLine.TrimEnd('\r'); + string trimmedLine = fullLine; + + int leadingSpaces = trimmedLine.IndexOf('/'); + if (leadingSpaces != -1) + { + fullLine = fullLine.Substring(leadingSpaces); + currentLocation += leadingSpaces; + trimmedLine = trimmedLine.Substring(leadingSpaces + 3); // Remove leading spaces and the "///" + trimmedLine = trimmedLine.Trim(); + + var span = Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(currentLocation, currentLocation + fullLine.Length); + var location = Microsoft.CodeAnalysis.Location.Create(trivia.SyntaxTree, span); + var commentType = CommentType.XmlDoc; + cx.CommentGenerator.AddComment(Create(cx, location, commentType, trimmedLine, fullLine)); + } + else + { + cx.ModelError("Unexpected comment format"); + } + currentLocation = nextLineLocation; + } + break; + + case SyntaxKind.SingleLineCommentTrivia: + { + string contents = trivia.ToString().Substring(2); + var commentType = CommentType.Singleline; + if (contents.Length > 0 && contents[0] == '/') + { + commentType = CommentType.XmlDoc; + contents = contents.Substring(1); // An XML comment. + } + cx.CommentGenerator.AddComment(Create(cx, trivia.GetLocation(), commentType, contents.Trim(), trivia.ToFullString())); + } + break; + case SyntaxKind.MultiLineDocumentationCommentTrivia: + case SyntaxKind.MultiLineCommentTrivia: + /* We receive a single SyntaxTrivia for a multiline block spanning several lines. + So we split it into separate lines + */ + text = trivia.ToFullString(); + + split = text.Split('\n'); + currentLocation = trivia.GetLocation().SourceSpan.Start; + + for (int line = 0; line < split.Length; ++line) + { + string fullLine = split[line]; + var nextLineLocation = currentLocation + fullLine.Length + 1; + fullLine = fullLine.TrimEnd('\r'); + string trimmedLine = fullLine; + if (line == 0) trimmedLine = trimmedLine.Substring(2); + if (line == split.Length - 1) trimmedLine = trimmedLine.Substring(0, trimmedLine.Length - 2); + trimmedLine = trimmedLine.Trim(); + + var span = Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(currentLocation, currentLocation + fullLine.Length); + var location = Microsoft.CodeAnalysis.Location.Create(trivia.SyntaxTree, span); + var commentType = line == 0 ? CommentType.Multiline : CommentType.MultilineContinuation; + cx.CommentGenerator.AddComment(Create(cx, location, commentType, trimmedLine, fullLine)); + currentLocation = nextLineLocation; + } + break; + // Strangely, these are reported as SingleLineCommentTrivia. + case SyntaxKind.DocumentationCommentExteriorTrivia: + cx.ModelError("Unhandled comment type {0} for {1}", trivia.Kind(), trivia); + break; + } + } + + Extraction.Entities.Location location; + + public override void Populate() + { + location = Context.Create(Location); + Context.Emit(Tuples.commentline(this, Type == CommentType.MultilineContinuation ? CommentType.Multiline : Type, Text, RawText)); + Context.Emit(Tuples.commentline_location(this, location)); + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => location.symbol; + + public override bool NeedsPopulation => true; + + public override IId Id + { + get + { + var loc = Context.Create(Location); + return new Key(loc, ";commentline"); + } + } + + static CommentLine Create(Context cx, Microsoft.CodeAnalysis.Location loc, CommentType type, string text, string raw) => CommentLineFactory.Instance.CreateEntity(cx, loc, type, text, raw); + + class CommentLineFactory : ICachedEntityFactory<(Microsoft.CodeAnalysis.Location, CommentType, string, string), CommentLine> + { + public static readonly CommentLineFactory Instance = new CommentLineFactory(); + + public CommentLine Create(Context cx, (Microsoft.CodeAnalysis.Location, CommentType, string, string) init) => + new CommentLine(cx, init.Item1, init.Item2, init.Item3, init.Item4); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs new file mode 100644 index 000000000000..d96db4b46246 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs @@ -0,0 +1,158 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Util; +using System.Linq; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + public class Constructor : Method + { + Constructor(Context cx, IMethodSymbol init) + : base(cx, init) { } + + public override void Populate() + { + PopulateMethod(); + ExtractModifiers(); + ContainingType.ExtractGenerics(); + + Context.Emit(Tuples.constructors(this, symbol.ContainingType.Name, ContainingType, (Constructor)OriginalDefinition)); + Context.Emit(Tuples.constructor_location(this, Location)); + + if (symbol.IsImplicitlyDeclared) + { + var lineCounts = new LineCounts() { Total = 2, Code = 1, Comment = 0 }; + Context.Emit(Tuples.numlines(this, lineCounts)); + } + ExtractCompilerGenerated(); + } + + protected override void ExtractInitializers() + { + // Do not extract initializers for constructed types. + if (!IsSourceDeclaration) return; + + var syntax = Syntax; + var initializer = syntax == null ? null : syntax.Initializer; + + if (initializer == null) return; + + Type initializerType; + var symbolInfo = Context.GetSymbolInfo(initializer); + + switch (initializer.Kind()) + { + case SyntaxKind.BaseConstructorInitializer: + initializerType = Type.Create(Context, symbol.ContainingType.BaseType); + break; + case SyntaxKind.ThisConstructorInitializer: + initializerType = ContainingType; + break; + default: + Context.ModelError(initializer, "Unknown initializer"); + return; + } + + var initInfo = new ExpressionInfo(Context, + initializerType, + Context.Create(initializer.ThisOrBaseKeyword.GetLocation()), + Kinds.ExprKind.CONSTRUCTOR_INIT, + this, + -1, + false, + null); + + var init = new Expression(initInfo); + + var target = Constructor.Create(Context, (IMethodSymbol)symbolInfo.Symbol); + + if (target == null) + { + Context.ModelError(symbol, "Unable to resolve call"); + return; + } + + Context.Emit(Tuples.expr_call(init, target)); + + int child = 0; + foreach (var arg in initializer.ArgumentList.Arguments) + { + Expression.Create(Context, arg.Expression, init, child++); + } + } + + ConstructorDeclarationSyntax Syntax + { + get + { + return symbol.DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + FirstOrDefault(); + } + } + + public new static Constructor Create(Context cx, IMethodSymbol constructor) + { + if (constructor == null) return null; + + switch (constructor.MethodKind) + { + case MethodKind.StaticConstructor: + case MethodKind.Constructor: + return ConstructorFactory.Instance.CreateEntity(cx, constructor); + default: + throw new InternalError(constructor, "Attempt to create a Constructor from a symbol that isn't a constructor"); + } + } + + public override IId Id + { + get + { + return new Key(tb => + { + if (symbol.IsStatic) tb.Append("static"); + tb.Append(ContainingType); + AddParametersToId(Context, tb, symbol); + tb.Append("; constructor"); + }); + } + } + + ConstructorDeclarationSyntax GetSyntax() => + symbol.DeclaringSyntaxReferences.Select(r => r.GetSyntax()).OfType().FirstOrDefault(); + + public override Microsoft.CodeAnalysis.Location FullLocation => ReportingLocation; + + public override Microsoft.CodeAnalysis.Location ReportingLocation + { + get + { + var syn = GetSyntax(); + if (syn != null) + { + return syn.Identifier.GetLocation(); + } + else if (symbol.IsImplicitlyDeclared) + { + return ContainingType.ReportingLocation; + } + else + { + return symbol.ContainingType.Locations.FirstOrDefault(); + } + } + } + + class ConstructorFactory : ICachedEntityFactory + { + public static readonly ConstructorFactory Instance = new ConstructorFactory(); + + public Constructor Create(Context cx, IMethodSymbol init) => new Constructor(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs new file mode 100644 index 000000000000..e365ca53ae0c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs @@ -0,0 +1,37 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Conversion : UserOperator + { + Conversion(Context cx, IMethodSymbol init) + : base(cx, init) { } + + public new static Conversion Create(Context cx, IMethodSymbol symbol) => + ConversionFactory.Instance.CreateEntity(cx, symbol); + + public override Microsoft.CodeAnalysis.Location ReportingLocation + { + get + { + return symbol. + DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + Select(s => s.FixedLocation()). + Concat(symbol.Locations). + FirstOrDefault(); + } + } + + class ConversionFactory : ICachedEntityFactory + { + public static readonly ConversionFactory Instance = new ConversionFactory(); + + public Conversion Create(Context cx, IMethodSymbol init) => new Conversion(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs new file mode 100644 index 000000000000..5a4b90933519 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs @@ -0,0 +1,35 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Destructor : Method + { + Destructor(Context cx, IMethodSymbol init) + : base(cx, init) { } + + public override void Populate() + { + PopulateMethod(); + ExtractModifiers(); + ContainingType.ExtractGenerics(); + + Context.Emit(Tuples.destructors(this, string.Format("~{0}", symbol.ContainingType.Name), ContainingType, OriginalDefinition(Context, this, symbol))); + Context.Emit(Tuples.destructor_location(this, Location)); + } + + static new Destructor OriginalDefinition(Context cx, Destructor original, IMethodSymbol symbol) + { + return symbol.OriginalDefinition == null || Equals(symbol.OriginalDefinition, symbol) ? original : Create(cx, symbol.OriginalDefinition); + } + + public new static Destructor Create(Context cx, IMethodSymbol symbol) => + DestructorFactory.Instance.CreateEntity(cx, symbol); + + class DestructorFactory : ICachedEntityFactory + { + public static readonly DestructorFactory Instance = new DestructorFactory(); + + public Destructor Create(Context cx, IMethodSymbol init) => new Destructor(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs new file mode 100644 index 000000000000..86a22c704a7f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs @@ -0,0 +1,76 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Event : CachedSymbol + { + Event(Context cx, IEventSymbol init) + : base(cx, init) { } + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(ContainingType); + tb.Append("."); + Method.AddExplicitInterfaceQualifierToId(Context, tb, symbol.ExplicitInterfaceImplementations); + tb.Append(symbol.Name); + tb.Append(";event"); + }); + } + } + + public override void Populate() + { + var type = Type.Create(Context, symbol.Type); + Context.Emit(Tuples.events(this, symbol.GetName(), ContainingType, type.TypeRef, Create(Context, symbol.OriginalDefinition))); + + var adder = symbol.AddMethod; + if (adder != null) + EventAccessor.Create(Context, adder); + + var remover = symbol.RemoveMethod; + if (remover != null) + EventAccessor.Create(Context, remover); + + ExtractModifiers(); + BindComments(); + + var declSyntaxReferences = IsSourceDeclaration + ? symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()).ToArray() + : Enumerable.Empty(); + + foreach (var explicitInterface in symbol.ExplicitInterfaceImplementations.Select(impl => Type.Create(Context, impl.ContainingType))) + { + Context.Emit(Tuples.explicitly_implements(this, explicitInterface.TypeRef)); + + foreach (var syntax in declSyntaxReferences.OfType()) + TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier.Name, this, explicitInterface); + } + + foreach (var l in Locations) + Context.Emit(Tuples.event_location(this, l)); + + foreach (var syntaxType in declSyntaxReferences.OfType(). + Select(d => d.Parent). + OfType(). + Select(syntax => syntax.Type)) + TypeMention.Create(Context, syntaxType, this, type); + } + + public static Event Create(Context cx, IEventSymbol symbol) => EventFactory.Instance.CreateEntity(cx, symbol); + + class EventFactory : ICachedEntityFactory + { + public static readonly EventFactory Instance = new EventFactory(); + + public Event Create(Context cx, IEventSymbol init) => new Event(cx, init); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs new file mode 100644 index 000000000000..7e6cb01a8b60 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs @@ -0,0 +1,64 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class EventAccessor : Accessor + { + EventAccessor(Context cx, IMethodSymbol init) + : base(cx, init) { } + + /// + /// Gets the event symbol associated with this accessor. + /// + IEventSymbol EventSymbol => symbol.AssociatedSymbol as IEventSymbol; + + public override void Populate() + { + PopulateMethod(); + ContainingType.ExtractGenerics(); + + var @event = EventSymbol; + if (@event == null) + { + Context.ModelError(symbol, "Unhandled event accessor associated symbol"); + return; + } + + var parent = Event.Create(Context, @event); + int kind; + EventAccessor unboundAccessor; + if (symbol.Equals(@event.AddMethod)) + { + kind = 1; + unboundAccessor = Create(Context, @event.OriginalDefinition.AddMethod); + } + else if (symbol.Equals(@event.RemoveMethod)) + { + kind = 2; + unboundAccessor = Create(Context, @event.OriginalDefinition.RemoveMethod); + } + else + { + Context.ModelError(symbol, "Undhandled event accessor kind"); + return; + } + + Context.Emit(Tuples.event_accessors(this, kind, symbol.Name, parent, unboundAccessor)); + + foreach (var l in Locations) + Context.Emit(Tuples.event_accessor_location(this, l)); + + Overrides(); + } + + public new static EventAccessor Create(Context cx, IMethodSymbol symbol) => + EventAccessorFactory.Instance.CreateEntity(cx, symbol); + + class EventAccessorFactory : ICachedEntityFactory + { + public static readonly EventAccessorFactory Instance = new EventAccessorFactory(); + + public EventAccessor Create(Context cx, IMethodSymbol init) => new EventAccessor(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs new file mode 100644 index 000000000000..f52e241f0317 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expression.cs @@ -0,0 +1,527 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + public interface IExpressionParentEntity : IEntity + { + /// + /// Whether this entity is the parent of a top-level expression. + /// + bool IsTopLevelParent { get; } + } + + class Expression : FreshEntity, IExpressionParentEntity + { + public readonly Type Type; + public readonly Extraction.Entities.Location Location; + public readonly ExprKind Kind; + + internal Expression(IExpressionInfo info) + : base(info.Context) + { + Location = info.Location; + Kind = info.Kind; + Type = info.Type ?? NullType.Create(cx); + + cx.Emit(Tuples.expressions(this, Kind, Type.TypeRef)); + if (info.Parent.IsTopLevelParent) + cx.Emit(Tuples.expr_parent_top_level(this, info.Child, info.Parent)); + else + cx.Emit(Tuples.expr_parent(this, info.Child, info.Parent)); + cx.Emit(Tuples.expr_location(this, Location)); + + if (info.IsCompilerGenerated) + cx.Emit(Tuples.expr_compiler_generated(this)); + + if (info.ExprValue is string value) + cx.Emit(Tuples.expr_value(this, value)); + + Type.ExtractGenerics(); + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => Location.symbol; + + bool IExpressionParentEntity.IsTopLevelParent => false; + + /// + /// Gets a string represention of a constant value. + /// + /// The value. + /// The string representation. + public static string ValueAsString(object value) + { + return value == null ? "null" : value is bool ? ((bool)value ? "true" : "false") : value.ToString(); + } + + /// + /// Creates an expression from a syntax node. + /// Inserts type conversion as required. + /// + /// The extraction context. + /// The node to extract. + /// The parent entity. + /// The child index. + /// A type hint. + /// The new expression. + public static Expression Create(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child) => + CreateFromNode(new ExpressionNodeInfo(cx, node, parent, child)); + + public static Expression CreateFromNode(ExpressionNodeInfo info) => Expressions.ImplicitCast.Create(info); + + /// + /// Creates an expression from a syntax node. + /// Inserts type conversion as required. + /// Population is deferred to avoid overflowing the stack. + /// + /// The extraction context. + /// The node to extract. + /// The parent entity. + /// The child index. + /// A type hint. + public static void CreateDeferred(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child) + { + if (ContainsPattern(node)) + // Expressions with patterns should be created right away, as they may introduce + // local variables referenced in `LocalVariable::GetAlreadyCreated()` + Create(cx, node, parent, child); + else + cx.PopulateLater(() => Create(cx, node, parent, child)); + } + + static bool ContainsPattern(SyntaxNode node) => + node is PatternSyntax || node is VariableDesignationSyntax || node.ChildNodes().Any(ContainsPattern); + + /// + /// Adapt the operator kind depending on whether it's a dynamic call or a user-operator call. + /// + /// + /// + /// + /// + public static ExprKind UnaryOperatorKind(Context cx, ExprKind originalKind, ExpressionSyntax node) => + GetCallType(cx, node).AdjustKind(originalKind); + + /// + /// If the expression calls an operator, add an expr_call() + /// to show the target of the call. Also note the dynamic method + /// name if available. + /// + /// Context + /// The expression. + public void OperatorCall(ExpressionSyntax node) + { + var @operator = cx.GetSymbolInfo(node); + var method = @operator.Symbol as IMethodSymbol; + + if (GetCallType(cx, node) == CallType.Dynamic) + { + UserOperator.OperatorSymbol(method.Name, out string operatorName); + cx.Emit(Tuples.dynamic_member_name(this, operatorName)); + return; + } + + if (method != null) + cx.Emit(Tuples.expr_call(this, Method.Create(cx, method))); + } + + public enum CallType + { + None, + BuiltInOperator, + Dynamic, + UserOperator + } + + /// + /// Determine what type of method was called for this expression. + /// + /// The context. + /// The expression + /// The call type. + public static CallType GetCallType(Context cx, ExpressionSyntax node) + { + var @operator = cx.GetSymbolInfo(node); + + if (@operator.Symbol != null) + { + var method = @operator.Symbol as IMethodSymbol; + + var containingSymbol = method.ContainingSymbol as ITypeSymbol; + if (containingSymbol != null && containingSymbol.TypeKind == Microsoft.CodeAnalysis.TypeKind.Dynamic) + { + return CallType.Dynamic; + } + + switch (method.MethodKind) + { + case MethodKind.BuiltinOperator: + if (method.ContainingType != null && method.ContainingType.TypeKind == Microsoft.CodeAnalysis.TypeKind.Delegate) + return CallType.UserOperator; + return CallType.BuiltInOperator; + default: + return CallType.UserOperator; + } + } + + return CallType.None; + } + + + public static bool IsDynamic(Context cx, ExpressionSyntax node) + { + var ti = cx.GetTypeInfo(node).ConvertedType; + return ti != null && ti.TypeKind == Microsoft.CodeAnalysis.TypeKind.Dynamic; + } + + /// + /// Given b in a?.b.c, return a. + /// + /// A MemberBindingExpression. + /// The qualifier of the conditional access. + protected static ExpressionSyntax FindConditionalQualifier(ExpressionSyntax node) + { + for (SyntaxNode n = node; n != null; n = n.Parent) + { + var conditionalAccess = n.Parent as ConditionalAccessExpressionSyntax; + + if (conditionalAccess != null && conditionalAccess.WhenNotNull == n) + return conditionalAccess.Expression; + } + + throw new InternalError(node, "Unable to locate a ConditionalAccessExpression"); + } + + public void MakeConditional() + { + cx.Emit(Tuples.conditional_access(this)); + } + + public void PopulateArguments(Context cx, BaseArgumentListSyntax args, int child) + { + foreach (var arg in args.Arguments) + PopulateArgument(cx, arg, child++); + } + + private void PopulateArgument(Context cx, ArgumentSyntax arg, int child) + { + var expr = Create(cx, arg.Expression, this, child); + int mode; + switch (arg.RefOrOutKeyword.Kind()) + { + case SyntaxKind.RefKeyword: + mode = 1; + break; + case SyntaxKind.OutKeyword: + mode = 2; + break; + case SyntaxKind.None: + mode = 0; + break; + default: + throw new InternalError(arg, "Unknown argument type"); + } + cx.Emit(Tuples.expr_argument(expr, mode)); + + if (arg.NameColon != null) + { + cx.Emit(Tuples.expr_argument_name(expr, arg.NameColon.Name.Identifier.Text)); + } + } + + public override string ToString() => Label.ToString(); + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } + + static class CallTypeExtensions + { + /// + /// Adjust the expression kind to match this call type. + /// + public static ExprKind AdjustKind(this Expression.CallType ct, ExprKind k) + { + switch (ct) + { + case Expression.CallType.Dynamic: + case Expression.CallType.UserOperator: + return ExprKind.OPERATOR_INVOCATION; + default: + return k; + } + } + } + + abstract class Expression : Expression + where SyntaxNode : ExpressionSyntax + { + public readonly SyntaxNode Syntax; + + protected Expression(ExpressionNodeInfo info) + : base(info) + { + Syntax = (SyntaxNode)info.Node; + } + + /// + /// Populates expression-type specific relations in the trap file. The general relations + /// expressions and expr_location are populated by the constructor + /// (should not fail), so even if expression-type specific population fails (e.g., in + /// standalone extraction), the expression created via + /// will + /// still be valid. + /// + protected abstract void Populate(); + + protected Expression TryPopulate() + { + cx.Try(Syntax, null, Populate); + return this; + } + } + + /// + /// Holds all information required to create an Expression entity. + /// + interface IExpressionInfo + { + Context Context { get; } + + /// + /// The type of the expression. + /// + Type Type { get; } + + /// + /// The location of the expression. + /// + Extraction.Entities.Location Location { get; } + + /// + /// The kind of the expression. + /// + ExprKind Kind { get; } + + /// + /// The parent of the expression. + /// + IExpressionParentEntity Parent { get; } + + /// + /// The child index of the expression. + /// + int Child { get; } + + /// + /// Holds if this is an implicit expression. + /// + bool IsCompilerGenerated { get; } + + /// + /// Gets a string representation of the value. + /// null is encoded as the string "null". + /// If the expression does not have a value, then this + /// is null. + /// + string ExprValue { get; } + } + + /// + /// Explicitly constructed expression information. + /// + class ExpressionInfo : IExpressionInfo + { + public Context Context { get; } + public Type Type { get; } + public Extraction.Entities.Location Location { get; } + public ExprKind Kind { get; } + public IExpressionParentEntity Parent { get; } + public int Child { get; } + public bool IsCompilerGenerated { get; } + public string ExprValue { get; } + + public ExpressionInfo(Context cx, Type type, Extraction.Entities.Location location, ExprKind kind, IExpressionParentEntity parent, int child, bool isCompilerGenerated, string value) + { + Context = cx; + Type = type; + Location = location; + Kind = kind; + Parent = parent; + Child = child; + ExprValue = value; + IsCompilerGenerated = isCompilerGenerated; + } + } + + /// + /// Expression information constructed from a syntax node. + /// + class ExpressionNodeInfo : IExpressionInfo + { + public ExpressionNodeInfo(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child) : + this(cx, node, parent, child, cx.GetTypeInfo(node)) + { + } + + public ExpressionNodeInfo(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child, TypeInfo typeInfo) + { + Context = cx; + Node = node; + Parent = parent; + Child = child; + TypeInfo = typeInfo; + Conversion = cx.Model(node).GetConversion(node); + } + + public ExpressionNodeInfo(Context cx, ExpressionSyntax node, IExpressionParentEntity parent, int child, ITypeSymbol type) : + this(cx, node, parent, child) + { + Type = Type.Create(cx, type); + } + + public Context Context { get; } + public ExpressionSyntax Node { get; private set; } + public IExpressionParentEntity Parent { get; set; } + public int Child { get; set; } + public TypeInfo TypeInfo { get; } + public Microsoft.CodeAnalysis.CSharp.Conversion Conversion { get; } + + public ITypeSymbol ResolvedType => Context.DisambiguateType(TypeInfo.Type); + public ITypeSymbol ConvertedType => Context.DisambiguateType(TypeInfo.ConvertedType); + + public ITypeSymbol ExpressionType + { + get + { + var type = ResolvedType; + + if (type == null) + type = Context.DisambiguateType(TypeInfo.Type ?? TypeInfo.ConvertedType); + + // Roslyn workaround: It can't work out the type of "new object[0]" + // Clearly a bug. + if (type != null && type.TypeKind == Microsoft.CodeAnalysis.TypeKind.Error) + { + var arrayCreation = Node as ArrayCreationExpressionSyntax; + if (arrayCreation != null) + { + var elementType = Context.GetType(arrayCreation.Type.ElementType); + + if (elementType != null) + return Context.Compilation.CreateArrayTypeSymbol(elementType, arrayCreation.Type.RankSpecifiers.Count); + } + + Context.ModelError(Node, "Failed to determine type"); + } + + return type; + } + } + + Microsoft.CodeAnalysis.Location location; + + public Microsoft.CodeAnalysis.Location CodeAnalysisLocation + { + get + { + if (location == null) + location = Node.FixedLocation(); + return location; + } + set + { + location = value; + } + } + + public SemanticModel Model => Context.Model(Node); + + public string ExprValue + { + get + { + var c = Model.GetConstantValue(Node); + return c.HasValue ? Expression.ValueAsString(c.Value) : null; + } + } + + Type cachedType; + + public Type Type + { + get + { + if (cachedType == null) + cachedType = Type.Create(Context, ExpressionType); + return cachedType; + } + set + { + cachedType = value; + } + } + + Extraction.Entities.Location cachedLocation; + + public Extraction.Entities.Location Location + { + get + { + if (cachedLocation == null) + cachedLocation = Context.Create(CodeAnalysisLocation); + return cachedLocation; + } + + set + { + cachedLocation = value; + } + } + + public ExprKind Kind { get; set; } = ExprKind.UNKNOWN; + + public bool IsCompilerGenerated { get; set; } + + public ExpressionNodeInfo SetParent(IExpressionParentEntity parent, int child) + { + Parent = parent; + Child = child; + return this; + } + + public ExpressionNodeInfo SetKind(ExprKind kind) + { + Kind = kind; + return this; + } + + public ExpressionNodeInfo SetType(Type type) + { + Type = type; + return this; + } + + public ExpressionNodeInfo SetNode(ExpressionSyntax node) + { + Node = node; + return this; + } + + SymbolInfo cachedSymbolInfo; + + public SymbolInfo SymbolInfo + { + get + { + if (cachedSymbolInfo.Symbol == null && cachedSymbolInfo.CandidateReason == CandidateReason.None) + cachedSymbolInfo = Model.GetSymbolInfo(Node); + return cachedSymbolInfo; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs new file mode 100644 index 000000000000..60921d683c1d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs @@ -0,0 +1,55 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Access : Expression + { + static ExprKind AccessKind(Context cx, ISymbol symbol) + { + switch (symbol.Kind) + { + case SymbolKind.TypeParameter: + case SymbolKind.NamedType: + return ExprKind.TYPE_ACCESS; + + case SymbolKind.Field: + return ExprKind.FIELD_ACCESS; + + case SymbolKind.Property: + return ExprKind.PROPERTY_ACCESS; + + case SymbolKind.Event: + return ExprKind.EVENT_ACCESS; + + case SymbolKind.Method: + return ExprKind.METHOD_ACCESS; + + case SymbolKind.Local: + case SymbolKind.RangeVariable: + return ExprKind.LOCAL_VARIABLE_ACCESS; + + case SymbolKind.Parameter: + return ExprKind.PARAMETER_ACCESS; + + default: + cx.ModelError(symbol, "Unhandled access kind '{0}'", symbol.Kind); + return ExprKind.UNKNOWN; + } + } + + Access(ExpressionNodeInfo info, ISymbol symbol, bool implicitThis, IEntity target) + : base(info.SetKind(AccessKind(info.Context, symbol))) + { + cx.Emit(Tuples.expr_access(this, target)); + + if (implicitThis && !symbol.IsStatic) + { + This.CreateImplicit(cx, Type.Create(cx, symbol.ContainingType), Location, this, -1); + } + } + + public static Expression Create(ExpressionNodeInfo info, ISymbol symbol, bool implicitThis, IEntity target) => new Access(info, symbol, implicitThis, target); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArgList.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArgList.cs new file mode 100644 index 000000000000..7b8fc26f0a4c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArgList.cs @@ -0,0 +1,18 @@ +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class ArgList : Expression + { + ArgList(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.UNKNOWN)) { } + + protected override void Populate() + { + throw new NotImplementedException(); + } + + public static ArgList Create(ExpressionNodeInfo info) => new ArgList(info); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs new file mode 100644 index 000000000000..c6f1a76a9d06 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ArrayCreation.cs @@ -0,0 +1,108 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + abstract class ArrayCreation : Expression where SyntaxNode : ExpressionSyntax + { + protected ArrayCreation(ExpressionNodeInfo info) : base(info) { } + } + + class StackAllocArrayCreation : ArrayCreation + { + StackAllocArrayCreation(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.ARRAY_CREATION)) { } + + public static Expression Create(ExpressionNodeInfo info) => new StackAllocArrayCreation(info).TryPopulate(); + + protected override void Populate() + { + var arrayType = Syntax.Type as ArrayTypeSyntax; + + if (arrayType == null) + { + cx.ModelError(Syntax, "Unexpected array type"); + return; + } + + var child = 0; + + foreach (var rank in arrayType.RankSpecifiers.SelectMany(rs => rs.Sizes)) + { + Create(cx, rank, this, child++); + } + + cx.Emit(Tuples.explicitly_sized_array_creation(this)); + } + } + + class ExplicitArrayCreation : ArrayCreation + { + ExplicitArrayCreation(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.ARRAY_CREATION)) { } + + public static Expression Create(ExpressionNodeInfo info) => new ExplicitArrayCreation(info).TryPopulate(); + + protected override void Populate() + { + var child = 0; + bool explicitlySized = false; + + foreach (var rank in Syntax.Type.RankSpecifiers.SelectMany(rs => rs.Sizes)) + { + if (rank is OmittedArraySizeExpressionSyntax) + { + // Create an expression which simulates the explicit size of the array + + if (Syntax.Initializer != null) + { + // An implicitly-sized array must have an initializer. + // Guard it just in case. + var size = Syntax.Initializer.Expressions.Count; + + var info = new ExpressionInfo( + cx, + Type.Create(cx, cx.Compilation.GetSpecialType(Microsoft.CodeAnalysis.SpecialType.System_Int32)), + Location, + ExprKind.INT_LITERAL, + this, + child, + false, + size.ToString()); + + new Expression(info); + } + } + else + { + Create(cx, rank, this, child); + explicitlySized = true; + } + child++; + } + if (Syntax.Initializer != null) + { + ArrayInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, -1)); + } + + if (explicitlySized) + cx.Emit(Tuples.explicitly_sized_array_creation(this)); + } + } + + class ImplicitArrayCreation : ArrayCreation + { + ImplicitArrayCreation(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.ARRAY_CREATION)) { } + + public static Expression Create(ExpressionNodeInfo info) => new ImplicitArrayCreation(info).TryPopulate(); + + protected override void Populate() + { + if (Syntax.Initializer != null) + { + ArrayInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, -1)); + } + + cx.Emit(Tuples.implicitly_typed_array_creation(this)); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs new file mode 100644 index 000000000000..921787cabc5e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Assignment.cs @@ -0,0 +1,154 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Assignment : Expression + { + Assignment(ExpressionNodeInfo info) + : base(info.SetKind(GetKind(info.Context, (AssignmentExpressionSyntax)info.Node))) + { + } + + public static Assignment Create(ExpressionNodeInfo info) + { + var ret = new Assignment(info); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + var operatorKind = OperatorKind; + if (operatorKind.HasValue) + { + // Convert assignment such as `a += b` into `a = a + b`. + var simpleAssignExpr = new Expression(new ExpressionInfo(cx, Type, Location, ExprKind.SIMPLE_ASSIGN, this, 2, false, null)); + Create(cx, Syntax.Left, simpleAssignExpr, 1); + var opexpr = new Expression(new ExpressionInfo(cx, Type, Location, operatorKind.Value, simpleAssignExpr, 0, false, null)); + Create(cx, Syntax.Left, opexpr, 0); + Create(cx, Syntax.Right, opexpr, 1); + opexpr.OperatorCall(Syntax); + } + else + { + Create(cx, Syntax.Left, this, 1); + Create(cx, Syntax.Right, this, 0); + + if (Kind == ExprKind.ADD_EVENT || Kind == ExprKind.REMOVE_EVENT) + { + OperatorCall(Syntax); + } + } + } + + static ExprKind GetAssignmentOperation(Context cx, AssignmentExpressionSyntax syntax) + { + switch (syntax.OperatorToken.Kind()) + { + case SyntaxKind.PlusEqualsToken: + return ExprKind.ASSIGN_ADD; + case SyntaxKind.MinusEqualsToken: + return ExprKind.ASSIGN_SUB; + case SyntaxKind.EqualsToken: + return ExprKind.SIMPLE_ASSIGN; + case SyntaxKind.BarEqualsToken: + return ExprKind.ASSIGN_OR; + case SyntaxKind.AmpersandEqualsToken: + return ExprKind.ASSIGN_AND; + case SyntaxKind.CaretEqualsToken: + return ExprKind.ASSIGN_XOR; + case SyntaxKind.AsteriskEqualsToken: + return ExprKind.ASSIGN_MUL; + case SyntaxKind.PercentEqualsToken: + return ExprKind.ASSIGN_REM; + case SyntaxKind.SlashEqualsToken: + return ExprKind.ASSIGN_DIV; + case SyntaxKind.LessThanLessThanEqualsToken: + return ExprKind.ASSIGN_LSHIFT; + case SyntaxKind.GreaterThanGreaterThanEqualsToken: + return ExprKind.ASSIGN_RSHIFT; + default: + cx.ModelError(syntax, "Unrecognised assignment type " + GetKind(cx, syntax)); + return ExprKind.UNKNOWN; + } + } + + static ExprKind GetKind(Context cx, AssignmentExpressionSyntax syntax) + { + var leftSymbol = cx.GetSymbolInfo(syntax.Left); + bool assignEvent = leftSymbol.Symbol != null && leftSymbol.Symbol is IEventSymbol; + var kind = GetAssignmentOperation(cx, syntax); + var leftType = cx.GetType(syntax.Left); + + if (leftType != null && leftType.SpecialType != SpecialType.None) + { + // In Mono, the builtin types did not specify their operator invocation + // even though EVERY operator has an invocation in C#. (This is a flaw in the dbscheme and should be fixed). + return kind; + } + + if (kind == ExprKind.ASSIGN_ADD && assignEvent) + { + return ExprKind.ADD_EVENT; + } + + if (kind == ExprKind.ASSIGN_SUB && assignEvent) + { + return ExprKind.REMOVE_EVENT; + } + + return kind; + } + + /// + /// Gets the kind of this assignment operator (null if the + /// assignment is not an assignment operator). For example, the operator + /// kind of `*=` is `*`. + /// + ExprKind? OperatorKind + { + get + { + var kind = Kind; + if (kind == ExprKind.REMOVE_EVENT || kind == ExprKind.ADD_EVENT || kind == ExprKind.SIMPLE_ASSIGN) + return null; + + if (CallType.AdjustKind(kind) == ExprKind.OPERATOR_INVOCATION) + return ExprKind.OPERATOR_INVOCATION; + + switch (kind) + { + case ExprKind.ASSIGN_ADD: + return ExprKind.ADD; + case ExprKind.ASSIGN_AND: + return ExprKind.BIT_AND; + case ExprKind.ASSIGN_DIV: + return ExprKind.DIV; + case ExprKind.ASSIGN_LSHIFT: + return ExprKind.LSHIFT; + case ExprKind.ASSIGN_MUL: + return ExprKind.MUL; + case ExprKind.ASSIGN_OR: + return ExprKind.BIT_OR; + case ExprKind.ASSIGN_REM: + return ExprKind.REM; + case ExprKind.ASSIGN_RSHIFT: + return ExprKind.RSHIFT; + case ExprKind.ASSIGN_SUB: + return ExprKind.SUB; + case ExprKind.ASSIGN_XOR: + return ExprKind.BIT_XOR; + default: + cx.ModelError(Syntax, "Couldn't unfold assignment of type " + kind); + return ExprKind.UNKNOWN; + } + } + } + + public new CallType CallType => GetCallType(cx, Syntax); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Await.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Await.cs new file mode 100644 index 000000000000..d7c758a35392 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Await.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Await : Expression + { + Await(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.AWAIT)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Await(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Base.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Base.cs new file mode 100644 index 000000000000..e67c18245f65 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Base.cs @@ -0,0 +1,12 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Base : Expression + { + Base(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.BASE_ACCESS)) { } + + public static Base Create(ExpressionNodeInfo info) => new Base(info); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Binary.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Binary.cs new file mode 100644 index 000000000000..afe1f351b21d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Binary.cs @@ -0,0 +1,61 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Binary : Expression + { + Binary(ExpressionNodeInfo info) + : base(info.SetKind(GetKind(info.Context, (BinaryExpressionSyntax)info.Node))) + { + } + + public static Expression Create(ExpressionNodeInfo info) => new Binary(info).TryPopulate(); + + protected override void Populate() + { + OperatorCall(Syntax); + CreateDeferred(cx, Syntax.Left, this, 0); + CreateDeferred(cx, Syntax.Right, this, 1); + } + + static ExprKind GetKind(Context cx, BinaryExpressionSyntax node) + { + var k = GetBinaryTokenKind(cx, node.OperatorToken.Kind()); + return GetCallType(cx, node).AdjustKind(k); + } + + static ExprKind GetBinaryTokenKind(Context cx, SyntaxKind kind) + { + switch (kind) + { + case SyntaxKind.LessThanToken: return ExprKind.LT; + case SyntaxKind.PlusToken: return ExprKind.ADD; + case SyntaxKind.LessThanEqualsToken: return ExprKind.LE; + case SyntaxKind.GreaterThanToken: return ExprKind.GT; + case SyntaxKind.AsteriskToken: return ExprKind.MUL; + case SyntaxKind.AmpersandAmpersandToken: return ExprKind.LOG_AND; + case SyntaxKind.EqualsEqualsToken: return ExprKind.EQ; + case SyntaxKind.PercentToken: return ExprKind.REM; + case SyntaxKind.MinusToken: return ExprKind.SUB; + case SyntaxKind.AmpersandToken: return ExprKind.BIT_AND; + case SyntaxKind.BarToken: return ExprKind.BIT_OR; + case SyntaxKind.SlashToken: return ExprKind.DIV; + case SyntaxKind.ExclamationEqualsToken: return ExprKind.NE; + case SyntaxKind.AsKeyword: return ExprKind.AS; + case SyntaxKind.IsKeyword: return ExprKind.IS; + case SyntaxKind.BarBarToken: return ExprKind.LOG_OR; + case SyntaxKind.GreaterThanEqualsToken: return ExprKind.GE; + case SyntaxKind.GreaterThanGreaterThanToken: return ExprKind.RSHIFT; + case SyntaxKind.LessThanLessThanToken: return ExprKind.LSHIFT; + case SyntaxKind.CaretToken: return ExprKind.BIT_XOR; + case SyntaxKind.QuestionQuestionToken: return ExprKind.NULL_COALESCING; + // !! And the rest + default: + cx.ModelError("Unhandled operator type {0}", kind); + return ExprKind.UNKNOWN; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs new file mode 100644 index 000000000000..8a99f15f5bf3 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Cast.cs @@ -0,0 +1,29 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Cast : Expression + { + Cast(ExpressionNodeInfo info) : base(info.SetKind(UnaryOperatorKind(info.Context, ExprKind.CAST, info.Node))) { } + + public static Expression Create(ExpressionNodeInfo info) => new Cast(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + + if (Kind == ExprKind.CAST) + // Type cast + TypeAccess.Create(new ExpressionNodeInfo(cx, Syntax.Type, this, 1)); + else + { + // Type conversion + OperatorCall(Syntax); + TypeMention.Create(cx, Syntax.Type, this, Type); + } + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => Syntax.GetLocation(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Checked.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Checked.cs new file mode 100644 index 000000000000..53c8f103dfac --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Checked.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Checked : Expression + { + Checked(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.CHECKED)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Checked(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Conditional.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Conditional.cs new file mode 100644 index 000000000000..5206e555ce1d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Conditional.cs @@ -0,0 +1,19 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Conditional : Expression + { + Conditional(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.CONDITIONAL)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Conditional(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Condition, this, 0); + Create(cx, Syntax.WhenTrue, this, 1); + Create(cx, Syntax.WhenFalse, this, 2); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs new file mode 100644 index 000000000000..b0470496b3e5 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Default.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Default : Expression + { + Default(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.DEFAULT)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Default(info).TryPopulate(); + + protected override void Populate() + { + TypeAccess.Create(cx, Syntax.Type, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs new file mode 100644 index 000000000000..32a8f373ef8a --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Discard.cs @@ -0,0 +1,16 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Discard : Expression + { + public Discard(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.DISCARD)) + { + } + + protected override void Populate() + { + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs new file mode 100644 index 000000000000..b979f8ca3a9b --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ElementAccess.cs @@ -0,0 +1,95 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + abstract class ElementAccess : Expression + { + protected ElementAccess(ExpressionNodeInfo info, ExpressionSyntax qualifier, BracketedArgumentListSyntax argumentList) + : base(info.SetKind(GetKind(info.Context, qualifier))) + { + Qualifier = qualifier; + ArgumentList = argumentList; + } + + readonly ExpressionSyntax Qualifier; + readonly BracketedArgumentListSyntax ArgumentList; + + protected override void Populate() + { + if (Kind == ExprKind.POINTER_INDIRECTION) + { + var qualifierInfo = new ExpressionNodeInfo(cx, Qualifier, this, 0); + var add = new Expression(new ExpressionInfo(cx, qualifierInfo.Type, Location, ExprKind.ADD, this, 0, false, null)); + qualifierInfo.SetParent(add, 0); + CreateFromNode(qualifierInfo); + PopulateArguments(cx, ArgumentList, 1); + } + else + { + var child = -1; + Create(cx, Qualifier, this, child++); + foreach (var a in ArgumentList.Arguments) + { + cx.Extract(a, this, child++); + } + + var symbolInfo = cx.GetSymbolInfo(base.Syntax); + + var indexer = symbolInfo.Symbol as IPropertySymbol; + if (indexer != null) + { + cx.Emit(Tuples.expr_access(this, Indexer.Create(cx, indexer))); + } + } + } + + public sealed override Microsoft.CodeAnalysis.Location ReportingLocation => base.ReportingLocation; + + static ExprKind GetKind(Context cx, ExpressionSyntax qualifier) + { + var qualifierType = cx.GetType(qualifier); + + // This is a compilation error, so make a guess and continue. + if (qualifierType == null) return ExprKind.ARRAY_ACCESS; + + if (qualifierType.TypeKind == Microsoft.CodeAnalysis.TypeKind.Pointer) + { + // Convert expressions of the form a[b] into *(a+b) + return ExprKind.POINTER_INDIRECTION; + } + + return IsDynamic(cx, qualifier) ? + ExprKind.DYNAMIC_ELEMENT_ACCESS : + qualifierType.TypeKind == Microsoft.CodeAnalysis.TypeKind.Array ? + ExprKind.ARRAY_ACCESS : + ExprKind.INDEXER_ACCESS; + } + } + + class NormalElementAccess : ElementAccess + { + NormalElementAccess(ExpressionNodeInfo info) + : base(info, ((ElementAccessExpressionSyntax)info.Node).Expression, ((ElementAccessExpressionSyntax)info.Node).ArgumentList) { } + + public static Expression Create(ExpressionNodeInfo info) => new NormalElementAccess(info).TryPopulate(); + } + + class BindingElementAccess : ElementAccess + { + BindingElementAccess(ExpressionNodeInfo info) + : base(info, FindConditionalQualifier(info.Node), ((ElementBindingExpressionSyntax)info.Node).ArgumentList) { } + + public static Expression Create(ExpressionNodeInfo info) => new BindingElementAccess(info).TryPopulate(); + + protected override void Populate() + { + base.Populate(); + MakeConditional(); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Factory.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Factory.cs new file mode 100644 index 000000000000..86e7164d4d8b --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Factory.cs @@ -0,0 +1,242 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + internal static class Factory + { + internal static Expression Create(ExpressionNodeInfo info) + { + // Some expressions can be extremely deep (e.g. string + string + string ...) + // to the extent that the stack has been known to overflow. + using (info.Context.StackGuard) + { + if (info.Node == null) + { + info.Context.ModelError("Attempt to create a null expression"); + return new Unknown(info); + } + + switch (info.Node.Kind()) + { + case SyntaxKind.AddExpression: + case SyntaxKind.SubtractExpression: + case SyntaxKind.LessThanExpression: + case SyntaxKind.LessThanOrEqualExpression: + case SyntaxKind.GreaterThanExpression: + case SyntaxKind.GreaterThanOrEqualExpression: + case SyntaxKind.MultiplyExpression: + case SyntaxKind.LogicalAndExpression: + case SyntaxKind.EqualsExpression: + case SyntaxKind.ModuloExpression: + case SyntaxKind.BitwiseAndExpression: + case SyntaxKind.BitwiseOrExpression: + case SyntaxKind.DivideExpression: + case SyntaxKind.NotEqualsExpression: + case SyntaxKind.LogicalOrExpression: + case SyntaxKind.IsExpression: + case SyntaxKind.AsExpression: + case SyntaxKind.RightShiftExpression: + case SyntaxKind.LeftShiftExpression: + case SyntaxKind.ExclusiveOrExpression: + case SyntaxKind.CoalesceExpression: + return Binary.Create(info); + + case SyntaxKind.FalseLiteralExpression: + case SyntaxKind.TrueLiteralExpression: + case SyntaxKind.StringLiteralExpression: + case SyntaxKind.NullLiteralExpression: + case SyntaxKind.NumericLiteralExpression: + case SyntaxKind.CharacterLiteralExpression: + case SyntaxKind.DefaultLiteralExpression: + return Literal.Create(info); + + case SyntaxKind.InvocationExpression: + return Invocation.Create(info); + + case SyntaxKind.PostIncrementExpression: + return PostfixUnary.Create(info.SetKind(ExprKind.POST_INCR), ((PostfixUnaryExpressionSyntax)info.Node).Operand); + + case SyntaxKind.PostDecrementExpression: + return PostfixUnary.Create(info.SetKind(ExprKind.POST_DECR), ((PostfixUnaryExpressionSyntax)info.Node).Operand); + + case SyntaxKind.AwaitExpression: + return Await.Create(info); + + case SyntaxKind.ElementAccessExpression: + return NormalElementAccess.Create(info); + + case SyntaxKind.SimpleAssignmentExpression: + case SyntaxKind.OrAssignmentExpression: + case SyntaxKind.AndAssignmentExpression: + case SyntaxKind.SubtractAssignmentExpression: + case SyntaxKind.AddAssignmentExpression: + case SyntaxKind.MultiplyAssignmentExpression: + case SyntaxKind.ExclusiveOrAssignmentExpression: + case SyntaxKind.LeftShiftAssignmentExpression: + case SyntaxKind.RightShiftAssignmentExpression: + case SyntaxKind.DivideAssignmentExpression: + case SyntaxKind.ModuloAssignmentExpression: + return Assignment.Create(info); + + case SyntaxKind.ObjectCreationExpression: + return ExplicitObjectCreation.Create(info); + + case SyntaxKind.ArrayCreationExpression: + return ExplicitArrayCreation.Create(info); + + case SyntaxKind.ObjectInitializerExpression: + return ObjectInitializer.Create(info); + + case SyntaxKind.ArrayInitializerExpression: + return ImplicitArrayInitializer.Create(info); + + case SyntaxKind.CollectionInitializerExpression: + return CollectionInitializer.Create(info); + + case SyntaxKind.ConditionalAccessExpression: + return MemberAccess.Create(info, (ConditionalAccessExpressionSyntax)info.Node); + + case SyntaxKind.SimpleMemberAccessExpression: + return MemberAccess.Create(info, (MemberAccessExpressionSyntax)info.Node); + + case SyntaxKind.UnaryMinusExpression: + return Unary.Create(info.SetKind(ExprKind.MINUS)); + + case SyntaxKind.UnaryPlusExpression: + return Unary.Create(info.SetKind(ExprKind.PLUS)); + + case SyntaxKind.SimpleLambdaExpression: + return Lambda.Create(info, (SimpleLambdaExpressionSyntax)info.Node); + + case SyntaxKind.ParenthesizedLambdaExpression: + return Lambda.Create(info, (ParenthesizedLambdaExpressionSyntax)info.Node); + + case SyntaxKind.ConditionalExpression: + return Conditional.Create(info); + + case SyntaxKind.CastExpression: + return Cast.Create(info); + + case SyntaxKind.ParenthesizedExpression: + return Create(info.SetNode(((ParenthesizedExpressionSyntax)info.Node).Expression)); + + case SyntaxKind.PointerType: + case SyntaxKind.ArrayType: + case SyntaxKind.PredefinedType: + case SyntaxKind.NullableType: + return TypeAccess.Create(info); + + case SyntaxKind.TypeOfExpression: + return TypeOf.Create(info); + + case SyntaxKind.QualifiedName: + case SyntaxKind.IdentifierName: + case SyntaxKind.AliasQualifiedName: + case SyntaxKind.GenericName: + return Name.Create(info); + + case SyntaxKind.LogicalNotExpression: + return Unary.Create(info.SetKind(ExprKind.LOG_NOT)); + + case SyntaxKind.BitwiseNotExpression: + return Unary.Create(info.SetKind(ExprKind.BIT_NOT)); + + case SyntaxKind.PreIncrementExpression: + return Unary.Create(info.SetKind(ExprKind.PRE_INCR)); + + case SyntaxKind.PreDecrementExpression: + return Unary.Create(info.SetKind(ExprKind.PRE_DECR)); + + case SyntaxKind.ThisExpression: + return This.CreateExplicit(info); + + case SyntaxKind.AddressOfExpression: + return Unary.Create(info.SetKind(ExprKind.ADDRESS_OF)); + + case SyntaxKind.PointerIndirectionExpression: + return Unary.Create(info.SetKind(ExprKind.POINTER_INDIRECTION)); + + case SyntaxKind.DefaultExpression: + return Default.Create(info); + + case SyntaxKind.CheckedExpression: + return Checked.Create(info); + + case SyntaxKind.UncheckedExpression: + return Unchecked.Create(info); + + case SyntaxKind.BaseExpression: + return Base.Create(info); + + case SyntaxKind.AnonymousMethodExpression: + return Lambda.Create(info, (AnonymousMethodExpressionSyntax)info.Node); + + case SyntaxKind.ImplicitArrayCreationExpression: + return ImplicitArrayCreation.Create(info); + + case SyntaxKind.AnonymousObjectCreationExpression: + return ImplicitObjectCreation.Create(info); + + case SyntaxKind.ComplexElementInitializerExpression: + return CollectionInitializer.Create(info); + + case SyntaxKind.SizeOfExpression: + return SizeOf.Create(info); + + case SyntaxKind.PointerMemberAccessExpression: + return PointerMemberAccess.Create(info); + + case SyntaxKind.QueryExpression: + return Query.Create(info); + + case SyntaxKind.InterpolatedStringExpression: + return InterpolatedString.Create(info); + + case SyntaxKind.MemberBindingExpression: + return MemberAccess.Create(info, (MemberBindingExpressionSyntax)info.Node); + + case SyntaxKind.ElementBindingExpression: + return BindingElementAccess.Create(info); + + case SyntaxKind.StackAllocArrayCreationExpression: + return StackAllocArrayCreation.Create(info); + + case SyntaxKind.ArgListExpression: + return ArgList.Create(info); + + case SyntaxKind.RefTypeExpression: + return RefType.Create(info); + + case SyntaxKind.RefValueExpression: + return RefValue.Create(info); + + case SyntaxKind.MakeRefExpression: + return MakeRef.Create(info); + + case SyntaxKind.ThrowExpression: + return Throw.Create(info); + + case SyntaxKind.DeclarationExpression: + return VariableDeclaration.Create(info.Context, (DeclarationExpressionSyntax)info.Node, info.Parent, info.Child); + + case SyntaxKind.TupleExpression: + return Tuple.Create(info); + + case SyntaxKind.RefExpression: + return Ref.Create(info); + + case SyntaxKind.IsPatternExpression: + return IsPattern.Create(info); + + default: + info.Context.ModelError(info.Node, "Unhandled expression '{0}' of kind '{1}'", info.Node, info.Node.Kind()); + return new Unknown(info); + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs new file mode 100644 index 000000000000..f92102151ba4 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ImplicitCast.cs @@ -0,0 +1,92 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class ImplicitCast : Expression + { + public Expression Expr + { + get; + private set; + } + + public ImplicitCast(ExpressionNodeInfo info) + : base(new ExpressionInfo(info.Context, Type.Create(info.Context, info.ConvertedType), info.Location, ExprKind.CAST, info.Parent, info.Child, true, info.ExprValue)) + { + Expr = Factory.Create(new ExpressionNodeInfo(cx, info.Node, this, 0)); + } + + public ImplicitCast(ExpressionNodeInfo info, IMethodSymbol method) + : base(new ExpressionInfo(info.Context, Type.Create(info.Context, info.ConvertedType), info.Location, ExprKind.OPERATOR_INVOCATION, info.Parent, info.Child, true, info.ExprValue) ) + { + Expr = Factory.Create(info.SetParent(this, 0)); + + var target = Method.Create(cx, method); + if (target != null) + cx.Emit(Tuples.expr_call(this, target)); + else + cx.ModelError(info.Node, "Failed to resolve target for operator invocation"); + } + + /// + /// Creates a new expression, adding casts as required. + /// + /// The extraction context. + /// The expression node. + /// The parent of the expression. + /// The child number. + /// A type hint. + /// A new expression. + public static Expression Create(ExpressionNodeInfo info) + { + var resolvedType = info.ResolvedType; + var convertedType = info.ConvertedType; + var conversion = info.Conversion; + + if (conversion.MethodSymbol != null) + { + bool convertedToDelegate = Type.IsDelegate(convertedType); + + if (convertedToDelegate) + { + var objectCreation = info.Parent as ExplicitObjectCreation; + bool isExplicitConversion = objectCreation != null && objectCreation.Kind == ExprKind.EXPLICIT_DELEGATE_CREATION; + + if (!isExplicitConversion) + { + info.Kind = ExprKind.IMPLICIT_DELEGATE_CREATION; + var parent = new Expression(info); + return Factory.Create(new ExpressionNodeInfo(info.Context, info.Node, parent, 0)); + } + + info.Kind = ExprKind.UNKNOWN; + return Factory.Create(info); + } + + if (resolvedType != null) + return new ImplicitCast(info, conversion.MethodSymbol); + } + + bool implicitUpcast = conversion.IsImplicit && + convertedType != null && + !conversion.IsBoxing && + ( + resolvedType == null || + conversion.IsReference || + convertedType.SpecialType == SpecialType.System_Object) + ; + + if (!conversion.IsIdentity && !implicitUpcast) + { + return new ImplicitCast(info); + } + + // Default: Just create the expression without a conversion. + return Factory.Create(info); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs new file mode 100644 index 000000000000..317ce29310ee --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Initializer.cs @@ -0,0 +1,137 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + abstract class Initializer : Expression + { + protected Initializer(ExpressionNodeInfo info) : base(info) { } + } + + class ArrayInitializer : Expression + { + ArrayInitializer(ExpressionNodeInfo info) : base(info.SetType(Type.Create(info.Context, null)).SetKind(ExprKind.ARRAY_INIT)) { } + + public static Expression Create(ExpressionNodeInfo info) => new ArrayInitializer(info).TryPopulate(); + + protected override void Populate() + { + var child = 0; + foreach (var e in Syntax.Expressions) + { + if (e.Kind() == SyntaxKind.ArrayInitializerExpression) + { + // Recursively create another array initializer + Create(new ExpressionNodeInfo(cx, (InitializerExpressionSyntax)e, this, child++)); + } + else + { + // Create the expression normally. + Create(cx, e, this, child++); + } + } + } + } + + // Array initializer { ..., ... }. + class ImplicitArrayInitializer : Initializer + { + ImplicitArrayInitializer(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.ARRAY_CREATION)) { } + + public static Expression Create(ExpressionNodeInfo info) => new ImplicitArrayInitializer(info).TryPopulate(); + + protected override void Populate() + { + ArrayInitializer.Create(new ExpressionNodeInfo(cx, Syntax, this, -1)); + cx.Emit(Tuples.implicitly_typed_array_creation(this)); + } + } + + class ObjectInitializer : Initializer + { + ObjectInitializer(ExpressionNodeInfo info) + : base(info.SetKind(ExprKind.OBJECT_INIT)) { } + + public static Expression Create(ExpressionNodeInfo info) => new ObjectInitializer(info).TryPopulate(); + + protected override void Populate() + { + var child = 0; + + foreach (var init in Syntax.Expressions) + { + var assignment = init as AssignmentExpressionSyntax; + + if (assignment != null) + { + var assignmentEntity = new Expression(new ExpressionNodeInfo(cx, init, this, child++).SetKind(ExprKind.SIMPLE_ASSIGN)); + + CreateFromNode(new ExpressionNodeInfo(cx, assignment.Right, assignmentEntity, 0)); + + var target = cx.GetSymbolInfo(assignment.Left); + if (target.Symbol == null) + { + cx.ModelError(assignment, "Unknown object initializer"); + new Unknown(new ExpressionNodeInfo(cx, assignment.Left, assignmentEntity, 1)); + } + else + { + Access.Create(new ExpressionNodeInfo(cx, assignment.Left, assignmentEntity, 1), target.Symbol, false, cx.CreateEntity(target.Symbol)); + } + } + else + { + cx.ModelError(init, "Unexpected object initialization"); + Create(cx, init, this, child++); + } + } + } + } + + class CollectionInitializer : Initializer + { + CollectionInitializer(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.COLLECTION_INIT)) { } + + public static Expression Create(ExpressionNodeInfo info) => new CollectionInitializer(info).TryPopulate(); + + protected override void Populate() + { + var child = 0; + foreach (var i in Syntax.Expressions) + { + var collectionInfo = cx.Model(Syntax).GetCollectionInitializerSymbolInfo(i); + var addMethod = Method.Create(cx, collectionInfo.Symbol as IMethodSymbol); + var voidType = Type.Create(cx, cx.Compilation.GetSpecialType(SpecialType.System_Void)); + + var invocation = new Expression(new ExpressionInfo(cx, voidType, cx.Create(i.GetLocation()), ExprKind.METHOD_INVOCATION, this, child++, false, null)); + + if (addMethod != null) + cx.Emit(Tuples.expr_call(invocation, addMethod)); + else + cx.ModelError(Syntax, "Unable to find an Add() method for collection initializer"); + + if (i.Kind() == SyntaxKind.ComplexElementInitializerExpression) + { + // Arrays of the form new Foo { { 1,2 }, { 3, 4 } } + // where the arguments { 1, 2 } are passed to the Add() method. + + var init = (InitializerExpressionSyntax)i; + + int addChild = 0; + foreach (var arg in init.Expressions) + { + Create(cx, arg, invocation, addChild++); + } + } + else + { + Create(cx, i, invocation, 0); + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs new file mode 100644 index 000000000000..fa24766bb8ff --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/InterpolatedString.cs @@ -0,0 +1,36 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class InterpolatedString : Expression + { + InterpolatedString(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.INTERPOLATED_STRING)) { } + + public static Expression Create(ExpressionNodeInfo info) => new InterpolatedString(info).TryPopulate(); + + protected override void Populate() + { + var child = 0; + foreach (var c in Syntax.Contents) + { + switch (c.Kind()) + { + case SyntaxKind.Interpolation: + var interpolation = (InterpolationSyntax)c; + Create(cx, interpolation.Expression, this, child++); + break; + case SyntaxKind.InterpolatedStringText: + // Create a string literal + var interpolatedText = (InterpolatedStringTextSyntax)c; + new Expression(new ExpressionInfo(cx, Type, cx.Create(c.GetLocation()), ExprKind.STRING_LITERAL, this, child++, false, interpolatedText.TextToken.Text)); + break; + default: + throw new InternalError(c, "Unhandled interpolation kind {0}", c.Kind()); + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs new file mode 100644 index 000000000000..750034dc7715 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Invocation.cs @@ -0,0 +1,181 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Invocation : Expression + { + Invocation(ExpressionNodeInfo info) + : base(info.SetKind(GetKind(info))) + { + this.info = info; + } + + readonly ExpressionNodeInfo info; + + public static Expression Create(ExpressionNodeInfo info) => new Invocation(info).TryPopulate(); + + protected override void Populate() + { + if (IsNameof(Syntax)) + { + PopulateArguments(cx, Syntax.ArgumentList, 0); + return; + } + + var child = -1; + string memberName = null; + var target = TargetSymbol; + switch (Syntax.Expression) + { + case MemberAccessExpressionSyntax memberAccess: + memberName = memberAccess.Name.Identifier.Text; + if (Syntax.Expression.Kind() == SyntaxKind.SimpleMemberAccessExpression) + // Qualified method call; `x.M()` + Create(cx, memberAccess.Expression, this, child++); + else + // Pointer member access; `x->M()` + Create(cx, Syntax.Expression, this, child++); + break; + case MemberBindingExpressionSyntax memberBinding: + // Conditionally qualified method call; `x?.M()` + memberName = memberBinding.Name.Identifier.Text; + Create(cx, FindConditionalQualifier(memberBinding), this, child++); + MakeConditional(); + break; + case SimpleNameSyntax simpleName when (Kind == ExprKind.METHOD_INVOCATION): + // Unqualified method call; `M()` + memberName = simpleName.Identifier.Text; + if (target != null && !target.IsStatic) + { + // Implicit `this` qualifier; add explicitly + var callingMethod = cx.Model(Syntax).GetEnclosingSymbol(Location.symbol.SourceSpan.Start) as IMethodSymbol; + + if (callingMethod == null) + cx.ModelError(Syntax, "Couldn't determine implicit this type"); + else + This.CreateImplicit(cx, Type.Create(cx, callingMethod.ContainingType), Location, this, child++); + } + else + { + // No implicit `this` qualifier + child++; + } + break; + default: + // Delegate call; `d()` + Create(cx, Syntax.Expression, this, child++); + break; + } + + var isDynamicCall = IsDynamicCall(info); + if (isDynamicCall) + { + if (memberName != null) + cx.Emit(Tuples.dynamic_member_name(this, memberName)); + else + cx.ModelError(Syntax, "Unable to get name for dynamic call."); + } + + PopulateArguments(cx, Syntax.ArgumentList, child); + + if (target == null) + { + if (!isDynamicCall && !IsDelegateCall(info)) + cx.ModelError(Syntax, "Unable to resolve target for call. (Compilation error?)"); + return; + } + + var targetKey = Method.Create(cx, target); + cx.Emit(Tuples.expr_call(this, targetKey)); + } + + static bool IsDynamicCall(ExpressionNodeInfo info) + { + // Either the qualifier (Expression) is dynamic, + // or one of the arguments is dynamic. + var node = (InvocationExpressionSyntax)info.Node; + return !IsDelegateCall(info) && + (IsDynamic(info.Context, node.Expression) || node.ArgumentList.Arguments.Any(arg => IsDynamic(info.Context, arg.Expression))); + } + + public SymbolInfo SymbolInfo => info.SymbolInfo; + + public IMethodSymbol TargetSymbol + { + get + { + var si = SymbolInfo; + + if (si.Symbol != null) return si.Symbol as IMethodSymbol; + + if (si.CandidateReason == CandidateReason.OverloadResolutionFailure) + { + // This seems to be a bug in Roslyn + // For some reason, typeof(X).InvokeMember(...) fails to resolve the correct + // InvokeMember() method, even though the number of parameters clearly identifies the correct method + + var candidates = si.CandidateSymbols. + OfType(). + Where(method => method.Parameters.Length >= Syntax.ArgumentList.Arguments.Count). + Where(method => method.Parameters.Count(p => !p.HasExplicitDefaultValue) <= Syntax.ArgumentList.Arguments.Count); + + return cx.Extractor.Standalone ? + candidates.FirstOrDefault() : + candidates.SingleOrDefault(); + } + + return si.Symbol as IMethodSymbol; + } + } + + static bool IsDelegateCall(ExpressionNodeInfo info) + { + var si = info.SymbolInfo; + + if (si.CandidateReason == CandidateReason.OverloadResolutionFailure && + si.CandidateSymbols.OfType().All(s => s.MethodKind == MethodKind.DelegateInvoke)) + return true; + + // Delegate variable is a dynamic + var node = (InvocationExpressionSyntax)info.Node; + if (si.CandidateReason == CandidateReason.LateBound && + node.Expression is IdentifierNameSyntax && + IsDynamic(info.Context, node.Expression) && + si.Symbol == null) + return true; + + return si.Symbol != null && + si.Symbol.Kind == SymbolKind.Method && + ((IMethodSymbol)si.Symbol).MethodKind == MethodKind.DelegateInvoke; + } + + static bool IsLocalFunctionInvocation(ExpressionNodeInfo info) + { + var target = info.SymbolInfo.Symbol as IMethodSymbol; + return target != null && target.MethodKind == MethodKind.LocalFunction; + } + + static ExprKind GetKind(ExpressionNodeInfo info) + { + return IsNameof((InvocationExpressionSyntax)info.Node) ? + ExprKind.NAMEOF : + IsDelegateCall(info) ? + ExprKind.DELEGATE_INVOCATION : + IsLocalFunctionInvocation(info) ? + ExprKind.LOCAL_FUNCTION_INVOCATION : + ExprKind.METHOD_INVOCATION; + } + + static bool IsNameof(InvocationExpressionSyntax syntax) + { + // Odd that this is not a separate expression type. + // Maybe it will be in the future. + var id = syntax.Expression as IdentifierNameSyntax; + return id != null && id.Identifier.Text == "nameof"; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/IsPattern.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/IsPattern.cs new file mode 100644 index 000000000000..e26605931cab --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/IsPattern.cs @@ -0,0 +1,46 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Kinds; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class IsPattern : Expression + { + IsPattern(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.IS)) + { + } + + protected override void Populate() + { + var constantPattern = Syntax.Pattern as ConstantPatternSyntax; + if (constantPattern != null) + { + Create(cx, Syntax.Expression, this, 0); + Create(cx, constantPattern.Expression, this, 3); + return; + } + + var pattern = Syntax.Pattern as DeclarationPatternSyntax; + + if (pattern == null) + { + throw new InternalError(Syntax, "Is-pattern not handled"); + } + + Create(cx, Syntax.Expression, this, 0); + TypeAccess.Create(cx, pattern.Type, this, 1); + + var symbol = cx.Model(Syntax).GetDeclaredSymbol(pattern.Designation) as ILocalSymbol; + if (symbol != null) + { + var type = Type.Create(cx, symbol.Type); + var isVar = pattern.Type.IsVar; + VariableDeclaration.Create(cx, symbol, type, cx.Create(pattern.GetLocation()), cx.Create(pattern.Designation.GetLocation()), isVar, this, 2); + } + } + + public static Expression Create(ExpressionNodeInfo info) => new IsPattern(info).TryPopulate(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs new file mode 100644 index 000000000000..3a82004c5b74 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Lambda.cs @@ -0,0 +1,56 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; +using System.Collections.Generic; +using Semmle.Util; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Lambda : Expression, IStatementParentEntity + { + bool IStatementParentEntity.IsTopLevelParent => false; + + protected override void Populate() { } + + void VisitParameter(Context cx, ParameterSyntax p) + { + var symbol = cx.Model(p).GetDeclaredSymbol(p); + Parameter.Create(cx, symbol, this); + } + + Lambda(ExpressionNodeInfo info, CSharpSyntaxNode body, IEnumerable @params) + : base(info) + { + // No need to use `Populate` as the population happens later + cx.PopulateLater(() => + { + foreach (var param in @params) + VisitParameter(cx, param); + + if (body is ExpressionSyntax) + Create(cx, (ExpressionSyntax)body, this, 0); + else if (body is BlockSyntax) + Statements.Block.Create(cx, (BlockSyntax)body, this, 0); + else + cx.ModelError(body, "Unhandled lambda body"); + }); + } + + Lambda(ExpressionNodeInfo info, ParenthesizedLambdaExpressionSyntax node) + : this(info.SetKind(ExprKind.LAMBDA), node.Body, node.ParameterList.Parameters) { } + + public static Lambda Create(ExpressionNodeInfo info, ParenthesizedLambdaExpressionSyntax node) => new Lambda(info, node); + + Lambda(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node) + : this(info.SetKind(ExprKind.LAMBDA), node.Body, Enumerators.Singleton(node.Parameter)) { } + + public static Lambda Create(ExpressionNodeInfo info, SimpleLambdaExpressionSyntax node) => new Lambda(info, node); + + Lambda(ExpressionNodeInfo info, AnonymousMethodExpressionSyntax node) : + this(info.SetKind(ExprKind.ANONYMOUS_METHOD), node.Body, node.ParameterList == null ? Enumerable.Empty() : node.ParameterList.Parameters) { } + + public static Lambda Create(ExpressionNodeInfo info, AnonymousMethodExpressionSyntax node) => new Lambda(info, node); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs new file mode 100644 index 000000000000..5dcc6564dfe6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Literal.cs @@ -0,0 +1,72 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp.Populators; +using Microsoft.CodeAnalysis.CSharp; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Literal : Expression + { + Literal(ExpressionNodeInfo info) : base(info.SetKind(GetKind(info)) ) { } + + public static Expression Create(ExpressionNodeInfo info) => new Literal(info).TryPopulate(); + + protected override void Populate() { } + + static ExprKind GetKind(ExpressionNodeInfo info) + { + switch(info.Node.Kind()) + { + case SyntaxKind.DefaultLiteralExpression: + return ExprKind.DEFAULT; + case SyntaxKind.NullLiteralExpression: + info.Type = Type.Create(info.Context, null); // Don't use converted type. + return ExprKind.NULL_LITERAL; + } + + var type = info.Type.symbol; + + switch (type.SpecialType) + { + case SpecialType.System_Boolean: + return ExprKind.BOOL_LITERAL; + + case SpecialType.System_Int16: + case SpecialType.System_Int32: + case SpecialType.System_Byte: // ?? + return ExprKind.INT_LITERAL; + + case SpecialType.System_Char: + return ExprKind.CHAR_LITERAL; + + case SpecialType.System_Decimal: + return ExprKind.DECIMAL_LITERAL; + + case SpecialType.System_Double: + return ExprKind.DOUBLE_LITERAL; + + case SpecialType.System_Int64: + return ExprKind.LONG_LITERAL; + + case SpecialType.System_Single: + return ExprKind.FLOAT_LITERAL; + + case SpecialType.System_String: + return ExprKind.STRING_LITERAL; + + case SpecialType.System_UInt16: + case SpecialType.System_UInt32: + case SpecialType.System_SByte: // ?? + return ExprKind.UINT_LITERAL; + + case SpecialType.System_UInt64: + return ExprKind.ULONG_LITERAL; + + default: + info.Context.ModelError(info.Node, "Unhandled literal type"); + return ExprKind.UNKNOWN; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MakeRef.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MakeRef.cs new file mode 100644 index 000000000000..168f413ae82f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MakeRef.cs @@ -0,0 +1,17 @@ +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class MakeRef : Expression + { + MakeRef(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.REF)) { } + + public static Expression Create(ExpressionNodeInfo info) => new MakeRef(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs new file mode 100644 index 000000000000..7b378f83210d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/MemberAccess.cs @@ -0,0 +1,101 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class MemberAccess : Expression + { + private MemberAccess(ExpressionNodeInfo info, ExpressionSyntax qualifier, ISymbol target) : base(info) + { + Qualifier = Create(cx, qualifier, this, -1); + + if (target == null) + { + if (info.Kind != ExprKind.DYNAMIC_MEMBER_ACCESS) + cx.ModelError(info.Node, "Could not determine target for member access"); + } + else + { + cx.Emit(Tuples.expr_access(this, cx.CreateEntity(target))); + } + } + + public static Expression Create(ExpressionNodeInfo info, ConditionalAccessExpressionSyntax node) => + // The qualifier is located by walking the syntax tree. + // `node.WhenNotNull` will contain a MemberBindingExpressionSyntax, calling the method below. + Create(info.Context, node.WhenNotNull, info.Parent, info.Child); + + public static Expression Create(ExpressionNodeInfo info, MemberBindingExpressionSyntax node) + { + var expr = Create(info, FindConditionalQualifier(node), node.Name); + expr.MakeConditional(); + return expr; + } + + public static Expression Create(ExpressionNodeInfo info, MemberAccessExpressionSyntax node) => + Create(info, node.Expression, node.Name); + + static Expression Create(ExpressionNodeInfo info, ExpressionSyntax expression, SimpleNameSyntax name) + { + if (IsDynamic(info.Context, expression)) + { + var expr = new MemberAccess(info.SetKind(ExprKind.DYNAMIC_MEMBER_ACCESS), expression, null); + info.Context.Emit(Tuples.dynamic_member_name(expr, name.Identifier.Text)); + return expr; + } + + var target = info.SymbolInfo; + + if (target.CandidateReason == CandidateReason.OverloadResolutionFailure) + { + // Roslyn workaround. Even if we can't resolve a method, we know it's a method. + return Create(info.Context, expression, info.Parent, info.Child); + } + + var symbol = target.Symbol ?? info.Context.GetSymbolInfo(name).Symbol; + + if (symbol == null && target.CandidateSymbols.Length >= 1) + { + // Pick the first symbol. This could occur for something like `nameof(Foo.Bar)` + // where `Bar` is a method group. Technically, we don't know which symbol is accessed. + symbol = target.CandidateSymbols[0]; + } + + if (symbol == null) + { + info.Context.ModelError(info.Node, "Failed to determine symbol for member access"); + return new MemberAccess(info.SetKind(ExprKind.UNKNOWN), expression, symbol); + } + + ExprKind kind; + + switch (symbol.Kind) + { + case SymbolKind.Property: + kind = ExprKind.PROPERTY_ACCESS; + break; + case SymbolKind.Method: + kind = ExprKind.METHOD_ACCESS; + break; + case SymbolKind.Field: + kind = ExprKind.FIELD_ACCESS; + break; + case SymbolKind.NamedType: + return TypeAccess.Create(info); + case SymbolKind.Event: + kind = ExprKind.EVENT_ACCESS; + break; + default: + info.Context.ModelError(info.Node, "Unhandled symbol for member access"); + kind = ExprKind.UNKNOWN; + break; + } + return new MemberAccess(info.SetKind(kind), expression, symbol); + } + + public Expression Qualifier { get; private set; } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Name.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Name.cs new file mode 100644 index 000000000000..c76753789adf --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Name.cs @@ -0,0 +1,66 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + static class Name + { + public static Expression Create(ExpressionNodeInfo info) + { + var symbolInfo = info.Context.GetSymbolInfo(info.Node); + + var target = symbolInfo.Symbol; + + if (target == null && symbolInfo.CandidateReason == CandidateReason.OverloadResolutionFailure) + { + // The expression is probably a cast + target = info.Context.GetSymbolInfo((CSharpSyntaxNode)info.Node.Parent).Symbol; + } + + if (target == null && (symbolInfo.CandidateReason == CandidateReason.Ambiguous || symbolInfo.CandidateReason == CandidateReason.MemberGroup)) + { + // Pick one at random - they probably resolve to the same ID + target = symbolInfo.CandidateSymbols.First(); + } + + if (target == null) + { + info.Context.ModelError(info.Node, "Failed to resolve name"); + return new Unknown(info); + } + + // There is a very strange bug in Microsoft.CodeAnalysis whereby + // target.Kind throws System.InvalidOperationException for Discard symbols. + // So, short-circuit that test here. + // Ideally this would be another case in the switch statement below. + if (target is IDiscardSymbol) + return new Discard(info); + + switch (target.Kind) + { + case SymbolKind.TypeParameter: + case SymbolKind.NamedType: + return TypeAccess.Create(info); + + case SymbolKind.Property: + case SymbolKind.Field: + case SymbolKind.Event: + case SymbolKind.Method: + return Access.Create(info, target, true, info.Context.CreateEntity(target)); + + case SymbolKind.Local: + case SymbolKind.RangeVariable: + return Access.Create(info, target, false, LocalVariable.GetAlreadyCreated(info.Context, target)); + + case SymbolKind.Parameter: + return Access.Create(info, target, false, Parameter.GetAlreadyCreated(info.Context, (IParameterSymbol)target)); + + default: + throw new InternalError(info.Node, "Unhandled identifier kind '{0}'", target.Kind); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation.cs new file mode 100644 index 000000000000..e90d04df2008 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/ObjectCreation.cs @@ -0,0 +1,136 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + abstract class ObjectCreation : Expression where SyntaxNode : ExpressionSyntax + { + protected ObjectCreation(ExpressionNodeInfo info) + : base(info) { } + } + + // new Foo(...) { ... }. + class ExplicitObjectCreation : ObjectCreation + { + static bool IsDynamicObjectCreation(Context cx, ObjectCreationExpressionSyntax node) + { + return node.ArgumentList != null && node.ArgumentList.Arguments.Any(arg => IsDynamic(cx, arg.Expression)); + } + + static ExprKind GetKind(Context cx, ObjectCreationExpressionSyntax node) + { + var si = cx.Model(node).GetSymbolInfo(node.Type); + return Type.IsDelegate(si.Symbol as INamedTypeSymbol) ? ExprKind.EXPLICIT_DELEGATE_CREATION : ExprKind.OBJECT_CREATION; + } + + ExplicitObjectCreation(ExpressionNodeInfo info) + : base(info.SetKind(GetKind(info.Context, (ObjectCreationExpressionSyntax)info.Node))) { } + + public static Expression Create(ExpressionNodeInfo info) => new ExplicitObjectCreation(info).TryPopulate(); + + protected override void Populate() + { + if (Syntax.ArgumentList != null) + { + PopulateArguments(cx, Syntax.ArgumentList, 0); + } + + var target = cx.Model(Syntax).GetSymbolInfo(Syntax); + var method = (IMethodSymbol)target.Symbol; + + if (method != null) + { + cx.Emit(Tuples.expr_call(this, Method.Create(cx, method))); + } + + if (IsDynamicObjectCreation(cx, Syntax)) + { + var name = GetDynamicName(Syntax.Type); + if (name.HasValue) + cx.Emit(Tuples.dynamic_member_name(this, name.Value.Text)); + else + cx.ModelError(Syntax, "Unable to get name for dynamic object creation."); + } + + if (Syntax.Initializer != null) + { + switch (Syntax.Initializer.Kind()) + { + case SyntaxKind.CollectionInitializerExpression: + CollectionInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, -1) { Type = Type }); + break; + case SyntaxKind.ObjectInitializerExpression: + ObjectInitializer.Create(new ExpressionNodeInfo(cx, Syntax.Initializer, this, -1) { Type = Type }); + break; + default: + cx.ModelError("Unhandled initializer in object creation"); + break; + } + } + + TypeMention.Create(cx, Syntax.Type, this, Type); + } + + static SyntaxToken? GetDynamicName(CSharpSyntaxNode name) + { + switch (name.Kind()) + { + case SyntaxKind.IdentifierName: + return ((IdentifierNameSyntax)name).Identifier; + case SyntaxKind.GenericName: + return ((GenericNameSyntax)name).Identifier; + case SyntaxKind.QualifiedName: + // We ignore any qualifiers, for now + return GetDynamicName(((QualifiedNameSyntax)name).Right); + default: + return null; + } + } + } + + class ImplicitObjectCreation : ObjectCreation + { + public ImplicitObjectCreation(ExpressionNodeInfo info) + : base(info.SetKind(ExprKind.OBJECT_CREATION)) { } + + public static Expression Create(ExpressionNodeInfo info) => + new ImplicitObjectCreation(info).TryPopulate(); + + protected override void Populate() + { + var target = cx.GetSymbolInfo(Syntax); + var method = (IMethodSymbol)target.Symbol; + + if (method != null) + { + cx.Emit(Tuples.expr_call(this, Method.Create(cx, method))); + } + var child = 0; + + Expression objectInitializer = Syntax.Initializers.Any() ? + new Expression(new ExpressionInfo(cx, Type, Location, ExprKind.OBJECT_INIT, this, -1, false, null)) : + null; + + foreach (var init in Syntax.Initializers) + { + // Create an "assignment" + var property = cx.Model(init).GetDeclaredSymbol(init); + var propEntity = Property.Create(cx, property); + var type = Type.Create(cx, property.Type); + var loc = cx.Create(init.GetLocation()); + + var assignment = new Expression(new ExpressionInfo(cx, type, loc, ExprKind.SIMPLE_ASSIGN, objectInitializer, child++, false, null)); + Create(cx, init.Expression, assignment, 0); + Property.Create(cx, property); + + var access = new Expression(new ExpressionInfo(cx, type, loc, ExprKind.PROPERTY_ACCESS, assignment, 1, false, null)); + cx.Emit(Tuples.expr_access(access, propEntity)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PointerMemberAccess.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PointerMemberAccess.cs new file mode 100644 index 000000000000..6087c9c4df2d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PointerMemberAccess.cs @@ -0,0 +1,19 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class PointerMemberAccess : Expression + { + PointerMemberAccess(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.POINTER_INDIRECTION)) { } + + public static Expression Create(ExpressionNodeInfo info) => new PointerMemberAccess(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + + // !! We do not currently look at the member (or store the member name). + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PostfixUnary.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PostfixUnary.cs new file mode 100644 index 000000000000..a596ab93c435 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/PostfixUnary.cs @@ -0,0 +1,32 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class PostfixUnary : Expression + { + PostfixUnary(ExpressionNodeInfo info, ExprKind kind, ExpressionSyntax operand) + : base(info.SetKind(UnaryOperatorKind(info.Context, kind, info.Node))) + { + Operand = operand; + OperatorKind = kind; + } + + readonly ExpressionSyntax Operand; + readonly ExprKind OperatorKind; + + public static Expression Create(ExpressionNodeInfo info, ExpressionSyntax operand) => new PostfixUnary(info, info.Kind, operand).TryPopulate(); + + protected override void Populate() + { + Create(cx, Operand, this, 0); + OperatorCall(Syntax); + + if ((OperatorKind == ExprKind.POST_INCR || OperatorKind == ExprKind.POST_DECR) && + Kind == ExprKind.OPERATOR_INVOCATION) + { + cx.Emit(Tuples.mutator_invocation_mode(this, 2)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs new file mode 100644 index 000000000000..d5c8ead52fd1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Query.cs @@ -0,0 +1,260 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; +using Semmle.Extraction.Kinds; +using System.Collections.Generic; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + static class Query + { + /// + /// An expression representing a call in a LINQ query. + /// + /// + /// + /// This code has some problems because the expression kind isn't correct - it should really be + /// a new ExprKind. The other issue is that the argument to the call is technically a lambda, + /// rather than the sub-expression itself. + /// + private class QueryCall : Expression + { + public QueryCall(Context cx, IMethodSymbol method, SyntaxNode clause, IExpressionParentEntity parent, int child) + : base(new ExpressionInfo(cx, Type.Create(cx, method?.ReturnType), cx.Create(clause.GetLocation()), ExprKind.METHOD_INVOCATION, parent, child, false, null)) + { + if (method != null) + cx.Emit(Tuples.expr_call(this, Method.Create(cx, method))); + } + } + + /// + /// Represents a chain of method calls (the operand being recursive). + /// + class ClauseCall + { + public ClauseCall operand; + public IMethodSymbol method; + public readonly List arguments = new List(); + public SyntaxNode node; + public ISymbol declaration; + public SyntaxToken name; + public ISymbol intoDeclaration; + + public ExpressionSyntax Expr => arguments.First(); + + public ClauseCall WithClause(IMethodSymbol newMethod, SyntaxNode newNode, SyntaxToken newName = default(SyntaxToken), ISymbol newDeclaration = null) + { + return new ClauseCall + { + operand = this, + method = newMethod, + node = newNode, + name = newName, + declaration = newDeclaration + }; + } + + public ClauseCall AddArgument(ExpressionSyntax arg) + { + if (arg != null) + arguments.Add(arg); + return this; + } + + public ClauseCall WithInto(ISymbol into) + { + intoDeclaration = into; + return this; + } + + Expression DeclareRangeVariable(Context cx, IExpressionParentEntity parent, int child, bool getElement) + { + return DeclareRangeVariable(cx, parent, child, getElement, declaration); + } + + void DeclareIntoVariable(Context cx, IExpressionParentEntity parent, int intoChild, bool getElement) + { + if (intoDeclaration != null) + DeclareRangeVariable(cx, parent, intoChild, getElement, intoDeclaration); + } + + Expression DeclareRangeVariable(Context cx, IExpressionParentEntity parent, int child, bool getElement, ISymbol variableSymbol) + { + var type = Type.Create(cx, cx.GetType(Expr)); + Semmle.Extraction.Entities.Location nameLoc; + + Type declType; + if (getElement) + { + var from = node as FromClauseSyntax; + declType = from != null && from.Type != null + ? Type.Create(cx, cx.GetType(from.Type)) + : type.ElementType; + } + else + declType = type; + + var decl = VariableDeclaration.Create(cx, + variableSymbol, + declType, + cx.Create(node.GetLocation()), + nameLoc = cx.Create(name.GetLocation()), + true, + parent, + child + ); + + Expression.Create(cx, Expr, decl, 0); + + var access = new Expression(new ExpressionInfo(cx, type, nameLoc, ExprKind.LOCAL_VARIABLE_ACCESS, decl, 1, false, null)); + cx.Emit(Tuples.expr_access(access, LocalVariable.GetAlreadyCreated(cx, variableSymbol))); + + return decl; + } + + void PopulateArguments(Context cx, QueryCall callExpr, int child) + { + foreach (var e in arguments) + { + Expression.Create(cx, e, callExpr, child++); + } + } + + public Expression Populate(Context cx, IExpressionParentEntity parent, int child) + { + if (declaration != null) // The first "from" clause, or a "let" clause + { + if (operand == null) + { + return DeclareRangeVariable(cx, parent, child, true); + } + else + { + if (method == null) + cx.ModelError(node, "Unable to determine target of query expression"); + + var callExpr = new QueryCall(cx, method, node, parent, child); + operand.Populate(cx, callExpr, 0); + DeclareRangeVariable(cx, callExpr, 1, false); + PopulateArguments(cx, callExpr, 2); + DeclareIntoVariable(cx, callExpr, 2 + arguments.Count, false); + return callExpr; + } + } + else + { + var callExpr = new QueryCall(cx, method, node, parent, child); + operand.Populate(cx, callExpr, 0); + PopulateArguments(cx, callExpr, 1); + return callExpr; + } + } + } + + /// + /// Construct a "syntax tree" representing the LINQ query. + /// + /// + /// The extraction context. + /// The query expression. + /// A "syntax tree" of the query. + static ClauseCall ConstructQueryExpression(Context cx, QueryExpressionSyntax node) + { + var info = cx.Model(node).GetQueryClauseInfo(node.FromClause); + var method = info.OperationInfo.Symbol as IMethodSymbol; + + ClauseCall clauseExpr = new ClauseCall + { + declaration = cx.Model(node).GetDeclaredSymbol(node.FromClause), + name = node.FromClause.Identifier, + method = method, + node = node.FromClause + }.AddArgument(node.FromClause.Expression); + + foreach (var qc in node.Body.Clauses) + { + info = cx.Model(node).GetQueryClauseInfo(qc); + + method = info.OperationInfo.Symbol as IMethodSymbol; + + switch (qc.Kind()) + { + case SyntaxKind.OrderByClause: + var orderByClause = (OrderByClauseSyntax)qc; + foreach (var ordering in orderByClause.Orderings) + { + method = cx.Model(node).GetSymbolInfo(ordering).Symbol as IMethodSymbol; + + clauseExpr = clauseExpr.WithClause(method, orderByClause).AddArgument(ordering.Expression); + + if (method == null) + cx.ModelError(ordering, "Could not determine method call for orderby clause"); + } + break; + case SyntaxKind.WhereClause: + var whereClause = (WhereClauseSyntax)qc; + clauseExpr = clauseExpr.WithClause(method, whereClause).AddArgument(whereClause.Condition); + break; + case SyntaxKind.FromClause: + var fromClause = (FromClauseSyntax)qc; + clauseExpr = clauseExpr. + WithClause(method, fromClause, fromClause.Identifier, cx.Model(node).GetDeclaredSymbol(fromClause)). + AddArgument(fromClause.Expression); + break; + case SyntaxKind.LetClause: + var letClause = (LetClauseSyntax)qc; + clauseExpr = clauseExpr.WithClause(method, letClause, letClause.Identifier, cx.Model(node).GetDeclaredSymbol(letClause)). + AddArgument(letClause.Expression); + break; + case SyntaxKind.JoinClause: + var joinClause = (JoinClauseSyntax)qc; + + clauseExpr = clauseExpr.WithClause(method, joinClause, joinClause.Identifier, cx.Model(node).GetDeclaredSymbol(joinClause)). + AddArgument(joinClause.InExpression). + AddArgument(joinClause.LeftExpression). + AddArgument(joinClause.RightExpression); + + if (joinClause.Into != null) + { + var into = cx.Model(node).GetDeclaredSymbol(joinClause.Into); + clauseExpr.WithInto(into); + } + + break; + default: + throw new InternalError(qc, "Unhandled query clause of kind {0}", qc.Kind()); + } + } + + method = cx.Model(node).GetSymbolInfo(node.Body.SelectOrGroup).Symbol as IMethodSymbol; + + var selectClause = node.Body.SelectOrGroup as SelectClauseSyntax; + var groupClause = node.Body.SelectOrGroup as GroupClauseSyntax; + + clauseExpr = new ClauseCall { operand = clauseExpr, method = method, node = node.Body.SelectOrGroup }; + + if (selectClause != null) + { + clauseExpr.AddArgument(selectClause.Expression); + } + else if (groupClause != null) + { + clauseExpr. + AddArgument(groupClause.GroupExpression). + AddArgument(groupClause.ByExpression); + } + else + { + throw new InternalError(node, "Failed to process select/group by clause"); + } + + return clauseExpr; + } + + public static Expression Create(ExpressionNodeInfo info) => + ConstructQueryExpression(info.Context, (QueryExpressionSyntax)info.Node).Populate(info.Context, info.Parent, info.Child); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Ref.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Ref.cs new file mode 100644 index 000000000000..a5e4e905970b --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Ref.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Ref : Expression + { + Ref(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.REF)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Ref(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefType.cs new file mode 100644 index 000000000000..d1190f5d7e01 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefType.cs @@ -0,0 +1,17 @@ +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class RefType : Expression + { + RefType(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.UNKNOWN)) { } + + public static Expression Create(ExpressionNodeInfo info) => new RefType(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefValue.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefValue.cs new file mode 100644 index 000000000000..df3ee3643468 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/RefValue.cs @@ -0,0 +1,18 @@ +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class RefValue : Expression + { + RefValue(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.REF)) { } + + public static Expression Create(ExpressionNodeInfo info) => new RefValue(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + Create(cx, Syntax.Type, this, 1); // A type-access + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Sizeof.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Sizeof.cs new file mode 100644 index 000000000000..095b4672f2db --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Sizeof.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class SizeOf : Expression + { + SizeOf(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.SIZEOF)) { } + + public static Expression Create(ExpressionNodeInfo info) => new SizeOf(info).TryPopulate(); + + protected override void Populate() + { + TypeAccess.Create(cx, Syntax.Type, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/This.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/This.cs new file mode 100644 index 000000000000..cc9514cd6394 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/This.cs @@ -0,0 +1,16 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class This : Expression + { + This(IExpressionInfo info) : base(info) { } + + public static This CreateImplicit(Context cx, Type @class, Location loc, IExpressionParentEntity parent, int child) => + new This(new ExpressionInfo(cx, @class, loc, Kinds.ExprKind.THIS_ACCESS, parent, child, true, null)); + + public static This CreateExplicit(ExpressionNodeInfo info) => new This(info.SetKind(ExprKind.THIS_ACCESS)); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Throw.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Throw.cs new file mode 100644 index 000000000000..5f365fc82da2 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Throw.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Throw : Expression + { + Throw(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.THROW)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Throw(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Tuple.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Tuple.cs new file mode 100644 index 000000000000..7fa86d88ebcf --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Tuple.cs @@ -0,0 +1,24 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Tuple : Expression + { + public static Expression Create(ExpressionNodeInfo info) => new Tuple(info).TryPopulate(); + + Tuple(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.TUPLE)) + { + } + + protected override void Populate() + { + int child = 0; + foreach (var argument in Syntax.Arguments.Select(a => a.Expression)) + { + Expression.Create(cx, argument, this, child++); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeAccess.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeAccess.cs new file mode 100644 index 000000000000..ee6997b6431f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeAccess.cs @@ -0,0 +1,38 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class TypeAccess : Expression + { + TypeAccess(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.TYPE_ACCESS)) { } + + protected override void Populate() + { + switch (Syntax.Kind()) + { + case SyntaxKind.SimpleMemberAccessExpression: + var maes = (MemberAccessExpressionSyntax)Syntax; + if (Type.ContainingType == null) + { + // namespace qualifier + TypeMention.Create(cx, maes.Name, this, Type, Syntax.GetLocation()); + } + else + { + // type qualifier + TypeMention.Create(cx, maes.Name, this, Type); + Create(cx, maes.Expression, this, -1); + } + return; + default: + TypeMention.Create(cx, (TypeSyntax)Syntax, this, Type); + return; + } + } + + public static Expression Create(ExpressionNodeInfo info) => new TypeAccess(info).TryPopulate(); + } +} + diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeOf.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeOf.cs new file mode 100644 index 000000000000..f343614bfdec --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/TypeOf.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class TypeOf : Expression + { + TypeOf(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.TYPEOF)) { } + + public static Expression Create(ExpressionNodeInfo info) => new TypeOf(info).TryPopulate(); + + protected override void Populate() + { + TypeAccess.Create(cx, Syntax.Type, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unary.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unary.cs new file mode 100644 index 000000000000..6f518477e2a0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unary.cs @@ -0,0 +1,35 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Unary : Expression + { + Unary(ExpressionNodeInfo info, ExprKind kind) + : base(info.SetKind(UnaryOperatorKind(info.Context, info.Kind, info.Node))) + { + OperatorKind = kind; + } + + readonly ExprKind OperatorKind; + + public static Unary Create(ExpressionNodeInfo info) + { + var ret = new Unary(info, info.Kind); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Create(cx, Syntax.Operand, this, 0); + OperatorCall(Syntax); + + if ((OperatorKind == ExprKind.PRE_INCR || OperatorKind == ExprKind.PRE_DECR) && + Kind == ExprKind.OPERATOR_INVOCATION) + { + cx.Emit(Tuples.mutator_invocation_mode(this, 1)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unchecked.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unchecked.cs new file mode 100644 index 000000000000..73ec785cd9fc --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unchecked.cs @@ -0,0 +1,17 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Unchecked : Expression + { + Unchecked(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.UNCHECKED)) { } + + public static Expression Create(ExpressionNodeInfo info) => new Unchecked(info).TryPopulate(); + + protected override void Populate() + { + Create(cx, Syntax.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unknown.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unknown.cs new file mode 100644 index 000000000000..c547626e6393 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Unknown.cs @@ -0,0 +1,10 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class Unknown : Expression + { + public Unknown(ExpressionNodeInfo info) : base(info.SetKind(ExprKind.UNKNOWN)) { } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/VariableDeclaration.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/VariableDeclaration.cs new file mode 100644 index 000000000000..70d6cdfe0724 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/VariableDeclaration.cs @@ -0,0 +1,145 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Kinds; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Expressions +{ + class VariableDeclaration : Expression + { + VariableDeclaration(IExpressionInfo info) : base(info) { } + + public static VariableDeclaration Create(Context cx, ISymbol symbol, Type type, Extraction.Entities.Location exprLocation, Extraction.Entities.Location declLocation, bool isVar, IExpressionParentEntity parent, int child) + { + var ret = new VariableDeclaration(new ExpressionInfo(cx, type, exprLocation, ExprKind.LOCAL_VAR_DECL, parent, child, false, null)); + cx.Try(null, null, () => LocalVariable.Create(cx, symbol, ret, isVar, declLocation)); + return ret; + } + + static VariableDeclaration CreateSingle(Context cx, DeclarationExpressionSyntax node, SingleVariableDesignationSyntax designation, IExpressionParentEntity parent, int child) + { + bool isVar = node.Type.IsVar; + + var variableSymbol = cx.Model(designation).GetDeclaredSymbol(designation) as ILocalSymbol; + if (variableSymbol == null) + { + cx.ModelError(node, "Failed to determine local variable"); + return new VariableDeclaration(cx, node, null, isVar, parent, child); + } + + var type = Type.Create(cx, variableSymbol.Type); + var location = cx.Create(designation.GetLocation()); + + var ret = new VariableDeclaration(cx, designation, type, isVar, parent, child); + cx.Try(null, null, () => LocalVariable.Create(cx, variableSymbol, ret, isVar, location)); + return ret; + } + + /// + /// Create a tuple expression representing a parenthesized variable declaration. + /// That is, we consider `var (x, y) = ...` to be equivalent to `(var x, var y) = ...`. + /// + static Expression CreateParenthesized(Context cx, DeclarationExpressionSyntax node, ParenthesizedVariableDesignationSyntax designation, IExpressionParentEntity parent, int child) + { + var type = Type.Create(cx, null); // Should ideally be a corresponding tuple type + var tuple = new Expression(new ExpressionInfo(cx, type, cx.Create(node.GetLocation()), ExprKind.TUPLE, parent, child, false, null)); + + cx.Try(null, null, () => + { + var child0 = 0; + foreach (var variable in designation.Variables) + Create(cx, node, variable, tuple, child0++); + }); + + return tuple; + } + + static Expression Create(Context cx, DeclarationExpressionSyntax node, VariableDesignationSyntax designation, IExpressionParentEntity parent, int child) + { + var single = designation as SingleVariableDesignationSyntax; + if (single != null) + return CreateSingle(cx, node, single, parent, child); + + var paren = designation as ParenthesizedVariableDesignationSyntax; + if (paren != null) + return CreateParenthesized(cx, node, paren, parent, child); + + var discard = designation as DiscardDesignationSyntax; + if (discard != null) + { + var type = cx.Model(node).GetTypeInfo(node).Type; + return new VariableDeclaration(cx, node, Type.Create(cx, type), node.Type.IsVar, parent, child); + } + + cx.ModelError(node, "Failed to determine designation type"); + return new VariableDeclaration(cx, node, null, node.Type.IsVar, parent, child); + } + + public static Expression Create(Context cx, DeclarationExpressionSyntax node, IExpressionParentEntity parent, int child) + { + return Create(cx, node, node.Designation, parent, child); + } + + VariableDeclaration(Context cx, CSharpSyntaxNode d, Type type, bool isVar, IExpressionParentEntity parent, int child) + : base(new ExpressionInfo(cx, type, cx.Create(d.FixedLocation()), ExprKind.LOCAL_VAR_DECL, parent, child, false, null)) + { + } + + public static VariableDeclaration Create(Context cx, CatchDeclarationSyntax d, bool isVar, IExpressionParentEntity parent, int child) + { + var type = Type.Create(cx, cx.Model(d).GetDeclaredSymbol(d).Type); + var ret = new VariableDeclaration(cx, d, type, isVar, parent, child); + cx.Try(d, null, () => + { + var id = d.Identifier; + var declSymbol = cx.Model(d).GetDeclaredSymbol(d); + var location = cx.Create(id.GetLocation()); + LocalVariable.Create(cx, declSymbol, ret, isVar, location); + TypeMention.Create(cx, d.Type, ret, type); + }); + return ret; + } + + public static VariableDeclaration Create(Context cx, VariableDeclaratorSyntax d, Type type, bool isVar, IExpressionParentEntity parent, int child) + { + var ret = new VariableDeclaration(cx, d, type, isVar, parent, child); + cx.Try(d, null, () => + { + var id = d.Identifier; + var declSymbol = cx.Model(d).GetDeclaredSymbol(d); + var location = cx.Create(id.GetLocation()); + var localVar = LocalVariable.Create(cx, declSymbol, ret, isVar, location); + + if (d.Initializer != null) + { + Create(cx, d.Initializer.Value, ret, 0); + + // Create an access + var access = new Expression(new ExpressionInfo(cx, type, location, ExprKind.LOCAL_VARIABLE_ACCESS, ret, 1, false, null)); + cx.Emit(Tuples.expr_access(access, localVar)); + } + + var decl = d.Parent as VariableDeclarationSyntax; + if (decl != null) + TypeMention.Create(cx, decl.Type, ret, type); + }); + return ret; + } + } + + static class VariableDeclarations + { + public static void Populate(Context cx, VariableDeclarationSyntax decl, IExpressionParentEntity parent, int child, int childIncrement = 1) + { + var type = Type.Create(cx, cx.GetType(decl.Type)); + + foreach (var v in decl.Variables) + { + VariableDeclaration.Create(cx, v, type, decl.Type.IsVar, parent, child); + child += childIncrement; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs new file mode 100644 index 000000000000..541d49968e69 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs @@ -0,0 +1,101 @@ +using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp.Populators; +using System.Linq; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Field : CachedSymbol, IExpressionParentEntity + { + Field(Context cx, IFieldSymbol init) + : base(cx, init) + { + type = new Lazy(() => Type.Create(cx, symbol.Type)); + } + + public static Field Create(Context cx, IFieldSymbol field) => FieldFactory.Instance.CreateEntity(cx, field); + + // Do not populate backing fields. + // Populate Tuple fields. + public override bool NeedsPopulation => + (base.NeedsPopulation && !symbol.IsImplicitlyDeclared) || symbol.ContainingType.IsTupleType; + + public override void Populate() + { + ExtractAttributes(); + ContainingType.ExtractGenerics(); + + Field unboundFieldKey = Field.Create(Context, symbol.OriginalDefinition); + Context.Emit(Tuples.fields(this, (symbol.IsConst ? 2 : 1), symbol.Name, ContainingType, Type.TypeRef, unboundFieldKey)); + + ExtractModifiers(); + + if (symbol.IsVolatile) + Modifier.HasModifier(Context, this, "volatile"); + + if (symbol.IsConst) + { + Modifier.HasModifier(Context, this, "const"); + + if (symbol.HasConstantValue) + { + Context.Emit(Tuples.constant_value(this, Expression.ValueAsString(symbol.ConstantValue))); + } + } + + foreach (var l in Locations) + Context.Emit(Tuples.field_location(this, l)); + + if (!IsSourceDeclaration || !symbol.FromSource()) + return; + + Context.BindComments(this, Location.symbol); + + int child = 0; + foreach (var initializer in + symbol.DeclaringSyntaxReferences. + Select(n => n.GetSyntax()). + OfType(). + Where(n => n.Initializer != null)) + { + Context.PopulateLater(() => + { + Expression.CreateFromNode(new ExpressionNodeInfo(Context, initializer.Initializer.Value, this, child++)); + }); + } + + foreach (var initializer in symbol.DeclaringSyntaxReferences. + Select(n => n.GetSyntax()). + OfType(). + Where(n => n.EqualsValue != null)) + { + // Mark fields that have explicit initializers. + var expr = new Expression(new ExpressionInfo(Context, Type, Context.Create(initializer.EqualsValue.Value.FixedLocation()), Kinds.ExprKind.FIELD_ACCESS, this, child++, false, null)); + Context.Emit(Tuples.expr_access(expr, this)); + } + + if (IsSourceDeclaration) + foreach (var syntax in symbol.DeclaringSyntaxReferences. + Select(d => d.GetSyntax()).OfType(). + Select(d => d.Parent).OfType()) + TypeMention.Create(Context, syntax.Type, this, Type); + } + + readonly Lazy type; + public Type Type => type.Value; + + public override IId Id => new Key(ContainingType, ".", symbol.Name, ";field"); + + bool IExpressionParentEntity.IsTopLevelParent => true; + + class FieldFactory : ICachedEntityFactory + { + public static readonly FieldFactory Instance = new FieldFactory(); + + public Field Create(Context cx, IFieldSymbol init) => new Field(cx, init); + } + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.PushesLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs new file mode 100644 index 000000000000..8fc7e20ccafc --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs @@ -0,0 +1,138 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Indexer : Property, IExpressionParentEntity + { + protected Indexer(Context cx, IPropertySymbol init) + : base(cx, init) { } + + Indexer OriginalDefinition => IsSourceDeclaration ? this : Create(Context, symbol.OriginalDefinition); + + public override void Populate() + { + var type = Type.Create(Context, symbol.Type); + Context.Emit(Tuples.indexers(this, symbol.GetName(useMetadataName: true), ContainingType, type.TypeRef, OriginalDefinition)); + foreach (var l in Locations) + Context.Emit(Tuples.indexer_location(this, l)); + + var getter = symbol.GetMethod; + var setter = symbol.SetMethod; + + if (getter == null && setter == null) + Context.ModelError(symbol, "No indexer accessor defined"); + + if (getter != null) + { + Getter = Accessor.Create(Context, getter); + } + + if (setter != null) + { + Setter = Accessor.Create(Context, setter); + } + + for (var i = 0; i < symbol.Parameters.Length; ++i) + { + var original = Parameter.Create(Context, symbol.OriginalDefinition.Parameters[i], OriginalDefinition); + Parameter.Create(Context, symbol.Parameters[i], this, original); + } + + if (getter != null) + { + Getter = Accessor.Create(Context, getter); + Context.Emit(Tuples.accessors(Getter, 1, getter.Name, this, Getter.OriginalDefinition)); + + Context.Emit(Tuples.accessor_location(Getter, Getter.Location)); + Getter.Overrides(); + Getter.ExtractModifiers(); + } + + if (setter != null) + { + Setter = Accessor.Create(Context, setter); + Context.Emit(Tuples.accessors(Setter, 2, setter.Name, this, Setter.OriginalDefinition)); + + Context.Emit(Tuples.accessor_location(Setter, Setter.Location)); + Setter.Overrides(); + Setter.ExtractModifiers(); + } + + if (IsSourceDeclaration) + { + var expressionBody = ExpressionBody; + if (expressionBody != null) + { + // The expression may need to reference parameters in the getter. + // So we need to arrange that the expression is populated after the getter. + Context.PopulateLater(() => Expression.CreateFromNode(new ExpressionNodeInfo(Context, expressionBody, this, 0) { Type = Type.Create(Context, symbol.Type) })); + } + } + + ExtractModifiers(); + BindComments(); + + var declSyntaxReferences = IsSourceDeclaration + ? symbol.DeclaringSyntaxReferences. + Select(d => d.GetSyntax()).OfType().ToArray() + : Enumerable.Empty(); + + foreach (var explicitInterface in symbol.ExplicitInterfaceImplementations.Select(impl => Type.Create(Context, impl.ContainingType))) + { + Context.Emit(Tuples.explicitly_implements(this, explicitInterface.TypeRef)); + + foreach (var syntax in declSyntaxReferences) + TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier.Name, this, explicitInterface); + } + + + foreach (var syntax in declSyntaxReferences) + TypeMention.Create(Context, syntax.Type, this, type); + } + + + public static new Indexer Create(Context cx, IPropertySymbol prop) => IndexerFactory.Instance.CreateEntity(cx, prop); + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(ContainingType); + tb.Append("."); + tb.Append(symbol.MetadataName); + tb.Append("("); + tb.BuildList(",", symbol.Parameters, (p, tb0) => tb0.Append(Type.Create(Context, p.Type))); + tb.Append(");indexer"); + }); + } + } + + public override Microsoft.CodeAnalysis.Location FullLocation + { + get + { + return + symbol. + DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + Select(s => s.GetLocation()). + Concat(symbol.Locations). + First(); + } + } + + bool IExpressionParentEntity.IsTopLevelParent => true; + + class IndexerFactory : ICachedEntityFactory + { + public static readonly IndexerFactory Instance = new IndexerFactory(); + + public Indexer Create(Context cx, IPropertySymbol init) => new Indexer(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs new file mode 100644 index 000000000000..7d808e0fd3af --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs @@ -0,0 +1,48 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class LocalFunction : Method + { + LocalFunction(Context cx, IMethodSymbol init) : base(cx, init) + { + } + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(Location); + if (symbol.IsGenericMethod && !IsSourceDeclaration) + { + tb.Append("<"); + tb.BuildList(",", symbol.TypeArguments, (ta, tb0) => AddSignatureTypeToId(Context, tb0, symbol, ta)); + tb.Append(">"); + } + + tb.Append(";localfunction"); + }); + } + } + + public static new LocalFunction Create(Context cx, IMethodSymbol field) => LocalFunctionFactory.Instance.CreateEntity(cx, field); + + class LocalFunctionFactory : ICachedEntityFactory + { + public static readonly LocalFunctionFactory Instance = new LocalFunctionFactory(); + + public LocalFunction Create(Context cx, IMethodSymbol init) => new LocalFunction(cx, init); + } + + public override void Populate() + { + PopulateMethod(); + var originalDefinition = IsSourceDeclaration ? this : Create(Context, symbol.OriginalDefinition); + var returnType = Type.Create(Context, symbol.ReturnType); + Context.Emit(Tuples.local_functions(this, symbol.Name, returnType, originalDefinition)); + ExtractRefReturn(); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs new file mode 100644 index 000000000000..163f81040a5c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs @@ -0,0 +1,95 @@ +using System; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + class LocalVariable : CachedSymbol + { + LocalVariable(Context cx, ISymbol init, Expression parent, bool isVar, Extraction.Entities.Location declLocation) + : base(cx, init) + { + Parent = parent; + IsVar = isVar; + DeclLocation = declLocation; + } + + readonly Expression Parent; + readonly bool IsVar; + readonly Extraction.Entities.Location DeclLocation; + + public override IId Id => new Key(Parent, "_", symbol.Name, ";localvar"); + + public override void Populate() + { + Context.Emit(Tuples.localvars( + this, + IsRef ? 3 : IsConst ? 2 : 1, + symbol.Name, + IsVar ? 1 : 0, + Type.TypeRef, + Parent)); + + Context.Emit(Tuples.localvar_location(this, DeclLocation)); + + DefineConstantValue(); + } + + public static LocalVariable Create(Context cx, ISymbol local, Expression parent, bool isVar, Extraction.Entities.Location declLocation) + { + return LocalVariableFactory.Instance.CreateEntity(cx, local, parent, isVar, declLocation); + } + + /// + /// Gets the local variable entity for which must + /// already have been created. + /// + public static LocalVariable GetAlreadyCreated(Context cx, ISymbol local) => LocalVariableFactory.Instance.CreateEntity(cx, local, null, false, null); + + bool IsConst + { + get + { + var local = symbol as ILocalSymbol; + return local != null && local.IsConst; + } + } + + bool IsRef + { + get + { + var local = symbol as ILocalSymbol; + return local != null && local.IsRef; + } + } + + Type Type + { + get + { + var local = symbol as ILocalSymbol; + return local == null ? Parent.Type : Type.Create(Context, local.Type); + } + } + + void DefineConstantValue() + { + var local = symbol as ILocalSymbol; + if (local != null && local.HasConstantValue) + { + Context.Emit(Tuples.constant_value(this, Expression.ValueAsString(local.ConstantValue))); + } + } + + class LocalVariableFactory : ICachedEntityFactory<(ISymbol, Expression, bool, Extraction.Entities.Location), LocalVariable> + { + public static readonly LocalVariableFactory Instance = new LocalVariableFactory(); + + public LocalVariable Create(Context cx, (ISymbol, Expression, bool, Extraction.Entities.Location) init) => + new LocalVariable(cx, init.Item1, init.Item2, init.Item3, init.Item4); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NeedsLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs new file mode 100644 index 000000000000..513cb52565da --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs @@ -0,0 +1,368 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + public abstract class Method : CachedSymbol, IExpressionParentEntity, IStatementParentEntity + { + public Method(Context cx, IMethodSymbol init) + : base(cx, init) { } + + protected void ExtractParameters() + { + var originalMethod = OriginalDefinition; + IEnumerable parameters = symbol.Parameters; + IEnumerable originalParameters = originalMethod.symbol.Parameters; + + if (IsReducedExtension) + { + if (this == originalMethod) + { + // Non-generic reduced extensions must be extracted exactly like the + // non-reduced counterparts + parameters = symbol.ReducedFrom.Parameters; + } + else + { + // Constructed reduced extensions are special because their non-reduced + // counterparts are not constructed. Therefore, we need to manually add + // the `this` parameter based on the type of the receiver + var originalThisParamSymbol = originalMethod.symbol.Parameters.First(); + var originalThisParam = Parameter.Create(Context, originalThisParamSymbol, originalMethod); + ConstructedExtensionParameter.Create(Context, this, originalThisParam); + originalParameters = originalParameters.Skip(1); + } + } + + foreach (var p in parameters.Zip(originalParameters, (paramSymbol, originalParam) => new { paramSymbol, originalParam })) + { + var original = Equals(p.paramSymbol, p.originalParam) ? null : Parameter.Create(Context, p.originalParam, originalMethod); + Parameter.Create(Context, p.paramSymbol, this, original); + } + + if (symbol.IsVararg) + { + // Mono decided that "__arglist" should be an explicit parameter, + // so now we need to populate it. + + VarargsParam.Create(Context, this); + } + } + + /// + /// Extracts constructor initializers. + /// + protected virtual void ExtractInitializers() + { + // Normal methods don't have initializers, + // so there's nothing to extract. + } + + void ExtractMethodBody() + { + if (!IsSourceDeclaration) + return; + + var block = Block; + var expr = ExpressionBody; + + if (block != null || expr != null) + Context.PopulateLater( + () => + { + ExtractInitializers(); + if (block != null) + Statements.Block.Create(Context, block, this, 0); + else + Expression.Create(Context, expr, this, 0); + + Context.NumberOfLines(symbol, this); + }); + } + + public void Overrides() + { + foreach (var explicitInterface in symbol.ExplicitInterfaceImplementations. + Where(sym => sym.MethodKind == MethodKind.Ordinary). + Select(impl => Type.Create(Context, impl.ContainingType))) + { + Context.Emit(Tuples.explicitly_implements(this, explicitInterface.TypeRef)); + + if (IsSourceDeclaration) + foreach (var syntax in symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()).OfType()) + TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier.Name, this, explicitInterface); + } + + if (symbol.OverriddenMethod != null) + { + Context.Emit(Tuples.overrides(this, Method.Create(Context, symbol.OverriddenMethod))); + } + } + + /// + /// Factored out to share logic between `Method` and `UserOperator`. + /// + protected static void BuildMethodId(Method m, ITrapBuilder tb) + { + tb.Append(m.ContainingType); + + AddExplicitInterfaceQualifierToId(m.Context, tb, m.symbol.ExplicitInterfaceImplementations); + + tb.Append(".").Append(m.symbol.Name); + + if (m.symbol.IsGenericMethod) + { + if (Equals(m.symbol, m.symbol.OriginalDefinition)) + { + tb.Append("`").Append(m.symbol.TypeParameters.Length); + } + else + { + tb.Append("<"); + tb.BuildList(",", m.symbol.TypeArguments, (ta, tb0) => AddSignatureTypeToId(m.Context, tb0, m.symbol, ta)); + tb.Append(">"); + } + } + + AddParametersToId(m.Context, tb, m.symbol); + switch (m.symbol.MethodKind) + { + case MethodKind.PropertyGet: + tb.Append(";getter"); + break; + case MethodKind.PropertySet: + tb.Append(";setter"); + break; + case MethodKind.EventAdd: + tb.Append(";adder"); + break; + case MethodKind.EventRaise: + tb.Append(";raiser"); + break; + case MethodKind.EventRemove: + tb.Append(";remover"); + break; + default: + tb.Append(";method"); + break; + } + } + + public override IId Id => new Key(tb => BuildMethodId(this, tb)); + + /// + /// Adds an appropriate label ID to the trap builder + /// for the type belonging to the signature of method + /// . + /// + /// For methods without type parameters this will always add the key of the + /// corresponding type. + /// + /// For methods with type parameters, this will add the key of the + /// corresponding type if the type does *not* contain one of the method + /// type parameters, otherwise it will add a textual representation of + /// the type. This distinction is required because type parameter IDs + /// refer to their declaring methods. + /// + /// Example: + /// + /// + /// int Count<T>(IEnumerable items) + /// + /// + /// The label definitions for Count (#4) and T + /// (#5) will look like: + /// + /// + /// #1=<label for System.Int32> + /// #2=<label for type containing Count> + /// #3=<label for IEnumerable`1> + /// #4=@"{#1} {#2}.Count`2(#3);method" + /// #5=@"{#4}T;typeparameter" + /// + /// + /// Note how int is referenced in the label definition #3 for + /// Count, while T[] is represented textually in order + /// to make the reference to #3 in the label definition #4 for + /// T valid. + /// + protected static void AddSignatureTypeToId(Context cx, ITrapBuilder tb, IMethodSymbol method, ITypeSymbol type) + { + if (type.ContainsTypeParameters(cx, method)) + type.BuildTypeId(cx, tb, (cx0, tb0, type0) => AddSignatureTypeToId(cx, tb0, method, type0)); + else + tb.Append(Type.Create(cx, type)); + } + + protected static void AddParametersToId(Context cx, ITrapBuilder tb, IMethodSymbol method) + { + tb.Append("("); + tb.AppendList(",", AddParameterPartsToId(cx, tb, method)); + tb.Append(")"); + } + + // This is a slight abuse of ITrapBuilder.AppendList(). + // yield return "" is used to insert a list separator + // at the desired location. + static IEnumerable AddParameterPartsToId(Context cx, ITrapBuilder tb, IMethodSymbol method) + { + if (method.MethodKind == MethodKind.ReducedExtension) + { + AddSignatureTypeToId(cx, tb, method, method.ReceiverType); + yield return ""; // The next yield return outputs a "," + } + + foreach (var param in method.Parameters) + { + yield return ""; // Maybe print "," + AddSignatureTypeToId(cx, tb, method, param.Type); + switch (param.RefKind) + { + case RefKind.Out: + yield return " out"; + break; + case RefKind.Ref: + yield return " ref"; + break; + } + } + + if (method.IsVararg) + { + yield return "__arglist"; + } + } + + public static void AddExplicitInterfaceQualifierToId(Context cx, ITrapBuilder tb, IEnumerable explicitInterfaceImplementations) + { + if (explicitInterfaceImplementations.Any()) + { + tb.AppendList(",", explicitInterfaceImplementations.Select(impl => cx.CreateEntity(impl.ContainingType))); + } + } + + public virtual string Name => symbol.Name; + + /// + /// Creates a method of the appropriate subtype. + /// + /// + /// + /// + public static Method Create(Context cx, IMethodSymbol methodDecl) + { + if (methodDecl == null) return null; + + switch (methodDecl.MethodKind) + { + case MethodKind.StaticConstructor: + case MethodKind.Constructor: + return Constructor.Create(cx, methodDecl); + case MethodKind.ReducedExtension: + case MethodKind.Ordinary: + case MethodKind.DelegateInvoke: + case MethodKind.ExplicitInterfaceImplementation: + return OrdinaryMethod.Create(cx, methodDecl); + case MethodKind.Destructor: + return Destructor.Create(cx, methodDecl); + case MethodKind.PropertyGet: + case MethodKind.PropertySet: + return Accessor.Create(cx, methodDecl); + case MethodKind.EventAdd: + case MethodKind.EventRemove: + return EventAccessor.Create(cx, methodDecl); + case MethodKind.UserDefinedOperator: + case MethodKind.BuiltinOperator: + return UserOperator.Create(cx, methodDecl); + case MethodKind.Conversion: + return Conversion.Create(cx, methodDecl); + case MethodKind.AnonymousFunction: + throw new InternalError(methodDecl, "Attempt to create a lambda"); + case MethodKind.LocalFunction: + return LocalFunction.Create(cx, methodDecl); + default: + throw new InternalError(methodDecl, "Unhandled method '{0}' of kind '{1}'", methodDecl, methodDecl.MethodKind); + } + } + + public Method OriginalDefinition => + IsReducedExtension + ? Create(Context, symbol.ReducedFrom) + : Create(Context, symbol.OriginalDefinition); + + public override Microsoft.CodeAnalysis.Location FullLocation => ReportingLocation; + + public override bool IsSourceDeclaration => symbol.IsSourceDeclaration(); + + /// + /// Whether this method has type parameters. + /// + public bool IsGeneric => symbol.IsGenericMethod; + + /// + /// Whether this method has unbound type parameters. + /// + public bool IsUnboundGeneric => IsGeneric && Equals(symbol.ConstructedFrom, symbol); + + public bool IsBoundGeneric => IsGeneric && !IsUnboundGeneric; + + bool IsReducedExtension => symbol.MethodKind == MethodKind.ReducedExtension; + + protected IMethodSymbol ConstructedFromSymbol => symbol.ConstructedFrom.ReducedFrom ?? symbol.ConstructedFrom; + + bool IExpressionParentEntity.IsTopLevelParent => true; + + bool IStatementParentEntity.IsTopLevelParent => true; + + protected void ExtractGenerics() + { + var isFullyConstructed = IsBoundGeneric; + + if (IsGeneric) + { + int child = 0; + + if (isFullyConstructed) + { + Context.Emit(Tuples.is_constructed(this)); + Context.Emit(Tuples.constructed_generic(this, Method.Create(Context, ConstructedFromSymbol))); + foreach (var tp in symbol.TypeArguments) + { + Context.Emit(Tuples.type_arguments(Type.Create(Context, tp), child++, this)); + } + } + else + { + Context.Emit(Tuples.is_generic(this)); + foreach (var typeParam in symbol.TypeParameters.Select(tp => TypeParameter.Create(Context, tp))) + { + Context.Emit(Tuples.type_parameters(typeParam, child++, this)); + } + } + } + } + + protected void ExtractRefReturn() + { + if (symbol.ReturnsByRef) + Context.Emit(Tuples.ref_returns(this)); + if (symbol.ReturnsByRefReadonly) + Context.Emit(Tuples.ref_readonly_returns(this)); + } + + protected void PopulateMethod() + { + // Common population code for all callables + BindComments(); + ExtractAttributes(); + ExtractParameters(); + ExtractMethodBody(); + ExtractGenerics(); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.PushesLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs new file mode 100644 index 000000000000..bca30cea96d1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs @@ -0,0 +1,169 @@ +using Microsoft.CodeAnalysis; +using System; +using System.Reflection; + +namespace Semmle.Extraction.CSharp.Entities +{ + /// + /// Provide a "Key" object to allow modifiers to exist as entities in the extractor + /// hash map. (Raw strings would work as keys but might clash with other types). + /// + class ModifierKey : Object + { + public readonly string name; + + public ModifierKey(string m) + { + name = m; + } + + public override bool Equals(Object obj) + { + return obj.GetType() == GetType() && name == ((ModifierKey)obj).name; + } + + public override int GetHashCode() => 13 * name.GetHashCode(); + } + + class Modifier : Extraction.CachedEntity + { + Modifier(Context cx, ModifierKey init) + : base(cx, init) { } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => null; + + public override IId Id => new Key(symbol.name, ";modifier"); + + public override bool NeedsPopulation => true; + + public override void Populate() + { + Context.Emit(new Tuple("modifiers", Label, symbol.name)); + } + + public static string AccessbilityModifier(Accessibility access) + { + switch (access) + { + case Accessibility.Private: + return "private"; + case Accessibility.Protected: + return "protected"; + case Accessibility.Public: + return "public"; + case Accessibility.Internal: + return "internal"; + default: + throw new InternalError("Unavailable modifier combination"); + } + } + + public static void HasAccessibility(Context cx, IEntity type, Accessibility access) + { + switch (access) + { + case Accessibility.Private: + case Accessibility.Public: + case Accessibility.Protected: + case Accessibility.Internal: + HasModifier(cx, type, Modifier.AccessbilityModifier(access)); + break; + case Accessibility.NotApplicable: + break; + case Accessibility.ProtectedOrInternal: + HasModifier(cx, type, "protected"); + HasModifier(cx, type, "internal"); + break; + case Accessibility.ProtectedAndInternal: + HasModifier(cx, type, "protected"); + HasModifier(cx, type, "private"); + break; + default: + throw new InternalError("Unhandled Microsoft.CodeAnalysis.Accessibility value: {0}", access); + } + } + + public static void HasModifier(Context cx, IEntity target, string modifier) + { + cx.Emit(Tuples.has_modifiers(target, Modifier.Create(cx, modifier))); + } + + public static void ExtractModifiers(Context cx, IEntity key, ISymbol symbol) + { + bool interfaceDefinition = symbol.ContainingType != null + && symbol.ContainingType.Kind == SymbolKind.NamedType + && symbol.ContainingType.TypeKind == TypeKind.Interface; + + Modifier.HasAccessibility(cx, key, symbol.DeclaredAccessibility); + if (symbol.Kind == SymbolKind.ErrorType) + cx.Emit(Tuples.has_modifiers(key, Modifier.Create(cx, Accessibility.Public))); + + if (symbol.IsAbstract && (symbol.Kind != SymbolKind.NamedType || ((INamedTypeSymbol)symbol).TypeKind != TypeKind.Interface) && !interfaceDefinition) + Modifier.HasModifier(cx, key, "abstract"); + + if (symbol.IsSealed) + HasModifier(cx, key, "sealed"); + + bool fromSource = symbol.DeclaringSyntaxReferences.Length > 0; + + if (symbol.IsStatic && !(symbol.Kind == SymbolKind.Field && ((IFieldSymbol)symbol).IsConst && !fromSource)) + HasModifier(cx, key, "static"); + + if (symbol.IsVirtual) + HasModifier(cx, key, "virtual"); + + // For some reason, method in interfaces are "virtual", not "abstract" + if (symbol.IsAbstract && interfaceDefinition) + HasModifier(cx, key, "virtual"); + + if (symbol.Kind == SymbolKind.Field && ((IFieldSymbol)symbol).IsReadOnly) + HasModifier(cx, key, "readonly"); + + if (symbol.IsOverride) + HasModifier(cx, key, "override"); + + if (symbol.Kind == SymbolKind.Method && ((IMethodSymbol)symbol).IsAsync) + HasModifier(cx, key, "async"); + + if (symbol.IsExtern) + HasModifier(cx, key, "extern"); + + foreach (var modifier in symbol.GetSourceLevelModifiers()) + HasModifier(cx, key, modifier); + + if (symbol.Kind == SymbolKind.NamedType) + { + INamedTypeSymbol nt = symbol as INamedTypeSymbol; + if (nt.TypeKind == TypeKind.Struct) + { + // Sadly, these properties are internal so cannot be accessed directly. + // This seems to be a deficiency in the model. + var readonlyProperty = nt.GetType().GetProperty("IsReadOnly", BindingFlags.NonPublic | BindingFlags.Instance); + var isByRefProperty = nt.GetType().GetProperty("IsByRefLikeType", BindingFlags.NonPublic | BindingFlags.Instance); + + bool isReadOnly = (bool)readonlyProperty.GetValue(nt); + bool isByRefLikeType = (bool)isByRefProperty.GetValue(nt); + + if (isReadOnly) + HasModifier(cx, key, "readonly"); + if (isByRefLikeType) + HasModifier(cx, key, "ref"); + } + } + } + + public static Modifier Create(Context cx, string modifier) => + ModifierFactory.Instance.CreateEntity(cx, new ModifierKey(modifier)); + + public static Modifier Create(Context cx, Accessibility access) => + ModifierFactory.Instance.CreateEntity(cx, new ModifierKey(AccessbilityModifier(access))); + + class ModifierFactory : ICachedEntityFactory + { + public static readonly ModifierFactory Instance = new ModifierFactory(); + + public Modifier Create(Context cx, ModifierKey init) => new Modifier(cx, init); + } + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs new file mode 100644 index 000000000000..f66ac8e6d605 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs @@ -0,0 +1,45 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Namespace : CachedEntity + { + Namespace(Context cx, INamespaceSymbol init) + : base(cx, init) { } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => null; + + public override void Populate() + { + Context.Emit(Tuples.namespaces(this, symbol.Name)); + + if (symbol.ContainingNamespace != null) + { + Namespace parent = Create(Context, symbol.ContainingNamespace); + Context.Emit(Tuples.parent_namespace(this, parent)); + } + } + + public override bool NeedsPopulation => true; + + public override IId Id + { + get + { + return symbol.IsGlobalNamespace ? new Key(";namespace") : + new Key(Create(Context, symbol.ContainingNamespace), ".", symbol.Name, ";namespace"); + } + } + + public static Namespace Create(Context cx, INamespaceSymbol ns) => NamespaceFactory.Instance.CreateEntity(cx, ns); + + class NamespaceFactory : ICachedEntityFactory + { + public static readonly NamespaceFactory Instance = new NamespaceFactory(); + + public Namespace Create(Context cx, INamespaceSymbol init) => new Namespace(cx, init); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs new file mode 100644 index 000000000000..d889361649d6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs @@ -0,0 +1,41 @@ + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class NamespaceDeclaration : FreshEntity + { + public NamespaceDeclaration(Context cx, NamespaceDeclarationSyntax node, NamespaceDeclaration parent) + : base(cx) + { + var ns = Namespace.Create(cx, (INamespaceSymbol)cx.Model(node).GetSymbolInfo(node.Name).Symbol); + cx.Emit(Tuples.namespace_declarations(this, ns)); + cx.Emit(Tuples.namespace_declaration_location(this, cx.Create(node.Name.GetLocation()))); + + var visitor = new Populators.TypeOrNamespaceVisitor(cx, this); + + foreach (var member in node.Members.Cast().Concat(node.Usings)) + { + member.Accept(visitor); + } + + if (parent != null) + { + cx.Emit(Tuples.parent_namespace_declaration(this, parent)); + } + } + + public void Extract(Context cx, NamespaceDeclarationSyntax decl) + { + decl.Accept(new Populators.TypeOrNamespaceVisitor(cx, this)); + } + + public static NamespaceDeclaration Create(Context cx, NamespaceDeclarationSyntax decl, NamespaceDeclaration parent) => new NamespaceDeclaration(cx, decl, parent); + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs new file mode 100644 index 000000000000..88895025446b --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs @@ -0,0 +1,60 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class OrdinaryMethod : Method + { + OrdinaryMethod(Context cx, IMethodSymbol init) + : base(cx, init) { } + + public override string Name => symbol.GetName(); + + public IMethodSymbol SourceDeclaration + { + get + { + var reducedFrom = symbol.ReducedFrom ?? symbol; + return reducedFrom.OriginalDefinition; + } + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => symbol.GetSymbolLocation(); + + public override void Populate() + { + PopulateMethod(); + ExtractModifiers(); + ContainingType.ExtractGenerics(); + + var returnType = Type.Create(Context, symbol.ReturnType); + Context.Emit(Tuples.methods(this, Name, ContainingType, returnType.TypeRef, OriginalDefinition)); + + if (IsSourceDeclaration) + foreach (var declaration in symbol.DeclaringSyntaxReferences.Select(s => s.GetSyntax()).OfType()) + { + Context.BindComments(this, declaration.Identifier.GetLocation()); + TypeMention.Create(Context, declaration.ReturnType, this, returnType); + } + + foreach (var l in Locations) + Context.Emit(Tuples.method_location(this, l)); + + ExtractGenerics(); + Overrides(); + ExtractRefReturn(); + ExtractCompilerGenerated(); + } + + public new static OrdinaryMethod Create(Context cx, IMethodSymbol method) => OrdinaryMethodFactory.Instance.CreateEntity(cx, method); + + class OrdinaryMethodFactory : ICachedEntityFactory + { + public static readonly OrdinaryMethodFactory Instance = new OrdinaryMethodFactory(); + + public OrdinaryMethod Create(Context cx, IMethodSymbol init) => new OrdinaryMethod(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs new file mode 100644 index 000000000000..b5858d5c5e94 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs @@ -0,0 +1,290 @@ +using System; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp.Populators; +using System.Linq; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + public class Parameter : CachedSymbol, IExpressionParentEntity + { + protected IEntity Parent; + protected readonly Parameter Original; + + protected Parameter(Context cx, IParameterSymbol init, IEntity parent, Parameter original) + : base(cx, init) + { + Parent = parent; + Original = original ?? this; + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => symbol.GetSymbolLocation(); + + public enum Kind + { + None, Ref, Out, Params, This, In + } + + protected virtual int Ordinal + { + get + { + // For some reason, methods of kind ReducedExtension + // omit the "this" parameter, so the parameters are + // actually numbered from 1. + // This is to be consistent from the original (unreduced) extension method. + var method = symbol.ContainingSymbol as IMethodSymbol; + bool isReducedExtension = method != null && method.MethodKind == MethodKind.ReducedExtension; + return symbol.Ordinal + (isReducedExtension ? 1 : 0); + } + } + + Kind ParamKind + { + get + { + switch (symbol.RefKind) + { + case RefKind.Out: + return Kind.Out; + case RefKind.Ref: + return Kind.Ref; + case RefKind.In: + return Kind.In; + default: + if (symbol.IsParams) return Kind.Params; + + if (Ordinal == 0) + { + var method = symbol.ContainingSymbol as IMethodSymbol; + if (method != null && method.IsExtensionMethod) return Kind.This; + } + return Kind.None; + } + } + } + + public static Parameter Create(Context cx, IParameterSymbol param, IEntity parent, Parameter original = null) => + ParameterFactory.Instance.CreateEntity(cx, param, parent, original); + + /// + /// Gets the parameter entity for which must + /// already have been created. + /// + public static Parameter GetAlreadyCreated(Context cx, IParameterSymbol param) => + ParameterFactory.Instance.CreateEntity(cx, param, null, null); + + public override IId Id + { + get + { + // This is due to a bug in Roslyn when ValueTuple.cs is extracted. + // The parameter symbols don't match up properly so we don't have a parent. + if (Parent == null) + Parent = Method.Create(Context, symbol.ContainingSymbol as IMethodSymbol); + return new Key(Parent, "_", Ordinal, ";parameter"); + } + } + + public override bool NeedsPopulation => true; + + string Name + { + get + { + // Very rarely, two parameters have the same name according to the data model. + // This breaks our database constraints. + // Generate an impossible name to ensure that it doesn't conflict. + int conflictingCount = symbol.ContainingSymbol.GetParameters().Count(p => p.Ordinal < symbol.Ordinal && p.Name == symbol.Name); + return conflictingCount > 0 ? symbol.Name + "`" + conflictingCount : symbol.Name; + } + } + + public override void Populate() + { + ExtractAttributes(); + + if (symbol.Name != Original.symbol.Name) + Context.ModelError(symbol, "Inconsistent parameter declaration"); + + var type = Type.Create(Context, symbol.Type); + Context.Emit(Tuples.@params(this, Name, type.TypeRef, Ordinal, ParamKind, Parent, Original)); + + foreach (var l in symbol.Locations) + Context.Emit(Tuples.param_location(this, Context.Create(l))); + + if (!IsSourceDeclaration || !symbol.FromSource()) + return; + + BindComments(); + + if (IsSourceDeclaration) + foreach (var syntax in symbol.DeclaringSyntaxReferences. + Select(d => d.GetSyntax()). + OfType(). + Where(s => s.Type != null)) + TypeMention.Create(Context, syntax.Type, this, type); + + if (symbol.HasExplicitDefaultValue && Context.Defines(symbol)) + { + // This is a slight bug in the dbscheme + // We should really define param_default(param, string) + // And use parameter child #0 to encode the default expression. + var defaultValue = GetParameterDefaultValue(symbol); + if (defaultValue == null) + { + // In case this parameter belongs to an accessor of an indexer, we need + // to get the default value from the corresponding parameter belonging + // to the indexer itself + var method = (IMethodSymbol)symbol.ContainingSymbol; + if (method != null) + { + var i = method.Parameters.IndexOf(symbol); + var indexer = (IPropertySymbol)method.AssociatedSymbol; + if (indexer != null) + defaultValue = GetParameterDefaultValue(indexer.Parameters[i]); + } + } + + if (defaultValue != null) + { + Context.PopulateLater(() => + { + Expression.Create(Context, defaultValue.Value, this, 0); + }); + } + } + } + + public override bool IsSourceDeclaration => symbol.IsSourceDeclaration(); + + bool IExpressionParentEntity.IsTopLevelParent => true; + + static EqualsValueClauseSyntax GetParameterDefaultValue(IParameterSymbol parameter) + { + var syntax = parameter.DeclaringSyntaxReferences.Select(@ref => @ref.GetSyntax()).OfType().FirstOrDefault(); + return syntax != null ? syntax.Default : null; + } + + class ParameterFactory : ICachedEntityFactory<(IParameterSymbol, IEntity, Parameter), Parameter> + { + public static readonly ParameterFactory Instance = new ParameterFactory(); + + public Parameter Create(Context cx, (IParameterSymbol, IEntity, Parameter) init) => new Parameter(cx, init.Item1, init.Item2, init.Item3); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } + + class VarargsType : Type + { + VarargsType(Context cx) + : base(cx, null) { } + + public override void Populate() + { + Context.Emit(Tuples.types(this, Kinds.TypeKind.ARGLIST, "__arglist")); + Context.Emit(Tuples.parent_namespace(this, Namespace.Create(Context, Context.Compilation.GlobalNamespace))); + Modifier.HasModifier(Context, this, "public"); + } + + public override bool NeedsPopulation => true; + + public sealed override IId Id => new Key("__arglist;type"); + + public override int GetHashCode() + { + return 98735267; + } + + public override bool Equals(object obj) + { + return obj != null && obj.GetType() == typeof(VarargsType); + } + + public static VarargsType Create(Context cx) => VarargsTypeFactory.Instance.CreateEntity(cx, null); + + class VarargsTypeFactory : ICachedEntityFactory + { + public static readonly VarargsTypeFactory Instance = new VarargsTypeFactory(); + + public VarargsType Create(Context cx, string init) => new VarargsType(cx); + } + } + + class VarargsParam : Parameter + { + VarargsParam(Context cx, Method methodKey) + : base(cx, null, methodKey, null) { } + + public override void Populate() + { + var typeKey = VarargsType.Create(Context); + // !! Maybe originaldefinition is wrong + Context.Emit(Tuples.@params(this, "", typeKey, Ordinal, Kind.None, Parent, this)); + Context.Emit(Tuples.param_location(this, GeneratedLocation.Create(Context))); + } + + protected override int Ordinal => ((Method)Parent).OriginalDefinition.symbol.Parameters.Length; + + public override int GetHashCode() + { + return 9873567; + } + + public override bool Equals(object obj) + { + return obj != null && obj.GetType() == typeof(VarargsParam); + } + + public static VarargsParam Create(Context cx, Method method) => VarargsParamFactory.Instance.CreateEntity(cx, method); + + class VarargsParamFactory : ICachedEntityFactory + { + public static readonly VarargsParamFactory Instance = new VarargsParamFactory(); + + public VarargsParam Create(Context cx, Method init) => new VarargsParam(cx, init); + } + } + + class ConstructedExtensionParameter : Parameter + { + readonly ITypeSymbol ConstructedType; + + ConstructedExtensionParameter(Context cx, Method method, Parameter original) + : base(cx, original.symbol, method, original) + { + ConstructedType = method.symbol.ReceiverType; + } + + public override void Populate() + { + var typeKey = Type.Create(Context, ConstructedType); + Context.Emit(Tuples.@params(this, Original.symbol.Name, typeKey.TypeRef, 0, Kind.This, Parent, Original)); + Context.Emit(Tuples.param_location(this, Original.Location)); + } + + public override int GetHashCode() => symbol.GetHashCode() + 31 * ConstructedType.GetHashCode(); + + public override bool Equals(object obj) + { + var other = obj as ConstructedExtensionParameter; + if (other == null || other.GetType() != typeof(ConstructedExtensionParameter)) + return false; + + return Equals(symbol, other.symbol) && Equals(ConstructedType, other.ConstructedType); + } + + public static ConstructedExtensionParameter Create(Context cx, Method method, Parameter parameter) => + ExtensionParamFactory.Instance.CreateEntity(cx, (method, parameter)); + + class ExtensionParamFactory : ICachedEntityFactory<(Method, Parameter), ConstructedExtensionParameter> + { + public static readonly ExtensionParamFactory Instance = new ExtensionParamFactory(); + + public ConstructedExtensionParameter Create(Context cx, (Method, Parameter) init) => + new ConstructedExtensionParameter(cx, init.Item1, init.Item2); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs new file mode 100644 index 000000000000..7b26e21be457 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs @@ -0,0 +1,119 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class Property : CachedSymbol, IExpressionParentEntity + { + protected Property(Context cx, IPropertySymbol init) + : base(cx, init) { } + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(ContainingType); + tb.Append("."); + Method.AddExplicitInterfaceQualifierToId(Context, tb, symbol.ExplicitInterfaceImplementations); + tb.Append(symbol.Name); + tb.Append(";property"); + }); + } + } + + protected Accessor Getter { get; set; } + protected Accessor Setter { get; set; } + + public override void Populate() + { + ExtractAttributes(); + ExtractModifiers(); + BindComments(); + + var type = Type.Create(Context, symbol.Type); + Context.Emit(Tuples.properties(this, symbol.GetName(), ContainingType, type.TypeRef, Create(Context, symbol.OriginalDefinition))); + + var getter = symbol.GetMethod; + if (getter != null) + Getter = Accessor.Create(Context, getter); + + var setter = symbol.SetMethod; + if (setter != null) + Setter = Accessor.Create(Context, setter); + + var declSyntaxReferences = IsSourceDeclaration ? + symbol.DeclaringSyntaxReferences. + Select(d => d.GetSyntax()).OfType().ToArray() + : Enumerable.Empty(); + + foreach (var explicitInterface in symbol.ExplicitInterfaceImplementations.Select(impl => Type.Create(Context, impl.ContainingType))) + { + Context.Emit(Tuples.explicitly_implements(this, explicitInterface.TypeRef)); + + foreach (var syntax in declSyntaxReferences) + TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier.Name, this, explicitInterface); + } + + foreach (var l in Locations) + Context.Emit(Tuples.property_location(this, l)); + + if (IsSourceDeclaration && symbol.FromSource()) + { + var expressionBody = ExpressionBody; + if (expressionBody != null) + { + Context.PopulateLater(() => Expression.Create(Context, expressionBody, this, 0)); + } + + foreach (var initializer in declSyntaxReferences. + Select(n => n.Initializer). + Where(i => i != null). + Select(i => i.Value)) + { + Context.PopulateLater(() => Expression.Create(Context, initializer, this, 1)); + } + + foreach (var syntax in declSyntaxReferences) + TypeMention.Create(Context, syntax.Type, this, type); + } + } + + public override Microsoft.CodeAnalysis.Location FullLocation + { + get + { + return + symbol. + DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + Select(s => s.GetLocation()). + Concat(symbol.Locations). + First(); + } + } + + bool IExpressionParentEntity.IsTopLevelParent => true; + + public static Property Create(Context cx, IPropertySymbol prop) + { + return prop.IsIndexer ? Indexer.Create(cx, prop) : PropertyFactory.Instance.CreateEntity(cx, prop); + } + + public void VisitDeclaration(Context cx, PropertyDeclarationSyntax p) + { + } + + class PropertyFactory : ICachedEntityFactory + { + public static readonly PropertyFactory Instance = new PropertyFactory(); + + public Property Create(Context cx, IPropertySymbol init) => new Property(cx, init); + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement.cs new file mode 100644 index 000000000000..8fb755c18059 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statement.cs @@ -0,0 +1,79 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities +{ + /// + /// Whether this entity is the parent of a top-level statement. + /// + public interface IStatementParentEntity : IEntity + { + bool IsTopLevelParent { get; } + } + + abstract class Statement : FreshEntity, IExpressionParentEntity, IStatementParentEntity + { + protected Statement(Context cx) : base(cx) { } + + public static Statement Create(Context cx, StatementSyntax node, Statement parent, int child) => + Statements.Factory.Create(cx, node, parent, child); + + /// + /// How many statements does this take up in a block. + /// The default is 1, however labelled statements can be more. + /// + public virtual int NumberOfStatements => 1; + + public override Microsoft.CodeAnalysis.Location ReportingLocation => GetStatementSyntax().GetLocation(); + + bool IExpressionParentEntity.IsTopLevelParent => false; + + bool IStatementParentEntity.IsTopLevelParent => false; + + protected abstract CSharpSyntaxNode GetStatementSyntax(); + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NeedsLabel; + } + + abstract class Statement : Statement where TSyntax : CSharpSyntaxNode + { + protected readonly TSyntax Stmt; + + protected override CSharpSyntaxNode GetStatementSyntax() => Stmt; + + protected Statement(Context cx, TSyntax stmt, Kinds.StmtKind kind, IStatementParentEntity parent, int child, Location location) + : base(cx) + { + Stmt = stmt; + cx.BindComments(this, location.symbol); + cx.Emit(Tuples.statements(this, kind)); + if (parent.IsTopLevelParent) + cx.Emit(Tuples.stmt_parent_top_level(this, child, parent)); + else + cx.Emit(Tuples.stmt_parent(this, child, parent)); + cx.Emit(Tuples.stmt_location(this, location)); + } + + protected Statement(Context cx, TSyntax stmt, Kinds.StmtKind kind, IStatementParentEntity parent, int child) + : this(cx, stmt, kind, parent, child, cx.Create(stmt.FixedLocation())) { } + + /// + /// Populates statement-type specific relations in the trap file. The general relations + /// statements and stmt_location are populated by the constructor + /// (should not fail), so even if statement-type specific population fails (e.g., in + /// standalone extraction), the statement created via + /// will still + /// be valid. + /// + protected abstract void Populate(); + + protected void TryPopulate() + { + cx.Try(Stmt, null, Populate); + } + + public override string ToString() => Label.ToString(); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Block.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Block.cs new file mode 100644 index 000000000000..575dbab2e4eb --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Block.cs @@ -0,0 +1,28 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Block : Statement + { + Block(Context cx, BlockSyntax block, IStatementParentEntity parent, int child) + : base(cx, block, StmtKind.BLOCK, parent, child) { } + + public static Block Create(Context cx, BlockSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Block(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + var child = 0; + foreach (var childStmt in Stmt.Statements.Select(c => Statement.Create(cx, c, this, child))) + { + child += childStmt.NumberOfStatements; + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Break.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Break.cs new file mode 100644 index 000000000000..571411698058 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Break.cs @@ -0,0 +1,20 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Break : Statement + { + Break(Context cx, BreakStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.BREAK, parent, child) { } + + public static Break Create(Context cx, BreakStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Break(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() { } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Case.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Case.cs new file mode 100644 index 000000000000..088b60ee1691 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Case.cs @@ -0,0 +1,105 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + abstract class Case : Statement where TSyntax : SwitchLabelSyntax + { + protected Case(Context cx, TSyntax node, Switch parent, int child) + : base(cx, node, StmtKind.CASE, parent, child, cx.Create(node.GetLocation())) { } + + public static Statement Create(Context cx, SwitchLabelSyntax node, Switch parent, int child) + { + switch (node.Kind()) + { + case SyntaxKind.CaseSwitchLabel: + return CaseLabel.Create(cx, (CaseSwitchLabelSyntax)node, parent, child); + case SyntaxKind.DefaultSwitchLabel: + return CaseDefault.Create(cx, (DefaultSwitchLabelSyntax)node, parent, child); + case SyntaxKind.CasePatternSwitchLabel: + return CasePattern.Create(cx, (CasePatternSwitchLabelSyntax)node, parent, child); + default: + throw new InternalError(node, "Unhandled case label"); + } + } + } + + class CaseLabel : Case + { + CaseLabel(Context cx, CaseSwitchLabelSyntax node, Switch parent, int child) + : base(cx, node, parent, child) { } + + protected override void Populate() + { + var value = Stmt.Value; + Expression.Create(cx, value, this, 0); + Switch.LabelForValue(cx.Model(Stmt).GetConstantValue(value).Value); + } + + public static CaseLabel Create(Context cx, CaseSwitchLabelSyntax node, Switch parent, int child) + { + var ret = new CaseLabel(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + } + + class CaseDefault : Case + { + CaseDefault(Context cx, DefaultSwitchLabelSyntax node, Switch parent, int child) + : base(cx, node, parent, child) { } + + protected override void Populate() { } + + public static CaseDefault Create(Context cx, DefaultSwitchLabelSyntax node, Switch parent, int child) + { + var ret = new CaseDefault(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + } + + class CasePattern : Case + { + CasePattern(Context cx, CasePatternSwitchLabelSyntax node, Switch parent, int child) + : base(cx, node, parent, child) { } + + protected override void Populate() + { + switch(Stmt.Pattern) + { + case DeclarationPatternSyntax declarationPattern: + var symbol = cx.Model(Stmt).GetDeclaredSymbol(declarationPattern.Designation) as ILocalSymbol; + if (symbol != null) + { + var type = Type.Create(cx, symbol.Type); + var isVar = declarationPattern.Type.IsVar; + Expressions.VariableDeclaration.Create(cx, symbol, type, cx.Create(declarationPattern.GetLocation()), cx.Create(declarationPattern.Designation.GetLocation()), isVar, this, 0); + } + + Expressions.TypeAccess.Create(cx, declarationPattern.Type, this, 1); + break; + case ConstantPatternSyntax pattern: + Expression.Create(cx, pattern.Expression, this, 0); + break; + default: + throw new InternalError(Stmt, "Case pattern not handled"); + } + + if (Stmt.WhenClause != null) + { + Expression.Create(cx, Stmt.WhenClause.Condition, this, 2); + } + } + + public static CasePattern Create(Context cx, CasePatternSwitchLabelSyntax node, Switch parent, int child) + { + var ret = new CasePattern(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Catch.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Catch.cs new file mode 100644 index 000000000000..f85e0659dbaa --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Catch.cs @@ -0,0 +1,51 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Catch : Statement + { + static readonly string SystemExceptionName = typeof(System.Exception).ToString(); + + Catch(Context cx, CatchClauseSyntax node, Try parent, int child) + : base(cx, node, StmtKind.CATCH, parent, child, cx.Create(node.GetLocation())) { } + + protected override void Populate() + { + bool isSpecificCatchClause = Stmt.Declaration != null; + bool hasVariableDeclaration = isSpecificCatchClause && Stmt.Declaration.Identifier.RawKind != 0; + + if (hasVariableDeclaration) // A catch clause of the form 'catch(Ex ex) { ... }' + { + var decl = Expressions.VariableDeclaration.Create(cx, Stmt.Declaration, false, this, 0); + cx.Emit(Tuples.catch_type(this, decl.Type.TypeRef, true)); + } + else if (isSpecificCatchClause) // A catch clause of the form 'catch(Ex) { ... }' + { + cx.Emit(Tuples.catch_type(this, Type.Create(cx, cx.GetType(Stmt.Declaration.Type)).TypeRef, true)); + } + else // A catch clause of the form 'catch { ... }' + { + var exception = Type.Create(cx, cx.Compilation.GetTypeByMetadataName(SystemExceptionName)); + cx.Emit(Tuples.catch_type(this, exception, false)); + } + + if (Stmt.Filter != null) + { + // For backward compatibility, the catch filter clause is child number 2. + Expression.Create(cx, Stmt.Filter.FilterExpression, this, 2); + } + + Create(cx, Stmt.Block, this, 1); + } + + public static Catch Create(Context cx, CatchClauseSyntax node, Try parent, int child) + { + var ret = new Catch(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Checked.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Checked.cs new file mode 100644 index 000000000000..55ce9d800580 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Checked.cs @@ -0,0 +1,23 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Checked : Statement + { + Checked(Context cx, CheckedStatementSyntax stmt, IStatementParentEntity parent, int child) + : base(cx, stmt, StmtKind.CHECKED, parent, child) { } + + public static Checked Create(Context cx, CheckedStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Checked(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Statement.Create(cx, Stmt.Block, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Continue.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Continue.cs new file mode 100644 index 000000000000..70038e239ff6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Continue.cs @@ -0,0 +1,21 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Continue : Statement + { + Continue(Context cx, ContinueStatementSyntax stmt, IStatementParentEntity parent, int child) + : base(cx, stmt, StmtKind.CONTINUE, parent, child) { } + + public static Continue Create(Context cx, ContinueStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Continue(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() { } + } +} + diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Do.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Do.cs new file mode 100644 index 000000000000..56cbb0a22e1f --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Do.cs @@ -0,0 +1,25 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Do : Statement + { + Do(Context cx, DoStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.DO, parent, child, cx.Create(node.GetLocation())) { } + + public static Do Create(Context cx, DoStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Do(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Create(cx, Stmt.Statement, this, 1); + Expression.Create(cx, Stmt.Condition, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Empty.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Empty.cs new file mode 100644 index 000000000000..7b32ec7d09d0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Empty.cs @@ -0,0 +1,20 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Empty : Statement + { + Empty(Context cx, EmptyStatementSyntax block, IStatementParentEntity parent, int child) + : base(cx, block, StmtKind.EMPTY, parent, child) { } + + public static Empty Create(Context cx, EmptyStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Empty(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() { } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ExpressionStatement.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ExpressionStatement.cs new file mode 100644 index 000000000000..de9a19859878 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ExpressionStatement.cs @@ -0,0 +1,25 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class ExpressionStatement : Statement + { + ExpressionStatement(Context cx, ExpressionStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, Kinds.StmtKind.EXPR, parent, child) { } + + public static ExpressionStatement Create(Context cx, ExpressionStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new ExpressionStatement(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + if (Stmt.Expression != null) + Expression.Create(cx, Stmt.Expression, this, 0); + else + cx.ModelError(Stmt, "Invalid expression statement"); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Factory.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Factory.cs new file mode 100644 index 000000000000..a804b0445068 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Factory.cs @@ -0,0 +1,73 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + static class Factory + { + internal static Statement Create(Context cx, StatementSyntax node, Statement parent, int child) + { + switch (node.Kind()) + { + case SyntaxKind.ForStatement: + return For.Create(cx, (ForStatementSyntax)node, parent, child); + case SyntaxKind.ExpressionStatement: + return ExpressionStatement.Create(cx, (ExpressionStatementSyntax)node, parent, child); + case SyntaxKind.UsingStatement: + return Using.Create(cx, (UsingStatementSyntax)node, parent, child); + case SyntaxKind.LocalDeclarationStatement: + return LocalDeclaration.Create(cx, (LocalDeclarationStatementSyntax)node, parent, child); + case SyntaxKind.Block: + return Block.Create(cx, (BlockSyntax)node, parent, child); + case SyntaxKind.ReturnStatement: + return Return.Create(cx, (ReturnStatementSyntax)node, parent, child); + case SyntaxKind.SwitchStatement: + return Switch.Create(cx, (SwitchStatementSyntax)node, parent, child); + case SyntaxKind.BreakStatement: + return Break.Create(cx, (BreakStatementSyntax)node, parent, child); + case SyntaxKind.IfStatement: + return If.Create(cx, (IfStatementSyntax)node, parent, child); + case SyntaxKind.WhileStatement: + return While.Create(cx, (WhileStatementSyntax)node, parent, child); + case SyntaxKind.DoStatement: + return Do.Create(cx, (DoStatementSyntax)node, parent, child); + case SyntaxKind.YieldReturnStatement: + return Yield.Create(cx, (YieldStatementSyntax)node, parent, child); + case SyntaxKind.ThrowStatement: + return Throw.Create(cx, (ThrowStatementSyntax)node, parent, child); + case SyntaxKind.TryStatement: + return Try.Create(cx, (TryStatementSyntax)node, parent, child); + case SyntaxKind.EmptyStatement: + return Empty.Create(cx, (EmptyStatementSyntax)node, parent, child); + case SyntaxKind.FixedStatement: + return Fixed.Create(cx, (FixedStatementSyntax)node, parent, child); + case SyntaxKind.LockStatement: + return Lock.Create(cx, (LockStatementSyntax)node, parent, child); + case SyntaxKind.GotoDefaultStatement: + case SyntaxKind.GotoStatement: + case SyntaxKind.GotoCaseStatement: + return Goto.Create(cx, (GotoStatementSyntax)node, parent, child); + case SyntaxKind.LabeledStatement: + return Labeled.Create(cx, (LabeledStatementSyntax)node, parent, child); + case SyntaxKind.CheckedStatement: + return Checked.Create(cx, (CheckedStatementSyntax)node, parent, child); + case SyntaxKind.UncheckedStatement: + return Unchecked.Create(cx, (CheckedStatementSyntax)node, parent, child); + case SyntaxKind.ForEachStatement: + return ForEach.Create(cx, (ForEachStatementSyntax)node, parent, child); + case SyntaxKind.YieldBreakStatement: + return Yield.Create(cx, (YieldStatementSyntax)node, parent, child); + case SyntaxKind.ContinueStatement: + return Continue.Create(cx, (ContinueStatementSyntax)node, parent, child); + case SyntaxKind.UnsafeStatement: + return Unsafe.Create(cx, (UnsafeStatementSyntax)node, parent, child); + case SyntaxKind.LocalFunctionStatement: + return LocalFunction.Create(cx, (LocalFunctionStatementSyntax)node, parent, child); + case SyntaxKind.ForEachVariableStatement: + return ForEachVariable.Create(cx, (ForEachVariableStatementSyntax)node, parent, child); + default: + throw new InternalError(node, "Unhandled statement of kind '{0}'", node.Kind()); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Fixed.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Fixed.cs new file mode 100644 index 000000000000..4e9e4ae4ac94 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Fixed.cs @@ -0,0 +1,25 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities.Expressions; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Fixed : Statement + { + Fixed(Context cx, FixedStatementSyntax @fixed, IStatementParentEntity parent, int child) + : base(cx, @fixed, StmtKind.FIXED, parent, child) { } + + public static Fixed Create(Context cx, FixedStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Fixed(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + VariableDeclarations.Populate(cx, Stmt.Declaration, this, -1, childIncrement: -1); + Create(cx, Stmt.Statement, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/For.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/For.cs new file mode 100644 index 000000000000..2c918af83ef6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/For.cs @@ -0,0 +1,45 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities.Expressions; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class For : Statement + { + For(Context cx, ForStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.FOR, parent, child) { } + + public static For Create(Context cx, ForStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new For(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + int child = -1; + + if (Stmt.Declaration != null) + VariableDeclarations.Populate(cx, Stmt.Declaration, this, child, childIncrement: -1); + + foreach (var init in Stmt.Initializers) + { + Expression.Create(cx, init, this, child--); + } + + Statement.Create(cx, Stmt.Statement, this, 1 + Stmt.Incrementors.Count); + + child = 1; + foreach (var inc in Stmt.Incrementors) + { + Expression.Create(cx, inc, this, child++); + } + + if (Stmt.Condition != null) + { + Expression.Create(cx, Stmt.Condition, this, 0); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ForEach.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ForEach.cs new file mode 100644 index 000000000000..2c3076757c26 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/ForEach.cs @@ -0,0 +1,56 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class ForEach : Statement + { + ForEach(Context cx, ForEachStatementSyntax stmt, IStatementParentEntity parent, int child) + : base(cx, stmt, StmtKind.FOREACH, parent, child) { } + + public static ForEach Create(Context cx, ForEachStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new ForEach(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Expression, this, 1); + + var typeSymbol = cx.Model(Stmt).GetDeclaredSymbol(Stmt); + var type = Type.Create(cx, typeSymbol.Type); + + var location = cx.Create(Stmt.Identifier.GetLocation()); + + if (typeSymbol.Name != "_") + Expressions.VariableDeclaration.Create(cx, typeSymbol, type, location, location, Stmt.Type.IsVar, this, 0); + TypeMention.Create(cx, Stmt.Type, this, type); + + Statement.Create(cx, Stmt.Statement, this, 2); + } + } + + class ForEachVariable : Statement + { + ForEachVariable(Context cx, ForEachVariableStatementSyntax stmt, IStatementParentEntity parent, int child) + : base(cx, stmt, StmtKind.FOREACH, parent, child) { } + + public static ForEachVariable Create(Context cx, ForEachVariableStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new ForEachVariable(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Variable, this, 0); + Expression.Create(cx, Stmt.Expression, this, 1); + Statement.Create(cx, Stmt.Statement, this, 2); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Goto.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Goto.cs new file mode 100644 index 000000000000..e5fdb2f637f7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Goto.cs @@ -0,0 +1,57 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + /// + /// A goto, goto case or goto default. + /// + class Goto : Statement + { + static StmtKind GetKind(GotoStatementSyntax node) + { + switch (node.CaseOrDefaultKeyword.Kind()) + { + case SyntaxKind.None: return StmtKind.GOTO; + case SyntaxKind.DefaultKeyword: return StmtKind.GOTO_DEFAULT; + case SyntaxKind.CaseKeyword: return StmtKind.GOTO_CASE; + default: throw new InternalError(node, "Unhandled goto statement kind {0}", node.CaseOrDefaultKeyword.Kind()); + } + } + + Goto(Context cx, GotoStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, GetKind(node), parent, child) { } + + public static Goto Create(Context cx, GotoStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Goto(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + switch (GetKind(Stmt)) + { + case StmtKind.GOTO: + var target = ((IdentifierNameSyntax)Stmt.Expression).Identifier.Text; + cx.Emit(Tuples.exprorstmt_name(this, target)); + break; + case StmtKind.GOTO_CASE: + Expr = Expression.Create(cx, Stmt.Expression, this, 0); + ConstantValue = Switch.LabelForValue(cx.Model(Stmt).GetConstantValue(Stmt.Expression).Value); + break; + case StmtKind.GOTO_DEFAULT: + ConstantValue = Switch.DefaultLabel; + break; + } + } + + public Expression Expr { get; private set; } + + public object ConstantValue { get; private set; } + + public bool IsDefault => ConstantValue == Switch.DefaultLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/If.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/If.cs new file mode 100644 index 000000000000..6cdd486c3638 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/If.cs @@ -0,0 +1,28 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class If : Statement + { + If(Context cx, IfStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.IF, parent, child) { } + + public static If Create(Context cx, IfStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new If(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Condition, this, 0); + + Create(cx, Stmt.Statement, this, 1); + + if (Stmt.Else != null) + Create(cx, Stmt.Else.Statement, this, 2); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Labeled.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Labeled.cs new file mode 100644 index 000000000000..30c4726c52ef --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Labeled.cs @@ -0,0 +1,38 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Labeled : Statement + { + readonly Statement Parent; + readonly int Child; + + Labeled(Context cx, LabeledStatementSyntax stmt, Statement parent, int child) + : base(cx, stmt, StmtKind.LABEL, parent, child) + { + Parent = parent; + Child = child; + } + + public static Labeled Create(Context cx, LabeledStatementSyntax node, Statement parent, int child) + { + var ret = new Labeled(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + cx.Emit(Tuples.exprorstmt_name(this, Stmt.Identifier.ToString())); + + // For compatilibty with the Mono extractor, make insert the labelled statement into the same block + // as this one. The parent MUST be a block statement. + labelledStmt = Statement.Create(cx, Stmt.Statement, Parent, Child + 1); + } + + Statement labelledStmt; + + public override int NumberOfStatements => 1 + labelledStmt.NumberOfStatements; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalDeclaration.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalDeclaration.cs new file mode 100644 index 000000000000..671b4d7d99e3 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalDeclaration.cs @@ -0,0 +1,25 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities.Expressions; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class LocalDeclaration : Statement + { + LocalDeclaration(Context cx, LocalDeclarationStatementSyntax declStmt, IStatementParentEntity parent, int child) + : base(cx, declStmt, declStmt.IsConst ? StmtKind.CONST_DECL : StmtKind.VAR_DECL, parent, child) { } + + public static LocalDeclaration Create(Context cx, LocalDeclarationStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new LocalDeclaration(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + VariableDeclarations.Populate(cx, Stmt.Declaration, this, 0); + cx.BindComments(this, Stmt.GetLocation()); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalFunction.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalFunction.cs new file mode 100644 index 000000000000..e4a553f4400a --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/LocalFunction.cs @@ -0,0 +1,48 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class LocalFunction : Statement + { + LocalFunction(Context cx, LocalFunctionStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.LOCAL_FUNCTION, parent, child, cx.Create(node.GetLocation())) { } + + public static LocalFunction Create(Context cx, LocalFunctionStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new LocalFunction(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + /// + /// Gets the IMethodSymbol for this local function statement. + /// + IMethodSymbol Symbol + { + get + { + // Ideally model.GetDeclaredSymbol(Stmt) would do + // the right thing but it doesn't exist. + // So instead, we have to do the lookup via GetEnclosingSymbol. + + var m = cx.Model(Stmt); + var body = Stmt.Body == null ? Stmt.ExpressionBody : (CSharpSyntaxNode)Stmt.Body; + return m.GetEnclosingSymbol(body.GetLocation().SourceSpan.Start) as IMethodSymbol; + } + } + + /// + /// Gets the function defined by this local statement. + /// + Entities.LocalFunction Function => Entities.LocalFunction.Create(cx, Symbol); + + protected override void Populate() + { + cx.Emit(Tuples.local_function_stmts(this, Function)); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Lock.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Lock.cs new file mode 100644 index 000000000000..e916f9ba6018 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Lock.cs @@ -0,0 +1,24 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Lock : Statement + { + Lock(Context cx, LockStatementSyntax @lock, IStatementParentEntity parent, int child) + : base(cx, @lock, StmtKind.LOCK, parent, child) { } + + public static Lock Create(Context cx, LockStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Lock(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Expression, this, 0); + Statement.Create(cx, Stmt.Statement, this, 1); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Return.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Return.cs new file mode 100644 index 000000000000..2616b8749971 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Return.cs @@ -0,0 +1,24 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Return : Statement + { + Return(Context cx, ReturnStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.RETURN, parent, child) { } + + public static Return Create(Context cx, ReturnStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Return(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + if (Stmt.Expression != null) + Expression.Create(cx, Stmt.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Switch.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Switch.cs new file mode 100644 index 000000000000..06e5364a84a0 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Switch.cs @@ -0,0 +1,49 @@ +using System.Linq; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Switch : Statement + { + static readonly object NullLabel = new object(); + public static readonly object DefaultLabel = new object(); + + // Sometimes, the literal "null" is used as a label. + // This is inconveniently represented by the "null" object. + // This cannot be stored in a Dictionary<>, so substitute an object which can be. + public static object LabelForValue(object label) + { + return label ?? NullLabel; + } + + Switch(Context cx, SwitchStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.SWITCH, parent, child) { } + + public static Switch Create(Context cx, SwitchStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Switch(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Expression, this, 0); + int childIndex = 0; + + foreach (var section in Stmt.Sections) + { + foreach (var stmt in section.Labels.Select(label => Case.Create(cx, label, this, childIndex))) + { + childIndex += stmt.NumberOfStatements; + } + + foreach (var stmt in section.Statements.Select(s => Create(cx, s, this, childIndex))) + { + childIndex += stmt.NumberOfStatements; + } + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Throw.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Throw.cs new file mode 100644 index 000000000000..98e3c37c1dcf --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Throw.cs @@ -0,0 +1,24 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Throw : Statement + { + Throw(Context cx, ThrowStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.THROW, parent, child) { } + + public static Throw Create(Context cx, ThrowStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Throw(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + if (Stmt.Expression != null) + Expression.Create(cx, Stmt.Expression, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Try.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Try.cs new file mode 100644 index 000000000000..4c0134839d90 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Try.cs @@ -0,0 +1,47 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Try : Statement + { + Try(Context cx, TryStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.TRY, parent, child) { } + + public static Try Create(Context cx, TryStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Try(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + var child = 1; + foreach (var c in Stmt.Catches) + { + Catch.Create(cx, c, this, child++); + } + + Create(cx, Stmt.Block, this, 0); + + if (Stmt.Finally != null) + { + Create(cx, Stmt.Finally.Block, this, -1); + } + } + + public static SyntaxNodeOrToken NextNode(SyntaxNode node) + { + for (var i = node.Parent.ChildNodesAndTokens().GetEnumerator(); i.MoveNext();) + { + if (i.Current == node) + { + return i.MoveNext() ? i.Current : null; + } + } + return null; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unchecked.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unchecked.cs new file mode 100644 index 000000000000..797d4021708e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unchecked.cs @@ -0,0 +1,23 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Unchecked : Statement + { + Unchecked(Context cx, CheckedStatementSyntax stmt, IStatementParentEntity parent, int child) + : base(cx, stmt, StmtKind.UNCHECKED, parent, child) { } + + public static Unchecked Create(Context cx, CheckedStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Unchecked(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Statement.Create(cx, Stmt.Block, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unsafe.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unsafe.cs new file mode 100644 index 000000000000..55385c4e604e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Unsafe.cs @@ -0,0 +1,23 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Unsafe : Statement + { + Unsafe(Context cx, UnsafeStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.UNSAFE, parent, child) { } + + public static Unsafe Create(Context cx, UnsafeStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Unsafe(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Create(cx, Stmt.Block, this, 0); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Using.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Using.cs new file mode 100644 index 000000000000..596de684de2c --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Using.cs @@ -0,0 +1,31 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities.Expressions; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Using : Statement + { + Using(Context cx, UsingStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.USING, parent, child) { } + + public static Using Create(Context cx, UsingStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Using(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + if (Stmt.Declaration != null) + VariableDeclarations.Populate(cx, Stmt.Declaration, this, -1, childIncrement: -1); + + if (Stmt.Expression != null) + Expression.Create(cx, Stmt.Expression, this, 0); + + if (Stmt.Statement != null) + Statement.Create(cx, Stmt.Statement, this, 1); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/While.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/While.cs new file mode 100644 index 000000000000..40ba9190b332 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/While.cs @@ -0,0 +1,24 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class While : Statement + { + While(Context cx, WhileStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.WHILE, parent, child) { } + + public static While Create(Context cx, WhileStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new While(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + Expression.Create(cx, Stmt.Condition, this, 0); + Create(cx, Stmt.Statement, this, 1); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Yield.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Yield.cs new file mode 100644 index 000000000000..a1d45af7de24 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Statements/Yield.cs @@ -0,0 +1,26 @@ +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Kinds; + +namespace Semmle.Extraction.CSharp.Entities.Statements +{ + class Yield : Statement + { + Yield(Context cx, YieldStatementSyntax node, IStatementParentEntity parent, int child) + : base(cx, node, StmtKind.YIELD, parent, child) { } + + public static Yield Create(Context cx, YieldStatementSyntax node, IStatementParentEntity parent, int child) + { + var ret = new Yield(cx, node, parent, child); + ret.TryPopulate(); + return ret; + } + + protected override void Populate() + { + if (Stmt.Expression != null) + { + Expression.Create(cx, Stmt.Expression, this, 0); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Symbol.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Symbol.cs new file mode 100644 index 000000000000..0c56855db818 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Symbol.cs @@ -0,0 +1,102 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + public abstract class CachedSymbol : CachedEntity where T : ISymbol + { + public CachedSymbol(Context cx, T init) + : base(cx, init) { } + + public virtual Type ContainingType => symbol.ContainingType != null ? Type.Create(Context, symbol.ContainingType) : null; + + public void ExtractModifiers() + { + Modifier.ExtractModifiers(Context, this, symbol); + } + + protected void ExtractAttributes() + { + // Only extract attributes for source declarations + if (ReferenceEquals(symbol, symbol.OriginalDefinition)) + Attribute.ExtractAttributes(Context, symbol, this); + } + + protected void ExtractCompilerGenerated() + { + if (symbol.IsImplicitlyDeclared) + Context.Emit(Tuples.compiler_generated(this)); + } + + /// + /// The location which is stored in the database and is used when highlighing source code. + /// It's generally short, e.g. a method name. + /// + public override Microsoft.CodeAnalysis.Location ReportingLocation => symbol.Locations.FirstOrDefault(); + + /// + /// The full text span of the entity, e.g. for binding comments. + /// + public virtual Microsoft.CodeAnalysis.Location FullLocation => symbol.Locations.FirstOrDefault(); + + public virtual IEnumerable Locations + { + get + { + var loc = ReportingLocation; + if (loc != null) + { + // Some built in operators lack locations, so loc is null. + yield return Context.Create(ReportingLocation); + if (Context.Extractor.OutputPath != null && loc.Kind == LocationKind.SourceFile) + yield return Assembly.CreateOutputAssembly(Context); + } + } + } + + /// + /// Bind comments to this symbol. + /// Comments are only bound to source declarations. + /// + protected void BindComments() + { + if (!symbol.IsImplicitlyDeclared && IsSourceDeclaration && symbol.FromSource()) + Context.BindComments(this, FullLocation); + } + + public BlockSyntax Block + { + get + { + return symbol. + DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + SelectMany(n => n.ChildNodes()). + OfType(). + FirstOrDefault(); + } + } + + public ExpressionSyntax ExpressionBody + { + get + { + return symbol. + DeclaringSyntaxReferences. + SelectMany(r => r.GetSyntax().ChildNodes()). + OfType(). + Select(arrow => arrow.Expression). + FirstOrDefault(); + } + } + + public virtual bool IsSourceDeclaration => symbol.IsSourceDeclaration(); + + public override bool NeedsPopulation => Context.Defines(symbol); + + public Extraction.Entities.Location Location => Context.Create(ReportingLocation); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/TypeMention.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/TypeMention.cs new file mode 100644 index 000000000000..9b32bc359e84 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/TypeMention.cs @@ -0,0 +1,93 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using Semmle.Util; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class TypeMention : FreshEntity + { + readonly TypeSyntax Syntax; + readonly IEntity Parent; + readonly Type Type; + readonly Microsoft.CodeAnalysis.Location Loc; + + TypeMention(Context cx, TypeSyntax syntax, IEntity parent, Type type, Microsoft.CodeAnalysis.Location loc = null) + : base(cx) + { + Syntax = syntax; + Parent = parent; + Type = type; + Loc = loc; + } + + void Populate() + { + switch (Syntax.Kind()) + { + case SyntaxKind.ArrayType: + var ats = (ArrayTypeSyntax)Syntax; + var at = (ArrayType)Type; + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + Create(cx, ats.ElementType, this, at.ElementType); + return; + case SyntaxKind.NullableType: + var nts = (NullableTypeSyntax)Syntax; + var nt = (NamedType)Type; + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + Create(cx, nts.ElementType, this, nt.TypeArguments[0]); + return; + case SyntaxKind.TupleType: + var tts = (TupleTypeSyntax)Syntax; + var tt = (TupleType)Type; + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + tts.Elements.Zip(tt.TupleElements, (s, t) => Create(cx, s.Type, this, t.Type)).Enumerate(); + return; + case SyntaxKind.PointerType: + var pts = (PointerTypeSyntax)Syntax; + var pt = (PointerType)Type; + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + Create(cx, pts.ElementType, this, pt.PointedAtType); + return; + case SyntaxKind.GenericName: + var gns = (GenericNameSyntax)Syntax; + Emit(Loc ?? gns.Identifier.GetLocation(), Parent, Type); + gns.TypeArgumentList.Arguments.Zip(Type.TypeMentions, (s, t) => Create(cx, s, this, t)).Enumerate(); + return; + case SyntaxKind.QualifiedName: + if (Type.ContainingType == null) + { + // namespace qualifier + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + } + else + { + // Type qualifier + var qns = (QualifiedNameSyntax)Syntax; + var right = Create(cx, qns.Right, Parent, Type); + Create(cx, qns.Left, right, Type.ContainingType); + } + return; + default: + Emit(Loc ?? Syntax.GetLocation(), Parent, Type); + return; + } + } + + void Emit(Microsoft.CodeAnalysis.Location loc, IEntity parent, Type type) + { + cx.Emit(Tuples.type_mention(this, type.TypeRef, parent)); + cx.Emit(Tuples.type_mention_location(this, cx.Create(loc))); + } + + public static TypeMention Create(Context cx, TypeSyntax syntax, IEntity parent, Type type, Microsoft.CodeAnalysis.Location loc = null) + { + var ret = new TypeMention(cx, syntax, parent, type, loc); + cx.Try(syntax, null, () => ret.Populate()); + return ret; + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs new file mode 100644 index 000000000000..e791c4b27db5 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs @@ -0,0 +1,53 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class ArrayType : Type + { + ArrayType(Context cx, IArrayTypeSymbol init) + : base(cx, init) + { + element = Create(cx, symbol.ElementType); + } + + readonly Type element; + + public int Rank => symbol.Rank; + + public override Type ElementType => element; + + public override int Dimension => 1 + element.Dimension; + + // All array types are extracted because they won't + // be extracted in their defining assembly. + public override bool NeedsPopulation => true; + + public override void Populate() + { + Context.Emit(Tuples.array_element_type(this, Dimension, Rank, element.TypeRef)); + ExtractType(); + } + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(element); + symbol.BuildArraySuffix(tb); + tb.Append(";type"); + }); + } + } + + public static ArrayType Create(Context cx, IArrayTypeSymbol symbol) => ArrayTypeFactory.Instance.CreateEntity(cx, symbol); + + class ArrayTypeFactory : ICachedEntityFactory + { + public static readonly ArrayTypeFactory Instance = new ArrayTypeFactory(); + + public ArrayType Create(Context cx, IArrayTypeSymbol init) => new ArrayType(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs new file mode 100644 index 000000000000..8c1f95bd4237 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs @@ -0,0 +1,35 @@ +using Microsoft.CodeAnalysis; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class DynamicType : Type + { + DynamicType(Context cx, IDynamicTypeSymbol init) + : base(cx, init) { } + + public static DynamicType Create(Context cx, IDynamicTypeSymbol type) => DynamicTypeFactory.Instance.CreateEntity(cx, type); + + public override Microsoft.CodeAnalysis.Location ReportingLocation => Context.Compilation.ObjectType.Locations.FirstOrDefault(); + + public override void Populate() + { + Context.Emit(Tuples.types(this, Kinds.TypeKind.DYNAMIC, "dynamic")); + Context.Emit(Tuples.type_location(this, Location)); + + Context.Emit(Tuples.has_modifiers(this, Modifier.Create(Context, "public"))); + Context.Emit(Tuples.parent_namespace(this, Namespace.Create(Context, Context.Compilation.GlobalNamespace))); + } + + static readonly Key id = new Key("dynamic;type"); + + public override IId Id => id; + + class DynamicTypeFactory : ICachedEntityFactory + { + public static readonly DynamicTypeFactory Instance = new DynamicTypeFactory(); + + public DynamicType Create(Context cx, IDynamicTypeSymbol init) => new DynamicType(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs new file mode 100644 index 000000000000..a5dba66c9342 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs @@ -0,0 +1,204 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class NamedType : Type + { + NamedType(Context cx, INamedTypeSymbol init) + : base(cx, init) + { + typeArgumentsLazy = new Lazy(() => symbol.TypeArguments.Select(t => Create(cx, t)).ToArray()); + } + + public static NamedType Create(Context cx, INamedTypeSymbol type) => NamedTypeFactory.Instance.CreateEntity(cx, type); + + public override bool NeedsPopulation => base.NeedsPopulation || symbol.TypeKind == TypeKind.Error; + + public override void Populate() + { + if (symbol.TypeKind == TypeKind.Error) + { + Context.Extractor.MissingType(symbol.ToString()); + return; + } + + Context.Emit(Tuples.typeref_type((NamedTypeRef)TypeRef, this)); + + if (symbol.IsGenericType) + { + if (symbol.IsBoundNullable()) + { + // An instance of Nullable + Context.Emit(Tuples.nullable_underlying_type(this, Create(Context, symbol.TypeArguments[0]).TypeRef)); + } + else if (symbol.IsReallyUnbound()) + { + Context.Emit(Tuples.is_generic(this)); + + for (int i = 0; i < symbol.TypeParameters.Length; ++i) + { + TypeParameter.Create(Context, symbol.TypeParameters[i]); + var param = symbol.TypeParameters[i]; + var typeParameter = TypeParameter.Create(Context, param); + Context.Emit(Tuples.type_parameters(typeParameter, i, this)); + } + } + else + { + Context.Emit(Tuples.is_constructed(this)); + Context.Emit(Tuples.constructed_generic(this, Type.Create(Context, symbol.ConstructedFrom).TypeRef)); + + for (int i = 0; i < symbol.TypeArguments.Length; ++i) + { + Context.Emit(Tuples.type_arguments(TypeArguments[i].TypeRef, i, this)); + } + } + } + + ExtractType(); + + if (symbol.EnumUnderlyingType != null) + { + Context.Emit(Tuples.enum_underlying_type(this, Type.Create(Context, symbol.EnumUnderlyingType).TypeRef)); + } + + // Class location + if (!symbol.IsGenericType || symbol.IsReallyUnbound()) + { + foreach (var l in Locations) + Context.Emit(Tuples.type_location(this, l)); + } + } + + readonly Lazy typeArgumentsLazy; + public Type[] TypeArguments => typeArgumentsLazy.Value; + + public override IEnumerable TypeMentions => TypeArguments; + + public override IEnumerable Locations + { + get + { + foreach (var l in GetLocations(symbol)) + yield return Context.Create(l); + + if (Context.Extractor.OutputPath != null && symbol.DeclaringSyntaxReferences.Any()) + yield return Assembly.CreateOutputAssembly(Context); + } + } + + static IEnumerable GetLocations(INamedTypeSymbol type) + { + return type.Locations. + Where(l => l.IsInMetadata). + Concat( + type. + DeclaringSyntaxReferences. + Select(loc => loc.GetSyntax()). + OfType(). + Select(l => l.FixedLocation()) + ); + } + + public override Microsoft.CodeAnalysis.Location ReportingLocation => GetLocations(symbol).FirstOrDefault(); + + public override IId Id + { + get + { + return new Key(tb => + { + // All syntactic sub terms (types) are referenced by key in the ID + symbol.BuildTypeId(Context, tb, (cx0, tb0, sub) => tb0.Append(Create(cx0, sub))); + tb.Append(";type"); + }); + } + } + + /// + /// Returns the element type in an Enumerable/IEnumerable + /// + /// Extraction context. + /// The enumerable type. + /// The element type, or null. + static ITypeSymbol GetElementType(Context cx, INamedTypeSymbol type) + { + return GetEnumerableType(cx, type) ?? + type.AllInterfaces. + Where(i => i.OriginalDefinition.SpecialType == SpecialType.System_Collections_Generic_IEnumerable_T). + Concat(type.AllInterfaces.Where(i => i.SpecialType == SpecialType.System_Collections_IEnumerable)). + Select(i => GetEnumerableType(cx, i)). + FirstOrDefault(); + } + + static ITypeSymbol GetEnumerableType(Context cx, INamedTypeSymbol type) + { + return type.SpecialType == SpecialType.System_Collections_IEnumerable ? + cx.Compilation.ObjectType : + type.OriginalDefinition.SpecialType == SpecialType.System_Collections_Generic_IEnumerable_T ? + type.TypeArguments.First() : + null; + } + + public override Type ElementType + { + get + { + var elementType = GetElementType(Context, symbol); + return elementType == null ? null : Create(Context, elementType); + } + } + + class NamedTypeFactory : ICachedEntityFactory + { + public static readonly NamedTypeFactory Instance = new NamedTypeFactory(); + + public NamedType Create(Context cx, INamedTypeSymbol init) => new NamedType(cx, init); + } + + public override Type TypeRef => NamedTypeRef.Create(Context, symbol); + } + + class NamedTypeRef : Type + { + readonly Type referencedType; + + public NamedTypeRef(Context cx, INamedTypeSymbol symbol) : base(cx, symbol) + { + referencedType = Type.Create(cx, symbol); + } + + public static NamedTypeRef Create(Context cx, INamedTypeSymbol type) => NamedTypeRefFactory.Instance.CreateEntity(cx, type); + + class NamedTypeRefFactory : ICachedEntityFactory + { + public static readonly NamedTypeRefFactory Instance = new NamedTypeRefFactory(); + + public NamedTypeRef Create(Context cx, INamedTypeSymbol init) => new NamedTypeRef(cx, init); + } + + public override bool NeedsPopulation => true; + + public override IId Id + { + get + { + return new Key(tb => + { + tb.Append(referencedType).Append(";typeref"); + }); + } + } + + public override void Populate() + { + Context.Emit(Tuples.typerefs(this, symbol.Name)); + } + }; +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs new file mode 100644 index 000000000000..55833879224d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs @@ -0,0 +1,35 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class NullType : Type + { + NullType(Context cx) + : base(cx, null) { } + + public override void Populate() + { + Context.Emit(Tuples.types(this, Kinds.TypeKind.NULL, "null")); + } + + public override IId Id => new Key(";type"); + + public override bool NeedsPopulation => true; + + public override int GetHashCode() => 987354; + + public override bool Equals(object obj) + { + return obj != null && obj.GetType() == typeof(NullType); + } + + public static NullType Create(Context cx) => NullTypeFactory.Instance.CreateEntity(cx, null); + + class NullTypeFactory : ICachedEntityFactory + { + public static readonly NullTypeFactory Instance = new NullTypeFactory(); + + public NullType Create(Context cx, ITypeSymbol init) => new NullType(cx); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs new file mode 100644 index 000000000000..3f722b05e6f6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs @@ -0,0 +1,36 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp.Entities +{ + class PointerType : Type + { + PointerType(Context cx, IPointerTypeSymbol init) + : base(cx, init) + { + PointedAtType = Create(cx, symbol.PointedAtType); + } + + public override IId Id => new Key(PointedAtType, "*;type"); + + // All pointer types are extracted because they won't + // be extracted in their defining assembly. + public override bool NeedsPopulation => true; + + public override void Populate() + { + Context.Emit(Tuples.pointer_referent_type(this, PointedAtType.TypeRef)); + ExtractType(); + } + + public Type PointedAtType { get; private set; } + + public static PointerType Create(Context cx, IPointerTypeSymbol symbol) => PointerTypeFactory.Instance.CreateEntity(cx, symbol); + + class PointerTypeFactory : ICachedEntityFactory + { + public static readonly PointerTypeFactory Instance = new PointerTypeFactory(); + + public PointerType Create(Context cx, IPointerTypeSymbol init) => new PointerType(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs new file mode 100644 index 000000000000..b28846123e68 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs @@ -0,0 +1,68 @@ +using Microsoft.CodeAnalysis; +using Semmle.Extraction.Entities; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + /// + /// A tuple type, which is a C# type but not a .Net type. + /// Tuples have the underlying type System.ValueTuple. + /// + class TupleType : Type + { + public static TupleType Create(Context cx, INamedTypeSymbol type) => TupleTypeFactory.Instance.CreateEntity(cx, type); + + class TupleTypeFactory : ICachedEntityFactory + { + public static readonly TupleTypeFactory Instance = new TupleTypeFactory(); + + public TupleType Create(Context cx, INamedTypeSymbol init) => new TupleType(cx, init); + } + + TupleType(Context cx, INamedTypeSymbol init) : base(cx, init) + { + tupleElementsLazy = new Lazy(() => symbol.TupleElements.Select(t => Field.Create(cx, t)).ToArray()); + } + + // All tuple types are "local types" + public override bool NeedsPopulation => true; + + public override IId Id + { + get + { + return new Key(tb => + { + symbol.BuildTypeId(Context, tb, (cx0, tb0, sub) => tb0.Append(Create(cx0, sub))); + tb.Append(";tuple"); + }); + } + } + + public override void Populate() + { + ExtractType(); + ExtractGenerics(); + + var underlyingType = NamedType.Create(Context, symbol.TupleUnderlyingType); + Context.Emit(Tuples.tuple_underlying_type(this, underlyingType)); + + int index = 0; + foreach (var element in TupleElements) + Context.Emit(Tuples.tuple_element(this, index++, element)); + + // Note: symbol.Locations seems to be very inconsistent + // about what locations are available for a tuple type. + // Sometimes it's the source code, and sometimes it's empty. + foreach (var l in symbol.Locations) + Context.Emit(Tuples.type_location(this, Context.Create(l))); + } + + readonly Lazy tupleElementsLazy; + public Field[] TupleElements => tupleElementsLazy.Value; + + public override IEnumerable TypeMentions => TupleElements.Select(e => e.Type); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs new file mode 100644 index 000000000000..ac1a7e7f10eb --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs @@ -0,0 +1,327 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Util; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + public abstract class Type : CachedSymbol + { + public Type(Context cx, ITypeSymbol init) + : base(cx, init) { } + + public virtual Type ElementType => null; + + public override bool NeedsPopulation => + base.NeedsPopulation || symbol.TypeKind == TypeKind.Dynamic || symbol.TypeKind == TypeKind.TypeParameter; + + public static bool ConstructedOrParentIsConstructed(INamedTypeSymbol symbol) + { + return !Equals(symbol, symbol.OriginalDefinition) || + symbol.ContainingType != null && ConstructedOrParentIsConstructed(symbol.ContainingType); + } + + static Kinds.TypeKind GetClassType(Context cx, ITypeSymbol t) + { + switch (t.SpecialType) + { + case SpecialType.System_Int32: return Kinds.TypeKind.INT; + case SpecialType.System_UInt32: return Kinds.TypeKind.UINT; + case SpecialType.System_Int16: return Kinds.TypeKind.SHORT; + case SpecialType.System_UInt16: return Kinds.TypeKind.USHORT; + case SpecialType.System_UInt64: return Kinds.TypeKind.ULONG; + case SpecialType.System_Int64: return Kinds.TypeKind.LONG; + case SpecialType.System_Void: return Kinds.TypeKind.VOID; + case SpecialType.System_Double: return Kinds.TypeKind.DOUBLE; + case SpecialType.System_Byte: return Kinds.TypeKind.BYTE; + case SpecialType.System_SByte: return Kinds.TypeKind.SBYTE; + case SpecialType.System_Boolean: return Kinds.TypeKind.BOOL; + case SpecialType.System_Char: return Kinds.TypeKind.CHAR; + case SpecialType.System_Decimal: return Kinds.TypeKind.DECIMAL; + case SpecialType.System_Single: return Kinds.TypeKind.FLOAT; + case SpecialType.System_IntPtr: return Kinds.TypeKind.INT_PTR; + default: + if (t.IsBoundNullable()) return Kinds.TypeKind.NULLABLE; + switch (t.TypeKind) + { + case TypeKind.Class: return Kinds.TypeKind.CLASS; + case TypeKind.Struct: + return ((INamedTypeSymbol)t).IsTupleType ? Kinds.TypeKind.TUPLE : Kinds.TypeKind.STRUCT; + case TypeKind.Interface: return Kinds.TypeKind.INTERFACE; + case TypeKind.Array: return Kinds.TypeKind.ARRAY; + case TypeKind.Enum: return Kinds.TypeKind.ENUM; + case TypeKind.Delegate: return Kinds.TypeKind.DELEGATE; + case TypeKind.Pointer: return Kinds.TypeKind.POINTER; + default: + cx.ModelError(t, "Unhandled type kind '{0}'", t.TypeKind); + return Kinds.TypeKind.UNKNOWN; + } + } + } + + class DisplayNameTrapBuilder : ITrapBuilder + { + public readonly List Fragments = new List(); + + public ITrapBuilder Append(object arg) + { + Fragments.Add(arg.ToString()); + return this; + } + + public ITrapBuilder Append(string arg) + { + Fragments.Add(arg); + return this; + } + + public ITrapBuilder AppendLine() + { + throw new NotImplementedException(); + } + } + + protected void ExtractType() + { + ExtractAttributes(); + + var tb = new DisplayNameTrapBuilder(); + symbol.BuildDisplayName(Context, tb); + Context.Emit(Tuples.types(this, GetClassType(Context, symbol), tb.Fragments.ToArray())); + + // Visit base types + var baseTypes = new List(); + if (symbol.BaseType != null) + { + Type baseKey = Create(Context, symbol.BaseType); + Context.Emit(Tuples.extend(this, baseKey.TypeRef)); + if (symbol.TypeKind != TypeKind.Struct) + baseTypes.Add(baseKey); + } + + if (base.symbol.TypeKind == TypeKind.Interface) + { + Context.Emit(Tuples.extend(this, Create(Context, Context.Compilation.ObjectType))); + } + + if (!(base.symbol is IArrayTypeSymbol)) + { + foreach (var t in base.symbol.Interfaces.Select(i=>Create(Context, i))) + { + Context.Emit(Tuples.implement(this, t.TypeRef)); + baseTypes.Add(t); + } + } + + var containingType = ContainingType; + if (containingType != null && symbol.Kind != SymbolKind.TypeParameter) + { + Type originalDefinition = symbol.TypeKind == TypeKind.Error ? this : Create(Context, symbol.OriginalDefinition); + Context.Emit(Tuples.nested_types(this, containingType, originalDefinition)); + } + else if (symbol.ContainingNamespace != null) + { + Context.Emit(Tuples.parent_namespace(this, Namespace.Create(Context, symbol.ContainingNamespace))); + } + + if (symbol is IArrayTypeSymbol) + { + // They are in the namespace of the original object + ITypeSymbol elementType = ((IArrayTypeSymbol)symbol).ElementType; + INamespaceSymbol ns = elementType.TypeKind == TypeKind.TypeParameter ? Context.Compilation.GlobalNamespace : elementType.ContainingNamespace; + if (ns != null) + Context.Emit(Tuples.parent_namespace(this, Namespace.Create(Context, ns))); + } + + if (symbol is IPointerTypeSymbol) + { + ITypeSymbol elementType = ((IPointerTypeSymbol)symbol).PointedAtType; + INamespaceSymbol ns = elementType.TypeKind == TypeKind.TypeParameter ? Context.Compilation.GlobalNamespace : elementType.ContainingNamespace; + + if (ns != null) + Context.Emit(Tuples.parent_namespace(this, Namespace.Create(Context, ns))); + } + + if (symbol.BaseType != null && symbol.BaseType.SpecialType == SpecialType.System_MulticastDelegate) + { + // This is a delegate. + // The method "Invoke" has the return type. + var invokeMethod = ((INamedTypeSymbol)symbol).DelegateInvokeMethod; + + // Copy the parameters from the "Invoke" method to the delegate type + for (var i = 0; i < invokeMethod.Parameters.Length; ++i) + { + var param = invokeMethod.Parameters[i]; + var originalParam = invokeMethod.OriginalDefinition.Parameters[i]; + var originalParamEntity = Equals(param, originalParam) ? null : + DelegateTypeParameter.Create(Context, originalParam, Create(Context, ((INamedTypeSymbol)symbol).ConstructedFrom)); + DelegateTypeParameter.Create(Context, param, this, originalParamEntity); + } + + var returnKey = Create(Context, invokeMethod.ReturnType); + Context.Emit(Tuples.delegate_return_type(this, returnKey.TypeRef)); + if (invokeMethod.ReturnsByRef) + Context.Emit(Tuples.ref_returns(this)); + if (invokeMethod.ReturnsByRefReadonly) + Context.Emit(Tuples.ref_readonly_returns(this)); + } + + Modifier.ExtractModifiers(Context, this, symbol); + + if (IsSourceDeclaration && symbol.FromSource()) + { + var declSyntaxReferences = symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()).ToArray(); + + var baseLists = declSyntaxReferences.OfType().Select(c => c.BaseList); + baseLists = baseLists.Concat(declSyntaxReferences.OfType().Select(c => c.BaseList)); + baseLists = baseLists.Concat(declSyntaxReferences.OfType().Select(c => c.BaseList)); + + baseLists. + Where(bl => bl != null). + SelectMany(bl => bl.Types). + Zip(baseTypes.Where(bt => bt.symbol.SpecialType != SpecialType.System_Object), + (s, t) => TypeMention.Create(Context, s.Type, this, t)). + Enumerate(); + } + } + + /// + /// Called to extract all members and nested types. + /// This is called on each member of a namespace, + /// in either source code or an assembly. + /// + public void ExtractRecursive() + { + foreach (var l in symbol.DeclaringSyntaxReferences.Select(s => s.GetSyntax().GetLocation())) + { + Context.BindComments(this, l); + } + + foreach (var member in symbol.GetMembers()) + { + switch (member.Kind) + { + case SymbolKind.NamedType: + Create(Context, (ITypeSymbol)member).ExtractRecursive(); + break; + default: + Context.CreateEntity(member); + break; + } + } + } + + /// + /// Extracts all members and nested types of this type. + /// + public void ExtractGenerics() + { + if (symbol == null || !NeedsPopulation || !Context.ExtractGenerics(this)) + return; + + var members = new List(); + + foreach (var member in symbol.GetMembers()) + members.Add(member); + foreach (var member in symbol.GetTypeMembers()) + members.Add(member); + + // Mono extractor puts all BASE interface members as members of the current interface. + + if (symbol.TypeKind == TypeKind.Interface) + { + foreach (var baseInterface in symbol.Interfaces) + { + foreach (var member in baseInterface.GetMembers()) + members.Add(member); + foreach (var member in baseInterface.GetTypeMembers()) + members.Add(member); + } + } + + foreach (var member in members) + { + Context.CreateEntity(member); + } + + if (symbol.BaseType != null) + Create(Context, symbol.BaseType).ExtractGenerics(); + + foreach (var i in symbol.Interfaces) + { + Create(Context, i).ExtractGenerics(); + } + } + + public void ExtractRecursive(IEntity parent) + { + if (symbol.ContainingSymbol.Kind == SymbolKind.Namespace && !symbol.ContainingNamespace.IsGlobalNamespace) + { + Context.Emit(Tuples.parent_namespace_declaration(this, (NamespaceDeclaration)parent)); + } + + ExtractRecursive(); + } + + public static Type Create(Context cx, ITypeSymbol type) + { + type = cx.DisambiguateType(type); + const bool errorTypeIsNull = false; + return type == null || (errorTypeIsNull && type.TypeKind == TypeKind.Error) ? + NullType.Create(cx) : (Type)cx.CreateEntity(type); + } + + public virtual int Dimension => 0; + + public static bool IsDelegate(ITypeSymbol symbol) => + symbol != null && symbol.TypeKind == TypeKind.Delegate; + + /// + /// A copy of a delegate "Invoke" method parameter used for the delgate + /// type. + /// + class DelegateTypeParameter : Parameter + { + DelegateTypeParameter(Context cx, IParameterSymbol init, IEntity parent, Parameter original) + : base(cx, init, parent, original) { } + + new public static DelegateTypeParameter Create(Context cx, IParameterSymbol param, IEntity parent, Parameter original = null) => + DelegateTypeParameterFactory.Instance.CreateEntity(cx, (param, parent, original)); + + class DelegateTypeParameterFactory : ICachedEntityFactory<(IParameterSymbol, IEntity, Parameter), DelegateTypeParameter> + { + public static readonly DelegateTypeParameterFactory Instance = new DelegateTypeParameterFactory(); + + public DelegateTypeParameter Create(Context cx, (IParameterSymbol, IEntity, Parameter) init) => + new DelegateTypeParameter(cx, init.Item1, init.Item2, init.Item3); + } + } + + /// + /// Gets a reference to this type, if the type + /// is defined in another assembly. + /// + public virtual Type TypeRef => this; + + public virtual IEnumerable TypeMentions + { + get + { + yield break; + } + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } + + abstract class Type : Type where T : ITypeSymbol + { + public Type(Context cx, T init) + : base(cx, init) { } + + public new T symbol => (T)base.symbol; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs new file mode 100644 index 000000000000..1ee72f27c136 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs @@ -0,0 +1,131 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.Entities; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + enum Variance + { + None = 0, + Out = 1, + In = 2 + } + + class TypeParameter : Type + { + TypeParameter(Context cx, ITypeParameterSymbol init) + : base(cx, init) { } + + static readonly string valueTypeName = typeof(System.ValueType).ToString(); + + public override void Populate() + { + var constraints = new TypeParameterConstraints(Context); + Context.Emit(Tuples.type_parameter_constraints(constraints, this)); + + if (symbol.HasReferenceTypeConstraint) + Context.Emit(Tuples.general_type_parameter_constraints(constraints, 1)); + + if (symbol.HasValueTypeConstraint) + Context.Emit(Tuples.general_type_parameter_constraints(constraints, 2)); + + if (symbol.HasConstructorConstraint) + Context.Emit(Tuples.general_type_parameter_constraints(constraints, 3)); + + ITypeSymbol baseType = symbol.HasValueTypeConstraint ? + Context.Compilation.GetTypeByMetadataName(valueTypeName) : + Context.Compilation.ObjectType; + + var constraintTypes = new List(); + foreach (var abase in symbol.ConstraintTypes) + { + if (abase.TypeKind != TypeKind.Interface) + baseType = abase; + var t = Create(Context, abase); + Context.Emit(Tuples.specific_type_parameter_constraints(constraints, t.TypeRef)); + constraintTypes.Add(t); + } + + Context.Emit(Tuples.types(this, Semmle.Extraction.Kinds.TypeKind.TYPE_PARAMETER, symbol.Name)); + Context.Emit(Tuples.extend(this, Create(Context, baseType).TypeRef)); + + Namespace parentNs = Namespace.Create(Context, symbol.TypeParameterKind == TypeParameterKind.Method ? Context.Compilation.GlobalNamespace : symbol.ContainingNamespace); + Context.Emit(Tuples.parent_namespace(this, parentNs)); + + foreach (var l in symbol.Locations) + { + Context.Emit(Tuples.type_location(this, Context.Create(l))); + } + + if (this.IsSourceDeclaration) + { + var declSyntaxReferences = symbol.DeclaringSyntaxReferences.Select(d => d.GetSyntax()). + Select(s => s.Parent).Where(p => p != null).Select(p => p.Parent).ToArray(); + var clauses = declSyntaxReferences.OfType().SelectMany(m => m.ConstraintClauses); + clauses = clauses.Concat(declSyntaxReferences.OfType().SelectMany(c => c.ConstraintClauses)); + clauses = clauses.Concat(declSyntaxReferences.OfType().SelectMany(c => c.ConstraintClauses)); + clauses = clauses.Concat(declSyntaxReferences.OfType().SelectMany(c => c.ConstraintClauses)); + int i = 0; + foreach (var clause in clauses.Where(c => c.Name.ToString() == symbol.Name)) + { + TypeMention.Create(Context, clause.Name, this, this); + foreach (var constraint in clause.Constraints.OfType()) + TypeMention.Create(Context, constraint.Type, this, constraintTypes[i++]); + } + } + } + + static public TypeParameter Create(Context cx, ITypeParameterSymbol p) => + TypeParameterFactory.Instance.CreateEntity(cx, p); + + /// + /// The variance of this type parameter. + /// + public Variance Variance + { + get + { + switch (symbol.Variance) + { + case VarianceKind.None: return Variance.None; + case VarianceKind.Out: return Variance.Out; + case VarianceKind.In: return Variance.In; + default: + throw new InternalError("Unexpected VarianceKind {0}", symbol.Variance); + } + } + } + + public override IId Id + { + get + { + string kind; + IEntity containingEntity; + switch (symbol.TypeParameterKind) + { + case TypeParameterKind.Method: + kind = "methodtypeparameter"; + containingEntity = Method.Create(Context, (IMethodSymbol)symbol.ContainingSymbol); + break; + case TypeParameterKind.Type: + kind = "typeparameter"; + containingEntity = Create(Context, symbol.ContainingType); + break; + default: + throw new InternalError(symbol, "Unhandled type parameter kind {0}", symbol.TypeParameterKind); + } + return new Key(containingEntity, "_", symbol.Ordinal, ";", kind); + } + } + + class TypeParameterFactory : ICachedEntityFactory + { + public static readonly TypeParameterFactory Instance = new TypeParameterFactory(); + + public TypeParameter Create(Context cx, ITypeParameterSymbol init) => new TypeParameter(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameterConstraints.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameterConstraints.cs new file mode 100644 index 000000000000..425fdc1fe71a --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameterConstraints.cs @@ -0,0 +1,10 @@ +namespace Semmle.Extraction.CSharp.Entities +{ + class TypeParameterConstraints : FreshEntity + { + public TypeParameterConstraints(Context cx) + : base(cx) { } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs new file mode 100644 index 000000000000..2208eec201d4 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs @@ -0,0 +1,207 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + class UserOperator : Method + { + protected UserOperator(Context cx, IMethodSymbol init) + : base(cx, init) { } + + public override void Populate() + { + PopulateMethod(); + ExtractModifiers(); + + var returnType = Type.Create(Context, symbol.ReturnType); + Context.Emit(Tuples.operators(this, + symbol.Name, + OperatorSymbol(Context, symbol.Name), + ContainingType, + returnType.TypeRef, + (UserOperator)OriginalDefinition)); + + foreach (var l in Locations) + Context.Emit(Tuples.operator_location(this, l)); + + if (IsSourceDeclaration) + { + var declSyntaxReferences = symbol.DeclaringSyntaxReferences.Select(s => s.GetSyntax()).ToArray(); + foreach (var declaration in declSyntaxReferences.OfType()) + TypeMention.Create(Context, declaration.ReturnType, this, returnType); + foreach (var declaration in declSyntaxReferences.OfType()) + TypeMention.Create(Context, declaration.Type, this, returnType); + } + + ContainingType.ExtractGenerics(); + } + + public override bool NeedsPopulation => Context.Defines(symbol) || IsImplicitOperator(out _); + + public override Type ContainingType + { + get + { + IsImplicitOperator(out var containingType); + return Type.Create(Context, containingType); + } + } + + public override IId Id + { + get + { + return new Key(tb => + { + AddSignatureTypeToId(Context, tb, symbol, symbol.ReturnType); // Needed for op_explicit(), which differs only by return type. + tb.Append(" "); + BuildMethodId(this, tb); + }); + } + } + + /// + /// For some reason, some operators are missing from the Roslyn database of mscorlib. + /// This method returns true for such operators. + /// + /// The type containing this operator. + /// + bool IsImplicitOperator(out ITypeSymbol containingType) + { + containingType = symbol.ContainingType; + if (containingType != null) + { + var containingNamedType = containingType as INamedTypeSymbol; + return containingNamedType == null || !containingNamedType.MemberNames.Contains(symbol.Name); + } + + var pointerType = symbol.Parameters.Select(p => p.Type).OfType().FirstOrDefault(); + if (pointerType != null) + { + containingType = pointerType; + return true; + } + + Context.ModelError(symbol, "Unexpected implicit operator"); + return true; + } + + /// + /// Convert an operator method name in to a symbolic name. + /// A return value indicates whether the conversion succeeded. + /// + /// The method name. + /// The converted operator name. + public static bool OperatorSymbol(string methodName, out string operatorName) + { + var success = true; + switch (methodName) + { + case "op_LogicalNot": + operatorName = "!"; + break; + case "op_BitwiseAnd": + operatorName = "&"; + break; + case "op_Equality": + operatorName = "=="; + break; + case "op_Inequality": + operatorName = "!="; + break; + case "op_UnaryPlus": + case "op_Addition": + operatorName = "+"; + break; + case "op_UnaryNegation": + case "op_Subtraction": + operatorName = "-"; + break; + case "op_Multiply": + operatorName = "*"; + break; + case "op_Division": + operatorName = "/"; + break; + case "op_Modulus": + operatorName = "%"; + break; + case "op_GreaterThan": + operatorName = ">"; + break; + case "op_GreaterThanOrEqual": + operatorName = ">="; + break; + case "op_LessThan": + operatorName = "<"; + break; + case "op_LessThanOrEqual": + operatorName = "<="; + break; + case "op_Decrement": + operatorName = "--"; + break; + case "op_Increment": + operatorName = "++"; + break; + case "op_Implicit": + operatorName = "implicit conversion"; + break; + case "op_Explicit": + operatorName = "explicit conversion"; + break; + case "op_OnesComplement": + operatorName = "~"; + break; + case "op_RightShift": + operatorName = ">>"; + break; + case "op_LeftShift": + operatorName = "<<"; + break; + case "op_BitwiseOr": + operatorName = "|"; + break; + case "op_ExclusiveOr": + operatorName = "^"; + break; + case "op_True": + operatorName = "true"; + break; + case "op_False": + operatorName = "false"; + break; + default: + operatorName = methodName; + success = false; + break; + } + return success; + } + + /// + /// Converts a method name into a symbolic name. + /// Logs an error if the name is not found. + /// + /// Extractor context. + /// The method name. + /// The converted name. + public static string OperatorSymbol(Context cx, string methodName) + { + string result; + if (!OperatorSymbol(methodName, out result)) + cx.ModelError("Unhandled operator name in OperatorSymbol(): '{0}'", methodName); + return result; + } + + public new static UserOperator Create(Context cx, IMethodSymbol symbol) => UserOperatorFactory.Instance.CreateEntity(cx, symbol); + + class UserOperatorFactory : ICachedEntityFactory + { + public static readonly UserOperatorFactory Instance = new UserOperatorFactory(); + + public UserOperator Create(Context cx, IMethodSymbol init) => new UserOperator(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs new file mode 100644 index 000000000000..ea1258ab59a2 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs @@ -0,0 +1,61 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Populators; +using Semmle.Extraction.Entities; +using System.Collections.Generic; + +namespace Semmle.Extraction.CSharp.Entities +{ + class UsingDirective : FreshEntity + { + readonly UsingDirectiveSyntax node; + + public UsingDirective(Context cx, UsingDirectiveSyntax usingDirective, NamespaceDeclaration parent) + : base(cx) + { + node = usingDirective; + var info = cx.Model(node).GetSymbolInfo(usingDirective.Name); + + if (usingDirective.StaticKeyword.Kind() == SyntaxKind.None) + { + // A normal using + var namespaceSymbol = info.Symbol as INamespaceSymbol; + + if (namespaceSymbol == null) + { + cx.Extractor.MissingNamespace(usingDirective.Name.ToFullString()); + cx.ModelError(usingDirective, "Namespace not found"); + return; + } + else + { + var ns = Namespace.Create(cx, namespaceSymbol); + cx.Emit(Tuples.using_namespace_directives(this, ns)); + cx.Emit(Tuples.using_directive_location(this, cx.Create(ReportingLocation))); + } + } + else + { + // A "using static" + Type m = Type.Create(cx, (ITypeSymbol)info.Symbol); + cx.Emit(Tuples.using_static_directives(this, m.TypeRef)); + cx.Emit(Tuples.using_directive_location(this, cx.Create(ReportingLocation))); + } + + if (parent != null) + { + cx.Emit(Tuples.parent_namespace_declaration(this, parent)); + } + } + + public sealed override Microsoft.CodeAnalysis.Location ReportingLocation => node.GetLocation(); + + public IEnumerable GetTuples() + { + yield break; + } + + public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs new file mode 100644 index 000000000000..10a145d9e2f1 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs @@ -0,0 +1,407 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Text; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Semmle.Util; +using System.Text; +using System.Diagnostics; +using System.Threading.Tasks; +using Semmle.Util.Logging; +using System.Collections.Concurrent; + +namespace Semmle.Extraction.CSharp +{ + public static class Extractor + { + public enum ExitCode + { + Ok, // Everything worked perfectly + Errors, // Trap was generated but there were processing errors + Failed // Trap could not be generated + } + + class LogProgressMonitor : IProgressMonitor + { + readonly ILogger Logger; + + public LogProgressMonitor(ILogger logger) + { + Logger = logger; + } + + public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action) + { + Logger.Log(Severity.Info, " {0} -> {1} ({2})", source, output, + action == AnalysisAction.Extracted ? time.ToString() : action == AnalysisAction.Excluded ? "excluded" : "up to date"); + } + + public void MissingNamespace(string @namespace) { } + + public void MissingSummary(int types, int namespaces) { } + + public void MissingType(string type) { } + } + + /// + /// Command-line driver for the extractor. + /// + /// + /// + /// The extractor can be invoked in one of two ways: Either as an "analyser" passed in via the /a + /// option to csc.exe, or as a stand-alone executable. In this case, we need to faithfully + /// drive Roslyn in the way that csc.exe would. + /// + /// + /// Command line arguments as passed to csc.exe + /// + public static ExitCode Run(string[] args) + { + var commandLineArguments = Options.CreateWithEnvironment(args); + var fileLogger = new FileLogger(commandLineArguments.Verbosity, GetCSharpLogPath()); + var logger = commandLineArguments.Console + ? new CombinedLogger(new ConsoleLogger(commandLineArguments.Verbosity), fileLogger) + : (ILogger)fileLogger; + + if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !commandLineArguments.ClrTracer) + { + logger.Log(Severity.Info, "Skipping extraction since already extracted from the CLR tracer"); + return ExitCode.Ok; + } + + using (var analyser = new Analyser(new LogProgressMonitor(logger), logger)) + using (var references = new BlockingCollection()) + { + try + { + var compilerVersion = new CompilerVersion(commandLineArguments); + + bool preserveSymlinks = Environment.GetEnvironmentVariable("SEMMLE_PRESERVE_SYMLINKS") == "true"; + var canonicalPathCache = CanonicalPathCache.Create(logger, 1000, preserveSymlinks ? CanonicalPathCache.Symlinks.Preserve : CanonicalPathCache.Symlinks.Follow); + + if (compilerVersion.SkipExtraction) + { + logger.Log(Severity.Warning, " Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason); + return ExitCode.Ok; + } + + var argsWithResponse = AddDefaultResponse(compilerVersion.CscRsp, commandLineArguments.CompilerArguments); + + var cwd = Directory.GetCurrentDirectory(); + var compilerArguments = CSharpCommandLineParser.Default.Parse( + argsWithResponse, + cwd, + compilerVersion.FrameworkPath, + compilerVersion.AdditionalReferenceDirectories + ); + + if (compilerArguments == null) + { + var sb = new StringBuilder(); + sb.Append(" Failed to parse command line: ").AppendList(" ", args); + logger.Log(Severity.Error, sb.ToString()); + ++analyser.CompilationErrors; + return ExitCode.Failed; + } + + var referenceTasks = ResolveReferences(compilerArguments, analyser, canonicalPathCache, references); + + var syntaxTrees = new List(); + var syntaxTreeTasks = ReadSyntaxTrees( + compilerArguments.SourceFiles. + Select(src => canonicalPathCache.GetCanonicalPath(src.Path)), + analyser, + compilerArguments.ParseOptions, + compilerArguments.Encoding, + syntaxTrees); + + var sw = new Stopwatch(); + sw.Start(); + + Parallel.Invoke( + new ParallelOptions { MaxDegreeOfParallelism = commandLineArguments.Threads }, + referenceTasks.Interleave(syntaxTreeTasks).ToArray()); + + if (syntaxTrees.Count == 0) + { + logger.Log(Severity.Error, " No source files"); + ++analyser.CompilationErrors; + analyser.LogDiagnostics(); + return ExitCode.Failed; + } + + var compilation = CSharpCompilation.Create( + compilerArguments.CompilationName, + syntaxTrees, + references, + compilerArguments.CompilationOptions. + WithAssemblyIdentityComparer(DesktopAssemblyIdentityComparer.Default). + WithStrongNameProvider(new DesktopStrongNameProvider(compilerArguments.KeyFileSearchPaths)) + // csc.exe (CSharpCompiler.cs) also provides WithMetadataReferenceResolver, + // WithXmlReferenceResolver and + // WithSourceReferenceResolver. + // These would be needed if we hadn't explicitly provided the source/references + // already. + ); + + analyser.Initialize(compilerArguments, compilation, commandLineArguments); + analyser.AnalyseReferences(); + + foreach (var tree in compilation.SyntaxTrees) + { + analyser.AnalyseTree(tree); + } + + sw.Stop(); + logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed); + + sw.Restart(); + analyser.PerformExtraction(commandLineArguments.Threads); + sw.Stop(); + logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed); + + return analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors; + } + catch (Exception e) + { + logger.Log(Severity.Error, " Unhandled exception: {0}", e); + return ExitCode.Errors; + } + } + } + + internal static bool SuppressDefaultResponseFile(IEnumerable args) + { + return args.Any(arg => new[] { "/noconfig", "-noconfig" }.Contains(arg.ToLowerInvariant())); + } + + /// + /// Adds @csc.rsp to the argument list to mimic csc.exe. + /// + /// The full pathname of csc.rsp. + /// The other command line arguments. + /// Modified list of arguments. + static IEnumerable AddDefaultResponse(string responseFile, IEnumerable args) + { + return SuppressDefaultResponseFile(args) && File.Exists(responseFile) ? + args : + new[] { "@" + responseFile }.Concat(args); + } + + /// + /// Gets the complete list of locations to locate references. + /// + /// Command line arguments. + /// List of directories. + static IEnumerable FixedReferencePaths(Microsoft.CodeAnalysis.CommandLineArguments args) + { + // See https://msdn.microsoft.com/en-us/library/s5bac5fx.aspx + // on how csc resolves references. Basically, + // 1) Current working directory. This is the directory from which the compiler is invoked. + // 2) The common language runtime system directory. + // 3) Directories specified by / lib. + // 4) Directories specified by the LIB environment variable. + + yield return args.BaseDirectory; + + foreach (var r in args.ReferencePaths) + yield return r; + + var lib = System.Environment.GetEnvironmentVariable("LIB"); + if (lib != null) + yield return lib; + } + + static MetadataReference MakeReference(CommandLineReference reference, string path) + { + return MetadataReference.CreateFromFile(path).WithProperties(reference.Properties); + } + + /// + /// Construct tasks for resolving references (possibly in parallel). + /// + /// The resolved references will be added (thread-safely) to the supplied + /// list . + /// + static IEnumerable ResolveReferences(Microsoft.CodeAnalysis.CommandLineArguments args, Analyser analyser, CanonicalPathCache canonicalPathCache, BlockingCollection ret) + { + var referencePaths = new Lazy(() => FixedReferencePaths(args).ToArray()); + return args.MetadataReferences.Select(clref => () => + { + if (Path.IsPathRooted(clref.Reference)) + { + if (File.Exists(clref.Reference)) + { + var reference = MakeReference(clref, canonicalPathCache.GetCanonicalPath(clref.Reference)); + ret.Add(reference); + } + else + { + lock (analyser) + { + analyser.Logger.Log(Severity.Error, " Reference '{0}' does not exist", clref.Reference); + ++analyser.CompilationErrors; + } + } + } + else + { + bool referenceFound = false; + { + foreach (var composed in referencePaths.Value. + Select(path => Path.Combine(path, clref.Reference)). + Where(path => File.Exists(path)). + Select(path => canonicalPathCache.GetCanonicalPath(path))) + { + referenceFound = true; + var reference = MakeReference(clref, composed); + ret.Add(reference); + break; + } + if (!referenceFound) + { + lock (analyser) + { + analyser.Logger.Log(Severity.Error, " Unable to resolve reference '{0}'", clref.Reference); + ++analyser.CompilationErrors; + } + } + } + } + }); + } + + /// + /// Construct tasks for reading source code files (possibly in parallel). + /// + /// The constructed syntax trees will be added (thread-safely) to the supplied + /// list . + /// + static IEnumerable ReadSyntaxTrees(IEnumerable sources, Analyser analyser, CSharpParseOptions parseOptions, Encoding encoding, IList ret) + { + return sources.Select(path => () => + { + try + { + using (var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var st = CSharpSyntaxTree.ParseText(SourceText.From(file, encoding), parseOptions, path); + lock (ret) + ret.Add(st); + } + } + catch (IOException ex) + { + lock (analyser) + { + analyser.Logger.Log(Severity.Error, " Unable to open source file {0}: {1}", path, ex.Message); + ++analyser.CompilationErrors; + } + } + }); + } + + public static void ExtractStandalone( + IEnumerable sources, + IEnumerable referencePaths, + IProgressMonitor pm, + ILogger logger, + CommonOptions options) + { + using (var analyser = new Analyser(pm, logger)) + using (var references = new BlockingCollection()) + { + try + { + var referenceTasks = referencePaths.Select(path => () => + { + var reference = MetadataReference.CreateFromFile(path); + references.Add(reference); + }); + + var syntaxTrees = new List(); + var syntaxTreeTasks = ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees); + + var sw = new Stopwatch(); + sw.Start(); + + Parallel.Invoke( + new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, + referenceTasks.Interleave(syntaxTreeTasks).ToArray()); + + if (syntaxTrees.Count == 0) + { + analyser.Logger.Log(Severity.Error, " No source files"); + ++analyser.CompilationErrors; + } + + var compilation = CSharpCompilation.Create( + "csharp.dll", + syntaxTrees, + references + ); + + analyser.InitializeStandalone(compilation, options); + analyser.AnalyseReferences(); + + foreach (var tree in compilation.SyntaxTrees) + { + analyser.AnalyseTree(tree); + } + + sw.Stop(); + analyser.Logger.Log(Severity.Info, " Models constructed in {0}", sw.Elapsed); + + sw.Restart(); + analyser.PerformExtraction(options.Threads); + sw.Stop(); + analyser.Logger.Log(Severity.Info, " Extraction took {0}", sw.Elapsed); + + foreach (var type in analyser.MissingNamespaces) + { + pm.MissingNamespace(type); + } + + foreach (var type in analyser.MissingTypes) + { + pm.MissingType(type); + } + + pm.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count()); + } + catch (Exception e) + { + analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", e); + } + } + } + + /// + /// Gets the path to the `csharp.log` file written to by the C# extractor. + /// + public static string GetCSharpLogPath() + { + string snapshot = Environment.GetEnvironmentVariable("ODASA_SNAPSHOT"); + string buildErrorDir = Environment.GetEnvironmentVariable("ODASA_BUILD_ERROR_DIR"); + string traps = Environment.GetEnvironmentVariable("TRAP_FOLDER"); + string output = "csharp.log"; + if (!string.IsNullOrEmpty(snapshot)) + { + snapshot = Path.Combine(snapshot, "log"); + return Path.Combine(snapshot, output); + } + if (!string.IsNullOrEmpty(buildErrorDir)) + { + // Used by `qltest` + return Path.Combine(buildErrorDir, output); + } + if (!string.IsNullOrEmpty(traps)) + { + return Path.Combine(traps, output); + } + return output; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Kinds/ExprKind.cs b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/ExprKind.cs new file mode 100644 index 000000000000..daa05a494ae6 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/ExprKind.cs @@ -0,0 +1,110 @@ +namespace Semmle.Extraction.Kinds +{ + /// + /// This enum has been auto-generated from the C# DB scheme - do not edit. + /// Auto-generate command: `genkindenum.pl expr` + /// + public enum ExprKind + { + BOOL_LITERAL = 1, + CHAR_LITERAL = 2, + DECIMAL_LITERAL = 3, + INT_LITERAL = 4, + LONG_LITERAL = 5, + UINT_LITERAL = 6, + ULONG_LITERAL = 7, + FLOAT_LITERAL = 8, + DOUBLE_LITERAL = 9, + STRING_LITERAL = 10, + NULL_LITERAL = 11, + THIS_ACCESS = 12, + BASE_ACCESS = 13, + LOCAL_VARIABLE_ACCESS = 14, + PARAMETER_ACCESS = 15, + FIELD_ACCESS = 16, + PROPERTY_ACCESS = 17, + METHOD_ACCESS = 18, + EVENT_ACCESS = 19, + INDEXER_ACCESS = 20, + ARRAY_ACCESS = 21, + TYPE_ACCESS = 22, + TYPEOF = 23, + METHOD_INVOCATION = 24, + DELEGATE_INVOCATION = 25, + OPERATOR_INVOCATION = 26, + CAST = 27, + OBJECT_CREATION = 28, + EXPLICIT_DELEGATE_CREATION = 29, + IMPLICIT_DELEGATE_CREATION = 30, + ARRAY_CREATION = 31, + DEFAULT = 32, + PLUS = 33, + MINUS = 34, + BIT_NOT = 35, + LOG_NOT = 36, + POST_INCR = 37, + POST_DECR = 38, + PRE_INCR = 39, + PRE_DECR = 40, + MUL = 41, + DIV = 42, + REM = 43, + ADD = 44, + SUB = 45, + LSHIFT = 46, + RSHIFT = 47, + LT = 48, + GT = 49, + LE = 50, + GE = 51, + EQ = 52, + NE = 53, + BIT_AND = 54, + BIT_XOR = 55, + BIT_OR = 56, + LOG_AND = 57, + LOG_OR = 58, + IS = 59, + AS = 60, + NULL_COALESCING = 61, + CONDITIONAL = 62, + SIMPLE_ASSIGN = 63, + ASSIGN_ADD = 64, + ASSIGN_SUB = 65, + ASSIGN_MUL = 66, + ASSIGN_DIV = 67, + ASSIGN_REM = 68, + ASSIGN_AND = 69, + ASSIGN_XOR = 70, + ASSIGN_OR = 71, + ASSIGN_LSHIFT = 72, + ASSIGN_RSHIFT = 73, + OBJECT_INIT = 74, + COLLECTION_INIT = 75, + ARRAY_INIT = 76, + CHECKED = 77, + UNCHECKED = 78, + CONSTRUCTOR_INIT = 79, + ADD_EVENT = 80, + REMOVE_EVENT = 81, + PAR = 82, + LOCAL_VAR_DECL = 83, + LAMBDA = 84, + ANONYMOUS_METHOD = 85, + NAMESPACE = 86, + DYNAMIC_ELEMENT_ACCESS = 92, + DYNAMIC_MEMBER_ACCESS = 93, + POINTER_INDIRECTION = 100, + ADDRESS_OF = 101, + SIZEOF = 102, + AWAIT = 103, + NAMEOF = 104, + INTERPOLATED_STRING = 105, + UNKNOWN = 106, + THROW = 107, + TUPLE = 108, + LOCAL_FUNCTION_INVOCATION = 109, + REF = 110, + DISCARD = 111 + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Kinds/StmtKind.cs b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/StmtKind.cs new file mode 100644 index 000000000000..1ecf005a3606 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/StmtKind.cs @@ -0,0 +1,39 @@ +namespace Semmle.Extraction.Kinds +{ + /// + /// This enum has been auto-generated from the C# DB scheme - do not edit. + /// + public enum StmtKind + { + BLOCK = 1, + EXPR = 2, + IF = 3, + SWITCH = 4, + WHILE = 5, + DO = 6, + FOR = 7, + FOREACH = 8, + BREAK = 9, + CONTINUE = 10, + GOTO = 11, + GOTO_CASE = 12, + GOTO_DEFAULT = 13, + THROW = 14, + RETURN = 15, + YIELD = 16, + TRY = 17, + CHECKED = 18, + UNCHECKED = 19, + LOCK = 20, + USING = 21, + VAR_DECL = 22, + CONST_DECL = 23, + EMPTY = 24, + UNSAFE = 25, + FIXED = 26, + LABEL = 27, + CATCH = 28, + CASE = 29, + LOCAL_FUNCTION = 30 + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Kinds/TypeKind.cs b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/TypeKind.cs new file mode 100644 index 000000000000..6387193db3eb --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Kinds/TypeKind.cs @@ -0,0 +1,39 @@ +namespace Semmle.Extraction.Kinds +{ + /// + /// This enum has been auto-generated from the C# DB scheme - do not edit. + /// + public enum TypeKind + { + BOOL = 1, + CHAR = 2, + DECIMAL = 3, + SBYTE = 4, + SHORT = 5, + INT = 6, + LONG = 7, + BYTE = 8, + USHORT = 9, + UINT = 10, + ULONG = 11, + FLOAT = 12, + DOUBLE = 13, + ENUM = 14, + STRUCT = 15, + CLASS = 17, + INTERFACE = 19, + DELEGATE = 20, + NULL = 21, + TYPE_PARAMETER = 22, + POINTER = 23, + NULLABLE = 24, + ARRAY = 25, + VOID = 26, + INT_PTR = 27, + UINT_PTR = 28, + DYNAMIC = 29, + ARGLIST = 30, + UNKNOWN = 31, + TUPLE = 32 + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Options.cs b/csharp/extractor/Semmle.Extraction.CSharp/Options.cs new file mode 100644 index 000000000000..8fa7be08b675 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Options.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp +{ + public sealed class Options : CommonOptions + { + /// + /// The compiler exe, or null if unspecified. + /// + public string CompilerName; + + /// + /// Specified .Net Framework dir, or null if unspecified. + /// + public string Framework; + + /// + /// All other arguments passed to the compilation. + /// + public IList CompilerArguments = new List(); + + /// + /// Holds if the extractor was launched from the CLR tracer. + /// + public bool ClrTracer = false; + + public static Options CreateWithEnvironment(string[] arguments) + { + var options = new Options(); + var extractionOptions = Environment.GetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS") ?? + Environment.GetEnvironmentVariable("LGTM_INDEX_EXTRACTOR"); + + var argsList = new List(arguments); + + if (!string.IsNullOrEmpty(extractionOptions)) + argsList.AddRange(extractionOptions.Split(' ')); + + options.ParseArguments(argsList); + return options; + } + + public override bool handleArgument(string argument) + { + CompilerArguments.Add(argument); + return true; + } + + public override void invalidArgument(string argument) + { + // Unrecognised arguments are passed to the compiler. + CompilerArguments.Add(argument); + } + + public override bool handleOption(string key, string value) + { + switch (key) + { + case "compiler": + CompilerName = value; + return true; + case "framework": + Framework = value; + return true; + default: + return base.handleOption(key, value); + } + } + + public override bool handleFlag(string flag, bool value) + { + switch (flag) + { + case "clrtracer": + ClrTracer = value; + return true; + default: + return base.handleFlag(flag, value); + } + } + + private Options() + { + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/Ast.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Ast.cs new file mode 100644 index 000000000000..0ee8117ff8c9 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Ast.cs @@ -0,0 +1,69 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities; + +namespace Semmle.Extraction.CSharp.Populators +{ + class Ast : CSharpSyntaxVisitor + { + readonly Context cx; + readonly IExpressionParentEntity parent; + readonly int child; + + public Ast(Context cx, IExpressionParentEntity parent, int child) + { + this.cx = cx; + this.parent = parent; + this.child = child; + } + + public override void DefaultVisit(SyntaxNode node) + { + cx.ModelError(node, "Unhandled syntax node {0}", node.Kind()); + } + + public override void VisitPropertyDeclaration(PropertyDeclarationSyntax node) + { + ((Property)parent).VisitDeclaration(cx, node); + } + + + public override void VisitArgumentList(ArgumentListSyntax node) + { + int c = 0; + foreach (var m in node.Arguments) + { + cx.Extract(m, parent, c++); + } + } + + public override void VisitArgument(ArgumentSyntax node) + { + Expression.Create(cx, node.Expression, parent, child); + } + } + + public static class AstExtensions + { + public static void Extract(this Context cx, CSharpSyntaxNode node, IExpressionParentEntity parent, int child) + { + using (cx.StackGuard) + { + try + { + node.Accept(new Ast(cx, parent, child)); + } + catch (System.Exception e) + { + cx.ModelError(node, "Exception processing syntax node of type {0}: {1}", node.Kind(), e); + } + } + } + + public static void Extract(this Context cx, SyntaxNode node, IEntity parent, int child) + { + cx.Extract(((CSharpSyntaxNode)node), parent, child); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/Comments.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Comments.cs new file mode 100644 index 000000000000..8f6bb6f52063 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Comments.cs @@ -0,0 +1,33 @@ +using Semmle.Extraction.CommentProcessing; +using System; + +namespace Semmle.Extraction.CSharp.Populators +{ + /// + /// Populators for comments. + /// + public static class Comments + { + public static void ExtractComments(this Context cx, ICommentGenerator gen) + { + cx.Try(null, null, () => + { + gen.GenerateBindings((entity, duplicationGuardKey, block, binding) => + { + var commentBlock = Entities.CommentBlock.Create(cx, block); + Action a = () => + { + commentBlock.BindTo(entity, binding); + }; + // When the duplication guard key exists, it means that the entity is guarded against + // trap duplication (). + // We must therefore also guard comment construction. + if (duplicationGuardKey != null) + cx.WithDuplicationGuard(duplicationGuardKey, a); + else + a(); + }); + }); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/CompilationUnit.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/CompilationUnit.cs new file mode 100644 index 000000000000..dd87f483ddfe --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/CompilationUnit.cs @@ -0,0 +1,125 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Extraction.CSharp.Entities; +using Semmle.Extraction.Entities; +using Semmle.Util.Logging; + +namespace Semmle.Extraction.CSharp.Populators +{ + public class TypeContainerVisitor : CSharpSyntaxVisitor + { + protected readonly Context cx; + protected readonly IEntity parent; + + public TypeContainerVisitor(Context cx, IEntity parent) + { + this.cx = cx; + this.parent = parent; + } + + public override void DefaultVisit(SyntaxNode node) + { + throw new InternalError(node, "Unhandled top-level syntax node"); + } + + public override void VisitDelegateDeclaration(DelegateDeclarationSyntax node) + { + Entities.NamedType.Create(cx, cx.Model(node).GetDeclaredSymbol(node)).ExtractRecursive(parent); + } + + public override void VisitClassDeclaration(ClassDeclarationSyntax classDecl) + { + Entities.Type.Create(cx, cx.Model(classDecl).GetDeclaredSymbol(classDecl)).ExtractRecursive(parent); + } + + public override void VisitStructDeclaration(StructDeclarationSyntax node) + { + Entities.Type.Create(cx, cx.Model(node).GetDeclaredSymbol(node)).ExtractRecursive(parent); + } + + public override void VisitEnumDeclaration(EnumDeclarationSyntax node) + { + Entities.Type.Create(cx, cx.Model(node).GetDeclaredSymbol(node)).ExtractRecursive(parent); + } + + public override void VisitInterfaceDeclaration(InterfaceDeclarationSyntax node) + { + Entities.Type.Create(cx, cx.Model(node).GetDeclaredSymbol(node)).ExtractRecursive(parent); + } + + public override void VisitAttributeList(AttributeListSyntax node) + { + if (cx.Extractor.Standalone) return; + + var outputAssembly = Assembly.CreateOutputAssembly(cx); + foreach (var attribute in node.Attributes) + { + var ae = new Attribute(cx, attribute, outputAssembly); + cx.BindComments(ae, attribute.GetLocation()); + } + } + } + + class TypeOrNamespaceVisitor : TypeContainerVisitor + { + public TypeOrNamespaceVisitor(Context cx, IEntity parent) + : base(cx, parent) { } + + public override void VisitUsingDirective(UsingDirectiveSyntax usingDirective) + { + // Only deal with "using namespace" not "using X = Y" + if (usingDirective.Alias == null) + new UsingDirective(cx, usingDirective, (NamespaceDeclaration)parent); + } + + public override void VisitNamespaceDeclaration(NamespaceDeclarationSyntax node) + { + NamespaceDeclaration.Create(cx, node, (NamespaceDeclaration)parent); + } + } + + class CompilationUnitVisitor : TypeOrNamespaceVisitor + { + public CompilationUnitVisitor(Context cx) + : base(cx, null) { } + + public override void VisitExternAliasDirective(ExternAliasDirectiveSyntax node) + { + // This information is not yet extracted. + cx.Extractor.Message(new Message { severity = Severity.Info, message = "Ignoring extern alias directive" }); + } + + public override void VisitCompilationUnit(CompilationUnitSyntax compilationUnit) + { + foreach (var m in compilationUnit.ChildNodes()) + { + cx.Try(m, null, () => ((CSharpSyntaxNode)m).Accept(this)); + } + + // Gather comments: + foreach (SyntaxTrivia trivia in compilationUnit.DescendantTrivia(compilationUnit.Span)) + { + CommentLine.Extract(cx, trivia); + } + + foreach (var trivia in compilationUnit.GetLeadingTrivia()) + { + CommentLine.Extract(cx, trivia); + } + + foreach (var trivia in compilationUnit.GetTrailingTrivia()) + { + CommentLine.Extract(cx, trivia); + } + } + } + + public class CompilationUnit + { + public static void Extract(Context cx, SyntaxNode unit) + { + ((CSharpSyntaxNode)unit).Accept(new CompilationUnitVisitor(cx)); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/Locations.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Locations.cs new file mode 100644 index 000000000000..a9a4863a2e44 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Locations.cs @@ -0,0 +1,104 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Populators +{ + public static class LocationExtensions + { + /// + /// Manually extend a location. + /// + /// The location to extend. + /// The node to extend the location to. + /// Extended location. + public static Location ExtendLocation(this Location l1, SyntaxNode n2) + { + if (n2 == null) + { + return l1; + } + else + { + var l2 = n2.FixedLocation(); + int start = System.Math.Min(l1.SourceSpan.Start, l2.SourceSpan.Start); + int end = System.Math.Max(l1.SourceSpan.End, l2.SourceSpan.End); + return Location.Create(n2.SyntaxTree, new Microsoft.CodeAnalysis.Text.TextSpan(start, end - start)); + } + } + + /// + /// Adjust the location of some syntax nodes + /// to make them more suitable for displaying results. + /// Sometimes we do not wish to highlight the whole node, + /// so select a sub-node such as the name. + /// !! Refactor this into each entity. + /// + /// The syntax node. + /// The fixed location. + public static Location FixedLocation(this SyntaxNode node) + { + Location result; + switch (node.Kind()) + { + case SyntaxKind.EqualsValueClause: + result = ((EqualsValueClauseSyntax)node).Value.FixedLocation(); + break; + case SyntaxKind.OperatorDeclaration: + { + var decl = (OperatorDeclarationSyntax)node; + result = decl.OperatorKeyword.GetLocation().ExtendLocation(decl.ParameterList); + break; + } + case SyntaxKind.ConversionOperatorDeclaration: + { + var decl = (ConversionOperatorDeclarationSyntax)node; + result = decl.OperatorKeyword.GetLocation(); + break; + } + case SyntaxKind.DelegateDeclaration: + { + var decl = (DelegateDeclarationSyntax)node; + return decl.Identifier.GetLocation().ExtendLocation(decl.TypeParameterList); + } + case SyntaxKind.ClassDeclaration: + case SyntaxKind.StructDeclaration: + case SyntaxKind.InterfaceDeclaration: + { + var decl = (TypeDeclarationSyntax)node; + return decl.Identifier.GetLocation().ExtendLocation(decl.TypeParameterList); + } + case SyntaxKind.EnumDeclaration: + return ((EnumDeclarationSyntax)node).Identifier.GetLocation(); + case SyntaxKind.MethodDeclaration: + { + var decl = (MethodDeclarationSyntax)node; + return decl.Identifier.GetLocation().ExtendLocation(decl.TypeParameterList); + } + case SyntaxKind.ConstructorDeclaration: + { + var decl = (ConstructorDeclarationSyntax)node; + return decl.Identifier.GetLocation(); + } + case SyntaxKind.ParenthesizedExpression: + return ((ParenthesizedExpressionSyntax)node).Expression.FixedLocation(); + case SyntaxKind.CatchDeclaration: + return ((CatchDeclarationSyntax)node).Identifier.GetLocation(); + case SyntaxKind.LabeledStatement: + return ((LabeledStatementSyntax)node).Identifier.GetLocation(); + default: + result = node.GetLocation(); + break; + } + return result; + } + + public static Location GetSymbolLocation(this ISymbol symbol) + { + return symbol.DeclaringSyntaxReferences.Any() ? + symbol.DeclaringSyntaxReferences.First().GetSyntax().FixedLocation() : + symbol.Locations.First(); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/Methods.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Methods.cs new file mode 100644 index 000000000000..109d07f6f7d2 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Methods.cs @@ -0,0 +1,64 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp.Populators +{ + public static class MethodExtensions + { + class AstLineCounter : CSharpSyntaxVisitor + { + public override LineCounts DefaultVisit(SyntaxNode node) + { + string text = node.SyntaxTree.GetText().GetSubText(node.GetLocation().SourceSpan).ToString(); + return Semmle.Util.LineCounter.ComputeLineCounts(text); + } + + public override LineCounts VisitMethodDeclaration(MethodDeclarationSyntax method) + { + return Visit(method.Identifier, method.Body ?? (SyntaxNode)method.ExpressionBody); + } + + public LineCounts Visit(SyntaxToken identifier, SyntaxNode body) + { + int start = identifier.GetLocation().SourceSpan.Start; + int end = body.GetLocation().SourceSpan.End - 1; + + var textSpan = new Microsoft.CodeAnalysis.Text.TextSpan(start, end - start); + + string text = body.SyntaxTree.GetText().GetSubText(textSpan) + "\r\n"; + return Semmle.Util.LineCounter.ComputeLineCounts(text); + } + + public override LineCounts VisitConstructorDeclaration(ConstructorDeclarationSyntax method) + { + return Visit(method.Identifier, (SyntaxNode)method.Body ?? method.ExpressionBody); + } + + public override LineCounts VisitDestructorDeclaration(DestructorDeclarationSyntax method) + { + return Visit(method.Identifier, (SyntaxNode)method.Body ?? method.ExpressionBody); + } + + public override LineCounts VisitOperatorDeclaration(OperatorDeclarationSyntax node) + { + return Visit(node.OperatorToken, node.Body ?? (SyntaxNode)node.ExpressionBody); + } + } + + public static void NumberOfLines(this Context cx, ISymbol symbol, IEntity callable) + { + foreach (var decl in symbol.DeclaringSyntaxReferences) + { + cx.NumberOfLines((CSharpSyntaxNode)decl.GetSyntax(), callable); + } + } + + public static void NumberOfLines(this Context cx, CSharpSyntaxNode node, IEntity callable) + { + var lineCounts = node.Accept(new AstLineCounter()); + cx.Emit(Tuples.numlines(callable, lineCounts)); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/Symbols.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Symbols.cs new file mode 100644 index 000000000000..0a62e4beaf05 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/Symbols.cs @@ -0,0 +1,136 @@ +using System; +using System.Linq; +using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp.Entities; + +namespace Semmle.Extraction.CSharp.Populators +{ + class Symbols : SymbolVisitor + { + readonly Context cx; + + public Symbols(Context cx) + { + this.cx = cx; + } + + public override IEntity DefaultVisit(ISymbol symbol) => throw new InternalError(symbol, "Unhandled symbol '{0}' of kind '{1}'", symbol, symbol.Kind); + + public override IEntity VisitArrayType(IArrayTypeSymbol array) => ArrayType.Create(cx, array); + + public override IEntity VisitMethod(IMethodSymbol methodDecl) + { + return Method.Create(cx, methodDecl); + } + + public override IEntity VisitField(IFieldSymbol field) => Field.Create(cx, field); + + public override IEntity VisitNamedType(INamedTypeSymbol type) => + type.IsTupleType ? TupleType.Create(cx, type) : (IEntity)NamedType.Create(cx, type); + + public override IEntity VisitNamespace(INamespaceSymbol ns) => Namespace.Create(cx, ns); + + public override IEntity VisitParameter(IParameterSymbol param) => Parameter.GetAlreadyCreated(cx, param); + + public override IEntity VisitProperty(IPropertySymbol symbol) => Property.Create(cx, symbol); + + public override IEntity VisitEvent(IEventSymbol symbol) => Event.Create(cx, symbol); + + public override IEntity VisitTypeParameter(ITypeParameterSymbol param) => TypeParameter.Create(cx, param); + + public override IEntity VisitPointerType(IPointerTypeSymbol symbol) => PointerType.Create(cx, symbol); + + public override IEntity VisitDynamicType(IDynamicTypeSymbol symbol) => DynamicType.Create(cx, symbol); + } + + public static class SymbolsExtensions + { + public static IEntity CreateEntity(this Context cx, ISymbol symbol) + { + if (symbol == null) return null; + + using (cx.StackGuard) + { + try + { + return symbol.Accept(new Symbols(cx)); + } + catch (Exception e) + { + cx.ModelError(symbol, "Exception processing symbol '{2}' of type '{0}': {1}", symbol.Kind, e, symbol); + return null; + } + } + } + + /// + /// Tries to recover from an ErrorType. + /// + /// + /// Extraction context. + /// The type to disambiguate. + /// + public static ITypeSymbol DisambiguateType(this Context cx, ITypeSymbol type) + { + /* A type could not be determined. + * Sometimes this happens due to a missing reference, + * or sometimes because the same type is defined in multiple places. + * + * In the case that a symbol is multiply-defined, Roslyn tells you which + * symbols are candidates. It usually resolves to the same DB entity, + * so it's reasonably safe to just pick a candidate. + * + * The conservative option would be to resolve all error types as null. + */ + + var errorType = type as IErrorTypeSymbol; + + return errorType != null && errorType.CandidateSymbols.Any() ? + errorType.CandidateSymbols.First() as ITypeSymbol : + type; + } + + public static TypeInfo GetTypeInfo(this Context cx, Microsoft.CodeAnalysis.CSharp.CSharpSyntaxNode node) => + cx.Model(node).GetTypeInfo(node); + + public static SymbolInfo GetSymbolInfo(this Context cx, Microsoft.CodeAnalysis.CSharp.CSharpSyntaxNode node) => + cx.Model(node).GetSymbolInfo(node); + + /// + /// Gets the symbol for a particular syntax node. + /// Throws an exception if the symbol is not found. + /// + /// + /// + /// This gives a nicer message than a "null pointer exception", + /// and should be used where we require a symbol to be resolved. + /// + /// + /// The extraction context. + /// The syntax node. + /// The resolved symbol. + public static ISymbol GetSymbol(this Context cx, Microsoft.CodeAnalysis.CSharp.CSharpSyntaxNode node) + { + var info = GetSymbolInfo(cx, node); + if (info.Symbol == null) + { + throw new InternalError(node, "Could not resolve symbol"); + } + + return info.Symbol; + } + + /// + /// Determines the type of a node, or null + /// if the type could not be determined. + /// + /// Extractor context. + /// The node to determine. + /// The type symbol of the node, or null. + public static ITypeSymbol GetType(this Context cx, Microsoft.CodeAnalysis.CSharp.CSharpSyntaxNode node) + { + var info = GetTypeInfo(cx, node); + return cx.DisambiguateType(info.Type); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp/Properties/AssemblyInfo.cs new file mode 100644 index 000000000000..bb2ea1f4882d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.CSharp")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Semmle.Extraction.CSharp")] +[assembly: AssemblyCopyright("Copyright © 2015")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("09d7c55d-5ed3-4af8-9b9f-8f9342533ee9")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj b/csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj new file mode 100644 index 000000000000..9cf59fd7a288 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Semmle.Extraction.CSharp.csproj @@ -0,0 +1,25 @@ + + + + netcoreapp2.0 + Semmle.Extraction.CSharp + Semmle.Extraction.CSharp + false + true + + + + + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs b/csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs new file mode 100644 index 000000000000..0a2dacdec1c8 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs @@ -0,0 +1,371 @@ +using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp.Entities; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp +{ + static class SymbolExtensions + { + /// + /// Gets the name of this symbol. + /// + /// If the symbol implements an explicit interface, only the + /// name of the member being implemented is included, not the + /// explicit prefix. + /// + public static string GetName(this ISymbol symbol, bool useMetadataName = false) + { + var name = useMetadataName ? symbol.MetadataName : symbol.Name; + return symbol.CanBeReferencedByName ? name : name.Substring(symbol.Name.LastIndexOf('.') + 1); + } + + /// + /// Gets the source-level modifiers belonging to this symbol, if any. + /// + public static IEnumerable GetSourceLevelModifiers(this ISymbol symbol) + { + var methodModifiers = + symbol.DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + SelectMany(md => md.Modifiers); + var typeModifers = + symbol.DeclaringSyntaxReferences. + Select(r => r.GetSyntax()). + OfType(). + SelectMany(cd => cd.Modifiers); + return methodModifiers.Concat(typeModifers).Select(m => m.Text); + } + + /// + /// Holds if this type symbol contains a type parameter from the + /// declaring generic . + /// + public static bool ContainsTypeParameters(this ITypeSymbol type, Context cx, ISymbol declaringGeneric) + { + using (cx.StackGuard) + { + switch (type.TypeKind) + { + case TypeKind.Array: + var array = (IArrayTypeSymbol)type; + return array.ElementType.ContainsTypeParameters(cx, declaringGeneric); + case TypeKind.Class: + case TypeKind.Interface: + case TypeKind.Struct: + case TypeKind.Enum: + case TypeKind.Delegate: + case TypeKind.Error: + var named = (INamedTypeSymbol)type; + if (named.IsTupleType) + named = named.TupleUnderlyingType; + if (named.ContainingType != null && named.ContainingType.ContainsTypeParameters(cx, declaringGeneric)) + return true; + return named.TypeArguments.Any(arg => arg.ContainsTypeParameters(cx, declaringGeneric)); + case TypeKind.Pointer: + var ptr = (IPointerTypeSymbol)type; + return ptr.PointedAtType.ContainsTypeParameters(cx, declaringGeneric); + case TypeKind.TypeParameter: + var tp = (ITypeParameterSymbol)type; + var declaringGen = tp.TypeParameterKind == TypeParameterKind.Method ? tp.DeclaringMethod : (ISymbol)tp.DeclaringType; + return Equals(declaringGen, declaringGeneric); + default: + return false; + } + } + } + + /// + /// Constructs a unique string for this type symbol. + /// + /// The supplied action is applied to the + /// syntactic sub terms of this type (if any). + /// + /// The extraction context. + /// The trap builder used to store the result. + /// The action to apply to syntactic sub terms of this type. + public static void BuildTypeId(this ITypeSymbol type, Context cx, ITrapBuilder tb, Action subTermAction) + { + if (type.SpecialType != SpecialType.None) + { + /* + * Use the keyword ("int" etc) for the built-in types. + * This makes the IDs shorter and means that all built-in types map to + * the same entities (even when using multiple versions of mscorlib). + */ + tb.Append(type.ToDisplayString()); + return; + } + + using (cx.StackGuard) + { + switch (type.TypeKind) + { + case TypeKind.Array: + var array = (IArrayTypeSymbol)type; + subTermAction(cx, tb, array.ElementType); + array.BuildArraySuffix(tb); + return; + case TypeKind.Class: + case TypeKind.Interface: + case TypeKind.Struct: + case TypeKind.Enum: + case TypeKind.Delegate: + case TypeKind.Error: + var named = (INamedTypeSymbol)type; + named.BuildNamedTypeId(cx, tb, subTermAction); + return; + case TypeKind.Pointer: + var ptr = (IPointerTypeSymbol)type; + subTermAction(cx, tb, ptr.PointedAtType); + tb.Append("*"); + return; + case TypeKind.TypeParameter: + var tp = (ITypeParameterSymbol)type; + tb.Append(tp.Name); + return; + case TypeKind.Dynamic: + tb.Append("dynamic"); + return; + default: + throw new InternalError(type, "Unhandled type kind '{0}'", type.TypeKind); + } + } + } + + /// + /// Constructs an array suffix string for this array type symbol. + /// + /// The trap builder used to store the result. + public static void BuildArraySuffix(this IArrayTypeSymbol array, ITrapBuilder tb) + { + tb.Append("["); + for (int i = 0; i < array.Rank - 1; i++) + tb.Append(","); + tb.Append("]"); + } + + static void BuildNamedTypeId(this INamedTypeSymbol named, Context cx, ITrapBuilder tb, Action subTermAction) + { + if (named.IsTupleType) + { + tb.Append("("); + tb.BuildList(",", named.TupleElements, + (f, tb0) => + { + tb.Append(f.Name).Append(":"); + subTermAction(cx, tb0, f.Type); + } + ); + tb.Append(")"); + return; + } + + if (named.ContainingType != null) + { + subTermAction(cx, tb, named.ContainingType); + tb.Append("."); + } + else if (named.ContainingNamespace != null) + { + named.ContainingNamespace.BuildNamespace(cx, tb); + } + + if (named.IsAnonymousType) + named.BuildAnonymousName(cx, tb, subTermAction, true); + else if (named.TypeParameters.IsEmpty) + tb.Append(named.Name); + else if (IsReallyUnbound(named)) + tb.Append(named.Name).Append("`").Append(named.TypeParameters.Length); + else + { + subTermAction(cx, tb, named.ConstructedFrom); + tb.Append("<"); + tb.BuildList(",", named.TypeArguments, (ta, tb0) => subTermAction(cx, tb0, ta)); + tb.Append(">"); + } + } + + static void BuildNamespace(this INamespaceSymbol ns, Context cx, ITrapBuilder tb) + { + // Only include the assembly information in each type ID + // for normal extractions. This is because standalone extractions + // lack assembly information or may be ambiguous. + bool prependAssemblyToTypeId = !cx.Extractor.Standalone && ns.ContainingAssembly != null; + + if (prependAssemblyToTypeId) + { + // Note that we exclude the revision number as this has + // been observed to be unstable. + var assembly = ns.ContainingAssembly.Identity; + tb.Append(assembly.Name).Append("_"). + Append(assembly.Version.Major).Append("."). + Append(assembly.Version.Minor).Append("."). + Append(assembly.Version.Build).Append("::"); + } + + tb.Append(Namespace.Create(cx, ns)).Append("."); + } + + static void BuildAnonymousName(this ITypeSymbol type, Context cx, ITrapBuilder tb, Action subTermAction, bool includeParamName) + { + var buildParam = includeParamName + ? (prop, tb0) => + { + tb0.Append(prop.Name).Append(" "); + subTermAction(cx, tb0, prop.Type); + } + : (Action)((prop, tb0) => subTermAction(cx, tb0, prop.Type)); + int memberCount = type.GetMembers().OfType().Count(); + int hackTypeNumber = memberCount == 1 ? 1 : 0; + tb.Append("<>__AnonType"); + tb.Append(hackTypeNumber); + tb.Append("<"); + tb.BuildList(",", type.GetMembers().OfType(), buildParam); + tb.Append(">"); + } + + /// + /// Constructs a display name string for this type symbol. + /// + /// The trap builder used to store the result. + public static void BuildDisplayName(this ITypeSymbol type, Context cx, ITrapBuilder tb) + { + using (cx.StackGuard) + { + switch (type.TypeKind) + { + case TypeKind.Array: + var array = (IArrayTypeSymbol)type; + var elementType = array.ElementType; + if (elementType.MetadataName.IndexOf("`") >= 0) + { + tb.Append(elementType.Name); + return; + } + elementType.BuildDisplayName(cx, tb); + array.BuildArraySuffix(tb); + return; + case TypeKind.Class: + case TypeKind.Interface: + case TypeKind.Struct: + case TypeKind.Enum: + case TypeKind.Delegate: + case TypeKind.Error: + var named = (INamedTypeSymbol)type; + named.BuildNamedTypeDisplayName(cx, tb); + return; + case TypeKind.Pointer: + var ptr = (IPointerTypeSymbol)type; + ptr.PointedAtType.BuildDisplayName(cx, tb); + tb.Append("*"); + return; + case TypeKind.TypeParameter: + tb.Append(type.Name); + return; + case TypeKind.Dynamic: + tb.Append("dynamic"); + return; + default: + throw new InternalError(type, "Unhandled type kind '{0}'", type.TypeKind); + } + } + } + + public static void BuildNamedTypeDisplayName(this INamedTypeSymbol namedType, Context cx, ITrapBuilder tb) + { + if (namedType.IsTupleType) + { + tb.Append("("); + tb.BuildList(",", namedType.TupleElements.Select(f => f.Type), + (t, tb0) => t.BuildDisplayName(cx, tb0) + ); + + tb.Append(")"); + return; + } + + if (namedType.IsAnonymousType) + { + namedType.BuildAnonymousName(cx, tb, (cx0, tb0, sub) => sub.BuildDisplayName(cx0, tb0), false); + } + + tb.Append(namedType.Name); + if (namedType.IsGenericType && namedType.TypeKind != TypeKind.Error && namedType.TypeArguments.Any()) + { + tb.Append("<"); + tb.BuildList(",", namedType.TypeArguments, (p, tb0) => + { + if (IsReallyBound(namedType)) + p.BuildDisplayName(cx, tb0); + }); + tb.Append(">"); + } + } + + public static bool IsReallyUnbound(this INamedTypeSymbol type) => + Equals(type.ConstructedFrom, type) || type.IsUnboundGenericType; + + public static bool IsReallyBound(this INamedTypeSymbol type) => !IsReallyUnbound(type); + + /// + /// Holds if this type is of the form int? or + /// System.Nullable. + /// + public static bool IsBoundNullable(this ITypeSymbol type) => + type.SpecialType == SpecialType.None && type.OriginalDefinition.IsUnboundNullable(); + + /// + /// Holds if this type is System.Nullable. + /// + public static bool IsUnboundNullable(this ITypeSymbol type) => + type.SpecialType == SpecialType.System_Nullable_T; + + /// + /// Gets the parameters of a method or property. + /// + /// The list of parameters, or an empty list. + public static IEnumerable GetParameters(this ISymbol parameterizable) + { + if (parameterizable is IMethodSymbol) + return ((IMethodSymbol)parameterizable).Parameters; + + if (parameterizable is IPropertySymbol) + return ((IPropertySymbol)parameterizable).Parameters; + + return Enumerable.Empty(); + } + + /// + /// Holds if this symbol is defined in a source code file. + /// + public static bool FromSource(this ISymbol symbol) => symbol.Locations.Any(l => l.IsInSource); + + /// + /// Holds if this symbol is a source declaration. + /// + public static bool IsSourceDeclaration(this ISymbol symbol) => Equals(symbol, symbol.OriginalDefinition); + + /// + /// Holds if this method is a source declaration. + /// + public static bool IsSourceDeclaration(this IMethodSymbol method) => + IsSourceDeclaration((ISymbol)method) && Equals(method, method.ConstructedFrom) && method.ReducedFrom == null; + + /// + /// Holds if this parameter is a source declaration. + /// + public static bool IsSourceDeclaration(this IParameterSymbol parameter) + { + var method = parameter.ContainingSymbol as IMethodSymbol; + if (method != null) + return method.IsSourceDeclaration(); + var property = parameter.ContainingSymbol as IPropertySymbol; + if (property != null && property.IsIndexer) + return property.IsSourceDeclaration(); + return true; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs b/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs new file mode 100644 index 000000000000..2d118256a080 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs @@ -0,0 +1,215 @@ +using Semmle.Extraction.CommentProcessing; +using Semmle.Extraction.CSharp.Entities; +using Semmle.Extraction.Entities; +using Semmle.Extraction.Kinds; +using Semmle.Util; + +namespace Semmle.Extraction.CSharp +{ + /// + /// Methods for creating DB tuples. + /// + /// + /// + /// Notice how the parameters to the tuples are well typed. + /// In an idea world, each tuple would be its own type, as a typed entity. However + /// that seems to be overkill. + /// + internal static class Tuples + { + internal static Tuple accessor_location(Accessor accessorKey, Location location) => new Tuple("accessor_location", accessorKey, location); + + internal static Tuple accessors(Accessor accessorKey, int kind, string name, Property propKey, Accessor unboundAccessor) => new Tuple("accessors", accessorKey, kind, name, propKey, unboundAccessor); + + internal static Tuple array_element_type(ArrayType array, int dimension, int rank, Type elementType) => new Tuple("array_element_type", array, dimension, rank, elementType); + + internal static Tuple attributes(Attribute attribute, Type attributeType, IEntity entity) => new Tuple("attributes", attribute, attributeType, entity); + + internal static Tuple attribute_location(Attribute attribute, Location location) => new Tuple("attribute_location", attribute, location); + + internal static Tuple catch_type(Entities.Statements.Catch @catch, Type type, bool explicityCaught) => new Tuple("catch_type", @catch, type, explicityCaught ? 1 : 2); + + internal static Tuple commentblock(CommentBlock k) => new Tuple("commentblock", k); + + internal static Tuple commentblock_binding(CommentBlock commentBlock, Label entity, Binding binding) => new Tuple("commentblock_binding", commentBlock, entity, binding); + + internal static Tuple commentblock_child(CommentBlock commentBlock, CommentLine commentLine, int child) => new Tuple("commentblock_child", commentBlock, commentLine, child); + + internal static Tuple commentblock_location(CommentBlock k, Location l) => new Tuple("commentblock_location", k, l); + + internal static Tuple commentline(CommentLine commentLine, CommentType type, string text, string rawtext) => new Tuple("commentline", commentLine, type, text, rawtext); + + internal static Tuple commentline_location(CommentLine commentLine, Location location) => new Tuple("commentline_location", commentLine, location); + + internal static Tuple compiler_generated(IEntity entity) => new Tuple("compiler_generated", entity); + + internal static Tuple conditional_access(Expression access) => new Tuple("conditional_access", access); + + internal static Tuple constant_value(IEntity field, string value) => new Tuple("constant_value", field, value); + + internal static Tuple constructed_generic(IEntity constructedTypeOrMethod, IEntity unboundTypeOrMethod) => new Tuple("constructed_generic", constructedTypeOrMethod, unboundTypeOrMethod); + + internal static Tuple constructor_location(Constructor constructor, Location location) => new Tuple("constructor_location", constructor, location); + + internal static Tuple constructors(Constructor key, string name, Type definingType, Constructor originalDefinition) => new Tuple("constructors", key, name, definingType, originalDefinition); + + internal static Tuple delegate_return_type(Type delegateKey, Type returnType) => new Tuple("delegate_return_type", delegateKey, returnType); + + internal static Tuple destructor_location(Destructor destructor, Location location) => new Tuple("destructor_location", destructor, location); + + internal static Tuple destructors(Destructor destructor, string name, Type containingType, Destructor original) => new Tuple("destructors", destructor, name, containingType, original); + + internal static Tuple dynamic_member_name(Expression e, string name) => new Tuple("dynamic_member_name", e, name); + + internal static Tuple enum_underlying_type(Type @enum, Type type) => new Tuple("enum_underlying_type", @enum, type); + + internal static Tuple event_accessor_location(EventAccessor accessor, Location location) => new Tuple("event_accessor_location", accessor, location); + + internal static Tuple event_accessors(EventAccessor accessorKey, int type, string name, Event eventKey, EventAccessor unboundAccessor) => new Tuple("event_accessors", accessorKey, type, name, eventKey, unboundAccessor); + + internal static Tuple event_location(Event eventKey, Location locationKey) => new Tuple("event_location", eventKey, locationKey); + + internal static Tuple events(Event eventKey, string name, Type declaringType, Type memberType, Event originalDefinition) => new Tuple("events", eventKey, name, declaringType, memberType, originalDefinition); + + internal static Tuple explicitly_implements(IEntity member, Type @interface) => new Tuple("explicitly_implements", member, @interface); + + internal static Tuple explicitly_sized_array_creation(Expression array) => new Tuple("explicitly_sized_array_creation", array); + + internal static Tuple expr_compiler_generated(Expression expr) => new Tuple("expr_compiler_generated", expr); + + internal static Tuple expr_location(Expression exprKey, Location location) => new Tuple("expr_location", exprKey, location); + + internal static Tuple expr_access(Expression expr, IEntity access) => new Tuple("expr_access", expr, access); + + internal static Tuple expr_argument(Expression expr, int mode) => new Tuple("expr_argument", expr, mode); + + internal static Tuple expr_argument_name(Expression expr, string name) => new Tuple("expr_argument_name", expr, name); + + internal static Tuple expr_call(Expression expr, Method target) => new Tuple("expr_call", expr, target); + + internal static Tuple expr_parent(Expression exprKey, int child, IExpressionParentEntity parent) => new Tuple("expr_parent", exprKey, child, parent); + + internal static Tuple expr_parent_top_level(Expression exprKey, int child, IExpressionParentEntity parent) => new Tuple("expr_parent_top_level", exprKey, child, parent); + + internal static Tuple expr_value(Expression exprKey, string value) => new Tuple("expr_value", exprKey, value); + + internal static Tuple expressions(Expression expr, ExprKind kind, Type exprType) => new Tuple("expressions", expr, kind, exprType); + + internal static Tuple exprorstmt_name(IEntity expr, string name) => new Tuple("exprorstmt_name", expr, name); + + internal static Tuple extend(Type type, Type super) => new Tuple("extend", type, super); + + internal static Tuple field_location(Field field, Location location) => new Tuple("field_location", field, location); + + internal static Tuple fields(Field field, int @const, string name, Type declaringType, Type fieldType, Field unboundKey) => new Tuple("fields", field, @const, name, declaringType, fieldType, unboundKey); + + + internal static Tuple general_type_parameter_constraints(TypeParameterConstraints constraints, int hasKind) => new Tuple("general_type_parameter_constraints", constraints, hasKind); + + internal static Tuple has_modifiers(IEntity entity, Modifier modifier) => new Tuple("has_modifiers", entity, modifier); + + internal static Tuple implement(Type type, Type @interface) => new Tuple("implement", type, @interface); + + internal static Tuple implicitly_typed_array_creation(Expression array) => new Tuple("implicitly_typed_array_creation", array); + + internal static Tuple indexer_location(Indexer indexer, Location location) => new Tuple("indexer_location", indexer, location); + + internal static Tuple indexers(Indexer propKey, string name, Type declaringType, Type memberType, Indexer unboundProperty) => new Tuple("indexers", propKey, name, declaringType, memberType, unboundProperty); + + internal static Tuple is_constructed(IEntity typeOrMethod) => new Tuple("is_constructed", typeOrMethod); + + internal static Tuple is_generic(IEntity typeOrMethod) => new Tuple("is_generic", typeOrMethod); + + internal static Tuple jump_step(IEntity origin, IEntity src, Statement dest) => new Tuple("jump_step", origin, src, dest); + + internal static Tuple local_function_stmts(Entities.Statements.LocalFunction fnStmt, LocalFunction fn) => new Tuple("local_function_stmts", fnStmt, fn); + + internal static Tuple local_functions(LocalFunction fn, string name, Type returnType, LocalFunction unboundFn) => new Tuple("local_functions", fn, name, returnType, unboundFn); + + internal static Tuple localvar_location(LocalVariable var, Location location) => new Tuple("localvar_location", var, location); + + internal static Tuple localvars(LocalVariable key, int @const, string name, int @var, Type type, Expression expr) => new Tuple("localvars", key, @const, name, @var, type, expr); + + internal static Tuple method_location(Method method, Location location) => new Tuple("method_location", method, location); + + internal static Tuple methods(Method method, string name, Type declType, Type retType, Method originalDefinition) => new Tuple("methods", method, name, declType, retType, originalDefinition); + + internal static Tuple mutator_invocation_mode(Expression expr, int mode) => new Tuple("mutator_invocation_mode", expr, mode); + + internal static Tuple namespace_declaration_location(NamespaceDeclaration decl, Location location) => new Tuple("namespace_declaration_location", decl, location); + + internal static Tuple namespace_declarations(NamespaceDeclaration decl, Namespace ns) => new Tuple("namespace_declarations", decl, ns); + + internal static Tuple namespaces(Namespace ns, string name) => new Tuple("namespaces", ns, name); + + internal static Tuple nested_types(Type typeKey, Type declaringTypeKey, Type unboundTypeKey) => new Tuple("nested_types", typeKey, declaringTypeKey, unboundTypeKey); + + internal static Tuple nullable_underlying_type(Type nullableType, Type underlyingType) => new Tuple("nullable_underlying_type", nullableType, underlyingType); + + internal static Tuple numlines(IEntity label, LineCounts lineCounts) => new Tuple("numlines", label, lineCounts.Total, lineCounts.Code, lineCounts.Comment); + + internal static Tuple operator_location(UserOperator @operator, Location location) => new Tuple("operator_location", @operator, location); + + internal static Tuple operators(UserOperator method, string methodName, string symbol, Type classKey, Type returnType, UserOperator originalDefinition) => new Tuple("operators", method, methodName, symbol, classKey, returnType, originalDefinition); + + internal static Tuple overrides(Method overriding, Method overridden) => new Tuple("overrides", overriding, overridden); + + internal static Tuple param_location(Parameter param, Location location) => new Tuple("param_location", param, location); + + internal static Tuple @params(Parameter param, string name, Type type, int child, Parameter.Kind mode, IEntity method, Parameter originalDefinition) => new Tuple("params", param, name, type, child, mode, method, originalDefinition); + + internal static Tuple parent_namespace(IEntity type, Namespace parent) => new Tuple("parent_namespace", type, parent); + + internal static Tuple parent_namespace_declaration(IEntity item, NamespaceDeclaration parent) => new Tuple("parent_namespace_declaration", item, parent); + + internal static Tuple pointer_referent_type(PointerType pointerType, Type referentType) => new Tuple("pointer_referent_type", pointerType, referentType); + + internal static Tuple property_location(Property property, Location location) => new Tuple("property_location", property, location); + + internal static Tuple properties(Property propKey, string name, Type declaringType, Type memberType, Property unboundProperty) => new Tuple("properties", propKey, name, declaringType, memberType, unboundProperty); + + internal static Tuple ref_returns(IEntity method) => new Tuple("ref_returns", method); + + internal static Tuple ref_readonly_returns(IEntity method) => new Tuple("ref_readonly_returns", method); + + internal static Tuple statements(Statement stmt, StmtKind kind) => new Tuple("statements", stmt, kind); + + internal static Tuple specific_type_parameter_constraints(TypeParameterConstraints constraints, Type baseType) => new Tuple("specific_type_parameter_constraints", constraints, baseType); + + internal static Tuple successors(IEntity from, IEntity to) => new Tuple("successors", from, to); + + internal static Tuple stmt_location(Statement stmt, Location location) => new Tuple("stmt_location", stmt, location); + + internal static Tuple stmt_parent(Statement stmt, int child, IStatementParentEntity parent) => new Tuple("stmt_parent", stmt, child, parent); + + internal static Tuple stmt_parent_top_level(Statement stmt, int child, IStatementParentEntity parent) => new Tuple("stmt_parent_top_level", stmt, child, parent); + + internal static Tuple tuple_element(TupleType type, int index, Field field) => new Tuple("tuple_element", type, index, field); + + internal static Tuple tuple_underlying_type(TupleType type, NamedType underlying) => new Tuple("tuple_underlying_type", type, underlying); + + internal static Tuple type_mention(TypeMention ta, Type type, IEntity parent) => new Tuple("type_mention", ta, type, parent); + + internal static Tuple type_mention_location(TypeMention ta, Location loc) => new Tuple("type_mention_location", ta, loc); + + internal static Tuple type_arguments(Type arg, int n, IEntity typeOrMethod) => new Tuple("type_arguments", arg, n, typeOrMethod); + + internal static Tuple type_location(Type type, Location location) => new Tuple("type_location", type, location); + + internal static Tuple type_parameter_constraints(TypeParameterConstraints constraints, TypeParameter typeParam) => new Tuple("type_parameter_constraints", constraints, typeParam); + + internal static Tuple type_parameters(TypeParameter param, int child, IEntity typeOrMethod) => new Tuple("type_parameters", param, child, typeOrMethod, param.Variance); + + internal static Tuple typeref_type(NamedTypeRef typeref, Type type) => new Tuple("typeref_type", typeref, type); + + internal static Tuple typerefs(NamedTypeRef type, string name) => new Tuple("typerefs", type, name); + + internal static Tuple types(Type type, TypeKind kind, params string[] name) => new Tuple("types", type, kind, name); + + internal static Tuple using_namespace_directives(UsingDirective @using, Namespace ns) => new Tuple("using_namespace_directives", @using, ns); + + internal static Tuple using_directive_location(UsingDirective @using, Location location) => new Tuple("using_directive_location", @using, location); + + internal static Tuple using_static_directives(UsingDirective @using, Type type) => new Tuple("using_static_directives", @using, type); + } +} diff --git a/csharp/extractor/Semmle.Extraction.Tests/Layout.cs b/csharp/extractor/Semmle.Extraction.Tests/Layout.cs new file mode 100644 index 000000000000..631a7b89a07d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/Layout.cs @@ -0,0 +1,200 @@ +using System.IO; +using Xunit; +using Semmle.Util.Logging; +using System.Runtime.InteropServices; + +namespace Semmle.Extraction.Tests +{ + public class Layout + { + readonly ILogger Logger = new LoggerMock(); + + [Fact] + public void TestDefaultLayout() + { + var layout = new Semmle.Extraction.Layout(null, null, null); + var project = layout.LookupProjectOrNull("foo.cs"); + + // All files are mapped when there's no layout file. + Assert.True(layout.FileInLayout("foo.cs")); + + // Test trap filename + var tmpDir = Path.GetTempPath(); + Directory.SetCurrentDirectory(tmpDir); + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + // `Directory.SetCurrentDirectory()` doesn't seem to work on macOS, + // so disable this test on macOS, for now + Assert.NotEqual(Directory.GetCurrentDirectory(), tmpDir); + return; + } + var f1 = project.GetTrapPath(Logger, "foo.cs"); + var g1 = TrapWriter.NestPaths(Logger, tmpDir, "foo.cs.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.Equal(f1, g1); + + // Test trap file generation + var trapwriterFilename = project.GetTrapPath(Logger, "foo.cs"); + using (var trapwriter = project.CreateTrapWriter(Logger, "foo.cs", false)) + { + trapwriter.Emit("1=*"); + Assert.False(File.Exists(trapwriterFilename)); + } + Assert.True(File.Exists(trapwriterFilename)); + File.Delete(trapwriterFilename); + } + + [Fact] + public void TestLayoutFile() + { + File.WriteAllLines("layout.txt", new string[] + { + "# Section", + "TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap"), + "ODASA_DB=snapshot\\db-csharp", + "SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive"), + "ODASA_BUILD_ERROR_DIR=snapshot\build-errors", + "-foo.cs", + "bar.cs", + "-excluded", + "excluded/foo.cs", + "included" + }); + + var layout = new Semmle.Extraction.Layout(null, null, "layout.txt"); + + // Test general pattern matching + Assert.True(layout.FileInLayout("bar.cs")); + Assert.False(layout.FileInLayout("foo.cs")); + Assert.False(layout.FileInLayout("goo.cs")); + Assert.False(layout.FileInLayout("excluded/bar.cs")); + Assert.True(layout.FileInLayout("excluded/foo.cs")); + Assert.True(layout.FileInLayout("included/foo.cs")); + + // Test the trap file + var project = layout.LookupProjectOrNull("bar.cs"); + var trapwriterFilename = project.GetTrapPath(Logger, "bar.cs"); + Assert.Equal(TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\trap"), "bar.cs.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE), + trapwriterFilename); + + // Test the source archive + var trapWriter = project.CreateTrapWriter(Logger, "bar.cs", false); + trapWriter.Archive("layout.txt", System.Text.Encoding.ASCII); + var writtenFile = TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\archive"), "layout.txt", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.True(File.Exists(writtenFile)); + File.Delete("layout.txt"); + } + + [Fact] + public void TestTrapOverridesLayout() + { + // When you specify both a trap file and a layout, use the trap file. + var layout = new Semmle.Extraction.Layout(Path.GetFullPath("snapshot\\trap"), null, "something.txt"); + Assert.True(layout.FileInLayout("bar.cs")); + var f1 = layout.LookupProjectOrNull("foo.cs").GetTrapPath(Logger, "foo.cs"); + var g1 = TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\trap"), "foo.cs.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.Equal(f1, g1); + } + + [Fact] + public void TestMultipleSections() + { + File.WriteAllLines("layout.txt", new string[] + { + "# Section 1", + "TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap1"), + "ODASA_DB=snapshot\\db-csharp", + "SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive1"), + "ODASA_BUILD_ERROR_DIR=snapshot\build-errors", + "foo.cs", + "# Section 2", + "TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap2"), + "ODASA_DB=snapshot\\db-csharp", + "SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive2"), + "ODASA_BUILD_ERROR_DIR=snapshot\build-errors", + "bar.cs", + }); + + var layout = new Semmle.Extraction.Layout(null, null, "layout.txt"); + + // Use Section 2 + Assert.True(layout.FileInLayout("bar.cs")); + var f1 = layout.LookupProjectOrNull("bar.cs").GetTrapPath(Logger, "bar.cs"); + var g1 = TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\trap2"), "bar.cs.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.Equal(f1, g1); + + // Use Section 1 + Assert.True(layout.FileInLayout("foo.cs")); + var f2 = layout.LookupProjectOrNull("foo.cs").GetTrapPath(Logger, "foo.cs"); + var g2 = TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\trap1"), "foo.cs.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.Equal(f2, g2); + + // boo.dll is not in the layout, so use layout from first section. + Assert.False(layout.FileInLayout("boo.dll")); + var f3 = layout.LookupProjectOrDefault("boo.dll").GetTrapPath(Logger, "boo.dll"); + var g3 = TrapWriter.NestPaths(Logger, Path.GetFullPath("snapshot\\trap1"), "boo.dll.trap.gz", TrapWriter.InnerPathComputation.ABSOLUTE); + Assert.Equal(f3, g3); + + // boo.cs is not in the layout, so return null + Assert.False(layout.FileInLayout("boo.cs")); + Assert.Null(layout.LookupProjectOrNull("boo.cs")); + } + + [Fact] + public void MissingLayout() + { + Assert.Throws(() => + new Semmle.Extraction.Layout(null, null, "nosuchfile.txt")); + } + + [Fact] + public void EmptyLayout() + { + File.Create("layout.txt").Close(); + Assert.Throws(() => + new Semmle.Extraction.Layout(null, null, "layout.txt")); + } + + [Fact] + public void InvalidLayout() + { + File.WriteAllLines("layout.txt", new string[] + { + "# Section 1" + }); + + Assert.Throws(() => + new Semmle.Extraction.Layout(null, null, "layout.txt")); + } + + class LoggerMock : ILogger + { + public void Dispose() { } + + public void Log(Severity s, string text) { } + + public void Log(Severity s, string text, params object[] args) { } + } + } + + static class TrapWriterTestExtensions + { + public static void Emit(this TrapWriter tw, string s) + { + tw.Emit(new StringTrapEmitter(s)); + } + + class StringTrapEmitter : ITrapEmitter + { + string Content; + public StringTrapEmitter(string content) + { + Content = content; + } + + public void EmitToTrapBuilder(ITrapBuilder tb) + { + tb.Append(Content); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.Tests/Options.cs b/csharp/extractor/Semmle.Extraction.Tests/Options.cs new file mode 100644 index 000000000000..50ea47ae9790 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/Options.cs @@ -0,0 +1,188 @@ +using Xunit; +using Semmle.Util.Logging; +using System; + +namespace Semmle.Extraction.Tests +{ + public class OptionsTests + { + CSharp.Options options; + CSharp.Standalone.Options standaloneOptions; + + public OptionsTests() + { + Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", ""); + Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", ""); + } + + [Fact] + public void DefaultOptions() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { }); + Assert.True(options.Cache); + Assert.False(options.CIL); + Assert.Null(options.Framework); + Assert.Null(options.CompilerName); + Assert.Empty(options.CompilerArguments); + Assert.True(options.Threads >= 1); + Assert.Equal(Verbosity.Info, options.Verbosity); + Assert.False(options.Console); + Assert.False(options.ClrTracer); + Assert.False(options.PDB); + Assert.False(options.Fast); + } + + [Fact] + public void Threads() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--threads", "3" }); + Assert.Equal(3, options.Threads); + } + + [Fact] + public void Cache() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--nocache" }); + Assert.False(options.Cache); + } + + [Fact] + public void CIL() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--cil" }); + Assert.True(options.CIL); + options = CSharp.Options.CreateWithEnvironment(new string[] { "--cil", "--nocil" }); + Assert.False(options.CIL); + } + + [Fact] + public void CompilerArguments() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "x", "y", "z" }); + Assert.Equal("x", options.CompilerArguments[0]); + Assert.Equal("y", options.CompilerArguments[1]); + Assert.Equal("z", options.CompilerArguments[2]); + } + + [Fact] + public void VerbosityTests() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbose" }); + Assert.Equal(Verbosity.Debug, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "0" }); + Assert.Equal(Verbosity.Off, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "1" }); + Assert.Equal(Verbosity.Error, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "2" }); + Assert.Equal(Verbosity.Warning, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "3" }); + Assert.Equal(Verbosity.Info, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "4" }); + Assert.Equal(Verbosity.Debug, options.Verbosity); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "5" }); + Assert.Equal(Verbosity.Trace, options.Verbosity); + + Assert.Throws(() => CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "X" })); + } + + [Fact] + public void Console() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--console" }); + Assert.True(options.Console); + } + + [Fact] + public void PDB() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--pdb" }); + Assert.True(options.PDB); + } + + [Fact] + public void Compiler() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--compiler", "foo" }); + Assert.Equal("foo", options.CompilerName); + } + + [Fact] + public void Framework() + { + options = CSharp.Options.CreateWithEnvironment(new string[] { "--framework", "foo" }); + Assert.Equal("foo", options.Framework); + } + + [Fact] + public void EnvironmentVariables() + { + Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", "--cil c"); + options = CSharp.Options.CreateWithEnvironment(new string[] { "a", "b" }); + Assert.True(options.CIL); + Assert.Equal("a", options.CompilerArguments[0]); + Assert.Equal("b", options.CompilerArguments[1]); + Assert.Equal("c", options.CompilerArguments[2]); + + Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", ""); + Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--nocil"); + options = CSharp.Options.CreateWithEnvironment(new string[] { "--cil" }); + Assert.False(options.CIL); + } + + [Fact] + public void StandaloneDefaults() + { + standaloneOptions = CSharp.Standalone.Options.Create(new string[] { }); + Assert.Equal(0, standaloneOptions.DllDirs.Count); + Assert.True(standaloneOptions.UseNuGet); + Assert.True(standaloneOptions.UseMscorlib); + Assert.False(standaloneOptions.SkipExtraction); + Assert.Null(standaloneOptions.SolutionFile); + Assert.True(standaloneOptions.ScanNetFrameworkDlls); + Assert.False(standaloneOptions.Errors); + } + + [Fact] + public void StandaloneOptions() + { + standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--references:foo", "--silent", "--skip-nuget", "--skip-dotnet", "--exclude", "bar", "--nostdlib" }); + Assert.Equal("foo", standaloneOptions.DllDirs[0]); + Assert.Equal("bar", standaloneOptions.Excludes[0]); + Assert.Equal(Verbosity.Off, standaloneOptions.Verbosity); + Assert.False(standaloneOptions.UseNuGet); + Assert.False(standaloneOptions.UseMscorlib); + Assert.False(standaloneOptions.ScanNetFrameworkDlls); + Assert.False(standaloneOptions.Errors); + Assert.False(standaloneOptions.Help); + } + + [Fact] + public void InvalidOptions() + { + standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--references:foo", "--silent", "--no-such-option" }); + Assert.True(standaloneOptions.Errors); + } + + [Fact] + public void ShowingHelp() + { + standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--help" }); + Assert.False(standaloneOptions.Errors); + Assert.True(standaloneOptions.Help); + } + + [Fact] + public void Fast() + { + Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--fast"); + options = CSharp.Options.CreateWithEnvironment(new string[] {}); + Assert.True(options.Fast); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.Tests/Properties/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000000..f87cf947b4d2 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Semmle.Extraction.Tests")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Semmle.Extraction.Tests")] +[assembly: AssemblyCopyright("Copyright © 2016")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("23237396-31ef-41f8-b466-ee96ddd7b7bc")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj b/csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj new file mode 100644 index 000000000000..24914080bcf8 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/Semmle.Extraction.Tests.csproj @@ -0,0 +1,21 @@ + + + + Exe + netcoreapp2.0 + false + + + + + + + + + + + + + + + diff --git a/csharp/extractor/Semmle.Extraction.Tests/TrapWriter.cs b/csharp/extractor/Semmle.Extraction.Tests/TrapWriter.cs new file mode 100644 index 000000000000..fd7f77f427b4 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.Tests/TrapWriter.cs @@ -0,0 +1,67 @@ +using Xunit; +using Semmle.Util.Logging; +using Semmle.Util; +using System.Runtime.InteropServices; +using System.IO; + +namespace Semmle.Extraction.Tests +{ + public class TrapWriterTests + { + [Fact] + public void NestedPaths() + { + string tempDir = System.IO.Path.GetTempPath(); + string root1, root2, root3; + + if(Win32.IsWindows()) + { + root1 = "E:"; + root2 = "e:"; + root3 = @"\"; + } + else + { + root1 = "/E_"; + root2 = "/e_"; + root3 = "/"; + } + + string formattedTempDir = tempDir.Replace('/', '\\').Replace(':', '_').Trim('\\'); + + var logger = new LoggerMock(); + System.IO.Directory.SetCurrentDirectory(tempDir); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + // `Directory.SetCurrentDirectory()` doesn't seem to work on macOS, + // so disable this test on macOS, for now + Assert.NotEqual(Directory.GetCurrentDirectory(), tempDir); + return; + } + + Assert.Equal($@"C:\Temp\source_archive\{formattedTempDir}\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", "def.cs", TrapWriter.InnerPathComputation.ABSOLUTE).Replace('/','\\')); + + Assert.Equal(@"C:\Temp\source_archive\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", "def.cs", TrapWriter.InnerPathComputation.RELATIVE).Replace('/', '\\')); + + Assert.Equal(@"C:\Temp\source_archive\E_\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root1}\source\def.cs", TrapWriter.InnerPathComputation.ABSOLUTE).Replace('/', '\\')); + + Assert.Equal(@"C:\Temp\source_archive\e_\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root2}\source\def.cs", TrapWriter.InnerPathComputation.RELATIVE).Replace('/', '\\')); + + Assert.Equal(@"C:\Temp\source_archive\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs", TrapWriter.InnerPathComputation.ABSOLUTE).Replace('/', '\\')); + + Assert.Equal(@"C:\Temp\source_archive\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}source\def.cs", TrapWriter.InnerPathComputation.RELATIVE).Replace('/', '\\')); + + Assert.Equal(@"C:\Temp\source_archive\diskstation\share\source\def.cs", TrapWriter.NestPaths(logger, @"C:\Temp\source_archive", $@"{root3}{root3}diskstation\share\source\def.cs", TrapWriter.InnerPathComputation.ABSOLUTE).Replace('/', '\\')); + } + + class LoggerMock : ILogger + { + public void Dispose() { } + + public void Log(Severity s, string text) { } + + public void Log(Severity s, string text, params object[] args) { } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction/CommentProcessing.cs b/csharp/extractor/Semmle.Extraction/CommentProcessing.cs new file mode 100644 index 000000000000..443cc6791224 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction/CommentProcessing.cs @@ -0,0 +1,482 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; +using Semmle.Util; + +namespace Semmle.Extraction.CommentProcessing +{ + // The lexical type of the comment. + public enum CommentType + { + Singleline, // Comment starting // ... + XmlDoc, // Comment starting /// ... + Multiline, // Comment starting /* ..., even if the comment only spans one line. + MultilineContinuation // The second and subsequent lines of comment in a multiline comment. + }; + + // Relationship between a comment and a program element. + public enum Binding + { + Parent, // The parent element of a comment + Best, // The most likely element associated with a comment + Before, // The element before the comment + After // The element after the comment + }; + + /// + /// A single line of text in a comment. + /// + public interface ICommentLine + { + Location Location { get; } + CommentType Type { get; } + + // Trimmed text of the comment. + string Text { get; } + + // Complete text of the comment including leading/trailing whitespace and comment markers. + string RawText { get; } + } + + /// + /// A block of comment lines combined into one unit. + /// + public interface ICommentBlock + { + Location Location { get; } + IList CommentLines { get; } + } + + /// + /// Output for generated comment associations. + /// + /// The label of the element + /// The duplication guard key of the element, if any + /// The comment block associated with the element + /// The relationship between the commentblock and the element + public delegate void CommentBinding(Label elementLabel, Key duplicationGuardKey, ICommentBlock commentBlock, Binding binding); + + /// + /// Used by the populator to generate binding information between comments and program elements. + /// + public interface ICommentGenerator + { + /// + /// Registers the location of a program element to associate comments with. + /// Can be called in any order. + /// + /// Label of the element. + /// The duplication guard key of the element, if any. + /// Location of the element. + void RegisterElementLocation(Label elementLabel, Key duplicationGuardKey, Location location); + + void AddComment(ICommentLine comment); + + /// + /// Generate all binding information. + /// + /// Receiver of the binding information. + void GenerateBindings(CommentBinding cb); + } + + static class LocationExtension + { + public static int StartLine(this Location loc) => loc.GetLineSpan().Span.Start.Line; + + public static int StartColumn(this Location loc) => loc.GetLineSpan().Span.Start.Character; + + + public static int EndLine(this Location loc) => loc.GetLineSpan().Span.End.Line; + + /// + /// Whether one Location outer completely contains another Location inner. + /// + /// The outer location. + /// The inner location + /// Whether inner is completely container in outer. + public static bool Contains(this Location outer, Location inner) + { + bool sameFile = outer.SourceTree == inner.SourceTree; + bool startsBefore = outer.SourceSpan.Start <= inner.SourceSpan.Start; + bool endsAfter = outer.SourceSpan.End >= inner.SourceSpan.End; + return sameFile && startsBefore && endsAfter; + } + + /// + /// Whether one Location ends before another starts. + /// + /// The Location coming before + /// The Location coming after + /// Whether 'before' comes before 'after'. + public static bool Before(this Location before, Location after) + { + bool sameFile = before.SourceTree == after.SourceTree; + bool endsBefore = before.SourceSpan.End <= after.SourceSpan.Start; + return sameFile && endsBefore; + } + } + + /// + /// Implements the comment processor. + /// Registers locations of comments and program elements, + /// then generates binding information. + /// + class CommentProcessor : ICommentGenerator + { + public void AddComment(ICommentLine comment) + { + comments[comment.Location] = comment; + } + + // Comments sorted by location. + readonly SortedDictionary comments = new SortedDictionary(new LocationComparer()); + + // Program elements sorted by location. + readonly SortedDictionary elements = new SortedDictionary(new LocationComparer()); + + readonly Dictionary duplicationGuardKeys = new Dictionary(); + + Key GetDuplicationGuardKey(Label label) + { + Key duplicationGuardKey; + if (duplicationGuardKeys.TryGetValue(label, out duplicationGuardKey)) + return duplicationGuardKey; + return null; + } + + class LocationComparer : IComparer + { + public int Compare(Location l1, Location l2) => CommentProcessor.Compare(l1, l2); + } + + /// + /// Comparer for two locations, allowing them to be inserted into a sorted list. + /// + /// First location + /// Second location + /// <0 if l1 before l2, >0 if l1 after l2, else 0. + static int Compare(Location l1, Location l2) + { + int diff = l1.SourceTree == l2.SourceTree ? 0 : l1.SourceTree.FilePath.CompareTo(l2.SourceTree.FilePath); + if (diff != 0) return diff; + diff = l1.SourceSpan.Start - l2.SourceSpan.Start; + if (diff != 0) return diff; + return l1.SourceSpan.End - l2.SourceSpan.End; + } + + /// + /// Called by the populator when there is a program element which can have comments. + /// + /// The label of the element in the trap file. + /// The duplication guard key of the element, if any. + /// The location of the element. + public void RegisterElementLocation(Label elementLabel, Key duplicationGuardKey, Location loc) + { + if (loc != null && loc.IsInSource) + elements[loc] = elementLabel; + if (duplicationGuardKey != null) + duplicationGuardKeys[elementLabel] = duplicationGuardKey; + } + + // Ensure that commentBlock and element refer to the same file + // which can happen when processing multiple files. + void EnsureSameFile(ICommentBlock commentBlock, ref KeyValuePair? element) + { + if (element != null && element.Value.Key.SourceTree != commentBlock.Location.SourceTree) + element = null; + } + + /// + /// Generate the bindings between a comment and program elements. + /// Called once for each commentBlock. + /// + /// + /// The comment block. + /// The element before the comment block. + /// The element after the comment block. + /// The parent element of the comment block. + /// Output binding information. + void GenerateBindings( + ICommentBlock commentBlock, + KeyValuePair? previousElement, + KeyValuePair? nextElement, + KeyValuePair? parentElement, + CommentBinding cb + ) + { + EnsureSameFile(commentBlock, ref previousElement); + EnsureSameFile(commentBlock, ref nextElement); + EnsureSameFile(commentBlock, ref parentElement); + + if (previousElement != null) + { + var key = previousElement.Value.Value; + cb(key, GetDuplicationGuardKey(key), commentBlock, Binding.Before); + } + + if (nextElement != null) + { + var key = nextElement.Value.Value; + cb(key, GetDuplicationGuardKey(key), commentBlock, Binding.After); + } + + if (parentElement != null) + { + var key = parentElement.Value.Value; + cb(key, GetDuplicationGuardKey(key), commentBlock, Binding.Parent); + } + + // Heuristic to decide which is the "best" element associated with the comment. + KeyValuePair? bestElement; + + if (previousElement != null && previousElement.Value.Key.EndLine() == commentBlock.Location.StartLine()) + { + // 1. If the comment is on the same line as the previous element, use that + bestElement = previousElement; + } + else if (nextElement != null && nextElement.Value.Key.StartLine() == commentBlock.Location.EndLine()) + { + // 2. If the comment is on the same line as the next element, use that + bestElement = nextElement; + } + else if (nextElement != null && previousElement != null && + previousElement.Value.Key.EndLine() + 1 == commentBlock.Location.StartLine() && + commentBlock.Location.EndLine() + 1 == nextElement.Value.Key.StartLine()) + { + // 3. If comment is equally between two elements, use the parentElement + // because it's ambiguous whether the comment refers to the next or previous element + bestElement = parentElement; + } + else if (nextElement != null && nextElement.Value.Key.StartLine() == commentBlock.Location.EndLine() + 1) + { + // 4. If there is no gap after the comment, use "nextElement" + bestElement = nextElement; + } + else if (previousElement != null && previousElement.Value.Key.EndLine() + 1 == commentBlock.Location.StartLine()) + { + // 5. If there is no gap before the comment, use previousElement + bestElement = previousElement; + } + else + { + // 6. Otherwise, bind the comment to the parent block. + bestElement = parentElement; + + /* if parentElement==null, then there is no best element. The comment is effectively orphaned. + * + * This can be caused by comments that are not in a type declaration. + * Due to restrictions in the dbscheme, the comment cannot be associated with the "file" + * which is not an element, and the "using" declarations are not emitted by the extractor. + */ + } + + if (bestElement != null) + { + var label = bestElement.Value.Value; + cb(label, GetDuplicationGuardKey(label), commentBlock, Binding.Best); + } + } + + // Stores element nesting information in a stack. + // Top of stack = most nested element, based on Location. + class ElementStack + { + // Invariant: the top of the stack must be contained by items below it. + readonly Stack> elementStack = new Stack>(); + + /// + /// Add a new element to the stack. + /// + /// The stack is maintained. + /// The new element to push. + public void Push(KeyValuePair value) + { + // Maintain the invariant by popping existing elements + while (elementStack.Count > 0 && !elementStack.Peek().Key.Contains(value.Key)) + elementStack.Pop(); + + elementStack.Push(value); + } + + /// + /// Locate the parent of a comment with location l. + /// + /// The location of the comment. + /// An element completely containing l, or null if none found. + public KeyValuePair? FindParent(Location l) => + elementStack.Where(v => v.Key.Contains(l)).FirstOrNull(); + + /// + /// Finds the element on the stack immediately preceding the comment at l. + /// + /// The location of the comment. + /// The element before l, or null. + public KeyValuePair? FindBefore(Location l) + { + return elementStack. + Where(v => v.Key.SourceSpan.End < l.SourceSpan.Start). + LastOrNull(); + } + + /// + /// Finds the element after the comment. + /// + /// The location of the comment. + /// The next element. + /// The next element. + public KeyValuePair? FindAfter(Location comment, KeyValuePair? next) + { + var p = FindParent(comment); + return next.HasValue && p.HasValue && p.Value.Key.Before(next.Value.Key) ? null : next; + } + } + + // Generate binding information for one CommentBlock. + void GenerateBindings( + ICommentBlock block, + ElementStack elementStack, + KeyValuePair? nextElement, + CommentBinding cb + ) + { + if (block.CommentLines.Count > 0) + { + GenerateBindings( + block, + elementStack.FindBefore(block.Location), + elementStack.FindAfter(block.Location, nextElement), + elementStack.FindParent(block.Location), + cb); + } + } + + /// + /// Process comments up until nextElement. + /// Group comments into blocks, and associate blocks with elements. + /// + /// Enumerator for all comments in the program. + /// The next element in the list. + /// A stack of nested program elements. + /// Where to send the results. + /// true if there are more comments to process, false otherwise. + bool GenerateBindings( + IEnumerator> commentEnumerator, + KeyValuePair? nextElement, + ElementStack elementStack, + CommentBinding cb + ) + { + CommentBlock block = new CommentBlock(); + + // Iterate comments until the commentEnumerator has gone past nextElement + while (nextElement == null || Compare(commentEnumerator.Current.Value.Location, nextElement.Value.Key) < 0) + { + if (!block.CombinesWith(commentEnumerator.Current.Value)) + { + // Start of a new block, so generate the bindings for the old block first. + GenerateBindings(block, elementStack, nextElement, cb); + block = new CommentBlock(); + } + + block.AddCommentLine(commentEnumerator.Current.Value); + + // Get the next comment. + if (!commentEnumerator.MoveNext()) + { + // If there are no more comments, generate the remaining bindings and return false. + GenerateBindings(block, elementStack, nextElement, cb); + return false; + } + } + + GenerateBindings(block, elementStack, nextElement, cb); + return true; + } + + /// + /// Merge comments into blocks and associate comment blocks with program elements. + /// + /// Callback for the binding information + public void GenerateBindings(CommentBinding cb) + { + /* Algorithm: + * Do a merge of elements and comments, which are both sorted in location order. + * + * Iterate through all elements, and iterate all comment lines between adjacent pairs of elements. + * Maintain a stack of elements, such that the top of the stack must be fully nested in the + * element below it. This enables comments to be associated with the "parent" element, as well as + * elements before, after and "best" element match for a comment. + * + * This is an O(n) algorithm because the list of elements and comments are traversed once. + * (Note that comment processing is O(n.log n) overall due to dictionary of elements and comments.) + */ + + ElementStack elementStack = new ElementStack(); + + using (IEnumerator> elementEnumerator = elements.GetEnumerator()) + using (IEnumerator> commentEnumerator = comments.GetEnumerator()) + { + if (!commentEnumerator.MoveNext()) + { + // There are no comments to process. + return; + } + + while (elementEnumerator.MoveNext()) + { + if (!GenerateBindings(commentEnumerator, elementEnumerator.Current, elementStack, cb)) + { + // No more comments to process. + return; + } + + elementStack.Push(elementEnumerator.Current); + } + + // Generate remaining comments at end of file + GenerateBindings(commentEnumerator, null, elementStack, cb); + } + } + } + + class CommentBlock : ICommentBlock + { + public IList CommentLines { get; } = new List(); + + public Location Location { get; private set; } + + /// + /// Determine whether commentlines should be merged. + /// + /// A comment line to be appended to this comment block. + /// Whether the new line should be appended to this block. + public bool CombinesWith(ICommentLine newLine) + { + if (CommentLines.Count == 0) return true; + + bool sameFile = Location.SourceTree == newLine.Location.SourceTree; + bool sameRow = Location.EndLine() == newLine.Location.StartLine(); + bool sameColumn = Location.EndLine() + 1 == newLine.Location.StartLine(); + bool nextRow = Location.StartColumn() == newLine.Location.StartColumn(); + bool adjacent = sameFile && (sameRow || (sameColumn && nextRow)); + + return + newLine.Type == CommentType.MultilineContinuation || + adjacent; + } + + /// + /// Adds a comment line to the this comment block. + /// + /// The line to add. + public void AddCommentLine(ICommentLine line) + { + Location = CommentLines.Count == 0 ? + line.Location : + Location.Create(line.Location.SourceTree, new TextSpan(Location.SourceSpan.Start, line.Location.SourceSpan.End - Location.SourceSpan.Start)); + CommentLines.Add(line); + } + } +} + diff --git a/csharp/extractor/Semmle.Extraction/Context.cs b/csharp/extractor/Semmle.Extraction/Context.cs new file mode 100644 index 000000000000..b66370e896b7 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction/Context.cs @@ -0,0 +1,496 @@ +using Microsoft.CodeAnalysis; +using System.Linq; +using Semmle.Extraction.CommentProcessing; +using System.Collections.Generic; +using System; +using Semmle.Util.Logging; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction +{ + /// + /// State which needs needs to be available throughout the extraction process. + /// There is one Context object per trap output file. + /// + public class Context + { + /// + /// Interface to various extraction functions, e.g. logger, trap writer. + /// + public readonly IExtractor Extractor; + + /// + /// The program database provided by Roslyn. + /// There's one per syntax tree, which makes things awkward. + /// + public SemanticModel Model(SyntaxNode node) + { + if (cachedModel == null || node.SyntaxTree != cachedModel.SyntaxTree) + { + cachedModel = Compilation.GetSemanticModel(node.SyntaxTree); + } + + return cachedModel; + } + + SemanticModel cachedModel; + + /// + /// Access to the trap file. + /// + public readonly TrapWriter TrapWriter; + + int NewId() => TrapWriter.IdCounter++; + + /// + /// Gets the cached label for the given entity, or creates a new + /// (cached) label if it hasn't already been created. The label + /// is set on the supplied object. + /// + /// true iff the label already existed. + public bool GetOrAddCachedLabel(ICachedEntity entity) + { + var id = GetId(entity); + if (id == null) + throw new InternalError("Attempt to create a null entity for {0}", entity.GetType()); + + Label existingLabel; + if (labelCache.TryGetValue(id, out existingLabel)) + { + entity.Label = existingLabel; + return true; + } + + entity.Label = new Label(NewId()); + DefineLabel(entity.Label, id); + labelCache[id] = entity.Label; + return false; + } + + /// + /// Should the given entity be extracted? + /// A second call to this method will always return false, + /// on the assumption that it would have been extracted on the first call. + /// + /// This is used to track the extraction of generics, which cannot be extracted + /// in a top-down manner. + /// + /// The entity to extract. + /// True only on the first call for a particular entity. + public bool ExtractGenerics(ICachedEntity entity) + { + if (extractedGenerics.Contains(entity.Label)) + { + return false; + } + else + { + extractedGenerics.Add(entity.Label); + return true; + } + } + + /// + /// Gets the ID belonging to cached entity . + /// + /// The ID itself is also cached, but unlike the label cache (which is used + /// to prevent reextraction/infinite loops), this is a pure performance + /// optimization. Moreover, the label cache is injective, which the ID cache + /// need not be. + /// + IId GetId(ICachedEntity entity) + { + IId id; + if (!idCache.TryGetValue(entity, out id)) + { + id = entity.Id; + idCache[entity] = id; + } + return id; + } + + /// + /// Creates a fresh label with ID "*", and set it on the + /// supplied object. + /// + public void AddFreshLabel(IEntity entity) + { + var label = new Label(NewId()); + TrapWriter.Emit(new DefineFreshLabelEmitter(label)); + entity.Label = label; + } + + readonly Dictionary labelCache = new Dictionary(); + readonly HashSet