diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs index 53542c970e9d..940cd647ed82 100644 --- a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs @@ -24,7 +24,7 @@ private static void ExtractAssembly(Layout layout, string assemblyPath, ILogger { var sw = new Stopwatch(); sw.Start(); - Entities.Assembly.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _); + Analyser.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _); sw.Stop(); logger.Log(Severity.Info, " {0} ({1})", assemblyPath, sw.Elapsed); } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs b/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs new file mode 100644 index 000000000000..dee801cc1718 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs @@ -0,0 +1,53 @@ +using Semmle.Util.Logging; +using System; +using Semmle.Util; +using Semmle.Extraction.CIL.Entities; + +namespace Semmle.Extraction.CIL +{ + public static class Analyser + { + private static void ExtractCIL(Extractor extractor, TrapWriter trapWriter, bool extractPdbs) + { + using var cilContext = new Context(extractor, trapWriter, extractor.OutputPath, extractPdbs); + cilContext.Populate(new Assembly(cilContext)); + cilContext.PopulateAll(); + } + + /// + /// Main entry point to the CIL extractor. + /// Call this to extract a given assembly. + /// + /// The trap layout. + /// The full path of the assembly to extract. + /// The logger. + /// True to overwrite existing trap file. + /// Whether to extract PDBs. + /// The path of the trap file. + /// Whether the file was extracted (false=cached). + public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted) + { + trapFile = ""; + extracted = false; + try + { + var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); + var pathTransformer = new PathTransformer(canonicalPathCache); + var extractor = new Extractor(false, assemblyPath, logger, pathTransformer); + var transformedAssemblyPath = pathTransformer.Transform(assemblyPath); + var project = layout.LookupProjectOrDefault(transformedAssemblyPath); + using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true); + trapFile = trapWriter.TrapFile; + if (nocache || !System.IO.File.Exists(trapFile)) + { + ExtractCIL(extractor, trapWriter, extractPdbs); + extracted = true; + } + } + catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] + { + logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs index 36e420e36281..65c72487b655 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs @@ -9,11 +9,11 @@ namespace Semmle.Extraction.CIL /// /// Provides methods for creating and caching various entities. /// - public sealed partial class Context + internal sealed partial class Context { private readonly Dictionary ids = new Dictionary(); - public T Populate(T e) where T : IExtractedEntity + internal T Populate(T e) where T : IExtractedEntity { if (e.Label.Valid) { @@ -27,10 +27,10 @@ public T Populate(T e) where T : IExtractedEntity } else { - e.Label = Cx.GetNewLabel(); - Cx.DefineLabel(e, Cx.TrapWriter.Writer, Cx.Extractor); + e.Label = GetNewLabel(); + DefineLabel(e); ids.Add(e, e.Label); - Cx.PopulateLater(() => + PopulateLater(() => { foreach (var c in e.Contents) c.Extract(this); @@ -42,7 +42,7 @@ public T Populate(T e) where T : IExtractedEntity if (debugLabels.TryGetValue(id, out var previousEntity)) { - Cx.Extractor.Message(new Message("Duplicate trap ID", id, null, severity: Util.Logging.Severity.Warning)); + Extractor.Message(new Message("Duplicate trap ID", id, null, severity: Util.Logging.Severity.Warning)); } else { @@ -74,9 +74,9 @@ public PrimitiveType Create(PrimitiveTypeCode code) { e = new PrimitiveType(this, code) { - Label = Cx.GetNewLabel() + Label = GetNewLabel() }; - Cx.DefineLabel(e, Cx.TrapWriter.Writer, Cx.Extractor); + DefineLabel(e); primitiveTypes[(int)code] = e; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.cs index 99113714b7c2..acb66dc32841 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.cs @@ -10,12 +10,10 @@ namespace Semmle.Extraction.CIL /// Adds additional context that is specific for CIL extraction. /// One context = one DLL/EXE. /// - public sealed partial class Context : IDisposable + internal sealed partial class Context : Extraction.Context, IDisposable { private readonly FileStream stream; private Entities.Assembly? assemblyNull; - - public Extraction.Context Cx { get; } public MetadataReader MdReader { get; } public PEReader PeReader { get; } public string AssemblyPath { get; } @@ -26,9 +24,9 @@ public Entities.Assembly Assembly } public PDB.IPdb? Pdb { get; } - public Context(Extraction.Context cx, string assemblyPath, bool extractPdbs) + public Context(Extractor extractor, TrapWriter trapWriter, string assemblyPath, bool extractPdbs) + : base(extractor, trapWriter) { - this.Cx = cx; this.AssemblyPath = assemblyPath; stream = File.OpenRead(assemblyPath); PeReader = new PEReader(stream, PEStreamOptions.PrefetchEntireImage); @@ -51,7 +49,7 @@ public Context(Extraction.Context cx, string assemblyPath, bool extractPdbs) Pdb = PDB.PdbReader.Create(assemblyPath, PeReader); if (Pdb != null) { - cx.Extractor.Logger.Log(Util.Logging.Severity.Info, string.Format("Found PDB information for {0}", assemblyPath)); + Extractor.Logger.Log(Util.Logging.Severity.Info, string.Format("Found PDB information for {0}", assemblyPath)); } } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs b/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs index 1105bc39cba3..d5344877264a 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs @@ -5,7 +5,7 @@ namespace Semmle.Extraction.CIL /// /// A generic context which does not contain any type parameters. /// - public class EmptyContext : IGenericContext + internal class EmptyContext : IGenericContext { public EmptyContext(Context cx) { diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs index ba332b2db97f..2191afef125f 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs @@ -1,18 +1,15 @@ using System.Reflection; using System.Globalization; using System.Collections.Generic; -using Semmle.Util.Logging; -using System; using Semmle.Extraction.Entities; using System.IO; -using Semmle.Util; namespace Semmle.Extraction.CIL.Entities { /// /// An assembly to extract. /// - public class Assembly : LabelledEntity, ILocation + internal class Assembly : LabelledEntity, ILocation { private readonly File file; private readonly AssemblyName assemblyName; @@ -76,7 +73,7 @@ public override IEnumerable Contents } catch (InternalError e) { - Cx.Cx.ExtractionError("Error processing type definition", e.Message, GeneratedLocation.Create(Cx.Cx), e.StackTrace); + Cx.ExtractionError("Error processing type definition", e.Message, GeneratedLocation.Create(Cx), e.StackTrace); } // Limitation of C#: Cannot yield return inside a try-catch. @@ -93,7 +90,7 @@ public override IEnumerable Contents } catch (InternalError e) { - Cx.Cx.ExtractionError("Error processing bytecode", e.Message, GeneratedLocation.Create(Cx.Cx), e.StackTrace); + Cx.ExtractionError("Error processing bytecode", e.Message, GeneratedLocation.Create(Cx), e.StackTrace); } if (product != null) @@ -101,49 +98,5 @@ public override IEnumerable Contents } } } - - private static void ExtractCIL(Extraction.Context cx, string assemblyPath, bool extractPdbs) - { - using var cilContext = new Context(cx, assemblyPath, extractPdbs); - cilContext.Populate(new Assembly(cilContext)); - cilContext.Cx.PopulateAll(); - } - - /// - /// Main entry point to the CIL extractor. - /// Call this to extract a given assembly. - /// - /// The trap layout. - /// The full path of the assembly to extract. - /// The logger. - /// True to overwrite existing trap file. - /// Whether to extract PDBs. - /// The path of the trap file. - /// Whether the file was extracted (false=cached). - public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted) - { - trapFile = ""; - extracted = false; - try - { - var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); - var pathTransformer = new PathTransformer(canonicalPathCache); - var extractor = new Extractor(false, assemblyPath, logger, pathTransformer); - var transformedAssemblyPath = pathTransformer.Transform(assemblyPath); - var project = layout.LookupProjectOrDefault(transformedAssemblyPath); - using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true); - trapFile = trapWriter.TrapFile; - if (nocache || !System.IO.File.Exists(trapFile)) - { - var cx = new Extraction.Context(extractor, trapWriter); - ExtractCIL(cx, assemblyPath, extractPdbs); - extracted = true; - } - } - catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] - { - logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex)); - } - } } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs index d79215f0adf6..8bc93bb52b8d 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs @@ -45,7 +45,7 @@ public override IEnumerable Contents } catch { - Cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Info, + Cx.Extractor.Logger.Log(Util.Logging.Severity.Info, $"Attribute decoding is partial. Decoding attribute {constructor.DeclaringType.GetQualifiedName()} failed on {@object}."); yield break; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs index 080227a63b92..d94b94df8a80 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs @@ -5,7 +5,7 @@ namespace Semmle.Extraction.CIL /// /// A CIL entity which has been extracted. /// - public interface IExtractedEntity : IExtractionProduct, IEntity + internal interface IExtractedEntity : IExtractionProduct, IEntity { /// /// The contents of the entity. diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs index 6383d7e1b4ca..231a311a78cb 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs @@ -12,7 +12,7 @@ namespace Semmle.Extraction.CIL /// - Enumerate Contents to produce more extraction products /// - Extract these until there is nothing left to extract /// - public interface IExtractionProduct + internal interface IExtractionProduct { /// /// Perform further extraction/population of this item as necessary. diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs index 21c69f7aeb49..65171e63cd03 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs @@ -6,7 +6,7 @@ namespace Semmle.Extraction.CIL /// When we decode a type/method signature, we need access to /// generic parameters. /// - public interface IGenericContext + internal interface IGenericContext { Context Cx { get; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs index a66ecbb1fab1..26f6c00cd0b4 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs @@ -8,14 +8,13 @@ namespace Semmle.Extraction.CIL /// An entity that needs to be populated during extraction. /// This assigns a key and optionally extracts its contents. /// - public abstract class LabelledEntity : Extraction.LabelledEntity, IExtractedEntity + internal abstract class LabelledEntity : Extraction.LabelledEntity, IExtractedEntity { // todo: with .NET 5 this can override the base context, and change the return type. - public Context Cx { get; } + public Context Cx => (Context)base.Context; - protected LabelledEntity(Context cx) : base(cx.Cx) + protected LabelledEntity(Context cx) : base(cx) { - this.Cx = cx; } public override Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException(); diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs index 5657f072c9c8..58c15dd21f61 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs @@ -14,7 +14,7 @@ public Tuple(string name, params object[] args) public void Extract(Context cx) { - cx.Cx.Emit(tuple); + cx.TrapWriter.Emit(tuple); } public override string ToString() => tuple.ToString(); diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs index d8c18d291eba..42d3824c00ce 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs @@ -7,14 +7,13 @@ namespace Semmle.Extraction.CIL /// An entity that has contents to extract. There is no need to populate /// a key as it's done in the contructor. /// - public abstract class UnlabelledEntity : Extraction.UnlabelledEntity, IExtractedEntity + internal abstract class UnlabelledEntity : Extraction.UnlabelledEntity, IExtractedEntity { // todo: with .NET 5 this can override the base context, and change the return type. - public Context Cx { get; } + public Context Cx => (Context)base.Context; - protected UnlabelledEntity(Context cx) : base(cx.Cx) + protected UnlabelledEntity(Context cx) : base(cx) { - Cx = cx; } public override Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException(); diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs index be2ab5da4d88..6e2fb90beae2 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs @@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A constructed type. /// - public sealed class ConstructedType : Type + internal sealed class ConstructedType : Type { private readonly Type unboundGenericType; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/CustomAttributeDecoder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/CustomAttributeDecoder.cs index 94d6cc0ab6d1..9007e0b8b90a 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/CustomAttributeDecoder.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/CustomAttributeDecoder.cs @@ -40,11 +40,11 @@ public PrimitiveTypeCode GetUnderlyingEnumType(Type type) if (wellKnownEnums.TryGetValue(name, out var code)) { - cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Debug, $"Using hard coded underlying enum type for {name}"); + cx.Extractor.Logger.Log(Util.Logging.Severity.Debug, $"Using hard coded underlying enum type for {name}"); return code; } - cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Info, $"Couldn't get underlying enum type for {name}"); + cx.Extractor.Logger.Log(Util.Logging.Severity.Info, $"Couldn't get underlying enum type for {name}"); // We can't fall back to Int32, because the type returned here defines how many bytes are read from the // stream and how those bytes are interpreted. diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs index 4b612ae86af7..8e440795092c 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs @@ -3,7 +3,7 @@ namespace Semmle.Extraction.CIL.Entities { - public class File : LabelledEntity, IFileOrFolder + internal class File : LabelledEntity, IFileOrFolder { protected string OriginalPath { get; } protected PathTransformer.ITransformedPath TransformedPath { get; } @@ -11,7 +11,7 @@ public class File : LabelledEntity, IFileOrFolder public File(Context cx, string path) : base(cx) { this.OriginalPath = path; - TransformedPath = cx.Cx.Extractor.PathTransformer.Transform(OriginalPath); + TransformedPath = Cx.Extractor.PathTransformer.Transform(OriginalPath); } public override void WriteId(TextWriter trapFile) diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs index 2294a525e4b5..a502a0057248 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs @@ -3,7 +3,7 @@ namespace Semmle.Extraction.CIL.Entities { - public sealed class Folder : LabelledEntity, IFileOrFolder + internal sealed class Folder : LabelledEntity, IFileOrFolder { private readonly PathTransformer.ITransformedPath transformedPath; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs index 033010e18a36..9cfcf444e664 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs @@ -476,7 +476,7 @@ public IEnumerable JumpContents(Dictionary // TODO: Find a solution to this. // For now, just log the error - Cx.Cx.ExtractionError("A CIL instruction jumps outside the current method", null, Extraction.Entities.GeneratedLocation.Create(Cx.Cx), "", Util.Logging.Severity.Warning); + Cx.ExtractionError("A CIL instruction jumps outside the current method", null, Extraction.Entities.GeneratedLocation.Create(Cx), "", Util.Logging.Severity.Warning); } } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs index 5d498c5ac548..6cabce18795f 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs @@ -7,7 +7,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A namespace. /// - public sealed class Namespace : TypeContainer + internal sealed class Namespace : TypeContainer { public Namespace? ParentNamespace { get; } public string Name { get; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs index 07cfdab6543a..97e9948f6a7d 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs @@ -2,7 +2,7 @@ namespace Semmle.Extraction.CIL.Entities { - public class PdbSourceFile : File + internal class PdbSourceFile : File { private readonly PDB.ISourceFile file; @@ -21,9 +21,9 @@ public override IEnumerable Contents var text = file.Contents; if (text == null) - Cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, string.Format("PDB source file {0} could not be found", OriginalPath)); + Cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, string.Format("PDB source file {0} could not be found", OriginalPath)); else - Cx.Cx.TrapWriter.Archive(TransformedPath, text); + Cx.TrapWriter.Archive(TransformedPath, text); yield return Tuples.file_extraction_mode(this, 2); } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs index 661ae0f535bb..d807c5778de1 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs @@ -5,7 +5,7 @@ namespace Semmle.Extraction.CIL.Entities { - public sealed class PrimitiveType : Type + internal sealed class PrimitiveType : Type { private readonly PrimitiveTypeCode typeCode; public PrimitiveType(Context cx, PrimitiveTypeCode tc) : base(cx) diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs index 318cac149304..ff67844121ae 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs @@ -4,7 +4,7 @@ namespace Semmle.Extraction.CIL.Entities { - public sealed class PdbSourceLocation : LabelledEntity, ILocation + internal sealed class PdbSourceLocation : LabelledEntity, ILocation { private readonly Location location; private readonly PdbSourceFile file; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs index ad5a6ababaa3..9b4527770315 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs @@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A type. /// - public abstract class Type : TypeContainer, IMember + internal abstract class Type : TypeContainer, IMember { internal const string AssemblyTypeNameSeparator = "::"; internal const string PrimitiveTypePrefix = "builtin" + AssemblyTypeNameSeparator + "System."; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs index 106b6fe83be7..e201e0652559 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs @@ -6,7 +6,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// Base class for all type containers (namespaces, types, methods). /// - public abstract class TypeContainer : LabelledEntity, IGenericContext + internal abstract class TypeContainer : LabelledEntity, IGenericContext { protected TypeContainer(Context cx) : base(cx) { diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs index 461141ae7190..3f9df14a8918 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs @@ -11,7 +11,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A type defined in the current assembly. /// - public sealed class TypeDefinitionType : Type + internal sealed class TypeDefinitionType : Type { private readonly TypeDefinitionHandle handle; private readonly TypeDefinition td; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs index 49a15965dd0c..51677bf0bb9f 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs @@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A type reference, to a type in a referenced assembly. /// - public sealed class TypeReferenceType : Type + internal sealed class TypeReferenceType : Type { private readonly TypeReferenceHandle handle; private readonly TypeReference tr; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs index 4417becc58d3..d1022d58f0cb 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs @@ -8,7 +8,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// Decodes a type signature and produces a Type, for use by DecodeSignature() and friends. /// - public class TypeSignatureDecoder : ISignatureTypeProvider + internal class TypeSignatureDecoder : ISignatureTypeProvider { private readonly Context cx; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs index b04b429993d7..5ef8bd24d4e2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs @@ -292,7 +292,7 @@ private void DoAnalyseReferenceAssembly(PortableExecutableReference r) AnalyseNamespace(cx, module.GlobalNamespace); } - Entities.Attribute.ExtractAttributes(cx, assembly, Extraction.Entities.Assembly.Create(cx, assembly.GetSymbolLocation())); + Entities.Attribute.ExtractAttributes(cx, assembly, Entities.Assembly.Create(cx, assembly.GetSymbolLocation())); cx.PopulateAll(); } @@ -310,7 +310,7 @@ private void DoExtractCIL(PortableExecutableReference r) { var stopwatch = new Stopwatch(); stopwatch.Start(); - CIL.Entities.Assembly.ExtractCIL(layout, r.FilePath, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted); + CIL.Analyser.ExtractCIL(layout, r.FilePath, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted); stopwatch.Stop(); ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate); } @@ -374,7 +374,7 @@ private void DoExtractTree(SyntaxTree tree) var cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree), AddAssemblyTrapPrefix); // Ensure that the file itself is populated in case the source file is totally empty var root = tree.GetRoot(); - Extraction.Entities.File.Create(cx, root.SyntaxTree.FilePath); + Entities.File.Create(cx, root.SyntaxTree.FilePath); var csNode = (CSharpSyntaxNode)root; csNode.Accept(new CompilationUnitVisitor(cx)); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs b/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs new file mode 100644 index 000000000000..2673ed0d7ef3 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs @@ -0,0 +1,19 @@ +namespace Semmle.Extraction.CSharp +{ + /// + /// A factory for creating cached entities. + /// + internal abstract class CachedEntityFactory + : Extraction.CachedEntityFactory where TEntity : CachedEntity + { + /// + /// Initializes the entity, but does not generate any trap code. + /// + public sealed override TEntity Create(Extraction.Context cx, TInit init) + { + return Create((Context)cx, init); + } + + public abstract TEntity Create(Context cx, TInit init); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBinding.cs b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBinding.cs new file mode 100644 index 000000000000..22518ba49301 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBinding.cs @@ -0,0 +1,13 @@ +namespace Semmle.Extraction.CSharp +{ + /// + /// Describes the relationship between a comment and a program element. + /// + public enum CommentBinding + { + Parent, // The parent element of a comment + Best, // The most likely element associated with a comment + Before, // The element before the comment + After // The element after the comment + }; +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBlock.cs b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBlock.cs new file mode 100644 index 000000000000..c7bb7c27008d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBlock.cs @@ -0,0 +1,59 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; +using Semmle.Extraction.CSharp.Entities; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Comments +{ + internal class CommentBlock + { + private readonly List lines; + + public IEnumerable CommentLines => lines; + + public Location Location { get; private set; } + + public CommentBlock(CommentLine firstLine) + { + lines = new List { firstLine }; + Location = firstLine.Location; + } + + /// + /// Determine whether commentlines should be merged. + /// + /// A comment line to be appended to this comment block. + /// Whether the new line should be appended to this block. + public bool CombinesWith(CommentLine newLine) + { + if (!CommentLines.Any()) + return true; + + var sameFile = Location.SourceTree == newLine.Location.SourceTree; + var sameRow = Location.EndLine() == newLine.Location.StartLine(); + var sameColumn = Location.EndLine() + 1 == newLine.Location.StartLine(); + var nextRow = Location.StartColumn() == newLine.Location.StartColumn(); + var adjacent = sameFile && (sameRow || (sameColumn && nextRow)); + + return + newLine.Type == CommentLineType.MultilineContinuation || + adjacent; + } + + /// + /// Adds a comment line to the this comment block. + /// + /// The line to add. + public void AddCommentLine(CommentLine line) + { + Location = !lines.Any() + ? line.Location + : Location.Create( + line.Location.SourceTree!, + new TextSpan(Location.SourceSpan.Start, line.Location.SourceSpan.End - Location.SourceSpan.Start)); + + lines.Add(line); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentLineType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentLineType.cs new file mode 100644 index 000000000000..8506ca701b3e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentLineType.cs @@ -0,0 +1,13 @@ +namespace Semmle.Extraction.CSharp +{ + /// + /// The type of a single comment line. + /// + public enum CommentLineType + { + Singleline, // Comment starting // ... + XmlDoc, // Comment starting /// ... + Multiline, // Comment starting /* ..., even if the comment only spans one line. + MultilineContinuation // The second and subsequent lines of comment in a multiline comment. + }; +} diff --git a/csharp/extractor/Semmle.Extraction/CommentProcessing.cs b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentProcessor.cs similarity index 81% rename from csharp/extractor/Semmle.Extraction/CommentProcessing.cs rename to csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentProcessor.cs index 201ac7051361..0f99c228a330 100644 --- a/csharp/extractor/Semmle.Extraction/CommentProcessing.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentProcessor.cs @@ -1,32 +1,32 @@ using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.Text; +using Semmle.Extraction.CSharp.Entities; using Semmle.Util; using System.Collections.Generic; using System.Linq; -namespace Semmle.Extraction.CommentProcessing +namespace Semmle.Extraction.CSharp { /// /// Implements the comment processor for associating comments with program elements. /// Registers locations of comments and program elements, /// then generates binding information. /// - internal class CommentProcessor : ICommentGenerator + internal class CommentProcessor { - public void AddComment(ICommentLine comment) + public void AddComment(CommentLine comment) { comments[comment.Location] = comment; } // Comments sorted by location. - private readonly SortedDictionary comments = new SortedDictionary(new LocationComparer()); + private readonly SortedDictionary comments = new SortedDictionary(new LocationComparer()); // Program elements sorted by location. private readonly SortedDictionary elements = new SortedDictionary(new LocationComparer()); private readonly Dictionary duplicationGuardKeys = new Dictionary(); - private Key? GetDuplicationGuardKey(Label label) + private Key GetDuplicationGuardKey(Label label) { if (duplicationGuardKeys.TryGetValue(label, out var duplicationGuardKey)) return duplicationGuardKey; @@ -35,7 +35,7 @@ public void AddComment(ICommentLine comment) private class LocationComparer : IComparer { - public int Compare(Location? l1, Location? l2) => CommentProcessor.Compare(l1, l2); + public int Compare(Location l1, Location l2) => CommentProcessor.Compare(l1, l2); } /// @@ -44,7 +44,7 @@ private class LocationComparer : IComparer /// First location /// Second location /// <0 if l1 before l2, >0 if l1 after l2, else 0. - private static int Compare(Location? l1, Location? l2) + private static int Compare(Location l1, Location l2) { if (object.ReferenceEquals(l1, l2)) return 0; @@ -68,7 +68,7 @@ private static int Compare(Location? l1, Location? l2) /// The label of the element in the trap file. /// The duplication guard key of the element, if any. /// The location of the element. - public void AddElement(Label elementLabel, Key? duplicationGuardKey, Location loc) + public void AddElement(Label elementLabel, Key duplicationGuardKey, Location loc) { if (loc != null && loc.IsInSource) elements[loc] = elementLabel; @@ -78,7 +78,7 @@ public void AddElement(Label elementLabel, Key? duplicationGuardKey, Location lo // Ensure that commentBlock and element refer to the same file // which can happen when processing multiple files. - private static void EnsureSameFile(ICommentBlock commentBlock, ref KeyValuePair? element) + private static void EnsureSameFile(Comments.CommentBlock commentBlock, ref KeyValuePair? element) { if (element != null && element.Value.Key.SourceTree != commentBlock.Location.SourceTree) element = null; @@ -95,7 +95,7 @@ private static void EnsureSameFile(ICommentBlock commentBlock, ref KeyValuePair< /// The parent element of the comment block. /// Output binding information. private void GenerateBindings( - ICommentBlock commentBlock, + Comments.CommentBlock commentBlock, KeyValuePair? previousElement, KeyValuePair? nextElement, KeyValuePair? parentElement, @@ -231,7 +231,7 @@ public void Push(KeyValuePair value) // Generate binding information for one CommentBlock. private void GenerateBindings( - ICommentBlock block, + Comments.CommentBlock block, ElementStack elementStack, KeyValuePair? nextElement, CommentBindingCallback cb @@ -259,25 +259,25 @@ CommentBindingCallback cb /// Where to send the results. /// true if there are more comments to process, false otherwise. private bool GenerateBindings( - IEnumerator> commentEnumerator, + IEnumerator> commentEnumerator, KeyValuePair? nextElement, ElementStack elementStack, CommentBindingCallback cb ) { - CommentBlock? block = null; + Comments.CommentBlock block = null; // Iterate comments until the commentEnumerator has gone past nextElement while (nextElement == null || Compare(commentEnumerator.Current.Value.Location, nextElement.Value.Key) < 0) { if (block is null) - block = new CommentBlock(commentEnumerator.Current.Value); + block = new Comments.CommentBlock(commentEnumerator.Current.Value); if (!block.CombinesWith(commentEnumerator.Current.Value)) { // Start of a new block, so generate the bindings for the old block first. GenerateBindings(block, elementStack, nextElement, cb); - block = new CommentBlock(commentEnumerator.Current.Value); + block = new Comments.CommentBlock(commentEnumerator.Current.Value); } else { @@ -320,7 +320,7 @@ public void GenerateBindings(CommentBindingCallback cb) var elementStack = new ElementStack(); using IEnumerator> elementEnumerator = elements.GetEnumerator(); - using IEnumerator> commentEnumerator = comments.GetEnumerator(); + using IEnumerator> commentEnumerator = comments.GetEnumerator(); if (!commentEnumerator.MoveNext()) { // There are no comments to process. @@ -343,54 +343,12 @@ public void GenerateBindings(CommentBindingCallback cb) } } - internal class CommentBlock : ICommentBlock - { - private readonly List lines; - - public IEnumerable CommentLines => lines; - - public Location Location { get; private set; } - - public CommentBlock(ICommentLine firstLine) - { - lines = new List { firstLine }; - Location = firstLine.Location; - } - - /// - /// Determine whether commentlines should be merged. - /// - /// A comment line to be appended to this comment block. - /// Whether the new line should be appended to this block. - public bool CombinesWith(ICommentLine newLine) - { - if (!CommentLines.Any()) - return true; - - var sameFile = Location.SourceTree == newLine.Location.SourceTree; - var sameRow = Location.EndLine() == newLine.Location.StartLine(); - var sameColumn = Location.EndLine() + 1 == newLine.Location.StartLine(); - var nextRow = Location.StartColumn() == newLine.Location.StartColumn(); - var adjacent = sameFile && (sameRow || (sameColumn && nextRow)); - - return - newLine.Type == CommentLineType.MultilineContinuation || - adjacent; - } - - /// - /// Adds a comment line to the this comment block. - /// - /// The line to add. - public void AddCommentLine(ICommentLine line) - { - Location = !lines.Any() - ? line.Location - : Location.Create( - line.Location.SourceTree!, - new TextSpan(Location.SourceSpan.Start, line.Location.SourceSpan.End - Location.SourceSpan.Start)); - - lines.Add(line); - } - } + /// + /// Callback for generated comment associations. + /// + /// The label of the element + /// The duplication guard key of the element, if any + /// The comment block associated with the element + /// The relationship between the commentblock and the element + internal delegate void CommentBindingCallback(Label elementLabel, Key duplicationGuardKey, Comments.CommentBlock commentBlock, CommentBinding binding); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Context.cs b/csharp/extractor/Semmle.Extraction.CSharp/Context.cs new file mode 100644 index 000000000000..314706c1e479 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Context.cs @@ -0,0 +1,144 @@ +using Microsoft.CodeAnalysis; +using System; +using System.Diagnostics.CodeAnalysis; +using Semmle.Extraction.Entities; +using System.Collections.Generic; + +namespace Semmle.Extraction.CSharp +{ + /// + /// State that needs to be available throughout the extraction process. + /// There is one Context object per trap output file. + /// + internal class Context : Extraction.Context + { + /// + /// The program database provided by Roslyn. + /// There's one per syntax tree, which makes things awkward. + /// + public SemanticModel GetModel(SyntaxNode node) + { + // todo: when this context belongs to a SourceScope, the syntax tree can be retrieved from the scope, and + // the node parameter could be removed. Is there any case when we pass in a node that's not from the current + // tree? + if (cachedModel == null || node.SyntaxTree != cachedModel.SyntaxTree) + { + cachedModel = Compilation.GetSemanticModel(node.SyntaxTree); + } + + return cachedModel; + } + + private SemanticModel cachedModel; + + /// + /// The current compilation unit. + /// + public Compilation Compilation { get; } + + internal CommentProcessor CommentGenerator { get; } = new CommentProcessor(); + + public Context(Extraction.Extractor e, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix) + : base(e, trapWriter, addAssemblyTrapPrefix) + { + Compilation = c; + this.scope = scope; + } + + public bool FromSource => scope is SourceScope; + + private readonly IExtractionScope scope; + + public bool IsAssemblyScope => scope is AssemblyScope; + + private SyntaxTree SourceTree => scope is SourceScope sc ? sc.SourceTree : null; + + /// + /// Whether the given symbol needs to be defined in this context. + /// This is the case if the symbol is contained in the source/assembly, or + /// of the symbol is a constructed generic. + /// + /// The symbol to populate. + public bool Defines(ISymbol symbol) => + !SymbolEqualityComparer.Default.Equals(symbol, symbol.OriginalDefinition) || + scope.InScope(symbol); + + public override void WithDuplicationGuard(Key key, Action a) + { + if (IsAssemblyScope) + { + // No need for a duplication guard when extracting assemblies, + // and the duplication guard could lead to method bodies being missed + // depending on trap import order. + a(); + } + else + { + base.WithDuplicationGuard(key, a); + } + } + + public override Extraction.Entities.Location CreateLocation() + { + return SourceTree == null + ? GeneratedLocation.Create(this) + : CreateLocation(Microsoft.CodeAnalysis.Location.Create(SourceTree, Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(0, 0))); + } + + public override Extraction.Entities.Location CreateLocation(Microsoft.CodeAnalysis.Location location) + { + return (location == null || location.Kind == LocationKind.None) + ? GeneratedLocation.Create(this) + : location.IsInSource + ? Entities.NonGeneratedSourceLocation.Create(this, location) + : Entities.Assembly.Create(this, location); + } + + /// + /// Register a program entity which can be bound to comments. + /// + /// Extractor context. + /// Program entity. + /// Location of the entity. + public void BindComments(Entity entity, Microsoft.CodeAnalysis.Location l) + { + var duplicationGuardKey = GetCurrentTagStackKey(); + CommentGenerator.AddElement(entity.Label, duplicationGuardKey, l); + } + + protected override bool IsEntityDuplicationGuarded(IEntity entity, [NotNullWhen(true)] out Extraction.Entities.Location loc) + { + if (CreateLocation(entity.ReportingLocation) is Entities.NonGeneratedSourceLocation l) + { + loc = l; + return true; + } + + loc = null; + return false; + } + + private readonly HashSet