From 5fca946678978214259193778c5b321053182a0e Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Tue, 16 Feb 2021 22:09:01 +0100 Subject: [PATCH 1/4] C#: Split 'Context' class between CIL and source extraction --- .../Entities/Base/Tuple.cs | 2 +- .../Semmle.Extraction.CSharp/Analyser.cs | 4 +- .../CachedEntityFactory.cs | 21 +++ .../Comments/CommentBinding.cs | 13 ++ .../Comments/CommentBlock.cs | 59 ++++++++ .../Comments/CommentLineType.cs | 13 ++ .../Comments/CommentProcessor.cs} | 92 ++++-------- .../Semmle.Extraction.CSharp/Context.cs | 120 +++++++++++++++ .../Entities/Accessor.cs | 4 +- .../Entities/Assembly.cs | 18 ++- .../Entities/Attribute.cs | 4 +- .../Entities/CachedEntity.cs | 12 ++ .../Entities/{Symbol.cs => CachedSymbol.cs} | 7 +- .../Entities/CommentBlock.cs | 11 +- .../Entities/CommentLine.cs | 7 +- .../Entities/Compilations/Compilation.cs | 10 +- .../Entities/Constructor.cs | 4 +- .../Entities/Conversion.cs | 4 +- .../Entities/Destructor.cs | 4 +- .../Entities/Event.cs | 4 +- .../Entities/EventAccessor.cs | 4 +- .../Entities/Field.cs | 4 +- .../Semmle.Extraction.CSharp/Entities/File.cs | 83 +++++++++++ .../Entities/FreshEntity.cs | 13 ++ .../Entities/Indexer.cs | 4 +- .../Entities/LocalFunction.cs | 4 +- .../Entities/LocalVariable.cs | 4 +- .../Entities/Modifier.cs | 4 +- .../Entities/Namespace.cs | 4 +- .../Entities/NamespaceDeclaration.cs | 4 +- .../Entities/NonGeneratedSourceLocation.cs | 66 +++++++++ .../Entities/OrdinaryMethod.cs | 4 +- .../Entities/Parameter.cs | 16 +- .../PreprocessorDirectives/LineDirective.cs | 2 +- .../PragmaChecksumDirective.cs | 2 +- .../Entities/Property.cs | 4 +- .../Entities/Types/ArrayType.cs | 4 +- .../Entities/Types/DynamicType.cs | 4 +- .../Entities/Types/FunctionPointerType.cs | 4 +- .../Entities/Types/NamedType.cs | 12 +- .../Entities/Types/NullType.cs | 4 +- .../Entities/Types/Nullability.cs | 4 +- .../Entities/Types/PointerType.cs | 4 +- .../Entities/Types/TupleType.cs | 4 +- .../Entities/Types/Type.cs | 4 +- .../Entities/Types/TypeParameter.cs | 4 +- .../Entities/UserOperator.cs | 4 +- .../Populators/CommentPopulator.cs | 5 +- .../Populators/TypeContainerVisitor.cs | 2 +- .../Semmle.Extraction.CSharp/Tuples.cs | 16 +- .../extractor/Semmle.Extraction/Comments.cs | 105 -------------- csharp/extractor/Semmle.Extraction/Context.cs | 137 ++++-------------- ...ntityFactory.cs => CachedEntityFactory.cs} | 7 +- ...ns.cs => CachedEntityFactoryExtensions.cs} | 6 +- .../Semmle.Extraction/Entities/File.cs | 104 +------------ .../Semmle.Extraction/Entities/Folder.cs | 6 +- .../Entities/GeneratedFile.cs | 31 ++++ .../Entities/GeneratedLocation.cs | 6 +- .../Entities/SourceLocation.cs | 62 -------- csharp/extractor/Semmle.Extraction/Tuples.cs | 26 +--- 60 files changed, 624 insertions(+), 576 deletions(-) create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBinding.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBlock.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentLineType.cs rename csharp/extractor/{Semmle.Extraction/CommentProcessing.cs => Semmle.Extraction.CSharp/Comments/CommentProcessor.cs} (81%) create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Context.cs rename csharp/extractor/{Semmle.Extraction => Semmle.Extraction.CSharp}/Entities/Assembly.cs (76%) create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedEntity.cs rename csharp/extractor/Semmle.Extraction.CSharp/Entities/{Symbol.cs => CachedSymbol.cs} (97%) create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/FreshEntity.cs create mode 100644 csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs delete mode 100644 csharp/extractor/Semmle.Extraction/Comments.cs rename csharp/extractor/Semmle.Extraction/Entities/Base/{ICachedEntityFactory.cs => CachedEntityFactory.cs} (53%) rename csharp/extractor/Semmle.Extraction/Entities/Base/{ICachedEntityFactoryExtensions.cs => CachedEntityFactoryExtensions.cs} (88%) create mode 100644 csharp/extractor/Semmle.Extraction/Entities/GeneratedFile.cs diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs index 5657f072c9c8..9de3ecfaf621 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs @@ -14,7 +14,7 @@ public Tuple(string name, params object[] args) public void Extract(Context cx) { - cx.Cx.Emit(tuple); + cx.Cx.TrapWriter.Emit(tuple); } public override string ToString() => tuple.ToString(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs index b04b429993d7..cf4f4e0ebf00 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs @@ -292,7 +292,7 @@ private void DoAnalyseReferenceAssembly(PortableExecutableReference r) AnalyseNamespace(cx, module.GlobalNamespace); } - Entities.Attribute.ExtractAttributes(cx, assembly, Extraction.Entities.Assembly.Create(cx, assembly.GetSymbolLocation())); + Entities.Attribute.ExtractAttributes(cx, assembly, Entities.Assembly.Create(cx, assembly.GetSymbolLocation())); cx.PopulateAll(); } @@ -374,7 +374,7 @@ private void DoExtractTree(SyntaxTree tree) var cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree), AddAssemblyTrapPrefix); // Ensure that the file itself is populated in case the source file is totally empty var root = tree.GetRoot(); - Extraction.Entities.File.Create(cx, root.SyntaxTree.FilePath); + Entities.File.Create(cx, root.SyntaxTree.FilePath); var csNode = (CSharpSyntaxNode)root; csNode.Accept(new CompilationUnitVisitor(cx)); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs b/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs new file mode 100644 index 000000000000..648d90be67ff --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs @@ -0,0 +1,21 @@ +using Microsoft.CodeAnalysis; + +namespace Semmle.Extraction.CSharp +{ + /// + /// A factory for creating cached entities. + /// + public abstract class CachedEntityFactory + : Extraction.CachedEntityFactory where TEntity : CachedEntity + { + /// + /// Initializes the entity, but does not generate any trap code. + /// + public sealed override TEntity Create(Extraction.Context cx, TInit init) + { + return Create((Context)cx, init); + } + + public abstract TEntity Create(Context cx, TInit init); + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBinding.cs b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBinding.cs new file mode 100644 index 000000000000..22518ba49301 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBinding.cs @@ -0,0 +1,13 @@ +namespace Semmle.Extraction.CSharp +{ + /// + /// Describes the relationship between a comment and a program element. + /// + public enum CommentBinding + { + Parent, // The parent element of a comment + Best, // The most likely element associated with a comment + Before, // The element before the comment + After // The element after the comment + }; +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBlock.cs b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBlock.cs new file mode 100644 index 000000000000..c7bb7c27008d --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentBlock.cs @@ -0,0 +1,59 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; +using Semmle.Extraction.CSharp.Entities; +using System.Collections.Generic; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Comments +{ + internal class CommentBlock + { + private readonly List lines; + + public IEnumerable CommentLines => lines; + + public Location Location { get; private set; } + + public CommentBlock(CommentLine firstLine) + { + lines = new List { firstLine }; + Location = firstLine.Location; + } + + /// + /// Determine whether commentlines should be merged. + /// + /// A comment line to be appended to this comment block. + /// Whether the new line should be appended to this block. + public bool CombinesWith(CommentLine newLine) + { + if (!CommentLines.Any()) + return true; + + var sameFile = Location.SourceTree == newLine.Location.SourceTree; + var sameRow = Location.EndLine() == newLine.Location.StartLine(); + var sameColumn = Location.EndLine() + 1 == newLine.Location.StartLine(); + var nextRow = Location.StartColumn() == newLine.Location.StartColumn(); + var adjacent = sameFile && (sameRow || (sameColumn && nextRow)); + + return + newLine.Type == CommentLineType.MultilineContinuation || + adjacent; + } + + /// + /// Adds a comment line to the this comment block. + /// + /// The line to add. + public void AddCommentLine(CommentLine line) + { + Location = !lines.Any() + ? line.Location + : Location.Create( + line.Location.SourceTree!, + new TextSpan(Location.SourceSpan.Start, line.Location.SourceSpan.End - Location.SourceSpan.Start)); + + lines.Add(line); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentLineType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentLineType.cs new file mode 100644 index 000000000000..8506ca701b3e --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentLineType.cs @@ -0,0 +1,13 @@ +namespace Semmle.Extraction.CSharp +{ + /// + /// The type of a single comment line. + /// + public enum CommentLineType + { + Singleline, // Comment starting // ... + XmlDoc, // Comment starting /// ... + Multiline, // Comment starting /* ..., even if the comment only spans one line. + MultilineContinuation // The second and subsequent lines of comment in a multiline comment. + }; +} diff --git a/csharp/extractor/Semmle.Extraction/CommentProcessing.cs b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentProcessor.cs similarity index 81% rename from csharp/extractor/Semmle.Extraction/CommentProcessing.cs rename to csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentProcessor.cs index 201ac7051361..0f99c228a330 100644 --- a/csharp/extractor/Semmle.Extraction/CommentProcessing.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Comments/CommentProcessor.cs @@ -1,32 +1,32 @@ using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.Text; +using Semmle.Extraction.CSharp.Entities; using Semmle.Util; using System.Collections.Generic; using System.Linq; -namespace Semmle.Extraction.CommentProcessing +namespace Semmle.Extraction.CSharp { /// /// Implements the comment processor for associating comments with program elements. /// Registers locations of comments and program elements, /// then generates binding information. /// - internal class CommentProcessor : ICommentGenerator + internal class CommentProcessor { - public void AddComment(ICommentLine comment) + public void AddComment(CommentLine comment) { comments[comment.Location] = comment; } // Comments sorted by location. - private readonly SortedDictionary comments = new SortedDictionary(new LocationComparer()); + private readonly SortedDictionary comments = new SortedDictionary(new LocationComparer()); // Program elements sorted by location. private readonly SortedDictionary elements = new SortedDictionary(new LocationComparer()); private readonly Dictionary duplicationGuardKeys = new Dictionary(); - private Key? GetDuplicationGuardKey(Label label) + private Key GetDuplicationGuardKey(Label label) { if (duplicationGuardKeys.TryGetValue(label, out var duplicationGuardKey)) return duplicationGuardKey; @@ -35,7 +35,7 @@ public void AddComment(ICommentLine comment) private class LocationComparer : IComparer { - public int Compare(Location? l1, Location? l2) => CommentProcessor.Compare(l1, l2); + public int Compare(Location l1, Location l2) => CommentProcessor.Compare(l1, l2); } /// @@ -44,7 +44,7 @@ private class LocationComparer : IComparer /// First location /// Second location /// <0 if l1 before l2, >0 if l1 after l2, else 0. - private static int Compare(Location? l1, Location? l2) + private static int Compare(Location l1, Location l2) { if (object.ReferenceEquals(l1, l2)) return 0; @@ -68,7 +68,7 @@ private static int Compare(Location? l1, Location? l2) /// The label of the element in the trap file. /// The duplication guard key of the element, if any. /// The location of the element. - public void AddElement(Label elementLabel, Key? duplicationGuardKey, Location loc) + public void AddElement(Label elementLabel, Key duplicationGuardKey, Location loc) { if (loc != null && loc.IsInSource) elements[loc] = elementLabel; @@ -78,7 +78,7 @@ public void AddElement(Label elementLabel, Key? duplicationGuardKey, Location lo // Ensure that commentBlock and element refer to the same file // which can happen when processing multiple files. - private static void EnsureSameFile(ICommentBlock commentBlock, ref KeyValuePair? element) + private static void EnsureSameFile(Comments.CommentBlock commentBlock, ref KeyValuePair? element) { if (element != null && element.Value.Key.SourceTree != commentBlock.Location.SourceTree) element = null; @@ -95,7 +95,7 @@ private static void EnsureSameFile(ICommentBlock commentBlock, ref KeyValuePair< /// The parent element of the comment block. /// Output binding information. private void GenerateBindings( - ICommentBlock commentBlock, + Comments.CommentBlock commentBlock, KeyValuePair? previousElement, KeyValuePair? nextElement, KeyValuePair? parentElement, @@ -231,7 +231,7 @@ public void Push(KeyValuePair value) // Generate binding information for one CommentBlock. private void GenerateBindings( - ICommentBlock block, + Comments.CommentBlock block, ElementStack elementStack, KeyValuePair? nextElement, CommentBindingCallback cb @@ -259,25 +259,25 @@ CommentBindingCallback cb /// Where to send the results. /// true if there are more comments to process, false otherwise. private bool GenerateBindings( - IEnumerator> commentEnumerator, + IEnumerator> commentEnumerator, KeyValuePair? nextElement, ElementStack elementStack, CommentBindingCallback cb ) { - CommentBlock? block = null; + Comments.CommentBlock block = null; // Iterate comments until the commentEnumerator has gone past nextElement while (nextElement == null || Compare(commentEnumerator.Current.Value.Location, nextElement.Value.Key) < 0) { if (block is null) - block = new CommentBlock(commentEnumerator.Current.Value); + block = new Comments.CommentBlock(commentEnumerator.Current.Value); if (!block.CombinesWith(commentEnumerator.Current.Value)) { // Start of a new block, so generate the bindings for the old block first. GenerateBindings(block, elementStack, nextElement, cb); - block = new CommentBlock(commentEnumerator.Current.Value); + block = new Comments.CommentBlock(commentEnumerator.Current.Value); } else { @@ -320,7 +320,7 @@ public void GenerateBindings(CommentBindingCallback cb) var elementStack = new ElementStack(); using IEnumerator> elementEnumerator = elements.GetEnumerator(); - using IEnumerator> commentEnumerator = comments.GetEnumerator(); + using IEnumerator> commentEnumerator = comments.GetEnumerator(); if (!commentEnumerator.MoveNext()) { // There are no comments to process. @@ -343,54 +343,12 @@ public void GenerateBindings(CommentBindingCallback cb) } } - internal class CommentBlock : ICommentBlock - { - private readonly List lines; - - public IEnumerable CommentLines => lines; - - public Location Location { get; private set; } - - public CommentBlock(ICommentLine firstLine) - { - lines = new List { firstLine }; - Location = firstLine.Location; - } - - /// - /// Determine whether commentlines should be merged. - /// - /// A comment line to be appended to this comment block. - /// Whether the new line should be appended to this block. - public bool CombinesWith(ICommentLine newLine) - { - if (!CommentLines.Any()) - return true; - - var sameFile = Location.SourceTree == newLine.Location.SourceTree; - var sameRow = Location.EndLine() == newLine.Location.StartLine(); - var sameColumn = Location.EndLine() + 1 == newLine.Location.StartLine(); - var nextRow = Location.StartColumn() == newLine.Location.StartColumn(); - var adjacent = sameFile && (sameRow || (sameColumn && nextRow)); - - return - newLine.Type == CommentLineType.MultilineContinuation || - adjacent; - } - - /// - /// Adds a comment line to the this comment block. - /// - /// The line to add. - public void AddCommentLine(ICommentLine line) - { - Location = !lines.Any() - ? line.Location - : Location.Create( - line.Location.SourceTree!, - new TextSpan(Location.SourceSpan.Start, line.Location.SourceSpan.End - Location.SourceSpan.Start)); - - lines.Add(line); - } - } + /// + /// Callback for generated comment associations. + /// + /// The label of the element + /// The duplication guard key of the element, if any + /// The comment block associated with the element + /// The relationship between the commentblock and the element + internal delegate void CommentBindingCallback(Label elementLabel, Key duplicationGuardKey, Comments.CommentBlock commentBlock, CommentBinding binding); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Context.cs b/csharp/extractor/Semmle.Extraction.CSharp/Context.cs new file mode 100644 index 000000000000..181795fbe910 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Context.cs @@ -0,0 +1,120 @@ +using Microsoft.CodeAnalysis; +using System; +using System.Diagnostics.CodeAnalysis; +using Semmle.Extraction.Entities; + +namespace Semmle.Extraction.CSharp +{ + /// + /// State that needs to be available throughout the extraction process. + /// There is one Context object per trap output file. + /// + public class Context : Extraction.Context + { + /// + /// The program database provided by Roslyn. + /// There's one per syntax tree, which makes things awkward. + /// + public SemanticModel GetModel(SyntaxNode node) + { + // todo: when this context belongs to a SourceScope, the syntax tree can be retrieved from the scope, and + // the node parameter could be removed. Is there any case when we pass in a node that's not from the current + // tree? + if (cachedModel == null || node.SyntaxTree != cachedModel.SyntaxTree) + { + cachedModel = Compilation.GetSemanticModel(node.SyntaxTree); + } + + return cachedModel; + } + + private SemanticModel cachedModel; + + /// + /// The current compilation unit. + /// + public Compilation Compilation { get; } + + internal CommentProcessor CommentGenerator { get; } = new CommentProcessor(); + + public Context(Extraction.Extractor e, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix) + : base(e, trapWriter, addAssemblyTrapPrefix) + { + Compilation = c; + this.scope = scope; + } + + public bool FromSource => scope is SourceScope; + + private readonly IExtractionScope scope; + + public bool IsAssemblyScope => scope is AssemblyScope; + + public SyntaxTree SourceTree => scope is SourceScope sc ? sc.SourceTree : null; + + /// + /// Whether the given symbol needs to be defined in this context. + /// This is the case if the symbol is contained in the source/assembly, or + /// of the symbol is a constructed generic. + /// + /// The symbol to populate. + public bool Defines(ISymbol symbol) => + !SymbolEqualityComparer.Default.Equals(symbol, symbol.OriginalDefinition) || + scope.InScope(symbol); + + public override void WithDuplicationGuard(Key key, Action a) + { + if (IsAssemblyScope) + { + // No need for a duplication guard when extracting assemblies, + // and the duplication guard could lead to method bodies being missed + // depending on trap import order. + a(); + } + else + { + base.WithDuplicationGuard(key, a); + } + } + + public override Extraction.Entities.Location CreateLocation() + { + return SourceTree == null + ? GeneratedLocation.Create(this) + : CreateLocation(Microsoft.CodeAnalysis.Location.Create(SourceTree, Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(0, 0))); + } + + public override Extraction.Entities.Location CreateLocation(Microsoft.CodeAnalysis.Location location) + { + return (location == null || location.Kind == LocationKind.None) + ? GeneratedLocation.Create(this) + : location.IsInSource + ? Entities.NonGeneratedSourceLocation.Create(this, location) + : Entities.Assembly.Create(this, location); + } + + /// + /// Register a program entity which can be bound to comments. + /// + /// Extractor context. + /// Program entity. + /// Location of the entity. + public void BindComments(Entity entity, Microsoft.CodeAnalysis.Location l) + { + var duplicationGuardKey = GetCurrentTagStackKey(); + CommentGenerator.AddElement(entity.Label, duplicationGuardKey, l); + } + + protected override bool IsEntityDuplicationGuarded(IEntity entity, [NotNullWhen(true)] out Extraction.Entities.Location loc) + { + if (CreateLocation(entity.ReportingLocation) is Entities.NonGeneratedSourceLocation l) + { + loc = l; + return true; + } + + loc = null; + return false; + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs index d5a20e8eef67..910e7dc2995c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Accessor.cs @@ -86,11 +86,11 @@ public override void Populate(TextWriter trapFile) public static new Accessor Create(Context cx, IMethodSymbol symbol) => AccessorFactory.Instance.CreateEntityFromSymbol(cx, symbol); - private class AccessorFactory : ICachedEntityFactory + private class AccessorFactory : CachedEntityFactory { public static AccessorFactory Instance { get; } = new AccessorFactory(); - public Accessor Create(Context cx, IMethodSymbol init) => new Accessor(cx, init); + public override Accessor Create(Context cx, IMethodSymbol init) => new Accessor(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction/Entities/Assembly.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs similarity index 76% rename from csharp/extractor/Semmle.Extraction/Entities/Assembly.cs rename to csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs index 42fe1f2b30b6..57ae72420d3b 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/Assembly.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs @@ -1,14 +1,18 @@ using Microsoft.CodeAnalysis; +using Semmle.Extraction.CSharp; using System.IO; -namespace Semmle.Extraction.Entities +namespace Semmle.Extraction.CSharp.Entities { - public class Assembly : Location + public class Assembly : Extraction.Entities.Location { + // todo: this can be changed to an override after the .NET 5 upgrade + private new Context Context => (Context)base.Context; + private readonly string assemblyPath; private readonly IAssemblySymbol assembly; - private Assembly(Context cx, Microsoft.CodeAnalysis.Location? init) + private Assembly(Context cx, Microsoft.CodeAnalysis.Location init) : base(cx, init) { if (init == null) @@ -40,7 +44,7 @@ public override void Populate(TextWriter trapFile) public override int GetHashCode() => Symbol == null ? 91187354 : Symbol.GetHashCode(); - public override bool Equals(object? obj) + public override bool Equals(object obj) { if (obj is Assembly other && other.GetType() == typeof(Assembly)) return Equals(Symbol, other.Symbol); @@ -48,13 +52,13 @@ public override bool Equals(object? obj) return false; } - public static Location Create(Context cx, Microsoft.CodeAnalysis.Location loc) => AssemblyConstructorFactory.Instance.CreateEntity(cx, loc, loc); + public static Extraction.Entities.Location Create(Context cx, Microsoft.CodeAnalysis.Location loc) => AssemblyConstructorFactory.Instance.CreateEntity(cx, loc, loc); - private class AssemblyConstructorFactory : ICachedEntityFactory + private class AssemblyConstructorFactory : CachedEntityFactory { public static AssemblyConstructorFactory Instance { get; } = new AssemblyConstructorFactory(); - public Assembly Create(Context cx, Microsoft.CodeAnalysis.Location? init) => new Assembly(cx, init); + public override Assembly Create(Context cx, Microsoft.CodeAnalysis.Location init) => new Assembly(cx, init); } private static readonly object outputAssemblyCacheKey = new object(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs index ba1d20937651..c82f35de2175 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs @@ -136,11 +136,11 @@ public static Attribute Create(Context cx, AttributeData attributeData, IEntity return AttributeFactory.Instance.CreateEntity(cx, attributeData, init); } - private class AttributeFactory : ICachedEntityFactory<(AttributeData attributeData, IEntity receiver), Attribute> + private class AttributeFactory : CachedEntityFactory<(AttributeData attributeData, IEntity receiver), Attribute> { public static readonly AttributeFactory Instance = new AttributeFactory(); - public Attribute Create(Context cx, (AttributeData attributeData, IEntity receiver) init) => + public override Attribute Create(Context cx, (AttributeData attributeData, IEntity receiver) init) => new Attribute(cx, init.attributeData, init.receiver); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedEntity.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedEntity.cs new file mode 100644 index 000000000000..3296880f698b --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedEntity.cs @@ -0,0 +1,12 @@ +namespace Semmle.Extraction.CSharp.Entities +{ + public abstract class CachedEntity : Extraction.CachedEntity + { + // todo: this can be changed to an override after the .NET 5 upgrade + protected new Context Context => (Context)base.Context; + + protected CachedEntity(Context context, T symbol) : base(context, symbol) + { + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Symbol.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs similarity index 97% rename from csharp/extractor/Semmle.Extraction.CSharp/Entities/Symbol.cs rename to csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs index 2ac5c6e90e5d..3be67e7587f7 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Symbol.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs @@ -11,8 +11,13 @@ namespace Semmle.Extraction.CSharp.Entities { public abstract class CachedSymbol : CachedEntity where T : ISymbol { + // todo: this can be changed to an override after the .NET 5 upgrade + protected new Context Context => (Context)base.Context; + protected CachedSymbol(Context cx, T init) - : base(cx, init) { } + : base(cx, init) + { + } public virtual Type ContainingType => Symbol.ContainingType != null ? Type.Create(Context, Symbol.ContainingType) : null; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs index e06b6aebe7f0..a3394885d1cc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs @@ -1,12 +1,11 @@ -using Semmle.Extraction.CommentProcessing; using Semmle.Extraction.Entities; using System.IO; namespace Semmle.Extraction.CSharp.Entities { - internal class CommentBlock : CachedEntity + internal class CommentBlock : CachedEntity { - private CommentBlock(Context cx, ICommentBlock init) + private CommentBlock(Context cx, Comments.CommentBlock init) : base(cx, init) { } public override void Populate(TextWriter trapFile) @@ -35,13 +34,13 @@ public void BindTo(Label entity, CommentBinding binding) Context.TrapWriter.Writer.commentblock_binding(this, entity, binding); } - public static CommentBlock Create(Context cx, ICommentBlock block) => CommentBlockFactory.Instance.CreateEntity(cx, block, block); + public static CommentBlock Create(Context cx, Comments.CommentBlock block) => CommentBlockFactory.Instance.CreateEntity(cx, block, block); - private class CommentBlockFactory : ICachedEntityFactory + private class CommentBlockFactory : CachedEntityFactory { public static CommentBlockFactory Instance { get; } = new CommentBlockFactory(); - public CommentBlock Create(Context cx, ICommentBlock init) => new CommentBlock(cx, init); + public override CommentBlock Create(Context cx, Comments.CommentBlock init) => new CommentBlock(cx, init); } public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs index 102ea262b397..27491792b7f2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs @@ -1,10 +1,9 @@ -using Semmle.Extraction.CommentProcessing; using Semmle.Extraction.Entities; using System.IO; namespace Semmle.Extraction.CSharp.Entities { - internal class CommentLine : CachedEntity<(Microsoft.CodeAnalysis.Location, string)>, ICommentLine + internal class CommentLine : CachedEntity<(Microsoft.CodeAnalysis.Location, string)> { private CommentLine(Context cx, Microsoft.CodeAnalysis.Location loc, CommentLineType type, string text, string raw) : base(cx, (loc, text)) @@ -44,11 +43,11 @@ internal static CommentLine Create(Context cx, Microsoft.CodeAnalysis.Location l return CommentLineFactory.Instance.CreateEntity(cx, init, init); } - private class CommentLineFactory : ICachedEntityFactory<(Microsoft.CodeAnalysis.Location, CommentLineType, string, string), CommentLine> + private class CommentLineFactory : CachedEntityFactory<(Microsoft.CodeAnalysis.Location, CommentLineType, string, string), CommentLine> { public static CommentLineFactory Instance { get; } = new CommentLineFactory(); - public CommentLine Create(Context cx, (Microsoft.CodeAnalysis.Location, CommentLineType, string, string) init) => + public override CommentLine Create(Context cx, (Microsoft.CodeAnalysis.Location, CommentLineType, string, string) init) => new CommentLine(cx, init.Item1, init.Item2, init.Item3, init.Item4); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs index 800e35d5ff42..f9c7357fe28b 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs @@ -31,7 +31,7 @@ private Compilation(Context cx) : base(cx, null) public override void Populate(TextWriter trapFile) { - var assembly = Extraction.Entities.Assembly.CreateOutputAssembly(Context); + var assembly = Assembly.CreateOutputAssembly(Context); trapFile.compilations(this, FileUtils.ConvertToUnix(Compilation.Settings.Cwd)); trapFile.compilation_assembly(this, assembly); @@ -45,7 +45,7 @@ public override void Populate(TextWriter trapFile) // Files index = 0; - foreach (var file in Context.Compilation.SyntaxTrees.Select(tree => Extraction.Entities.File.Create(Context, tree.FilePath))) + foreach (var file in Context.Compilation.SyntaxTrees.Select(tree => File.Create(Context, tree.FilePath))) { trapFile.compilation_compiling_files(this, index++, file); } @@ -54,7 +54,7 @@ public override void Populate(TextWriter trapFile) index = 0; foreach (var file in Context.Compilation.References .OfType() - .Select(r => Extraction.Entities.File.Create(Context, r.FilePath))) + .Select(r => File.Create(Context, r.FilePath))) { trapFile.compilation_referencing_files(this, index++, file); } @@ -90,11 +90,11 @@ public override void WriteId(TextWriter trapFile) public override bool NeedsPopulation => Context.IsAssemblyScope; - private class CompilationFactory : ICachedEntityFactory + private class CompilationFactory : CachedEntityFactory { public static CompilationFactory Instance { get; } = new CompilationFactory(); - public Compilation Create(Context cx, object init) => new Compilation(cx); + public override Compilation Create(Context cx, object init) => new Compilation(cx); } private static readonly object compilationCacheKey = new object(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs index d04e80324ff9..fc24bf2be614 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs @@ -145,11 +145,11 @@ public override Microsoft.CodeAnalysis.Location ReportingLocation } } - private class ConstructorFactory : ICachedEntityFactory + private class ConstructorFactory : CachedEntityFactory { public static ConstructorFactory Instance { get; } = new ConstructorFactory(); - public Constructor Create(Context cx, IMethodSymbol init) => new Constructor(cx, init); + public override Constructor Create(Context cx, IMethodSymbol init) => new Constructor(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs index 05a752cb35cb..4a245207bfee 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Conversion.cs @@ -26,11 +26,11 @@ public override Microsoft.CodeAnalysis.Location ReportingLocation } } - private class ConversionFactory : ICachedEntityFactory + private class ConversionFactory : CachedEntityFactory { public static ConversionFactory Instance { get; } = new ConversionFactory(); - public Conversion Create(Context cx, IMethodSymbol init) => new Conversion(cx, init); + public override Conversion Create(Context cx, IMethodSymbol init) => new Conversion(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs index 0862baa36008..2bf33e6a88ab 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Destructor.cs @@ -26,11 +26,11 @@ public override void Populate(TextWriter trapFile) public static new Destructor Create(Context cx, IMethodSymbol symbol) => DestructorFactory.Instance.CreateEntityFromSymbol(cx, symbol); - private class DestructorFactory : ICachedEntityFactory + private class DestructorFactory : CachedEntityFactory { public static DestructorFactory Instance { get; } = new DestructorFactory(); - public Destructor Create(Context cx, IMethodSymbol init) => new Destructor(cx, init); + public override Destructor Create(Context cx, IMethodSymbol init) => new Destructor(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs index fe095ed44625..ad86c26d2816 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs @@ -65,11 +65,11 @@ public override void Populate(TextWriter trapFile) public static Event Create(Context cx, IEventSymbol symbol) => EventFactory.Instance.CreateEntityFromSymbol(cx, symbol); - private class EventFactory : ICachedEntityFactory + private class EventFactory : CachedEntityFactory { public static EventFactory Instance { get; } = new EventFactory(); - public Event Create(Context cx, IEventSymbol init) => new Event(cx, init); + public override Event Create(Context cx, IEventSymbol init) => new Event(cx, init); } public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs index 82be6a6e406a..2ee2a0ebc086 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/EventAccessor.cs @@ -55,11 +55,11 @@ public override void Populate(TextWriter trapFile) public static new EventAccessor Create(Context cx, IMethodSymbol symbol) => EventAccessorFactory.Instance.CreateEntityFromSymbol(cx, symbol); - private class EventAccessorFactory : ICachedEntityFactory + private class EventAccessorFactory : CachedEntityFactory { public static EventAccessorFactory Instance { get; } = new EventAccessorFactory(); - public EventAccessor Create(Context cx, IMethodSymbol init) => new EventAccessor(cx, init); + public override EventAccessor Create(Context cx, IMethodSymbol init) => new EventAccessor(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs index 1c435ebd7d16..59cb0a9d6b1e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs @@ -130,11 +130,11 @@ public override void WriteId(TextWriter trapFile) bool IExpressionParentEntity.IsTopLevelParent => true; - private class FieldFactory : ICachedEntityFactory + private class FieldFactory : CachedEntityFactory { public static FieldFactory Instance { get; } = new FieldFactory(); - public Field Create(Context cx, IFieldSymbol init) => new Field(cx, init); + public override Field Create(Context cx, IFieldSymbol init) => new Field(cx, init); } public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.PushesLabel; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs new file mode 100644 index 000000000000..6c0c33ff3923 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/File.cs @@ -0,0 +1,83 @@ +using Microsoft.CodeAnalysis; +using Semmle.Util; +using System; +using System.IO; +using System.Linq; + +namespace Semmle.Extraction.CSharp.Entities +{ + public class File : Extraction.Entities.File + { + // todo: this can be changed to an override after the .NET 5 upgrade + private new Context Context => (Context)base.Context; + + protected File(Context cx, string path) + : base(cx, path) + { + } + + public override void Populate(TextWriter trapFile) + { + trapFile.files(this, TransformedPath.Value, TransformedPath.NameWithoutExtension, TransformedPath.Extension); + + if (TransformedPath.ParentDirectory is PathTransformer.ITransformedPath dir) + trapFile.containerparent(Extraction.Entities.Folder.Create(Context, dir), this); + + var trees = Context.Compilation.SyntaxTrees.Where(t => t.FilePath == originalPath); + + if (trees.Any()) + { + foreach (var text in trees.Select(tree => tree.GetText())) + { + var rawText = text.ToString() ?? ""; + var lineCounts = LineCounter.ComputeLineCounts(rawText); + if (rawText.Length > 0 && rawText[rawText.Length - 1] != '\n') + lineCounts.Total++; + + trapFile.numlines(this, lineCounts); + Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default); + } + } + else if (IsPossiblyTextFile()) + { + try + { + System.Text.Encoding encoding; + var lineCount = 0; + using (var sr = new StreamReader(originalPath, detectEncodingFromByteOrderMarks: true)) + { + while (sr.ReadLine() != null) + { + lineCount++; + } + encoding = sr.CurrentEncoding; + } + + trapFile.numlines(this, new LineCounts() { Total = lineCount, Code = 0, Comment = 0 }); + Context.TrapWriter.Archive(originalPath, TransformedPath, encoding ?? System.Text.Encoding.Default); + } + catch (Exception exc) + { + Context.ExtractionError($"Couldn't read file: {originalPath}. {exc.Message}", null, null, exc.StackTrace); + } + } + + trapFile.file_extraction_mode(this, Context.Extractor.Standalone ? 1 : 0); + } + + private bool IsPossiblyTextFile() + { + var extension = TransformedPath.Extension.ToLowerInvariant(); + return !extension.Equals("dll") && !extension.Equals("exe"); + } + + public static File Create(Context cx, string path) => FileFactory.Instance.CreateEntity(cx, (typeof(File), path), path); + + private class FileFactory : CachedEntityFactory + { + public static FileFactory Instance { get; } = new FileFactory(); + + public override File Create(Context cx, string init) => new File(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/FreshEntity.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/FreshEntity.cs new file mode 100644 index 000000000000..3776dc9552aa --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/FreshEntity.cs @@ -0,0 +1,13 @@ +namespace Semmle.Extraction.CSharp.Entities +{ + public abstract class FreshEntity : Extraction.FreshEntity + { + // todo: this can be changed to an override after the .NET 5 upgrade + protected new Context Context => (Context)base.Context; + + protected FreshEntity(Context cx) + : base(cx) + { + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs index 04bc319c93e1..4ee0301ee35e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs @@ -98,11 +98,11 @@ public override Microsoft.CodeAnalysis.Location FullLocation bool IExpressionParentEntity.IsTopLevelParent => true; - private class IndexerFactory : ICachedEntityFactory + private class IndexerFactory : CachedEntityFactory { public static IndexerFactory Instance { get; } = new IndexerFactory(); - public Indexer Create(Context cx, IPropertySymbol init) => new Indexer(cx, init); + public override Indexer Create(Context cx, IPropertySymbol init) => new Indexer(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs index efc50f7f85f1..a94401f2a933 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalFunction.cs @@ -22,11 +22,11 @@ public override void WriteQuotedId(TextWriter trapFile) public static new LocalFunction Create(Context cx, IMethodSymbol field) => LocalFunctionFactory.Instance.CreateEntityFromSymbol(cx, field); - private class LocalFunctionFactory : ICachedEntityFactory + private class LocalFunctionFactory : CachedEntityFactory { public static LocalFunctionFactory Instance { get; } = new LocalFunctionFactory(); - public LocalFunction Create(Context cx, IMethodSymbol init) => new LocalFunction(cx, init); + public override LocalFunction Create(Context cx, IMethodSymbol init) => new LocalFunction(cx, init); } public override void Populate(TextWriter trapFile) diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs index 4f9a0c4f7f6c..4317364b48cd 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs @@ -53,11 +53,11 @@ private void DefineConstantValue(TextWriter trapFile) } } - private class LocalVariableFactory : ICachedEntityFactory + private class LocalVariableFactory : CachedEntityFactory { public static LocalVariableFactory Instance { get; } = new LocalVariableFactory(); - public LocalVariable Create(Context cx, ISymbol init) => new LocalVariable(cx, init); + public override LocalVariable Create(Context cx, ISymbol init) => new LocalVariable(cx, init); } public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NeedsLabel; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs index b644558f799e..aac42b56f6ea 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs @@ -140,11 +140,11 @@ public static Modifier Create(Context cx, Accessibility access) return ModifierFactory.Instance.CreateEntity(cx, (typeof(Modifier), modifier), modifier); } - private class ModifierFactory : ICachedEntityFactory + private class ModifierFactory : CachedEntityFactory { public static ModifierFactory Instance { get; } = new ModifierFactory(); - public Modifier Create(Context cx, string init) => new Modifier(cx, init); + public override Modifier Create(Context cx, string init) => new Modifier(cx, init); } public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs index 31e66656eb37..497fab17eedc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Namespace.cs @@ -36,11 +36,11 @@ public override void WriteId(TextWriter trapFile) public static Namespace Create(Context cx, INamespaceSymbol ns) => NamespaceFactory.Instance.CreateEntityFromSymbol(cx, ns); - private class NamespaceFactory : ICachedEntityFactory + private class NamespaceFactory : CachedEntityFactory { public static NamespaceFactory Instance { get; } = new NamespaceFactory(); - public Namespace Create(Context cx, INamespaceSymbol init) => new Namespace(cx, init); + public override Namespace Create(Context cx, INamespaceSymbol init) => new Namespace(cx, init); } public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs index 3a19417008a1..43c6a65bdb17 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs @@ -52,11 +52,11 @@ public static NamespaceDeclaration Create(Context cx, NamespaceDeclarationSyntax return NamespaceDeclarationFactory.Instance.CreateEntity(cx, decl, init); } - private class NamespaceDeclarationFactory : ICachedEntityFactory<(NamespaceDeclarationSyntax decl, NamespaceDeclaration parent), NamespaceDeclaration> + private class NamespaceDeclarationFactory : CachedEntityFactory<(NamespaceDeclarationSyntax decl, NamespaceDeclaration parent), NamespaceDeclaration> { public static readonly NamespaceDeclarationFactory Instance = new NamespaceDeclarationFactory(); - public NamespaceDeclaration Create(Context cx, (NamespaceDeclarationSyntax decl, NamespaceDeclaration parent) init) => + public override NamespaceDeclaration Create(Context cx, (NamespaceDeclarationSyntax decl, NamespaceDeclaration parent) init) => new NamespaceDeclaration(cx, init.decl, init.parent); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs new file mode 100644 index 000000000000..ee62be6f4055 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NonGeneratedSourceLocation.cs @@ -0,0 +1,66 @@ +using Microsoft.CodeAnalysis; +using System.IO; + +namespace Semmle.Extraction.CSharp.Entities +{ + public class NonGeneratedSourceLocation : Extraction.Entities.SourceLocation + { + // todo: this can be changed to an override after the .NET 5 upgrade + private new Context Context => (Context)base.Context; + + protected NonGeneratedSourceLocation(Context cx, Location init) + : base(cx, init) + { + Position = init.GetLineSpan(); + FileEntity = File.Create(Context, Position.Path); + } + + public static NonGeneratedSourceLocation Create(Context cx, Location loc) => SourceLocationFactory.Instance.CreateEntity(cx, loc, loc); + + public override void Populate(TextWriter trapFile) + { + trapFile.locations_default(this, FileEntity, + Position.Span.Start.Line + 1, Position.Span.Start.Character + 1, + Position.Span.End.Line + 1, Position.Span.End.Character); + + var mapped = Symbol!.GetMappedLineSpan(); + if (mapped.HasMappedPath && mapped.IsValid) + { + var mappedLoc = Create(Context, Location.Create(mapped.Path, default, mapped.Span)); + + trapFile.locations_mapped(this, mappedLoc); + } + } + + public FileLinePositionSpan Position + { + get; + } + + public File FileEntity + { + get; + } + + public override void WriteId(TextWriter trapFile) + { + trapFile.Write("loc,"); + trapFile.WriteSubId(FileEntity); + trapFile.Write(','); + trapFile.Write(Position.Span.Start.Line + 1); + trapFile.Write(','); + trapFile.Write(Position.Span.Start.Character + 1); + trapFile.Write(','); + trapFile.Write(Position.Span.End.Line + 1); + trapFile.Write(','); + trapFile.Write(Position.Span.End.Character); + } + + private class SourceLocationFactory : CachedEntityFactory + { + public static SourceLocationFactory Instance { get; } = new SourceLocationFactory(); + + public override NonGeneratedSourceLocation Create(Context cx, Location init) => new NonGeneratedSourceLocation(cx, init); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs index 90f17d70ea41..bd10b75aba45 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs @@ -55,11 +55,11 @@ public override void Populate(TextWriter trapFile) public static new OrdinaryMethod Create(Context cx, IMethodSymbol method) => OrdinaryMethodFactory.Instance.CreateEntityFromSymbol(cx, method); - private class OrdinaryMethodFactory : ICachedEntityFactory + private class OrdinaryMethodFactory : CachedEntityFactory { public static OrdinaryMethodFactory Instance { get; } = new OrdinaryMethodFactory(); - public OrdinaryMethod Create(Context cx, IMethodSymbol init) => new OrdinaryMethod(cx, init); + public override OrdinaryMethod Create(Context cx, IMethodSymbol init) => new OrdinaryMethod(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs index b1b7a5444fec..7ca9ce06c375 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs @@ -177,11 +177,11 @@ private static EqualsValueClauseSyntax GetParameterDefaultValue(IParameterSymbol return syntax?.Default; } - private class ParameterFactory : ICachedEntityFactory<(IParameterSymbol, IEntity, Parameter), Parameter> + private class ParameterFactory : CachedEntityFactory<(IParameterSymbol, IEntity, Parameter), Parameter> { public static ParameterFactory Instance { get; } = new ParameterFactory(); - public Parameter Create(Context cx, (IParameterSymbol, IEntity, Parameter) init) => new Parameter(cx, init.Item1, init.Item2, init.Item3); + public override Parameter Create(Context cx, (IParameterSymbol, IEntity, Parameter) init) => new Parameter(cx, init.Item1, init.Item2, init.Item3); } public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.OptionalLabel; @@ -218,11 +218,11 @@ public override bool Equals(object obj) public static VarargsType Create(Context cx) => VarargsTypeFactory.Instance.CreateEntity(cx, typeof(VarargsType), null); - private class VarargsTypeFactory : ICachedEntityFactory + private class VarargsTypeFactory : CachedEntityFactory { public static VarargsTypeFactory Instance { get; } = new VarargsTypeFactory(); - public VarargsType Create(Context cx, string init) => new VarargsType(cx); + public override VarargsType Create(Context cx, string init) => new VarargsType(cx); } } @@ -253,11 +253,11 @@ public override bool Equals(object obj) public static VarargsParam Create(Context cx, Method method) => VarargsParamFactory.Instance.CreateEntity(cx, typeof(VarargsParam), method); - private class VarargsParamFactory : ICachedEntityFactory + private class VarargsParamFactory : CachedEntityFactory { public static VarargsParamFactory Instance { get; } = new VarargsParamFactory(); - public VarargsParam Create(Context cx, Method init) => new VarargsParam(cx, init); + public override VarargsParam Create(Context cx, Method init) => new VarargsParam(cx, init); } } @@ -281,11 +281,11 @@ public override void Populate(TextWriter trapFile) public static ConstructedExtensionParameter Create(Context cx, Method method, Parameter parameter) => ExtensionParamFactory.Instance.CreateEntity(cx, (new SymbolEqualityWrapper(parameter.Symbol), new SymbolEqualityWrapper(method.Symbol.ReceiverType)), (method, parameter)); - private class ExtensionParamFactory : ICachedEntityFactory<(Method, Parameter), ConstructedExtensionParameter> + private class ExtensionParamFactory : CachedEntityFactory<(Method, Parameter), ConstructedExtensionParameter> { public static ExtensionParamFactory Instance { get; } = new ExtensionParamFactory(); - public ConstructedExtensionParameter Create(Context cx, (Method, Parameter) init) => + public override ConstructedExtensionParameter Create(Context cx, (Method, Parameter) init) => new ConstructedExtensionParameter(cx, init.Item1, init.Item2); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineDirective.cs index 470f54f379a5..a5af19ab5098 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineDirective.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/LineDirective.cs @@ -31,7 +31,7 @@ protected override void PopulatePreprocessor(TextWriter trapFile) if (!string.IsNullOrWhiteSpace(trivia.File.ValueText)) { - var file = Extraction.Entities.File.Create(Context, trivia.File.ValueText); + var file = File.Create(Context, trivia.File.ValueText); trapFile.directive_line_file(this, file); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs index 572b77e2c73c..caa77ceec337 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PragmaChecksumDirective.cs @@ -12,7 +12,7 @@ public PragmaChecksumDirective(Context cx, PragmaChecksumDirectiveTriviaSyntax t protected override void PopulatePreprocessor(TextWriter trapFile) { - var file = Extraction.Entities.File.Create(Context, trivia.File.ValueText); + var file = File.Create(Context, trivia.File.ValueText); trapFile.pragma_checksums(this, file, trivia.Guid.ToString(), trivia.Bytes.ToString()); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs index 085047d0812d..5bef8bdcbf93 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs @@ -124,11 +124,11 @@ public static Property Create(Context cx, IPropertySymbol prop) return isIndexer ? Indexer.Create(cx, prop) : PropertyFactory.Instance.CreateEntityFromSymbol(cx, prop); } - private class PropertyFactory : ICachedEntityFactory + private class PropertyFactory : CachedEntityFactory { public static PropertyFactory Instance { get; } = new PropertyFactory(); - public Property Create(Context cx, IPropertySymbol init) => new Property(cx, init); + public override Property Create(Context cx, IPropertySymbol init) => new Property(cx, init); } public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.PushesLabel; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs index 62a10396b4bd..483814304287 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/ArrayType.cs @@ -40,11 +40,11 @@ public override void WriteId(TextWriter trapFile) public static ArrayType Create(Context cx, IArrayTypeSymbol symbol) => ArrayTypeFactory.Instance.CreateEntityFromSymbol(cx, symbol); - private class ArrayTypeFactory : ICachedEntityFactory + private class ArrayTypeFactory : CachedEntityFactory { public static ArrayTypeFactory Instance { get; } = new ArrayTypeFactory(); - public ArrayType Create(Context cx, IArrayTypeSymbol init) => new ArrayType(cx, init); + public override ArrayType Create(Context cx, IArrayTypeSymbol init) => new ArrayType(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs index 2516afe037bc..741ee0f7cb0c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs @@ -27,11 +27,11 @@ public override void WriteId(TextWriter trapFile) trapFile.Write("dynamic;type"); } - private class DynamicTypeFactory : ICachedEntityFactory + private class DynamicTypeFactory : CachedEntityFactory { public static DynamicTypeFactory Instance { get; } = new DynamicTypeFactory(); - public DynamicType Create(Context cx, IDynamicTypeSymbol init) => new DynamicType(cx, init); + public override DynamicType Create(Context cx, IDynamicTypeSymbol init) => new DynamicType(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/FunctionPointerType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/FunctionPointerType.cs index 903de837a733..2bb9cd3a1b66 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/FunctionPointerType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/FunctionPointerType.cs @@ -32,11 +32,11 @@ public override void Populate(TextWriter trapFile) public static FunctionPointerType Create(Context cx, IFunctionPointerTypeSymbol symbol) => FunctionPointerTypeFactory.Instance.CreateEntityFromSymbol(cx, symbol); - private class FunctionPointerTypeFactory : ICachedEntityFactory + private class FunctionPointerTypeFactory : CachedEntityFactory { public static FunctionPointerTypeFactory Instance { get; } = new FunctionPointerTypeFactory(); - public FunctionPointerType Create(Context cx, IFunctionPointerTypeSymbol init) => new FunctionPointerType(cx, init); + public override FunctionPointerType Create(Context cx, IFunctionPointerTypeSymbol init) => new FunctionPointerType(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs index 4c250e22692c..e9a8816b6e25 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs @@ -149,18 +149,18 @@ public override void WriteQuotedId(TextWriter trapFile) base.WriteQuotedId(trapFile); } - private class NamedTypeFactory : ICachedEntityFactory + private class NamedTypeFactory : CachedEntityFactory { public static NamedTypeFactory Instance { get; } = new NamedTypeFactory(); - public NamedType Create(Context cx, INamedTypeSymbol init) => new NamedType(cx, init, false); + public override NamedType Create(Context cx, INamedTypeSymbol init) => new NamedType(cx, init, false); } - private class UnderlyingTupleTypeFactory : ICachedEntityFactory + private class UnderlyingTupleTypeFactory : CachedEntityFactory { public static UnderlyingTupleTypeFactory Instance { get; } = new UnderlyingTupleTypeFactory(); - public NamedType Create(Context cx, INamedTypeSymbol init) => new NamedType(cx, init, true); + public override NamedType Create(Context cx, INamedTypeSymbol init) => new NamedType(cx, init, true); } // Do not create typerefs of constructed generics as they are always in the current trap file. @@ -186,11 +186,11 @@ public static NamedTypeRef Create(Context cx, INamedTypeSymbol type) => // `NamedType`s and `NamedTypeRef`s NamedTypeRefFactory.Instance.CreateEntity(cx, (typeof(NamedTypeRef), new SymbolEqualityWrapper(type)), type); - private class NamedTypeRefFactory : ICachedEntityFactory + private class NamedTypeRefFactory : CachedEntityFactory { public static NamedTypeRefFactory Instance { get; } = new NamedTypeRefFactory(); - public NamedTypeRef Create(Context cx, INamedTypeSymbol init) => new NamedTypeRef(cx, init); + public override NamedTypeRef Create(Context cx, INamedTypeSymbol init) => new NamedTypeRef(cx, init); } public override bool NeedsPopulation => true; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs index 33f7c44d21f9..584d64afbb92 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NullType.cs @@ -29,11 +29,11 @@ public override bool Equals(object obj) public static Type Create(Context cx) => NullTypeFactory.Instance.CreateEntity(cx, typeof(NullType), null); - private class NullTypeFactory : ICachedEntityFactory + private class NullTypeFactory : CachedEntityFactory { public static NullTypeFactory Instance { get; } = new NullTypeFactory(); - public NullType Create(Context cx, ITypeSymbol init) => new NullType(cx); + public override NullType Create(Context cx, ITypeSymbol init) => new NullType(cx); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Nullability.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Nullability.cs index a0d9e2a1f8c2..37278a7cf889 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Nullability.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Nullability.cs @@ -127,11 +127,11 @@ public override void WriteId(TextWriter trapFile) public static NullabilityEntity Create(Context cx, Nullability init) => NullabilityFactory.Instance.CreateEntity(cx, init, init); - private class NullabilityFactory : ICachedEntityFactory + private class NullabilityFactory : CachedEntityFactory { public static NullabilityFactory Instance { get; } = new NullabilityFactory(); - public NullabilityEntity Create(Context cx, Nullability init) => new NullabilityEntity(cx, init); + public override NullabilityEntity Create(Context cx, Nullability init) => new NullabilityEntity(cx, init); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs index dd9b749f53e7..635172a0e620 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/PointerType.cs @@ -31,11 +31,11 @@ public override void Populate(TextWriter trapFile) public static PointerType Create(Context cx, IPointerTypeSymbol symbol) => PointerTypeFactory.Instance.CreateEntityFromSymbol(cx, symbol); - private class PointerTypeFactory : ICachedEntityFactory + private class PointerTypeFactory : CachedEntityFactory { public static PointerTypeFactory Instance { get; } = new PointerTypeFactory(); - public PointerType Create(Context cx, IPointerTypeSymbol init) => new PointerType(cx, init); + public override PointerType Create(Context cx, IPointerTypeSymbol init) => new PointerType(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs index 18321f84d32f..a936481ccf52 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs @@ -15,11 +15,11 @@ internal class TupleType : Type { public static TupleType Create(Context cx, INamedTypeSymbol type) => TupleTypeFactory.Instance.CreateEntityFromSymbol(cx, type); - private class TupleTypeFactory : ICachedEntityFactory + private class TupleTypeFactory : CachedEntityFactory { public static TupleTypeFactory Instance { get; } = new TupleTypeFactory(); - public TupleType Create(Context cx, INamedTypeSymbol init) => new TupleType(cx, init); + public override TupleType Create(Context cx, INamedTypeSymbol init) => new TupleType(cx, init); } private TupleType(Context cx, INamedTypeSymbol init) : base(cx, init) diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs index 3931c6ef1d83..22a5f9e1284b 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs @@ -287,11 +287,11 @@ private DelegateTypeParameter(Context cx, IParameterSymbol init, IEntity parent, // `DelegateTypeParameter`s and `Parameter`s DelegateTypeParameterFactory.Instance.CreateEntity(cx, (typeof(DelegateTypeParameter), new SymbolEqualityWrapper(param)), (param, parent, original)); - private class DelegateTypeParameterFactory : ICachedEntityFactory<(IParameterSymbol, IEntity, Parameter), DelegateTypeParameter> + private class DelegateTypeParameterFactory : CachedEntityFactory<(IParameterSymbol, IEntity, Parameter), DelegateTypeParameter> { public static DelegateTypeParameterFactory Instance { get; } = new DelegateTypeParameterFactory(); - public DelegateTypeParameter Create(Context cx, (IParameterSymbol, IEntity, Parameter) init) => + public override DelegateTypeParameter Create(Context cx, (IParameterSymbol, IEntity, Parameter) init) => new DelegateTypeParameter(cx, init.Item1, init.Item2, init.Item3); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs index fc8ac7e39d80..1d52fb5a5764 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs @@ -127,11 +127,11 @@ public override void WriteId(TextWriter trapFile) trapFile.Write(kind); } - private class TypeParameterFactory : ICachedEntityFactory + private class TypeParameterFactory : CachedEntityFactory { public static TypeParameterFactory Instance { get; } = new TypeParameterFactory(); - public TypeParameter Create(Context cx, ITypeParameterSymbol init) => new TypeParameter(cx, init); + public override TypeParameter Create(Context cx, ITypeParameterSymbol init) => new TypeParameter(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs index 2643363773bf..e4ca68100175 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs @@ -185,11 +185,11 @@ public static string OperatorSymbol(Context cx, string methodName) public static new UserOperator Create(Context cx, IMethodSymbol symbol) => UserOperatorFactory.Instance.CreateEntityFromSymbol(cx, symbol); - private class UserOperatorFactory : ICachedEntityFactory + private class UserOperatorFactory : CachedEntityFactory { public static UserOperatorFactory Instance { get; } = new UserOperatorFactory(); - public UserOperator Create(Context cx, IMethodSymbol init) => new UserOperator(cx, init); + public override UserOperator Create(Context cx, IMethodSymbol init) => new UserOperator(cx, init); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/CommentPopulator.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/CommentPopulator.cs index 325746274ff4..eca18ebc1ef9 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Populators/CommentPopulator.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/CommentPopulator.cs @@ -1,6 +1,5 @@ using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; -using Semmle.Extraction.CommentProcessing; using Semmle.Extraction.CSharp.Entities; using System; @@ -9,9 +8,9 @@ namespace Semmle.Extraction.CSharp.Populators /// /// Populators for comments. /// - public static class CommentPopulator + internal static class CommentPopulator { - public static void ExtractCommentBlocks(Context cx, ICommentGenerator gen) + public static void ExtractCommentBlocks(Context cx, CommentProcessor gen) { cx.Try(null, null, () => { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Populators/TypeContainerVisitor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Populators/TypeContainerVisitor.cs index ededf2978b99..8dafeb393d66 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Populators/TypeContainerVisitor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Populators/TypeContainerVisitor.cs @@ -1,7 +1,7 @@ using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; -using Semmle.Extraction.Entities; +using Semmle.Extraction.CSharp.Entities; using System; using System.Collections.Generic; using System.IO; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs b/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs index df39d6a836c0..72beb7b6cde9 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs @@ -1,5 +1,4 @@ using Microsoft.CodeAnalysis.CSharp.Syntax; -using Semmle.Extraction.CommentProcessing; using Semmle.Extraction.CSharp.Entities; using Semmle.Extraction.CSharp.Entities.Expressions; using Semmle.Extraction.Entities; @@ -18,6 +17,11 @@ namespace Semmle.Extraction.CSharp /// internal static class Tuples { + internal static void assemblies(this System.IO.TextWriter trapFile, Assembly assembly, Extraction.Entities.File file, string identifier, string name, string version) + { + trapFile.WriteTuple("assemblies", assembly, file, identifier, name, version); + } + internal static void accessor_location(this TextWriter trapFile, Accessor accessorKey, Location location) { trapFile.WriteTuple("accessor_location", accessorKey, location); @@ -721,5 +725,15 @@ internal static void directive_define_symbols(this TextWriter trapFile, DefineSy { trapFile.WriteTuple("directive_define_symbols", symb, name); } + + internal static void locations_mapped(this System.IO.TextWriter trapFile, NonGeneratedSourceLocation l1, Location l2) + { + trapFile.WriteTuple("locations_mapped", l1, l2); + } + + internal static void file_extraction_mode(this System.IO.TextWriter trapFile, Entities.File file, int mode) + { + trapFile.WriteTuple("file_extraction_mode", file, mode); + } } } diff --git a/csharp/extractor/Semmle.Extraction/Comments.cs b/csharp/extractor/Semmle.Extraction/Comments.cs deleted file mode 100644 index 457d478ca4c5..000000000000 --- a/csharp/extractor/Semmle.Extraction/Comments.cs +++ /dev/null @@ -1,105 +0,0 @@ -using Microsoft.CodeAnalysis; -using System.Collections.Generic; - -namespace Semmle.Extraction.CommentProcessing -{ - /// - /// The type of a single comment line. - /// - public enum CommentLineType - { - Singleline, // Comment starting // ... - XmlDoc, // Comment starting /// ... - Multiline, // Comment starting /* ..., even if the comment only spans one line. - MultilineContinuation // The second and subsequent lines of comment in a multiline comment. - }; - - /// - /// Describes the relationship between a comment and a program element. - /// - public enum CommentBinding - { - Parent, // The parent element of a comment - Best, // The most likely element associated with a comment - Before, // The element before the comment - After // The element after the comment - }; - - /// - /// A single line in a comment. - /// - public interface ICommentLine - { - /// - /// The location of this comment line. - /// - Location Location { get; } - - /// - /// The type of this comment line. - /// - CommentLineType Type { get; } - - /// - /// The text body of this comment line, excluding comment delimiter and leading and trailing whitespace. - /// - string Text { get; } - - /// - /// Full text of the comment including leading/trailing whitespace and comment delimiters. - /// - string RawText { get; } - } - - /// - /// A block of comment lines combined into one unit. - /// - public interface ICommentBlock - { - /// - /// The full span of this comment block. - /// - Location Location { get; } - - /// - /// The individual lines in the comment. - /// - IEnumerable CommentLines { get; } - } - - /// - /// Callback for generated comment associations. - /// - /// The label of the element - /// The duplication guard key of the element, if any - /// The comment block associated with the element - /// The relationship between the commentblock and the element - public delegate void CommentBindingCallback(Label elementLabel, Key? duplicationGuardKey, ICommentBlock commentBlock, CommentBinding binding); - - /// - /// Computes the binding information between comments and program elements. - /// - public interface ICommentGenerator - { - /// - /// Registers the location of a program element to associate comments with. - /// This can be called in any order. - /// - /// Label of the element. - /// The duplication guard key of the element, if any. - /// Location of the element. - void AddElement(Label elementLabel, Key? duplicationGuardKey, Location location); - - /// - /// Registers a line of comment. - /// - /// The comment to register. - void AddComment(ICommentLine comment); - - /// - /// Computes the binding information and calls `cb` with all of the comment binding information. - /// - /// Receiver of the binding information. - void GenerateBindings(CommentBindingCallback cb); - } -} diff --git a/csharp/extractor/Semmle.Extraction/Context.cs b/csharp/extractor/Semmle.Extraction/Context.cs index a4b84d5d047e..f30ecdfcdfa7 100644 --- a/csharp/extractor/Semmle.Extraction/Context.cs +++ b/csharp/extractor/Semmle.Extraction/Context.cs @@ -1,9 +1,9 @@ using Microsoft.CodeAnalysis; -using Semmle.Extraction.CommentProcessing; using Semmle.Extraction.Entities; using Semmle.Util.Logging; using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; @@ -20,22 +20,6 @@ public class Context /// public Extractor Extractor { get; } - /// - /// The program database provided by Roslyn. - /// There's one per syntax tree, which makes things awkward. - /// - public SemanticModel GetModel(SyntaxNode node) - { - if (cachedModel == null || node.SyntaxTree != cachedModel.SyntaxTree) - { - cachedModel = Compilation.GetSemanticModel(node.SyntaxTree); - } - - return cachedModel; - } - - private SemanticModel? cachedModel; - /// /// Access to the trap file. /// @@ -88,11 +72,11 @@ private void CheckEntityHasUniqueLabel(string id, CachedEntity entity) public Label GetNewLabel() => new Label(GetNewId()); - public TEntity CreateEntity(ICachedEntityFactory factory, object cacheKey, TInit init) + public TEntity CreateEntity(CachedEntityFactory factory, object cacheKey, TInit init) where TEntity : CachedEntity => - cacheKey is ISymbol s ? CreateEntity(factory, s, init, symbolEntityCache) : CreateEntity(factory, cacheKey, init, objectEntityCache); + cacheKey is ISymbol s ? CreateEntity(factory, s, init, symbolEntityCache) : CreateEntity(factory, cacheKey, init, objectEntityCache); - public TEntity CreateEntityFromSymbol(ICachedEntityFactory factory, TSymbol init) + public TEntity CreateEntityFromSymbol(CachedEntityFactory factory, TSymbol init) where TSymbol : ISymbol where TEntity : CachedEntity => CreateEntity(factory, init, init, symbolEntityCache); @@ -104,7 +88,7 @@ public TEntity CreateEntityFromSymbol(ICachedEntityFactoryThe initializer for the entity. /// The dictionary to use for caching. /// The new/existing entity. - private TEntity CreateEntity(ICachedEntityFactory factory, TCacheKey cacheKey, TInit init, IDictionary dictionary) + private TEntity CreateEntity(CachedEntityFactory factory, TCacheKey cacheKey, TInit init, IDictionary dictionary) where TCacheKey : notnull where TEntity : CachedEntity { @@ -185,11 +169,11 @@ public void AddFreshLabel(Entity entity) /// The action to run. public void PopulateLater(Action a) { - if (tagStack.Count > 0) + var key = GetCurrentTagStackKey(); + if (key is object) { // If we are currently executing with a duplication guard, then the same // guard must be used for the deferred action - var key = tagStack.Peek(); populateQueue.Enqueue(() => WithDuplicationGuard(key, a)); } else @@ -220,45 +204,13 @@ public void PopulateAll() } } - /// - /// The current compilation unit. - /// - public Compilation Compilation { get; } - - public Context(Extractor e, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool addAssemblyTrapPrefix) + public Context(Extractor e, TrapWriter trapWriter, bool addAssemblyTrapPrefix = false) { Extractor = e; - Compilation = c; - this.scope = scope; TrapWriter = trapWriter; ShouldAddAssemblyTrapPrefix = addAssemblyTrapPrefix; } - public Context(Extractor e, TrapWriter trapWriter) - : this(e, null, trapWriter, null, false) - { - } - - public bool FromSource => scope is SourceScope; - - public ICommentGenerator CommentGenerator { get; } = new CommentProcessor(); - - private readonly IExtractionScope scope; - - public bool IsAssemblyScope => scope is AssemblyScope; - - public SyntaxTree? SourceTree => scope is SourceScope sc ? sc.SourceTree : null; - - /// - /// Whether the given symbol needs to be defined in this context. - /// This is the case if the symbol is contained in the source/assembly, or - /// of the symbol is a constructed generic. - /// - /// The symbol to populate. - public bool Defines(ISymbol symbol) => - !SymbolEqualityComparer.Default.Equals(symbol, symbol.OriginalDefinition) || - scope.InScope(symbol); - private int currentRecursiveDepth = 0; private const int maxRecursiveDepth = 150; @@ -361,7 +313,7 @@ public void Populate(ISymbol? optionalSymbol, CachedEntity entity) throw new InternalError("Unexpected TrapStackBehaviour"); } - var a = duplicationGuard && CreateLocation(entity.ReportingLocation) is NonGeneratedSourceLocation loc + var a = duplicationGuard && IsEntityDuplicationGuarded(entity, out var loc) //CreateLocation(entity.ReportingLocation) is NonGeneratedSourceLocation loc ? (Action)(() => WithDuplicationGuard(new Key(entity, loc), () => entity.Populate(TrapWriter.Writer))) : (Action)(() => this.Try(null, optionalSymbol, () => entity.Populate(TrapWriter.Writer))); @@ -371,46 +323,34 @@ public void Populate(ISymbol? optionalSymbol, CachedEntity entity) a(); } + protected virtual bool IsEntityDuplicationGuarded(IEntity entity, [NotNullWhen(returnValue: true)] out Entities.Location? loc) + { + loc = null; + return false; + } + /// /// Runs the given action , guarding for trap duplication /// based on key . /// - public void WithDuplicationGuard(Key key, Action a) + public virtual void WithDuplicationGuard(Key key, Action a) { - if (IsAssemblyScope) + tagStack.Push(key); + TrapWriter.Emit(new PushEmitter(key)); + try { - // No need for a duplication guard when extracting assemblies, - // and the duplication guard could lead to method bodies being missed - // depending on trap import order. a(); } - else + finally { - tagStack.Push(key); - TrapWriter.Emit(new PushEmitter(key)); - try - { - a(); - } - finally - { - TrapWriter.Emit(new PopEmitter()); - tagStack.Pop(); - } + TrapWriter.Emit(new PopEmitter()); + tagStack.Pop(); } } - /// - /// Register a program entity which can be bound to comments. - /// - /// Extractor context. - /// Program entity. - /// Location of the entity. - public void BindComments(Entity entity, Microsoft.CodeAnalysis.Location l) - { - var duplicationGuardKey = tagStack.Count > 0 ? tagStack.Peek() : null; - CommentGenerator.AddElement(entity.Label, duplicationGuardKey, l); - } + protected Key? GetCurrentTagStackKey() => tagStack.Count > 0 + ? tagStack.Peek() + : null; /// /// Log an extraction error. @@ -528,29 +468,10 @@ public void Try(SyntaxNode? node, ISymbol? symbol, Action a) } } - /// - /// Write the given tuple to the trap file. - /// - /// Tuple to write. - public void Emit(Tuple tuple) - { - TrapWriter.Emit(tuple); - } + public virtual Entities.Location CreateLocation() => + GeneratedLocation.Create(this); - public Entities.Location CreateLocation() - { - return SourceTree == null - ? GeneratedLocation.Create(this) - : CreateLocation(Microsoft.CodeAnalysis.Location.Create(SourceTree, Microsoft.CodeAnalysis.Text.TextSpan.FromBounds(0, 0))); - } - - public Entities.Location CreateLocation(Microsoft.CodeAnalysis.Location? location) - { - return (location == null || location.Kind == LocationKind.None) - ? GeneratedLocation.Create(this) - : location.IsInSource - ? NonGeneratedSourceLocation.Create(this, location) - : Assembly.Create(this, location); - } + public virtual Entities.Location CreateLocation(Microsoft.CodeAnalysis.Location? location) => + CreateLocation(); } } diff --git a/csharp/extractor/Semmle.Extraction/Entities/Base/ICachedEntityFactory.cs b/csharp/extractor/Semmle.Extraction/Entities/Base/CachedEntityFactory.cs similarity index 53% rename from csharp/extractor/Semmle.Extraction/Entities/Base/ICachedEntityFactory.cs rename to csharp/extractor/Semmle.Extraction/Entities/Base/CachedEntityFactory.cs index 7c03dd61002e..ccd01835c6f0 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/Base/ICachedEntityFactory.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/Base/CachedEntityFactory.cs @@ -1,14 +1,15 @@ +using Microsoft.CodeAnalysis; + namespace Semmle.Extraction { /// /// A factory for creating cached entities. /// - /// The type of the initializer. - public interface ICachedEntityFactory where TEntity : CachedEntity + public abstract class CachedEntityFactory where TEntity : CachedEntity { /// /// Initializes the entity, but does not generate any trap code. /// - TEntity Create(Context cx, TInit init); + public abstract TEntity Create(Context cx, TInit init); } } diff --git a/csharp/extractor/Semmle.Extraction/Entities/Base/ICachedEntityFactoryExtensions.cs b/csharp/extractor/Semmle.Extraction/Entities/Base/CachedEntityFactoryExtensions.cs similarity index 88% rename from csharp/extractor/Semmle.Extraction/Entities/Base/ICachedEntityFactoryExtensions.cs rename to csharp/extractor/Semmle.Extraction/Entities/Base/CachedEntityFactoryExtensions.cs index b311fccb7ab2..f8a08298cca6 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/Base/ICachedEntityFactoryExtensions.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/Base/CachedEntityFactoryExtensions.cs @@ -2,7 +2,7 @@ namespace Semmle.Extraction { - public static class ICachedEntityFactoryExtensions + public static class CachedEntityFactoryExtensions { /// /// Creates and populates a new entity, or returns the existing one from the cache, @@ -15,7 +15,7 @@ public static class ICachedEntityFactoryExtensions /// The key used for caching. /// The initializer for the entity. /// The entity. - public static TEntity CreateEntity(this ICachedEntityFactory factory, Context cx, object cacheKey, TInit init) + public static TEntity CreateEntity(this CachedEntityFactory factory, Context cx, object cacheKey, TInit init) where TEntity : CachedEntity => cx.CreateEntity(factory, cacheKey, init); /// @@ -28,7 +28,7 @@ public static TEntity CreateEntity(this ICachedEntityFactoryThe extractor context. /// The initializer for the entity. /// The entity. - public static TEntity CreateEntityFromSymbol(this ICachedEntityFactory factory, Context cx, TSymbol init) + public static TEntity CreateEntityFromSymbol(this CachedEntityFactory factory, Context cx, TSymbol init) where TSymbol : ISymbol where TEntity : CachedEntity => cx.CreateEntityFromSymbol(factory, init); } diff --git a/csharp/extractor/Semmle.Extraction/Entities/File.cs b/csharp/extractor/Semmle.Extraction/Entities/File.cs index 9ad618512a47..ed8881f0d2f8 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/File.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/File.cs @@ -1,126 +1,30 @@ -using Semmle.Util; using System; -using System.IO; -using System.Linq; namespace Semmle.Extraction.Entities { - public class File : CachedEntity + public abstract class File : CachedEntity { - private File(Context cx, string path) + protected File(Context cx, string path) : base(cx, path) { originalPath = path; transformedPathLazy = new Lazy(() => Context.Extractor.PathTransformer.Transform(originalPath)); } - private readonly string originalPath; + protected readonly string originalPath; private readonly Lazy transformedPathLazy; - private PathTransformer.ITransformedPath TransformedPath => transformedPathLazy.Value; + protected PathTransformer.ITransformedPath TransformedPath => transformedPathLazy.Value; public override bool NeedsPopulation => true; - public override void Populate(TextWriter trapFile) - { - trapFile.files(this, TransformedPath.Value, TransformedPath.NameWithoutExtension, TransformedPath.Extension); - - if (TransformedPath.ParentDirectory is PathTransformer.ITransformedPath dir) - trapFile.containerparent(Folder.Create(Context, dir), this); - - var trees = Context.Compilation.SyntaxTrees.Where(t => t.FilePath == originalPath); - - if (trees.Any()) - { - foreach (var text in trees.Select(tree => tree.GetText())) - { - var rawText = text.ToString() ?? ""; - var lineCounts = LineCounter.ComputeLineCounts(rawText); - if (rawText.Length > 0 && rawText[rawText.Length - 1] != '\n') - lineCounts.Total++; - - trapFile.numlines(this, lineCounts); - Context.TrapWriter.Archive(originalPath, TransformedPath, text.Encoding ?? System.Text.Encoding.Default); - } - } - else if (IsPossiblyTextFile()) - { - try - { - System.Text.Encoding encoding; - var lineCount = 0; - using (var sr = new StreamReader(originalPath, detectEncodingFromByteOrderMarks: true)) - { - while (sr.ReadLine() != null) - { - lineCount++; - } - encoding = sr.CurrentEncoding; - } - - trapFile.numlines(this, new LineCounts() { Total = lineCount, Code = 0, Comment = 0 }); - Context.TrapWriter.Archive(originalPath, TransformedPath, encoding ?? System.Text.Encoding.Default); - } - catch (Exception exc) - { - Context.ExtractionError($"Couldn't read file: {originalPath}. {exc.Message}", null, null, exc.StackTrace); - } - } - - trapFile.file_extraction_mode(this, Context.Extractor.Standalone ? 1 : 0); - } - - private bool IsPossiblyTextFile() - { - var extension = TransformedPath.Extension.ToLowerInvariant(); - return !extension.Equals("dll") && !extension.Equals("exe"); - } - public override void WriteId(System.IO.TextWriter trapFile) { trapFile.Write(TransformedPath.DatabaseId); trapFile.Write(";sourcefile"); } - public static File Create(Context cx, string path) => FileFactory.Instance.CreateEntity(cx, (typeof(File), path), path); - - public static File CreateGenerated(Context cx) => GeneratedFile.Create(cx); - - private class GeneratedFile : File - { - private GeneratedFile(Context cx) : base(cx, "") { } - - public override bool NeedsPopulation => true; - - public override void Populate(TextWriter trapFile) - { - trapFile.files(this, "", "", ""); - } - - public override void WriteId(TextWriter trapFile) - { - trapFile.Write("GENERATED;sourcefile"); - } - - public static GeneratedFile Create(Context cx) => - GeneratedFileFactory.Instance.CreateEntity(cx, typeof(GeneratedFile), null); - - private class GeneratedFileFactory : ICachedEntityFactory - { - public static GeneratedFileFactory Instance { get; } = new GeneratedFileFactory(); - - public GeneratedFile Create(Context cx, string? init) => new GeneratedFile(cx); - } - } - public override Microsoft.CodeAnalysis.Location? ReportingLocation => null; - private class FileFactory : ICachedEntityFactory - { - public static FileFactory Instance { get; } = new FileFactory(); - - public File Create(Context cx, string init) => new File(cx, init); - } - public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; } } diff --git a/csharp/extractor/Semmle.Extraction/Entities/Folder.cs b/csharp/extractor/Semmle.Extraction/Entities/Folder.cs index c84a6247d69e..82364c987e73 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/Folder.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/Folder.cs @@ -2,7 +2,7 @@ namespace Semmle.Extraction.Entities { - internal sealed class Folder : CachedEntity + public sealed class Folder : CachedEntity { private Folder(Context cx, PathTransformer.ITransformedPath init) : base(cx, init) { } @@ -26,11 +26,11 @@ public static Folder Create(Context cx, PathTransformer.ITransformedPath folder) public override Microsoft.CodeAnalysis.Location? ReportingLocation => null; - private class FolderFactory : ICachedEntityFactory + private class FolderFactory : CachedEntityFactory { public static FolderFactory Instance { get; } = new FolderFactory(); - public Folder Create(Context cx, PathTransformer.ITransformedPath init) => new Folder(cx, init); + public override Folder Create(Context cx, PathTransformer.ITransformedPath init) => new Folder(cx, init); } public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; diff --git a/csharp/extractor/Semmle.Extraction/Entities/GeneratedFile.cs b/csharp/extractor/Semmle.Extraction/Entities/GeneratedFile.cs new file mode 100644 index 000000000000..db458e574a57 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction/Entities/GeneratedFile.cs @@ -0,0 +1,31 @@ +using System.IO; + +namespace Semmle.Extraction.Entities +{ + internal class GeneratedFile : File + { + private GeneratedFile(Context cx) : base(cx, "") { } + + public override bool NeedsPopulation => true; + + public override void Populate(TextWriter trapFile) + { + trapFile.files(this, "", "", ""); + } + + public override void WriteId(TextWriter trapFile) + { + trapFile.Write("GENERATED;sourcefile"); + } + + public static GeneratedFile Create(Context cx) => + GeneratedFileFactory.Instance.CreateEntity(cx, typeof(GeneratedFile), null); + + private class GeneratedFileFactory : CachedEntityFactory + { + public static GeneratedFileFactory Instance { get; } = new GeneratedFileFactory(); + + public override GeneratedFile Create(Context cx, string? init) => new GeneratedFile(cx); + } + } +} diff --git a/csharp/extractor/Semmle.Extraction/Entities/GeneratedLocation.cs b/csharp/extractor/Semmle.Extraction/Entities/GeneratedLocation.cs index 6d05263aecd1..cb1d994597c9 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/GeneratedLocation.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/GeneratedLocation.cs @@ -9,7 +9,7 @@ public class GeneratedLocation : SourceLocation private GeneratedLocation(Context cx) : base(cx, null) { - generatedFile = File.CreateGenerated(cx); + generatedFile = GeneratedFile.Create(cx); } public override void Populate(TextWriter trapFile) @@ -30,11 +30,11 @@ public override void WriteId(TextWriter trapFile) public static GeneratedLocation Create(Context cx) => GeneratedLocationFactory.Instance.CreateEntity(cx, typeof(GeneratedLocation), null); - private class GeneratedLocationFactory : ICachedEntityFactory + private class GeneratedLocationFactory : CachedEntityFactory { public static GeneratedLocationFactory Instance { get; } = new GeneratedLocationFactory(); - public GeneratedLocation Create(Context cx, string? init) => new GeneratedLocation(cx); + public override GeneratedLocation Create(Context cx, string? init) => new GeneratedLocation(cx); } } } diff --git a/csharp/extractor/Semmle.Extraction/Entities/SourceLocation.cs b/csharp/extractor/Semmle.Extraction/Entities/SourceLocation.cs index 4e1726d6d079..d126f5521658 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/SourceLocation.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/SourceLocation.cs @@ -1,7 +1,3 @@ -using System; -using System.IO; -using Microsoft.CodeAnalysis; - namespace Semmle.Extraction.Entities { public abstract class SourceLocation : Location @@ -12,62 +8,4 @@ protected SourceLocation(Context cx, Microsoft.CodeAnalysis.Location? init) : ba public override bool NeedsPopulation => true; } - - public class NonGeneratedSourceLocation : SourceLocation - { - protected NonGeneratedSourceLocation(Context cx, Microsoft.CodeAnalysis.Location init) - : base(cx, init) - { - Position = init.GetLineSpan(); - FileEntity = File.Create(base.Context, Position.Path); - } - - public static Location Create(Context cx, Microsoft.CodeAnalysis.Location loc) => SourceLocationFactory.Instance.CreateEntity(cx, loc, loc); - - public override void Populate(TextWriter trapFile) - { - trapFile.locations_default(this, FileEntity, - Position.Span.Start.Line + 1, Position.Span.Start.Character + 1, - Position.Span.End.Line + 1, Position.Span.End.Character); - - var mapped = Symbol!.GetMappedLineSpan(); - if (mapped.HasMappedPath && mapped.IsValid) - { - var mappedLoc = Create(Context, Microsoft.CodeAnalysis.Location.Create(mapped.Path, default, mapped.Span)); - - trapFile.locations_mapped(this, mappedLoc); - } - } - - public FileLinePositionSpan Position - { - get; - } - - public File FileEntity - { - get; - } - - public override void WriteId(System.IO.TextWriter trapFile) - { - trapFile.Write("loc,"); - trapFile.WriteSubId(FileEntity); - trapFile.Write(','); - trapFile.Write(Position.Span.Start.Line + 1); - trapFile.Write(','); - trapFile.Write(Position.Span.Start.Character + 1); - trapFile.Write(','); - trapFile.Write(Position.Span.End.Line + 1); - trapFile.Write(','); - trapFile.Write(Position.Span.End.Character); - } - - private class SourceLocationFactory : ICachedEntityFactory - { - public static SourceLocationFactory Instance { get; } = new SourceLocationFactory(); - - public SourceLocation Create(Context cx, Microsoft.CodeAnalysis.Location init) => new NonGeneratedSourceLocation(cx, init); - } - } } diff --git a/csharp/extractor/Semmle.Extraction/Tuples.cs b/csharp/extractor/Semmle.Extraction/Tuples.cs index 8e9016cb6b68..49c14df76434 100644 --- a/csharp/extractor/Semmle.Extraction/Tuples.cs +++ b/csharp/extractor/Semmle.Extraction/Tuples.cs @@ -6,34 +6,24 @@ namespace Semmle.Extraction /// /// Methods for creating DB tuples. /// - internal static class Tuples + public static class Tuples { - public static void assemblies(this System.IO.TextWriter trapFile, Assembly assembly, File file, string identifier, string name, string version) - { - trapFile.WriteTuple("assemblies", assembly, file, identifier, name, version); - } - public static void containerparent(this System.IO.TextWriter trapFile, Folder parent, IEntity child) { trapFile.WriteTuple("containerparent", parent, child); } - public static void extractor_messages(this System.IO.TextWriter trapFile, ExtractionMessage error, Semmle.Util.Logging.Severity severity, string origin, string errorMessage, string entityText, Location location, string stackTrace) + internal static void extractor_messages(this System.IO.TextWriter trapFile, ExtractionMessage error, Semmle.Util.Logging.Severity severity, string origin, string errorMessage, string entityText, Location location, string stackTrace) { trapFile.WriteTuple("extractor_messages", error, (int)severity, origin, errorMessage, entityText, location, stackTrace); } - internal static void file_extraction_mode(this System.IO.TextWriter trapFile, Entities.File file, int mode) - { - trapFile.WriteTuple("file_extraction_mode", file, mode); - } - public static void files(this System.IO.TextWriter trapFile, File file, string fullName, string name, string extension) { trapFile.WriteTuple("files", file, fullName, name, extension, 0); } - public static void folders(this System.IO.TextWriter trapFile, Folder folder, string path, string name) + internal static void folders(this System.IO.TextWriter trapFile, Folder folder, string path, string name) { trapFile.WriteTuple("folders", folder, path, name); } @@ -42,15 +32,5 @@ public static void locations_default(this System.IO.TextWriter trapFile, SourceL { trapFile.WriteTuple("locations_default", label, file, startLine, startCol, endLine, endCol); } - - public static void locations_mapped(this System.IO.TextWriter trapFile, SourceLocation l1, Location l2) - { - trapFile.WriteTuple("locations_mapped", l1, l2); - } - - public static void numlines(this System.IO.TextWriter trapFile, IEntity label, LineCounts lineCounts) - { - trapFile.WriteTuple("numlines", label, lineCounts.Total, lineCounts.Code, lineCounts.Comment); - } } } From 539fdf952a823822cf90b5f2f0987d78631e7938 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 17 Feb 2021 09:51:00 +0100 Subject: [PATCH 2/4] Extend base context in CIL project --- .../Semmle.Extraction.CIL/Context.Factories.cs | 12 ++++++------ csharp/extractor/Semmle.Extraction.CIL/Context.cs | 10 ++++------ .../Semmle.Extraction.CIL/Entities/Assembly.cs | 13 ++++++------- .../Semmle.Extraction.CIL/Entities/Attribute.cs | 2 +- .../Entities/Base/LabelledEntity.cs | 5 ++--- .../Semmle.Extraction.CIL/Entities/Base/Tuple.cs | 2 +- .../Entities/Base/UnlabelledEntity.cs | 5 ++--- .../Entities/CustomAttributeDecoder.cs | 4 ++-- .../Semmle.Extraction.CIL/Entities/File.cs | 2 +- .../Semmle.Extraction.CIL/Entities/Instruction.cs | 2 +- .../Semmle.Extraction.CIL/Entities/PdbSourceFile.cs | 4 ++-- 11 files changed, 28 insertions(+), 33 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs index 36e420e36281..1017b2f1a19f 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs @@ -27,10 +27,10 @@ public T Populate(T e) where T : IExtractedEntity } else { - e.Label = Cx.GetNewLabel(); - Cx.DefineLabel(e, Cx.TrapWriter.Writer, Cx.Extractor); + e.Label = GetNewLabel(); + DefineLabel(e, TrapWriter.Writer, Extractor); ids.Add(e, e.Label); - Cx.PopulateLater(() => + PopulateLater(() => { foreach (var c in e.Contents) c.Extract(this); @@ -42,7 +42,7 @@ public T Populate(T e) where T : IExtractedEntity if (debugLabels.TryGetValue(id, out var previousEntity)) { - Cx.Extractor.Message(new Message("Duplicate trap ID", id, null, severity: Util.Logging.Severity.Warning)); + Extractor.Message(new Message("Duplicate trap ID", id, null, severity: Util.Logging.Severity.Warning)); } else { @@ -74,9 +74,9 @@ public PrimitiveType Create(PrimitiveTypeCode code) { e = new PrimitiveType(this, code) { - Label = Cx.GetNewLabel() + Label = GetNewLabel() }; - Cx.DefineLabel(e, Cx.TrapWriter.Writer, Cx.Extractor); + DefineLabel(e, TrapWriter.Writer, Extractor); primitiveTypes[(int)code] = e; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.cs index 99113714b7c2..a0d6ad1ab6f2 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.cs @@ -10,12 +10,10 @@ namespace Semmle.Extraction.CIL /// Adds additional context that is specific for CIL extraction. /// One context = one DLL/EXE. /// - public sealed partial class Context : IDisposable + public sealed partial class Context : Extraction.Context, IDisposable { private readonly FileStream stream; private Entities.Assembly? assemblyNull; - - public Extraction.Context Cx { get; } public MetadataReader MdReader { get; } public PEReader PeReader { get; } public string AssemblyPath { get; } @@ -26,9 +24,9 @@ public Entities.Assembly Assembly } public PDB.IPdb? Pdb { get; } - public Context(Extraction.Context cx, string assemblyPath, bool extractPdbs) + public Context(Extractor extractor, TrapWriter trapWriter, string assemblyPath, bool extractPdbs) + : base(extractor, trapWriter) { - this.Cx = cx; this.AssemblyPath = assemblyPath; stream = File.OpenRead(assemblyPath); PeReader = new PEReader(stream, PEStreamOptions.PrefetchEntireImage); @@ -51,7 +49,7 @@ public Context(Extraction.Context cx, string assemblyPath, bool extractPdbs) Pdb = PDB.PdbReader.Create(assemblyPath, PeReader); if (Pdb != null) { - cx.Extractor.Logger.Log(Util.Logging.Severity.Info, string.Format("Found PDB information for {0}", assemblyPath)); + Extractor.Logger.Log(Util.Logging.Severity.Info, string.Format("Found PDB information for {0}", assemblyPath)); } } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs index ba332b2db97f..feff55836c09 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs @@ -76,7 +76,7 @@ public override IEnumerable Contents } catch (InternalError e) { - Cx.Cx.ExtractionError("Error processing type definition", e.Message, GeneratedLocation.Create(Cx.Cx), e.StackTrace); + Cx.ExtractionError("Error processing type definition", e.Message, GeneratedLocation.Create(Cx), e.StackTrace); } // Limitation of C#: Cannot yield return inside a try-catch. @@ -93,7 +93,7 @@ public override IEnumerable Contents } catch (InternalError e) { - Cx.Cx.ExtractionError("Error processing bytecode", e.Message, GeneratedLocation.Create(Cx.Cx), e.StackTrace); + Cx.ExtractionError("Error processing bytecode", e.Message, GeneratedLocation.Create(Cx), e.StackTrace); } if (product != null) @@ -102,11 +102,11 @@ public override IEnumerable Contents } } - private static void ExtractCIL(Extraction.Context cx, string assemblyPath, bool extractPdbs) + private static void ExtractCIL(Extractor extractor, TrapWriter trapWriter, bool extractPdbs) { - using var cilContext = new Context(cx, assemblyPath, extractPdbs); + using var cilContext = new Context(extractor, trapWriter, extractor.OutputPath, extractPdbs); cilContext.Populate(new Assembly(cilContext)); - cilContext.Cx.PopulateAll(); + cilContext.PopulateAll(); } /// @@ -135,8 +135,7 @@ public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger trapFile = trapWriter.TrapFile; if (nocache || !System.IO.File.Exists(trapFile)) { - var cx = new Extraction.Context(extractor, trapWriter); - ExtractCIL(cx, assemblyPath, extractPdbs); + ExtractCIL(extractor, trapWriter, extractPdbs); extracted = true; } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs index d79215f0adf6..8bc93bb52b8d 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs @@ -45,7 +45,7 @@ public override IEnumerable Contents } catch { - Cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Info, + Cx.Extractor.Logger.Log(Util.Logging.Severity.Info, $"Attribute decoding is partial. Decoding attribute {constructor.DeclaringType.GetQualifiedName()} failed on {@object}."); yield break; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs index a66ecbb1fab1..80ef03a18215 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs @@ -11,11 +11,10 @@ namespace Semmle.Extraction.CIL public abstract class LabelledEntity : Extraction.LabelledEntity, IExtractedEntity { // todo: with .NET 5 this can override the base context, and change the return type. - public Context Cx { get; } + public Context Cx => (Context)base.Context; - protected LabelledEntity(Context cx) : base(cx.Cx) + protected LabelledEntity(Context cx) : base(cx) { - this.Cx = cx; } public override Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException(); diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs index 9de3ecfaf621..58c15dd21f61 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/Tuple.cs @@ -14,7 +14,7 @@ public Tuple(string name, params object[] args) public void Extract(Context cx) { - cx.Cx.TrapWriter.Emit(tuple); + cx.TrapWriter.Emit(tuple); } public override string ToString() => tuple.ToString(); diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs index d8c18d291eba..3c9924cc98fb 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs @@ -10,11 +10,10 @@ namespace Semmle.Extraction.CIL public abstract class UnlabelledEntity : Extraction.UnlabelledEntity, IExtractedEntity { // todo: with .NET 5 this can override the base context, and change the return type. - public Context Cx { get; } + public Context Cx => (Context)base.Context; - protected UnlabelledEntity(Context cx) : base(cx.Cx) + protected UnlabelledEntity(Context cx) : base(cx) { - Cx = cx; } public override Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException(); diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/CustomAttributeDecoder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/CustomAttributeDecoder.cs index 94d6cc0ab6d1..9007e0b8b90a 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/CustomAttributeDecoder.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/CustomAttributeDecoder.cs @@ -40,11 +40,11 @@ public PrimitiveTypeCode GetUnderlyingEnumType(Type type) if (wellKnownEnums.TryGetValue(name, out var code)) { - cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Debug, $"Using hard coded underlying enum type for {name}"); + cx.Extractor.Logger.Log(Util.Logging.Severity.Debug, $"Using hard coded underlying enum type for {name}"); return code; } - cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Info, $"Couldn't get underlying enum type for {name}"); + cx.Extractor.Logger.Log(Util.Logging.Severity.Info, $"Couldn't get underlying enum type for {name}"); // We can't fall back to Int32, because the type returned here defines how many bytes are read from the // stream and how those bytes are interpreted. diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs index 4b612ae86af7..ba447aebc3a9 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs @@ -11,7 +11,7 @@ public class File : LabelledEntity, IFileOrFolder public File(Context cx, string path) : base(cx) { this.OriginalPath = path; - TransformedPath = cx.Cx.Extractor.PathTransformer.Transform(OriginalPath); + TransformedPath = Cx.Extractor.PathTransformer.Transform(OriginalPath); } public override void WriteId(TextWriter trapFile) diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs index 033010e18a36..9cfcf444e664 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs @@ -476,7 +476,7 @@ public IEnumerable JumpContents(Dictionary // TODO: Find a solution to this. // For now, just log the error - Cx.Cx.ExtractionError("A CIL instruction jumps outside the current method", null, Extraction.Entities.GeneratedLocation.Create(Cx.Cx), "", Util.Logging.Severity.Warning); + Cx.ExtractionError("A CIL instruction jumps outside the current method", null, Extraction.Entities.GeneratedLocation.Create(Cx), "", Util.Logging.Severity.Warning); } } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs index 07cfdab6543a..5dd5dfa37e6c 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs @@ -21,9 +21,9 @@ public override IEnumerable Contents var text = file.Contents; if (text == null) - Cx.Cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, string.Format("PDB source file {0} could not be found", OriginalPath)); + Cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, string.Format("PDB source file {0} could not be found", OriginalPath)); else - Cx.Cx.TrapWriter.Archive(TransformedPath, text); + Cx.TrapWriter.Archive(TransformedPath, text); yield return Tuples.file_extraction_mode(this, 2); } From 841ef9a4ae8cbd7885772096a21443bf9b714122 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 17 Feb 2021 10:41:53 +0100 Subject: [PATCH 3/4] Make derived 'Context' classes internal and adjust visibility of members in base 'Context' --- .../Semmle.Extraction.CIL.Driver/Program.cs | 2 +- .../Semmle.Extraction.CIL/Analyser.cs | 53 ++++++++++++++++++ .../Context.Factories.cs | 8 +-- .../Semmle.Extraction.CIL/Context.cs | 2 +- .../Semmle.Extraction.CIL/EmptyContext.cs | 2 +- .../Entities/Assembly.cs | 48 +---------------- .../Entities/Base/IExtractedEntity.cs | 2 +- .../Entities/Base/IExtractionProduct.cs | 2 +- .../Entities/Base/IGenericContext.cs | 2 +- .../Entities/Base/LabelledEntity.cs | 2 +- .../Entities/Base/UnlabelledEntity.cs | 2 +- .../Entities/ConstructedType.cs | 2 +- .../Semmle.Extraction.CIL/Entities/File.cs | 2 +- .../Semmle.Extraction.CIL/Entities/Folder.cs | 2 +- .../Entities/Namespace.cs | 2 +- .../Entities/PdbSourceFile.cs | 2 +- .../Entities/PrimitiveType.cs | 2 +- .../Entities/SourceLocation.cs | 2 +- .../Semmle.Extraction.CIL/Entities/Type.cs | 2 +- .../Entities/TypeContainer.cs | 2 +- .../Entities/TypeDefinitionType.cs | 2 +- .../Entities/TypeReferenceType.cs | 2 +- .../Entities/TypeSignatureDecoder.cs | 2 +- .../Semmle.Extraction.CSharp/Analyser.cs | 2 +- .../CachedEntityFactory.cs | 4 +- .../Semmle.Extraction.CSharp/Context.cs | 28 +++++++++- .../Entities/Assembly.cs | 2 +- .../Entities/CachedEntity.cs | 2 +- .../Entities/CachedSymbol.cs | 2 +- .../Entities/Compilations/Compilation.cs | 2 +- .../Entities/Constructor.cs | 2 +- .../Entities/Expressions/Binary.cs | 7 +-- .../ObjectCreation/BaseObjectCreation.cs | 2 +- .../Semmle.Extraction.CSharp/Entities/File.cs | 2 +- .../Entities/FreshEntity.cs | 2 +- .../Entities/Method.cs | 2 +- .../Entities/NonGeneratedSourceLocation.cs | 2 +- .../Entities/Parameter.cs | 2 +- .../Entities/Types/Nullability.cs | 4 +- .../Entities/Types/Type.cs | 2 +- .../Entities/UserOperator.cs | 7 +-- .../Populators/TypeContainerVisitor.cs | 2 +- csharp/extractor/Semmle.Extraction/Context.cs | 54 ++++++------------- 43 files changed, 145 insertions(+), 136 deletions(-) create mode 100644 csharp/extractor/Semmle.Extraction.CIL/Analyser.cs diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs index 53542c970e9d..940cd647ed82 100644 --- a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs @@ -24,7 +24,7 @@ private static void ExtractAssembly(Layout layout, string assemblyPath, ILogger { var sw = new Stopwatch(); sw.Start(); - Entities.Assembly.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _); + Analyser.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _); sw.Stop(); logger.Log(Severity.Info, " {0} ({1})", assemblyPath, sw.Elapsed); } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs b/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs new file mode 100644 index 000000000000..dee801cc1718 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs @@ -0,0 +1,53 @@ +using Semmle.Util.Logging; +using System; +using Semmle.Util; +using Semmle.Extraction.CIL.Entities; + +namespace Semmle.Extraction.CIL +{ + public static class Analyser + { + private static void ExtractCIL(Extractor extractor, TrapWriter trapWriter, bool extractPdbs) + { + using var cilContext = new Context(extractor, trapWriter, extractor.OutputPath, extractPdbs); + cilContext.Populate(new Assembly(cilContext)); + cilContext.PopulateAll(); + } + + /// + /// Main entry point to the CIL extractor. + /// Call this to extract a given assembly. + /// + /// The trap layout. + /// The full path of the assembly to extract. + /// The logger. + /// True to overwrite existing trap file. + /// Whether to extract PDBs. + /// The path of the trap file. + /// Whether the file was extracted (false=cached). + public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted) + { + trapFile = ""; + extracted = false; + try + { + var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); + var pathTransformer = new PathTransformer(canonicalPathCache); + var extractor = new Extractor(false, assemblyPath, logger, pathTransformer); + var transformedAssemblyPath = pathTransformer.Transform(assemblyPath); + var project = layout.LookupProjectOrDefault(transformedAssemblyPath); + using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true); + trapFile = trapWriter.TrapFile; + if (nocache || !System.IO.File.Exists(trapFile)) + { + ExtractCIL(extractor, trapWriter, extractPdbs); + extracted = true; + } + } + catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] + { + logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs index 1017b2f1a19f..65c72487b655 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs @@ -9,11 +9,11 @@ namespace Semmle.Extraction.CIL /// /// Provides methods for creating and caching various entities. /// - public sealed partial class Context + internal sealed partial class Context { private readonly Dictionary ids = new Dictionary(); - public T Populate(T e) where T : IExtractedEntity + internal T Populate(T e) where T : IExtractedEntity { if (e.Label.Valid) { @@ -28,7 +28,7 @@ public T Populate(T e) where T : IExtractedEntity else { e.Label = GetNewLabel(); - DefineLabel(e, TrapWriter.Writer, Extractor); + DefineLabel(e); ids.Add(e, e.Label); PopulateLater(() => { @@ -76,7 +76,7 @@ public PrimitiveType Create(PrimitiveTypeCode code) { Label = GetNewLabel() }; - DefineLabel(e, TrapWriter.Writer, Extractor); + DefineLabel(e); primitiveTypes[(int)code] = e; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.cs index a0d6ad1ab6f2..acb66dc32841 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.cs @@ -10,7 +10,7 @@ namespace Semmle.Extraction.CIL /// Adds additional context that is specific for CIL extraction. /// One context = one DLL/EXE. /// - public sealed partial class Context : Extraction.Context, IDisposable + internal sealed partial class Context : Extraction.Context, IDisposable { private readonly FileStream stream; private Entities.Assembly? assemblyNull; diff --git a/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs b/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs index 1105bc39cba3..d5344877264a 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs @@ -5,7 +5,7 @@ namespace Semmle.Extraction.CIL /// /// A generic context which does not contain any type parameters. /// - public class EmptyContext : IGenericContext + internal class EmptyContext : IGenericContext { public EmptyContext(Context cx) { diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs index feff55836c09..2191afef125f 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs @@ -1,18 +1,15 @@ using System.Reflection; using System.Globalization; using System.Collections.Generic; -using Semmle.Util.Logging; -using System; using Semmle.Extraction.Entities; using System.IO; -using Semmle.Util; namespace Semmle.Extraction.CIL.Entities { /// /// An assembly to extract. /// - public class Assembly : LabelledEntity, ILocation + internal class Assembly : LabelledEntity, ILocation { private readonly File file; private readonly AssemblyName assemblyName; @@ -101,48 +98,5 @@ public override IEnumerable Contents } } } - - private static void ExtractCIL(Extractor extractor, TrapWriter trapWriter, bool extractPdbs) - { - using var cilContext = new Context(extractor, trapWriter, extractor.OutputPath, extractPdbs); - cilContext.Populate(new Assembly(cilContext)); - cilContext.PopulateAll(); - } - - /// - /// Main entry point to the CIL extractor. - /// Call this to extract a given assembly. - /// - /// The trap layout. - /// The full path of the assembly to extract. - /// The logger. - /// True to overwrite existing trap file. - /// Whether to extract PDBs. - /// The path of the trap file. - /// Whether the file was extracted (false=cached). - public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted) - { - trapFile = ""; - extracted = false; - try - { - var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); - var pathTransformer = new PathTransformer(canonicalPathCache); - var extractor = new Extractor(false, assemblyPath, logger, pathTransformer); - var transformedAssemblyPath = pathTransformer.Transform(assemblyPath); - var project = layout.LookupProjectOrDefault(transformedAssemblyPath); - using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true); - trapFile = trapWriter.TrapFile; - if (nocache || !System.IO.File.Exists(trapFile)) - { - ExtractCIL(extractor, trapWriter, extractPdbs); - extracted = true; - } - } - catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] - { - logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex)); - } - } } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs index 080227a63b92..d94b94df8a80 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs @@ -5,7 +5,7 @@ namespace Semmle.Extraction.CIL /// /// A CIL entity which has been extracted. /// - public interface IExtractedEntity : IExtractionProduct, IEntity + internal interface IExtractedEntity : IExtractionProduct, IEntity { /// /// The contents of the entity. diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs index 6383d7e1b4ca..231a311a78cb 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs @@ -12,7 +12,7 @@ namespace Semmle.Extraction.CIL /// - Enumerate Contents to produce more extraction products /// - Extract these until there is nothing left to extract /// - public interface IExtractionProduct + internal interface IExtractionProduct { /// /// Perform further extraction/population of this item as necessary. diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs index 21c69f7aeb49..65171e63cd03 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs @@ -6,7 +6,7 @@ namespace Semmle.Extraction.CIL /// When we decode a type/method signature, we need access to /// generic parameters. /// - public interface IGenericContext + internal interface IGenericContext { Context Cx { get; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs index 80ef03a18215..26f6c00cd0b4 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs @@ -8,7 +8,7 @@ namespace Semmle.Extraction.CIL /// An entity that needs to be populated during extraction. /// This assigns a key and optionally extracts its contents. /// - public abstract class LabelledEntity : Extraction.LabelledEntity, IExtractedEntity + internal abstract class LabelledEntity : Extraction.LabelledEntity, IExtractedEntity { // todo: with .NET 5 this can override the base context, and change the return type. public Context Cx => (Context)base.Context; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs index 3c9924cc98fb..42d3824c00ce 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs @@ -7,7 +7,7 @@ namespace Semmle.Extraction.CIL /// An entity that has contents to extract. There is no need to populate /// a key as it's done in the contructor. /// - public abstract class UnlabelledEntity : Extraction.UnlabelledEntity, IExtractedEntity + internal abstract class UnlabelledEntity : Extraction.UnlabelledEntity, IExtractedEntity { // todo: with .NET 5 this can override the base context, and change the return type. public Context Cx => (Context)base.Context; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs index be2ab5da4d88..6e2fb90beae2 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs @@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A constructed type. /// - public sealed class ConstructedType : Type + internal sealed class ConstructedType : Type { private readonly Type unboundGenericType; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs index ba447aebc3a9..8e440795092c 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs @@ -3,7 +3,7 @@ namespace Semmle.Extraction.CIL.Entities { - public class File : LabelledEntity, IFileOrFolder + internal class File : LabelledEntity, IFileOrFolder { protected string OriginalPath { get; } protected PathTransformer.ITransformedPath TransformedPath { get; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs index 2294a525e4b5..a502a0057248 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs @@ -3,7 +3,7 @@ namespace Semmle.Extraction.CIL.Entities { - public sealed class Folder : LabelledEntity, IFileOrFolder + internal sealed class Folder : LabelledEntity, IFileOrFolder { private readonly PathTransformer.ITransformedPath transformedPath; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs index 5d498c5ac548..6cabce18795f 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs @@ -7,7 +7,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A namespace. /// - public sealed class Namespace : TypeContainer + internal sealed class Namespace : TypeContainer { public Namespace? ParentNamespace { get; } public string Name { get; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs index 5dd5dfa37e6c..97e9948f6a7d 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs @@ -2,7 +2,7 @@ namespace Semmle.Extraction.CIL.Entities { - public class PdbSourceFile : File + internal class PdbSourceFile : File { private readonly PDB.ISourceFile file; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs index 661ae0f535bb..d807c5778de1 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs @@ -5,7 +5,7 @@ namespace Semmle.Extraction.CIL.Entities { - public sealed class PrimitiveType : Type + internal sealed class PrimitiveType : Type { private readonly PrimitiveTypeCode typeCode; public PrimitiveType(Context cx, PrimitiveTypeCode tc) : base(cx) diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs index 318cac149304..ff67844121ae 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs @@ -4,7 +4,7 @@ namespace Semmle.Extraction.CIL.Entities { - public sealed class PdbSourceLocation : LabelledEntity, ILocation + internal sealed class PdbSourceLocation : LabelledEntity, ILocation { private readonly Location location; private readonly PdbSourceFile file; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs index ad5a6ababaa3..9b4527770315 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs @@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A type. /// - public abstract class Type : TypeContainer, IMember + internal abstract class Type : TypeContainer, IMember { internal const string AssemblyTypeNameSeparator = "::"; internal const string PrimitiveTypePrefix = "builtin" + AssemblyTypeNameSeparator + "System."; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs index 106b6fe83be7..e201e0652559 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs @@ -6,7 +6,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// Base class for all type containers (namespaces, types, methods). /// - public abstract class TypeContainer : LabelledEntity, IGenericContext + internal abstract class TypeContainer : LabelledEntity, IGenericContext { protected TypeContainer(Context cx) : base(cx) { diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs index 461141ae7190..3f9df14a8918 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs @@ -11,7 +11,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A type defined in the current assembly. /// - public sealed class TypeDefinitionType : Type + internal sealed class TypeDefinitionType : Type { private readonly TypeDefinitionHandle handle; private readonly TypeDefinition td; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs index 49a15965dd0c..51677bf0bb9f 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs @@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A type reference, to a type in a referenced assembly. /// - public sealed class TypeReferenceType : Type + internal sealed class TypeReferenceType : Type { private readonly TypeReferenceHandle handle; private readonly TypeReference tr; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs index 4417becc58d3..d1022d58f0cb 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs @@ -8,7 +8,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// Decodes a type signature and produces a Type, for use by DecodeSignature() and friends. /// - public class TypeSignatureDecoder : ISignatureTypeProvider + internal class TypeSignatureDecoder : ISignatureTypeProvider { private readonly Context cx; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs index cf4f4e0ebf00..5ef8bd24d4e2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs @@ -310,7 +310,7 @@ private void DoExtractCIL(PortableExecutableReference r) { var stopwatch = new Stopwatch(); stopwatch.Start(); - CIL.Entities.Assembly.ExtractCIL(layout, r.FilePath, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted); + CIL.Analyser.ExtractCIL(layout, r.FilePath, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted); stopwatch.Stop(); ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs b/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs index 648d90be67ff..2673ed0d7ef3 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs @@ -1,11 +1,9 @@ -using Microsoft.CodeAnalysis; - namespace Semmle.Extraction.CSharp { /// /// A factory for creating cached entities. /// - public abstract class CachedEntityFactory + internal abstract class CachedEntityFactory : Extraction.CachedEntityFactory where TEntity : CachedEntity { /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Context.cs b/csharp/extractor/Semmle.Extraction.CSharp/Context.cs index 181795fbe910..314706c1e479 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Context.cs @@ -2,6 +2,7 @@ using System; using System.Diagnostics.CodeAnalysis; using Semmle.Extraction.Entities; +using System.Collections.Generic; namespace Semmle.Extraction.CSharp { @@ -9,7 +10,7 @@ namespace Semmle.Extraction.CSharp /// State that needs to be available throughout the extraction process. /// There is one Context object per trap output file. /// - public class Context : Extraction.Context + internal class Context : Extraction.Context { /// /// The program database provided by Roslyn. @@ -50,7 +51,7 @@ public Context(Extraction.Extractor e, Compilation c, TrapWriter trapWriter, IEx public bool IsAssemblyScope => scope is AssemblyScope; - public SyntaxTree SourceTree => scope is SourceScope sc ? sc.SourceTree : null; + private SyntaxTree SourceTree => scope is SourceScope sc ? sc.SourceTree : null; /// /// Whether the given symbol needs to be defined in this context. @@ -116,5 +117,28 @@ protected override bool IsEntityDuplicationGuarded(IEntity entity, [NotNullWhen( loc = null; return false; } + + private readonly HashSet