diff --git a/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs b/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs index dc1228ddb..4f6801dff 100644 --- a/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs +++ b/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs @@ -63,9 +63,9 @@ public interface IDocument: IPythonModule, IDisposable { void Update(IEnumerable changes); /// - /// Resets document buffer to the provided content or tries to load it if content is null, then parses and analyzes document. + /// Forces parse and analysis of the document. /// - void Reset(string content); + void Invalidate(); /// /// Provides collection of parsing errors, if any. diff --git a/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs b/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs index 21a2bdfb8..e05536806 100644 --- a/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs +++ b/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; +using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Parsing; namespace Microsoft.Python.Analysis.Documents { @@ -23,6 +24,8 @@ internal sealed class DocumentBuffer { private readonly object _lock = new object(); private StringBuilder _sb = new StringBuilder(); private string _content; + private bool _contentDropped; + private bool _populated; public int Version { get; private set; } @@ -34,16 +37,48 @@ public string Text { } } - public void Reset(int version, string content) { + /// + /// Clear buffer content to save memory. + /// The buffer cannot be modified after this point. + /// + public void Clear() { lock (_lock) { - Version = version; + _content = string.Empty; + _sb = null; + _contentDropped = true; + } + } + + /// + /// Advances content version of the buffer without changing the contents. + /// Typically used to invalidate analysis. + /// + public void MarkChanged() { + lock (_lock) { + Check.InvalidOperation(_populated, "Buffer is not populated."); + Check.InvalidOperation(!_contentDropped, "Buffer content was dropped and cannot be updated."); + Version++; + } + } + + /// + /// Populates buffer with initial content. This can ony happen once. + /// + /// + public void Populate(string content) { + lock (_lock) { + Check.InvalidOperation(!_populated, "Buffer is already populated."); + Check.InvalidOperation(!_contentDropped, "Buffer content was dropped and cannot be updated."); + Version = 0; _content = content ?? string.Empty; _sb = null; + _populated = true; } } public void Update(IEnumerable changes) { lock (_lock) { + Check.InvalidOperation(!_contentDropped, "Buffer content was dropped and cannot be updated."); _sb = _sb ?? new StringBuilder(_content); foreach (var change in changes) { @@ -59,9 +94,11 @@ public void Update(IEnumerable changes) { var start = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.Start, _sb.Length); var end = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.End, _sb.Length); + if (end > start) { _sb.Remove(start, end - start); } + if (!string.IsNullOrEmpty(change.InsertedText)) { _sb.Insert(start, change.InsertedText); } @@ -98,7 +135,6 @@ public IEnumerable GetNewLineLocations() { if (i == _sb.Length - 1) { yield return new NewLineLocation(i + 1, NewLineKind.None); } - break; } } diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index 810632abc..4e855397c 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -106,7 +106,7 @@ public IDocument OpenDocument(Uri uri, string content, string filePath = null) { }; entry = CreateDocument(mco); } - justOpened = TryOpenDocument(entry, content); + justOpened = TryOpenDocument(entry); document = entry.Document; } @@ -217,7 +217,7 @@ public void ReloadAll() { } foreach (var (_, entry) in opened) { - entry.Document.Reset(null); + entry.Document.Invalidate(); } } @@ -277,10 +277,9 @@ private bool TryAddModulePath(ModuleCreationOptions mco) { return true; } - private bool TryOpenDocument(DocumentEntry entry, string content) { + private bool TryOpenDocument(DocumentEntry entry) { if (!entry.Document.IsOpen) { entry.Document.IsOpen = true; - entry.Document.Reset(content); entry.LockCount++; return true; } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 92de189dd..34849835c 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -119,7 +119,7 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s } IsPersistent = creationOptions.IsPersistent; - InitializeContent(creationOptions.Content, 0); + InitializeContent(creationOptions.Content); } #region IPythonType @@ -315,14 +315,12 @@ public void Update(IEnumerable changes) { Services.GetService().InvalidateAnalysis(this); } - public void Reset(string content) { + public void Invalidate() { lock (_syncObj) { - if (content != Content) { - ContentState = State.None; - InitializeContent(content, _buffer.Version + 1); - } + ContentState = State.None; + _buffer.MarkChanged(); + Parse(); } - Services.GetService().InvalidateAnalysis(this); } @@ -451,7 +449,7 @@ public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { ContentState = State.Analyzed; if (ModuleType != ModuleType.User) { - _buffer.Reset(_buffer.Version, string.Empty); + _buffer.Clear(); } } @@ -487,7 +485,7 @@ public void AddAstNode(object o, Node n) { public void ClearContent() { lock (_syncObj) { if (ModuleType != ModuleType.User) { - _buffer.Reset(_buffer.Version, string.Empty); + _buffer.Clear(); _astMap.Clear(); } } @@ -515,18 +513,21 @@ protected virtual string LoadContent() { return null; // Keep content as null so module can be loaded later. } - private void InitializeContent(string content, int version) { + /// + /// Populates buffer with content. Content can either be provided, such as when + /// user opens the document or generated such as when module is compiled and + /// scraper will generate the content in the overridden LoadContent(). + /// + private void InitializeContent(string content) { lock (_syncObj) { - LoadContent(content, version); - - var startParse = ContentState < State.Parsing && (_parsingTask == null || version > 0); - if (startParse) { + LoadContent(content); + if (ContentState < State.Parsing && _parsingTask == null) { Parse(); } } } - private void LoadContent(string content, int version) { + private void LoadContent(string content) { if (ContentState < State.Loading) { try { if (IsPersistent) { @@ -534,7 +535,7 @@ private void LoadContent(string content, int version) { } else { content = content ?? LoadContent(); } - _buffer.Reset(version, content); + _buffer.Populate(content); ContentState = State.Loaded; } catch (IOException) { } catch (UnauthorizedAccessException) { } } diff --git a/src/Analysis/Ast/Test/DocumentBufferTests.cs b/src/Analysis/Ast/Test/DocumentBufferTests.cs index c0230ac18..c6e48b8a6 100644 --- a/src/Analysis/Ast/Test/DocumentBufferTests.cs +++ b/src/Analysis/Ast/Test/DocumentBufferTests.cs @@ -29,7 +29,7 @@ public class DocumentBufferTests { [TestMethod, Priority(0)] public void BasicDocumentBuffer() { var doc = new DocumentBuffer(); - doc.Reset(0, @"def f(x): + doc.Populate(@"def f(x): return def g(y): @@ -77,7 +77,7 @@ def g(y): public void ResetDocumentBuffer() { var doc = new DocumentBuffer(); - doc.Reset(0, string.Empty); + doc.Populate(string.Empty); Assert.AreEqual(string.Empty, doc.Text); doc.Update(new[] { @@ -86,18 +86,13 @@ public void ResetDocumentBuffer() { Assert.AreEqual("text", doc.Text); Assert.AreEqual(1, doc.Version); - - doc.Reset(0, @"abcdef"); - - Assert.AreEqual(@"abcdef", doc.Text); - Assert.AreEqual(0, doc.Version); } [TestMethod, Priority(0)] public void ReplaceAllDocumentBuffer() { var doc = new DocumentBuffer(); - doc.Reset(0, string.Empty); + doc.Populate(string.Empty); Assert.AreEqual(string.Empty, doc.Text); doc.Update(new[] { @@ -134,7 +129,7 @@ public void ReplaceAllDocumentBuffer() { [TestMethod, Priority(0)] public void DeleteMultipleDisjoint() { var doc = new DocumentBuffer(); - doc.Reset(0, @" + doc.Populate(@" line1 line2 line3 @@ -157,7 +152,7 @@ public void DeleteMultipleDisjoint() { [TestMethod, Priority(0)] public void InsertMultipleDisjoint() { var doc = new DocumentBuffer(); - doc.Reset(0, @" + doc.Populate(@" line line line @@ -180,7 +175,7 @@ public void InsertMultipleDisjoint() { [TestMethod, Priority(0)] public void DeleteAcrossLines() { var doc = new DocumentBuffer(); - doc.Reset(0, @" + doc.Populate(@" line1 line2 line3 @@ -199,7 +194,7 @@ public void DeleteAcrossLines() { [TestMethod, Priority(0)] public void SequentialChanges() { var doc = new DocumentBuffer(); - doc.Reset(0, @" + doc.Populate(@" line1 line2 line3 @@ -218,7 +213,7 @@ public void SequentialChanges() { [TestMethod, Priority(0)] public void InsertTopToBottom() { var doc = new DocumentBuffer(); - doc.Reset(0, @"linelinelineline"); + doc.Populate(@"linelinelineline"); doc.Update(new[] { DocumentChange.Insert("\n", new SourceLocation(1, 1)), DocumentChange.Insert("1\n", new SourceLocation(2, 5)), @@ -233,7 +228,7 @@ public void InsertTopToBottom() { [DataTestMethod, Priority(0)] public void NewLines(string s, NewLineLocation[] expected) { var doc = new DocumentBuffer(); - doc.Reset(0, s); + doc.Populate(s); var nls = doc.GetNewLineLocations().ToArray(); for (var i = 0; i < nls.Length; i++) { Assert.AreEqual(nls[i].Kind, expected[i].Kind); @@ -281,7 +276,18 @@ public IEnumerable GetData(MethodInfo methodInfo) => new NewLineLocation(12, NewLineKind.LineFeed), new NewLineLocation(13, NewLineKind.None) } + }, + new object[] { + "\r\na\r\n\r\n\r\n \r\n", + new NewLineLocation[] { + new NewLineLocation(2, NewLineKind.CarriageReturnLineFeed), + new NewLineLocation(5, NewLineKind.CarriageReturnLineFeed), + new NewLineLocation(7, NewLineKind.CarriageReturnLineFeed), + new NewLineLocation(9, NewLineKind.CarriageReturnLineFeed), + new NewLineLocation(12, NewLineKind.CarriageReturnLineFeed), + new NewLineLocation(13, NewLineKind.None) } + } }; public string GetDisplayName(MethodInfo methodInfo, object[] data) => data != null ? $"{methodInfo.Name} ({FormatName((string)data[0])})" : null;