Merge pull request #1319 from karthiknadig/bug107

Fixes #107 Added code to show data tip for  __foo variables and members.

.gitignore

@@ -20,3 +20,5 @@ Python/Product/Django/Templates/Projects/*/*RoleConfiguration.mht
 Python/Product/Django/Templates/Projects/*/ConfigureCloudService.ps1
 Python/Product/Django/Templates/Projects/*/ps.cmd
 __pycache__/
+/Python/*.VC.opendb
+/Python/*.VC.db

Common/Product/SharedProject/TaskExtensions.cs

@@ -16,6 +16,8 @@
 
 using System;
 using System.Threading.Tasks;
+using System.Windows.Threading;
+using Microsoft.VisualStudio.Threading;
 using Task = System.Threading.Tasks.Task;
 
 namespace Microsoft.VisualStudioTools.Infrastructure {
@@ -46,6 +48,36 @@ static class TaskExtensions {
             return task.GetAwaiter().GetResult();
         }
 
+        /// <summary>
+        /// Waits for a task to complete. If current thread is UI thread then switches thread 
+        /// context to <see cref="NoMessagePumpSyncContext"/>. If an exception occurs, the 
+        /// exception will be raised without being wrapped in a <see cref="AggregateException"/>. 
+        /// </summary>
+        public static void WaitAndUnwrapExceptions(this Task task, Dispatcher dispatcher) {
+            if (dispatcher.CheckAccess()) {
+                using (ThreadingTools.Apply(new NoMessagePumpSyncContext(), true)) {
+                    task.GetAwaiter().GetResult();
+                }
+            } else {
+                task.GetAwaiter().GetResult();
+            }
+        }
+
+        /// <summary>
+        /// Waits for a task to complete. If current thread is UI thread then switches thread 
+        /// context to <see cref="NoMessagePumpSyncContext"/>. If an exception occurs, the 
+        /// exception will be raised without being wrapped in a <see cref="AggregateException"/>. 
+        /// </summary>
+        public static T WaitAndUnwrapExceptions<T>(this Task<T> task, Dispatcher dispatcher) {
+            if (dispatcher.CheckAccess()) {
+                using (ThreadingTools.Apply(new NoMessagePumpSyncContext(), true)) {
+                    return task.GetAwaiter().GetResult();
+                }
+            } else {
+                return task.GetAwaiter().GetResult();
+            }
+        }
+
         /// <summary>
         /// Silently handles the specified exception.
         /// </summary>

Python/Product/Analysis/ModuleAnalysis.cs

@@ -816,7 +816,7 @@ GetMemberOptions options
         /// 
         /// If the expression is a member expression such as "fob.__bar" and the
         /// line number is inside of a class definition this will return a
-        /// MemberExpression with the mangled name like "fob.__ClassName_Bar".
+        /// MemberExpression with the mangled name like "fob._ClassName__bar".
         /// </summary>
         /// <param name="exprText">The expression to evaluate.</param>
         /// <param name="index">

Python/Product/Analysis/Parsing/Ast/MemberExpression.cs

@@ -76,7 +76,11 @@ public sealed class MemberExpression : Expression {
             res.Append('.');
             if (!this.IsIncompleteNode(ast)) {
                 res.Append(this.GetSecondWhiteSpaceDefaultNull(ast));
-                res.Append(this.GetVerbatimImage(ast) ?? _name);
+                if (format.UseVerbatimImage) {
+                    res.Append(this.GetVerbatimImage(ast) ?? _name);
+                } else {
+                    res.Append(_name);
+                }
             }
         }
 

Python/Product/Analysis/Parsing/Ast/NameExpression.cs

@@ -59,7 +59,11 @@ public class NameExpression : Expression {
 
         internal override void AppendCodeString(StringBuilder res, PythonAst ast, CodeFormattingOptions format) {
             format.ReflowComment(res, this.GetProceedingWhiteSpaceDefaultNull(ast));
-            res.Append(this.GetVerbatimImage(ast) ?? _name);
+            if (format.UseVerbatimImage) {
+                res.Append(this.GetVerbatimImage(ast) ?? _name);
+            } else {
+                res.Append(_name);
+            }
         }
     }
 }

Python/Product/Analysis/Parsing/CodeFormattingOptions.cs

@@ -277,6 +277,8 @@ public sealed class CodeFormattingOptions {
 
         #endregion
 
+        internal bool UseVerbatimImage { get; set; } = true;
+
         /// <summary>
         /// Appends one of 3 strings depending upon a code formatting setting.  The 3 settings are the on and off
         /// settings as well as the original formatting if the setting is not set.

Python/Product/Analyzer/Analyzer.csproj

@@ -71,6 +71,7 @@
     <Compile Include="Intellisense\AssignmentWalker.cs" />
     <Compile Include="Intellisense\ClassifierWalker.cs" />
     <Compile Include="Intellisense\CompletionOptions.cs" />
+    <Compile Include="Intellisense\DetectSideEffectsWalker.cs" />
     <Compile Include="Intellisense\EnclosingNodeWalker.cs" />
     <Compile Include="Intellisense\ExtractedMethodCreator.cs" />
     <Compile Include="Intellisense\ExtractMethodResult.cs" />

Python/Product/Analyzer/Intellisense/AnalysisProtocol.cs

@@ -851,5 +851,19 @@ internal class UnhandledExceptionEvent : Event {
             }
             public override string name => Name;
         }
+
+        public sealed class ExpressionForDataTipRequest : Request<ExpressionForDataTipResponse> {
+            public const string Command = "exprForDataTip";
+
+            public int fileId;
+            public string expr;
+            public int line, column, index;
+
+            public override string command => Command;
+        }
+
+        public sealed class ExpressionForDataTipResponse : Response {
+            public string expression;
+        }
     }
 }

Python/Product/Analyzer/Intellisense/DetectSideEffectsWalker.cs

@@ -0,0 +1,67 @@
+// Python Tools for Visual Studio
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using Microsoft.PythonTools.Parsing.Ast;
+
+namespace Microsoft.PythonTools.Intellisense {
+    /// <summary>
+    /// Walks the AST, and indicates if execution of the node has side effects
+    /// </summary>
+    internal class DetectSideEffectsWalker : PythonWalker {
+        public bool HasSideEffects { get; private set; }
+
+        public override bool Walk(AwaitExpression node) {
+            HasSideEffects = true;
+            return false;
+        }
+
+        public override bool Walk(CallExpression node) {
+            HasSideEffects = true;
+            return false;
+        }
+
+        public override bool Walk(BackQuoteExpression node) {
+            HasSideEffects = true;
+            return false;
+        }
+
+        public override bool Walk(ErrorExpression node) {
+            HasSideEffects = true;
+            return false;
+        }
+
+        public override bool Walk(YieldExpression node) {
+            HasSideEffects = true;
+            return false;
+        }
+
+        public override bool Walk(YieldFromExpression node) {
+            HasSideEffects = true;
+            return false;
+        }
+
+        private static readonly HashSet<string> allowedCalls = new HashSet<string> {
+                "abs", "bool", "callable", "chr", "cmp", "complex", "divmod", "float", "format",
+                "getattr", "hasattr", "hash", "hex", "id", "int", "isinstance", "issubclass",
+                "len", "max", "min", "oct", "ord", "pow", "repr", "round", "str", "tuple", "type"
+            };
+
+        public static bool IsSideEffectFreeCall(string name) {
+            return allowedCalls.Contains(name);
+        }
+    }
+}

Python/Product/Analyzer/Intellisense/OutOfProcProjectAnalyzer.cs

@@ -185,6 +185,7 @@ sealed class OutOfProcProjectAnalyzer : IDisposable {
                 case AP.ValueDescriptionRequest.Command: response = GetValueDescriptions((AP.ValueDescriptionRequest)request); break;
                 case AP.ExtensionRequest.Command: response = ExtensionRequest((AP.ExtensionRequest)request); break;
                 case AP.InitializeRequest.Command: response = Initialize((AP.InitializeRequest)request); break;
+                case AP.ExpressionForDataTipRequest.Command: response = ExpressionForDataTip((AP.ExpressionForDataTipRequest)request); break;
                 default:
                     throw new InvalidOperationException("Unknown command");
             }
@@ -1156,6 +1157,32 @@ class ImportStatementWalker : PythonWalker {
             };
         }
 
+        private Response ExpressionForDataTip(AP.ExpressionForDataTipRequest request) {
+            var pyEntry = _projectFiles[request.fileId] as IPythonProjectEntry;
+
+            string dataTipExpression = null;
+            var options = new CodeFormattingOptions() {
+                UseVerbatimImage = false
+            };
+
+            if (pyEntry != null && pyEntry.Analysis != null) {
+                var ast = pyEntry.Analysis.GetAstFromText(request.expr, new SourceLocation(request.index, request.line, request.column));
+                var expr = Statement.GetExpression(ast.Body);
+
+                if (ast != null) {
+                    var walker = new DetectSideEffectsWalker();
+                    ast.Walk(walker);
+                    if (!walker.HasSideEffects) {
+                        dataTipExpression = expr?.ToCodeString(ast, new CodeFormattingOptions() { UseVerbatimImage = false });
+                    }
+                }
+            }
+
+            return new AP.ExpressionForDataTipResponse() {
+                expression = dataTipExpression
+            };
+        }
+
         private Response AnalyzeExpression(AP.AnalyzeExpressionRequest request) {
             var pyEntry = _projectFiles[request.fileId] as IPythonProjectEntry;
             AP.AnalysisReference[] references;

Python/Product/Analyzer/Intellisense/ProximityExpressionWalker.cs

@@ -106,12 +106,6 @@ internal class ProximityExpressionWalker : PythonWalker {
             }
         }
 
-        private static readonly HashSet<string> allowedCalls = new HashSet<string> {
-                "abs", "bool", "callable", "chr", "cmp", "complex", "divmod", "float", "format",
-                "getattr", "hasattr", "hash", "hex", "id", "int", "isinstance", "issubclass",
-                "len", "max", "min", "oct", "ord", "pow", "repr", "round", "str", "tuple", "type"
-            };
-
         public override void PostWalk(CallExpression node) {
             if (IsInRange(node) && node.Target != null) {
                 // For call nodes, we don't want either the call nor the called function to show up,
@@ -128,7 +122,7 @@ internal class ProximityExpressionWalker : PythonWalker {
 
                 // Hardcode some commonly used side-effect-free builtins to show even in calls.
                 var name = node.Target as NameExpression;
-                if (name != null && allowedCalls.Contains(name.Name) && node.Args.All(arg => IsValidTarget(arg.Expression))) {
+                if (name != null && DetectSideEffectsWalker.IsSideEffectFreeCall(name.Name) && node.Args.All(arg => IsValidTarget(arg.Expression))) {
                     _expressions.Add(node, null);
                 }
             }
@@ -141,40 +135,6 @@ internal class ProximityExpressionWalker : PythonWalker {
             return isct0 <= isct1;
         }
 
-        private class DetectSideEffectsWalker : PythonWalker {
-            public bool HasSideEffects { get; private set; }
-
-            public override bool Walk(AwaitExpression node) {
-                HasSideEffects = true;
-                return false;
-            }
-
-            public override bool Walk(CallExpression node) {
-                HasSideEffects = true;
-                return false;
-            }
-
-            public override bool Walk(BackQuoteExpression node) {
-                HasSideEffects = true;
-                return false;
-            }
-
-            public override bool Walk(ErrorExpression node) {
-                HasSideEffects = true;
-                return false;
-            }
-
-            public override bool Walk(YieldExpression node) {
-                HasSideEffects = true;
-                return false;
-            }
-
-            public override bool Walk(YieldFromExpression node) {
-                HasSideEffects = true;
-                return false;
-            }
-        }
-
         private bool IsValidTarget(Node node) {
             if (node == null || node is ConstantExpression || node is NameExpression) {
                 return true;

Python/Product/PythonTools/PythonTools/Extensions.cs

@@ -32,6 +32,7 @@
 using Microsoft.PythonTools.Intellisense;
 using Microsoft.PythonTools.Interpreter;
 using Microsoft.PythonTools.InterpreterList;
+using Microsoft.PythonTools.Parsing;
 using Microsoft.PythonTools.Parsing.Ast;
 using Microsoft.PythonTools.Project;
 using Microsoft.PythonTools.Repl;
@@ -50,7 +51,6 @@
 using Microsoft.VisualStudio.Utilities;
 using Microsoft.VisualStudioTools;
 using Microsoft.VisualStudioTools.Project;
-using Microsoft.PythonTools.Parsing;
 
 namespace Microsoft.PythonTools {
     public static class Extensions {

Python/Product/PythonTools/PythonTools/Intellisense/ProjectAnalyzer.cs

@@ -1960,6 +1960,33 @@ class ExprWalker : PythonWalker {
             return new AnalysisVariable(type, location);
         }
 
+        internal static async Task<string> ExpressionForDataTipAsync(IServiceProvider serviceProvider, ITextView view, SnapshotSpan span) {
+            var analysis = GetApplicableExpression(
+                view.GetAnalysisEntry(span.Start.Snapshot.TextBuffer, serviceProvider),
+                span.Start
+            );
+            string result = null;
+
+            if (analysis != null) {
+                var location = analysis.Location;
+                var req = new AP.ExpressionForDataTipRequest() {
+                    expr = span.GetText(),
+                    column = location.Column,
+                    index = location.Index,
+                    line = location.Line,
+                    fileId = analysis.Entry.FileId,
+                };
+
+                var resp = await analysis.Entry.Analyzer.SendRequestAsync(req).ConfigureAwait(false);
+
+                if (resp != null) {
+                    result = resp.expression;
+                }
+            }
+
+            return result;
+        }
+
         class ApplicableExpression {
             public readonly string Text;
             public readonly AnalysisEntry Entry;

Python/Product/PythonTools/PythonTools/Navigation/TextViewFilter.cs

@@ -15,6 +15,7 @@
 // permissions and limitations under the License.
 
 using System;
+using System.Windows.Threading;
 using Microsoft.PythonTools.Intellisense;
 using Microsoft.VisualStudio;
 using Microsoft.VisualStudio.ComponentModelHost;
@@ -24,6 +25,7 @@
 using Microsoft.VisualStudio.Text;
 using Microsoft.VisualStudio.Text.Editor;
 using Microsoft.VisualStudio.TextManager.Interop;
+using Microsoft.VisualStudioTools.Infrastructure;
 using IServiceProvider = System.IServiceProvider;
 
 namespace Microsoft.PythonTools.Language {
@@ -34,13 +36,15 @@ namespace Microsoft.PythonTools.Language {
     public sealed class TextViewFilter : IOleCommandTarget, IVsTextViewFilter {
         private IVsEditorAdaptersFactoryService _vsEditorAdaptersFactoryService;
         private IVsDebugger _debugger;
+        private IServiceProvider _serviceProvider;
         private readonly IOleCommandTarget _next;
         private readonly IVsTextLines _vsTextLines;
         private readonly IWpfTextView _wpfTextView;
 
         public TextViewFilter(IServiceProvider serviceProvider, IVsTextView vsTextView) {
             var compModel = (IComponentModel)serviceProvider.GetService(typeof(SComponentModel));
             _vsEditorAdaptersFactoryService = compModel.GetService<IVsEditorAdaptersFactoryService>();
+            _serviceProvider = serviceProvider;
             _debugger = (IVsDebugger)serviceProvider.GetService(typeof(IVsDebugger));
 
             vsTextView.GetBuffer(out _vsTextLines);
@@ -114,16 +118,18 @@ public sealed class TextViewFilter : IOleCommandTarget, IVsTextViewFilter {
             var trackingSpan = snapshot.CreateTrackingSpan(snapshotSpan.Span, SpanTrackingMode.EdgeExclusive);
             var rep = new ReverseExpressionParser(snapshot, _wpfTextView.TextBuffer, trackingSpan);
             var exprSpan = rep.GetExpressionRange(forCompletion: false);
+            string expr = null;
             if (exprSpan != null) {
                 SnapshotPointToLineAndColumnNumber(exprSpan.Value.Start, out pSpan[0].iStartLine, out pSpan[0].iStartIndex);
                 SnapshotPointToLineAndColumnNumber(exprSpan.Value.End, out pSpan[0].iEndLine, out pSpan[0].iEndIndex);
+                expr = VsProjectAnalyzer.ExpressionForDataTipAsync(_serviceProvider, _wpfTextView, exprSpan.Value).WaitAndUnwrapExceptions(Dispatcher.CurrentDispatcher);
             } else {
                 // If it's not an expression, suppress the tip.
                 pbstrText = null;
                 return VSConstants.E_FAIL;
             }
 
-            return _debugger.GetDataTipValue(_vsTextLines, pSpan, null, out pbstrText);
+            return _debugger.GetDataTipValue(_vsTextLines, pSpan, expr, out pbstrText);
         }
 
         public int GetPairExtents(int iLine, int iIndex, TextSpan[] pSpan) {