Django Intellisense step 1

 Enables some basic DJango intellisense for templates, still more to go…
 Adds new DjangoCompletionSource, DjangoCompletionSourceProvider, DjangoIntellisenseControllerProvider, DjangoIntellisenseController
 The intellisense controller is still greatly lacking – e.g. there's no real way to commit yet.

 DjangoProject now specializes the appropriate DJango functions and participates in their analysis. This enables it to pick up the available filters and tags both from the core Django project as well as any extensions defined in a user's own app.

 Html Completions should only be available in the HTML section of the file, not in the template section

 Adds new AnalysisValue base class which allows 3rd-party code to interact w/ the analysis engine
 Adds CallInfo class which allows 3rd-party code to analyze calls

 Force modules to load before getting their locations

 Make ProjectAnalyzer public, rename to VsProjectAnalyzer, reduce its public API to only the APIs we want to support

 Increase the default cross module analysis limit

Release/Product/Python/Analysis/Analysis.csproj

@@ -264,6 +264,7 @@
     <Compile Include="Analyzer\AnalysisUnit.cs" />
     <Compile Include="Parsing\SourceLocation.cs" />
     <Compile Include="Unpickler.cs" />
+    <Compile Include="AnalysisValue.cs" />
     <Compile Include="Values\BoundBuiltinMethodInfo.cs" />
     <Compile Include="Values\BuiltinClassInfo.cs" />
     <Compile Include="Values\BuiltinEventInfo.cs" />
@@ -273,6 +274,7 @@
     <Compile Include="Values\BuiltinModule.cs" />
     <Compile Include="Values\BuiltinNamespace.cs" />
     <Compile Include="Values\BuiltinPropertyInfo.cs" />
+    <Compile Include="CallInfo.cs" />
     <Compile Include="Values\ClassInfo.cs" />
     <Compile Include="Values\ConstantInfo.cs" />
     <Compile Include="Values\GeneratorFunctionInfo.cs" />
@@ -307,7 +309,8 @@
     <Compile Include="Values\SequenceInfo.cs" />
     <Compile Include="Values\SetInfo.cs" />
     <Compile Include="Values\SliceInfo.cs" />
-    <Compile Include="Values\SpecializedBuiltinFunction.cs" />
+    <Compile Include="Values\SpecializedCallable.cs" />
+    <Compile Include="Values\SpecializedNamespace.cs" />
     <Compile Include="Values\TupleBuiltinClassInfo.cs" />
     <Compile Include="Values\UserDefinedInfo.cs" />
     <Compile Include="Values\Utils.cs" />

Release/Product/Python/Analysis/AnalysisValue.cs

@@ -0,0 +1,46 @@
+/* ****************************************************************************
+ *
+ * Copyright (c) Microsoft Corporation. 
+ *
+ * This source code is subject to terms and conditions of the Apache License, Version 2.0. A 
+ * copy of the license can be found in the License.html file at the root of this distribution. If 
+ * you cannot locate the Apache License, Version 2.0, please send an email to 
+ * vspython@microsoft.com. By using this source code in any fashion, you are agreeing to be bound 
+ * by the terms of the Apache License, Version 2.0.
+ *
+ * You must not remove this notice, or any other, from this software.
+ *
+ * ***************************************************************************/
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.PythonTools.Analysis {
+    /// <summary>
+    /// Base class for all analysis values.  Exposes the public API of the analysis engine.
+    /// 
+    /// New in 1.5.
+    /// </summary>
+    public class AnalysisValue {
+        internal AnalysisValue() {
+        }
+
+        /// <summary>
+        /// Gets a list of locations where this value is defined.
+        /// </summary>
+        public virtual IEnumerable<LocationInfo> Locations {
+            get { return LocationInfo.Empty; }
+        }
+
+        /// <summary>
+        /// Gets the constant value that this object represents, if it's a constant.
+        /// 
+        /// Returns Type.Missing if the value is not constant (because it returns null
+        /// if the type is None).
+        /// </summary>
+        /// <returns></returns>
+        public virtual object GetConstantValue() {
+            return Type.Missing;
+        }
+    }
+}

Release/Product/Python/Analysis/CallInfo.cs

@@ -0,0 +1,36 @@
+/* ****************************************************************************
+ *
+ * Copyright (c) Microsoft Corporation. 
+ *
+ * This source code is subject to terms and conditions of the Apache License, Version 2.0. A 
+ * copy of the license can be found in the License.html file at the root of this distribution. If 
+ * you cannot locate the Apache License, Version 2.0, please send an email to 
+ * vspython@microsoft.com. By using this source code in any fashion, you are agreeing to be bound 
+ * by the terms of the Apache License, Version 2.0.
+ *
+ * You must not remove this notice, or any other, from this software.
+ *
+ * ***************************************************************************/
+
+using System.Collections.Generic;
+using Microsoft.PythonTools.Analysis.Values;
+
+namespace Microsoft.PythonTools.Analysis {
+    public struct CallInfo {
+        private readonly ISet<Namespace>[] _args;
+
+        internal CallInfo(ISet<Namespace>[] args) {
+            _args = args;
+        }
+
+        public int NormalArgumentCount {
+            get {
+                return _args.Length;
+            }
+        }
+
+        public IEnumerable<AnalysisValue> GetArgument(int arg) {
+            return _args[arg];
+        }
+    }
+}

Release/Product/Python/Analysis/Interpreter/Default/CPythonModule.cs

@@ -214,12 +214,15 @@ class CPythonModule : IPythonModule2, IProjectEntry, ILocatedMember {
         }
 
         public string FilePath {
-            get { return _filename; }
+            get {
+                EnsureLoaded();
+                return _filename; 
+            }
         }
 
         public string GetLine(int lineNo) {
             lineNo--; // line is 1 based
-            string[] lines = File.ReadAllLines(_filename);
+            string[] lines = File.ReadAllLines(FilePath);
             if (lineNo < lines.Length) {
                 return lines[lineNo];
             }
@@ -247,7 +250,10 @@ class CPythonModule : IPythonModule2, IProjectEntry, ILocatedMember {
         #region ILocatedMember Members
 
         public IEnumerable<LocationInfo> Locations {
-            get { yield return new LocationInfo(this, 1, 1); }
+            get {
+                EnsureLoaded();
+                yield return new LocationInfo(this, 1, 1); 
+            }
         }
 
         #endregion

Release/Product/Python/Analysis/MemberResult.cs

@@ -116,5 +116,20 @@ public struct MemberResult {
                 return _vars();
             }
         }
+
+        /// <summary>
+        /// Gets the location(s) for the member(s) if they are available.
+        /// 
+        /// New in 1.5.
+        /// </summary>
+        public IEnumerable<LocationInfo> Locations {
+            get {
+                foreach (var ns in _vars()) {
+                    foreach (var location in ns.Locations) {
+                        yield return location;
+                    }
+                }
+            }
+        }
     }
 }

Release/Product/Python/Analysis/ProjectEntry.cs

@@ -204,6 +204,7 @@ internal sealed class ProjectEntry : IPythonProjectEntry, IProjectEntry2 {
             // collect top-level definitions first
             var walker = new OverviewWalker(this, unit);
             _tree.Walk(walker);
+            _myScope.Specialize();
 
             PublishPackageChildrenInPackage();
 

Release/Product/Python/Analysis/PythonAnalyzer.cs

@@ -169,7 +169,7 @@ public PythonAnalyzer(IPythonInterpreterFactory interpreterFactory)
             }
             return entry;
         }
-
+        
         /// <summary>
         /// Removes the specified project entry from the current analysis.
         /// 
@@ -202,17 +202,28 @@ public PythonAnalyzer(IPythonInterpreterFactory interpreterFactory)
             return entry;
         }
 
+        /// <summary>
+        /// Looks up the specified module by name.
+        /// </summary>
+        public MemberResult[] GetModule(string name) {
+            return GetModules(modName => modName != name);
+        }
+
         /// <summary>
         /// Gets a top-level list of all the available modules as a list of MemberResults.
         /// </summary>
         /// <returns></returns>
         public MemberResult[] GetModules(bool topLevelOnly = false) {
+            return GetModules(modName => topLevelOnly && modName.IndexOf('.') != -1);
+        }
+
+        private MemberResult[] GetModules(Func<string, bool> excludedPredicate) {
             var d = new Dictionary<string, List<ModuleLoadState>>();
             foreach (var keyValue in Modules) {
                 var modName = keyValue.Key;
                 var moduleRef = keyValue.Value;
 
-                if (topLevelOnly && modName.IndexOf('.') != -1) {
+                if (excludedPredicate(modName)) {
                     continue;
                 }
 
@@ -228,6 +239,10 @@ public PythonAnalyzer(IPythonInterpreterFactory interpreterFactory)
                 }
             }
 
+            return ModuleDictToMemberResult(d);
+        }
+
+        private static MemberResult[] ModuleDictToMemberResult(Dictionary<string, List<ModuleLoadState>> d) {
             var result = new MemberResult[d.Count];
             int pos = 0;
             foreach (var kvp in d) {
@@ -363,10 +378,18 @@ class LazyModuleEnumerator {
             return new MemberResult[0];
         }
 
+        public void SpecializeFunction(string moduleName, string name, Action<CallExpression, CallInfo> dlg) {
+            SpecializeFunction(moduleName, name, (call, unit, types) => { dlg(call, new CallInfo(types)); return null; });
+        }
+
         public void SpecializeFunction(string moduleName, string name, Action<CallExpression> dlg) {
             SpecializeFunction(moduleName, name, (call, unit, types) => { dlg(call); return null; });
         }
 
+        public void SpecializeFunction(string moduleName, string name, Action<PythonAnalyzer, CallExpression> dlg) {
+            SpecializeFunction(moduleName, name, (call, unit, types) => { dlg(this, call); return null; });
+        }
+
         /// <summary>
         /// Gets the list of directories which should be analyzed.
         /// 
@@ -448,17 +471,20 @@ class LazyModuleEnumerator {
         private void SpecializeFunction(string moduleName, string name, Func<CallExpression, AnalysisUnit, ISet<Namespace>[], ISet<Namespace>> dlg) {
             ModuleReference module;
 
+            int lastDot;
             if (Modules.TryGetValue(moduleName, out module)) {
-                BuiltinModule builtin = module.Module as BuiltinModule;
-                Debug.Assert(builtin != null);
-                if (builtin != null) {
-                    foreach (var v in builtin[name]) {
-                        BuiltinFunctionInfo funcInfo = v as BuiltinFunctionInfo;
-                        if (funcInfo != null && !(funcInfo is SpecializedBuiltinFunction)) {
-                            builtin[name] = new SpecializedBuiltinFunction(this, funcInfo.Function, dlg).SelfSet;
-                            break;
-                        }
-                    }
+                IModule mod = module.Module as IModule;
+                Debug.Assert(mod != null);
+                if (mod != null) {
+                    mod.SpecializeFunction(name, dlg);
+                }
+            } else if ((lastDot = moduleName.LastIndexOf('.')) != -1 && 
+                Modules.TryGetValue(moduleName.Substring(0, lastDot), out module)) {
+
+                IModule mod = module.Module as IModule;
+                Debug.Assert(mod != null);
+                if (mod != null) {
+                    mod.SpecializeFunction(moduleName.Substring(lastDot + 1, moduleName.Length - (lastDot + 1)) + "." + name, dlg);
                 }
             }
         }

Release/Product/Python/Analysis/Values/BuiltinModule.cs

@@ -118,6 +118,15 @@ public BuiltinModule(IPythonModule module, PythonAnalyzer projectState)
             }
         }
 
+        public void SpecializeFunction(string name, Func<CallExpression, AnalysisUnit, ISet<Namespace>[], ISet<Namespace>> dlg) {
+            foreach (var v in this[name]) {
+                if (!(v is SpecializedNamespace)) {
+                    this[name] = new SpecializedCallable(v, dlg).SelfSet;
+                    break;
+                }
+            }
+        }
+
         public override ILocatedMember GetLocatedMember() {
             return _interpreterModule as ILocatedMember;
         }

Release/Product/Python/Analysis/Values/IModule.cs

@@ -19,5 +19,7 @@ namespace Microsoft.PythonTools.Analysis.Values {
     interface IModule {
         IModule GetChildPackage(IModuleContext context, string name);
         IEnumerable<KeyValuePair<string, Namespace>> GetChildrenPackages(IModuleContext context);
+
+        void SpecializeFunction(string name, System.Func<Parsing.Ast.CallExpression, Interpreter.AnalysisUnit, ISet<Namespace>[], ISet<Namespace>> dlg);
     }
 }

Release/Product/Python/Analysis/Values/ModuleInfo.cs

@@ -31,6 +31,7 @@ internal class ModuleInfo : Namespace, IReferenceableContainer, IModule {
         private readonly WeakReference _weakModule;
         private readonly IModuleContext _context;
         private Dictionary<string, WeakReference> _packageModules;
+        private Dictionary<string, Func<CallExpression, AnalysisUnit, ISet<Namespace>[], ISet<Namespace>>> _specialized;
         private ModuleInfo _parentPackage;
         private DependentData _definition = new DependentData();
 
@@ -113,6 +114,50 @@ internal class ModuleInfo : Namespace, IReferenceableContainer, IModule {
             return null;
         }
 
+        public void SpecializeFunction(string name, Func<CallExpression, AnalysisUnit, ISet<Namespace>[], ISet<Namespace>> dlg) {
+            if (_specialized == null) {
+                _specialized = new Dictionary<string, Func<CallExpression, AnalysisUnit, ISet<Namespace>[], ISet<Namespace>>>();
+            }
+            _specialized[name] = dlg;
+        }
+
+        internal void Specialize() {
+            if (_specialized != null) {
+                foreach (var keyValue in _specialized) {
+                    SpecializeOneFunction(keyValue.Key, keyValue.Value);
+                }
+            }
+        }
+
+        private void SpecializeOneFunction(string name, Func<CallExpression, AnalysisUnit, ISet<Namespace>[], ISet<Namespace>> dlg) {
+            int lastIndex;
+            VariableDef def;
+            if (Scope.Variables.TryGetValue(name, out def)) {
+                SpecializeVariableDef(dlg, def);
+            } else if ((lastIndex = name.LastIndexOf('.')) != -1 && 
+                Scope.Variables.TryGetValue(name.Substring(0, lastIndex), out def)) {
+                    var methodName = name.Substring(lastIndex + 1, name.Length - (lastIndex + 1));
+                    foreach (var v in def.Types) {
+                        ClassInfo ci = v as ClassInfo;
+                        if (ci != null) {
+                            VariableDef methodDef;
+                            if (ci.Scope.Variables.TryGetValue(methodName, out methodDef)) {
+                                SpecializeVariableDef(dlg, methodDef);
+                            }
+                        }
+                    }
+            }
+        }
+
+        private static void SpecializeVariableDef(Func<CallExpression, AnalysisUnit, ISet<Namespace>[], ISet<Namespace>> dlg, VariableDef def) {
+            foreach (var v in def.Types) {
+                if (!(v is SpecializedNamespace) && v.DeclaringModule != null) {
+                    def.AddTypes(v.DeclaringModule, new SpecializedCallable(v, dlg).SelfSet);
+                    break;
+                }
+            }
+        }
+
         public override ISet<Namespace> GetMember(Node node, AnalysisUnit unit, string name) {
             ModuleDefinition.AddDependency(unit);
 
@@ -238,5 +283,6 @@ internal class ModuleInfo : Namespace, IReferenceableContainer, IModule {
         }
 
         #endregion
+
     }
 }

Release/Product/Python/Analysis/Values/Namespace.cs

@@ -25,7 +25,7 @@ namespace Microsoft.PythonTools.Analysis.Values {
     /// A namespace represents a set of variables and code.  Examples of 
     /// namespaces include top-level code, classes, and functions.
     /// </summary>
-    internal class Namespace : ISet<Namespace>, IAnalysisValue {
+    internal class Namespace : AnalysisValue, ISet<Namespace>, IAnalysisValue {
         [ThreadStatic] private static HashSet<Namespace> _processing;
 
         public Namespace() { }
@@ -46,10 +46,6 @@ internal class Namespace : ISet<Namespace>, IAnalysisValue {
             }
         }*/
 
-        public virtual IEnumerable<LocationInfo> Locations {
-            get { return LocationInfo.Empty; }
-        }
-
         public LocationInfo Location {
             get {
                 return Locations.FirstOrDefault();
@@ -83,10 +79,6 @@ internal class Namespace : ISet<Namespace>, IAnalysisValue {
             get { return PythonMemberType.Unknown; }
         }
 
-        public virtual object GetConstantValue() {
-            return Type.Missing;
-        }
-
         public virtual IDictionary<string, ISet<Namespace>> GetAllMembers(IModuleContext moduleContext) {
             return new Dictionary<string, ISet<Namespace>>();
         }