[WIP]Adding paket package resolution to fsi and VF# editor

VisualFSharp.sln

@@ -134,6 +134,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Utils", "Utils", "{D086C8C6
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "LanguageServiceProfiling", "vsintegration\Utils\LanguageServiceProfiling\LanguageServiceProfiling.fsproj", "{E7FA3A71-51AF-4FCA-9C2F-7C853E515903}"
 EndProject
+Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.DependencyManager.Paket", "src\fsharp\FSharp.DependencyManager.Paket\FSharp.DependencyManager.Paket.fsproj", "{25555554-522D-4CF7-97E4-BA940F0B18F3}"
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FSharp.UIResources", "vsintegration\src\FSharp.UIResources\FSharp.UIResources.csproj", "{C4586A06-1402-48BC-8E35-A1B8642F895B}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSharp_Analysis", "tests\service\data\CSharp_Analysis\CSharp_Analysis.csproj", "{887630A3-4B1D-40EA-B8B3-2D842E9C40DB}"
@@ -550,6 +551,18 @@ Global
 		{E7FA3A71-51AF-4FCA-9C2F-7C853E515903}.Release|Any CPU.Build.0 = Release|Any CPU
 		{E7FA3A71-51AF-4FCA-9C2F-7C853E515903}.Release|x86.ActiveCfg = Release|Any CPU
 		{E7FA3A71-51AF-4FCA-9C2F-7C853E515903}.Release|x86.Build.0 = Release|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Debug|x86.Build.0 = Debug|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Proto|Any CPU.ActiveCfg = Proto|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Proto|Any CPU.Build.0 = Proto|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Proto|x86.ActiveCfg = Proto|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Proto|x86.Build.0 = Proto|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Release|x86.ActiveCfg = Release|Any CPU
+		{25555554-522D-4CF7-97E4-BA940F0B18F3}.Release|x86.Build.0 = Release|Any CPU
 		{C4586A06-1402-48BC-8E35-A1B8642F895B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C4586A06-1402-48BC-8E35-A1B8642F895B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C4586A06-1402-48BC-8E35-A1B8642F895B}.Debug|x86.ActiveCfg = Debug|Any CPU
@@ -635,6 +648,7 @@ Global
 		{2E60864A-E3FF-4BCC-810F-DC7C34E6B236} = {CFE3259A-2D30-4EB0-80D5-E8B5F3D01449}
 		{D086C8C6-D00D-4C3B-9AB2-A4286C9F5922} = {4C7B48D7-19AF-4AE7-9D1D-3BB289D5480D}
 		{E7FA3A71-51AF-4FCA-9C2F-7C853E515903} = {D086C8C6-D00D-4C3B-9AB2-A4286C9F5922}
+		{25555554-522D-4CF7-97E4-BA940F0B18F3} = {B8DDA694-7939-42E3-95E5-265C2217C142}
 		{C4586A06-1402-48BC-8E35-A1B8642F895B} = {4C7B48D7-19AF-4AE7-9D1D-3BB289D5480D}
 		{887630A3-4B1D-40EA-B8B3-2D842E9C40DB} = {35636A82-401A-4C3A-B2AB-EB7DC5E9C268}
 		{35636A82-401A-4C3A-B2AB-EB7DC5E9C268} = {F7876C9B-FB6A-4EFB-B058-D6967DB75FB2}

fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <!-- Copyright (c) Microsoft Corporation.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information. -->
 <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
@@ -457,6 +457,12 @@
     <Compile Include="$(FSharpSourcesRoot)\fsharp\IlxGen.fs">
       <Link>CodeGen/IlxGen.fs</Link>
     </Compile>
+    <Compile Include="$(FSharpSourcesRoot)\fsharp\DependencyManager.Integration.fsi">
+      <Link>Driver\DependencyManager.Integration.fsi</Link>
+    </Compile>
+    <Compile Include="$(FSharpSourcesRoot)\fsharp\DependencyManager.Integration.fs">
+      <Link>Driver\DependencyManager.Integration.fs</Link>
+    </Compile>
     <Compile Include="$(FSharpSourcesRoot)\fsharp\CompileOps.fsi">
       <Link>Driver/CompileOps.fsi</Link>
     </Compile>

src/fsharp/CompileOps.fsi

@@ -258,9 +258,11 @@ type TcConfigBuilder =
       mutable light: bool option
       mutable conditionalCompilationDefines: string list
       /// Sources added into the build with #load
-      mutable loadedSources: (range * string) list
+      mutable loadedSources: (range * string * string) list
 
       mutable referencedDLLs: AssemblyReference  list
+      mutable packageManagerLines : Map<string,(string*range) list>
+
       mutable projectReferences : IProjectReference list
       mutable knownUnresolvedReferences : UnresolvedAssemblyReference list
       optimizeForMemory: bool
@@ -668,7 +670,7 @@ val RequireDLL : CompilationThreadToken * TcImports * TcEnv * thisAssemblyName:
 
 /// Processing # commands
 val ProcessMetaCommandsFromInput : 
-    (('T -> range * string -> 'T) * ('T -> range * string -> 'T) * ('T -> range * string -> unit)) 
+    (('T -> range * string -> 'T) * ('T -> range * string -> 'T) * ('T -> DependencyManagerIntegration.IDependencyManagerProvider * range * string -> 'T) * ('T -> range * string -> unit)) 
     -> TcConfigBuilder * Ast.ParsedInput * string * 'T 
     -> 'T
 

src/fsharp/DependencyManager.Integration.fs

@@ -0,0 +1,253 @@
+// Copyright (c) Microsoft Corporation.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+/// Helper members to integrate DependencyManagers into F# codebase
+module internal Microsoft.FSharp.Compiler.DependencyManagerIntegration
+
+open System
+open System.Reflection
+open System.IO
+open Microsoft.FSharp.Compiler.ErrorLogger
+
+#if FX_RESHAPED_REFLECTION
+    open PrimReflectionAdapters
+    open ReflectionAdapters    
+#endif
+
+// NOTE: this contains mostly members whose intents are :
+// * to keep ReferenceLoading.PaketHandler usable outside of F# (so it can be used in scriptcs & others)
+// * to minimize footprint of integration in fsi/CompileOps
+
+/// hardcoded to net461 as we don't have fsi on netcore
+let targetFramework = "net461"
+
+module ReflectionHelper =
+    let assemblyHasAttribute (theAssembly: Assembly) attributeName =
+        try
+            CustomAttributeExtensions.GetCustomAttributes(theAssembly)
+            |> Seq.tryFind (fun a -> a.GetType().Name = attributeName)
+            |> function | Some _ -> true | _ -> false
+        with | _ -> false
+
+    let getAttributeNamed (theType: Type) attributeName =
+        try
+#if FX_RESHAPED_REFLECTION
+            theType.GetTypeInfo().GetCustomAttributes false
+#else
+            theType.GetCustomAttributes false
+#endif
+            |> Seq.tryFind (fun a -> a.GetType().Name = attributeName)
+        with | _ -> None
+
+    let getInstanceProperty<'treturn> (theType: Type) indexParameterTypes propertyName =
+        try
+            let property = theType.GetProperty(propertyName, typeof<'treturn>)
+            if isNull property then
+                None
+            elif not (property.GetGetMethod().IsStatic)
+                 && property.GetIndexParameters() = indexParameterTypes
+            then
+                Some property
+            else
+                None
+        with | _ -> None
+
+    let getInstanceMethod<'treturn> (theType: Type) (parameterTypes: Type array) methodName =
+        try
+            let theMethod = theType.GetMethod(methodName, parameterTypes)
+            if isNull theMethod then
+                None
+            else
+                Some theMethod
+        with | _ -> None
+
+    let implements<'timplemented> (theType: Type) =
+        typeof<'timplemented>.IsAssignableFrom(theType)
+
+(* this is the loose contract for now, just to define the shape, but this is resolved through reflection *)
+type internal IDependencyManagerProvider =
+    inherit System.IDisposable
+    abstract Name : string
+    abstract ToolName: string
+    abstract Key: string
+    abstract ResolveDependencies : targetFramework: string * scriptDir: string * mainScriptName: string * scriptName: string * packageManagerTextLines: string seq -> string option * string list
+
+[<RequireQualifiedAccess>]
+type ReferenceType =
+| RegisteredDependencyManager of IDependencyManagerProvider
+| Library of string
+| UnknownType
+
+type ReflectionDependencyManagerProvider(theType: Type, nameProperty: PropertyInfo, toolNameProperty: PropertyInfo, keyProperty: PropertyInfo, resolveDeps: MethodInfo) =
+    let instance = Activator.CreateInstance(theType) :?> IDisposable
+    let nameProperty     = nameProperty.GetValue >> string
+    let toolNameProperty = toolNameProperty.GetValue >> string
+    let keyProperty      = keyProperty.GetValue >> string
+    static member InstanceMaker (theType: System.Type) = 
+        if not (ReflectionHelper.implements<IDisposable> theType) then None
+        else        
+        match ReflectionHelper.getAttributeNamed theType "FSharpDependencyManagerAttribute" with
+        | None -> None
+        | Some _ ->
+        match ReflectionHelper.getInstanceProperty<string> theType Array.empty "Name" with
+        | None -> None
+        | Some nameProperty ->
+        match ReflectionHelper.getInstanceProperty<string> theType Array.empty "ToolName" with
+        | None -> None
+        | Some toolNameProperty ->
+        match ReflectionHelper.getInstanceProperty<string> theType Array.empty "Key" with
+        | None -> None
+        | Some keyProperty ->
+        match ReflectionHelper.getInstanceMethod<string * string list> theType [|typeof<string>;typeof<string>;typeof<string>;typeof<string>;typeof<string seq>;|] "ResolveDependencies" with
+        | None -> None
+        | Some resolveDependenciesMethod ->
+            Some (fun () -> new ReflectionDependencyManagerProvider(theType, nameProperty, toolNameProperty, keyProperty, resolveDependenciesMethod) :> IDependencyManagerProvider)
+
+    interface IDependencyManagerProvider with
+        member __.Name     = instance |> nameProperty
+        member __.ToolName = instance |> toolNameProperty
+        member __.Key      = instance |> keyProperty
+        member __.ResolveDependencies(targetFramework, scriptDir, mainScriptName, scriptName, packageManagerTextLines) =
+            let arguments = [|box targetFramework; box scriptDir; box mainScriptName; box scriptName; box packageManagerTextLines|]
+            resolveDeps.Invoke(instance, arguments) :?> _
+    interface IDisposable with
+        member __.Dispose () = instance.Dispose()
+
+
+let assemblySearchPaths = lazy(
+    [ let assemblyLocation =
+#if FX_RESHAPED_REFLECTION
+          typeof<IDependencyManagerProvider>.GetTypeInfo().Assembly.Location
+#else
+          typeof<IDependencyManagerProvider>.Assembly.Location
+#endif  
+      yield Path.GetDirectoryName assemblyLocation
+      let executingAssembly = 
+#if FX_RESHAPED_REFLECTION
+          typeof<IDependencyManagerProvider>.GetTypeInfo().Assembly
+#else
+          Assembly.GetExecutingAssembly()
+#endif
+      yield Path.GetDirectoryName(executingAssembly.Location)
+#if FX_NO_APP_DOMAINS
+#else
+      let baseDir = AppDomain.CurrentDomain.BaseDirectory
+      yield baseDir 
+#endif
+    ]
+    |> List.distinct)
+
+let enumerateDependencyManagerAssembliesFromCurrentAssemblyLocation m =
+    assemblySearchPaths.Force()
+    |> Seq.collect (fun path -> Directory.EnumerateFiles(path,"*DependencyManager*.dll"))
+    |> Seq.choose (fun path -> 
+        try
+            Some(AbstractIL.Internal.Library.Shim.FileSystem.AssemblyLoadFrom path)
+        with 
+        | exn ->
+            warning(Error(FSComp.SR.couldNotLoadDependencyManagerExtenstion(path,exn.Message),m))
+            None)
+    |> Seq.filter (fun a -> ReflectionHelper.assemblyHasAttribute a "FSharpDependencyManagerAttribute")
+
+type ProjectDependencyManager() =
+    interface IDependencyManagerProvider with
+        member __.Name = "Project loader"
+        member __.ToolName = ""
+        member __.Key = "project"
+        member __.ResolveDependencies(_targetFramework:string, _scriptDir: string, _mainScriptName: string, _scriptName: string, _packageManagerTextLines: string seq) = 
+            None,[]
+
+    interface System.IDisposable with
+        member __.Dispose() = ()
+
+type RefDependencyManager() =
+    interface IDependencyManagerProvider with
+        member __.Name = "Ref library loader"
+        member __.ToolName = ""
+        member __.Key = "ref"
+        member __.ResolveDependencies(_targetFramework:string, _scriptDir: string, _mainScriptName: string, _scriptName: string, _packageManagerTextLines: string seq) = 
+            None,[]
+
+    interface System.IDisposable with
+        member __.Dispose() = ()
+
+type ImplDependencyManager() =
+    interface IDependencyManagerProvider with
+        member __.Name = "Impl library loader"
+        member __.ToolName = ""
+        member __.Key = "impl"
+        member __.ResolveDependencies(_targetFramework:string, _scriptDir: string, _mainScriptName: string, _scriptName: string, _packageManagerTextLines: string seq) = 
+            None,[]
+
+    interface System.IDisposable with
+        member __.Dispose() = ()
+
+let registeredDependencyManagers = ref None
+
+let RegisteredDependencyManagers m =
+    match !registeredDependencyManagers with
+    | Some managers -> managers
+    | None ->
+        let defaultProviders =
+            [new ProjectDependencyManager() :> IDependencyManagerProvider
+             new RefDependencyManager() :> IDependencyManagerProvider
+             new ImplDependencyManager() :> IDependencyManagerProvider]
+
+        let loadedProviders =
+            enumerateDependencyManagerAssembliesFromCurrentAssemblyLocation m
+            |> Seq.collect (fun a -> a.GetTypes())
+            |> Seq.choose ReflectionDependencyManagerProvider.InstanceMaker
+            |> Seq.map (fun maker -> maker ())
+
+        defaultProviders
+        |> Seq.append loadedProviders
+        |> Seq.map (fun pm -> pm.Key, pm)
+        |> Map.ofSeq
+
+let createPackageManagerUnknownError packageManagerKey m =
+    let registeredKeys = String.Join(", ", RegisteredDependencyManagers m |> Seq.map (fun kv -> kv.Value.Key))
+    let searchPaths = assemblySearchPaths.Force()
+    Error(FSComp.SR.packageManagerUnknown(packageManagerKey, String.Join(", ", searchPaths), registeredKeys),m)
+
+let tryFindDependencyManagerInPath m (path:string) : ReferenceType =
+    try
+        if path.Contains ":" && not (System.IO.Path.IsPathRooted path) then
+            let managers = RegisteredDependencyManagers m
+            match managers |> Seq.tryFind (fun kv -> path.StartsWith(kv.Value.Key + ":" )) with
+            | None ->
+                errorR(createPackageManagerUnknownError (path.Split(':').[0]) m)
+                ReferenceType.UnknownType
+            | Some kv -> ReferenceType.RegisteredDependencyManager kv.Value
+        else
+            ReferenceType.Library path
+    with 
+    | e -> 
+        errorR(Error(FSComp.SR.packageManagerError(e.Message),m))
+        ReferenceType.UnknownType
+
+let removeDependencyManagerKey (packageManagerKey:string) (path:string) = path.Substring(packageManagerKey.Length + 1).Trim()
+
+let tryFindDependencyManagerByKey m (key:string) : IDependencyManagerProvider option =
+    try
+        RegisteredDependencyManagers m |> Map.tryFind key
+    with 
+    | e -> 
+        errorR(Error(FSComp.SR.packageManagerError(e.Message),m))
+        None
+
+let resolve (packageManager:IDependencyManagerProvider) implicitIncludeDir mainScriptName fileName m packageManagerTextLines =
+    try
+        let loadScript,additionalIncludeFolders =
+            packageManager.ResolveDependencies(
+                targetFramework,
+                implicitIncludeDir,
+                mainScriptName,
+                fileName,
+                packageManagerTextLines)
+
+        Some(loadScript,additionalIncludeFolders)
+    with e ->
+        if e.InnerException <> null then
+            errorR(Error(FSComp.SR.packageManagerError(e.InnerException.Message),m))
+        else
+            errorR(Error(FSComp.SR.packageManagerError(e.Message),m))
+        None

src/fsharp/DependencyManager.Integration.fsi

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+/// Helper members to integrate DependencyManagers into F# codebase
+module internal Microsoft.FSharp.Compiler.DependencyManagerIntegration
+
+open Microsoft.FSharp.Compiler.Range
+
+type IDependencyManagerProvider =
+    inherit System.IDisposable
+    abstract Name : string
+    abstract ToolName: string
+    abstract Key: string
+    abstract ResolveDependencies : string * string * string * string * string seq -> string option * string list
+
+[<RequireQualifiedAccess>]
+type ReferenceType =
+| RegisteredDependencyManager of IDependencyManagerProvider
+| Library of string
+| UnknownType
+
+val tryFindDependencyManagerInPath : range -> string -> ReferenceType
+val tryFindDependencyManagerByKey : range -> string -> IDependencyManagerProvider option
+val removeDependencyManagerKey : string -> string -> string
+val createPackageManagerUnknownError : string -> range -> exn
+
+val resolve : IDependencyManagerProvider -> string -> string -> string -> range -> string seq -> (string option * string list) option

src/fsharp/FSComp.txt

@@ -1339,6 +1339,8 @@ tcGlobalsSystemTypeNotFound,"The system type '%s' was required but no referenced
 3213,typrelMemberHasMultiplePossibleDispatchSlots,"The member '%s' matches multiple overloads of the same method.\nPlease restrict it to one of the following:%s."
 3214,methodIsNotStatic,"Method or object constructor '%s' is not static"
 3215,parsUnexpectedSymbolEqualsInsteadOfIn,"Unexpected symbol '=' in expression. Did you intend to use 'for x in y .. z do' instead?"
+3216,packageManagerUnknown,"Package manager key '%s' was not registered in %s. Currently registered: %s"
+3217,packageManagerError,"%s"
 keywordDescriptionAbstract,"Indicates a method that either has no implementation in the type in which it is declared or that is virtual and has a default implementation."
 keyworkDescriptionAnd,"Used in mutually recursive bindings, in property declarations, and with multiple constraints on generic parameters."
 keywordDescriptionAs,"Used to give the current class object an object name. Also used to give a name to a whole pattern within a pattern match."
@@ -1420,3 +1422,4 @@ notAFunctionButMaybeIndexerWithName,"This value is not a function and cannot be
 notAFunctionButMaybeIndexer,"This expression is not a function and cannot be applied. Did you intend to access the indexer via expr.[index] instead?"
 3217,notAFunctionButMaybeIndexerErrorCode,""
 notAFunctionButMaybeDeclaration,"This value is not a function and cannot be applied. Did you forget to terminate a declaration?"
+3218,couldNotLoadDependencyManagerExtenstion,"The dependency manager extension %s could not be loaded. Message: %s"