Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

implement various performance optimizations #2688

Merged
merged 2 commits into from Aug 30, 2017
Merged

implement various performance optimizations #2688

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7018bba
implement various performance optimizations, fixes https://github.com…
matthid Aug 29, 2017
213e84b
fix build
matthid Aug 29, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions Paket.sln
View file
Open in desktop
Expand Up @@ -163,4 +163,7 @@ Global
GlobalSection(Performance) = preSolution
HasPerformanceSessions = true
EndGlobalSection
GlobalSection(Performance) = preSolution
HasPerformanceSessions = true
EndGlobalSection
EndGlobal
23 changes: 9 additions & 14 deletions src/Paket.Core/Installation/InstallProcess.fs
View file
Open in desktop
Expand Up @@ -203,14 +203,15 @@ let private applyBindingRedirects isFirstGroup createNewBindingFiles redirects c
None)

|> Seq.choose id
|> Seq.collect (fun p -> dependencyGraph |> getOrAdd p dependencies)
|> Seq.append (
|> Seq.map (fun p -> dependencyGraph |> getOrAdd p dependencies)
|> Set.unionMany
|> Set.union (
referenceFile.Groups
|> Seq.filter (fun g -> g.Key = groupName)
|> Seq.collect (fun g -> g.Value.NugetPackages |> List.map (fun p -> (groupName,p.Name)))
|> Seq.collect(fun (g,p) -> findDependencies(g,p,projectFile.FileName))
|> Seq.map (fun x -> x, projectFile.GetTargetProfile()))
|> Set.ofSeq
|> Seq.map (fun x -> x, projectFile.GetTargetProfile())
|> Set.ofSeq)
| None -> Set.empty

let bindingRedirects (projectFile : ProjectFile) =
Expand Down Expand Up @@ -330,11 +331,8 @@ let InstallIntoProjects(options : InstallerOptions, forceTouch, dependenciesFile
| None -> Choice2Of2 <| sprintf " - %s uses NuGet package %O, but it was not found in the paket.lock file in group %O.%s" referenceFile.FileName ps.Name kv.Key (lockFile.CheckIfPackageExistsInAnyGroup ps.Name)

match group, package with
| Choice1Of2 group, Choice1Of2 package ->
let resolvedSettings =
[package.Settings; group.Options.Settings]
|> List.fold (+) ps.Settings
((kv.Key,ps.Name), (package.Version,resolvedSettings))
| Choice1Of2 _, Choice1Of2 package ->
((kv.Key,ps.Name), (package.Version, ps.Settings + package.Settings))
|> Choice1Of2
| Choice2Of2 error1, Choice2Of2 error2 -> Choice2Of2 (error1 + "\n" + error2)
| Choice2Of2 error, _ | _, Choice2Of2 error -> Choice2Of2 error
Expand All @@ -361,13 +359,10 @@ let InstallIntoProjects(options : InstallerOptions, forceTouch, dependenciesFile
lockFile.Groups |> Map.containsKey groupName)
(fun (groupName,(_,parentSettings), dep) ->
let group = lockFile.Groups.[groupName]
match group.Resolution |> Map.tryFind dep with
match group.TryFind dep with
| None -> None
| Some p ->
let resolvedSettings =
[p.Settings; group.Options.Settings]
|> List.fold (+) parentSettings
Some ((groupName,p.Name), (p.Version,resolvedSettings)) )
Some ((groupName,p.Name), (p.Version,parentSettings + p.Settings)) )
(fun (groupName,(_,parentSettings), dep) ->
Some <| sprintf " - %s uses the group %O, but this group was not found in paket.lock." referenceFile.FileName groupName
)
Expand Down
66 changes: 40 additions & 26 deletions src/Paket.Core/PaketConfigFiles/ProjectFile.fs
View file
Open in desktop
Expand Up @@ -154,7 +154,11 @@ type ProjectFile =
OriginalText : string
Document : XmlDocument
ProjectNode : XmlNode
Language : ProjectLanguage }
Language : ProjectLanguage
// Caches for optimizations
mutable DefaultProperties : Map<string,string> option
// CalculatedMaps for optimizations
mutable CalculatedProperties : System.Collections.Concurrent.ConcurrentDictionary<Map<string,string>, Map<string,string>> }


[<CompilationRepresentation(CompilationRepresentationFlags.ModuleSuffix)>]
Expand Down Expand Up @@ -185,7 +189,8 @@ module ProjectFile =
ProjectNode = projectNode
OriginalText = Utils.normalizeXml doc
Language = LanguageEvaluation.getProjectLanguage doc (Path.GetFileName fullName)

DefaultProperties = None
CalculatedProperties = new System.Collections.Concurrent.ConcurrentDictionary<Map<string,string>, Map<string,string>>()
}

let loadFromFile(fileName:string) =
Expand Down Expand Up @@ -218,39 +223,45 @@ module ProjectFile =
with
| _ ->
project.Document.CreateElement (name, Constants.ProjectDefaultNameSpace)

let createNodeSet name text (project:ProjectFile) =
let node = createNode name project
node.InnerText <- text
node

let getReservedProperties (projectFile:ProjectFile) =
let projectFileInfo = FileInfo projectFile.FileName
let directoryNoRoot = Regex.Replace(projectFileInfo.FullName, "^.:\\\\?", "")
[
// Project file properties
"MSBuildProjectDirectory", projectFileInfo.DirectoryName
"MSBuildProjectDirectoryNoRoot", directoryNoRoot
"MSBuildProjectExtension", projectFileInfo.Extension
"MSBuildProjectFile", projectFileInfo.Name
"MSBuildProjectFullPath", projectFileInfo.FullName
"MSBuildProjectName", Path.GetFileNameWithoutExtension(projectFileInfo.FullName)

// This file properties (Potentially an Imported file)
"MSBuildThisFileDirectory", projectFileInfo.DirectoryName + (string Path.DirectorySeparatorChar)
"MSBuildThisFileDirectoryNoRoot", directoryNoRoot + (string Path.DirectorySeparatorChar)
"MSBuildThisFileExtension", projectFileInfo.Extension
"MSBuildThisFile", projectFileInfo.Name
"MSBuildThisFileFullPath", projectFileInfo.FullName
"MSBuildThisFileName", Path.GetFileNameWithoutExtension(projectFileInfo.FullName)

] |> Map.ofList
let private getReservedProperties (projectFile:ProjectFile) =
let calculateNew () =
let projectFileInfo = FileInfo projectFile.FileName
let directoryNoRoot = Regex.Replace(projectFileInfo.FullName, "^.:\\\\?", "")
[
// Project file properties
"MSBuildProjectDirectory", projectFileInfo.DirectoryName
"MSBuildProjectDirectoryNoRoot", directoryNoRoot
"MSBuildProjectExtension", projectFileInfo.Extension
"MSBuildProjectFile", projectFileInfo.Name
"MSBuildProjectFullPath", projectFileInfo.FullName
"MSBuildProjectName", Path.GetFileNameWithoutExtension(projectFileInfo.FullName)

// This file properties (Potentially an Imported file)
"MSBuildThisFileDirectory", projectFileInfo.DirectoryName + (string Path.DirectorySeparatorChar)
"MSBuildThisFileDirectoryNoRoot", directoryNoRoot + (string Path.DirectorySeparatorChar)
"MSBuildThisFileExtension", projectFileInfo.Extension
"MSBuildThisFile", projectFileInfo.Name
"MSBuildThisFileFullPath", projectFileInfo.FullName
"MSBuildThisFileName", Path.GetFileNameWithoutExtension(projectFileInfo.FullName)
] |> Map.ofList
match projectFile.DefaultProperties with
| Some p -> p
| None ->
let res = calculateNew()
projectFile.DefaultProperties <- Some res
res

/// Append two maps with the properties of the second replacing properties of the first
let private appendMap first second =
Map.fold (fun state key value -> Map.add key value state) first second

let getPropertyWithDefaults propertyName defaultProperties (projectFile:ProjectFile) =
let private calculatePropertyMap (projectFile:ProjectFile) defaultProperties =
let defaultProperties = appendMap defaultProperties (getReservedProperties projectFile)

let processPlaceholders (data : Map<string, string>) text =
Expand Down Expand Up @@ -463,7 +474,10 @@ module ProjectFile =
projectFile.Document
|> getDescendants "PropertyGroup"
|> Seq.fold handleElement defaultProperties

map

let getPropertyWithDefaults propertyName defaultProperties (projectFile:ProjectFile) =
let map = projectFile.CalculatedProperties.GetOrAdd(defaultProperties, calculatePropertyMap projectFile)
Map.tryFind propertyName map

let getProperty propertyName (projectFile:ProjectFile) =
Expand Down
75 changes: 51 additions & 24 deletions src/Paket.Core/Versioning/Requirements.fs
View file
Open in desktop
Expand Up @@ -450,46 +450,73 @@ module FrameworkRestriction =


let rec private And2 (left : FrameworkRestriction) (right : FrameworkRestriction) =
match left.OrFormulas with
| [] -> right
| [h] ->
right.OrFormulas
match left.OrFormulas, right.OrFormulas with
// false -> stay false
| [], _ -> true, left
| _, [] -> true, right
// true -> use the other
| _, [ { Literals = [] }] -> true, left
| [{ Literals = [] }], _ -> true, right
| otherFormulas, [h]
| [h], otherFormulas ->
false,
otherFormulas
|> List.map (fun andFormula -> { Literals = andFormula.Literals @ h.Literals } )
|> FrameworkRestriction.FromOrList
| h :: t ->
(And2 (FrameworkRestriction.FromOrList [h]) right).OrFormulas @ (And2 (FrameworkRestriction.FromOrList t) right).OrFormulas
| h :: t, _ ->
false,
((And2 (FrameworkRestriction.FromOrList [h]) right) |> snd).OrFormulas @ ((And2 (FrameworkRestriction.FromOrList t) right) |> snd).OrFormulas
|> FrameworkRestriction.FromOrList

let And (rst:FrameworkRestriction list) =
List.fold And2 NoRestriction rst
|> simplify

let isSimple, result =
List.fold
(fun (isSimplified, current) next -> let wasSimple, result = And2 current next in wasSimple && isSimplified, result)
(true, NoRestriction)
rst
if isSimple then result
else simplify result

let private Or2 (left : FrameworkRestriction) (right : FrameworkRestriction) =
left.OrFormulas @ right.OrFormulas
|> FrameworkRestriction.FromOrList

match left.OrFormulas, right.OrFormulas with
// false -> use the other
| [], _ -> true, right
| _, [] -> true, left
// true -> become true
| _, [ { Literals = [] }] -> true, right
| [{ Literals = [] }], _ -> true, left
| leftFormumas, rightFormulas ->
false,
leftFormumas @ rightFormulas
|> FrameworkRestriction.FromOrList

let Or (rst:FrameworkRestriction list) =
List.fold Or2 EmptySet rst
|> simplify

let isSimple, result =
List.fold
(fun (isSimplified, current) next -> let wasSimple, result = Or2 current next in wasSimple && isSimplified, result)
(true, EmptySet)
rst
if isSimple then result
else simplify result

//[<Obsolete ("Method is provided for completeness sake. But I don't think its needed")>]
//let Not (rst:FrameworkRestriction) =
// Unchecked.defaultof<_>

let Between (x, y) =
And2 (AtLeast x) (NotAtLeast y)
|> simplify
let isSimple, result = And2 (AtLeast x) (NotAtLeast y)
if isSimple then result else simplify result

let combineRestrictionsWithOr (x : FrameworkRestriction) y =
Or2 x y
|> simplify
let isSimple, result = Or2 x y
if isSimple then result else simplify result

let (|HasNoRestriction|_|) x =
if x = NoRestriction then Some () else None

let combineRestrictionsWithAnd (x : FrameworkRestriction) y =
And2 x y
|> simplify
let combineRestrictionsWithAnd (x : FrameworkRestriction) y =
let isSimple, result = And2 x y
if isSimple then result else simplify result

type FrameworkRestrictions =
| ExplicitRestriction of FrameworkRestriction
Expand Down
8 changes: 4 additions & 4 deletions src/Paket/Paket.fsproj
View file
Open in desktop
Expand Up @@ -31,8 +31,8 @@
<StartAction>Project</StartAction>
<StartProgram>paket.exe</StartProgram>
<StartAction>Project</StartAction>
<StartArguments>fix-nuspecs files src/Paket.Core.preview3/obj/Paket.Core.1.0.0.nuspec project-file src/Paket.Core.preview3/Paket.Core.fsproj</StartArguments>
<StartWorkingDirectory>C:\proj\Paket</StartWorkingDirectory>
<StartArguments>install</StartArguments>
<StartWorkingDirectory>C:\proj\testing\VS2017PerfIssue\</StartWorkingDirectory>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<Optimize>true</Optimize>
Expand All @@ -42,8 +42,8 @@
<WarningLevel>3</WarningLevel>
<DocumentationFile>
</DocumentationFile>
<StartArguments>fix-nuspecs files src/Paket.Core.preview3/obj/Paket.Core.1.0.0.nuspec project-file src/Paket.Core.preview3/Paket.Core.fsproj</StartArguments>
<StartWorkingDirectory>C:\proj\Paket</StartWorkingDirectory>
<StartArguments>install</StartArguments>
<StartWorkingDirectory>C:\proj\testing\VS2017PerfIssue\</StartWorkingDirectory>
</PropertyGroup>
<PropertyGroup>
<VisualStudioVersion Condition=" '$(VisualStudioVersion)' == '' ">14.0</VisualStudioVersion>
Expand Down
4 changes: 3 additions & 1 deletion tests/Paket.Tests/ProjectFile/FileBuildActionSpecs.fs
View file
Open in desktop
Expand Up @@ -10,7 +10,9 @@ let createProject name =
OriginalText = ""
Document = XmlDocument()
ProjectNode = null
Language = ProjectLanguage.Unknown }
Language = ProjectLanguage.Unknown
DefaultProperties = None
CalculatedProperties = new System.Collections.Concurrent.ConcurrentDictionary<_,_>() }

[<Test>]
let ``should recognize compilable files``() =
Expand Down
14 changes: 8 additions & 6 deletions tests/Paket.Tests/Simplifier/BasicScenarioSpecs.fs
View file
Open in desktop
Expand Up @@ -10,12 +10,14 @@ open Paket.TestHelpers
open Paket.InstallProcess

let dummyDir = System.IO.DirectoryInfo("C:/")
let dummyProjectFile =
let dummyProjectFile () =
{ FileName = ""
OriginalText = ""
Document = null
ProjectNode = null
Language = ProjectLanguage.Unknown }
Language = ProjectLanguage.Unknown
DefaultProperties = None
CalculatedProperties = new System.Collections.Concurrent.ConcurrentDictionary<_,_>() }

let lockFile1 = """
NUGET
Expand All @@ -40,7 +42,7 @@ nuget D 2.1""" |> DependenciesFile.FromSource

let projects1 = [
ReferencesFile.FromLines [|"A";"B";"C";"D"|]
ReferencesFile.FromLines [|"B";"C"|] ] |> List.zip [dummyProjectFile; dummyProjectFile]
ReferencesFile.FromLines [|"B";"C"|] ] |> List.zip [dummyProjectFile(); dummyProjectFile()]

[<Test>]
let ``should remove one level deep transitive dependencies from dep and ref files``() =
Expand Down Expand Up @@ -83,7 +85,7 @@ nuget F 1.0""" |> DependenciesFile.FromSource

let projects2 = [
ReferencesFile.FromLines [|"A";"B";"C";"D";"F"|]
ReferencesFile.FromLines [|"C";"D";"E"|] ] |> List.zip [dummyProjectFile; dummyProjectFile]
ReferencesFile.FromLines [|"C";"D";"E"|] ] |> List.zip [dummyProjectFile(); dummyProjectFile()]

[<Test>]
let ``should remove all transitive dependencies from dep file recursively``() =
Expand Down Expand Up @@ -161,7 +163,7 @@ nuget F 1.0""" |> DependenciesFile.FromSource

let projects3 = [
ReferencesFile.FromLines [|"A";"B";"C";"D";"F"|]
ReferencesFile.FromLines [|"C";"D";"E"|] ] |> List.zip [dummyProjectFile; dummyProjectFile]
ReferencesFile.FromLines [|"C";"D";"E"|] ] |> List.zip [dummyProjectFile(); dummyProjectFile()]

[<Test>]
let ``should remove all transitive dependencies from dep file with multiple groups``() =
Expand Down Expand Up @@ -224,7 +226,7 @@ nuget F 1.0""" |> DependenciesFile.FromSource

let projects4 = [
ReferencesFile.FromLines [|"group Foo";"A";"B";"C";"D";"F"|]
ReferencesFile.FromLines [|"group Foo";"C";"D";"E"|] ] |> List.zip [dummyProjectFile; dummyProjectFile]
ReferencesFile.FromLines [|"group Foo";"C";"D";"E"|] ] |> List.zip [dummyProjectFile(); dummyProjectFile()]

[<Test>]
let ``should remove all transitive dependencies from dep file and ref file with empty main group and non empty group foo``() =
Expand Down
27 changes: 27 additions & 0 deletions tests/Paket.Tests/Versioning/FrameworkRestrictionTests.fs
View file
Open in desktop
Expand Up @@ -5,6 +5,33 @@ open FsUnit
open NUnit.Framework
open Paket.Requirements

[<Test>]
let ``Simplify && (false) (< net45)`` () =
let toSimplify =
(FrameworkRestriction.And[
FrameworkRestriction.EmptySet
FrameworkRestriction.NotAtLeast (DotNetFramework FrameworkVersion.V4_5)])
toSimplify
|> shouldEqual (FrameworkRestriction.EmptySet)

[<Test>]
let ``Simplify && (< net45) (false)`` () =
let toSimplify =
(FrameworkRestriction.And[
FrameworkRestriction.NotAtLeast (DotNetFramework FrameworkVersion.V4_5)
FrameworkRestriction.EmptySet])
toSimplify
|> shouldEqual (FrameworkRestriction.EmptySet)

[<Test>]
let ``Simplify && (< net45) (true)`` () =
let toSimplify =
(FrameworkRestriction.And[
FrameworkRestriction.NotAtLeast (DotNetFramework FrameworkVersion.V4_5)
FrameworkRestriction.NoRestriction])
toSimplify
|> shouldEqual (FrameworkRestriction.NotAtLeast (DotNetFramework FrameworkVersion.V4_5))

[<Test>]
let ``Simplify && (true) (< net45)`` () =
let toSimplify =
Expand Down