From f62158bae5a300be60abf3d97ae7cb4f83e7267d Mon Sep 17 00:00:00 2001 From: Don Syme Date: Fri, 23 Mar 2018 15:06:48 +0000 Subject: [PATCH 1/3] update perf results --- tests/scripts/compiler-perf-results.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/compiler-perf-results.txt b/tests/scripts/compiler-perf-results.txt index 867991b488..46e50507e7 100644 --- a/tests/scripts/compiler-perf-results.txt +++ b/tests/scripts/compiler-perf-results.txt @@ -137,3 +137,4 @@ https://github.com/dsyme/visualfsharp.git range1 e49a https://github.com/Microsoft/visualfsharp master 221224e6d20bd835c2b9e01e0a52bf45e740a8d0 221224e6d20bd835c2b9e01e0a52bf45e740a8d0 MSRC-3617253 273.00 12.46 32.80 51.38 60.93 58.66 https://github.com/dsyme/visualfsharp.git weak2 35b7e2caed9b81e2ceb9de9f325ddeb550bf97d6 4df997507226caa272f2c7d4fbdc52eb71c8ead2 MSRC-3617253 257.26 11.24 30.62 48.03 57.80 57.60 https://github.com/Microsoft/visualfsharp master 4df997507226caa272f2c7d4fbdc52eb71c8ead2 4df997507226caa272f2c7d4fbdc52eb71c8ead2 MSRC-3617253 254.53 11.55 31.80 46.58 57.03 58.89 +https://github.com/Microsoft/visualfsharp master 65d87f0b2ee67e50503540aad5d4438fdde14fea 65d87f0b2ee67e50503540aad5d4438fdde14fea MSRC-3617253 257.92 10.44 32.62 45.45 56.68 58.53 From 48a2f59cc043620fcd209110111d6b49bb7b9251 Mon Sep 17 00:00:00 2001 From: Eugene Auduchinok Date: Sat, 24 Mar 2018 22:36:24 +0300 Subject: [PATCH 2/3] Remove duplicate illex.fsl include (#4621) --- fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj b/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj index ce45abbc31..39664cc417 100644 --- a/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj +++ b/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj @@ -278,10 +278,6 @@ ILXErase/EraseUnions.fs - - --unicode --lexlib Internal.Utilities.Text.Lexing - AbsIL/illex.fsl - --unicode --lexlib Internal.Utilities.Text.Lexing ParserAndUntypedAST/lex.fsl From 432784c98929b9f678fe15db2a84e1b7d12789d5 Mon Sep 17 00:00:00 2001 From: Don Syme Date: Tue, 27 Mar 2018 17:38:10 +0100 Subject: [PATCH 3/3] Fix FCS problem with duplicate ByteFile (#4629) * fix memory usage for FCS case * use explicit version flag for FCS until #3113 is fixed * fix build * use explicit version flag for FCS until #3113 is fixed * use WeakByteFile * add cache size parameter * add cache size parameter * fix build --- .../FSharp.Compiler.Service.Tests.fsproj | 2 +- fcs/README.md | 6 +- fcs/RELEASE_NOTES.md | 7 ++ fcs/fcs.props | 3 +- src/absil/ilread.fs | 97 ++++++++++--------- 5 files changed, 63 insertions(+), 52 deletions(-) diff --git a/fcs/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj b/fcs/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj index 148b5f4dad..88236b9dff 100644 --- a/fcs/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj +++ b/fcs/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj @@ -6,7 +6,7 @@ net46;netcoreapp2.0 - $(NoWarn);44; + $(NoWarn);44;75; true true false diff --git a/fcs/README.md b/fcs/README.md index d297185ee6..5110d7cbcc 100644 --- a/fcs/README.md +++ b/fcs/README.md @@ -60,9 +60,9 @@ which does things like: Yu can push the packages if you have permissions, either automatically using ``build Release`` or manually set APIKEY=... - .nuget\nuget.exe push release\fcs\FSharp.Compiler.Service.21.0.1.nupkg %APIKEY% -Source https://nuget.org - .nuget\nuget.exe push release\fcs\FSharp.Compiler.Service.MSBuild.v12.21.0.1.nupkg %APIKEY% -Source https://nuget.org - .nuget\nuget.exe push release\fcs\FSharp.Compiler.Service.ProjectCracker.21.0.1.nupkg %APIKEY% -Source https://nuget.org + ..\fsharp\.nuget\nuget.exe push %HOMEDRIVE%%HOMEPATH%\Downloads\FSharp.Compiler.Service.22.0.2.nupkg %APIKEY% -Source https://nuget.org + ..\fsharp\.nuget\nuget.exe push %HOMEDRIVE%%HOMEPATH%\Downloads\FSharp.Compiler.Service.MSBuild.v12.22.0.2.nupkg %APIKEY% -Source https://nuget.org + ..\fsharp\.nuget\nuget.exe push %HOMEDRIVE%%HOMEPATH%\Downloads\FSharp.Compiler.Service.ProjectCracker.22.0.2.nupkg %APIKEY% -Source https://nuget.org ### Use of Paket and FAKE diff --git a/fcs/RELEASE_NOTES.md b/fcs/RELEASE_NOTES.md index 59a8745a7e..df8f86645d 100644 --- a/fcs/RELEASE_NOTES.md +++ b/fcs/RELEASE_NOTES.md @@ -1,3 +1,10 @@ +#### 22.0.2 + * Use correct version number in DLLs (needed until https://github.com/Microsoft/visualfsharp/issues/3113 is fixed) + +#### 22.0.1 + * Integrate visualfsharp master + * Includes recent memory usage reduction work for ByteFile and ILAttributes + #### 21.0.1 * Use new .NET SDK project files * FSharp.Compiler.Service nuget now uses net45 and netstandard2.0 diff --git a/fcs/fcs.props b/fcs/fcs.props index c3e2198b38..4a4dfbc007 100644 --- a/fcs/fcs.props +++ b/fcs/fcs.props @@ -3,7 +3,8 @@ - 21.0.1 + 22.0.2 + --version:$(VersionPrefix) $(FSharpSourcesRoot)\..\packages\FSharp.Compiler.Tools.4.1.27\tools diff --git a/src/absil/ilread.fs b/src/absil/ilread.fs index d9d66d2464..91290dddf8 100644 --- a/src/absil/ilread.fs +++ b/src/absil/ilread.fs @@ -33,9 +33,11 @@ open System.Reflection let checking = false let logging = false -let _ = if checking then dprintn "warning : Ilread.checking is on" +let _ = if checking then dprintn "warning : ILBinaryReader.checking is on" let noStableFileHeuristic = try (System.Environment.GetEnvironmentVariable("FSharp_NoStableFileHeuristic") <> null) with _ -> false let alwaysMemoryMapFSC = try (System.Environment.GetEnvironmentVariable("FSharp_AlwaysMemoryMapCommandLineCompiler") <> null) with _ -> false +let stronglyHeldReaderCacheSizeDefault = 30 +let stronglyHeldReaderCacheSize = try (match System.Environment.GetEnvironmentVariable("FSharp_StronglyHeldBinaryReaderCacheSize") with null -> stronglyHeldReaderCacheSizeDefault | s -> int32 s) with _ -> stronglyHeldReaderCacheSizeDefault let singleOfBits (x:int32) = System.BitConverter.ToSingle(System.BitConverter.GetBytes(x), 0) let doubleOfBits (x:int64) = System.BitConverter.Int64BitsToDouble(x) @@ -346,7 +348,7 @@ type ByteFile(fileName: string, bytes:byte[]) = /// This is the default implementation used by F# Compiler Services when accessing "stable" binaries. It is not used /// by Visual Studio, where tryGetMetadataSnapshot provides a RawMemoryFile backed by Roslyn data. [] -type WeakByteFile(fileName: string) = +type WeakByteFile(fileName: string, chunk: (int * int) option) = do stats.weakByteFileCount <- stats.weakByteFileCount + 1 @@ -357,30 +359,29 @@ type WeakByteFile(fileName: string) = let weakBytes = new WeakReference (null) member __.FileName = fileName - /// Get the bytes for the file - member this.Get() = - let mutable tg = null - if not (weakBytes.TryGetTarget(&tg)) then - if FileSystem.GetLastWriteTimeShim(fileName) <> fileStamp then - errorR (Error (FSComp.SR.ilreadFileChanged fileName, range0)) - - tg <- FileSystem.ReadAllBytesShim fileName - weakBytes.SetTarget tg - tg + /// Get the bytes for the file interface BinaryFile with - override __.GetView() = - let mutable tg = null + + override this.GetView() = let strongBytes = + let mutable tg = null if not (weakBytes.TryGetTarget(&tg)) then if FileSystem.GetLastWriteTimeShim(fileName) <> fileStamp then - errorR (Error (FSComp.SR.ilreadFileChanged fileName, range0)) + error (Error (FSComp.SR.ilreadFileChanged fileName, range0)) + + let bytes = + match chunk with + | None -> FileSystem.ReadAllBytesShim fileName + | Some(start, length) -> File.ReadBinaryChunk (fileName, start, length) + + tg <- bytes + + weakBytes.SetTarget bytes - tg <- FileSystem.ReadAllBytesShim fileName - weakBytes.SetTarget tg tg - (ByteView(strongBytes) :> BinaryView) + (ByteView(strongBytes) :> BinaryView) let seekReadByte (mdv:BinaryView) addr = mdv.ReadByte addr @@ -3927,27 +3928,31 @@ type ILModuleReader(ilModule: ILModuleDef, ilAssemblyRefs: Lazy(0, areSimilar=(fun (x, y) -> x = y)) +type ILModuleReaderCacheKey = ILModuleReaderCacheKey of string * DateTime * ILScopeRef * bool * ReduceMemoryFlag * MetadataOnlyFlag +let ilModuleReaderCache = new AgedLookup(stronglyHeldReaderCacheSize, areSimilar=(fun (x, y) -> x = y)) let ilModuleReaderCacheLock = Lock() let stableFileHeuristicApplies fileName = not noStableFileHeuristic && try FileSystem.IsStableFileHeuristic fileName with _ -> false -let createByteFile opts fileName = +let createByteFileChunk opts fileName chunk = // If we're trying to reduce memory usage then we are willing to go back and re-read the binary, so we can use // a weakly-held handle to an array of bytes. if opts.reduceMemoryUsage = ReduceMemoryFlag.Yes && stableFileHeuristicApplies fileName then - WeakByteFile(fileName) :> BinaryFile + WeakByteFile(fileName, chunk) :> BinaryFile else - let bytes = FileSystem.ReadAllBytesShim(fileName) + let bytes = + match chunk with + | None -> FileSystem.ReadAllBytesShim fileName + | Some (start, length) -> File.ReadBinaryChunk(fileName, start, length) ByteFile(fileName, bytes) :> BinaryFile -let tryMemoryMap opts fileName = +let tryMemoryMapWholeFile opts fileName = let file = try MemoryMapFile.Create fileName :> BinaryFile with _ -> - createByteFile opts fileName + createByteFileChunk opts fileName None let disposer = { new IDisposable with member __.Dispose() = @@ -3963,17 +3968,16 @@ let OpenILModuleReaderFromBytes fileName bytes opts = let OpenILModuleReader fileName opts = // Pseudo-normalize the paths. - let ((_,writeStamp,_,_,_,_) as key), keyOk = + let (ILModuleReaderCacheKey (fullPath,writeStamp,_,_,_,_) as key), keyOk = try - (FileSystem.GetFullPathShim(fileName), - FileSystem.GetLastWriteTimeShim(fileName), - opts.ilGlobals.primaryAssemblyScopeRef, - opts.pdbPath.IsSome, - opts.reduceMemoryUsage, - opts.metadataOnly), true - with e -> - System.Diagnostics.Debug.Assert(false, sprintf "Failed to compute key in OpenILModuleReader cache for '%s'. Falling back to uncached." fileName) - ("", System.DateTime.UtcNow, ILScopeRef.Local, false, ReduceMemoryFlag.Yes, MetadataOnlyFlag.Yes), false + let fullPath = FileSystem.GetFullPathShim(fileName) + let writeTime = FileSystem.GetLastWriteTimeShim(fileName) + let key = ILModuleReaderCacheKey (fullPath, writeTime, opts.ilGlobals.primaryAssemblyScopeRef, opts.pdbPath.IsSome, opts.reduceMemoryUsage, opts.metadataOnly) + key, true + with exn -> + System.Diagnostics.Debug.Assert(false, sprintf "Failed to compute key in OpenILModuleReader cache for '%s'. Falling back to uncached. Error = %s" fileName (exn.ToString())) + let fakeKey = ILModuleReaderCacheKey(fileName, System.DateTime.UtcNow, ILScopeRef.Local, false, ReduceMemoryFlag.Yes, MetadataOnlyFlag.Yes) + fakeKey, false let cacheResult = if keyOk then @@ -3999,30 +4003,29 @@ let OpenILModuleReader fileName opts = // See if tryGetMetadata gives us a BinaryFile for the metadata section alone. let mdfileOpt = - match opts.tryGetMetadataSnapshot (fileName, writeStamp) with - | Some (obj, start, len) -> Some (RawMemoryFile(fileName, obj, start, len) :> BinaryFile) + match opts.tryGetMetadataSnapshot (fullPath, writeStamp) with + | Some (obj, start, len) -> Some (RawMemoryFile(fullPath, obj, start, len) :> BinaryFile) | None -> None // For metadata-only, always use a temporary, short-lived PE file reader, preferably over a memory mapped file. // Then use the metadata blob as the long-lived memory resource. - let disposer, pefileEager = tryMemoryMap opts fileName + let disposer, pefileEager = tryMemoryMapWholeFile opts fullPath use _disposer = disposer - let (metadataPhysLoc, metadataSize, peinfo, pectxtEager, pevEager, _pdb) = openPEFileReader (fileName, pefileEager, None) + let (metadataPhysLoc, metadataSize, peinfo, pectxtEager, pevEager, _pdb) = openPEFileReader (fullPath, pefileEager, None) let mdfile = match mdfileOpt with | Some mdfile -> mdfile | None -> // If tryGetMetadata doesn't give anything, then just read the metadata chunk out of the binary - let bytes = File.ReadBinaryChunk (fileName, metadataPhysLoc, metadataSize) - ByteFile(fileName, bytes) :> BinaryFile + createByteFileChunk opts fullPath (Some (metadataPhysLoc, metadataSize)) - let ilModule, ilAssemblyRefs = openPEMetadataOnly (fileName, peinfo, pectxtEager, pevEager, mdfile, reduceMemoryUsage, opts.ilGlobals) + let ilModule, ilAssemblyRefs = openPEMetadataOnly (fullPath, peinfo, pectxtEager, pevEager, mdfile, reduceMemoryUsage, opts.ilGlobals) new ILModuleReader(ilModule, ilAssemblyRefs, ignore) else // If we are not doing metadata-only, then just go ahead and read all the bytes and hold them either strongly or weakly // depending on the heuristic - let pefile = createByteFile opts fileName - let ilModule, ilAssemblyRefs, _pdb = openPE (fileName, pefile, None, reduceMemoryUsage, opts.ilGlobals) + let pefile = createByteFileChunk opts fullPath None + let ilModule, ilAssemblyRefs, _pdb = openPE (fullPath, pefile, None, reduceMemoryUsage, opts.ilGlobals) new ILModuleReader(ilModule, ilAssemblyRefs, ignore) if keyOk then @@ -4042,14 +4045,14 @@ let OpenILModuleReader fileName opts = // multi-proc build. So use memory mapping, but only for stable files. Other files // fill use an in-memory ByteFile let _disposer, pefile = - if alwaysMemoryMapFSC || stableFileHeuristicApplies fileName then - tryMemoryMap opts fileName + if alwaysMemoryMapFSC || stableFileHeuristicApplies fullPath then + tryMemoryMapWholeFile opts fullPath else - let pefile = createByteFile opts fileName + let pefile = createByteFileChunk opts fullPath None let disposer = { new IDisposable with member __.Dispose() = () } disposer, pefile - let ilModule, ilAssemblyRefs, pdb = openPE (fileName, pefile, opts.pdbPath, reduceMemoryUsage, opts.ilGlobals) + let ilModule, ilAssemblyRefs, pdb = openPE (fullPath, pefile, opts.pdbPath, reduceMemoryUsage, opts.ilGlobals) let ilModuleReader = new ILModuleReader(ilModule, ilAssemblyRefs, (fun () -> ClosePdbReader pdb)) // Readers with PDB reader disposal logic don't go in the cache. Note the PDB reader is only used in static linking.