diff --git a/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj b/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj index ce45abbc31..39664cc417 100644 --- a/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj +++ b/fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj @@ -278,10 +278,6 @@ ILXErase/EraseUnions.fs - - --unicode --lexlib Internal.Utilities.Text.Lexing - AbsIL/illex.fsl - --unicode --lexlib Internal.Utilities.Text.Lexing ParserAndUntypedAST/lex.fsl diff --git a/src/absil/ilread.fs b/src/absil/ilread.fs index d9d66d2464..91290dddf8 100755 --- a/src/absil/ilread.fs +++ b/src/absil/ilread.fs @@ -33,9 +33,11 @@ open System.Reflection let checking = false let logging = false -let _ = if checking then dprintn "warning : Ilread.checking is on" +let _ = if checking then dprintn "warning : ILBinaryReader.checking is on" let noStableFileHeuristic = try (System.Environment.GetEnvironmentVariable("FSharp_NoStableFileHeuristic") <> null) with _ -> false let alwaysMemoryMapFSC = try (System.Environment.GetEnvironmentVariable("FSharp_AlwaysMemoryMapCommandLineCompiler") <> null) with _ -> false +let stronglyHeldReaderCacheSizeDefault = 30 +let stronglyHeldReaderCacheSize = try (match System.Environment.GetEnvironmentVariable("FSharp_StronglyHeldBinaryReaderCacheSize") with null -> stronglyHeldReaderCacheSizeDefault | s -> int32 s) with _ -> stronglyHeldReaderCacheSizeDefault let singleOfBits (x:int32) = System.BitConverter.ToSingle(System.BitConverter.GetBytes(x), 0) let doubleOfBits (x:int64) = System.BitConverter.Int64BitsToDouble(x) @@ -346,7 +348,7 @@ type ByteFile(fileName: string, bytes:byte[]) = /// This is the default implementation used by F# Compiler Services when accessing "stable" binaries. It is not used /// by Visual Studio, where tryGetMetadataSnapshot provides a RawMemoryFile backed by Roslyn data. [] -type WeakByteFile(fileName: string) = +type WeakByteFile(fileName: string, chunk: (int * int) option) = do stats.weakByteFileCount <- stats.weakByteFileCount + 1 @@ -357,30 +359,29 @@ type WeakByteFile(fileName: string) = let weakBytes = new WeakReference (null) member __.FileName = fileName - /// Get the bytes for the file - member this.Get() = - let mutable tg = null - if not (weakBytes.TryGetTarget(&tg)) then - if FileSystem.GetLastWriteTimeShim(fileName) <> fileStamp then - errorR (Error (FSComp.SR.ilreadFileChanged fileName, range0)) - - tg <- FileSystem.ReadAllBytesShim fileName - weakBytes.SetTarget tg - tg + /// Get the bytes for the file interface BinaryFile with - override __.GetView() = - let mutable tg = null + + override this.GetView() = let strongBytes = + let mutable tg = null if not (weakBytes.TryGetTarget(&tg)) then if FileSystem.GetLastWriteTimeShim(fileName) <> fileStamp then - errorR (Error (FSComp.SR.ilreadFileChanged fileName, range0)) + error (Error (FSComp.SR.ilreadFileChanged fileName, range0)) + + let bytes = + match chunk with + | None -> FileSystem.ReadAllBytesShim fileName + | Some(start, length) -> File.ReadBinaryChunk (fileName, start, length) + + tg <- bytes + + weakBytes.SetTarget bytes - tg <- FileSystem.ReadAllBytesShim fileName - weakBytes.SetTarget tg tg - (ByteView(strongBytes) :> BinaryView) + (ByteView(strongBytes) :> BinaryView) let seekReadByte (mdv:BinaryView) addr = mdv.ReadByte addr @@ -3927,27 +3928,31 @@ type ILModuleReader(ilModule: ILModuleDef, ilAssemblyRefs: Lazy(0, areSimilar=(fun (x, y) -> x = y)) +type ILModuleReaderCacheKey = ILModuleReaderCacheKey of string * DateTime * ILScopeRef * bool * ReduceMemoryFlag * MetadataOnlyFlag +let ilModuleReaderCache = new AgedLookup(stronglyHeldReaderCacheSize, areSimilar=(fun (x, y) -> x = y)) let ilModuleReaderCacheLock = Lock() let stableFileHeuristicApplies fileName = not noStableFileHeuristic && try FileSystem.IsStableFileHeuristic fileName with _ -> false -let createByteFile opts fileName = +let createByteFileChunk opts fileName chunk = // If we're trying to reduce memory usage then we are willing to go back and re-read the binary, so we can use // a weakly-held handle to an array of bytes. if opts.reduceMemoryUsage = ReduceMemoryFlag.Yes && stableFileHeuristicApplies fileName then - WeakByteFile(fileName) :> BinaryFile + WeakByteFile(fileName, chunk) :> BinaryFile else - let bytes = FileSystem.ReadAllBytesShim(fileName) + let bytes = + match chunk with + | None -> FileSystem.ReadAllBytesShim fileName + | Some (start, length) -> File.ReadBinaryChunk(fileName, start, length) ByteFile(fileName, bytes) :> BinaryFile -let tryMemoryMap opts fileName = +let tryMemoryMapWholeFile opts fileName = let file = try MemoryMapFile.Create fileName :> BinaryFile with _ -> - createByteFile opts fileName + createByteFileChunk opts fileName None let disposer = { new IDisposable with member __.Dispose() = @@ -3963,17 +3968,16 @@ let OpenILModuleReaderFromBytes fileName bytes opts = let OpenILModuleReader fileName opts = // Pseudo-normalize the paths. - let ((_,writeStamp,_,_,_,_) as key), keyOk = + let (ILModuleReaderCacheKey (fullPath,writeStamp,_,_,_,_) as key), keyOk = try - (FileSystem.GetFullPathShim(fileName), - FileSystem.GetLastWriteTimeShim(fileName), - opts.ilGlobals.primaryAssemblyScopeRef, - opts.pdbPath.IsSome, - opts.reduceMemoryUsage, - opts.metadataOnly), true - with e -> - System.Diagnostics.Debug.Assert(false, sprintf "Failed to compute key in OpenILModuleReader cache for '%s'. Falling back to uncached." fileName) - ("", System.DateTime.UtcNow, ILScopeRef.Local, false, ReduceMemoryFlag.Yes, MetadataOnlyFlag.Yes), false + let fullPath = FileSystem.GetFullPathShim(fileName) + let writeTime = FileSystem.GetLastWriteTimeShim(fileName) + let key = ILModuleReaderCacheKey (fullPath, writeTime, opts.ilGlobals.primaryAssemblyScopeRef, opts.pdbPath.IsSome, opts.reduceMemoryUsage, opts.metadataOnly) + key, true + with exn -> + System.Diagnostics.Debug.Assert(false, sprintf "Failed to compute key in OpenILModuleReader cache for '%s'. Falling back to uncached. Error = %s" fileName (exn.ToString())) + let fakeKey = ILModuleReaderCacheKey(fileName, System.DateTime.UtcNow, ILScopeRef.Local, false, ReduceMemoryFlag.Yes, MetadataOnlyFlag.Yes) + fakeKey, false let cacheResult = if keyOk then @@ -3999,30 +4003,29 @@ let OpenILModuleReader fileName opts = // See if tryGetMetadata gives us a BinaryFile for the metadata section alone. let mdfileOpt = - match opts.tryGetMetadataSnapshot (fileName, writeStamp) with - | Some (obj, start, len) -> Some (RawMemoryFile(fileName, obj, start, len) :> BinaryFile) + match opts.tryGetMetadataSnapshot (fullPath, writeStamp) with + | Some (obj, start, len) -> Some (RawMemoryFile(fullPath, obj, start, len) :> BinaryFile) | None -> None // For metadata-only, always use a temporary, short-lived PE file reader, preferably over a memory mapped file. // Then use the metadata blob as the long-lived memory resource. - let disposer, pefileEager = tryMemoryMap opts fileName + let disposer, pefileEager = tryMemoryMapWholeFile opts fullPath use _disposer = disposer - let (metadataPhysLoc, metadataSize, peinfo, pectxtEager, pevEager, _pdb) = openPEFileReader (fileName, pefileEager, None) + let (metadataPhysLoc, metadataSize, peinfo, pectxtEager, pevEager, _pdb) = openPEFileReader (fullPath, pefileEager, None) let mdfile = match mdfileOpt with | Some mdfile -> mdfile | None -> // If tryGetMetadata doesn't give anything, then just read the metadata chunk out of the binary - let bytes = File.ReadBinaryChunk (fileName, metadataPhysLoc, metadataSize) - ByteFile(fileName, bytes) :> BinaryFile + createByteFileChunk opts fullPath (Some (metadataPhysLoc, metadataSize)) - let ilModule, ilAssemblyRefs = openPEMetadataOnly (fileName, peinfo, pectxtEager, pevEager, mdfile, reduceMemoryUsage, opts.ilGlobals) + let ilModule, ilAssemblyRefs = openPEMetadataOnly (fullPath, peinfo, pectxtEager, pevEager, mdfile, reduceMemoryUsage, opts.ilGlobals) new ILModuleReader(ilModule, ilAssemblyRefs, ignore) else // If we are not doing metadata-only, then just go ahead and read all the bytes and hold them either strongly or weakly // depending on the heuristic - let pefile = createByteFile opts fileName - let ilModule, ilAssemblyRefs, _pdb = openPE (fileName, pefile, None, reduceMemoryUsage, opts.ilGlobals) + let pefile = createByteFileChunk opts fullPath None + let ilModule, ilAssemblyRefs, _pdb = openPE (fullPath, pefile, None, reduceMemoryUsage, opts.ilGlobals) new ILModuleReader(ilModule, ilAssemblyRefs, ignore) if keyOk then @@ -4042,14 +4045,14 @@ let OpenILModuleReader fileName opts = // multi-proc build. So use memory mapping, but only for stable files. Other files // fill use an in-memory ByteFile let _disposer, pefile = - if alwaysMemoryMapFSC || stableFileHeuristicApplies fileName then - tryMemoryMap opts fileName + if alwaysMemoryMapFSC || stableFileHeuristicApplies fullPath then + tryMemoryMapWholeFile opts fullPath else - let pefile = createByteFile opts fileName + let pefile = createByteFileChunk opts fullPath None let disposer = { new IDisposable with member __.Dispose() = () } disposer, pefile - let ilModule, ilAssemblyRefs, pdb = openPE (fileName, pefile, opts.pdbPath, reduceMemoryUsage, opts.ilGlobals) + let ilModule, ilAssemblyRefs, pdb = openPE (fullPath, pefile, opts.pdbPath, reduceMemoryUsage, opts.ilGlobals) let ilModuleReader = new ILModuleReader(ilModule, ilAssemblyRefs, (fun () -> ClosePdbReader pdb)) // Readers with PDB reader disposal logic don't go in the cache. Note the PDB reader is only used in static linking.