Skip to content

Integrate https://github.com/Microsoft/visualfsharp/pull/4629 #855

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dsyme merged 4 commits into from
Mar 27, 2018
Merged

Integrate https://github.com/Microsoft/visualfsharp/pull/4629 #855

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f62158b
update perf results
dsyme Mar 23, 2018
48a2f59
Remove duplicate illex.fsl include (#4621)
auduchinok Mar 24, 2018
432784c
Fix FCS problem with duplicate ByteFile (#4629)
dsyme Mar 27, 2018
bd3ed35
integrate visualfsharp master
dsyme Mar 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions fcs/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,10 +278,6 @@
<Compile Include="$(FSharpSourcesRoot)/ilx/EraseUnions.fs">
<Link>ILXErase/EraseUnions.fs</Link>
</Compile>
<FsLex Include="$(FSharpSourcesRoot)\absil\illex.fsl">
<OtherFlags>--unicode --lexlib Internal.Utilities.Text.Lexing</OtherFlags>
<Link>AbsIL/illex.fsl</Link>
</FsLex>
<FsLex Include="$(FSharpSourcesRoot)\fsharp\lex.fsl">
<OtherFlags>--unicode --lexlib Internal.Utilities.Text.Lexing</OtherFlags>
<Link>ParserAndUntypedAST/lex.fsl</Link>
Expand Down
97 changes: 50 additions & 47 deletions src/absil/ilread.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,11 @@ open System.Reflection

let checking = false
let logging = false
let _ = if checking then dprintn "warning : Ilread.checking is on"
let _ = if checking then dprintn "warning : ILBinaryReader.checking is on"
let noStableFileHeuristic = try (System.Environment.GetEnvironmentVariable("FSharp_NoStableFileHeuristic") <> null) with _ -> false
let alwaysMemoryMapFSC = try (System.Environment.GetEnvironmentVariable("FSharp_AlwaysMemoryMapCommandLineCompiler") <> null) with _ -> false
let stronglyHeldReaderCacheSizeDefault = 30
let stronglyHeldReaderCacheSize = try (match System.Environment.GetEnvironmentVariable("FSharp_StronglyHeldBinaryReaderCacheSize") with null -> stronglyHeldReaderCacheSizeDefault | s -> int32 s) with _ -> stronglyHeldReaderCacheSizeDefault

let singleOfBits (x:int32) = System.BitConverter.ToSingle(System.BitConverter.GetBytes(x), 0)
let doubleOfBits (x:int64) = System.BitConverter.Int64BitsToDouble(x)
Expand Down Expand Up @@ -346,7 +348,7 @@ type ByteFile(fileName: string, bytes:byte[]) =
/// This is the default implementation used by F# Compiler Services when accessing "stable" binaries. It is not used
/// by Visual Studio, where tryGetMetadataSnapshot provides a RawMemoryFile backed by Roslyn data.
[<DebuggerDisplay("{FileName}")>]
type WeakByteFile(fileName: string) =
type WeakByteFile(fileName: string, chunk: (int * int) option) =

do stats.weakByteFileCount <- stats.weakByteFileCount + 1

Expand All @@ -357,30 +359,29 @@ type WeakByteFile(fileName: string) =
let weakBytes = new WeakReference<byte[]> (null)

member __.FileName = fileName
/// Get the bytes for the file
member this.Get() =
let mutable tg = null
if not (weakBytes.TryGetTarget(&tg)) then
if FileSystem.GetLastWriteTimeShim(fileName) <> fileStamp then
errorR (Error (FSComp.SR.ilreadFileChanged fileName, range0))

tg <- FileSystem.ReadAllBytesShim fileName
weakBytes.SetTarget tg
tg

/// Get the bytes for the file
interface BinaryFile with
override __.GetView() =
let mutable tg = null

override this.GetView() =
let strongBytes =
let mutable tg = null
if not (weakBytes.TryGetTarget(&tg)) then
if FileSystem.GetLastWriteTimeShim(fileName) <> fileStamp then
errorR (Error (FSComp.SR.ilreadFileChanged fileName, range0))
error (Error (FSComp.SR.ilreadFileChanged fileName, range0))

let bytes =
match chunk with
| None -> FileSystem.ReadAllBytesShim fileName
| Some(start, length) -> File.ReadBinaryChunk (fileName, start, length)

tg <- bytes

weakBytes.SetTarget bytes

tg <- FileSystem.ReadAllBytesShim fileName
weakBytes.SetTarget tg
tg
(ByteView(strongBytes) :> BinaryView)

(ByteView(strongBytes) :> BinaryView)


let seekReadByte (mdv:BinaryView) addr = mdv.ReadByte addr
Expand Down Expand Up @@ -3927,27 +3928,31 @@ type ILModuleReader(ilModule: ILModuleDef, ilAssemblyRefs: Lazy<ILAssemblyRef li

// ++GLOBAL MUTABLE STATE (concurrency safe via locking)
type ILModuleReaderCacheLockToken() = interface LockToken
let ilModuleReaderCache = new AgedLookup<ILModuleReaderCacheLockToken, (string * System.DateTime * ILScopeRef * bool * ReduceMemoryFlag * MetadataOnlyFlag), ILModuleReader>(0, areSimilar=(fun (x, y) -> x = y))
type ILModuleReaderCacheKey = ILModuleReaderCacheKey of string * DateTime * ILScopeRef * bool * ReduceMemoryFlag * MetadataOnlyFlag
let ilModuleReaderCache = new AgedLookup<ILModuleReaderCacheLockToken, ILModuleReaderCacheKey, ILModuleReader>(stronglyHeldReaderCacheSize, areSimilar=(fun (x, y) -> x = y))
let ilModuleReaderCacheLock = Lock()

let stableFileHeuristicApplies fileName =
not noStableFileHeuristic && try FileSystem.IsStableFileHeuristic fileName with _ -> false

let createByteFile opts fileName =
let createByteFileChunk opts fileName chunk =
// If we're trying to reduce memory usage then we are willing to go back and re-read the binary, so we can use
// a weakly-held handle to an array of bytes.
if opts.reduceMemoryUsage = ReduceMemoryFlag.Yes && stableFileHeuristicApplies fileName then
WeakByteFile(fileName) :> BinaryFile
WeakByteFile(fileName, chunk) :> BinaryFile
else
let bytes = FileSystem.ReadAllBytesShim(fileName)
let bytes =
match chunk with
| None -> FileSystem.ReadAllBytesShim fileName
| Some (start, length) -> File.ReadBinaryChunk(fileName, start, length)
ByteFile(fileName, bytes) :> BinaryFile

let tryMemoryMap opts fileName =
let tryMemoryMapWholeFile opts fileName =
let file =
try
MemoryMapFile.Create fileName :> BinaryFile
with _ ->
createByteFile opts fileName
createByteFileChunk opts fileName None
let disposer =
{ new IDisposable with
member __.Dispose() =
Expand All @@ -3963,17 +3968,16 @@ let OpenILModuleReaderFromBytes fileName bytes opts =

let OpenILModuleReader fileName opts =
// Pseudo-normalize the paths.
let ((_,writeStamp,_,_,_,_) as key), keyOk =
let (ILModuleReaderCacheKey (fullPath,writeStamp,_,_,_,_) as key), keyOk =
try
(FileSystem.GetFullPathShim(fileName),
FileSystem.GetLastWriteTimeShim(fileName),
opts.ilGlobals.primaryAssemblyScopeRef,
opts.pdbPath.IsSome,
opts.reduceMemoryUsage,
opts.metadataOnly), true
with e ->
System.Diagnostics.Debug.Assert(false, sprintf "Failed to compute key in OpenILModuleReader cache for '%s'. Falling back to uncached." fileName)
("", System.DateTime.UtcNow, ILScopeRef.Local, false, ReduceMemoryFlag.Yes, MetadataOnlyFlag.Yes), false
let fullPath = FileSystem.GetFullPathShim(fileName)
let writeTime = FileSystem.GetLastWriteTimeShim(fileName)
let key = ILModuleReaderCacheKey (fullPath, writeTime, opts.ilGlobals.primaryAssemblyScopeRef, opts.pdbPath.IsSome, opts.reduceMemoryUsage, opts.metadataOnly)
key, true
with exn ->
System.Diagnostics.Debug.Assert(false, sprintf "Failed to compute key in OpenILModuleReader cache for '%s'. Falling back to uncached. Error = %s" fileName (exn.ToString()))
let fakeKey = ILModuleReaderCacheKey(fileName, System.DateTime.UtcNow, ILScopeRef.Local, false, ReduceMemoryFlag.Yes, MetadataOnlyFlag.Yes)
fakeKey, false

let cacheResult =
if keyOk then
Expand All @@ -3999,30 +4003,29 @@ let OpenILModuleReader fileName opts =

// See if tryGetMetadata gives us a BinaryFile for the metadata section alone.
let mdfileOpt =
match opts.tryGetMetadataSnapshot (fileName, writeStamp) with
| Some (obj, start, len) -> Some (RawMemoryFile(fileName, obj, start, len) :> BinaryFile)
match opts.tryGetMetadataSnapshot (fullPath, writeStamp) with
| Some (obj, start, len) -> Some (RawMemoryFile(fullPath, obj, start, len) :> BinaryFile)
| None -> None

// For metadata-only, always use a temporary, short-lived PE file reader, preferably over a memory mapped file.
// Then use the metadata blob as the long-lived memory resource.
let disposer, pefileEager = tryMemoryMap opts fileName
let disposer, pefileEager = tryMemoryMapWholeFile opts fullPath
use _disposer = disposer
let (metadataPhysLoc, metadataSize, peinfo, pectxtEager, pevEager, _pdb) = openPEFileReader (fileName, pefileEager, None)
let (metadataPhysLoc, metadataSize, peinfo, pectxtEager, pevEager, _pdb) = openPEFileReader (fullPath, pefileEager, None)
let mdfile =
match mdfileOpt with
| Some mdfile -> mdfile
| None ->
// If tryGetMetadata doesn't give anything, then just read the metadata chunk out of the binary
let bytes = File.ReadBinaryChunk (fileName, metadataPhysLoc, metadataSize)
ByteFile(fileName, bytes) :> BinaryFile
createByteFileChunk opts fullPath (Some (metadataPhysLoc, metadataSize))

let ilModule, ilAssemblyRefs = openPEMetadataOnly (fileName, peinfo, pectxtEager, pevEager, mdfile, reduceMemoryUsage, opts.ilGlobals)
let ilModule, ilAssemblyRefs = openPEMetadataOnly (fullPath, peinfo, pectxtEager, pevEager, mdfile, reduceMemoryUsage, opts.ilGlobals)
new ILModuleReader(ilModule, ilAssemblyRefs, ignore)
else
// If we are not doing metadata-only, then just go ahead and read all the bytes and hold them either strongly or weakly
// depending on the heuristic
let pefile = createByteFile opts fileName
let ilModule, ilAssemblyRefs, _pdb = openPE (fileName, pefile, None, reduceMemoryUsage, opts.ilGlobals)
let pefile = createByteFileChunk opts fullPath None
let ilModule, ilAssemblyRefs, _pdb = openPE (fullPath, pefile, None, reduceMemoryUsage, opts.ilGlobals)
new ILModuleReader(ilModule, ilAssemblyRefs, ignore)

if keyOk then
Expand All @@ -4042,14 +4045,14 @@ let OpenILModuleReader fileName opts =
// multi-proc build. So use memory mapping, but only for stable files. Other files
// fill use an in-memory ByteFile
let _disposer, pefile =
if alwaysMemoryMapFSC || stableFileHeuristicApplies fileName then
tryMemoryMap opts fileName
if alwaysMemoryMapFSC || stableFileHeuristicApplies fullPath then
tryMemoryMapWholeFile opts fullPath
else
let pefile = createByteFile opts fileName
let pefile = createByteFileChunk opts fullPath None
let disposer = { new IDisposable with member __.Dispose() = () }
disposer, pefile

let ilModule, ilAssemblyRefs, pdb = openPE (fileName, pefile, opts.pdbPath, reduceMemoryUsage, opts.ilGlobals)
let ilModule, ilAssemblyRefs, pdb = openPE (fullPath, pefile, opts.pdbPath, reduceMemoryUsage, opts.ilGlobals)
let ilModuleReader = new ILModuleReader(ilModule, ilAssemblyRefs, (fun () -> ClosePdbReader pdb))

// Readers with PDB reader disposal logic don't go in the cache. Note the PDB reader is only used in static linking.
Expand Down