Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor Symbol Cache #384

Merged
merged 2 commits into from
May 24, 2019
Merged

Refactor Symbol Cache #384

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a8ac3f8
Refactor Symbol Cache
Krzysztof-Cieslak May 24, 2019
f9f6aa3
Comment out failing tests
Krzysztof-Cieslak May 24, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions paket.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -501,7 +501,7 @@ NUGET
Microsoft.NETCore.Targets (>= 1.1) - restriction: || (&& (== net461) (< net46)) (== netcoreapp2.0) (== netcoreapp2.1) (== netstandard2.0)
System.Runtime (>= 4.3) - restriction: || (&& (== net461) (< net46)) (== netcoreapp2.0) (== netcoreapp2.1) (== netstandard2.0)
Microsoft.Win32.Registry (4.5) - restriction: || (&& (== net461) (>= netcoreapp1.0)) (== netcoreapp2.0) (== netcoreapp2.1) (&& (== netstandard2.0) (>= netcoreapp1.0))
System.Memory (>= 4.5) - restriction: || (&& (== net461) (< net46) (>= netstandard2.0)) (&& (== net461) (>= netcoreapp2.0)) (&& (== net461) (>= uap10.1)) (== netcoreapp2.0) (&& (== netcoreapp2.1) (< netcoreapp2.0)) (&& (== netcoreapp2.1) (>= uap10.1)) (== netstandard2.0)
System.Memory (>= 4.5) - restriction: || (&& (== net461) (== netcoreapp2.1)) (&& (== net461) (< net46) (>= netstandard2.0)) (&& (== net461) (>= netcoreapp2.0)) (&& (== net461) (>= uap10.1)) (== netcoreapp2.0) (&& (== netcoreapp2.1) (< netcoreapp2.0)) (&& (== netcoreapp2.1) (>= uap10.1)) (== netstandard2.0)
System.Security.AccessControl (>= 4.5)
System.Security.Principal.Windows (>= 4.5)
Mono.Cecil (0.10.3)
Expand Down Expand Up @@ -926,7 +926,7 @@ NUGET
System.Threading (>= 4.3) - restriction: || (&& (== net461) (< net46)) (== netcoreapp2.0) (== netcoreapp2.1) (== netstandard2.0)
System.Threading.Tasks (>= 4.3) - restriction: || (&& (== net461) (< net46)) (== netcoreapp2.0) (== netcoreapp2.1) (== netstandard2.0)
System.Security.Cryptography.ProtectedData (4.5) - restriction: || (&& (== net461) (< net45) (>= netstandard2.0)) (== netcoreapp2.0) (== netcoreapp2.1) (== netstandard2.0)
System.Memory (>= 4.5) - restriction: || (&& (== net461) (< net46) (>= netstandard2.0)) (== netcoreapp2.0) (== netstandard2.0)
System.Memory (>= 4.5) - restriction: || (&& (== net461) (== netcoreapp2.1)) (&& (== net461) (< net46) (>= netstandard2.0)) (== netcoreapp2.0) (== netstandard2.0)
System.Security.Permissions (4.5) - restriction: || (&& (== net461) (>= monoandroid) (>= netstandard2.0)) (&& (== net461) (>= monotouch) (>= netstandard2.0)) (&& (== net461) (< net45) (>= netstandard2.0)) (&& (== net461) (>= netstandard2.0) (>= xamarinmac)) (&& (== net461) (>= netstandard2.0) (>= xamarintvos)) (&& (== net461) (>= netstandard2.0) (>= xamarinwatchos)) (== netcoreapp2.0) (== netcoreapp2.1) (== netstandard2.0)
System.Security.AccessControl (>= 4.5)
System.Security.Principal.Windows (4.5.1) - restriction: || (&& (== net461) (>= netcoreapp1.0)) (&& (== net461) (>= netcoreapp2.0)) (== netcoreapp2.0) (== netcoreapp2.1) (&& (== netstandard2.0) (>= netcoreapp1.0)) (&& (== netstandard2.0) (>= netcoreapp2.0))
Expand Down
111 changes: 90 additions & 21 deletions src/FsAutoComplete.Core/SymbolCache.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,88 @@ open System.Net
open System.IO
open Newtonsoft.Json

[<CLIMutable>]
type SymbolUseRange = {
FileName: string
StartLine: int
StartColumn: int
EndLine: int
EndColumn: int
IsFromDefinition: bool
IsFromAttribute : bool
IsFromComputationExpression : bool
IsFromDispatchSlotImplementation : bool
IsFromPattern : bool
IsFromType : bool
SymbolFullName: string
SymbolDisplayName: string
SymbolIsLocal: bool
}


module PersistenCacheImpl =
open Microsoft.Data.Sqlite
open Dapper
open System.Data

let mutable connection : SqliteConnection option = None

let insert (connection: SqliteConnection) file (sugs: SymbolUseRange[]) =
if connection.State <> ConnectionState.Open then connection.Open()
use tx = connection.BeginTransaction()
let delCmd = sprintf "DELETE FROM Symbols WHERE FileName=\"%s\"" file
let inserCmd =
sprintf "INSERT INTO SYMBOLS(FileName, StartLine, StartColumn, EndLine, EndColumn, IsFromDefinition, IsFromAttribute, IsFromComputationExpression, IsFromDispatchSlotImplementation, IsFromPattern, IsFromType, SymbolFullName, SymbolDisplayName, SymbolIsLocal) VALUES
(@FileName, @StartLine, @StartColumn, @EndLine, @EndColumn, @IsFromDefinition, @IsFromAttribute, @IsFromComputationExpression, @IsFromDispatchSlotImplementation, @IsFromPattern, @IsFromType, @SymbolFullName, @SymbolDisplayName, @SymbolIsLocal)"
connection.Execute(delCmd, transaction = tx) |> ignore
connection.Execute(inserCmd, sugs, transaction = tx) |> ignore
tx.Commit()

let loadAll (connection: SqliteConnection) =
if connection.State <> ConnectionState.Open then connection.Open()
let q = "SELECT * FROM SYMBOLS"
let res = connection.Query<SymbolUseRange>(q)
res

let loadFile (connection: SqliteConnection) file =
if connection.State <> ConnectionState.Open then connection.Open()
let q = sprintf "SELECT * FROM SYMBOLS WHERE FileName=\"%s\"" file
let res = connection.Query<SymbolUseRange>(q)
res

let initializeCache dir =
let connectionString = sprintf "Data Source=%s/.ionide/symbolCache.db" dir
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if we use /.ionide/obj , that will be git ignored by default by pratically all .gitignores, so less noise

Copy link
Contributor

@enricosada enricosada May 24, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another option is that it's passed as argument to FSAC.

So editors can put that:

  • or in their temp directory or custom dir
  • or can be in a dir inside %TEMP% generated for the workspace so valid until restart and automatically cleaned up.


let dir = Path.Combine(dir, ".ionide")
do if not (Directory.Exists dir) then Directory.CreateDirectory dir |> ignore
let dbPath = Path.Combine(dir, "symbolCache.db")
let dbExists = File.Exists dbPath
let connection = new SqliteConnection(connectionString)

do if not dbExists then
let fs = File.Create(dbPath)
fs.Close()
let cmd = "CREATE TABLE Symbols(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this doesnt work if the schema will be updated later to v2.
for example an older symbolcache exists on file, and that's will be used but schema is v1

FileName TEXT,
StartLine INT,
StartColumn INT,
EndLine INT,
EndColumn INT,
IsFromDefinition BOOLEAN,
IsFromAttribute BOOLEAN,
IsFromComputationExpression BOOLEAN,
IsFromDispatchSlotImplementation BOOLEAN,
IsFromPattern BOOLEAN,
IsFromType BOOLEAN,
SymbolFullName TEXT,
SymbolDisplayName TEXT,
SymbolIsLocal BOOLEAN
)"

connection.Execute(cmd)
|> ignore
connection

let makePostRequest (url : string) (requestBody : string) =
let req = WebRequest.CreateHttp url
req.CookieContainer <- new CookieContainer()
Expand All @@ -30,28 +112,11 @@ let makePostRequest (url : string) (requestBody : string) =

let mutable port = 0

[<CLIMutable>]
type SymbolUseRange = {
FileName: string
StartLine: int
StartColumn: int
EndLine: int
EndColumn: int
IsFromDefinition: bool
IsFromAttribute : bool
IsFromComputationExpression : bool
IsFromDispatchSlotImplementation : bool
IsFromPattern : bool
IsFromType : bool
SymbolFullName: string
SymbolDisplayName: string
SymbolIsLocal: bool
}

type SymbolCacheRequest = {
Filename: string
Uses: SymbolUseRange[]
}

let p =
let t = typeof<SymbolCacheRequest>
Path.GetDirectoryName t.Assembly.Location
Expand All @@ -62,6 +127,7 @@ let pid =

let startCache (dir : string) =
port <- Random().Next(9000,9999)
PersistenCacheImpl.connection <- Some (PersistenCacheImpl.initializeCache dir)
let si = ProcessStartInfo()
si.RedirectStandardOutput <- true
si.RedirectStandardError <- true
Expand Down Expand Up @@ -105,11 +171,14 @@ let fromSymbolUse (su : FSharpSymbolUse) =


let sendSymbols (serializer: Serializer) fn (symbols: FSharpSymbolUse[]) =
let request =
let sus =
symbols
|> Array.map(fromSymbolUse)
|> fun n -> {Filename = fn; Uses = n}
|> serializer

PersistenCacheImpl.connection
|> Option.iter (fun con -> PersistenCacheImpl.insert con fn sus )

let request = serializer {Filename = fn}

try
makePostRequest ("http://localhost:" + (string port) + "/updateSymbols") request
Expand Down
2 changes: 2 additions & 0 deletions src/FsAutoComplete.Core/paket.references
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,5 @@ ICSharpCode.Decompiler
FSharp.Analyzers.SDK
Microsoft.SourceLink.GitHub
FSharp.Data
Dapper
Microsoft.Data.Sqlite
86 changes: 19 additions & 67 deletions src/FsAutoComplete.SymbolCache/FsAutoComplete.SymbolCache.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ open System
open FsAutoComplete.Utils
open SymbolCache
open System.IO
open Microsoft.Data.Sqlite
open Dapper
open System.Data

open Suave.Logging


Expand All @@ -31,81 +29,44 @@ let state = ConcurrentDictionary<string, SymbolUseRange[]>()

type private PersistentStateMessage =
| Save of file : string * symbols : SymbolUseRange[]
| FillStateForFile of file : string
| FillState

type PersistentState (dir) =
let connectionString = sprintf "Data Source=%s/.ionide/symbolCache.db" dir

let dir = Path.Combine(dir, ".ionide")
do if not (Directory.Exists dir) then Directory.CreateDirectory dir |> ignore
let dbPath = Path.Combine(dir, "symbolCache.db")
let dbExists = File.Exists dbPath
let connection = new SqliteConnection(connectionString)

do if not dbExists then
let fs = File.Create(dbPath)
fs.Close()
let cmd = "CREATE TABLE Symbols(
FileName TEXT,
StartLine INT,
StartColumn INT,
EndLine INT,
EndColumn INT,
IsFromDefinition BOOLEAN,
IsFromAttribute BOOLEAN,
IsFromComputationExpression BOOLEAN,
IsFromDispatchSlotImplementation BOOLEAN,
IsFromPattern BOOLEAN,
IsFromType BOOLEAN,
SymbolFullName TEXT,
SymbolDisplayName TEXT,
SymbolIsLocal BOOLEAN
)"

connection.Execute(cmd)
|> ignore

type PersistentState (dir) =
let connection = SymbolCache.PersistenCacheImpl.initializeCache dir

let agent = MailboxProcessor.Start <| fun mb ->
let rec loop () = async {
let! msg = mb.Receive()
match msg with
| Save (file, sugs) ->
try
let d = DateTime.Now
printfn "[Debug] Updating DB for %s" file
if connection.State <> ConnectionState.Open then connection.Open()
use tx = connection.BeginTransaction()
let delCmd = sprintf "DELETE FROM Symbols WHERE FileName=\"%s\"" file
let inserCmd =
sprintf "INSERT INTO SYMBOLS(FileName, StartLine, StartColumn, EndLine, EndColumn, IsFromDefinition, IsFromAttribute, IsFromComputationExpression, IsFromDispatchSlotImplementation, IsFromPattern, IsFromType, SymbolFullName, SymbolDisplayName, SymbolIsLocal) VALUES
(@FileName, @StartLine, @StartColumn, @EndLine, @EndColumn, @IsFromDefinition, @IsFromAttribute, @IsFromComputationExpression, @IsFromDispatchSlotImplementation, @IsFromPattern, @IsFromType, @SymbolFullName, @SymbolDisplayName, @SymbolIsLocal)"
connection.Execute(delCmd, transaction = tx) |> ignore
connection.Execute(inserCmd, sugs, transaction = tx) |> ignore
tx.Commit()
let e = DateTime.Now
printfn "[Debug] Updating DB took %fms" (e-d).TotalMilliseconds
PersistenCacheImpl.insert connection file sugs
with
| ex ->
printfn "%s" ex.Message
printfn "%s" ex.StackTrace
return! loop()
| FillState ->
try
printfn "[Debug] Loading initial state"
let d = DateTime.Now
if connection.State <> ConnectionState.Open then connection.Open()
let q = "SELECT * FROM SYMBOLS"
let res = connection.Query<SymbolUseRange>(q)
res
PersistenCacheImpl.loadAll connection
|> Seq.groupBy (fun r -> r.FileName)
|> Seq.iter (fun (fn, lst) ->
let sms = lst |> Seq.toArray
state.AddOrUpdate(fn, sms, fun _ _ -> sms) |> ignore
)
let e = DateTime.Now
printfn "Loaded initial state in %fms" (e-d).TotalMilliseconds

with
| ex ->
printfn "%s" ex.Message
printfn "%s" ex.StackTrace
return! loop()
| FillStateForFile fn ->
try
let sms =
PersistenCacheImpl.loadFile connection fn
|> Seq.toArray
state.AddOrUpdate(fn, sms, fun _ _ -> sms) |> ignore
with
| ex ->
printfn "%s" ex.Message
Expand All @@ -115,12 +76,10 @@ type PersistentState (dir) =
loop ()

member __.Save(file, sugs) = Save (file, sugs) |> agent.Post
member __.Load(file) = FillStateForFile file |> agent.Post
member __.Initialize () = agent.Post FillState





type BackgroundFSharpCompilerServiceChecker() =
let checker =
FSharpChecker.Create(
Expand All @@ -139,7 +98,6 @@ module Commands =

let buildCacheForProject (onAdded : string -> SymbolUseRange[] -> unit) opts =
async {
let start = DateTime.Now
let! res = checker.CheckProject(opts)
let! results = res.GetAllUsesOfAllSymbols()
results
Expand All @@ -148,8 +106,6 @@ module Commands =
let sms = symbols |> Array.map (SymbolCache.fromSymbolUse)
state.AddOrUpdate(fn, sms, fun _ _ -> sms) |> onAdded fn
)
let e = DateTime.Now
printfn "Project %s took %fms" opts.ProjectFileName (e-start).TotalMilliseconds
} |> Async.RunSynchronously

let getSymbols symbolName =
Expand All @@ -170,10 +126,6 @@ module Commands =

writeJson uses

let updateSymbols (onAdded : string -> SymbolUseRange[] -> unit) (req: SymbolCacheRequest) =
state.AddOrUpdate(req.Filename, req.Uses, fun _ _ -> req.Uses)
|> onAdded req.Filename


let start port dir =
Directory.SetCurrentDirectory dir
Expand Down Expand Up @@ -232,7 +184,7 @@ let start port dir =
try
let d = DateTime.Now
let req = getResourceFromReq<SymbolCacheRequest> httpCtx.request
do Commands.updateSymbols (fun fn syms -> pS.Save(fn,syms) ) req
do pS.Load req.Filename
let e = DateTime.Now
printfn "[Debug] /updateSymbols request took %fms" (e-d).TotalMilliseconds
Response.response HttpCode.HTTP_200 [||] httpCtx
Expand Down