From 88c0d6d363458a420f44b34fb9118d1e2e11159a Mon Sep 17 00:00:00 2001 From: Martin Date: Thu, 21 Mar 2024 13:55:12 +0100 Subject: [PATCH] [LodRenderer] Handle exceptions in background threads --- .../Runtime/LodRenderer.fs | 224 +++++++++--------- 1 file changed, 115 insertions(+), 109 deletions(-) diff --git a/src/Aardvark.Rendering.GL/Runtime/LodRenderer.fs b/src/Aardvark.Rendering.GL/Runtime/LodRenderer.fs index 800e7ffc..e7593d47 100644 --- a/src/Aardvark.Rendering.GL/Runtime/LodRenderer.fs +++ b/src/Aardvark.Rendering.GL/Runtime/LodRenderer.fs @@ -1816,10 +1816,16 @@ type LodRenderer(manager : ResourceManager, config : LodRendererConfig, roots : dataSize = ref 0L } - + let inline startThreadSafe (name: string) (run: unit -> unit) = + startThread (fun () -> + try + run() + with exn -> + Log.error "[LodRenderer] %s thread faulted: %A" name exn + ) let cameraPrediction = - startThread (fun () -> + startThreadSafe "Camera prediction" (fun () -> let mutable lastTime = time() let mutable lastReport = time() let timer = new MultimediaTimer.Trigger(5) @@ -1884,7 +1890,7 @@ type LodRenderer(manager : ResourceManager, config : LodRendererConfig, roots : //let rootDeltas = ref HDeltaSet.empty let puller = - startThread (fun () -> + startThreadSafe "Puller" (fun () -> let timer = new MultimediaTimer.Trigger(20) let pickTrees = config.pickTrees @@ -1961,7 +1967,7 @@ type LodRenderer(manager : ResourceManager, config : LodRendererConfig, roots : ) let thread = - startThread (fun () -> + startThreadSafe "Update" (fun () -> let notConverged = MVar.create () //new ManualResetEventSlim(true) let cancel = System.Collections.Concurrent.ConcurrentDictionary() @@ -2032,126 +2038,126 @@ type LodRenderer(manager : ResourceManager, config : LodRendererConfig, roots : while not shutdown.IsCancellationRequested do timer.Wait() MVar.take notConverged - shutdown.Token.ThrowIfCancellationRequested() - //caller.EvaluateAlways AdaptiveToken.Top (fun token -> - let view = config.view.GetValue AdaptiveToken.Top - let proj = config.proj.GetValue AdaptiveToken.Top - let maxSplits = config.maxSplits.GetValue AdaptiveToken.Top + if not shutdown.IsCancellationRequested then + //caller.EvaluateAlways AdaptiveToken.Top (fun token -> + let view = config.view.GetValue AdaptiveToken.Top + let proj = config.proj.GetValue AdaptiveToken.Top + let maxSplits = config.maxSplits.GetValue AdaptiveToken.Top - deltas <- - let ops = reader.GetChanges AdaptiveToken.Top - HashSetDelta.combine deltas ops - - let toFree = - lock toFreeUniforms (fun () -> - let r = !toFreeUniforms - toFreeUniforms := HashSet.empty - r + deltas <- + let ops = reader.GetChanges AdaptiveToken.Top + HashSetDelta.combine deltas ops + + let toFree = + lock toFreeUniforms (fun () -> + let r = !toFreeUniforms + toFreeUniforms := HashSet.empty + r + ) + + toFree |> HashSet.iter ( fun node -> + freeRootId node + rootUniforms <- HashMap.remove node rootUniforms ) - - toFree |> HashSet.iter ( fun node -> - freeRootId node - rootUniforms <- HashMap.remove node rootUniforms - ) - if maxSplits > 0 then - let ops = - let d = deltas - deltas <- HashSetDelta.empty - d - - //if ops.Count > 0 then - // lock rootDeltas (fun () -> - // rootDeltas := HDeltaSet.combine !rootDeltas ops - // ) - // MVar.put changesPending () - - //if ops.Count > 0 then - // for o in ops do - // match o with - // | Add(_,v) -> - // Log.warn "add %A" v.root - // | Rem(_,v) -> - // Log.warn "rem %A" v.root - - let roots = - if ops.Count > 0 then - lock rootLock (fun () -> - for o in ops do - match o with - | Add(_,i) -> - let r = i.root - match HashMap.tryFind r roots with - | Some o -> - Log.error "[Lod] add of existing root %A" i.root - | None -> - let u = i.uniforms - rootUniforms <- HashMap.add r u rootUniforms - let rid = getRootId r - let load ct n = load ct rid n (fun _ _ r -> r) - roots <- HashMap.add r (TaskTree(load, rid)) roots - - | Rem(_,i) -> - let r = i.root - match HashMap.tryRemove r roots with - | Some (_, rest) -> + if maxSplits > 0 then + let ops = + let d = deltas + deltas <- HashSetDelta.empty + d + + //if ops.Count > 0 then + // lock rootDeltas (fun () -> + // rootDeltas := HDeltaSet.combine !rootDeltas ops + // ) + // MVar.put changesPending () + + //if ops.Count > 0 then + // for o in ops do + // match o with + // | Add(_,v) -> + // Log.warn "add %A" v.root + // | Rem(_,v) -> + // Log.warn "rem %A" v.root + + let roots = + if ops.Count > 0 then + lock rootLock (fun () -> + for o in ops do + match o with + | Add(_,i) -> + let r = i.root + match HashMap.tryFind r roots with + | Some o -> + Log.error "[Lod] add of existing root %A" i.root + | None -> + let u = i.uniforms + rootUniforms <- HashMap.add r u rootUniforms + let rid = getRootId r + let load ct n = load ct rid n (fun _ _ r -> r) + roots <- HashMap.add r (TaskTree(load, rid)) roots + + | Rem(_,i) -> + let r = i.root + match HashMap.tryRemove r roots with + | Some (_, rest) -> - roots <- rest - | None -> - Log.error "[Lod] remove of nonexisting root %A" i.root - - MVar.put changesPending () - roots - ) - else - lock rootLock (fun () -> roots) + roots <- rest + | None -> + Log.error "[Lod] remove of nonexisting root %A" i.root + + MVar.put changesPending () + roots + ) + else + lock rootLock (fun () -> roots) - let modelView (r : ILodTreeNode) = - let t = getRootTrafo r - subs.GetOrCreate(t, fun t -> t.AddMarkingCallback (MVar.put notConverged)) |> ignore - let m = t.GetValue() - m * view + let modelView (r : ILodTreeNode) = + let t = getRootTrafo r + subs.GetOrCreate(t, fun t -> t.AddMarkingCallback (MVar.put notConverged)) |> ignore + let m = t.GetValue() + m * view - let budget = config.budget.GetValue() - let splitfactor = config.splitfactor.GetValue() + let budget = config.budget.GetValue() + let splitfactor = config.splitfactor.GetValue() - let start = time() - let maxQ = - if budget < 0L then 1.0 - else fst (TreeHelpers.getMaxQuality budget splitfactor (Seq.map fst roots) modelView proj) + let start = time() + let maxQ = + if budget < 0L then 1.0 + else fst (TreeHelpers.getMaxQuality budget splitfactor (Seq.map fst roots) modelView proj) - let dt = time() - start - maxQualityTime.Insert(dt.TotalMilliseconds) |> ignore + let dt = time() - start + maxQualityTime.Insert(dt.TotalMilliseconds) |> ignore - let collapseIfNotSplit = maxQ < 1.0 - let start = time() - let queue = List() - for (k,v) in roots do - v.BuildQueue(state, collapseIfNotSplit, k, maxQ, splitfactor, modelView, proj, queue) + let collapseIfNotSplit = maxQ < 1.0 + let start = time() + let queue = List() + for (k,v) in roots do + v.BuildQueue(state, collapseIfNotSplit, k, maxQ, splitfactor, modelView, proj, queue) - let q, fin = TaskTree<_>.ProcessQueue(state, queue, maxQ, splitfactor, modelView, proj, maxSplits) - lastQ <- q - let dt = time() - start - expandTime.Insert dt.TotalMilliseconds |> ignore - - transact (fun () -> - config.stats.Value <- - { - quality = lastQ - maxQuality = maxQ - totalPrimitives = !state.dataSize - totalNodes = !state.totalNodes - usedMemory = pool.UsedMemory + inner.UsedMemory - allocatedMemory = pool.TotalMemory + inner.TotalMemory - renderTime = inner.AverageRenderTime - } - ) - if not fin then MVar.put notConverged () + let q, fin = TaskTree<_>.ProcessQueue(state, queue, maxQ, splitfactor, modelView, proj, maxSplits) + lastQ <- q + let dt = time() - start + expandTime.Insert dt.TotalMilliseconds |> ignore + + transact (fun () -> + config.stats.Value <- + { + quality = lastQ + maxQuality = maxQ + totalPrimitives = !state.dataSize + totalNodes = !state.totalNodes + usedMemory = pool.UsedMemory + inner.UsedMemory + allocatedMemory = pool.TotalMemory + inner.TotalMemory + renderTime = inner.AverageRenderTime + } + ) + if not fin then MVar.put notConverged () finally