break ties in MaxUCB randomly

src/POMCPOW.jl

@@ -119,9 +119,9 @@ Fields:
     default: `RandomActionGenerator(rng)`
 - `default_action::Any`:
     Function, action, or Policy used to determine the action if POMCP fails with exception `ex`.
-    If this is a Function `f`, `f(belief, ex)` will be called.
+    If this is a Function `f`, `f(pomdp, belief, ex)` will be called.
     If this is a Policy `p`, `action(p, belief)` will be called.
-    If it is an object `a`, `default_action(a, belief, ex)` will be called, and
+    If it is an object `a`, `default_action(a, pomdp, belief, ex)` will be called, and
     if this method is not implemented, `a` will be returned directly.
 """
 @with_kw mutable struct POMCPOWSolver <: AbstractPOMCPSolver

src/criteria.jl

@@ -2,11 +2,13 @@ struct MaxUCB
     c::Float64
 end
 
-function select_best(crit::MaxUCB, h_node::POWTreeObsNode)
+function select_best(crit::MaxUCB, h_node::POWTreeObsNode, rng)
     tree = h_node.tree
     h = h_node.node
     best_criterion_val = -Inf
-    local best_node
+    local best_node::Int
+    istied = false
+    local tied::Vector{Int}
     ltn = log(tree.total_n[h])
     for node in tree.tried[h]
         n = tree.n[node]
@@ -17,17 +19,29 @@ function select_best(crit::MaxUCB, h_node::POWTreeObsNode)
         else
             criterion_value = tree.v[node] + crit.c*sqrt(ltn/n)
         end
-        if criterion_value >= best_criterion_val
+        if criterion_value > best_criterion_val
             best_criterion_val = criterion_value
             best_node = node
+            istied = false
+        elseif criterion_value == best_criterion_val
+            if istied
+                push!(tied, node)
+            else
+                istied = true
+                tied = [best_node, node]
+            end
         end
     end
-    return best_node
+    if istied
+        return rand(rng, tied)
+    else
+        return best_node
+    end
 end
 
 struct MaxQ end
 
-function select_best(crit::MaxQ, h_node::POWTreeObsNode)
+function select_best(crit::MaxQ, h_node::POWTreeObsNode, rng)
     tree = h_node.tree
     h = h_node.node
     best_node = first(tree.tried[h])
@@ -44,7 +58,7 @@ end
 
 struct MaxTries end
 
-function select_best(crit::MaxTries, h_node::POWTreeObsNode)
+function select_best(crit::MaxTries, h_node::POWTreeObsNode, rng)
     tree = h_node.tree
     h = h_node.node
     best_node = first(tree.tried[h])

src/planner2.jl

@@ -35,7 +35,7 @@ function action{P,NBU}(pomcp::POMCPOWPlanner{P,NBU}, b)
     try
         a = search(pomcp, tree)
     catch ex
-        a = convert(A, default_action(pomcp.solver.default_action, b, ex))
+        a = convert(A, default_action(pomcp.solver.default_action, pomcp.problem, b, ex))
     end
     return a
 end
@@ -60,7 +60,7 @@ function search(pomcp::POMCPOWPlanner, tree::POMCPOWTree)
         throw(AllSamplesTerminal(tree.root_belief))
     end
 
-    best_node = select_best(pomcp.solver.final_criterion, POWTreeObsNode(tree,1))
+    best_node = select_best(pomcp.solver.final_criterion, POWTreeObsNode(tree,1), pomcp.solver.rng)
 
     return tree.a_labels[best_node]
 end

src/solver2.jl

@@ -53,7 +53,7 @@ function simulate{B,S,A,O}(pomcp::POMCPOWPlanner, h_node::POWTreeObsNode{B,A,O},
     end
     total_n = tree.total_n[h]
 
-    best_node = select_best(pomcp.criterion, h_node)
+    best_node = select_best(pomcp.criterion, h_node, pomcp.solver.rng)
     a = tree.a_labels[best_node]
 
     sp, o, r = generate_sor(pomcp.problem, s, a, sol.rng)