-
Notifications
You must be signed in to change notification settings - Fork 11
/
n-step semi-gradient TD.kt
62 lines (58 loc) · 1.59 KB
/
n-step semi-gradient TD.kt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
@file:Suppress("NAME_SHADOWING")
package lab.mars.rl.algo.func_approx.prediction
import lab.mars.rl.algo.ntd.MAX_N
import lab.mars.rl.model.*
import lab.mars.rl.util.buf.newBuf
import lab.mars.rl.util.log.debug
import lab.mars.rl.util.math.Σ
import lab.mars.rl.util.matrix.times
import org.apache.commons.math3.util.FastMath.min
import org.apache.commons.math3.util.FastMath.pow
fun <E> MDP.`n-step semi-gradient TD`(
v: ApproximateFunction<E>, π: Policy,
n: Int,
α: Double,
episodes: Int,
episodeListener: (Int, Int) -> Unit = { _, _ -> }) {
val _R = newBuf<Double>(min(n, MAX_N))
val _S = newBuf<State>(min(n, MAX_N))
for (episode in 1..episodes) {
log.debug { "$episode/$episodes" }
var step = 0
var n = n
var T = Int.MAX_VALUE
var t = 0
var s = started()
var a = π(s)
_R.clear();_R.append(0.0)
_S.clear();_S.append(s)
do {
step++
if (t >= n) {
_R.removeFirst()
_S.removeFirst()
}
if (t < T) {
val (s_next, reward) = a.sample()
_R.append(reward)
_S.append(s_next)
s = s_next
if (s.isTerminal) {
T = t + 1
val τ = t - n + 1
if (τ < 0) n = T //n is too large
} else
a = π(s)
}
val τ = t - n + 1
if (τ >= 0) {
var G = Σ(1..min(n, T - τ)) { pow(γ, it - 1) * _R[it] }
if (τ + n < T) G += pow(γ, n) * v(_S[n])
v.w += α * (G - v(_S[0])) * v.`∇`(_S[0])
}
t++
} while (τ < T - 1)
log.debug { "n=$n,T=$T" }
episodeListener(episode, step)
}
}