/
pau.js
83 lines (77 loc) · 2.13 KB
/
pau.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
import Layer from './base.js'
/**
* Pade activation unit layer
*/
export default class PadeActivationUnitLayer extends Layer {
/**
* @param {object} config config
* @param {number} [config.m] m
* @param {number} [config.n] n
* @param {number | number[]} [config.a] m
* @param {number | number[]} [config.b] b
*/
constructor({ m = 2, n = 2, a = 0.1, b = 0, ...rest }) {
super(rest)
this._m = m
this._n = n
this._a = Array.isArray(a) ? a : Array(m + 1).fill(a)
this._b = Array.isArray(b) ? b : Array(n).fill(b)
this._l2_decay = 0.001
}
calc(x) {
this._i = x
const o = x.copy()
o.map(
v =>
this._a.reduce((s, a, k) => s + a * v ** k, 0) /
(1 + Math.abs(this._b.reduce((s, b, k) => s + b * v ** (k + 1), 0)))
)
return o
}
grad(bo) {
this._bo = bo
const bi = bo.copy()
bi.broadcastOperate(this._i, (a, b) => {
const p = this._a.reduce((s, v, k) => s + v * b ** k, 0)
const A = this._b.reduce((s, v, k) => s + v * b ** (k + 1), 0)
const q = 1 + Math.abs(A)
return (
a *
(this._a.reduce((s, v, k) => s + v * k * b ** (k - 1), 0) / q -
Math.sign(A) * this._b.reduce((s, v, k) => s + v * (k + 1) * b ** k, 0) * (p / q ** 2))
)
})
return bi
}
update(optimizer) {
const sa = Array(this._m + 1).fill(0)
const sb = Array(this._n).fill(0)
for (let i = 0; i < this._i.length; i++) {
const p = this._a.reduce((s, v, k) => s + v * this._i.value[i] ** k, 0)
const A = this._b.reduce((s, v, k) => s + v * this._i.value[i] ** (k + 1), 0)
const q = 1 + Math.abs(A)
for (let k = 0; k < this._m; k++) {
sa[k] += this._bo.value[i] * this._i.value[i] ** k
}
for (let k = 0; k < this._n; k++) {
sb[k] += this._bo.value[i] * (-(this._i.value[i] ** (k + 1)) * Math.sign(A) * (p / q ** 2))
}
}
for (let k = 0; k < this._m + 1; k++) {
this._a[k] -= optimizer.delta(`a${k}`, sa[k] / this._i.length)
}
for (let k = 0; k < this._n; k++) {
this._b[k] -= optimizer.delta(`b${k + 1}`, sb[k] / this._i.length)
}
}
toObject() {
return {
type: 'pau',
m: this._m,
n: this._n,
a: this._a,
b: this._b,
}
}
}
PadeActivationUnitLayer.registLayer('pau')