-
Notifications
You must be signed in to change notification settings - Fork 0
/
demo.py
118 lines (86 loc) · 1.75 KB
/
demo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
# %% Imports
import jax
import jax.numpy as jnp
import matplotlib.pyplot as plt
import numpy as np
# %% JAX is like numpy
jnp.ones((4,4))
# %%
a = jnp.asarray(np.random.normal(size=(2, 4)))
# %% gelu
def gelu(x):
cdf = 0.5 * (1.0 + jnp.tanh(jnp.sqrt(2 / jnp.pi) * (x + 0.044715 * (x ** 3))))
return x * cdf
# %%
# %%
# %%
x = jnp.linspace(-7, 7, 200)
plt.plot(x, gelu(x))
# %% Taking gradients
# %%
# %%
# %% Vectorization
def printing_gelu(x):
print('shape of x is', x.shape)
return gelu(x)
# %%
# %%
# %% Linear Regression
xs = np.random.normal(size=(128, 1))
noise = 0.5 * np.random.normal(size=(128, 1))
ys = xs * 2 - 1 + noise
plt.scatter(xs, ys)
# %%
initial_weight = jnp.asarray(np.random.normal())
initial_bias = jnp.asarray(np.random.normal())
plt.scatter(xs, ys)
plt.plot(xs, initial_weight * xs + initial_bias, color='red')
# %%
# %%
# %%
def loss(weight, bias, x, y):
...
# %%
# %%
# %% Structured objects
from flax.struct import dataclass
@dataclass
class WeightBiasPair:
weight: jnp.ndarray
bias: jnp.ndarray
def loss(params, x, y):
...
...
plt.scatter(xs, ys)
plt.plot(xs, initial_params.weight * xs + initial_params.bias, color='red')
loss(initial_params, xs, ys)
# %%
# %%
# %% Gradients and parameter updates
# %%
# %%
# %% Training loop and JIT
# %%
# %%
# %% Learning rate adjustment
def train(params, x, y, lr):
for _ in range(100):
...
final_loss = loss(params, x, y)
return final_loss, params
final_loss, params = train(initial_params, xs, ys, 0.005)
plt.scatter(xs, ys)
plt.plot(xs, params.weight * xs + params.bias, color='red')
# %%
# %%
# %% JVP
# %%
# %%
# %% Optax
import optax
tx = optax.scale(-0.005)
state = tx.init(initial_params)
params = initial_params
...
# %%
# %%