/
positional-encoding.py
31 lines (25 loc) · 1.01 KB
/
positional-encoding.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
import numpy as np
import math
import matplotlib.pyplot as plt
class DefaultParameters:
DEFAULT_NUM_HEADS = 8
DEFAULT_WORD_WIDTH = 512
DEFAULT_PROJECTION_WIDTH = 64
DEFAULT_SCALE_FACTOR = 100
DEFAULT_FFNN_HIDDEN_LAYER_WIDTH = 2048
num_words = 10
positions = range(num_words)
dimensions = range(DefaultParameters.DEFAULT_WORD_WIDTH)
mesh = np.zeros([len(positions), len(dimensions)])
positional_encoding = lambda pos, dimension: math.sin(
pos / math.pow(10000, dimension / DefaultParameters.DEFAULT_WORD_WIDTH)) if (
dimension % 2 == 0) else math.cos(pos / math.pow(10000, (dimension - 1) / len(dimensions)))
plt.figure()
for dimension in dimensions:
for position in positions:
encoding = positional_encoding(position, dimension)
greyscale_encoding = (encoding + 1) / 2
mesh[position, dimension] = greyscale_encoding
# plt.plot(dimension, position, marker="s", color=(greyscale_encoding, greyscale_encoding, greyscale_encoding))
plt.pcolormesh(mesh)
plt.show()