-
Notifications
You must be signed in to change notification settings - Fork 566
/
model.py
71 lines (51 loc) · 2.3 KB
/
model.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import Wav2Vec2Config
from .torch_utils import get_mask_from_lengths
from .wav2vec2 import Wav2Vec2Model
class Audio2MeshModel(nn.Module):
def __init__(
self,
config
):
super().__init__()
out_dim = config['out_dim']
latent_dim = config['latent_dim']
model_path = config['model_path']
only_last_fetures = config['only_last_fetures']
from_pretrained = config['from_pretrained']
self._only_last_features = only_last_fetures
self.audio_encoder_config = Wav2Vec2Config.from_pretrained(model_path, local_files_only=True)
if from_pretrained:
self.audio_encoder = Wav2Vec2Model.from_pretrained(model_path, local_files_only=True)
else:
self.audio_encoder = Wav2Vec2Model(self.audio_encoder_config)
self.audio_encoder.feature_extractor._freeze_parameters()
hidden_size = self.audio_encoder_config.hidden_size
self.in_fn = nn.Linear(hidden_size, latent_dim)
self.out_fn = nn.Linear(latent_dim, out_dim)
nn.init.constant_(self.out_fn.weight, 0)
nn.init.constant_(self.out_fn.bias, 0)
def forward(self, audio, label, audio_len=None):
attention_mask = ~get_mask_from_lengths(audio_len) if audio_len else None
seq_len = label.shape[1]
embeddings = self.audio_encoder(audio, seq_len=seq_len, output_hidden_states=True,
attention_mask=attention_mask)
if self._only_last_features:
hidden_states = embeddings.last_hidden_state
else:
hidden_states = sum(embeddings.hidden_states) / len(embeddings.hidden_states)
layer_in = self.in_fn(hidden_states)
out = self.out_fn(layer_in)
return out, None
def infer(self, input_value, seq_len):
embeddings = self.audio_encoder(input_value, seq_len=seq_len, output_hidden_states=True)
if self._only_last_features:
hidden_states = embeddings.last_hidden_state
else:
hidden_states = sum(embeddings.hidden_states) / len(embeddings.hidden_states)
layer_in = self.in_fn(hidden_states)
out = self.out_fn(layer_in)
return out