fix the gradient backward issue when joint training with s3prl frontend

espnet · May 10, 2023 · 3434492 · 3434492
1 parent 33aa097
commit 3434492
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 8 deletions.
diff --git a/espnet2/asr/frontend/s3prl.py b/espnet2/asr/frontend/s3prl.py
@@ -50,14 +50,10 @@ def __init__(
             normalize=frontend_conf.get("normalize", False),
             extra_conf=frontend_conf.get("extra_conf", None),
         )
+        if getattr(upstream.upstream, "model", None):
+            if getattr(upstream.upstream.model, "feature_grad_mult", None):
+                upstream.upstream.model.feature_grad_mult = 1.0
         upstream.eval()
-        if getattr(
-            upstream.upstream, "model", None
-        ) is not None and upstream.upstream.model.__class__.__name__ in [
-            "Wav2Vec2Model",
-            "HubertModel",
-        ]:
-            upstream.upstream.model.encoder.layerdrop = 0.0
 
         if layer != -1:
             layer_selections = [layer]

diff --git a/tools/extra_path.sh b/tools/extra_path.sh
@@ -36,5 +36,4 @@ export PATH="${TOOL_DIR}"/festival/bin:"${PATH:-}"
 export PATH="${TOOL_DIR}"/ffmpeg-release:"${PATH:-}"
 export LD_LIBRARY_PATH="${TOOL_DIR}"/lib:"${TOOL_DIR}"/lib64:"${LD_LIBRARY_PATH:-}"
 export LD_LIBRARY_PATH="${TOOL_DIR}"/espeak-ng/lib:"${LD_LIBRARY_PATH:-}"
-export PYTHONPATH="${TOOL_DIR}"/s3prl:"${PYTHONPATH:-}"
 export PYTHONPATH="${TOOL_DIR}"/RawNet/python/RawNet3:"${TOOL_DIR}"/RawNet/python/RawNet3/models:"${PYTHONPATH:-}"