Merge pull request #5046 from kamo-naoyuki/ffmpeg

Add installers/install_ffmpeg.sh

doc/installation.md

@@ -1,17 +1,11 @@
 ## Installation
 ### Requirements
 
-- Python 3.6.1+
-- gcc 4.9+ for PyTorch1.0.0+
-
-Optionally, GPU environment requires the following libraries:
-
-- Cuda 8.0, 9.0, 9.1, 10.0 depending on each DNN library
-- Cudnn 6+, 7+
-- NCCL 2.0+ (for the use of multi-GPUs)
+- Python 3.7+
+- gcc 4.9+ for PyTorch1.4.0+
 
 (If you'll use anaconda environment at installation step2,
-the following packages are installed using Anaconda, so you can skip them.)
+the following packages are installed using conda, so you can skip them.)
 
 - cmake3 for some extensions
     ```sh
@@ -25,20 +19,6 @@ the following packages are installed using Anaconda, so you can skip them.)
     # For CentOS
     $ sudo yum install sox
     ```
-- sndfile
-    ```sh
-    # For Ubuntu
-    $ sudo apt-get install libsndfile1-dev
-    # For CentOS
-    $ sudo yum install libsndfile
-    ```
-- ffmpeg (This is not required when installing, but used in some recipes)
-    ```sh
-    # For Ubuntu
-    $ sudo apt-get install ffmpeg
-    # For CentOS
-    $ sudo yum install ffmpeg
-    ```
 - flac (This is not required when installing, but used in some recipes)
     ```sh
     # For Ubuntu
@@ -156,24 +136,24 @@ We also have [prebuilt Kaldi binaries](https://github.com/espnet/espnet/blob/mas
 
     We also have some scripts to generate `tools/activate_python.sh`.
 
-    - Option A) Setup Anaconda environment
+    - Option A) Setup conda environment
 
         ```sh
         $ cd <espnet-root>/tools
         $ ./setup_anaconda.sh [output-dir-name|default=venv] [conda-env-name|default=root] [python-version|default=none]
         # e.g.
-        $ ./setup_anaconda.sh anaconda espnet 3.8
+        $ ./setup_anaconda.sh miniconda espnet 3.8
         ```
 
         This script tries to create a new miniconda if the output directory doesn't exist.
-        If you already have Anaconda and you'll use it then,
+        If you already have conda and you'll use it then,
 
         ```sh
         $ cd <espnet-root>/tools
-        $ CONDA_TOOLS_DIR=$(dirname ${CONDA_EXE})/..
-        $ ./setup_anaconda.sh ${CONDA_TOOLS_DIR} [conda-env-name] [python-version]
+        $ CONDA_ROOT=${CONDA_EXE}/../..  # CONDA_EXE is an environment variable set by ${CONDA_ROOT}/etc/profile.d/conda.sh
+        $ ./setup_anaconda.sh ${CONDA_ROOT} [conda-env-name] [python-version]
         # e.g.
-        $ ./setup_anaconda.sh ${CONDA_TOOLS_DIR} espnet 3.8
+        $ ./setup_anaconda.sh ${CONDA_ROOT} espnet 3.8
         ```
 
     - Option B) Setup venv from system Python
@@ -195,7 +175,7 @@ We also have [prebuilt Kaldi binaries](https://github.com/espnet/espnet/blob/mas
 
         ```sh
         $ cd <espnet-root>/tools
-        $ rm -f activate_python.sh
+        $ rm -f activate_python.sh && touch activate_python.sh
         ```
 1. Install ESPnet
 
@@ -258,6 +238,6 @@ e.g.
 You can check whether your installation is successfully finished by
 ```sh
 cd <espnet-root>/tools
-. ./activate_python.sh; python3 check_install.py
+bash -c ". ./activate_python.sh; . ./extra_path.sh; python3 check_install.py"
 ```
 Note that this check is always called in the last stage of the above installation.

tools/Makefile

@@ -26,7 +26,7 @@ WITH_OMP=ON
 
 .PHONY: all clean
 
-all: showenv python conda_packages.done sctk sph2pipe check_install
+all: showenv python conda_packages.done ffmpeg.done sctk sph2pipe check_install
 
 python: activate_python.sh packaging.done espnet.done pytorch.done chainer.done fairscale.done torch_optimizer.done
 extra: warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pesq kenlm.done pyopenjtalk.done py3mmseg.done beamformit.done fairseq.done s3prl.done k2.done transformers.done phonemizer.done longformer.done muskit.done whisper.done rvad_fast.done
@@ -58,22 +58,29 @@ cmake.done: activate_python.sh
 flac.done: activate_python.sh
 	. ./activate_python.sh && { command -v flac || conda install -y libflac -c conda-forge; }
 	touch flac.done
-ffmpeg.done: activate_python.sh
-	. ./activate_python.sh && { command -v ffmpeg || conda install -y ffmpeg -c conda-forge; }
-	touch ffmpeg.done
 sox.done: activate_python.sh
 	. ./activate_python.sh && { command -v sox || conda install -y sox -c conda-forge; }
 	touch sox.done
 sndfile.done: activate_python.sh
-	. ./activate_python.sh && { python3 -c "from ctypes.util import find_library as F; assert F('sndfile') is not None" || conda install -y libsndfile -c conda-forge; }
+	# NOTE(kamo): The wheel version of Soundfile includes shared library and it is refered in preference.
+	# However, some old linuxs are not compatible with the wheel, so libsndfile is installed for this case.
+	. ./activate_python.sh && { conda install -y libsndfile -c conda-forge; }
 	touch sndfile.done
 ifneq ($(strip $(USE_CONDA)),)
-conda_packages.done: bc.done cmake.done flac.done ffmpeg.done sox.done sndfile.done
+conda_packages.done: bc.done cmake.done flac.done sox.done sndfile.done
 else
 conda_packages.done:
 endif
 	touch conda_packages.done
 
+ffmpeg.done: activate_python.sh
+ifneq ($(strip $(USE_CONDA)),)
+	. ./activate_python.sh && { command -v ffmpeg || conda install -y ffmpeg -c conda-forge; }
+else
+	. ./activate_python.sh && { command -v ffmpeg || ./installers/install_ffmpeg.sh; }
+endif
+	touch ffmpeg.done
+
 sctk: sctk/bin/sclite
 sctk/bin/sclite:
 	./installers/install_sctk.sh

tools/check_install.py

@@ -164,7 +164,7 @@ def main():
     print()
     print("Executables:")
 
-    pattern = re.compile(r"([0-9]+.[0-9]+.[0-9]+[^\s]*)\s*")
+    pattern = re.compile(r"([0-9]+\.[0-9]+(?:\.[0-9]+[^\s]*)?)\s*")
 
     for name, installer, version_option in executable_list:
         if shutil.which(name) is not None:

tools/extra_path.sh

@@ -33,6 +33,7 @@ export PATH="${TOOL_DIR}"/BeamformIt:"${PATH:-}"
 export PATH="${TOOL_DIR}"/espeak-ng/bin:"${PATH:-}"
 export PATH="${TOOL_DIR}"/MBROLA/Bin:"${PATH:-}"
 export PATH="${TOOL_DIR}"/festival/bin:"${PATH:-}"
+export PATH="${TOOL_DIR}"/ffmpeg-release:"${PATH:-}"
 export LD_LIBRARY_PATH="${TOOL_DIR}"/lib:"${TOOL_DIR}"/lib64:"${LD_LIBRARY_PATH:-}"
 export LD_LIBRARY_PATH="${TOOL_DIR}"/espeak-ng/lib:"${LD_LIBRARY_PATH:-}"
 export PYTHONPATH="${TOOL_DIR}"/s3prl:"${PYTHONPATH:-}"

tools/installers/install_ffmpeg.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ $# != 0 ]; then
+    echo "Usage: $0"
+    exit 1;
+fi
+
+unames="$(uname -s)"
+unamem="$(uname -m)"
+
+dirname=ffmpeg-release
+rm -rf ${dirname}
+
+if [[ ${unames} =~ Linux ]]; then
+    if [ "${unamem}" = x86_64 ]; then
+        unamem=amd64
+    fi
+    url="https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-${unamem}-static.tar.xz"
+    wget --no-check-certificate --trust-server-names "${url}"
+    tar xvf "ffmpeg-release-${unamem}-static.tar.xz"
+    ffmpegdir="$(ls -d ffmpeg-*-static)"
+    ln -sf "${ffmpegdir}" "${dirname}"
+elif [[ ${unames} =~ Darwin ]]; then
+    # bins="ffmpeg ffprobe ffplay ffserver"
+    bins="ffmpeg ffprobe ffplay"
+    for bin in ${bins}; do
+        url="https://evermeet.cx/ffmpeg/getrelease/${bin}/zip"
+        wget --no-check-certificate --trust-server-names "${url}" -O "${bin}-release.zip"
+        unzip -o "${bin}-*.zip" -d ${dirname}
+    done
+elif [[ ${unames} =~ MINGW || ${unames} =~ CYGWIN || ${unames} =~ MSYS ]]; then
+    # Windows
+    url=https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-essentials.zip
+    wget --no-check-certificate --trust-server-names "${url}" -O "ffmpeg-release-essentials_build.zip"
+    unzip -o ffmpeg-release-essentials_build.zip
+    ffmpegdir="$(ls -d ffmpeg-*-essentials_build)"
+    ln -sf "${ffmpegdir}"/bin "${dirname}"
+else
+    echo "$0: Warning: not supported platform: ${unames}"
+fi