diff --git a/CMakeLists.txt b/CMakeLists.txt
index c3bac20272..cb9c84bbf9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -52,13 +52,11 @@ endif()
 
 
 # Options
-option(BUILD_SOX "Build libsox statically" ON)
 option(BUILD_RIR "Enable RIR simulation" ON)
 option(BUILD_RNNT "Enable RNN transducer" ON)
 option(BUILD_ALIGN "Enable forced alignment" ON)
 option(BUILD_CUDA_CTC_DECODER "Build CUCTC decoder" OFF)
 option(BUILD_TORCHAUDIO_PYTHON_EXTENSION "Build Python extension" OFF)
-option(USE_FFMPEG "Enable ffmpeg-based features" OFF)
 option(USE_CUDA "Enable CUDA support" OFF)
 option(USE_ROCM "Enable ROCM support" OFF)
 option(USE_OPENMP "Enable OpenMP support" OFF)
@@ -166,14 +164,6 @@ else()
 endif()
 
 add_subdirectory(src/libtorchaudio)
-if (USE_FFMPEG)
-  if (DEFINED ENV{FFMPEG_ROOT})
-    add_subdirectory(third_party/ffmpeg/single)
-  else()
-    message(STATUS "Building FFmpeg integration with multi version support")
-    add_subdirectory(third_party/ffmpeg/multi)
-  endif()
-endif()
 if (BUILD_CUDA_CTC_DECODER)
   if (NOT USE_CUDA)
     message(FATAL "BUILD_CUDA_CTC_DECODER=1 but USE_CUDA=0.")
diff --git a/packaging/torchaudio/meta.yaml b/packaging/torchaudio/meta.yaml
index 555e214ee8..7ffa2c9d89 100644
--- a/packaging/torchaudio/meta.yaml
+++ b/packaging/torchaudio/meta.yaml
@@ -43,7 +43,6 @@ build:
     - USE_CUDA
     - CUDA_HOME
     - TORCH_CUDA_ARCH_LIST
-    - USE_FFMPEG
     - USE_OPENMP
     - MACOSX_DEPLOYMENT_TARGET
 
diff --git a/third_party/ffmpeg/multi/CMakeLists.txt b/third_party/ffmpeg/multi/CMakeLists.txt
index 11ea5d1dce..88f51adb94 100644
--- a/third_party/ffmpeg/multi/CMakeLists.txt
+++ b/third_party/ffmpeg/multi/CMakeLists.txt
@@ -87,16 +87,6 @@ elseif (UNIX)
       URL ${base_url}/2023-07-06/linux_x86_64/6.0.tar.gz
       URL_HASH SHA256=04d3916404bab5efadd29f68361b7d13ea71e6242c6473edcb747a41a9fb97a6
       )
-  else ()
-    # Possible case ppc64le (though it's not officially supported.)
-    message(
-      FATAL_ERROR
-      "${CMAKE_SYSTEM_PROCESSOR} is not supported for FFmpeg multi-version integration. "
-      "If you have FFmpeg libraries installed in the system,"
-      " setting FFMPEG_ROOT environment variable to the root directory of FFmpeg installation"
-      " (the directory where `include` and `lib` sub directories with corresponding headers"
-      " and library files are present) will invoke the FFmpeg single-version integration. "
-      "If you do not need the FFmpeg integration, setting USE_FFMPEG=0 will bypass the issue.")
   endif()
 elseif(MSVC)
   FetchContent_Declare(
diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py
index 66f816aa43..635b4ae496 100644
--- a/tools/setup_helpers/extension.py
+++ b/tools/setup_helpers/extension.py
@@ -34,10 +34,8 @@ def _get_build(var, default=False):
 
 
 _BUILD_CPP_TEST = _get_build("BUILD_CPP_TEST", False)
-_BUILD_SOX = False if platform.system() == "Windows" else _get_build("BUILD_SOX", True)
 _BUILD_RIR = _get_build("BUILD_RIR", True)
 _BUILD_RNNT = _get_build("BUILD_RNNT", True)
-_USE_FFMPEG = _get_build("USE_FFMPEG", True)
 _USE_ROCM = _get_build("USE_ROCM", torch.backends.cuda.is_built() and torch.version.hip is not None)
 _USE_CUDA = _get_build("USE_CUDA", torch.backends.cuda.is_built() and torch.version.hip is None)
 _BUILD_ALIGN = _get_build("BUILD_ALIGN", True)
@@ -102,7 +100,6 @@ def build_extension(self, ext):
             "-DCMAKE_VERBOSE_MAKEFILE=ON",
             f"-DPython_INCLUDE_DIR={distutils.sysconfig.get_python_inc()}",
             f"-DBUILD_CPP_TEST={'ON' if _BUILD_CPP_TEST else 'OFF'}",
-            f"-DBUILD_SOX:BOOL={'ON' if _BUILD_SOX else 'OFF'}",
             f"-DBUILD_RIR:BOOL={'ON' if _BUILD_RIR else 'OFF'}",
             f"-DBUILD_RNNT:BOOL={'ON' if _BUILD_RNNT else 'OFF'}",
             f"-DBUILD_ALIGN:BOOL={'ON' if _BUILD_ALIGN else 'OFF'}",
@@ -112,7 +109,6 @@ def build_extension(self, ext):
             f"-DUSE_ROCM:BOOL={'ON' if _USE_ROCM else 'OFF'}",
             f"-DUSE_CUDA:BOOL={'ON' if _USE_CUDA else 'OFF'}",
             f"-DUSE_OPENMP:BOOL={'ON' if _USE_OPENMP else 'OFF'}",
-            f"-DUSE_FFMPEG:BOOL={'ON' if _USE_FFMPEG else 'OFF'}",
         ]
         build_args = ["--target", "install"]
         # Pass CUDA architecture to cmake