diff --git a/onnxruntime/python/tools/transformers/models/whisper/README.md b/onnxruntime/python/tools/transformers/models/whisper/README.md
index 7a678f2734..b44124340a 100644
--- a/onnxruntime/python/tools/transformers/models/whisper/README.md
+++ b/onnxruntime/python/tools/transformers/models/whisper/README.md
@@ -10,6 +10,7 @@ Please note the package versions needed for using Whisper in the `requirements.t
   - Note that `torch` with CUDA enabled is not installed automatically. This is because `torch` should be installed with the CUDA version used on your machine. Please visit [the PyTorch website](https://pytorch.org/get-started/locally/) to download the `torch` version that is used with the CUDA version installed on your machine and satisfies the requirement listed in the file.
 - `requirements.txt`
   - Package versions needed in each of the above files
+- ffmpeg-python is also required, but please install it by source code with allowed codecs to avoid any patent risks.
 
 In addition to the above packages, you will need to install `ffmpeg` on your machine. Visit the [FFmpeg website](https://ffmpeg.org/) for details. You can also install it natively using package managers.
 
diff --git a/onnxruntime/python/tools/transformers/models/whisper/requirements.txt b/onnxruntime/python/tools/transformers/models/whisper/requirements.txt
index 9bbe0d7380..4cb8085017 100644
--- a/onnxruntime/python/tools/transformers/models/whisper/requirements.txt
+++ b/onnxruntime/python/tools/transformers/models/whisper/requirements.txt
@@ -1,7 +1,6 @@
 torch>=1.13.0
 transformers>=4.24.0
 openai-whisper
-ffmpeg-python
 datasets
 soundfile
 librosa
diff --git a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml
index 43dedbc394..8e28342aa6 100644
--- a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml
@@ -392,33 +392,3 @@ stages:
             '
       displayName: 'Convert Whisper Model'
       workingDirectory: $(Build.SourcesDirectory)
-
-    - script: |
-        docker run --rm --gpus all -v $(Build.SourcesDirectory):/workspace \
-           -v $(Build.BinariesDirectory)/ort-artifact/:/ort-artifact \
-           -v $(Agent.TempDirectory)/whisper_large_v3:/whisper_large_v3 \
-           onnxruntimepackagestest \
-            bash -c '
-              set -ex; \
-              pushd /workspace/onnxruntime/python/tools/transformers/ ; \
-              python3 -m pip install --upgrade pip ; \
-              pushd models/whisper ; \
-              python3 -m pip install -r requirements.txt ; \
-              popd ; \
-              python3 -m pip install /ort-artifact/*.whl ; \
-              python3 -m pip uninstall -y torch ; \
-              python3 -m pip install torch --index-url https://download.pytorch.org/whl/cu118 ; \
-              ls whisperlargev3; \
-              python3 -m models.whisper.benchmark \
-                  --benchmark-type ort \
-                  --audio-path models/whisper/test/1272-141231-0002.mp3 \
-                  --model-name openai/whisper-large-v3 \
-                  --ort-model-path /workspace/onnxruntime/python/tools/transformers/whisperlargev3/whisper_large_v3_beamsearch.onnx \
-                  --precision fp32 \
-                  --device cuda > ort_output.txt ; \
-              cat ort_output.txt ; \
-              diff ort_output.txt /workspace/onnxruntime/python/tools/transformers/models/whisper/test/whisper_ort_output.txt && exit 0 || exit 1
-              popd ; \
-            '
-      displayName: 'Test Whisper ONNX Model'
-      workingDirectory: $(Build.SourcesDirectory)
diff --git a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu
index c9038afc09..331eb64720 100644
--- a/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu
+++ b/tools/ci_build/github/linux/docker/Dockerfile.package_ubuntu_2004_gpu
@@ -25,10 +25,6 @@ RUN apt-get install -y --no-install-recommends \
     python3-dev \
     python3-wheel
 
-# Install ffmpeg, which couldn't be installed in UBI8
-# https://stackoverflow.com/questions/73597789/how-to-install-ffmpeg-on-ubi-docker-images
-RUN apt-get install -y --no-install-recommends ffmpeg
-
 RUN pip install --upgrade pip
 
 # Install TensorRT