diff --git a/onnxruntime/python/tools/transformers/models/stable_diffusion/test/astronaut_riding_error.png b/onnxruntime/python/tools/transformers/models/stable_diffusion/test/astronaut_riding_error.png
new file mode 100644
index 0000000000..37d639a511
Binary files /dev/null and b/onnxruntime/python/tools/transformers/models/stable_diffusion/test/astronaut_riding_error.png differ
diff --git a/onnxruntime/python/tools/transformers/models/stable_diffusion/test/check_image.py b/onnxruntime/python/tools/transformers/models/stable_diffusion/test/check_image.py
index da7f47b144..9a3615c1cb 100644
--- a/onnxruntime/python/tools/transformers/models/stable_diffusion/test/check_image.py
+++ b/onnxruntime/python/tools/transformers/models/stable_diffusion/test/check_image.py
@@ -14,6 +14,7 @@ def arg_parser():
     parser.add_argument("--image1", type=str, help="Path to image 1")
     parser.add_argument("--image2", type=str, help="Path to image 2")
     parser.add_argument("--cache_dir", type=str, help="Path to model cache directory")
+    parser.add_argument("--negative", action="store_true", help="match the unexpected image, for testing purpose")
     args = parser.parse_args()
     return args
 
@@ -62,11 +63,16 @@ def main():
     cache_dir = args.cache_dir
     score = round(generate_score(image1, image2, cache_dir), 2)
     print("similarity Score: ", {score})
-    if score < 97:
-        print(f"{image1} and {image2} are different")
-        raise SystemExit(1)
+    if args.negative:
+        if score > 97:
+            print("Why generated this incorrect image")
+            raise SystemExit(1)
     else:
-        print(f"{image1} and {image2} are same")
+        if score < 97:
+            print(f"{image1} and {image2} are different")
+            raise SystemExit(1)
+        else:
+            print(f"{image1} and {image2} are same")
 
 
 if __name__ == "__main__":
diff --git a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml
index 8019633066..31f3eac808 100644
--- a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml
@@ -244,6 +244,27 @@ stages:
         path: $(CLIP_MODEL_CACHE)
       displayName: Cache clip model
 
+    - script: |
+        docker run --rm --gpus all -v $PWD:/workspace \
+          -v $(CLIP_MODEL_CACHE):/model_cache:rw  \
+          nvcr.io/nvidia/pytorch:22.11-py3 \
+          bash -c '
+            set -ex; \
+            python3 --version; \
+            python3 -m pip install --upgrade pip; \
+            pushd /workspace/onnxruntime/python/tools/transformers/models/stable_diffusion/; \
+            image2=$(find $(pwd) -name "astronaut_riding_a_h*.png") ; \
+            pushd test; \
+            python3 -m pip install -r requirements.txt; \
+            echo check demo_txt2image.py generate image; \
+            python3 -u check_image.py --image1 astronaut_riding_error.png --image2 $image2 --cache_dir /model_cache --negative; \
+            popd ; \
+            popd ; \
+            '
+      displayName: 'Check if the generated image is wierd'
+      workingDirectory: $(Build.SourcesDirectory)
+      continueOnError: true
+
     - script: |
         docker run --rm --gpus all -v $PWD:/workspace \
           -v $(CLIP_MODEL_CACHE):/model_cache:rw  \