diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py
index ac34a5380c..6b0519d392 100755
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@@ -486,6 +486,14 @@ def install_python_deps(numpy_version=""):
                     'files.pythonhosted.org'] + dep_packages)
 
 
+# We need to install Torch to test certain functionalities of the ORT Python package
+def install_torch():
+    # Command works for both Windows
+    run_subprocess([sys.executable, '-m', 'pip', 'install', '--trusted-host',
+                    'files.pythonhosted.org', 'torch===1.5.1+cu101', 'torchvision===0.6.1+cu101',
+                    '-f', 'https://download.pytorch.org/whl/torch_stable.html'])
+
+
 def check_md5(filename, expected_md5):
     if not os.path.exists(filename):
         return False
@@ -1214,10 +1222,19 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
                 [sys.executable, 'onnxruntime_test_python.py'],
                 cwd=cwd, dll_path=dll_path)
 
-            iobinding_test = False
             # For CUDA enabled builds test IOBinding feature
-            if args.use_cuda:
+            # Limit testing to Windows non-ARM builds for now
+            iobinding_test = False
+            if args.use_cuda and not (args.arm or args.arm64):
+                # We need to have Torch installed to test the IOBinding feature
+                # which currently uses Torch's allocator to allocate GPU memory for testing
                 iobinding_test = True
+
+                # Try install Torch on Windows
+                if is_windows():
+                    log.info("Attempting to install Torch to test ORT's IOBinding feature")
+                    install_torch()
+
                 try:
                     import torch  # noqa
                 except ImportError as error:
@@ -1228,6 +1245,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
                         "The IOBinding tests will be skipped as it requires Torch.")
 
             if iobinding_test:
+                log.info("Testing IOBinding feature")
                 run_subprocess([sys.executable, 'onnxruntime_test_python_iobinding.py'], cwd=cwd, dll_path=dll_path)
 
             if not args.disable_ml_ops:
diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh
index eaa86c2fb7..1f54119de1 100755
--- a/tools/ci_build/github/linux/docker/scripts/install_deps.sh
+++ b/tools/ci_build/github/linux/docker/scripts/install_deps.sh
@@ -113,8 +113,8 @@ if [ $DEVICE_TYPE = "Normal" ]; then
     ${PYTHON_EXE} -m pip install sympy==1.1.1
 elif [ $DEVICE_TYPE = "gpu" ]; then
     ${PYTHON_EXE} -m pip install sympy==1.1.1
-    ${PYTHON_EXE} -m pip install --upgrade --pre torch==1.6.0.dev20200610 torchvision==0.7.0.dev20200610 -f https://download.pytorch.org/whl/nightly/cu101/torch_nightly.html
     if [[ $BUILD_EXTR_PAR = *--enable_training* ]]; then
+      ${PYTHON_EXE} -m pip install --upgrade --pre torch==1.6.0.dev20200610 torchvision==0.7.0.dev20200610 -f https://download.pytorch.org/whl/nightly/cu101/torch_nightly.html
       ${PYTHON_EXE} -m pip install  transformers==v2.10.0
       # transformers requires sklearn
       ${PYTHON_EXE} -m pip install sklearn