Fix the TRT GPU build job in python packaging pipeline (#16073)

1. Cherry-pick #16054 back to the main branch
2. Replace onnxruntime-gpu-winbuild-t4 with onnxruntime-Win2022-GPU-T4.
The later one has VS2022.

---------

Co-authored-by: Patrice Vignola <vignola.patrice@gmail.com>

onnxruntime/test/contrib_ops/attention_op_test_helper.cc

@@ -7,7 +7,8 @@
 namespace onnxruntime {
 namespace test {
 
-#if !defined(_MSC_VER) || defined(USE_DML)
+// Disable some tests in Windows since prefast build might crash with large test data.
+#if !defined(_MSC_VER)
 void GetWeight_64_3_64(std::vector<float>& weight_data) {
   weight_data = {
       -0.004707f, -0.006775f, 0.0009236f, 0.003067f, -0.00806f, 0.00779f, 0.0004425f, 0.00846f, 0.00048f,

onnxruntime/test/contrib_ops/attention_op_test_helper.h

@@ -43,7 +43,7 @@ struct AttentionTestData {
 };
 
 // Disable some tests in Windows since prefast build might crash with large test data.
-#if !defined(_MSC_VER) || defined(USE_DML)
+#if !defined(_MSC_VER)
 // Return packed weights and bias for input projection.
 void GetAttentionWeight(std::vector<float>& weight_data, int elements = 64 * 3 * 64, int offset = 0, int step = 1);
 void GetAttentionBias(std::vector<float>& bias_data, int elements = 3 * 64, int offset = 0, int step = 1);

onnxruntime/test/contrib_ops/multihead_attention_op_test.cc

@@ -452,7 +452,8 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, bool disable_cpu
   }
 }
 
-#if !defined(_MSC_VER) || defined(USE_DML)
+// Disable some tests in Windows since prefast build might crash with large test data.
+#if !defined(_MSC_VER)
 // Test fused cross attention kernel
 // It requires head_size > 32 and head_size <= 64 for T4 GPU; hidden_size == v_hidden_size.
 TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize40) {

tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml

@@ -113,10 +113,11 @@ stages:
           addToPath: true
           architecture: $(buildArch)
 
-      - task: PythonScript@0
-        displayName: 'Run a Python script'
+      - task: onebranch.pipeline.tsaoptions@1
+        displayName: 'OneBranch TSAOptions'
         inputs:
-          scriptPath: 'tools\ci_build\update_tsaoptions.py'
+          tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
+          appendSourceBranchName: false
 
       - template: set-nightly-build-option-variable-step.yml
 
@@ -270,33 +271,33 @@ stages:
   - ${{ if eq(parameters.enable_windows_gpu, true) }}:
       - template: py-win-gpu.yml
         parameters:
-          MACHINE_POOL: 'onnxruntime-gpu-winbuild-t4'
+          MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4'
           PYTHON_VERSION: '3.8'
-          EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.0.12.Windows10.x86_64.cuda-11.8" --cuda_version=11.8 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
+          EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.0.12.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
           ENV_SETUP_SCRIPT: setup_env_gpu.bat
           EP_NAME: gpu
 
       - template: py-win-gpu.yml
         parameters:
-          MACHINE_POOL: 'onnxruntime-gpu-winbuild-t4'
+          MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4'
           PYTHON_VERSION: '3.9'
-          EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.0.12.Windows10.x86_64.cuda-11.8" --cuda_version=11.8 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
+          EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.0.12.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
           ENV_SETUP_SCRIPT: setup_env_gpu.bat
           EP_NAME: gpu
 
       - template: py-win-gpu.yml
         parameters:
-          MACHINE_POOL: 'onnxruntime-gpu-winbuild-t4'
+          MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4'
           PYTHON_VERSION: '3.10'
-          EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.0.12.Windows10.x86_64.cuda-11.8" --cuda_version=11.8 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
+          EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.0.12.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
           ENV_SETUP_SCRIPT: setup_env_gpu.bat
           EP_NAME: gpu
 
       - template: py-win-gpu.yml
         parameters:
-          MACHINE_POOL: 'onnxruntime-gpu-winbuild-t4'
+          MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4'
           PYTHON_VERSION: '3.11'
-          EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.0.12.Windows10.x86_64.cuda-11.8" --cuda_version=11.8 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
+          EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.0.12.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
           ENV_SETUP_SCRIPT: setup_env_gpu.bat
           EP_NAME: gpu
 

tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml

@@ -45,6 +45,12 @@ jobs:
           addToPath: true
           architecture: 'x64'
 
+      - task: onebranch.pipeline.tsaoptions@1
+        displayName: 'OneBranch TSAOptions'
+        inputs:
+          tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
+          appendSourceBranchName: false
+
       - task: BatchScript@1
         displayName: 'setup env'
         inputs:
@@ -65,9 +71,9 @@ jobs:
 
       - template: download-deps.yml
 
-      - ${{ if contains(parameters.EP_BUILD_FLAGS, 'use_cuda') }}:
-        - powershell: |
-           azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory)
+      - powershell: |
+         azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory)
+
       - task: PythonScript@0
         displayName: 'Update deps.txt'
         inputs: