Update win-ci-pipeline.yml: enable xnnpack tests (#16244)

1. Enable xnnpack test 2. Change TSA database name from onnxruntime_master to onnxruntime_main. This is a leftover of renaming the "master" branch to "main" 3. Add two static analysis jobs for WinML and DML 4. Rename the machine pool "aiinfra-dml-winbuild" to "onnxruntime-Win2019-GPU-dml-A10", so that the internal and public ADO instances use the same machine pool name. 5. Move Windows GPU CI build pipeline from "onnxruntime-Win2022-GPU-T4" to "onnxruntime-Win2022-GPU-A10" machine pool, because we do not have enough T4 GPUs.
2026-05-14 20:48:00 +00:00 · 2023-06-14 19:12:42 -07:00 · 2023-06-14 19:12:42 -07:00 · dbc7a195b1
commit dbc7a195b1
parent 9be133231f
14 changed files with 130 additions and 71 deletions
--- a/.gdn/.gdntsa
+++ b/.gdn/.gdntsa
@ -1,3 +1,3 @@
 {
-    "codebaseName": "onnxruntime_master"
+    "codebaseName": "onnxruntime_main"
 }
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@ -3,8 +3,8 @@ name: Lint
 on:
  push:
    branches:
-      - master
      - main
+      - rel-*
  pull_request:

 jobs:
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@ -2,10 +2,14 @@ name: Linux_CI
 on:
  push:
    branches:
-      - master
      - main
+      - rel-*
  pull_request:

+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
  Onnxruntime-TVM:
    runs-on: ubuntu-latest
--- a/.github/workflows/sca.yml
+++ b/.github/workflows/sca.yml
@ -3,10 +3,11 @@ on:
  push:
    branches:
      - main
+      - rel-*
  pull_request:

 concurrency:
-  group: ${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

 env:
@ -14,7 +15,7 @@ env:
  AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4

 jobs:
-  Onnxruntime-SCA:
+  Onnxruntime-SCA-training-CUDA:
    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
    steps:
      - uses: actions/checkout@v3
@ -33,6 +34,11 @@ jobs:
        run: azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" cuda_sdk


+      - name: Delete build folder
+        run: |
+          if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
+          &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug
+
      # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
      - name: Build code
        env:
@ -49,3 +55,79 @@ jobs:
        with:
          sarif_file: ${{ github.workspace }}\output\MergeResult.sarif
          category: VS_SCA
+
+  # No python
+  Onnxruntime-SCA-win32-WINML-x64:
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: false
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.11.x'
+          architecture: 'x64'
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Delete build folder
+        run: |
+          if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
+          &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug
+
+      # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
+      - name: Build code
+        env:
+           CAExcludePath: 'C:\Program Files;D:\b;${{ github.workspace }}\cmake'
+        run:  python tools\ci_build\build.py --enable_training --build_java --compile_no_warning_as_error --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --cmake_extra_defines onnxruntime_USE_CUSTOM_STATIC_ANALYSIS_RULES=ON --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON --cmake_extra_defines onnxruntime_REDIRECT_STATIC_ANALYSIS_OUTPUTS_TO_FILE=ON --ms_experimental --use_dml --use_winml --disable_rtti --enable_wcos --build_shared_lib
+        
+      - name: Generate sarif
+        working-directory: D:\b
+        run: npx @microsoft/sarif-multitool merge *.sarif --recurse --output-directory=${{ github.workspace }}\output --output-file=MergeResult.sarif --merge-runs && dir ${{ github.workspace }}\output
+
+      - name: Upload SARIF to GitHub
+        uses: github/codeql-action/upload-sarif@v2
+        continue-on-error: true
+        with:
+          sarif_file: ${{ github.workspace }}\output\MergeResult.sarif
+          category: VS_SCA_WIN32_WINML_X64
+
+  # No java, No python
+  Onnxruntime-SCA-win32-WINML-x86:
+    runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: false
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.11.x'
+          architecture: 'x86'
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Delete build folder
+        run: |
+          if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
+          &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x86 -install_prefix D:\b\Debug\installed -build_config Debug
+
+      # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
+      - name: Build code
+        env:
+           CAExcludePath: 'C:\Program Files;D:\b;${{ github.workspace }}\cmake'
+        run:  python tools\ci_build\build.py --enable_training --compile_no_warning_as_error --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --cmake_extra_defines onnxruntime_USE_CUSTOM_STATIC_ANALYSIS_RULES=ON --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON --cmake_extra_defines onnxruntime_REDIRECT_STATIC_ANALYSIS_OUTPUTS_TO_FILE=ON --ms_experimental --use_dml --use_winml --disable_rtti --enable_wcos --build_shared_lib
+        
+      - name: Generate sarif
+        working-directory: D:\b
+        run: npx @microsoft/sarif-multitool merge *.sarif --recurse --output-directory=${{ github.workspace }}\output --output-file=MergeResult.sarif --merge-runs && dir ${{ github.workspace }}\output
+
+      - name: Upload SARIF to GitHub
+        uses: github/codeql-action/upload-sarif@v2
+        continue-on-error: true
+        with:
+          sarif_file: ${{ github.workspace }}\output\MergeResult.sarif
+          category: VS_SCA_WIN32_WINML_X86
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@ -2,10 +2,14 @@ name: Windows_CI
 on:
  push:
    branches:
-      - master
      - main
+      - rel-*
  pull_request:

+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
  Onnxruntime-TVM:
    runs-on: windows-latest
--- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
+++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
@ -794,7 +794,7 @@ stages:

 - template: nuget/templates/dml-vs-2022.yml
  parameters:
-    AgentPool : 'aiinfra-dml-winbuild'
+    AgentPool : 'onnxruntime-Win2019-GPU-dml-A10'
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    ArtifactName: 'drop-nuget-dml'
    StageName: 'Windows_CI_GPU_DML_Dev'
@ -817,7 +817,7 @@ stages:

 - template: nuget/templates/dml-vs-2022.yml
  parameters:
-    AgentPool : 'aiinfra-dml-winbuild'
+    AgentPool : 'onnxruntime-Win2019-GPU-dml-A10'
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    ArtifactName: 'drop-win-dml-x86-zip'
    StageName: 'Windows_CI_GPU_DML_Dev_x86'
@ -840,7 +840,7 @@ stages:

 - template: nuget/templates/dml-vs-2022.yml
  parameters:
-    AgentPool : 'aiinfra-dml-winbuild'
+    AgentPool : 'onnxruntime-Win2019-GPU-dml-A10'
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    ArtifactName: 'drop-win-dml-arm64-zip'
    StageName: 'Windows_CI_GPU_DML_Dev_arm64'
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml
@ -48,7 +48,7 @@ stages:
      ALLOW_RELEASED_ONNX_OPSET_ONLY: ${{ parameters.AllowReleasedOpsetOnly }}
      ${{ if eq(parameters.EnableLto, true) }}:
        build_py_lto_flag: --enable_lto
-      ${{ if or(eq(parameters.AgentPool, 'aiinfra-dml-winbuild'), eq(parameters.AgentPool, 'onnxruntime-Win-CPU-2019')) }}:
+      ${{ if or(eq(parameters.AgentPool, 'onnxruntime-Win2019-GPU-dml-A10'), eq(parameters.AgentPool, 'onnxruntime-Win-CPU-2019')) }}:
        vsVersion: '2019'

    steps:
@ -72,7 +72,13 @@ stages:

      - task: NodeTool@0
        inputs:
-          versionSpec: '16.x'
+          versionSpec: '18.x'
+
+      - task: onebranch.pipeline.tsaoptions@1
+        displayName: 'OneBranch TSAOptions'
+        inputs:
+          tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
+          appendSourceBranchName: false

      - task: UsePythonVersion@0
        inputs:
@ -272,9 +278,6 @@ stages:
        - template: ../../templates/compliance.yml
          parameters :
            msbuildPlatform: ${{ parameters.sln_platform }}
-            # parameter couldn't be compared with string, so I have to use boolean.
-            ${{ if eq(variables.vsVersion, '2019') }}:
-              vs2022: false

      - template: ../../templates/component-governance-component-detection-steps.yml
        parameters :
--- a/tools/ci_build/github/azure-pipelines/templates/compliance.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/compliance.yml
@ -4,11 +4,6 @@ parameters:
  type: string
  default: x64

- name: vs2022
-  displayName: Visual Studio Version
-  type: string
-  default: true
-
 steps:
 - task: CredScan@2
  displayName: 'Run CredScan'
@ -23,45 +18,10 @@ steps:
    arguments: 'analyze $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.dll  --recurse --verbose'
  continueOnError: true

- task: DeleteFiles@1
-  displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo'
-  inputs:
-    SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
-    Contents: |
-     **/*.obj
-     **/*.pdb
-     **/*.dll
-
-#Manually set msBuildCommandline so that we can also set CAExcludePath
- task: securedevelopmentteam.vss-secure-development-tools.build-task-prefast.SDLNativeRules@2
-  displayName: 'Run the PREfast SDL Native Rules for MSBuild'
-  inputs:
-    userProvideBuildInfo: msBuildInfo
-    msBuildArchitecture: x64
-    ${{ if eq(parameters.vs2022, false)}}:
-      msBuildVersion: 16.0
-      msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="${{parameters.msbuildPlatform}}" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
-    ${{ else }}:
-      msBuildVersion: 17.0
-      msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="${{parameters.msbuildPlatform}}" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
-  continueOnError: true
-
- task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1
-  displayName: 'Create Security Analysis Report'
-  inputs:
-    BinSkim: true
-    BinSkimBreakOn: WarningAbove
-    CredScan: true
-    SDLNativeRules: true
-
- task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
-  displayName: 'Publish Security Analysis Logs'
-  continueOnError: true
-
- task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1
+- task: TSAUpload@2
+  displayName: 'TSA upload'
  condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
-  displayName: 'TSA Upload'
  inputs:
-    tsaVersion: TsaV2
-    codeBaseName: 'onnxruntime_master'
+    GdnPublishTsaOnboard: false
+    GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
  continueOnError: true
--- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
@ -64,6 +64,7 @@ jobs:
    setVcvars: true
    ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
    DocUpdateNeeded: false  # Set to true during document generation if there are diffs
+    NVIDIA_TF32_OVERRIDE: '0'
    skipComponentGovernanceDetection: true
    DEPS_CACHE_DIR: $(Agent.TempDirectory)/deps_ccache
    ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@ -303,7 +303,7 @@ stages:

      - template: py-win-gpu.yml
        parameters:
-          MACHINE_POOL: 'aiinfra-dml-winbuild'
+          MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
          PYTHON_VERSION: '3.8'
          EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
          ENV_SETUP_SCRIPT: setup_env.bat
@ -311,7 +311,7 @@ stages:

      - template: py-win-gpu.yml
        parameters:
-          MACHINE_POOL: 'aiinfra-dml-winbuild'
+          MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
          PYTHON_VERSION: '3.9'
          EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
          ENV_SETUP_SCRIPT: setup_env.bat
@ -319,7 +319,7 @@ stages:

      - template: py-win-gpu.yml
        parameters:
-          MACHINE_POOL: 'aiinfra-dml-winbuild'
+          MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
          PYTHON_VERSION: '3.10'
          EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
          ENV_SETUP_SCRIPT: setup_env.bat
@ -327,7 +327,7 @@ stages:

      - template: py-win-gpu.yml
        parameters:
-          MACHINE_POOL: 'aiinfra-dml-winbuild'
+          MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
          PYTHON_VERSION: '3.11'
          EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
          ENV_SETUP_SCRIPT: setup_env.bat
--- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
@ -28,7 +28,7 @@ jobs:
    clean: all
  pool: ${{ parameters.MACHINE_POOL }}
  variables:
-    ${{if or(eq(parameters.MACHINE_POOL, 'onnxruntime-gpu-winbuild-T4'), eq(parameters.MACHINE_POOL, 'aiinfra-dml-winbuild'))}}:
+    ${{if or(eq(parameters.MACHINE_POOL, 'onnxruntime-gpu-winbuild-T4'), eq(parameters.MACHINE_POOL, 'onnxruntime-Win2019-GPU-dml-A10'))}}:
      VSGenerator: 'Visual Studio 16 2019'
    ${{else}}:
      VSGenerator: 'Visual Studio 17 2022'
--- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
@ -81,7 +81,7 @@ stages:
        msbuildPlatform: x64
        isX86: false
        job_name_suffix: x64_release
-        RunOnnxRuntimeTests: false
+        RunOnnxRuntimeTests: true
        RunStaticCodeAnalysis: false
        isTraining: false
        ORT_EP_NAME: XNNPACK
--- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
@ -36,7 +36,7 @@ stages:
        BuildConfig: 'RelWithDebInfo'
        EnvSetupScript: setup_env_cuda_11.bat
        buildArch: x64
-        additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75
+        additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
        msbuildPlatform: x64
        isX86: false
        job_name_suffix: x64_RelWithDebInfo
@ -44,7 +44,7 @@ stages:
        RunStaticCodeAnalysis: false
        ORT_EP_NAME: CUDA
        WITH_CACHE: true
-        MachinePool: onnxruntime-Win2022-GPU-T4
+        MachinePool: onnxruntime-Win2022-GPU-A10

 - stage: training
  dependsOn: []
@ -62,6 +62,7 @@ stages:
        RunStaticCodeAnalysis: false
        ORT_EP_NAME: CUDA
        WITH_CACHE: true
+        # Some unit tests crash on A10 GPUs. So this job still needs to use A10.
        MachinePool: onnxruntime-Win2022-GPU-T4
        isTraining: true

@ -81,7 +82,6 @@ stages:
        RunStaticCodeAnalysis: false
        ORT_EP_NAME: DML
        WITH_CACHE: true
-        # DirectML cannot run on T4 GPUs.
        MachinePool: onnxruntime-Win2019-GPU-dml-A10

 - stage: kernelDocumentation
@ -93,7 +93,7 @@ stages:
        EnvSetupScript: setup_env_cuda_11.bat
        buildArch: x64
        # note: need to specify `--gen_doc` when creating the build config so it has to be in additionalBuildFlags
-        additionalBuildFlags: --gen_doc validate --skip_tests --enable_pybind --use_dml --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF
+        additionalBuildFlags: --gen_doc validate --skip_tests --enable_pybind --use_dml --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF
        msbuildPlatform: x64
        isX86: false
        job_name_suffix: x64_RelWithDebInfo
@ -102,4 +102,4 @@ stages:
        GenerateDocumentation: true
        ORT_EP_NAME: CUDA # It doesn't really matter which EP is selected here since this stage is for documentation.
        WITH_CACHE: true
-        MachinePool: onnxruntime-Win2022-GPU-T4
+        MachinePool: onnxruntime-Win2022-GPU-A10
--- a/tools/ci_build/github/windows/setup_env_gpu.bat
+++ b/tools/ci_build/github/windows/setup_env_gpu.bat
@ -1,6 +1,11 @@
 REM Copyright (c) Microsoft Corporation. All rights reserved.
 REM Licensed under the MIT License.

-set PATH=C:\local\TensorRT-8.6.0.12.Windows10.x86_64.cuda-11.8\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\extras\CUPTI\lib64;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin;%PATH%
+if exist PATH=%AGENT_TEMPDIRECTORY%\v11.8\ {
+    set PATH=%AGENT_TEMPDIRECTORY%\v11.8\bin;%AGENT_TEMPDIRECTORY%\v11.8\extras\CUPTI\lib64;%PATH%
+} else {
+    set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\extras\CUPTI\lib64;%PATH%
+}
+set PATH=C:\local\TensorRT-8.6.0.12.Windows10.x86_64.cuda-11.8\lib;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin;%PATH%
 set GRADLE_OPTS=-Dorg.gradle.daemon=false
 set CUDA_MODULE_LOADING=LAZY