Common pre-build steps of Windows CI (#16970)

### Description Unify some pre-build common steps. ### Motivation and Context In the long run, other devs should only focus on build option and test commands. It would reduce mistakes and maintenance cost to use common template steps. There will be more PRs to achieve the goal.
2026-07-30 20:18:08 +00:00 · 2023-08-22 18:09:55 +08:00 · 2023-08-22 18:09:55 +08:00 · 61a79436e2
commit 61a79436e2
parent d5c565156d
5 changed files with 131 additions and 141 deletions
--- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml
@ -0,0 +1,94 @@
+parameters:
+- name: EnvSetupScript
+  type: string
+
+- name: BuildConfig
+  type: string
+
+- name: BuildArch
+  type: string
+
+- name: DownloadCUDA
+  type: boolean
+  default: false
+
+- name: WITHCACHE
+  type: boolean
+  default: false
+
+- name: MachinePool
+  type: string
+
+- name: Today
+  type: string
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '3.8'
+    addToPath: true
+    architecture: ${{parameters.BuildArch}}
+
+- script: |
+    python -m pip install -q setuptools wheel numpy flatbuffers
+  workingDirectory: '$(Build.BinariesDirectory)'
+  displayName: 'Install python modules'
+
+- template: ../download-deps.yml
+
+- task: PythonScript@0
+  displayName: 'Update deps.txt'
+  inputs:
+    scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
+    arguments: --new_dir $(Build.BinariesDirectory)/deps
+    workingDirectory: $(Build.BinariesDirectory)
+
+- template: set-winenv.yml
+  parameters:
+    EnvSetupScript: ${{parameters.EnvSetupScript}}
+    DownloadCUDA: ${{parameters.DownloadCUDA}}
+
+- ${{ if eq(parameters.WITHCACHE, true) }}:
+  - powershell: |
+      if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue)))
+      {
+        choco install ccache -y --version 4.7.4
+        $ccache_path = (Get-Command ccache).Source
+        $ccache_parent_dir = (Split-Path -parent $ccache_path)
+        Copy-Item "C:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.7.4-windows-x86_64\ccache.exe" -Destination "C:\ProgramData\chocolatey\bin\cl.exe"
+        Get-ChildItem $ccache_parent_dir
+        ccache --version
+      }
+    displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
+
+  - ${{ if eq(parameters.WITHCACHE, true) }}:
+    - task: Cache@2
+      # machinepool is used to ensure the compiler is same
+      inputs:
+        key: '"$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1, $(Build.SourcesDirectory)/tools/ci_build/github/windows/helpers.ps1'
+        path: $(Agent.TempDirectory)/deps_ccache
+        restoreKeys: |
+          "$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }}
+      displayName: Cache Task
+
+  - task: PowerShell@2
+    displayName: 'Install ONNX'
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
+      workingDirectory: '$(Build.BinariesDirectory)'
+      ${{ if eq(parameters.WITHCACHE, true) }}:
+        arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} -use_cache
+      ${{ else }}:
+        arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }}
+    ${{ if eq(parameters.WITHCACHE, true) }}:
+      env:
+        CCACHE_DIR: $(Agent.TempDirectory)/deps_ccache
+        CCACHE_COMPILERCHECK: content
+
+  - ${{ if eq(parameters.WITHCACHE, true) }}:
+    - powershell: |
+        ccache -sv
+        ccache -z
+      displayName: cache stat
+      env:
+        CCACHE_DIR: $(Agent.TempDirectory)/deps_ccache
--- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
@ -88,27 +88,24 @@ jobs:
    clean: true
    submodules: none

-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.8'
-      addToPath: true
-      architecture: ${{ parameters.buildArch }}
-
-  - template: ../download-deps.yml
-
-  - task: PythonScript@0
-    displayName: 'Update deps.txt'
-    inputs:
-      scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
-      arguments: --new_dir $(Build.BinariesDirectory)/deps
-      workingDirectory: $(Build.BinariesDirectory)
+  - template: win-ci-prebuild-steps.yml
+    parameters:
+      EnvSetupScript: ${{parameters.EnvSetupScript}}
+      ${{ if contains(parameters.additionalBuildFlags, 'use_cuda') }}:
+        DownloadCUDA: true
+      ${{ else }}:
+        DownloadCUDA: false
+      BuildArch: ${{parameters.buildArch}}
+      BuildConfig: ${{parameters.BuildConfig}}
+      MachinePool: ${{parameters.MachinePool}}
+      WithCache: ${{parameters.WITH_CACHE}}
+      Today: $(Today)

  - task: NodeTool@0
    inputs:
      versionSpec: '16.x'
      force32bit: ${{ parameters.isX86 }}

-
  # Our build machine doesn't have java x86
  - ${{ if eq(parameters.buildArch, 'x64') }}:
      - task: JavaToolInstaller@0
@ -117,12 +114,6 @@ jobs:
          jdkArchitectureOption: ${{ parameters.buildArch }}
          jdkSourceOption: 'PreInstalled'

-  - template: set-winenv.yml
-    parameters:
-      EnvSetupScript: ${{ parameters.EnvSetupScript }}
-      ${{ if contains(parameters.additionalBuildFlags, 'use_cuda') }}:
-        DownloadCUDA: true
-
  - script: |
     set ORT_DOXY_SRC=$(Build.SourcesDirectory)
     set ORT_DOXY_OUT=$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}
@ -132,54 +123,6 @@ jobs:
    workingDirectory: '$(Build.SourcesDirectory)'
    displayName: 'API Documentation Check and generate'

-  - script: |
-     python -m pip install -q setuptools wheel numpy
-    workingDirectory: '$(Build.BinariesDirectory)'
-    displayName: 'Install python modules'
-
-  - ${{ if eq(parameters.WITH_CACHE, true) }}:
-    - powershell: |
-        if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue)))
-        {
-          choco install ccache -y --version 4.7.4
-          $ccache_path = (Get-Command ccache).Source
-          $ccache_parent_dir = (Split-Path -parent $ccache_path)
-          Copy-Item "C:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.7.4-windows-x86_64\ccache.exe" -Destination "C:\ProgramData\chocolatey\bin\cl.exe"
-          Get-ChildItem $ccache_parent_dir
-          ccache --version
-        }
-      displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
-
-  - ${{ if or(eq(parameters.RunOnnxRuntimeTests, true), eq(parameters.GenerateDocumentation, true)) }}:
-      - ${{ if eq(parameters.WITH_CACHE, true) }}:
-        - task: Cache@2
-          # machinepool is used to ensure the compiler is same
-          inputs:
-            key: '"$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1, $(Build.SourcesDirectory)/tools/ci_build/github/windows/helpers.ps1'
-            path: $(DEPS_CACHE_DIR)
-            restoreKeys: |
-              "$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }}
-          displayName: Cache Task
-
-      - task: PowerShell@2
-        displayName: 'Install ONNX'
-        inputs:
-          filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
-          workingDirectory: '$(Build.BinariesDirectory)'
-          arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} ${{ variables['PS_CACHE_ARG'] }}
-        ${{ if eq(parameters.WITH_CACHE, true) }}:
-          env:
-            CCACHE_DIR: $(DEPS_CACHE_DIR)
-            CCACHE_COMPILERCHECK: content
-
-      - ${{ if eq(parameters.WITH_CACHE, true) }}:
-        - powershell: |
-            ccache -sv
-            ccache -z
-          displayName: cache stat
-          env:
-            CCACHE_DIR: $(DEPS_CACHE_DIR)
-
  - task: NuGetToolInstaller@0
    displayName: Use Nuget 5.7.0
    inputs:
--- a/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml
+++ b/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml
@ -15,39 +15,25 @@ jobs:
    buildArch: x64
    setVcvars: true
    ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
  timeoutInMinutes: 120
  workspace:
    clean: all
-  steps:    
-  - task: UsePythonVersion@0
-    inputs: 
-      versionSpec: '3.8' 
-      addToPath: true 
-      architecture: $(buildArch)
+  steps:
+  - template: win-ci-prebuild-steps.yml
+    parameters:
+      EnvSetupScript: $(EnvSetupScript)
+      DownloadCUDA: false
+      BuildArch: $(buildArch)
+      BuildConfig: $(BuildConfig)
+      MachinePool: 'onnxruntime-Win-CPU-2022'
+      WithCache: true
+      Today: $(Today)

  - task: NodeTool@0
    inputs:
      versionSpec: '16.x'

-  - task: BatchScript@1
-    displayName: 'setup env'
-    inputs:
-      filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
-      modifyEnvironment: true
-      workingFolder: '$(Build.BinariesDirectory)'
-
-  - script: |
-     python -m pip install -q setuptools wheel numpy
-    workingDirectory: '$(Build.BinariesDirectory)'
-    displayName: 'Install python modules'
-
-  - task: PowerShell@2
-    displayName: 'Install ONNX'
-    inputs:
-      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
-      workingDirectory: '$(Build.BinariesDirectory)'
-      arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig)
-
  - task: NuGetToolInstaller@0
    displayName: Use Nuget 5.7.0
    inputs:
--- a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
@ -16,32 +16,20 @@ jobs:
    EnvSetupScript: setup_env_cuda_11.bat
    buildArch: x64
    setVcvars: true
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
  timeoutInMinutes: 120
  workspace:
    clean: all
  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.8'
-      addToPath: true
-      architecture: $(buildArch)
-
-  - script: |
-     python -m pip install -q setuptools wheel numpy flatbuffers
-    workingDirectory: '$(Build.BinariesDirectory)'
-    displayName: 'Install python modules'
-
-  - template: templates/jobs/set-winenv.yml
+  - template: templates/jobs/win-ci-prebuild-steps.yml
    parameters:
      EnvSetupScript: $(EnvSetupScript)
      DownloadCUDA: true
-
-  - task: PowerShell@2
-    displayName: 'Install ONNX'
-    inputs:
-      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
-      workingDirectory: '$(Build.BinariesDirectory)'
-      arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig)
+      BuildArch: $(buildArch)
+      BuildConfig: $(BuildConfig)
+      MachinePool: 'onnxruntime-Win2022-GPU-T4'
+      WithCache: true
+      Today: $(Today)

  - task: PythonScript@0
    displayName: 'Build and test'
--- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
@ -32,41 +32,20 @@ jobs:
    BuildConfig: 'RelWithDebInfo'
    ALLOW_RELEASED_ONNX_OPSET_ONLY: '1'
    skipComponentGovernanceDetection: true
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
  timeoutInMinutes: 150
  workspace:
    clean: all
  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.8'
-      addToPath: true
-      architecture: $(buildArch)
-
-  - script: |
-     python -m pip install -q setuptools wheel numpy
-    workingDirectory: '$(Build.BinariesDirectory)'
-    displayName: 'Install python modules'
-
-  - template: templates/download-deps.yml
-
-  - task: PythonScript@0
-    displayName: 'Update deps.txt'
-    inputs:
-      scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
-      arguments: --new_dir $(Build.BinariesDirectory)/deps
-      workingDirectory: $(Build.BinariesDirectory)
-
-  - template: templates/jobs/set-winenv.yml
+  - template: templates/jobs/win-ci-prebuild-steps.yml
    parameters:
      EnvSetupScript: $(EnvSetupScript)
      DownloadCUDA: true
-
-  - task: PowerShell@2
-    displayName: 'Install ONNX'
-    inputs:
-      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
-      workingDirectory: '$(Build.BinariesDirectory)'
-      arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig)
+      BuildArch: $(buildArch)
+      BuildConfig: $(BuildConfig)
+      MachinePool: 'onnxruntime-Win2022-GPU-T4'
+      WithCache: true
+      Today: $(Today)

  - task: PythonScript@0
    displayName: 'Generate cmake config'