From 61a79436e22892bdd91a905389f12e0aee68132e Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Tue, 22 Aug 2023 18:09:55 +0800 Subject: [PATCH] Common pre-build steps of Windows CI (#16970) ### Description Unify some pre-build common steps. ### Motivation and Context In the long run, other devs should only focus on build option and test commands. It would reduce mistakes and maintenance cost to use common template steps. There will be more PRs to achieve the goal. --- .../templates/jobs/win-ci-prebuild-steps.yml | 94 +++++++++++++++++++ .../templates/jobs/win-ci-vs-2022-job.yml | 81 +++------------- .../azure-pipelines/win-ci-fuzz-testing.yml | 36 +++---- .../win-gpu-reduce-op-ci-pipeline.yml | 26 ++--- .../win-gpu-tensorrt-ci-pipeline.yml | 35 ++----- 5 files changed, 131 insertions(+), 141 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml new file mode 100644 index 0000000000..e29d9d2cab --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml @@ -0,0 +1,94 @@ +parameters: +- name: EnvSetupScript + type: string + +- name: BuildConfig + type: string + +- name: BuildArch + type: string + +- name: DownloadCUDA + type: boolean + default: false + +- name: WITHCACHE + type: boolean + default: false + +- name: MachinePool + type: string + +- name: Today + type: string + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '3.8' + addToPath: true + architecture: ${{parameters.BuildArch}} + +- script: | + python -m pip install -q setuptools wheel numpy flatbuffers + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + +- template: ../download-deps.yml + +- task: PythonScript@0 + displayName: 'Update deps.txt' + inputs: + scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py + arguments: --new_dir $(Build.BinariesDirectory)/deps + workingDirectory: $(Build.BinariesDirectory) + +- template: set-winenv.yml + parameters: + EnvSetupScript: ${{parameters.EnvSetupScript}} + DownloadCUDA: ${{parameters.DownloadCUDA}} + +- ${{ if eq(parameters.WITHCACHE, true) }}: + - powershell: | + if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue))) + { + choco install ccache -y --version 4.7.4 + $ccache_path = (Get-Command ccache).Source + $ccache_parent_dir = (Split-Path -parent $ccache_path) + Copy-Item "C:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.7.4-windows-x86_64\ccache.exe" -Destination "C:\ProgramData\chocolatey\bin\cl.exe" + Get-ChildItem $ccache_parent_dir + ccache --version + } + displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc + + - ${{ if eq(parameters.WITHCACHE, true) }}: + - task: Cache@2 + # machinepool is used to ensure the compiler is same + inputs: + key: '"$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1, $(Build.SourcesDirectory)/tools/ci_build/github/windows/helpers.ps1' + path: $(Agent.TempDirectory)/deps_ccache + restoreKeys: | + "$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} + displayName: Cache Task + + - task: PowerShell@2 + displayName: 'Install ONNX' + inputs: + filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1' + workingDirectory: '$(Build.BinariesDirectory)' + ${{ if eq(parameters.WITHCACHE, true) }}: + arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} -use_cache + ${{ else }}: + arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} + ${{ if eq(parameters.WITHCACHE, true) }}: + env: + CCACHE_DIR: $(Agent.TempDirectory)/deps_ccache + CCACHE_COMPILERCHECK: content + + - ${{ if eq(parameters.WITHCACHE, true) }}: + - powershell: | + ccache -sv + ccache -z + displayName: cache stat + env: + CCACHE_DIR: $(Agent.TempDirectory)/deps_ccache diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml index 1cd21ea199..e49c522b61 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml @@ -88,27 +88,24 @@ jobs: clean: true submodules: none - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - addToPath: true - architecture: ${{ parameters.buildArch }} - - - template: ../download-deps.yml - - - task: PythonScript@0 - displayName: 'Update deps.txt' - inputs: - scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py - arguments: --new_dir $(Build.BinariesDirectory)/deps - workingDirectory: $(Build.BinariesDirectory) + - template: win-ci-prebuild-steps.yml + parameters: + EnvSetupScript: ${{parameters.EnvSetupScript}} + ${{ if contains(parameters.additionalBuildFlags, 'use_cuda') }}: + DownloadCUDA: true + ${{ else }}: + DownloadCUDA: false + BuildArch: ${{parameters.buildArch}} + BuildConfig: ${{parameters.BuildConfig}} + MachinePool: ${{parameters.MachinePool}} + WithCache: ${{parameters.WITH_CACHE}} + Today: $(Today) - task: NodeTool@0 inputs: versionSpec: '16.x' force32bit: ${{ parameters.isX86 }} - # Our build machine doesn't have java x86 - ${{ if eq(parameters.buildArch, 'x64') }}: - task: JavaToolInstaller@0 @@ -117,12 +114,6 @@ jobs: jdkArchitectureOption: ${{ parameters.buildArch }} jdkSourceOption: 'PreInstalled' - - template: set-winenv.yml - parameters: - EnvSetupScript: ${{ parameters.EnvSetupScript }} - ${{ if contains(parameters.additionalBuildFlags, 'use_cuda') }}: - DownloadCUDA: true - - script: | set ORT_DOXY_SRC=$(Build.SourcesDirectory) set ORT_DOXY_OUT=$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }} @@ -132,54 +123,6 @@ jobs: workingDirectory: '$(Build.SourcesDirectory)' displayName: 'API Documentation Check and generate' - - script: | - python -m pip install -q setuptools wheel numpy - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - - ${{ if eq(parameters.WITH_CACHE, true) }}: - - powershell: | - if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue))) - { - choco install ccache -y --version 4.7.4 - $ccache_path = (Get-Command ccache).Source - $ccache_parent_dir = (Split-Path -parent $ccache_path) - Copy-Item "C:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.7.4-windows-x86_64\ccache.exe" -Destination "C:\ProgramData\chocolatey\bin\cl.exe" - Get-ChildItem $ccache_parent_dir - ccache --version - } - displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc - - - ${{ if or(eq(parameters.RunOnnxRuntimeTests, true), eq(parameters.GenerateDocumentation, true)) }}: - - ${{ if eq(parameters.WITH_CACHE, true) }}: - - task: Cache@2 - # machinepool is used to ensure the compiler is same - inputs: - key: '"$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1, $(Build.SourcesDirectory)/tools/ci_build/github/windows/helpers.ps1' - path: $(DEPS_CACHE_DIR) - restoreKeys: | - "$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} - displayName: Cache Task - - - task: PowerShell@2 - displayName: 'Install ONNX' - inputs: - filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1' - workingDirectory: '$(Build.BinariesDirectory)' - arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} ${{ variables['PS_CACHE_ARG'] }} - ${{ if eq(parameters.WITH_CACHE, true) }}: - env: - CCACHE_DIR: $(DEPS_CACHE_DIR) - CCACHE_COMPILERCHECK: content - - - ${{ if eq(parameters.WITH_CACHE, true) }}: - - powershell: | - ccache -sv - ccache -z - displayName: cache stat - env: - CCACHE_DIR: $(DEPS_CACHE_DIR) - - task: NuGetToolInstaller@0 displayName: Use Nuget 5.7.0 inputs: diff --git a/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml b/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml index 0eb78412de..f3a5728d65 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml @@ -15,39 +15,25 @@ jobs: buildArch: x64 setVcvars: true ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' + TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] timeoutInMinutes: 120 workspace: clean: all - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - addToPath: true - architecture: $(buildArch) + steps: + - template: win-ci-prebuild-steps.yml + parameters: + EnvSetupScript: $(EnvSetupScript) + DownloadCUDA: false + BuildArch: $(buildArch) + BuildConfig: $(BuildConfig) + MachinePool: 'onnxruntime-Win-CPU-2022' + WithCache: true + Today: $(Today) - task: NodeTool@0 inputs: versionSpec: '16.x' - - task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' - - - script: | - python -m pip install -q setuptools wheel numpy - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - - task: PowerShell@2 - displayName: 'Install ONNX' - inputs: - filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1' - workingDirectory: '$(Build.BinariesDirectory)' - arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig) - - task: NuGetToolInstaller@0 displayName: Use Nuget 5.7.0 inputs: diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml index 3ac98f4d87..aab8e5ca21 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml @@ -16,32 +16,20 @@ jobs: EnvSetupScript: setup_env_cuda_11.bat buildArch: x64 setVcvars: true + TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] timeoutInMinutes: 120 workspace: clean: all steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - addToPath: true - architecture: $(buildArch) - - - script: | - python -m pip install -q setuptools wheel numpy flatbuffers - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - - template: templates/jobs/set-winenv.yml + - template: templates/jobs/win-ci-prebuild-steps.yml parameters: EnvSetupScript: $(EnvSetupScript) DownloadCUDA: true - - - task: PowerShell@2 - displayName: 'Install ONNX' - inputs: - filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1' - workingDirectory: '$(Build.BinariesDirectory)' - arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig) + BuildArch: $(buildArch) + BuildConfig: $(BuildConfig) + MachinePool: 'onnxruntime-Win2022-GPU-T4' + WithCache: true + Today: $(Today) - task: PythonScript@0 displayName: 'Build and test' diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index 973fdb9a2a..3008cc2d86 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -32,41 +32,20 @@ jobs: BuildConfig: 'RelWithDebInfo' ALLOW_RELEASED_ONNX_OPSET_ONLY: '1' skipComponentGovernanceDetection: true + TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] timeoutInMinutes: 150 workspace: clean: all steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - addToPath: true - architecture: $(buildArch) - - - script: | - python -m pip install -q setuptools wheel numpy - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - - template: templates/download-deps.yml - - - task: PythonScript@0 - displayName: 'Update deps.txt' - inputs: - scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py - arguments: --new_dir $(Build.BinariesDirectory)/deps - workingDirectory: $(Build.BinariesDirectory) - - - template: templates/jobs/set-winenv.yml + - template: templates/jobs/win-ci-prebuild-steps.yml parameters: EnvSetupScript: $(EnvSetupScript) DownloadCUDA: true - - - task: PowerShell@2 - displayName: 'Install ONNX' - inputs: - filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1' - workingDirectory: '$(Build.BinariesDirectory)' - arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig) + BuildArch: $(buildArch) + BuildConfig: $(BuildConfig) + MachinePool: 'onnxruntime-Win2022-GPU-T4' + WithCache: true + Today: $(Today) - task: PythonScript@0 displayName: 'Generate cmake config'