From dc545d366dc2d139b8585c2acb2c8dedf519b0b6 Mon Sep 17 00:00:00 2001
From: Changming Sun <chasun@microsoft.com>
Date: Mon, 10 Jun 2024 12:33:49 -0700
Subject: [PATCH] Publish debug symbols for Windows python packages (#20973)

### Description
1. Publish debug symbols for Windows python packages. This PR will
publish them to ADO. Later on I will also replicate them to Microsoft
Symbol Server.
2. Build the packages in Release mode instead of RelWithDebInfo, to be
consistent with the other platforms(Linux/macOS/...)


### Motivation and Context
To help debug things. Sometimes we found an issue, but we couldn't debug
it because we didn't have symbols, and once we rebuilt the package
locally the issue was gone. This change would be helpful for such
scenarios.

Build log:
https://aiinfra.visualstudio.com/Lotus/_build?definitionId=841
---
 .../azure-pipelines/py-packaging-pipeline.yml |  1 +
 .../templates/py-packaging-stage.yml          | 62 ++++++++++++++-----
 .../azure-pipelines/templates/py-win-gpu.yml  | 40 ++++++++++--
 3 files changed, 83 insertions(+), 20 deletions(-)

diff --git a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml
index c1fde9eff6..df4ed796f2 100644
--- a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml
@@ -77,3 +77,4 @@ stages:
     build_py_parameters: ${{ parameters.build_py_parameters }}
     cmake_build_type: ${{ parameters.cmake_build_type }}
     qnn_sdk_version: ${{ parameters.qnn_sdk_version }}
+    publish_symbols: true
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
index f2bd0e6f16..0ac274566f 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@@ -45,7 +45,6 @@ parameters:
   type: boolean
   default: true
 
-# TODO: Now the Windows jobs use a different cmake build type. Consider to merge it.
 - name: cmake_build_type
   type: string
   displayName: 'Linux packages cmake build type. Linux Only.'
@@ -56,6 +55,10 @@ parameters:
    - RelWithDebInfo
    - MinSizeRel
 
+- name: publish_symbols
+  type: boolean
+  default: false
+
 # Only applies to QNN packages.
 - name: qnn_sdk_version
   type: string
@@ -117,8 +120,6 @@ stages:
       variables:
         OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
         EnvSetupScript: setup_env.bat
-        setVcvars: true
-        BuildConfig: 'RelWithDebInfo'
         ExtraParam: ${{ parameters.build_py_parameters }}
       timeoutInMinutes: 180
       workspace:
@@ -177,14 +178,14 @@ stages:
         inputs:
           filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
           workingDirectory: '$(Build.BinariesDirectory)'
-          arguments: -cpu_arch $(buildArch) -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig)
+          arguments: -cpu_arch $(buildArch) -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }}
 
       - task: PythonScript@0
         displayName: 'Generate cmake config'
         inputs:
           scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
           arguments: >
-            --config $(BuildConfig)
+            --config ${{ parameters.cmake_build_type }}
             --enable_lto
             --build_dir $(Build.BinariesDirectory)
             --skip_submodule_sync
@@ -199,19 +200,19 @@ stages:
       - task: VSBuild@1
         displayName: 'Build'
         inputs:
-          solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln'
+          solution: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\onnxruntime.sln'
           platform: $(MsbuildPlatform)
-          configuration: $(BuildConfig)
+          configuration: ${{ parameters.cmake_build_type }}
           msbuildArchitecture: $(buildArch)
           maximumCpuCount: true
           logProjectEvents: true
-          workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
+          workingFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}'
           createLogFile: true
 
       # Esrp signing
       - template: win-esrp-dll.yml
         parameters:
-          FolderPath: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)\onnxruntime\capi'
+          FolderPath: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\onnxruntime\capi'
           DisplayName: 'ESRP - Sign Native dlls'
           DoEsrp: true
           Pattern: '*.pyd,*.dll'
@@ -221,12 +222,12 @@ stages:
         inputs:
           scriptPath: '$(Build.SourcesDirectory)\setup.py'
           arguments: 'bdist_wheel ${{ parameters.build_py_parameters }} $(NightlyBuildOption)'
-          workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
+          workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
 
       - task: CopyFiles@2
         displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
         inputs:
-          SourceFolder: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)\dist'
+          SourceFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\dist'
           Contents: '*.whl'
           TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
@@ -257,13 +258,26 @@ stages:
          Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate}
          Remove-Item -Recurse -Force onnxruntime
          if ("$(ExtraParam)" -contains "--use_azure") {
-           $env:path="$(Build.BinariesDirectory)\$(BuildConfig)\_deps\vcpkg-src\installed\x64-windows\bin;$(Build.BinariesDirectory)\$(BuildConfig)\_deps\vcpkg-src\installed\x86-windows\bin;$env:path"
+           $env:path="$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\_deps\vcpkg-src\installed\x64-windows\bin;$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\_deps\vcpkg-src\installed\x86-windows\bin;$env:path"
            python onnxruntime_test_python_azure.py
          }
          python onnx_backend_test_series.py
-        workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
+        workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
         displayName: 'Run Python Tests'
-
+      - ${{ if eq(parameters.publish_symbols, true) }}:
+        - task: PublishSymbols@2
+          displayName: 'Publish symbols'
+          condition: and (succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rel-')))
+          inputs:
+            SymbolsFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
+            SearchPattern: |
+               onnxruntime_pybind11_state.pdb
+               onnxruntime_providers_shared.pdb
+            IndexSources: true
+            SymbolServerType: TeamServices
+            SymbolExpirationInDays: 3650
+            SymbolsArtifactName: 'win_cpu_$(PythonVersion)_$(buildArch)_$(Build.BuildNumber)'
+ 
       - task: TSAUpload@2
         displayName: 'TSA upload'
         condition: and(and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.8'))), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
@@ -288,6 +302,8 @@ stages:
         EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
         ENV_SETUP_SCRIPT: setup_env_gpu.bat
         EP_NAME: gpu
+        publish_symbols: ${{ parameters.publish_symbols }}
+        cmake_build_type: ${{ parameters.cmake_build_type }}
 
     - template: py-win-gpu.yml
       parameters:
@@ -296,6 +312,8 @@ stages:
         EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
         ENV_SETUP_SCRIPT: setup_env_gpu.bat
         EP_NAME: gpu
+        publish_symbols: ${{ parameters.publish_symbols }}
+        cmake_build_type: ${{ parameters.cmake_build_type }}
 
     - template: py-win-gpu.yml
       parameters:
@@ -304,6 +322,8 @@ stages:
         EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
         ENV_SETUP_SCRIPT: setup_env_gpu.bat
         EP_NAME: gpu
+        publish_symbols: ${{ parameters.publish_symbols }}
+        cmake_build_type: ${{ parameters.cmake_build_type }}
 
     - template: py-win-gpu.yml
       parameters:
@@ -312,6 +332,8 @@ stages:
         EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
         ENV_SETUP_SCRIPT: setup_env_gpu.bat
         EP_NAME: gpu
+        publish_symbols: ${{ parameters.publish_symbols }}
+        cmake_build_type: ${{ parameters.cmake_build_type }}
 
     - template: py-win-gpu.yml
       parameters:
@@ -320,6 +342,8 @@ stages:
         EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
         ENV_SETUP_SCRIPT: setup_env_gpu.bat
         EP_NAME: gpu
+        publish_symbols: ${{ parameters.publish_symbols }}
+        cmake_build_type: ${{ parameters.cmake_build_type }}
 
     - template: py-win-gpu.yml
       parameters:
@@ -328,6 +352,8 @@ stages:
         EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
         ENV_SETUP_SCRIPT: setup_env.bat
         EP_NAME: directml
+        publish_symbols: ${{ parameters.publish_symbols }}
+        cmake_build_type: ${{ parameters.cmake_build_type }}
 
     - template: py-win-gpu.yml
       parameters:
@@ -336,6 +362,8 @@ stages:
         EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
         ENV_SETUP_SCRIPT: setup_env.bat
         EP_NAME: directml
+        publish_symbols: ${{ parameters.publish_symbols }}
+        cmake_build_type: ${{ parameters.cmake_build_type }}
 
     - template: py-win-gpu.yml
       parameters:
@@ -344,6 +372,8 @@ stages:
         EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
         ENV_SETUP_SCRIPT: setup_env.bat
         EP_NAME: directml
+        publish_symbols: ${{ parameters.publish_symbols }}
+        cmake_build_type: ${{ parameters.cmake_build_type }}
 
     - template: py-win-gpu.yml
       parameters:
@@ -352,6 +382,8 @@ stages:
         EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
         ENV_SETUP_SCRIPT: setup_env.bat
         EP_NAME: directml
+        publish_symbols: ${{ parameters.publish_symbols }}
+        cmake_build_type: ${{ parameters.cmake_build_type }}
 
     - template: py-win-gpu.yml
       parameters:
@@ -360,6 +392,8 @@ stages:
         EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
         ENV_SETUP_SCRIPT: setup_env.bat
         EP_NAME: directml
+        publish_symbols: ${{ parameters.publish_symbols }}
+        cmake_build_type: ${{ parameters.cmake_build_type }}
 
 - ${{ if eq(parameters.enable_mac_cpu, true) }}:
   - stage: Python_Packaging_MacOS
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
index e5e72aacd0..0a36d620a0 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
@@ -38,6 +38,20 @@ parameters:
   type: string
   default: '0'
 
+- name: cmake_build_type
+  type: string
+  displayName: 'Linux packages cmake build type. Linux Only.'
+  default: 'Release'
+  values:
+   - Debug
+   - Release
+   - RelWithDebInfo
+   - MinSizeRel
+
+- name: publish_symbols
+  type: boolean
+  default: false
+
 stages:
   - stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
     dependsOn: []
@@ -121,7 +135,7 @@ stages:
             inputs:
               filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
               workingDirectory: '$(Build.BinariesDirectory)'
-              arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\RelWithDebInfo\installed -build_config RelWithDebInfo
+              arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }}
 
           - template: set-nightly-build-option-variable-step.yml
 
@@ -130,7 +144,7 @@ stages:
             inputs:
               scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
               arguments: >
-                --config RelWithDebInfo
+                --config ${{ parameters.cmake_build_type }}
                 --build_dir $(Build.BinariesDirectory)
                 --skip_submodule_sync
                 --cmake_generator "$(VSGenerator)"
@@ -146,7 +160,7 @@ stages:
             inputs:
               scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
               arguments: >
-                --config RelWithDebInfo
+                --config ${{ parameters.cmake_build_type }}
                 --build_dir $(Build.BinariesDirectory)
                 --parallel --build
                 $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
@@ -155,7 +169,7 @@ stages:
           # Esrp signing
           - template: win-esrp-dll.yml
             parameters:
-              FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi'
+              FolderPath: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\onnxruntime\capi'
               DisplayName: 'ESRP - Sign Native dlls'
               DoEsrp: true
               Pattern: '*.pyd,*.dll'
@@ -165,12 +179,12 @@ stages:
             inputs:
               scriptPath: '$(Build.SourcesDirectory)\setup.py'
               arguments: 'bdist_wheel ${{ parameters.BUILD_PY_PARAMETERS }} $(NightlyBuildOption) --wheel_name_suffix=${{ parameters.EP_NAME }}'
-              workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
+              workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
 
           - task: CopyFiles@2
             displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
             inputs:
-              SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
+              SourceFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\dist'
               Contents: '*.whl'
               TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
@@ -179,6 +193,20 @@ stages:
             inputs:
               ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
 
+          - ${{ if eq(parameters.publish_symbols, true) }}:
+            - task: PublishSymbols@2
+              displayName: 'Publish symbols'
+              condition: and (succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rel-')))
+              inputs:
+                SymbolsFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
+                SearchPattern: |
+                   onnxruntime_pybind11_state.pdb
+                   onnxruntime_providers_shared.pdb
+                IndexSources: true
+                SymbolServerType: TeamServices
+                SymbolExpirationInDays: 3650
+                SymbolsArtifactName: 'win_${{ parameters.EP_NAME }}_${{ parameters.PYTHON_VERSION }}_$(Build.BuildNumber)'
+
           - script: |
               7z x *.whl
             workingDirectory: '$(Build.ArtifactStagingDirectory)'