Update Python-Cuda-Publishing-Pipeline (#23253)

### Description 1. Currently Python-Cuda-Publishing-Pipeline only publishes Linux wheels, not Windows wheels. It is because recently we refactored the upstream pipeline("Python-CUDA-Packaging-Pipeline") to use 1ES PT. This PR fixed the issue 2. tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml no longer includes component-governance-component-detection-steps.yml , because 1ES PT already inserted such a thing 3. Delete tools/ci_build/github/windows/eager/requirements.txt because it is no longer used. ### Motivation and Context The "Python-CUDA-Packaging-Pipeline" is for CUDA 12. "Python CUDA ALT Packaging Pipeline" is for CUDA 11. The two pipelines are very similar, except the CUDA versions are different. Each of them has three parts: build, test, publish. "Python-CUDA-Packaging-Pipeline" is the first part: build. "Python CUDA12 Package Test Pipeline" is the second part. "Python-Cuda-Publishing-Pipeline" is the third part that publishes the packages to an internal ADO feed.
2026-06-25 02:50:42 +00:00 · 2025-01-06 11:50:58 -08:00 · 2025-01-06 11:50:58 -08:00 · c6cbda3257
commit c6cbda3257
parent c53c9caf17
9 changed files with 103 additions and 43 deletions
--- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml
@ -19,6 +19,39 @@ stages:
          python_wheel_suffix: '_gpu'
          timeout: 480
          docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241120.3
-          trt_version: '10.7.0.23-1.cuda11.8'
          cuda_version: '11.8'

+  - stage: Republish_Wheels
+    dependsOn:
+    jobs:
+    - job: Python_Publishing_GPU
+      pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
+      steps:
+      - checkout: none
+      - download: build
+        displayName: 'Download Pipeline Artifact - onnxruntime_gpu'
+        artifact: 'onnxruntime_gpu'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.10'
+        artifact: 'win_gpu_wheel_3.10'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.11'
+        artifact: 'win_gpu_wheel_3.11'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.12'
+        artifact: 'win_gpu_wheel_3.12'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.13'
+        artifact: 'win_gpu_wheel_3.13'
+        patterns: '*.whl'
+
+      - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \;
+        displayName: 'Merge files together'
+
+      - publish: $(Pipeline.Workspace)/build/onnxruntime_gpu
+        artifact: whl
+        displayName: Republish artifacts
--- a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml
@ -29,6 +29,12 @@ extends:
  template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
  parameters:
    sdl:
+      componentgovernance:
+        ignoreDirectories: '$(Build.Repository.LocalPath)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/benchmark,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11/tests,$(Build.Repository.LocalPath)/cmake/external/onnxruntime-extensions,$(Build.Repository.LocalPath)/js/react_native/e2e/node_modules,$(Build.Repository.LocalPath)/js/node_modules,$(Build.Repository.LocalPath)/onnxruntime-inference-examples,$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/benchmark,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11/tests,$(Build.SourcesDirectory)/cmake/external/onnxruntime-extensions,$(Build.SourcesDirectory)/js/react_native/e2e/node_modules,$(Build.SourcesDirectory)/js/node_modules,$(Build.SourcesDirectory)/onnxruntime-inference-examples,$(Build.BinariesDirectory)'
+        alertWarningLevel: High
+        failOnAlert: false
+        verbosity: Normal
+        timeout: 3600
      tsa:
        enabled: true
      codeSignValidation:
--- a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml
@ -21,3 +21,37 @@ stages:
          docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241120.3
          cuda_version: '12.2'

+  - stage: Republish_Wheels
+    dependsOn:
+    jobs:
+    - job: Python_Publishing_GPU
+      pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
+      steps:
+      - checkout: none
+      - download: build
+        displayName: 'Download Pipeline Artifact - onnxruntime_gpu'
+        artifact: 'onnxruntime_gpu'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.10'
+        artifact: 'win_gpu_wheel_3.10'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.11'
+        artifact: 'win_gpu_wheel_3.11'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.12'
+        artifact: 'win_gpu_wheel_3.12'
+        patterns: '*.whl'
+      - download: build
+        displayName: 'Download Pipeline Artifact - Win GPU 3.13'
+        artifact: 'win_gpu_wheel_3.13'
+        patterns: '*.whl'
+
+      - script: find $(Pipeline.Workspace) -name \*win_amd64.whl -exec mv {} $(Pipeline.Workspace)/build/onnxruntime_gpu \;
+        displayName: 'Merge files together'
+
+      - publish: $(Pipeline.Workspace)/build/onnxruntime_gpu
+        artifact: whl
+        displayName: Republish artifacts
--- a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml
@ -32,6 +32,12 @@ extends:
  template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
  parameters:
    sdl:
+      componentgovernance:
+        ignoreDirectories: '$(Build.Repository.LocalPath)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/benchmark,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11,$(Build.Repository.LocalPath)/cmake/external/onnx/third_party/pybind11/tests,$(Build.Repository.LocalPath)/cmake/external/onnxruntime-extensions,$(Build.Repository.LocalPath)/js/react_native/e2e/node_modules,$(Build.Repository.LocalPath)/js/node_modules,$(Build.Repository.LocalPath)/onnxruntime-inference-examples,$(Build.SourcesDirectory)/cmake/external/emsdk/upstream/emscripten/tests,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/benchmark,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11,$(Build.SourcesDirectory)/cmake/external/onnx/third_party/pybind11/tests,$(Build.SourcesDirectory)/cmake/external/onnxruntime-extensions,$(Build.SourcesDirectory)/js/react_native/e2e/node_modules,$(Build.SourcesDirectory)/js/node_modules,$(Build.SourcesDirectory)/onnxruntime-inference-examples,$(Build.BinariesDirectory)'
+        alertWarningLevel: High
+        failOnAlert: false
+        verbosity: Normal
+        timeout: 3600
      tsa:
        enabled: true
      codeSignValidation:
--- a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml
@ -1,7 +1,7 @@
 resources:
  pipelines:
  - pipeline: build
-    source: 'Python-CUDA-Packaging-Pipeline'
+    source: 'Python CUDA12 Package Test Pipeline'
    trigger:
      branches:
        include:
--- a/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml
@ -11,20 +11,21 @@ stages:
    steps:
    - checkout: none
    - download: build
-      displayName: 'Download Pipeline Artifact - onnxruntime_gpu'
-      artifact: 'onnxruntime_gpu'
+      displayName: 'Download Pipeline Artifact'
+      artifact: 'whl'
+      
    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.13'
      displayName: 'Use Python 3.x'
-    - script: 'pip install twine==3.4.2'
+    - script: 'pip install twine==6.0.1'
      displayName: 'Install Twine'
    - task: TwineAuthenticate@1
      displayName: 'Twine Authenticate '
      inputs:
        artifactFeed: PublicPackages/${{ parameters.artifact_feed }}
-    - script: 'python -m twine upload -r ${{ parameters.artifact_feed }} --config-file $(PYPIRC_PATH) --non-interactive --skip-existing *.whl'
-      workingDirectory: '$(Pipeline.Workspace)/build/onnxruntime_gpu'
-      displayName: 'Uploading wheels to ${{ parameters.artifact_feed }}'
-      retryCountOnTaskFailure: 3
-      env:
-        SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+
+    - script: 'python -m twine upload -r ${{ parameters.artifact_feed }} --config-file $(PYPIRC_PATH) --non-interactive *.whl'
+      workingDirectory: '$(Pipeline.Workspace)/build/whl'
+      displayName: 'Uploading wheels to ${{ parameters.artifact_feed }}'

--- a/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml
@ -39,16 +39,6 @@ stages:
      name: ${{ parameters.machine_pool }}
      os: linux
    templateContext:
-      codeSignValidation:
-        enabled: true
-        break: true
-      psscriptanalyzer:
-        enabled: true
-      sdl:
-        binskim:
-          enabled: true
-          scanOutputDirectoryOnly: true
-          targetPathPattern: '\".*.so\"'
      outputs:
      - output: pipelineArtifact
        targetPath: $(Build.ArtifactStagingDirectory)/dist
@ -57,9 +47,6 @@ stages:
        targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.cmake_build_type }}
        artifactName: linux_gpu_wheel_${{ parameters.arch }}
    variables:
-      # The build machine pool doesn't have dotnet, so it can't run CG.
-      - name: skipComponentGovernanceDetection
-        value: true
      - name: extra_build_args
        ${{ if ne(parameters.extra_build_arg, '') }}:
          value: -x ${{ parameters.extra_build_arg }}
--- a/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml
@ -50,16 +50,19 @@ stages:
        name: onnxruntime-Win-CPU-2022
        os: windows
      templateContext:
-        codeSignValidation:
-          enabled: true
-          break: true
-        psscriptanalyzer:
-          enabled: true
        sdl:
+          codeSignValidation:
+            enabled: true
+            # TODO: check why pyd file was not signed
+            break: false
+            additionalTargetsGlobPattern: f|**\*.pyd
+          psscriptanalyzer:
+            enabled: true
          binskim:
+            preReleaseVersion: '4.3.1'
            enabled: true
            scanOutputDirectoryOnly: true
-            targetPathPattern: '+:file|*.dll;-:file|DirectML.dll'
+            analyzeTargetGlob: '+:file|$(BUILD.STAGINGDIRECTORY)\**\*.pyd;+:file|$(BUILD.STAGINGDIRECTORY)\**\*.dll;-:file|DirectML.dll'
        outputs:
        - output: pipelineArtifact
          targetPath: $(Build.ArtifactStagingDirectory)
@ -86,7 +89,7 @@ stages:

          - checkout: self
            clean: true
-            submodules: recursive
+            submodules: none

          - template: ../templates/telemetry-steps.yml

@ -121,7 +124,7 @@ stages:
          - template: ../templates/set-nightly-build-option-variable-step.yml

          - task: PythonScript@0
-            displayName: 'Generate cmake config'
+            displayName: 'Build'
            inputs:
              scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
              arguments: >
@ -131,7 +134,7 @@ stages:
                --cmake_generator "$(VSGenerator)"
                --enable_pybind
                --enable_onnx_tests
-                --parallel --use_binskim_compliant_compile_flags --update --build
+                --parallel 8 --use_binskim_compliant_compile_flags --update --build --msvc_toolset 14.40
                $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} ${{ variables.trt_build_flag }}
              workingDirectory: '$(Build.BinariesDirectory)'

@ -162,9 +165,6 @@ stages:
            workingDirectory: '$(Build.ArtifactStagingDirectory)'
            displayName: 'unzip the package'

-          - template: ../templates/component-governance-component-detection-steps.yml
-            parameters:
-              condition: 'succeeded'

  - stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
    dependsOn: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
--- a/tools/ci_build/github/windows/eager/requirements.txt
+++ b/tools/ci_build/github/windows/eager/requirements.txt
@ -1,7 +0,0 @@
-setuptools
-wheel
-numpy==1.21.6 ; python_version < '3.9'
-numpy==2.1.2 ; python_version >= '3.9'
-typing_extensions
-torch==2.2.0
-parameterized