Create python packages for DML (#8061)

2026-06-01 23:30:35 +00:00 · 2021-06-16 16:59:12 -07:00 · 2021-06-16 16:59:12 -07:00 · 96989b83ee
commit 96989b83ee
parent d924fd205b
3 changed files with 118 additions and 34 deletions
--- a/setup.py
+++ b/setup.py
@ -54,8 +54,8 @@ rocm_version = None
 # The following arguments are mutually exclusive
 if parse_arg_remove_boolean(sys.argv, '--use_tensorrt'):
    package_name = 'onnxruntime-gpu-tensorrt' if not nightly_build else 'ort-trt-nightly'
-elif parse_arg_remove_boolean(sys.argv, '--use_cuda'):
-    package_name = 'onnxruntime-gpu' if not nightly_build else 'ort-gpu-nightly'
+elif wheel_name_suffix == 'gpu':
+    #TODO: how to support multiple CUDA versions?
    cuda_version = parse_arg_remove_string(sys.argv, '--cuda_version=')
 elif parse_arg_remove_boolean(sys.argv, '--use_rocm'):
    package_name = 'onnxruntime-rocm' if not nightly_build else 'ort-rocm-nightly'
@ -72,8 +72,7 @@ elif parse_arg_remove_boolean(sys.argv, '--use_acl'):
    package_name = 'onnxruntime-acl'
 elif parse_arg_remove_boolean(sys.argv, '--use_armnn'):
    package_name = 'onnxruntime-armnn'
-elif parse_arg_remove_boolean(sys.argv, '--use_dml'):
-    package_name = 'onnxruntime-dml'
+

 # PEP 513 defined manylinux1_x86_64 and manylinux1_i686
 # PEP 571 defined manylinux2010_x86_64 and manylinux2010_i686
@ -380,7 +379,7 @@ if local_version:
    version_number = version_number + local_version

 if wheel_name_suffix:
-    package_name = "{}_{}".format(package_name, wheel_name_suffix)
+    package_name = "{}-{}".format(package_name, wheel_name_suffix)

 cmd_classes = {}
 if bdist_wheel is not None:
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@ -1618,7 +1618,8 @@ def build_python_wheel(
        if use_tensorrt:
            args.append('--use_tensorrt')
        elif use_cuda:
-            args.append('--use_cuda')
+            # The following line assumes no other EP is enabled
+            args.append('--wheel_name_suffix=gpu')
            if cuda_version:
                args.append('--cuda_version={}'.format(cuda_version))
        elif use_rocm:
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@ -901,8 +901,7 @@ stages:
        EnvSetupScript: setup_env.bat
        buildArch: x64
        setVcvars: true
-        BuildConfig: 'Release'
-        GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
+        BuildConfig: 'Release'        
      timeoutInMinutes: 120
      workspace:
        clean: all
@ -983,11 +982,6 @@ stages:
          Contents: '*.whl'
          TargetFolder: '$(Build.ArtifactStagingDirectory)'

-      - script: |
-          rename *.whl *.zip
-        workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
-        displayName: 'Rename file extension for codesign validation'
-
      - task: PublishTestResults@2
        displayName: 'Publish unit test results'
        inputs:
@ -1001,6 +995,66 @@ stages:
        inputs:
          ArtifactName: onnxruntime

+      - task: DeleteFiles@1
+        displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo'
+        condition: and (succeeded(), eq(variables['PythonVersion'], '3.7'))
+        inputs:
+          SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
+          Contents: |
+            **/*.obj
+            **/*.pdb
+
+      - task: PythonScript@0
+        displayName: 'Regenerate cmake config with STATIC_ANALYSIS=ON'
+        condition: and (succeeded(), eq(variables['PythonVersion'], '3.7'))
+        inputs:
+          scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
+          arguments: >
+            --config RelWithDebInfo
+            --enable_lto
+            --build_dir $(Build.BinariesDirectory)
+            --skip_submodule_sync
+            --cmake_generator "Visual Studio 16 2019"
+            --enable_pybind
+            --enable_onnx_tests
+            ${{ parameters.build_py_parameters }}
+            --parallel
+            $(TelemetryOption)
+            --update
+            --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON
+          workingDirectory: '$(Build.BinariesDirectory)'
+
+      #Manually set msBuildCommandline so that we can also set CAExcludePath
+      - task: SDLNativeRules@2
+        displayName: 'Run the PREfast SDL Native Rules for MSBuild'
+        condition: and (succeeded(), eq(variables['PythonVersion'], '3.7'))
+        inputs:
+          userProvideBuildInfo: msBuildInfo
+          msBuildVersion: 16.0
+          msBuildArchitecture: x64
+          msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="x64" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64'
+        continueOnError: false
+
+      - script: |
+          rename *.whl *.zip
+        workingDirectory: '$(Build.ArtifactStagingDirectory)'
+        displayName: 'Rename file extension for codesign validation'
+
+      - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
+        displayName: 'Publish Security Analysis Logs'
+        condition: and (succeeded(), eq(variables['PythonVersion'], '3.7'))
+        inputs:
+          ArtifactName: CodeAnalysisLogs_windows_cpu
+        continueOnError: true
+
+      - task: TSAUpload@1
+        condition: and(and (succeeded(), eq(variables['PythonVersion'], '3.7')), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+        displayName: 'TSA Upload'
+        inputs:
+          tsaVersion: TsaV2
+          codeBaseName: 'onnxruntime_master'
+        continueOnError: true
+
      - template: component-governance-component-detection-steps.yml
        parameters:
          condition: 'succeeded'
@ -1018,18 +1072,48 @@ stages:
      variables:
        CUDA_VERSION: '11.1'
        buildArch: x64
-        EnvSetupScript: setup_env_cuda_11.bat
-        GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
      strategy:
        matrix:
-          Python36:
+          Python36_cuda:
            PythonVersion: '3.6'
-          Python37:
+            EpBuildFlags: --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.0.5.39\cuda" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80"
+            EnvSetupScript: setup_env_cuda_11.bat
+            EP_NAME: gpu
+          Python37_cuda:
            PythonVersion: '3.7'
-          Python38:
+            EpBuildFlags: --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.0.5.39\cuda" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80"
+            EnvSetupScript: setup_env_cuda_11.bat
+            EP_NAME: gpu
+          Python38_cuda:
            PythonVersion: '3.8'
-          Python39:
+            EpBuildFlags: --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.0.5.39\cuda" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80"
+            EnvSetupScript: setup_env_cuda_11.bat
+            EP_NAME: gpu
+          Python39_cuda:
            PythonVersion: '3.9'
+            EpBuildFlags: --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.0.5.39\cuda" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80"
+            EnvSetupScript: setup_env_cuda_11.bat
+            EP_NAME: gpu
+          Python36_dml:
+            PythonVersion: '3.6'
+            EpBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
+            EP_NAME: dml
+            EnvSetupScript: setup_env.bat
+          Python37_dml:
+            PythonVersion: '3.7'
+            EpBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
+            EP_NAME: dml
+            EnvSetupScript: setup_env.bat
+          Python38_dml:
+            PythonVersion: '3.8'
+            EpBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
+            EP_NAME: dml
+            EnvSetupScript: setup_env.bat
+          Python39_dml:
+            PythonVersion: '3.9'
+            EpBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
+            EP_NAME: dml
+            EnvSetupScript: setup_env.bat
      steps:
      - checkout: self
        clean: true
@ -1080,10 +1164,7 @@ stages:
            --enable_onnx_tests
            ${{ parameters.build_py_parameters }}
            --parallel
-            --use_cuda --cuda_version=$(CUDA_VERSION)
-            --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)"
-            --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.0.5.39\cuda"
-            $(TelemetryOption) --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80"
+            $(TelemetryOption) $(EpBuildFlags)
          workingDirectory: '$(Build.BinariesDirectory)'

      # Esrp signing
@ -1098,7 +1179,7 @@ stages:
        displayName: 'Build wheel'
        inputs:
          scriptPath: '$(Build.SourcesDirectory)\setup.py'
-          arguments: 'bdist_wheel --use_cuda ${{ parameters.build_py_parameters }} $(NightlyBuildOption)'
+          arguments: 'bdist_wheel ${{ parameters.build_py_parameters }} $(NightlyBuildOption) --wheel_name_suffix=$(EP_NAME)'
          workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'

      - task: CopyFiles@2
@ -1108,11 +1189,6 @@ stages:
          Contents: '*.whl'
          TargetFolder: '$(Build.ArtifactStagingDirectory)'

-      - script: |
-          rename *.whl *.zip
-        workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
-        displayName: 'Rename file extension for codesign validation'
-
      - task: PublishTestResults@2
        displayName: 'Publish unit test results'
        inputs:
@ -1134,7 +1210,6 @@ stages:
          Contents: |
            **/*.obj
            **/*.pdb
-            **/*.dll

      - task: PythonScript@0
        displayName: 'Regenerate cmake config with STATIC_ANALYSIS=ON'
@ -1150,10 +1225,7 @@ stages:
            --enable_onnx_tests
            ${{ parameters.build_py_parameters }}
            --parallel
-            --use_cuda --cuda_version=$(CUDA_VERSION)
-            --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)"
-            --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.0.5.39\cuda"
-            $(TelemetryOption)
+            $(TelemetryOption) $(EpBuildFlags)
            --update
            --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON
          workingDirectory: '$(Build.BinariesDirectory)'
@ -1167,6 +1239,18 @@ stages:
          msBuildVersion: 16.0
          msBuildArchitecture: x64
          msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="x64" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64'
+        continueOnError: false
+
+      - script: |
+          rename *.whl *.zip
+        workingDirectory: '$(Build.ArtifactStagingDirectory)'
+        displayName: 'Rename file extension for codesign validation'
+
+      - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
+        displayName: 'Publish Security Analysis Logs'
+        condition: and (succeeded(), eq(variables['PythonVersion'], '3.7'))
+        inputs:
+          ArtifactName: CodeAnalysisLogs_windows_gpu_$(EP_NAME)_$(PythonVersion)
        continueOnError: true

      - task: TSAUpload@1