From 11df22b59bde73cca30eec1d3198767b1599701d Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 20 May 2024 10:37:15 -0700 Subject: [PATCH] Reenabling Nuget Cuda Packaging Pipeline (#20688) ### Description ### Motivation and Context --- .../c-api-noopenmp-packaging-pipelines.yml | 142 ++------ .../cuda-packaging-pipeline.yml | 133 ++++++++ .../stages/nuget-combine-cuda-stage.yml | 314 +++++------------- .../stages/nuget-cuda-packaging-stage.yml | 239 +++++++++++++ .../stages/set_packaging_variables_stage.yml | 46 +++ 5 files changed, 537 insertions(+), 337 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/cuda-packaging-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml create mode 100644 tools/ci_build/github/azure-pipelines/stages/set_packaging_variables_stage.yml diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 1b7475311a..cae34d451c 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -116,47 +116,11 @@ variables: value: $(Agent.TempDirectory)\v12.2 stages: -- stage: Setup - jobs: - - job: Set_Variables - pool: - vmImage: ubuntu-latest - steps: - - checkout: none - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - - bash: | - # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote. - set +x - if [[ "${{ parameters.IsReleaseBuild }}" = True && "${{ parameters.PreReleaseVersionSuffixString }}" != "none" ]]; then - if [[ "${{ parameters.PreReleaseVersionSuffixNumber }}" -eq 0 ]]; then - echo "##vso[task.setvariable variable=ReleaseVersionSuffix;isOutput=true]-${{ parameters.PreReleaseVersionSuffixString }}" - else - echo "##vso[task.setvariable variable=ReleaseVersionSuffix;isOutput=true]-${{ parameters.PreReleaseVersionSuffixString }}.${{ parameters.PreReleaseVersionSuffixNumber }}" - fi - else - echo "##vso[task.setvariable variable=ReleaseVersionSuffix;isOutput=true]" - fi - name: Set_Release_Version_Suffix - - script: | - # Extracting hours and minutes - date=$(date +'%Y%m%d') - # Set the hhmm value as a pipeline variable - echo "##vso[task.setvariable variable=BuildDate;isOutput=true]$date" - displayName: 'Set Start Date as Variable' - name: Set_Build_Date - - - script: | - # Extracting hours and minutes - hhmm=$(date +'%H%M') - # Set the hhmm value as a pipeline variable - echo "##vso[task.setvariable variable=BuildTime;isOutput=true]$hhmm" - displayName: 'Set Start Time as Variable' - name: Set_Build_Time - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' +- template: stages/set_packaging_variables_stage.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }} + PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }} - stage: Debug dependsOn: Setup @@ -237,13 +201,28 @@ stages: AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos' BuildVariant: 'default' -- template: stages/nuget-linux-cuda-packaging-stage.yml +- template: stages/java-cuda-packaging-stage.yml parameters: CudaVersion: ${{ parameters.CudaVersion }} - docker_base_image: ${{ variables.docker_base_image }} - linux_trt_version: ${{ variables.linux_trt_version }} - buildJava: true - buildNodejs: true + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} + +- template: stages/nuget-combine-cuda-stage.yml + parameters: + DoCompliance: ${{ parameters.DoCompliance }} + CudaVersion: ${{ parameters.CudaVersion }} + docker_base_image: ${{ variables.docker_base_image }} + linux_trt_version: ${{ variables.linux_trt_version }} + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }} + win_trt_home: ${{ variables.win_trt_home }} + win_cuda_home: ${{ variables.win_cuda_home }} + DoEsrp: ${{ parameters.DoEsrp }} + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + buildJava: true + buildNodejs: true + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} # ROCm - stage: Linux_C_API_Packaging_ROCm_x64 @@ -311,29 +290,6 @@ stages: condition: 'succeeded' - template: templates/clean-agent-build-directory-step.yml -- template: stages/java-cuda-packaging-stage.yml - parameters: - CudaVersion: ${{ parameters.CudaVersion }} - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - -- template: stages/nuget-win-cuda-packaging-stage.yml - parameters: - CudaVersion: ${{ parameters.CudaVersion }} - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - DoEsrp: ${{ parameters.DoEsrp }} - DoCompliance: ${{ parameters.DoCompliance }} - UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }} - RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - win_trt_home: ${{ variables.win_trt_home }} - win_cuda_home: ${{ variables.win_cuda_home }} - -- template: stages/nuget-combine-cuda-stage.yml - parameters: - DoEsrp: ${{ parameters.DoEsrp }} - DoCompliance: ${{ parameters.DoCompliance }} - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - stage: NuGet_Packaging_ROCm dependsOn: @@ -556,50 +512,6 @@ stages: displayName: 'Clean Agent Directories' condition: always() -- template: nuget/templates/test_win.yml - parameters: - AgentPool: 'onnxruntime-Win2022-GPU-A10' - NugetPackageName: 'Microsoft.ML.OnnxRuntime.Gpu' - ArtifactSuffix: 'GPU' - StageSuffix: 'GPU' - Skipx86Tests: 'true' - CudaVersion: ${{ parameters.CudaVersion }} - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - -- template: nuget/templates/test_win.yml - parameters: - AgentPool: 'onnxruntime-Win2022-GPU-A10' - NugetPackageName: 'Microsoft.ML.OnnxRuntime.Gpu.Windows' - ArtifactSuffix: 'GPU' - StageSuffix: 'GPU' - MoreSuffix: '_Windows' - Skipx86Tests: 'true' - CudaVersion: ${{ parameters.CudaVersion }} - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - -- template: nuget/templates/test_linux.yml - parameters: - AgentPool: Onnxruntime-Linux-GPU-A10 - ArtifactSuffix: 'GPU' - StageSuffix: 'GPU' - NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu' - CudaVersion: ${{ parameters.CudaVersion }} - SpecificArtifact: ${{ parameters.specificArtifact }} - BuildId: ${{ parameters.BuildId }} - -- template: nuget/templates/test_linux.yml - parameters: - AgentPool: Onnxruntime-Linux-GPU-A10 - ArtifactSuffix: 'GPU' - StageSuffix: 'GPU' - MoreSuffix: '_Linux' - NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu.Linux' - CudaVersion: ${{ parameters.CudaVersion }} - SpecificArtifact: ${{ parameters.specificArtifact }} - BuildId: ${{ parameters.BuildId }} - - template: nuget/templates/test_linux.yml parameters: AgentPool: AMD-GPU @@ -813,13 +725,13 @@ stages: inputs: artifactName: 'drop-nuget-qnn-x64' targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x64' - + - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - QNN NuGet arm64' inputs: artifactName: 'drop-nuget-qnn-arm64' targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64' - + - task: PowerShell@2 displayName: 'Bundle NuGet' inputs: diff --git a/tools/ci_build/github/azure-pipelines/cuda-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/cuda-packaging-pipeline.yml new file mode 100644 index 0000000000..c86b92304e --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/cuda-packaging-pipeline.yml @@ -0,0 +1,133 @@ +parameters: + - name: RunOnnxRuntimeTests + displayName: Run Tests? + type: boolean + default: true + + - name: UseIncreasedTimeoutForTests + displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release. + type: boolean + default: false + + - name: DoCompliance + displayName: Run Compliance Tasks? + type: boolean + default: true + + - name: DoEsrp + displayName: Run code sign tasks? Must be true if you are doing an ONNX Runtime release + type: boolean + default: true + + - name: IsReleaseBuild + displayName: Is a release build? Set it to true if you are doing an ONNX Runtime release. + type: boolean + default: false + + - name: PreReleaseVersionSuffixString + displayName: Suffix added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the type of pre-release package. + type: string + values: + - alpha + - beta + - rc + - none + default: none + + - name: PreReleaseVersionSuffixNumber + displayName: Number added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the sequence of a pre-release package. + type: number + default: 0 + + # these 2 parameters are used for debugging. + - name: SpecificArtifact + displayName: Use Specific Artifact (Debugging only) + type: boolean + default: false + + - name: BuildId + displayName: Pipeline BuildId, you could find it in the URL + type: string + default: '0' + + - name: CudaVersion + displayName: CUDA version + type: string + default: '12.2' + values: + - 11.8 + - 12.2 + +variables: + - name: ReleaseVersionSuffix + value: '' + - name: docker_base_image + ${{ if eq(parameters.CudaVersion, '11.8') }}: + value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8 + ${{ if eq(parameters.CudaVersion, '12.2') }}: + value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8 + - name: linux_trt_version + ${{ if eq(parameters.CudaVersion, '11.8') }}: + value: 10.0.1.6-1.cuda11.8 + ${{ if eq(parameters.CudaVersion, '12.2') }}: + value: 10.0.1.6-1.cuda12.4 + - name: win_trt_home + ${{ if eq(parameters.CudaVersion, '11.8') }}: + value: $(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-11.8 + ${{ if eq(parameters.CudaVersion, '12.2') }}: + value: $(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-12.4 + - name: win_cuda_home + ${{ if eq(parameters.CudaVersion, '11.8') }}: + value: $(Agent.TempDirectory)\v11.8 + ${{ if eq(parameters.CudaVersion, '12.2') }}: + value: $(Agent.TempDirectory)\v12.2 +resources: + repositories: + - repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step + type: github + endpoint: ort-examples + name: microsoft/onnxruntime-inference-examples + - repository: manylinux + type: Github + endpoint: Microsoft + name: pypa/manylinux + ref: 5eda9aded5462201e6310105728d33016e637ea7 + +stages: + # Set ReleaseVersionSuffix + - template: stages/set_packaging_variables_stage.yml + parameters: + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }} + PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }} + + # this is needed for certain artifacts to be published + - stage: Linux_C_API_Packaging_CPU_x64 + dependsOn: [ ] + jobs: + - template: templates/c-api-linux-cpu.yml + parameters: + BaseImage: 'registry.access.redhat.com/ubi8/ubi' + OnnxruntimeArch: 'x64' + OnnxruntimeNodejsBindingArch: 'x64' + PoolName: 'onnxruntime-Ubuntu2204-AMD-CPU' + PackageJava: false + PackageNodeJS: false + + # Nuget Packaging + - template: stages/nuget-combine-cuda-stage.yml + parameters: + DoCompliance: ${{ parameters.DoCompliance }} + CudaVersion: ${{ parameters.CudaVersion }} + docker_base_image: ${{ variables.docker_base_image }} + linux_trt_version: ${{ variables.linux_trt_version }} + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }} + win_trt_home: ${{ variables.win_trt_home }} + win_cuda_home: ${{ variables.win_cuda_home }} + DoEsrp: ${{ parameters.DoEsrp }} + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} + buildJava: false + buildNodejs: false + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-combine-cuda-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-combine-cuda-stage.yml index 9f5ca3db31..364811f8da 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nuget-combine-cuda-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-combine-cuda-stage.yml @@ -8,232 +8,102 @@ parameters: - name: IsReleaseBuild type: boolean +- name: CudaVersion + type: string + +- name: docker_base_image + type: string + +- name: linux_trt_version + type: string + +- name: win_trt_home + type: string + +- name: win_cuda_home + type: string + +- name: UseIncreasedTimeoutForTests + type: boolean + +- name: RunOnnxRuntimeTests + type: boolean + +- name: buildJava + type: boolean + +- name: buildNodejs + type: boolean + +- name: SpecificArtifact + type: boolean + +- name: BuildId + type: string + stages: -######## Nuget ######## -# Win/Linux CUDA Combined packaging -- stage: NuGet_Packaging_GPU - dependsOn: - - Setup - - Windows_Packaging_CUDA - - Windows_Packaging_TensorRT - - Linux_C_API_Packaging_Combined_CUDA - condition: succeeded() - jobs: - - job: - workspace: - clean: all - pool: 'Onnxruntime-Win-CPU-2022' - variables: - breakCodesignValidationInjection: ${{ parameters.DoEsrp }} - ReleaseVersionSuffix: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']] - BuildDate: $[format('{0:yyyyMMdd}', pipeline.startTime)] - BuildTime: $[format('{0:HHmm}', pipeline.startTime)] +- template: nuget-linux-cuda-packaging-stage.yml + parameters: + CudaVersion: ${{ parameters.CudaVersion }} + docker_base_image: ${{ parameters.docker_base_image }} + linux_trt_version: ${{ parameters.linux_trt_version }} + buildJava: ${{ parameters.buildJava }} + buildNodejs: ${{ parameters.buildNodejs }} - steps: - - checkout: self - submodules: true - - template: ../templates/flex-downloadPipelineArtifact.yml - parameters: - StepName: 'Download Pipeline Artifact - NuGet' - ArtifactName: 'onnxruntime-win-x64-cuda' - TargetPath: '$(Build.BinariesDirectory)/nuget-artifact' +- template: nuget-win-cuda-packaging-stage.yml + parameters: + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }} + CudaVersion: ${{ parameters.CudaVersion }} + win_trt_home: ${{ parameters.win_trt_home }} + win_cuda_home: ${{ parameters.win_cuda_home }} - - template: ../templates/flex-downloadPipelineArtifact.yml - parameters: - StepName: 'Download Pipeline Artifact - NuGet' - ArtifactName: 'onnxruntime-win-x64-tensorrt' - TargetPath: '$(Build.BinariesDirectory)/nuget-artifact' +- template: nuget-cuda-packaging-stage.yml + parameters: + DoCompliance: ${{ parameters.DoCompliance }} + DoEsrp: ${{ parameters.DoEsrp }} + IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - - template: ../templates/flex-downloadPipelineArtifact.yml - parameters: - StepName: 'Download Pipeline Artifact - NuGet' - ArtifactName: 'onnxruntime-linux-x64-cuda' - TargetPath: '$(Build.BinariesDirectory)/nuget-artifact' +- template: ../nuget/templates/test_win.yml + parameters: + AgentPool : 'onnxruntime-Win2022-GPU-A10' + NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu' + ArtifactSuffix: 'GPU' + StageSuffix: 'GPU' + Skipx86Tests: 'true' + CudaVersion: ${{ parameters.CudaVersion }} + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} - - template: ../templates/flex-downloadPipelineArtifact.yml - parameters: - StepName: 'Download Pipeline Artifact - NuGet' - ArtifactName: 'onnxruntime-linux-x64-tensorrt' - TargetPath: '$(Build.BinariesDirectory)/nuget-artifact' +- template: ../nuget/templates/test_win.yml + parameters: + AgentPool : 'onnxruntime-Win2022-GPU-A10' + NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu.Windows' + ArtifactSuffix: 'GPU' + StageSuffix: 'GPU' + MoreSuffix: '_Windows' + Skipx86Tests: 'true' + CudaVersion: ${{ parameters.CudaVersion }} + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} - - template: ../templates/flex-downloadPipelineArtifact.yml - parameters: - StepName: 'Download Pipeline Artifact - NuGet' - ArtifactName: 'drop-extra' - TargetPath: '$(Build.BinariesDirectory)/extra-artifact' +- template: ../nuget/templates/test_linux.yml + parameters: + AgentPool : Onnxruntime-Linux-GPU-A10 + ArtifactSuffix: 'GPU' + StageSuffix: 'GPU' + NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu' + CudaVersion: ${{ parameters.CudaVersion }} + SpecificArtifact: ${{ parameters.specificArtifact }} + BuildId: ${{ parameters.BuildId }} - # Reconstruct the build dir - - task: PowerShell@2 - displayName: 'PS: Extract nuget files gpu' - inputs: - targetType: filePath - filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files_gpu.ps1 - - - script: | - dir - workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact' - displayName: 'List artifacts' - - - script: | - mklink /D /J models C:\local\models - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Create models link' - - - task: NuGetToolInstaller@0 - displayName: Use Nuget 6.2.1 - inputs: - versionSpec: 6.2.1 - - - task: PowerShell@2 - displayName: Install .NET 6 workloads - inputs: - targetType: 'inline' - script: | - dotnet workload install android ios macos - workingDirectory: '$(Build.SourcesDirectory)\csharp' - - - task: PowerShell@2 - displayName: Build .NET 6 targets using dotnet - inputs: - targetType: 'inline' - # we don't specify 'Any CPU' as the platform here because if we do it gets added to the output path - # e.g. csharp\src\Microsoft.ML.OnnxRuntime\bin\Any CPU\RelWithDebInfo\net6.0-ios\ - # which is inconsistent with the msbuild output path for the pre-.net6 targets - # e.g. csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\monoandroid11.0 - # and makes it harder to do the packing - # - # 'Any CPU' is the default (first 'mixed' platform specified in the csproj) so this should be fine. - script: | - dotnet build .\src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj -p:SelectedTargets=Net6 -p:Configuration=RelWithDebInfo -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId="Microsoft.ML.OnnxRuntime.Gpu" -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} -p:ReleaseVersionSuffix=$(ReleaseVersionSuffix) - workingDirectory: '$(Build.SourcesDirectory)\csharp' - - - task: MSBuild@1 - displayName: 'Restore NuGet Packages and create project.assets.json for pre-.net6 targets' - inputs: - solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - platform: 'Any CPU' - configuration: RelWithDebInfo - msbuildArguments: '-t:restore -p:SelectedTargets=PreNet6 -p:OrtPackageId="Microsoft.ML.OnnxRuntime.Gpu"' - workingDirectory: '$(Build.SourcesDirectory)\csharp' - - - task: MSBuild@1 - displayName: 'Build C# for pre-.net6 targets' - inputs: - solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - configuration: RelWithDebInfo - platform: 'Any CPU' - msbuildArguments: '-p:SelectedTargets=PreNet6 -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId="Microsoft.ML.OnnxRuntime.Gpu" -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} -p:ReleaseVersionSuffix=$(ReleaseVersionSuffix)' - workingDirectory: '$(Build.SourcesDirectory)\csharp' - - - template: ../templates/win-esrp-dll.yml - parameters: - FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo' - DisplayName: 'ESRP - Sign C# dlls' - DoEsrp: ${{ parameters.DoEsrp }} - - - task: MSBuild@1 - displayName: Update projects.assets.json with combined list of all target frameworks - inputs: - solution: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj' - platform: 'Any CPU' - configuration: RelWithDebInfo - msbuildArguments: '-t:restore -p:SelectedTargets=All -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu' - workingDirectory: '$(Build.SourcesDirectory)\csharp' - - - task: MSBuild@1 - displayName: 'Build Nuget Packages' - inputs: - solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj' - configuration: RelWithDebInfo - platform: 'Any CPU' - msbuildArguments: '-t:CreatePackage -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} - -p:ReleaseVersionSuffix=$(ReleaseVersionSuffix) -p:CurrentDate=$(BuildDate) -p:CurrentTime=$(BuildTime)' - workingDirectory: '$(Build.SourcesDirectory)\csharp' - - - task: BatchScript@1 - displayName: 'Add TensorRT header file to the native nuGet package' - inputs: - filename: $(Build.SourcesDirectory)\tools\ci_build\github\windows\bundle_nuget_with_native_headers.bat - workingFolder: $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo - - - task: CopyFiles@2 - displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' - Contents: '*.snupkg' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: CopyFiles@2 - displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' - Contents: '*.nupkg' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: CopyFiles@2 - displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo' - Contents: '*.nupkg' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - template: ../templates/esrp_nuget.yml - parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.ArtifactStagingDirectory)' - DoEsrp: ${{ parameters.DoEsrp }} - - - template: ../templates/validate-package.yml - parameters: - PackageType: 'nuget' - PackagePath: '$(Build.ArtifactStagingDirectory)' - PlatformsSupported: 'win-x64,linux-x64' - # 1* stands for version number. we use it to filter Gpu.Windows and Gpu.Linux packages - PackageName: 'Microsoft.ML.OnnxRuntime.Gpu.1*nupkg' - VerifyNugetSigning: false - - - template: ../templates/validate-package.yml - parameters: - PackageType: 'nuget' - PackagePath: '$(Build.ArtifactStagingDirectory)' - PackageName: 'Microsoft.ML.OnnxRuntime.Gpu.Windows.*nupkg' - PlatformsSupported: 'win-x64' - VerifyNugetSigning: false - - - template: ../templates/validate-package.yml - parameters: - PackageType: 'nuget' - PackagePath: '$(Build.ArtifactStagingDirectory)' - PackageName: 'Microsoft.ML.OnnxRuntime.Gpu.Linux.*nupkg' - PlatformsSupported: 'linux-x64' - VerifyNugetSigning: false - - - task: MSBuild@1 - displayName: 'Clean C#' - inputs: - solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - platform: 'Any CPU' - configuration: RelWithDebInfo - msbuildArguments: '-t:Clean -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu' - workingDirectory: '$(Build.SourcesDirectory)\csharp' - - - task: RoslynAnalyzers@2 - displayName: 'Run Roslyn Analyzers' - inputs: - userProvideBuildInfo: msBuildInfo - msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\msbuild.exe" $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln -p:configuration="RelWithDebInfo" -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu' - condition: and(succeeded(), eq('${{ parameters.DoCompliance }}', true)) - - - template: ../templates/component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget-GPU' - targetPath: '$(Build.ArtifactStagingDirectory)' - - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() \ No newline at end of file +- template: ../nuget/templates/test_linux.yml + parameters: + AgentPool : Onnxruntime-Linux-GPU-A10 + ArtifactSuffix: 'GPU' + StageSuffix: 'GPU' + MoreSuffix: '_Linux' + NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu.Linux' + CudaVersion: ${{ parameters.CudaVersion }} + SpecificArtifact: ${{ parameters.specificArtifact }} + BuildId: ${{ parameters.BuildId }} \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml new file mode 100644 index 0000000000..9f5ca3db31 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml @@ -0,0 +1,239 @@ +parameters: +- name: DoCompliance + type: boolean + +- name: DoEsrp + type: boolean + +- name: IsReleaseBuild + type: boolean + +stages: +######## Nuget ######## +# Win/Linux CUDA Combined packaging +- stage: NuGet_Packaging_GPU + dependsOn: + - Setup + - Windows_Packaging_CUDA + - Windows_Packaging_TensorRT + - Linux_C_API_Packaging_Combined_CUDA + condition: succeeded() + jobs: + - job: + workspace: + clean: all + pool: 'Onnxruntime-Win-CPU-2022' + variables: + breakCodesignValidationInjection: ${{ parameters.DoEsrp }} + ReleaseVersionSuffix: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']] + BuildDate: $[format('{0:yyyyMMdd}', pipeline.startTime)] + BuildTime: $[format('{0:HHmm}', pipeline.startTime)] + + steps: + - checkout: self + submodules: true + - template: ../templates/flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline Artifact - NuGet' + ArtifactName: 'onnxruntime-win-x64-cuda' + TargetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - template: ../templates/flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline Artifact - NuGet' + ArtifactName: 'onnxruntime-win-x64-tensorrt' + TargetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - template: ../templates/flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline Artifact - NuGet' + ArtifactName: 'onnxruntime-linux-x64-cuda' + TargetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - template: ../templates/flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline Artifact - NuGet' + ArtifactName: 'onnxruntime-linux-x64-tensorrt' + TargetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - template: ../templates/flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline Artifact - NuGet' + ArtifactName: 'drop-extra' + TargetPath: '$(Build.BinariesDirectory)/extra-artifact' + + # Reconstruct the build dir + - task: PowerShell@2 + displayName: 'PS: Extract nuget files gpu' + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files_gpu.ps1 + + - script: | + dir + workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact' + displayName: 'List artifacts' + + - script: | + mklink /D /J models C:\local\models + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Create models link' + + - task: NuGetToolInstaller@0 + displayName: Use Nuget 6.2.1 + inputs: + versionSpec: 6.2.1 + + - task: PowerShell@2 + displayName: Install .NET 6 workloads + inputs: + targetType: 'inline' + script: | + dotnet workload install android ios macos + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: PowerShell@2 + displayName: Build .NET 6 targets using dotnet + inputs: + targetType: 'inline' + # we don't specify 'Any CPU' as the platform here because if we do it gets added to the output path + # e.g. csharp\src\Microsoft.ML.OnnxRuntime\bin\Any CPU\RelWithDebInfo\net6.0-ios\ + # which is inconsistent with the msbuild output path for the pre-.net6 targets + # e.g. csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\monoandroid11.0 + # and makes it harder to do the packing + # + # 'Any CPU' is the default (first 'mixed' platform specified in the csproj) so this should be fine. + script: | + dotnet build .\src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj -p:SelectedTargets=Net6 -p:Configuration=RelWithDebInfo -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId="Microsoft.ML.OnnxRuntime.Gpu" -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} -p:ReleaseVersionSuffix=$(ReleaseVersionSuffix) + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: MSBuild@1 + displayName: 'Restore NuGet Packages and create project.assets.json for pre-.net6 targets' + inputs: + solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + platform: 'Any CPU' + configuration: RelWithDebInfo + msbuildArguments: '-t:restore -p:SelectedTargets=PreNet6 -p:OrtPackageId="Microsoft.ML.OnnxRuntime.Gpu"' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: MSBuild@1 + displayName: 'Build C# for pre-.net6 targets' + inputs: + solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + configuration: RelWithDebInfo + platform: 'Any CPU' + msbuildArguments: '-p:SelectedTargets=PreNet6 -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId="Microsoft.ML.OnnxRuntime.Gpu" -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} -p:ReleaseVersionSuffix=$(ReleaseVersionSuffix)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - template: ../templates/win-esrp-dll.yml + parameters: + FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo' + DisplayName: 'ESRP - Sign C# dlls' + DoEsrp: ${{ parameters.DoEsrp }} + + - task: MSBuild@1 + displayName: Update projects.assets.json with combined list of all target frameworks + inputs: + solution: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\Microsoft.ML.OnnxRuntime.csproj' + platform: 'Any CPU' + configuration: RelWithDebInfo + msbuildArguments: '-t:restore -p:SelectedTargets=All -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: MSBuild@1 + displayName: 'Build Nuget Packages' + inputs: + solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj' + configuration: RelWithDebInfo + platform: 'Any CPU' + msbuildArguments: '-t:CreatePackage -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} + -p:ReleaseVersionSuffix=$(ReleaseVersionSuffix) -p:CurrentDate=$(BuildDate) -p:CurrentTime=$(BuildTime)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: BatchScript@1 + displayName: 'Add TensorRT header file to the native nuGet package' + inputs: + filename: $(Build.SourcesDirectory)\tools\ci_build\github\windows\bundle_nuget_with_native_headers.bat + workingFolder: $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo + + - task: CopyFiles@2 + displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + Contents: '*.snupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - task: CopyFiles@2 + displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + Contents: '*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - task: CopyFiles@2 + displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo' + Contents: '*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - template: ../templates/esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.ArtifactStagingDirectory)' + DoEsrp: ${{ parameters.DoEsrp }} + + - template: ../templates/validate-package.yml + parameters: + PackageType: 'nuget' + PackagePath: '$(Build.ArtifactStagingDirectory)' + PlatformsSupported: 'win-x64,linux-x64' + # 1* stands for version number. we use it to filter Gpu.Windows and Gpu.Linux packages + PackageName: 'Microsoft.ML.OnnxRuntime.Gpu.1*nupkg' + VerifyNugetSigning: false + + - template: ../templates/validate-package.yml + parameters: + PackageType: 'nuget' + PackagePath: '$(Build.ArtifactStagingDirectory)' + PackageName: 'Microsoft.ML.OnnxRuntime.Gpu.Windows.*nupkg' + PlatformsSupported: 'win-x64' + VerifyNugetSigning: false + + - template: ../templates/validate-package.yml + parameters: + PackageType: 'nuget' + PackagePath: '$(Build.ArtifactStagingDirectory)' + PackageName: 'Microsoft.ML.OnnxRuntime.Gpu.Linux.*nupkg' + PlatformsSupported: 'linux-x64' + VerifyNugetSigning: false + + - task: MSBuild@1 + displayName: 'Clean C#' + inputs: + solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + platform: 'Any CPU' + configuration: RelWithDebInfo + msbuildArguments: '-t:Clean -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu' + workingDirectory: '$(Build.SourcesDirectory)\csharp' + + - task: RoslynAnalyzers@2 + displayName: 'Run Roslyn Analyzers' + inputs: + userProvideBuildInfo: msBuildInfo + msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\msbuild.exe" $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln -p:configuration="RelWithDebInfo" -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu' + condition: and(succeeded(), eq('${{ parameters.DoCompliance }}', true)) + + - template: ../templates/component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline NuGet Artifact' + inputs: + artifactName: 'drop-signed-nuget-GPU' + targetPath: '$(Build.ArtifactStagingDirectory)' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/stages/set_packaging_variables_stage.yml b/tools/ci_build/github/azure-pipelines/stages/set_packaging_variables_stage.yml new file mode 100644 index 0000000000..616dc5495d --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/stages/set_packaging_variables_stage.yml @@ -0,0 +1,46 @@ +parameters: + IsReleaseBuild: false + PreReleaseVersionSuffixString: 'none' + PreReleaseVersionSuffixNumber: 0 +stages: +- stage: Setup + jobs: + - job: Set_Variables + pool: + vmImage: ubuntu-latest + steps: + - checkout: none + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + - bash: | + # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote. + set +x + if [[ "${{ parameters.IsReleaseBuild }}" = True && "${{ parameters.PreReleaseVersionSuffixString }}" != "none" ]]; then + if [[ "${{ parameters.PreReleaseVersionSuffixNumber }}" -eq 0 ]]; then + echo "##vso[task.setvariable variable=ReleaseVersionSuffix;isOutput=true]-${{ parameters.PreReleaseVersionSuffixString }}" + else + echo "##vso[task.setvariable variable=ReleaseVersionSuffix;isOutput=true]-${{ parameters.PreReleaseVersionSuffixString }}.${{ parameters.PreReleaseVersionSuffixNumber }}" + fi + else + echo "##vso[task.setvariable variable=ReleaseVersionSuffix;isOutput=true]" + fi + name: Set_Release_Version_Suffix + - script: | + # Extracting hours and minutes + date=$(date +'%Y%m%d') + # Set the hhmm value as a pipeline variable + echo "##vso[task.setvariable variable=BuildDate;isOutput=true]$date" + displayName: 'Set Start Date as Variable' + name: Set_Build_Date + + - script: | + # Extracting hours and minutes + hhmm=$(date +'%H%M') + # Set the hhmm value as a pipeline variable + echo "##vso[task.setvariable variable=BuildTime;isOutput=true]$hhmm" + displayName: 'Set Start Time as Variable' + name: Set_Build_Time + - template: ../templates/component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded'