mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-29 23:06:41 +00:00
5f74f198c1
Merge CPU/GPU nuget pipeline. The old GPU nuget pipeline will be only for DML. TODO: the result GPU package contains PDB files for some of the DLLs, but not all. It is due to the refactoring of CUDA EP to pluggable DLLs. At that time we forgot to copy the PDB files. However, I can't add them in now. Because currently the package is already 220MB large. If the missed PDB files were added, then it will be oversize. nuget.org doesn't accept >250MB packages.
406 lines
15 KiB
YAML
406 lines
15 KiB
YAML
parameters:
|
|
- name: RunOnnxRuntimeTests
|
|
displayName: Run Tests?
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: DoCompliance
|
|
displayName: Run Compliance Tasks?
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: DoEsrp
|
|
displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release.
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: IsReleaseBuild
|
|
displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release.
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: AdditionalBuildFlags
|
|
displayName: Additional build flags for build.py
|
|
type: string
|
|
default: ''
|
|
|
|
- name: AdditionalWinBuildFlags
|
|
displayName: Additional build flags that just for Windows Builds
|
|
type: string
|
|
default: ''
|
|
|
|
- name: OrtNugetPackageId
|
|
displayName: Package name for nuget
|
|
type: string
|
|
default: 'Microsoft.ML.OnnxRuntime'
|
|
|
|
- name: BuildVariant
|
|
type: string
|
|
default: 'openmp'
|
|
|
|
jobs:
|
|
- job: Linux_C_API_Packaging_CPU_x64
|
|
workspace:
|
|
clean: all
|
|
timeoutInMinutes: 60
|
|
pool: 'Linux-CPU'
|
|
steps:
|
|
- template: set-version-number-variables-step.yml
|
|
- template: get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
|
|
Context: tools/ci_build/github/linux/docker
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
|
|
Repository: onnxruntimecpubuild
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \
|
|
/onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \
|
|
--skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-x64"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
|
|
- script: |
|
|
set -e -x
|
|
mkdir $(Build.ArtifactStagingDirectory)/testdata
|
|
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata
|
|
ls -al $(Build.ArtifactStagingDirectory)
|
|
displayName: 'Create Artifacts'
|
|
|
|
- task: PublishPipelineArtifact@0
|
|
displayName: 'Publish Pipeline Artifact'
|
|
inputs:
|
|
artifactName: 'drop-linux'
|
|
targetPath: '$(Build.ArtifactStagingDirectory)'
|
|
|
|
- template: c-api-artifacts-package-and-publish-steps-posix.yml
|
|
parameters:
|
|
buildConfig: 'Release'
|
|
artifactName: 'onnxruntime-linux-x64-$(OnnxRuntimeVersion)'
|
|
artifactNameNoVersionString: 'onnxruntime-linux-x64'
|
|
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
|
|
commitId: $(OnnxRuntimeGitCommitHash)
|
|
|
|
- template: component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- job: MacOS_C_API_Packaging_CPU_x64
|
|
workspace:
|
|
clean: all
|
|
variables:
|
|
MACOSX_DEPLOYMENT_TARGET: '10.13'
|
|
pool:
|
|
vmImage: 'macOS-10.15'
|
|
timeoutInMinutes: 120
|
|
steps:
|
|
- template: set-version-number-variables-step.yml
|
|
|
|
- script: |
|
|
set -e
|
|
pushd .
|
|
cd $(Build.SourcesDirectory)/cmake/external/protobuf
|
|
cmake ./cmake -DCMAKE_INSTALL_PREFIX=$(Build.BinariesDirectory)/protobuf -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo
|
|
make -j$(getconf _NPROCESSORS_ONLN)
|
|
make install
|
|
popd
|
|
export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH
|
|
export ONNX_ML=1
|
|
export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF"
|
|
sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
|
|
sudo xcode-select --switch /Applications/Xcode_12.4.app/Contents/Developer
|
|
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release
|
|
displayName: 'Build and Test MacOS'
|
|
- template: c-api-artifacts-package-and-publish-steps-posix.yml
|
|
parameters:
|
|
buildConfig: 'Release'
|
|
artifactName: 'onnxruntime-osx-x64-$(OnnxRuntimeVersion)'
|
|
artifactNameNoVersionString: 'onnxruntime-osx-x64'
|
|
libraryName: 'libonnxruntime.$(OnnxRuntimeVersion).dylib'
|
|
commitId: $(OnnxRuntimeGitCommitHash)
|
|
|
|
- script: |
|
|
set -e -x
|
|
mkdir $(Build.ArtifactStagingDirectory)/testdata
|
|
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata
|
|
displayName: 'Create Artifacts'
|
|
|
|
- task: PublishPipelineArtifact@0
|
|
displayName: 'Publish Pipeline Artifact'
|
|
inputs:
|
|
artifactName: 'drop-osx'
|
|
targetPath: '$(Build.ArtifactStagingDirectory)'
|
|
|
|
- template: component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
|
|
- template: win-ci.yml
|
|
parameters:
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
job_name_suffix: CPU_x86_${{ parameters.BuildVariant }}
|
|
EnvSetupScript: setup_env_x86.bat
|
|
buildArch: x86
|
|
msbuildPlatform: Win32
|
|
packageName: x86
|
|
buildparameter: ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}}
|
|
runTests: ${{ parameters.RunOnnxRuntimeTests }}
|
|
buildJava: false
|
|
|
|
- template: win-ci.yml
|
|
parameters:
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
job_name_suffix: CPU_arm_${{ parameters.BuildVariant }}
|
|
EnvSetupScript: setup_env.bat
|
|
buildArch: x64
|
|
msbuildPlatform: arm
|
|
packageName: arm
|
|
buildparameter: --arm ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}}
|
|
runTests: false
|
|
buildJava: false
|
|
|
|
- template: win-ci.yml
|
|
parameters:
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
job_name_suffix: CPU_arm64_${{ parameters.BuildVariant }}
|
|
EnvSetupScript: setup_env.bat
|
|
buildArch: x64
|
|
msbuildPlatform: arm64
|
|
packageName: arm64
|
|
buildparameter: --arm64 ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}}
|
|
runTests: false
|
|
buildJava: false
|
|
|
|
- template: win-ci.yml
|
|
parameters:
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
job_name_suffix: CPU_x64_${{ parameters.BuildVariant }}
|
|
EnvSetupScript: setup_env.bat
|
|
buildArch: x64
|
|
msbuildPlatform: x64
|
|
packageName: x64
|
|
buildparameter: ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}}
|
|
runTests: ${{ parameters.RunOnnxRuntimeTests }}
|
|
buildJava: false
|
|
|
|
- job: NuGet_Packaging_CPU
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-CPU-2021'
|
|
variables:
|
|
OrtPackageId: ${{ parameters.OrtNugetPackageId }}
|
|
breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
|
|
dependsOn:
|
|
- Linux_C_API_Packaging_CPU_x64
|
|
- MacOS_C_API_Packaging_CPU_x64
|
|
- Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }}
|
|
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
|
|
- Windows_Packaging_CPU_arm_${{ parameters.BuildVariant }}
|
|
- Windows_Packaging_CPU_arm64_${{ parameters.BuildVariant }}
|
|
condition: succeeded()
|
|
steps:
|
|
- checkout: self
|
|
submodules: true
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet'
|
|
inputs:
|
|
artifactName: 'onnxruntime-win-x64'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet'
|
|
inputs:
|
|
artifactName: 'onnxruntime-win-x86'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet'
|
|
inputs:
|
|
artifactName: 'onnxruntime-win-arm64'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet'
|
|
inputs:
|
|
artifactName: 'onnxruntime-win-arm'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet'
|
|
inputs:
|
|
artifactName: 'onnxruntime-osx-x64'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet'
|
|
inputs:
|
|
artifactName: 'onnxruntime-linux-x64'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet'
|
|
inputs:
|
|
artifactName: 'drop-extra'
|
|
targetPath: '$(Build.BinariesDirectory)/extra-artifact'
|
|
|
|
#Reconstruct the build dir
|
|
- task: PowerShell@2
|
|
displayName: 'PowerShell Script'
|
|
inputs:
|
|
targetType: filePath
|
|
filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files.ps1
|
|
|
|
- script: |
|
|
dir
|
|
workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact'
|
|
displayName: 'List artifacts'
|
|
|
|
- script: |
|
|
mklink /D /J models C:\local\models
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
displayName: 'Create models link'
|
|
|
|
- task: NuGetToolInstaller@0
|
|
displayName: Use Nuget 5.7.0
|
|
inputs:
|
|
versionSpec: 5.7.0
|
|
|
|
- task: DotNetCoreCLI@2
|
|
displayName: 'Restore nuget packages'
|
|
inputs:
|
|
command: restore
|
|
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
|
|
configuration: RelWithDebInfo
|
|
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
|
|
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
|
|
|
- task: DotNetCoreCLI@2
|
|
displayName: 'Build C#'
|
|
inputs:
|
|
command: build
|
|
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
|
|
configuration: RelWithDebInfo
|
|
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
|
|
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
|
|
|
- template: win-esrp-dll.yml
|
|
parameters:
|
|
FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
|
|
DisplayName: 'ESRP - Sign C# dlls'
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
|
|
- task: DotNetCoreCLI@2
|
|
displayName: 'Build Nuget Packages'
|
|
inputs:
|
|
command: build
|
|
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj'
|
|
configuration: RelWithDebInfo
|
|
arguments: '--configuration RelWithDebInfo -t:CreatePackage -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
|
|
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
|
|
|
- task: BatchScript@1
|
|
displayName: 'Bundle Native NuGet and other binaries'
|
|
inputs:
|
|
filename: $(Build.SourcesDirectory)\tools\ci_build\github\windows\bundle_dlls.bat
|
|
workingFolder: $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts
|
|
|
|
- task: CopyFiles@2
|
|
displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
|
|
inputs:
|
|
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts'
|
|
Contents: '*.nupkg'
|
|
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
|
|
|
- task: CopyFiles@2
|
|
displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
|
|
inputs:
|
|
SourceFolder: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
|
|
Contents: '*.nupkg'
|
|
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
|
|
|
- template: esrp_nuget.yml
|
|
parameters:
|
|
DisplayName: 'ESRP - sign NuGet package'
|
|
FolderPath: '$(Build.ArtifactStagingDirectory)'
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
|
|
- template: validate-nuget.yml
|
|
parameters:
|
|
NugetPath: '$(Build.ArtifactStagingDirectory)'
|
|
NugetPackage: 'Microsoft.ML.OnnxRuntime.*nupkg'
|
|
PlatformsSupported: 'win-x64,win-x86,linux-x64,osx-x64'
|
|
VerifyNugetSigning: false
|
|
|
|
- task: PublishPipelineArtifact@0
|
|
displayName: 'Publish Pipeline NuGet Artifact'
|
|
inputs:
|
|
artifactName: 'drop-signed-nuget-CPU'
|
|
targetPath: '$(Build.ArtifactStagingDirectory)'
|
|
|
|
- powershell: |
|
|
$counter = 0
|
|
Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.nupkg | foreach {7z x -y -o$counter $_.fullname;$counter++}
|
|
workingDirectory: '$(Build.ArtifactStagingDirectory)'
|
|
displayName: 'Unzip files to sanitize the PE files'
|
|
|
|
- task: DotNetCoreCLI@2
|
|
displayName: 'Build C#'
|
|
inputs:
|
|
command: custom
|
|
custom: clean
|
|
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
|
|
configuration: RelWithDebInfo
|
|
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
|
|
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
|
|
|
- task: BinSkim@4
|
|
displayName: 'Run BinSkim'
|
|
|
|
- task: RoslynAnalyzers@2
|
|
displayName: 'Run Roslyn Analyzers'
|
|
inputs:
|
|
userProvideBuildInfo: msBuildInfo
|
|
msBuildArchitecture: DotNetCore
|
|
msBuildCommandline: '"C:\Program Files\dotnet\dotnet.exe" build $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln --configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
|
|
condition: and(succeeded(), eq('${{ parameters.DoCompliance }}', true))
|
|
|
|
- template: component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
|
|
- template: ../nuget/templates/test_win.yml
|
|
parameters:
|
|
AgentPool : 'Win-CPU-2021'
|
|
Skipx86Tests : false
|
|
NugetPackageName : 'Microsoft.ML.OnnxRuntime'
|
|
ArtifactSuffix: 'CPU'
|
|
|
|
- template: ../nuget/templates/test_linux.yml
|
|
parameters:
|
|
AgentPool : Linux-CPU
|
|
NugetPackageName : 'Microsoft.ML.OnnxRuntime'
|
|
ArtifactSuffix: 'CPU'
|
|
|
|
- template: ../nuget/templates/test_macos.yml
|
|
parameters:
|
|
AgentPool : macOS-10.15
|
|
ArtifactSuffix: 'CPU'
|