From e45326b5dfec934d3deb9b3fea628d678dbde2c3 Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Mon, 2 Mar 2020 17:00:56 -0800 Subject: [PATCH] Create NuGet packaging pipeline for ORT Featurizers (#3125) Create a new pipeline to publish ORT with Featurizers Update pipeline for two separate packages. Change package names. --- .../nuget/templates/cpu-featurizers.yml | 116 ++++++++++++++++-- 1 file changed, 103 insertions(+), 13 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml index 7fa502fc81..4cd68dddf4 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml @@ -1,6 +1,7 @@ parameters: DoEsrp: 'false' - PackageName: 'Microsoft.ML.OnnxRuntime' + PackageNameNative: 'Microsoft.ML.ORTFeaturizers' + PackageNameManaged: 'Microsoft.ML.ORTFeaturizers.Managed' DoCompliance: 'false' jobs: @@ -9,7 +10,7 @@ jobs: AgentPool : 'Win-CPU-2019' ArtifactName: 'drop-nuget' JobName: 'Windows_CI_Dev' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --use_winml --cmake_generator "Visual Studio 16 2019"' BuildArch: 'x64' EnvSetupScript: 'setup_env.bat' sln_platform: 'x64' @@ -20,6 +21,7 @@ jobs: NuPackScript: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) mkdir $(Build.ArtifactStagingDirectory)\testdata copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata @@ -28,7 +30,7 @@ jobs: AgentPool : 'Win-CPU-2019' ArtifactName: 'drop-win-x86-zip' JobName: 'Windows_CI_Dev_x86' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --x86 --use_telemetry --use_winml --cmake_generator "Visual Studio 16 2019"' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --x86 --use_winml --cmake_generator "Visual Studio 16 2019"' BuildArch: 'x86' EnvSetupScript: 'setup_env_x86.bat' sln_platform: 'Win32' @@ -38,18 +40,80 @@ jobs: DoEsrp: ${{ parameters.DoEsrp }} NuPackScript: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage - cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ + cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\ ren *.nupkg win-x86.zip - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) mkdir $(Build.ArtifactStagingDirectory)\testdata copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata +- job: 'Linux_CI_Dev' + workspace: + clean: all + pool: $(AgentPoolLinux) + steps: + - template: ../../templates/set-version-number-variables-step.yml + - template: ../../templates/linux-set-variables-and-download.yml + - task: CmdLine@2 + inputs: + script: | + docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . + workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker + - task: CmdLine@2 + inputs: + script: | + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --use_featurizers --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + workingDirectory: $(Build.SourcesDirectory) + - script: | + set -e -x + mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 + cd $(Build.BinariesDirectory)/linux-x64 + zip -r linux-x64.zip linux-x64 + cp $(Build.BinariesDirectory)/linux-x64/linux*.zip $(Build.ArtifactStagingDirectory) + mkdir $(Build.ArtifactStagingDirectory)/testdata + cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata + ls -al $(Build.ArtifactStagingDirectory) + displayName: 'Create Artifacts' + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact' + inputs: + artifactName: 'drop-linux' + targetPath: '$(Build.ArtifactStagingDirectory)' + - template: ../../templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - template: ../../templates/clean-agent-build-directory-step.yml + +- template: ../../templates/mac-ci.yml + parameters: + AgentPool : $(AgentPoolMacOS) + JobName: 'MacOS_CI_Dev' + BuildCommand: 'python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --use_featurizers --build_shared_lib --use_openmp --enable_onnx_tests --config RelWithDebInfo' + DoNugetPack : 'true' + NuPackScript: | + set -e -x + mkdir $(Build.BinariesDirectory)/osx-x64 + find $(Build.BinariesDirectory) + cp $(Build.BinariesDirectory)/RelWithDebInfo/libonnxruntime.dylib $(Build.BinariesDirectory)/osx-x64/ + dsymutil $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib -o $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib.dSYM + strip -S -x $(Build.BinariesDirectory)/osx-x64/libonnxruntime.dylib + find $(Build.BinariesDirectory)/osx-x64 -ls + cwd=`pwd` + cd $(Build.BinariesDirectory) + zip -r osx-x64.zip osx-x64 + cp $(Build.BinariesDirectory)/osx-x64.zip $(Build.ArtifactStagingDirectory) + mkdir $(Build.ArtifactStagingDirectory)/testdata + cp $(Build.BinariesDirectory)/RelWithDebInfo/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata + cd $cwd + - job: NuGet_Packaging workspace: clean: all pool: 'Win-CPU-2019' dependsOn: - Windows_CI_Dev + - Windows_CI_Dev_x86 + - Linux_CI_Dev + - MacOS_CI_Dev condition: succeeded() steps: @@ -59,9 +123,25 @@ jobs: artifactName: 'drop-nuget' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - # XXX: This currently bundles only Windows. - # Add linux when linux build is fixed. - - template: bundle_featurizers_dlls.yml + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Win-x86' + inputs: + artifactName: 'drop-win-x86-zip' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Linux' + inputs: + artifactName: 'drop-linux' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - MacOS' + inputs: + artifactName: 'drop-osx' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - template: bundle_dlls.yml - template: ../../templates/esrp_nuget.yml parameters: @@ -72,7 +152,7 @@ jobs: - template: ../../templates/validate-nuget.yml parameters: NugetPath: '$(Build.ArtifactStagingDirectory)' - PlatformsSupported: 'win-x64,win-x86' + PlatformsSupported: 'win-x64,win-x86,linux-x64,osx-x64' VerifyNugetSigning: ${{ parameters.DoEsrp }} - task: PublishPipelineArtifact@0 @@ -81,8 +161,7 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.ArtifactStagingDirectory)' - # XXX: Only windows now -- template: test_win.yml +- template: test_all_os.yml - job: Publish_NuGet_Package_And_Report workspace: @@ -97,11 +176,22 @@ jobs: condition: and (succeeded(), and (${{ parameters.DoEsrp }}, eq(variables['Build.SourceBranch'], 'refs/heads/master'))) dependsOn: - NuGet_Test_Win + - NuGet_Test_Linux + - NuGet_Test_MacOS steps: - task: NuGetCommand@2 - displayName: 'NuGet push to ORT-NIGHTLY' + displayName: 'Copy Signed Native NuGet Package to ORT-NIGHTLY' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme inputs: command: 'push' - packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/Microsoft.ML.OnnxRuntime.$(NuGetPackageVersionNumber).nupkg' + packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/$(PackageNameNative).$(NuGetPackageVersionNumber).nupkg' + publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7' + + - task: NuGetCommand@2 + displayName: 'Copy Signed Managed NuGet Package to ORT-NIGHTLY' + condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme + inputs: + command: 'push' + packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/$(PackageNameManaged).$(NuGetPackageVersionNumber).nupkg' publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7'