From d3e5474c1d4624aefb9f06c710d32fe486e3a0af Mon Sep 17 00:00:00 2001 From: jignparm Date: Thu, 20 Jun 2019 19:41:30 -0700 Subject: [PATCH] Refactor CI pipelines - add GPU NuGet pipelines and ESRP code signing steps (#1247) * Simplify linux gpu pipeline * Refactor win-gpu-ci-pipeline.yml * Set cuda environment variables for testing and version * Remove variables from starter script * minor fix * Add GPU Nuget pipeline * Set DisableContribOps environment variable for Linux package tests * Add ESRP tasks * Add ESRP signing templates * Test out hardcode value of ERSP * Test out hardcode value of ERSP * Test out hardcode value of ERSP * Test out hardcode value of ERSP * test variable expansion * test variable expansion * test variable expansion * test variable expansion * test variable expansion * test variable expansion * test out variable expansion * test variable expansion * test variable expansion * test variable expansion * test variable expansion * test variable expansion * test variable expansion * test variable expansion * test variable expansion * update cpu pipeline to conditionally esrp sign * Set C# GPU tests to run only if env var is set * Refactor for easy parameter passing * refactored esrp templates * remove variables from template * Add packaging variables back to pipelines * update C# for cuda 10 * Merge vars ana parameters for gpu pipeline * remove vars from mklml pipeline * display envvars on terminal * Clean up C# cuda tests, and upgrade to Cuda10 * Introduce CUDNN_PATH pipeline varaible * YAML variable are always uppercased (not true with classic) * Update C# GPU test to be more meaningful * remove macos from gpu tests * remove debugging info for DisableContribOps option * Remove DisableContrib ops parameters -- use variables only * Fix typo from = to - * remove debug steps * fix typo * remove unused variable TESTONGPU from some templates * clean up CUDA env setup scripts * Remove CUDNN_PATH from setup_env_cuda.bat --- .../SessionOptions.cs | 2 +- ...ft.ML.OnnxRuntime.EndToEndTests.Gpu.csproj | 47 ------- .../runtest-docker-gpu.sh | 8 +- .../runtest-docker.sh | 2 +- .../runtest-gpu.bat | 31 ----- .../runtest-gpu.sh | 45 ------- .../runtest.bat | 6 +- .../InferenceTest.cs | 12 +- .../azure-pipelines/linux-gpu-ci-pipeline.yml | 46 +------ .../nuget/cpu-esrp-pipeline.yml | 11 ++ .../nuget/cpu-mklml-esrp-pipeline.yml | 14 ++ .../nuget/cpu-mklml-pipeline.yml | 14 ++ .../nuget/cpu-nocontribops-esrp-pipeline.yml | 14 ++ .../nuget/cpu-nocontribops-pipeline.yml | 14 ++ .../azure-pipelines/nuget/cpu-pipeline.yml | 11 ++ .../nuget/gpu-esrp-pipeline.yml | 15 +++ .../azure-pipelines/nuget/gpu-pipeline.yml | 15 +++ .../cpu-mklml.yml} | 19 ++- .../cpu-nocontribops.yml} | 26 ++-- .../cpu.yml} | 26 +++- .../azure-pipelines/nuget/templates/gpu.yml | 109 ++++++++++++++++ .../nuget/templates/test_linux.yml | 2 +- .../azure-pipelines/templates/esrp_dll.yml | 45 +++++++ .../azure-pipelines/templates/esrp_nuget.yml | 31 +++++ .../azure-pipelines/templates/linux-ci.yml | 1 - .../azure-pipelines/templates/win-ci.yml | 30 ++++- .../azure-pipelines/templates/win-x86-ci.yml | 14 ++ .../azure-pipelines/win-gpu-ci-pipeline.yml | 120 +++--------------- .../github/windows/setup_env_cuda.bat | 2 +- 29 files changed, 425 insertions(+), 307 deletions(-) delete mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.Gpu.csproj delete mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-gpu.bat delete mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-gpu.sh create mode 100644 tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-esrp-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml rename tools/ci_build/github/azure-pipelines/nuget/{cpu_win_linux_mac_x86_mklml.yml => templates/cpu-mklml.yml} (93%) rename tools/ci_build/github/azure-pipelines/nuget/{cpu_win_linux_mac_nocontribops.yml => templates/cpu-nocontribops.yml} (91%) rename tools/ci_build/github/azure-pipelines/nuget/{cpu_win_linux_mac_x86.yml => templates/cpu.yml} (90%) create mode 100644 tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml create mode 100644 tools/ci_build/github/azure-pipelines/templates/esrp_dll.yml create mode 100644 tools/ci_build/github/azure-pipelines/templates/esrp_nuget.yml diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs index 6ef6c60345..4ce708687e 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs @@ -14,7 +14,7 @@ namespace Microsoft.ML.OnnxRuntime { public IntPtr _nativePtr; protected static readonly Lazy _default = new Lazy(MakeSessionOptionWithCpuProvider); - private static string[] cudaDelayLoadedLibs = { "cublas64_91.dll", "cudnn64_7.dll" }; + private static string[] cudaDelayLoadedLibs = { "cublas64_100.dll", "cudnn64_7.dll" }; /// /// Constructs an empty SessionOptions diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.Gpu.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.Gpu.csproj deleted file mode 100644 index b39677bce6..0000000000 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.Gpu.csproj +++ /dev/null @@ -1,47 +0,0 @@ - - - - netcoreapp2.0 - win-x64 - false - $(MSBuildThisFileDirectory)..\.. - AnyCPU - bin\$(Configuration)\ - $(OnnxRuntimeCsharpRoot)\..\build\Windows - $(OnnxRuntimeBuildDirectory)\$(Configuration)\external\protobuf\cmake\$(Configuration) - $(OnnxRuntimeCsharpRoot)\..\cmake\external\onnx\onnx - $(OnnxRuntimeBuildDirectory)\$(Configuration)\$(Configuration) - - - - - - - - - - - - - - - - - Always - false - - - - - - - - - - - - - - - - diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh index ab50111bcc..fc0ea19ea5 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh @@ -9,11 +9,12 @@ set -x SOURCE_ROOT=$1 BUILD_DIR=$2 NUGET_REPO_DIRNAME=$3 # path relative to BUILD_DIR -#CUDA_VER=cuda10.0-cudnn7.3, cuda9.1-cudnn7.1 -CUDA_VER=${4:-cuda9.1-cudnn7.1} +PackageName=${PackageName:-Microsoft.ML.OnnxRuntime} +#CUDA_VER=cuda10.0-cudnn7.3, cuda9.1-cudnn7.1, cuda10.0-cudnn7.3 +CUDA_VER=${4:-cuda10.0-cudnn7.3} -IMAGE="ubuntu16.04-$CUDA_VER" PYTHON_VER=3.5 +IMAGE="ubuntu16.04-$CUDA_VER" OldDir=$(pwd) cd $SOURCE_ROOT/tools/ci_build/github/linux/docker @@ -37,6 +38,7 @@ docker run -h $HOSTNAME \ --volume "$HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime" \ -e "OnnxRuntimeBuildDirectory=/home/onnxruntimedev" \ -e "IsReleaseBuild=$IsReleaseBuild" \ + -e "PackageName=$PackageName" \ "onnxruntime-$IMAGE" \ /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-gpu.sh \ /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev & diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh index 63ec87585b..dd6bd136cd 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh @@ -38,7 +38,7 @@ docker run -h $HOSTNAME \ -e "OnnxRuntimeBuildDirectory=/home/onnxruntimedev" \ -e "IsReleaseBuild=$IsReleaseBuild" \ -e "PackageName=$PackageName" \ - -e "DisableContribOps=$DisableContribOps" \ + -e "DisableContribOps=$DISABLECONTRIBOPS" \ -e "RunTestCsharp=$RunTestCsharp" \ -e "RunTestNative=$RunTestNative" \ "onnxruntime-$IMAGE" \ diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-gpu.bat b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-gpu.bat deleted file mode 100644 index e10170ebaa..0000000000 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-gpu.bat +++ /dev/null @@ -1,31 +0,0 @@ -REM Copyright (c) Microsoft Corporation. All rights reserved. -REM Licensed under the MIT License. - -@echo off -SETLOCAL EnableDelayedExpansion - -set LocalNuGetRepo=%1 -IF "%2"=="" (SET TargetFramework=netcoreapp2.1) ELSE (SET TargetFramework=%2) -ECHO Target Framework is %TargetFramework% -REM WorkingDirectory is Build.SourcesDirectory\csharp -set /p MajorVersionNumber=<..\VERSION_NUMBER -set VersionSuffix= -IF NOT DEFINED IsReleaseBuild ( - FOR /F "tokens=* USEBACKQ" %%F IN (`git rev-parse --short HEAD`) DO ( - set VersionSuffix=-dev-%%F - ) -) - -set CurrentOnnxRuntimeVersion=%MajorVersionNumber%%VersionSuffix% -@echo %CurrentOnnxRuntimeVersion% -dotnet restore test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.Gpu.csproj -s %LocalNuGetRepo% --configfile .\Nuget.CSharp.config -if NOT errorlevel 0 ( - @echo "Failed to restore nuget packages for the test project" - Exit 1 -) - -dotnet test test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore -if NOT errorlevel 0 ( - @echo "Failed to build or execute the end-to-end test" - Exit 1 -) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-gpu.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-gpu.sh deleted file mode 100644 index 9fc2f9b34e..0000000000 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-gpu.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -LocalNuGetRepo=$1 -SourceRoot=$2 -BuildDir=$3 -#TestDataUrl=$4 -#TestDataChecksum=$5 - -#TestDataUrl and TestDataChecksum comes from the build env variable - -#echo "Downloading test data" -#python3 $SourceRoot/tools/ci_build/build.py --update --download_test_data --build_dir $BuildDir --test_data_url $TestDataUrl --test_data_checksum $TestDataChecksum -#if [ $? -ne 0 ]; then -# echo "Failed to download test data" -# exit 1 -#fi - -set -x - -OldDir=`pwd` -cd $SourceRoot - -MajorVersion=$(cat $SourceRoot/VERSION_NUMBER) -VersionSuffix= -if [ "$IsReleaseBuild" != "true" ]; then - VersionSuffix=-dev-$(git rev-parse --short=8 HEAD) -fi -export CurrentOnnxRuntimeVersion=$MajorVersion$VersionSuffix -echo "Current NuGet package version is $CurrentOnnxRuntimeVersion" - -dotnet restore $SourceRoot/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.Gpu.csproj -s $LocalNuGetRepo -s https://api.nuget.org/v3/index.json -if [ $? -ne 0 ]; then - echo "Failed to restore nuget packages for the test project" - exit 1 -fi - -dotnet test $SourceRoot/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.Gpu.csproj --no-restore -if [ $? -ne 0 ]; then - echo "Failed to build or execute the end-to-end test" - exit 1 -fi - -cd $OldDir diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat index 154d44f32b..08c335113e 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat @@ -30,8 +30,12 @@ IF NOT DEFINED IsReleaseBuild ( SET CurrentOnnxRuntimeVersion=%MajorVersionNumber%%VersionSuffix% +REM Update if CUDA lib paths if set +SET PATH=%CUDA_PATH%\bin;%CUDNN_PATH%\bin;%PATH% + @echo %CurrentOnnxRuntimeVersion% -%dn% restore test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj -s %LocalNuGetRepo% --configfile .\Nuget.CSharp.config +%dn% restore test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --configfile .\Nuget.CSharp.config --no-cache --packages test\Microsoft.ML.OnnxRuntime.EndToEndTests --source https://api.nuget.org/v3/index.json --source %LocalNuGetRepo% + IF NOT errorlevel 0 ( @echo "Failed to restore nuget packages for the test project" EXIT 1 diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index d739414b8b..31c0ee6743 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -661,15 +661,18 @@ namespace Microsoft.ML.OnnxRuntime.Tests { var gpu = Environment.GetEnvironmentVariable("TESTONGPU"); var tuple = OpenSessionSqueezeNet(0); // run on deviceID 0 + float[] expectedOutput = LoadTensorFromFile(@"bench.expected_out"); + using (var session = tuple.Item1) { var inputData = tuple.Item2; var tensor = tuple.Item3; var inputMeta = session.InputMetadata; var container = new List(); - container.Add(NamedOnnxValue.CreateFromTensor("input", tensor)); - var ex = Assert.Throws(() => session.Run(container)); - Assert.Contains("Missing Input", ex.Message); + container.Add(NamedOnnxValue.CreateFromTensor("data_0", tensor)); + var res = session.Run(container); + var resultArray = res.First().AsTensor().ToArray(); + Assert.Equal(expectedOutput, resultArray, new floatComparer()); } } @@ -779,7 +782,8 @@ namespace Microsoft.ML.OnnxRuntime.Tests { public GpuFact() { - if (System.Environment.GetEnvironmentVariable("TESTONGPU") == null) + var testOnGpu = System.Environment.GetEnvironmentVariable("TESTONGPU"); + if (testOnGpu == null || !testOnGpu.Equals("ON") ) { Skip = "GPU testing not enabled"; } diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index c19f646e6d..cbadf2c4e0 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -1,40 +1,8 @@ jobs: -- job: Linux_CI_GPU_Dev - pool: Linux-GPU-CUDA10 - steps: - - template: templates/set-test-data-variables-step.yml - - - task: CmdLine@2 - displayName: 'Clean untagged docker images' - inputs: - script: | - docker container prune -f - docker image prune -f - workingDirectory: $(Build.BinariesDirectory) - continueOnError: true - condition: always() - - - task: CmdLine@2 - displayName: 'Download azcopy' - inputs: - script: | - curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux' - tar -zxvf azcopy.tar.gz --strip 1 - workingDirectory: $(Build.BinariesDirectory) - - - task: PythonScript@0 - displayName: 'Download test data' - inputs: - scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) - pythonInterpreter: '/usr/bin/python3' - workingDirectory: $(Build.BinariesDirectory) - - - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -x "--build_wheel"' - displayName: 'Command Line Script' - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) - - - template: templates/clean-agent-build-directory-step.yml +- template: templates/linux-ci.yml + parameters: + AgentPool : 'Linux-GPU-CUDA10' + JobName: 'Linux_CI_GPU_Dev' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -x "--build_wheel"' + DoNugetPack: 'false' + ArtifactName: 'drop-linux' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml new file mode 100644 index 0000000000..023393d91b --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-pipeline.yml @@ -0,0 +1,11 @@ +# Defined as pipeline variables +# variables: +# AgentPoolWin : 'Win-CPU' +# AgentPoolLinux : 'Linux-CPU' +# AgentPoolMacOS : 'macOS-10.13' + +jobs: +- template: templates/cpu.yml + parameters: + AgentPool : $(AgentPoolWin) + DoEsrp: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml new file mode 100644 index 0000000000..335540163c --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-esrp-pipeline.yml @@ -0,0 +1,14 @@ +# Defined as pipeline variables +# variables: +# AgentPoolWin : 'Win-CPU' +# AgentPoolLinux : 'Linux-CPU' +# AgentPoolMacOS : 'macOS-10.13' + +variables: + PackageName: 'Microsoft.ML.OnnxRuntime.MKLML' + +jobs: +- template: templates/cpu-mklml.yml + parameters: + AgentPool : $(AgentPoolWin) + DoEsrp: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml new file mode 100644 index 0000000000..135469669c --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-mklml-pipeline.yml @@ -0,0 +1,14 @@ +# Defined as pipeline variables +# variables: +# AgentPoolWin : 'Win-CPU' +# AgentPoolLinux : 'Linux-CPU' +# AgentPoolMacOS : 'macOS-10.13' + +variables: + PackageName: 'Microsoft.ML.OnnxRuntime.MKLML' + +jobs: +- template: templates/cpu-mklml.yml + parameters: + AgentPool : $(AgentPoolWin) + DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-esrp-pipeline.yml new file mode 100644 index 0000000000..4f3a8ed7c7 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-esrp-pipeline.yml @@ -0,0 +1,14 @@ +# Defined as pipeline variables +# variables: +# AgentPoolWin : 'Win-CPU' +# AgentPoolLinux : 'Linux-CPU' +# AgentPoolMacOS : 'macOS-10.13' + +variables: + DisableContribOps: 'ON' + +jobs: +- template: templates/cpu-nocontribops.yml + parameters: + AgentPool : $(AgentPoolWin) + DoEsrp: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-pipeline.yml new file mode 100644 index 0000000000..2e7b341681 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-nocontribops-pipeline.yml @@ -0,0 +1,14 @@ +# Defined as pipeline variables +# variables: +# AgentPoolWin : 'Win-CPU' +# AgentPoolLinux : 'Linux-CPU' +# AgentPoolMacOS : 'macOS-10.13' + +variables: + DisableContribOps: 'ON' + +jobs: +- template: templates/cpu-nocontribops.yml + parameters: + AgentPool : $(AgentPoolWin) + DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml new file mode 100644 index 0000000000..a1baef7439 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-pipeline.yml @@ -0,0 +1,11 @@ +# Defined as pipeline variables +# variables: +# AgentPoolWin : 'Win-CPU' +# AgentPoolLinux : 'Linux-CPU' +# AgentPoolMacOS : 'macOS-10.13' + +jobs: +- template: templates/cpu.yml + parameters: + AgentPool : $(AgentPoolWin) + DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml new file mode 100644 index 0000000000..bfd6957159 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml @@ -0,0 +1,15 @@ +# Defined as pipeline variables +# variables: +# AgentPoolWin : 'Win-CPU-CUDA10' +# AgentPoolLinux : 'Linux-CPU' +# AgentPoolMacOS : 'macOS-10.13' + +variables: + PackageName: 'Microsoft.ML.OnnxRuntime.Gpu' + TESTONGPU: 'ON' + +jobs: +- template: templates/gpu.yml + parameters: + AgentPool : $(AgentPoolWin) + DoEsrp: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml new file mode 100644 index 0000000000..ef040845ee --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml @@ -0,0 +1,15 @@ +# Defined as pipeline variables +# variables: +# AgentPoolWin : 'Win-CPU-CUDA10' +# AgentPoolLinux : 'Linux-CPU' +# AgentPoolMacOS : 'macOS-10.13' + +variables: + PackageName: 'Microsoft.ML.OnnxRuntime.Gpu' + TESTONGPU: 'ON' + +jobs: +- template: templates/gpu.yml + parameters: + AgentPool : $(AgentPoolWin) + DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu_win_linux_mac_x86_mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml similarity index 93% rename from tools/ci_build/github/azure-pipelines/nuget/cpu_win_linux_mac_x86_mklml.yml rename to tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml index 9d46d771d0..1d7c5d1f30 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu_win_linux_mac_x86_mklml.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml @@ -4,11 +4,11 @@ # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' -variables: - PackageName: 'Microsoft.ML.OnnxRuntime.MKLML' +parameters: + DoEsrp: 'false' jobs: -- template: ../templates/win-ci.yml +- template: ../../templates/win-ci.yml parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' @@ -16,11 +16,12 @@ jobs: DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' + DoEsrp: ${{ parameters.DoEsrp }} NuPackScript: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:PackageId=Microsoft.ML.OnnxRuntime.MKLML copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) -- template: ../templates/linux-ci.yml +- template: ../../templates/linux-ci.yml parameters: AgentPool : $(AgentPoolLinux) JobName: 'Linux_CI_Dev' @@ -40,7 +41,7 @@ jobs: cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) ls -al $(Build.ArtifactStagingDirectory) -- template: ../templates/mac-ci.yml +- template: ../../templates/mac-ci.yml parameters: AgentPool : $(AgentPoolMacOS) JobName: 'MacOS_CI_Dev' @@ -129,10 +130,16 @@ jobs: copy $(Build.BinariesDirectory)\nuget-artifact\*.nupkg $(Build.ArtifactStagingDirectory) displayName: 'Bundle NuGet and other binaries' + - template: ../../templates/esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' + DoEsrp: ${{ parameters.DoEsrp }} + - task: PublishPipelineArtifact@0 displayName: 'Publish Pipeline NuGet Artifact' inputs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.ArtifactStagingDirectory)' -- template: templates/test_all_os.yml +- template: test_all_os.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu_win_linux_mac_nocontribops.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml similarity index 91% rename from tools/ci_build/github/azure-pipelines/nuget/cpu_win_linux_mac_nocontribops.yml rename to tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml index de0475b55e..114e632221 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu_win_linux_mac_nocontribops.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops.yml @@ -4,11 +4,11 @@ # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' -variables: - DisableContribOps: 'ON' +parameters: + DoEsrp: 'false' jobs: -- template: ../templates/win-ci.yml +- template: ../../templates/win-ci.yml parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' @@ -16,12 +16,13 @@ jobs: DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' + DoEsrp: ${{ parameters.DoEsrp }} NuPackScript: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) -- template: ../templates/win-x86-ci.yml +- template: ../../templates/win-x86-ci.yml parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev_x86' @@ -29,13 +30,14 @@ jobs: DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' + DoEsrp: ${{ parameters.DoEsrp }} NuPackScript: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ ren *.nupkg win-x86.zip copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) -- template: ../templates/linux-ci.yml +- template: ../../templates/linux-ci.yml parameters: AgentPool : $(AgentPoolLinux) JobName: 'Linux_CI_Dev' @@ -51,7 +53,7 @@ jobs: cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) ls -al $(Build.ArtifactStagingDirectory) -- template: ../templates/linux-ci.yml +- template: ../../templates/linux-ci.yml parameters: AgentPool : $(AgentPoolLinux) JobName: 'Linux_CI_Dev_x86' @@ -67,7 +69,7 @@ jobs: cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) ls -al $(Build.ArtifactStagingDirectory) -- template: ../templates/mac-ci.yml +- template: ../../templates/mac-ci.yml parameters: AgentPool : $(AgentPoolMacOS) JobName: 'MacOS_CI_Dev' @@ -132,7 +134,13 @@ jobs: targetPath: '$(Build.BinariesDirectory)/nuget-artifact' continueOnError: true - - template: templates/bundle_dlls.yml + - template: bundle_dlls.yml + + - template: ../../templates/esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' + DoEsrp: ${{ parameters.DoEsrp }} - task: PublishPipelineArtifact@0 displayName: 'Publish Pipeline NuGet Artifact' @@ -140,4 +148,4 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.ArtifactStagingDirectory)' -- template: templates/test_all_os.yml +- template: test_all_os.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu_win_linux_mac_x86.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml similarity index 90% rename from tools/ci_build/github/azure-pipelines/nuget/cpu_win_linux_mac_x86.yml rename to tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 08c5fed586..75caa0fc1d 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu_win_linux_mac_x86.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -4,8 +4,11 @@ # AgentPoolLinux : 'Linux-CPU' # AgentPoolMacOS : 'macOS-10.13' +parameters: + DoEsrp: 'false' + jobs: -- template: ../templates/win-ci.yml +- template: ../../templates/win-ci.yml parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev' @@ -13,12 +16,13 @@ jobs: DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' + DoEsrp: ${{ parameters.DoEsrp }} NuPackScript: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) -- template: ../templates/win-x86-ci.yml +- template: ../../templates/win-x86-ci.yml parameters: AgentPool : $(AgentPoolWin) JobName: 'Windows_CI_Dev_x86' @@ -26,13 +30,14 @@ jobs: DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' + DoEsrp: ${{ parameters.DoEsrp }} NuPackScript: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage cd $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\ ren *.nupkg win-x86.zip copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*zip $(Build.ArtifactStagingDirectory) -- template: ../templates/linux-ci.yml +- template: ../../templates/linux-ci.yml parameters: AgentPool : $(AgentPoolLinux) JobName: 'Linux_CI_Dev' @@ -48,7 +53,7 @@ jobs: cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) ls -al $(Build.ArtifactStagingDirectory) -- template: ../templates/linux-ci.yml +- template: ../../templates/linux-ci.yml parameters: AgentPool : $(AgentPoolLinux) JobName: 'Linux_CI_Dev_x86' @@ -64,7 +69,7 @@ jobs: cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) ls -al $(Build.ArtifactStagingDirectory) -- template: ../templates/mac-ci.yml +- template: ../../templates/mac-ci.yml parameters: AgentPool : $(AgentPoolMacOS) JobName: 'MacOS_CI_Dev' @@ -129,7 +134,14 @@ jobs: targetPath: '$(Build.BinariesDirectory)/nuget-artifact' continueOnError: true - - template: templates/bundle_dlls.yml + - template: bundle_dlls.yml + + - template: ../../templates/esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' + DoEsrp: ${{ parameters.DoEsrp }} + - task: PublishPipelineArtifact@0 displayName: 'Publish Pipeline NuGet Artifact' @@ -137,4 +149,4 @@ jobs: artifactName: 'drop-signed-nuget' targetPath: '$(Build.ArtifactStagingDirectory)' -- template: templates/test_all_os.yml +- template: test_all_os.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml new file mode 100644 index 0000000000..20610d93bd --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -0,0 +1,109 @@ +# Defined as pipeline variables +# variables: +# AgentPoolWin : 'Win-CPU-CUDA10' +# AgentPoolLinux : 'Linux-CPU' +# AgentPoolMacOS : 'macOS-10.13' + +parameters: + DoEsrp: 'false' + PackageName: 'Microsoft.ML.OnnxRuntime.Gpu' + +jobs: +- template: ../../templates/win-ci.yml + parameters: + AgentPool : $(AgentPoolWin) + JobName: 'Windows_CI_GPU_Dev' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --msvc_toolset=14.11' + DoDebugBuild: 'false' + DoNugetPack : 'true' + DoCompliance: 'false' + DoEsrp: ${{ parameters.DoEsrp }} + BuildArch: 'amd64' + SetVcvars: 'true' + MsbuildArguments: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\' + EnvSetupScript: 'setup_env_cuda.bat' + CudaVersion: '10.0' + NuPackScript: | + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:PackageId=Microsoft.ML.OnnxRuntime.Gpu + copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + + +- template: ../../templates/linux-ci.yml + parameters: + AgentPool : $(AgentPoolLinux) + JobName: 'Linux_CI_GPU_Dev' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -x ""' + DoNugetPack : 'true' + ArtifactName: 'drop-linux' + NuPackScript: | + set -e -x + mkdir $(Build.BinariesDirectory)/linux-x64 + cp $(Build.BinariesDirectory)/Release/libonnxruntime.so $(Build.BinariesDirectory)/linux-x64 + cd $(Build.BinariesDirectory) + zip -r linux-x64.zip linux-x64 + cp $(Build.BinariesDirectory)/*.zip $(Build.ArtifactStagingDirectory) + ls -al $(Build.ArtifactStagingDirectory) + + +- job: NuGet_Packaging + pool: $(AgentPoolWin) + dependsOn: + - Windows_CI_GPU_Dev + - Linux_CI_GPU_Dev + condition: succeeded() + steps: + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'drop-nuget' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - Linux' + inputs: + artifactName: 'drop-linux' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + continueOnError: true + + - script: | + pushd $(Build.BinariesDirectory)\nuget-artifact + dir + powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe" + powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe" + set PATH=%CD%;%PATH% + FOR /R %%i IN (*.nupkg) do ( + rename %%~ni.nupkg %%~ni.zip + unzip %%~ni.zip -d %%~ni + del /Q %%~ni.zip + unzip linux-x64.zip -d linux-x64 + mkdir %%~ni\runtimes\linux-x64 + mkdir %%~ni\runtimes\linux-x64\native + move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so + pushd %%~ni + zip -r ..\%%~ni.zip . + popd + move %%~ni.zip %%~ni.nupkg + ) + popd + copy $(Build.BinariesDirectory)\nuget-artifact\*.nupkg $(Build.ArtifactStagingDirectory) + displayName: 'Bundle NuGet and other binaries' + + - template: ../../templates/esrp_nuget.yml + parameters: + DisplayName: 'ESRP - sign NuGet package' + FolderPath: '$(Build.BinariesDirectory)/nuget-artifact' + DoEsrp: ${{ parameters.DoEsrp }} + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline NuGet Artifact' + inputs: + artifactName: 'drop-signed-nuget' + targetPath: '$(Build.ArtifactStagingDirectory)' + +- template: test_win.yml + parameters: + AgentPool : $(AgentPoolWin) +- template: test_linux.yml + parameters: + AgentPool : $(AgentPoolLinux) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index ce7b6456c9..1eabf3fdf8 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -1,5 +1,5 @@ parameters: - AgentPool : 'Linux-CPU' + AgentPool: 'Linux-CPU' jobs: - job: NuGet_Test_Linux diff --git a/tools/ci_build/github/azure-pipelines/templates/esrp_dll.yml b/tools/ci_build/github/azure-pipelines/templates/esrp_dll.yml new file mode 100644 index 0000000000..4185d4a80d --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/esrp_dll.yml @@ -0,0 +1,45 @@ +parameters: + FolderPath: '' + DisplayName: '' + DoEsrp: 'false' + +steps: +- ${{ if eq(parameters['DoEsrp'], 'true') }}: + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ${{ parameters.DisplayName }} + inputs: + ConnectedServiceName: 'OnnxRuntime CodeSign 20171127' + FolderPath: ${{ parameters.FolderPath }} + Pattern: '*.dll' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "signtool.exe", + "toolVersion": "6.2.9304.0" + } + ] diff --git a/tools/ci_build/github/azure-pipelines/templates/esrp_nuget.yml b/tools/ci_build/github/azure-pipelines/templates/esrp_nuget.yml new file mode 100644 index 0000000000..876a8ee3ac --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/esrp_nuget.yml @@ -0,0 +1,31 @@ +parameters: + FolderPath: '' + DisplayName: '' + DoEsrp: 'false' + +steps: +- ${{ if eq(parameters['DoEsrp'], 'true') }}: + - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 + displayName: ${{ parameters.DisplayName }} + inputs: + ConnectedServiceName: 'OnnxRuntime CodeSign 20171127' + FolderPath: ${{ parameters.FolderPath }} + Pattern: '*.nupkg' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetSign", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetVerify", + "parameters": [ ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml index 17d6d2b01b..74351e86d4 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml @@ -4,7 +4,6 @@ parameters: BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_tvm --build_wheel"' DoNugetPack: 'false' NuPackScript: '' - ZipDir : 'linux-x64' ArtifactName: 'drop-linux' jobs: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index fee15babcb..8ca6418322 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -6,7 +6,13 @@ parameters: JobName: 'Windows_CI_Dev' DoNugetPack: 'false' NuPackScript : '' + DoEsrp: 'false' DoTestCoverage: 'false' + BuildArch: 'x64' + SetVcvars: 'false' + MsbuildArguments: '/m' + EnvSetupScript: 'setup_env.bat' + CudaVersion: '' jobs: - job: ${{ parameters.JobName }} @@ -17,14 +23,15 @@ jobs: BuildCommand: ${{ parameters.BuildCommand }} OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' DotNetExe: 'dotnet.exe' + CUDA_VERSION: ${{ parameters.CudaVersion }} steps: - template: set-test-data-variables-step.yml - template: windows-build-tools-setup-steps.yml parameters: - EnvSetupScript: 'setup_env.bat' - buildArch: 'x64' - setVcvars: false + EnvSetupScript: ${{ parameters.EnvSetupScript }} + buildArch: ${{ parameters.BuildArch }} + setVcvars: ${{ parameters.SetVcvars }} - task: CmdLine@1 displayName: 'Download test data and generate cmake config' @@ -41,7 +48,7 @@ jobs: solution: '$(Build.BinariesDirectory)\Debug\onnxruntime.sln' platform: 'x64' configuration: 'Debug' - msbuildArguments: '/m' + msbuildArguments: ${{ parameters.MsbuildArguments }} msbuildArchitecture: 'x64' logProjectEvents: true workingFolder: '$(Build.BinariesDirectory)\Debug' @@ -88,7 +95,7 @@ jobs: solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' platform: 'x64' configuration: 'RelWithDebInfo' - msbuildArguments: '/m' + msbuildArguments: ${{ parameters.MsbuildArguments }} msbuildArchitecture: 'x64' logProjectEvents: true workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' @@ -135,6 +142,19 @@ jobs: arguments: 'x64 -vcvars_ver=14.11' modifyEnvironment: true + # Esrp signing + - template: esrp_dll.yml + parameters: + FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo' + DisplayName: 'ESRP - Sign Native dlls' + DoEsrp: ${{ parameters.DoEsrp }} + + - template: esrp_dll.yml + parameters: + FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo' + DisplayName: 'ESRP - Sign C# dlls' + DoEsrp: ${{ parameters.DoEsrp }} + - script: | ${{ parameters.NuPackScript }} workingDirectory: '$(Build.SourcesDirectory)\csharp' diff --git a/tools/ci_build/github/azure-pipelines/templates/win-x86-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-x86-ci.yml index 5a91d5d2e6..43fc4693a3 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-x86-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-x86-ci.yml @@ -6,6 +6,7 @@ parameters: JobName: 'Windows_CI_Dev_x86' DoNugetPack: 'false' NuPackScript : '' + DoEsrp: 'false' jobs: - job: ${{ parameters.JobName }} @@ -131,6 +132,19 @@ jobs: arguments: 'x86 -vcvars_ver=14.11' modifyEnvironment: true + # Esrp signing + - template: esrp_dll.yml + parameters: + FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo' + DisplayName: 'ESRP - Sign Native dlls' + DoEsrp: ${{ parameters.DoEsrp }} + + - template: esrp_dll.yml + parameters: + FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo' + DisplayName: 'ESRP - Sign C# dlls' + DoEsrp: ${{ parameters.DoEsrp }} + - script: | ${{ parameters.NuPackScript }} workingDirectory: '$(Build.SourcesDirectory)\csharp' diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index 078d954a98..d4930faf19 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -1,106 +1,16 @@ jobs: -- job: Windows_CI_GPU_Dev - variables: - buildDirectory: '$(Build.BinariesDirectory)' - CUDA_VERSION: '10.0' - steps: - - template: templates/set-test-data-variables-step.yml - - template: templates/windows-build-tools-setup-steps.yml - parameters: - EnvSetupScript: 'setup_env.bat' - buildArch: 'amd64' - setVcvars: true - - - task: CmdLine@1 - displayName: 'Download test data and generate cmake config' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11' - workingDirectory: "$(Build.BinariesDirectory)" - - - task: VSBuild@1 - displayName: 'Build Debug' - inputs: - solution: '$(Build.BinariesDirectory)\Debug\onnxruntime.sln' - platform: 'x64' - configuration: 'Debug' - msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\' - msbuildArchitecture: 'x64' - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\Debug' - - task: BatchScript@1 - displayName: 'Test Debug' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' - workingFolder: '$(Build.BinariesDirectory)' - - task: VSBuild@1 - displayName: 'Build C# Debug' - inputs: - solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - configuration: 'Debug' - platform: 'any cpu' - restoreNugetPackages: false - msbuildArchitecture: 'x64' - workingFolder: '$(Build.SourcesDirectory)\csharp' - msbuildArgs: '/m /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory)' - - - task: VSTest@2 - displayName: 'VsTest - C# Debug' - inputs: - testAssemblyVer2: '**\bin\Debug\**\*Tests.dll' - searchFolder: '$(Build.SourcesDirectory)\csharp\test' - runInParallel: true - configuration: Debug - - - task: VSBuild@1 - displayName: 'Build Release' - inputs: - solution: '$(Build.BinariesDirectory)\Release\onnxruntime.sln' - platform: 'x64' - configuration: 'Release' - msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\' - msbuildArchitecture: 'x64' - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\Release' - - - task: BatchScript@1 - displayName: 'Test Release' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --build_shared_lib --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test' - workingFolder: "$(Build.BinariesDirectory)" - - - task: VSBuild@1 - displayName: 'Build c# Release' - inputs: - solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - platform: 'any cpu' - configuration: 'Release' - msbuildArchitecture: 'x64' - restoreNugetPackages: false - workingFolder: '$(Build.SourcesDirectory)\csharp' - msbuildArgs: '/m /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory)' - - - task: VSTest@2 - displayName: 'VsTest - C# Release' - inputs: - testAssemblyVer2: '**\bin\Release\**\*Tests.dll' - searchFolder: '$(Build.SourcesDirectory)\csharp\test' - runInParallel: true - configuration: Release - - - task: PublishTestResults@2 - displayName: 'Publish unit test results' - inputs: - testResultsFiles: '**\*.results.xml' - searchFolder: '$(Build.BinariesDirectory)' - testRunTitle: 'Unit Test Run' - condition: succeededOrFailed() - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) - - - template: templates/clean-agent-build-directory-step.yml - +- template: templates/win-ci.yml + parameters: + AgentPool : 'Win-GPU-CUDA10' + DoDebugBuild: 'true' + DoCompliance: 'false' + BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --use_mkldnn --use_mkldnn --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --msvc_toolset=14.11' + JobName: 'Windows_CI_GPU_Dev' + DoNugetPack: 'false' + NuPackScript : '' + DoTestCoverage: 'false' + BuildArch: 'amd64' + SetVcvars: 'true' + MsbuildArguments: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\' + EnvSetupScript: 'setup_env_cuda.bat' + CudaVersion: '10.0' diff --git a/tools/ci_build/github/windows/setup_env_cuda.bat b/tools/ci_build/github/windows/setup_env_cuda.bat index af2a3d343d..7f87a11990 100644 --- a/tools/ci_build/github/windows/setup_env_cuda.bat +++ b/tools/ci_build/github/windows/setup_env_cuda.bat @@ -1 +1 @@ -set PATH=%BUILD_BINARIESDIRECTORY%\packages\python;%BUILD_BINARIESDIRECTORY%\packages\python\DLLs;%BUILD_BINARIESDIRECTORY%\packages\python\Library\bin;%BUILD_BINARIESDIRECTORY%\packages\python\script;C:\local\cuda_10.0.130_win10\bin;%PATH% +set PATH=%BUILD_BINARIESDIRECTORY%\packages\python;%BUILD_BINARIESDIRECTORY%\packages\python\DLLs;%BUILD_BINARIESDIRECTORY%\packages\python\Library\bin;%BUILD_BINARIESDIRECTORY%\packages\python\script;C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda\bin;C:\local\cuda_10.0.130_win10\bin;%PATH%