From 2cfee5744b6b2c5d1f7eb96dc1b391255b9abc04 Mon Sep 17 00:00:00 2001
From: Ori Levari <ori.levari@microsoft.com>
Date: Wed, 27 Nov 2019 11:36:20 -0800
Subject: [PATCH] Layer dev release pipeline (#2488)

Adds winml binaries to existing cpu nuget package, and creates new gpu dml nuget package with winml binaries and DML EP.
---
 cmake/winml_unittests.cmake                   |  2 ++
 .../Microsoft.ML.OnnxRuntime.csproj           | 22 +++++++++++++
 .../azure-pipelines/nuget/templates/cpu.yml   |  4 +--
 .../azure-pipelines/nuget/templates/gpu.yml   | 31 +++++++++++++++++--
 4 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/cmake/winml_unittests.cmake b/cmake/winml_unittests.cmake
index cd89135366..d078465c55 100644
--- a/cmake/winml_unittests.cmake
+++ b/cmake/winml_unittests.cmake
@@ -55,6 +55,8 @@ endfunction()
 
 file(GLOB winml_test_common_src CONFIGURE_DEPENDS "${WINML_TEST_SRC_DIR}/common/*.cpp")
 add_library(winml_test_common STATIC ${winml_test_common_src})
+add_dependencies(winml_test_common onnx)
+add_dependencies(winml_test_common winml_api)
 set_winml_target_properties(winml_test_common)
 
 file(GLOB winml_test_api_src CONFIGURE_DEPENDS "${WINML_TEST_SRC_DIR}/api/*.cpp")
diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
index ba2e9e7f23..45009e6c8e 100644
--- a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
+++ b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
@@ -55,6 +55,14 @@
           Visible="false"
     />
     <None Include="$(OnnxRuntimeCsharpRoot)\..\include\onnxruntime\core\providers\cuda\cuda_provider_factory.h"
+          Condition="'$(OrtPackageId)' != 'Microsoft.ML.OnnxRuntime.DirectML'"
+          PackagePath="\build\native\include"
+          Pack="true"
+          CopyToOutputDirectory="Never"
+          Visible="false"
+    />
+    <None Include="$(OnnxRuntimeCsharpRoot)\..\include\onnxruntime\core\providers\dml\dml_provider_factory.h"
+          Condition="'$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime.DirectML'"
           PackagePath="\build\native\include"
           Pack="true"
           CopyToOutputDirectory="Never"
@@ -79,6 +87,20 @@
           CopyToOutputDirectory="Always"
           Visible="false"
     />
+    <None Include="$(NativeBuildOutputDir)\windows.ai.machinelearning.dll"
+          Condition="Exists('$(NativeBuildOutputDir)\windows.ai.machinelearning.dll')"
+          PackagePath="\runtimes\win-$(TargetArchitecture)\native"
+          Pack="true"
+          CopyToOutputDirectory="Always"
+          Visible="false"
+    />
+    <None Include="$(NativeBuildOutputDir)\windows.ai.machinelearning.pdb"
+          Condition="Exists('$(NativeBuildOutputDir)\windows.ai.machinelearning.pdb')"
+          PackagePath="\runtimes\win-$(TargetArchitecture)\native"
+          Pack="true"
+          CopyToOutputDirectory="Always"
+          Visible="false"
+    />
     <None Include="$(NativeBuildOutputDir)\mkldnn.dll"
           Condition="Exists('$(NativeBuildOutputDir)\mkldnn.dll')"
           PackagePath="\runtimes\win-$(TargetArchitecture)\native"
diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
index 702a20254a..5ef6436f8e 100644
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml
@@ -13,7 +13,7 @@ jobs:
   parameters:
     AgentPool : $(AgentPoolWin)
     JobName: 'Windows_CI_Dev'
-    BuildCommand:  '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests'
+    BuildCommand:  '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_winml'
     DoDebugBuild: 'false'
     DoNugetPack : 'true'
     DoCompliance: ${{ parameters.DoCompliance }}
@@ -28,7 +28,7 @@ jobs:
   parameters:
     AgentPool : $(AgentPoolWin)
     JobName: 'Windows_CI_Dev_x86'
-    BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib  --build_csharp --enable_onnx_tests --x86'
+    BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib  --build_csharp --enable_onnx_tests --use_winml --x86'
     DoDebugBuild: 'false'
     DoNugetPack : 'true'
     DoCompliance: 'false'
diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml
index 81914ad074..46c2b13aca 100644
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml
@@ -12,7 +12,7 @@ jobs:
 - template: ../../templates/win-ci.yml
   parameters:
     AgentPool : $(AgentPoolWin)
-    JobName: 'Windows_CI_GPU_Dev'
+    JobName: 'Windows_CI_GPU_CUDA_Dev'
     BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --build_shared_lib  --build_csharp --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --cuda_version 10.0'
     DoDebugBuild: 'false'
     DoNugetPack : 'true'
@@ -23,6 +23,7 @@ jobs:
     MsbuildArguments: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\'
     EnvSetupScript: 'setup_env_cuda.bat'
     CudaVersion: '10.0'
+    ArtifactName: drop-nuget-cuda
     OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu'
     NuPackScript: |
      msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu
@@ -30,6 +31,24 @@ jobs:
      mkdir $(Build.ArtifactStagingDirectory)\testdata
      copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
 
+- template: ../../templates/win-ci.yml
+  parameters:
+    AgentPool : $(AgentPoolWin)
+    JobName: 'Windows_CI_GPU_DML_Dev'
+    BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe  --enable_pybind --build_shared_lib  --build_csharp --enable_onnx_tests --use_winml --use_dml'
+    DoDebugBuild: 'false'
+    DoNugetPack : 'true'
+    DoCompliance: 'false'
+    DoEsrp: ${{ parameters.DoEsrp }}
+    BuildArch: 'amd64'
+    SetVcvars: 'true'
+    MsbuildArguments: '/m'
+    ArtifactName: drop-nuget-dml
+    OrtPackageId: 'Microsoft.ML.OnnxRuntime.DirectML'
+    NuPackScript: |
+     msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML
+     copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory)
+
 - job: 'Linux_CI_GPU_Dev'
   pool: $(AgentPoolLinux)
   steps:    
@@ -67,7 +86,8 @@ jobs:
 - job: NuGet_Packaging
   pool: $(AgentPoolWin)
   dependsOn:
-  - Windows_CI_GPU_Dev
+  - Windows_CI_GPU_CUDA_Dev
+  - Windows_CI_GPU_DML_Dev
   - Linux_CI_GPU_Dev
   condition: succeeded()
   steps:  
@@ -78,6 +98,13 @@ jobs:
       targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
     continueOnError: true
 
+  - task: DownloadPipelineArtifact@0
+    displayName: 'Download Pipeline Artifact - NuGet'
+    inputs:
+      artifactName: 'drop-nuget-dml'
+      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
+    continueOnError: true
+
   - task: DownloadPipelineArtifact@0
     displayName: 'Download Pipeline Artifact - Linux'
     inputs: