From ee5ca27ae2b9d3994bc95287facf51d45b9fd180 Mon Sep 17 00:00:00 2001 From: Sheil Kumar Date: Tue, 14 Jul 2020 14:52:39 -0700 Subject: [PATCH] Split Microsoft.AI.MachineLearning.nupkg in a NuGet package and symbol NuGet package (#4503) * add threadpool interface * generate snupkgs * include_pdb check * fix snupkg generation * Add task to merge snupkgs * folder exists * check dir * revert thread pool stuff Co-authored-by: Sheil Kumar --- csharp/OnnxRuntime.CSharp.proj | 2 +- .../nuget/templates/windowsai.yml | 151 +++++++++++++----- .../templates/windowsai-nuget-build.yml | 8 +- 3 files changed, 114 insertions(+), 47 deletions(-) diff --git a/csharp/OnnxRuntime.CSharp.proj b/csharp/OnnxRuntime.CSharp.proj index a5058e9c07..f3b9a73b3f 100644 --- a/csharp/OnnxRuntime.CSharp.proj +++ b/csharp/OnnxRuntime.CSharp.proj @@ -134,7 +134,7 @@ CMake creates a target to this project - + diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml index aa3c088091..0a87019ba3 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml @@ -142,37 +142,37 @@ jobs: script: | Add-Type -AssemblyName "System.IO.Compression.FileSystem" - $packages = (Get-ChildItem -Filter Microsoft.AI.MachineLearning* -Recurse) - $x64_nuget_package_name = $packages[0].Name - $x64_nuget_package = $packages[0].FullName - $x64_unzipped_directory_root = $packages[0].Directory.FullName - $x64_unzipped_directory = [System.IO.Path]::Combine($x64_unzipped_directory_root, [System.IO.Path]::GetFileNameWithoutExtension($x64_nuget_package)) - [System.IO.Compression.ZipFile]::ExtractToDirectory($x64_nuget_package, $x64_unzipped_directory) - - $packages = (Get-ChildItem ..\nuget-artifact-x86 -Filter Microsoft.AI.MachineLearning* -Recurse) - $x86_nuget_package = $packages[0].FullName - $x86_unzipped_directory_root = $packages[0].Directory.FullName - $x86_unzipped_directory = [System.IO.Path]::Combine($x86_unzipped_directory_root, [System.IO.Path]::GetFileNameWithoutExtension($x86_nuget_package)) - [System.IO.Compression.ZipFile]::ExtractToDirectory($x86_nuget_package, $x86_unzipped_directory) + $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $x64_nuget_package_name = $nupkgs[0].Name + $x64_nuget_package = $nupkgs[0].FullName + $x64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $x64_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x64_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($x64_nuget_package, $x64_nupkg_unzipped_directory) - $packages = (Get-ChildItem ..\nuget-artifact-arm64 -Filter Microsoft.AI.MachineLearning* -Recurse) - $arm64_nuget_package = $packages[0].FullName - $arm64_unzipped_directory_root = $packages[0].Directory.FullName - $arm64_unzipped_directory = [System.IO.Path]::Combine($arm64_unzipped_directory_root, [System.IO.Path]::GetFileNameWithoutExtension($arm64_nuget_package)) - [System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_nuget_package, $arm64_unzipped_directory) + $nupkgs = (Get-ChildItem ..\nuget-artifact-x86 -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $x86_nuget_package = $nupkgs[0].FullName + $x86_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $x86_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x86_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($x86_nuget_package, $x86_nupkg_unzipped_directory) - $packages = (Get-ChildItem ..\nuget-artifact-arm -Filter Microsoft.AI.MachineLearning* -Recurse) - $arm_nuget_package = $packages[0].FullName - $arm_unzipped_directory_root = $packages[0].Directory.FullName - $arm_unzipped_directory = [System.IO.Path]::Combine($arm_unzipped_directory_root, [System.IO.Path]::GetFileNameWithoutExtension($arm_nuget_package)) - [System.IO.Compression.ZipFile]::ExtractToDirectory($arm_nuget_package, $arm_unzipped_directory) + $nupkgs = (Get-ChildItem ..\nuget-artifact-arm64 -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $arm64_nuget_package = $nupkgs[0].FullName + $arm64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $arm64_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm64_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_nuget_package, $arm64_nupkg_unzipped_directory) - $x86_runtime_path_old = [System.IO.Path]::Combine($x86_unzipped_directory, 'runtimes', 'win-x86', 'native') - $x86_runtime_path_new = [System.IO.Path]::Combine($x64_unzipped_directory, 'runtimes', 'win-x86', 'native') - $arm64_runtime_path_old = [System.IO.Path]::Combine($arm64_unzipped_directory, 'runtimes', 'win-arm64', 'native') - $arm64_runtime_path_new = [System.IO.Path]::Combine($x64_unzipped_directory, 'runtimes', 'win-arm64', 'native') - $arm_runtime_path_old = [System.IO.Path]::Combine($arm_unzipped_directory, 'runtimes', 'win-arm', 'native') - $arm_runtime_path_new = [System.IO.Path]::Combine($x64_unzipped_directory, 'runtimes', 'win-arm', 'native') + $nupkgs = (Get-ChildItem ..\nuget-artifact-arm -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $arm_nuget_package = $nupkgs[0].FullName + $arm_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $arm_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($arm_nuget_package, $arm_nupkg_unzipped_directory) + + $x86_runtime_path_old = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory, 'runtimes', 'win-x86', 'native') + $x86_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', 'native') + $arm64_runtime_path_old = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', 'native') + $arm64_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', 'native') + $arm_runtime_path_old = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory, 'runtimes', 'win-arm', 'native') + $arm_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm', 'native') New-Item -Path $x86_runtime_path_new -ItemType Directory New-Item -Path $arm64_runtime_path_new -ItemType Directory @@ -180,39 +180,106 @@ jobs: Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.dll')) $x86_runtime_path_new Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.lib')) $x86_runtime_path_new - Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.pdb')) $x86_runtime_path_new Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $x86_runtime_path_new Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $x86_runtime_path_new - Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $x86_runtime_path_new Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'directml.dll')) $x86_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.dll')) $arm64_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.lib')) $arm64_runtime_path_new - Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.pdb')) $arm64_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $arm64_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $arm64_runtime_path_new - Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm64_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.dll')) $arm_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.lib')) $arm_runtime_path_new - Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.pdb')) $arm_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $arm_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $arm_runtime_path_new - Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm_runtime_path_new $merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'merged') - New-Item -Path $merged_nuget_path -ItemType Directory + if (!(Test-Path $merged_nuget_path)) { + New-Item -Path $merged_nuget_path -ItemType Directory + } $merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $x64_nuget_package_name) - $zip_tool_directory = [System.IO.Path]::Combine($x64_unzipped_directory_root, 'ziptool') - New-Item -Path $zip_tool_directory -ItemType Directory + $zip_tool_directory = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory_root, 'ziptool') + if (!(Test-Path $zip_tool_directory)) { + New-Item -Path $zip_tool_directory -ItemType Directory + } $zip_tool = [System.IO.Path]::Combine($zip_tool_directory, 'zip.exe') Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile $zip_tool - Start-Process -FilePath $zip_tool -ArgumentList "-r $merged_nuget ." -WorkingDirectory $x64_unzipped_directory -NoNewWindow -Wait + Start-Process -FilePath $zip_tool -ArgumentList "-r $merged_nuget ." -WorkingDirectory $x64_nupkg_unzipped_directory -NoNewWindow -Wait + workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64 + - task: PowerShell@2 + displayName: 'Bundle Symbols NuGet' + inputs: + targetType: 'inline' + script: | + Add-Type -AssemblyName "System.IO.Compression.FileSystem" + + $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse) + $x64_nuget_package_name = $nupkgs[0].Name + $x64_nuget_package = $nupkgs[0].FullName + $x64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $x64_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($x64_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($x64_nuget_package, $x64_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-x86 -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse) + $x86_nuget_package = $nupkgs[0].FullName + $x86_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $x86_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($x86_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($x86_nuget_package, $x86_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-arm64 -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse) + $arm64_nuget_package = $nupkgs[0].FullName + $arm64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $arm64_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($arm64_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_nuget_package, $arm64_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-arm -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse) + $arm_nuget_package = $nupkgs[0].FullName + $arm_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $arm_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($arm_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($arm_nuget_package, $arm_nupkg_unzipped_directory) + + $x86_runtime_path_old = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory, 'runtimes', 'win-x86', 'native') + $x86_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', 'native') + $arm64_runtime_path_old = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', 'native') + $arm64_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', 'native') + $arm_runtime_path_old = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory, 'runtimes', 'win-arm', 'native') + $arm_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm', 'native') + + New-Item -Path $x86_runtime_path_new -ItemType Directory + New-Item -Path $arm64_runtime_path_new -ItemType Directory + New-Item -Path $arm_runtime_path_new -ItemType Directory + + Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.pdb')) $x86_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $x86_runtime_path_new + + Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.pdb')) $arm64_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm64_runtime_path_new + + Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.pdb')) $arm_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm_runtime_path_new + + $merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'merged') + if (!(Test-Path $merged_nuget_path)) { + New-Item -Path $merged_nuget_path -ItemType Directory + } + + $merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $x64_nuget_package_name) + + $zip_tool_directory = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory_root, 'ziptool') + if (!(Test-Path $zip_tool_directory)) { + New-Item -Path $zip_tool_directory -ItemType Directory + } + + $zip_tool = [System.IO.Path]::Combine($zip_tool_directory, 'zip.exe') + + Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile $zip_tool + Start-Process -FilePath $zip_tool -ArgumentList "-r $merged_nuget ." -WorkingDirectory $x64_nupkg_unzipped_directory -NoNewWindow -Wait workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64 - template: ../../templates/esrp_nuget.yml @@ -247,8 +314,8 @@ jobs: script: | Add-Type -AssemblyName "System.IO.Compression.FileSystem" - $packages = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*) - $merged_nuget_package = $packages[0] + $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*) + $merged_nuget_package = $nupkgs[0] $merged_nuget_package_name = $merged_nuget_package.Name $matched_name = ($merged_nuget_package_name -match "Microsoft.AI.MachineLearning.(?.*).nupkg") $package_version = $matches['version'] @@ -279,8 +346,8 @@ jobs: script: | Add-Type -AssemblyName "System.IO.Compression.FileSystem" - $packages = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*) - $merged_nuget_package = $packages[0] + $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*) + $merged_nuget_package = $nupkgs[0] $merged_nuget_package_name = $merged_nuget_package.Name $matched_name = ($merged_nuget_package_name -match "Microsoft.AI.MachineLearning.(?.*).nupkg") $package_version = $matches['version'] diff --git a/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml b/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml index 5358ac8ee6..4a3e01d048 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml @@ -150,32 +150,32 @@ steps: - ${{ if eq(parameters.BuildArch, 'x64') }}: - script: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreateWindowsAIPackage /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory) /p:OnnxRuntimeSourceDirectory=$(Build.SourcesDirectory) - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.AI.MachineLearning\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.snupkg $(Build.ArtifactStagingDirectory) workingDirectory: '$(Build.SourcesDirectory)\csharp' displayName: 'Create NuGet Package' - ${{ if eq(parameters.BuildArch, 'x86') }}: - script: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreateWindowsAIPackage /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory) /p:OnnxRuntimeSourceDirectory=$(Build.SourcesDirectory) /p:TargetArchitecture=x86 - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.AI.MachineLearning\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.snupkg $(Build.ArtifactStagingDirectory) workingDirectory: '$(Build.SourcesDirectory)\csharp' displayName: 'Create NuGet Package' - ${{ if eq(parameters.BuildArch, 'arm64') }}: - script: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreateWindowsAIPackage /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory) /p:OnnxRuntimeSourceDirectory=$(Build.SourcesDirectory) /p:TargetArchitecture=arm64 /p:ProtocDirectory=$(Build.BinariesDirectory)\host_protoc\Release - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.AI.MachineLearning\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.snupkg $(Build.ArtifactStagingDirectory) workingDirectory: '$(Build.SourcesDirectory)\csharp' displayName: 'Create NuGet Package' - ${{ if eq(parameters.BuildArch, 'arm') }}: - script: | msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreateWindowsAIPackage /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory) /p:OnnxRuntimeSourceDirectory=$(Build.SourcesDirectory) /p:TargetArchitecture=arm /p:ProtocDirectory=$(Build.BinariesDirectory)\host_protoc\Release - copy $(Build.SourcesDirectory)\csharp\src\Microsoft.AI.MachineLearning\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.snupkg $(Build.ArtifactStagingDirectory) workingDirectory: '$(Build.SourcesDirectory)\csharp' displayName: 'Create NuGet Package'