onnxruntime/.pipelines/OneBranch.Nuget-WindowsAI-Pipeline.Official.yml
Yi Zhang 5a0e5237f5
Fix onebranch exception in code signing (#21088)
### Description
Fix regression caused by
https://github.com/microsoft/onnxruntime/pull/20995



### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
2024-06-19 12:07:17 +08:00

355 lines
21 KiB
YAML

parameters:
- name: UploadSymbols
displayName: Upload Symbols to Microsoft symbol server?
type: boolean
default: false
trigger: none
variables:
CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)]
DEBIAN_FRONTEND: noninteractive
ComponentDetection.Timeout: 1200
resources:
repositories:
- repository: templates
type: git
name: OneBranch.Pipelines/GovernedTemplates
ref: refs/heads/main
extends:
template: v2/OneBranch.Official.CrossPlat.yml@templates
parameters:
nugetPublishing:
feeds:
- name: PublicPackages/ORT-Nightly
files_to_publish: '*.nupkg;!*.symbols.nupkg'
continueOnConflict: true
git:
submodules: false
globalSdl: # https://aka.ms/obpipelines/sdl
asyncSdl:
enabled: false
tsa:
enabled: true
prefast:
enabled: false
cg:
failOnAlert: false
policheck:
break: true # always break the build on policheck issues. You can disable it by setting to 'false'
exclusionsFile: '$(Build.SourcesDirectory)\tools\ci_build\policheck_exclusions.xml'
# suppression:
# suppressionFile: $(Build.SourcesDirectory)\.gdn\global.gdnsuppress
stages:
- stage: Windows_Build
jobs:
- template: .pipelines/windowsai-steps.yml@self
parameters:
BuildArch: x64
- template: .pipelines/windowsai-steps.yml@self
parameters:
BuildArch: x86
PythonPackageName: pythonx86
- template: .pipelines/windowsai-steps.yml@self
parameters:
BuildArch: arm64
- template: .pipelines/windowsai-steps.yml@self
parameters:
BuildArch: x64
Runtime: static
- template: .pipelines/windowsai-steps.yml@self
parameters:
BuildArch: x86
PythonPackageName: pythonx86
Runtime: static
- template: .pipelines/windowsai-steps.yml@self
parameters:
BuildArch: arm64
Runtime: static
- job: NuGet_Packaging
pool:
type: windows
variables:
ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
ob_sdl_binskim_break: false
ob_symbolsPublishing_enabled: ${{ parameters.UploadSymbols }}
ob_symbolsPublishing_symbolsFolder: $(Build.SourcesDirectory)/unzipped
dependsOn:
- Windows_Packaging_x64_dynamic
- Windows_Packaging_x86_dynamic
- Windows_Packaging_arm64_dynamic
- Windows_Packaging_x64_static
- Windows_Packaging_x86_static
- Windows_Packaging_arm64_static
condition: succeeded()
steps:
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML x64'
inputs:
artifactName: 'drop_Windows_Build_Windows_Packaging_x64_dynamic'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x64'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML x86'
inputs:
artifactName: 'drop_Windows_Build_Windows_Packaging_x86_dynamic'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x86'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML arm64'
inputs:
artifactName: 'drop_Windows_Build_Windows_Packaging_arm64_dynamic'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML x64 StaticRuntime'
inputs:
artifactName: 'drop_Windows_Build_Windows_Packaging_x64_static'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x64-static-runtime'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML x86 StaticRuntime'
inputs:
artifactName: 'drop_Windows_Build_Windows_Packaging_x86_static'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x86-static-runtime'
- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet DirectML arm64 StaticRuntime'
inputs:
artifactName: 'drop_Windows_Build_Windows_Packaging_arm64_static'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64-static-runtime'
- task: PowerShell@2
displayName: 'Bundle NuGet and other binaries'
inputs:
targetType: 'inline'
script: |
Add-Type -AssemblyName "System.IO.Compression.FileSystem"
$nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$x64_nuget_package_name = $nupkgs[0].Name
$x64_nuget_package = $nupkgs[0].FullName
$x64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x64_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x64_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x64_nuget_package, $x64_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-x64-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$x64_static_runtime_nuget_package = $nupkgs[0].FullName
$x64_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x64_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x64_static_runtime_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x64_static_runtime_nuget_package, $x64_static_runtime_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-x86 -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$x86_nuget_package = $nupkgs[0].FullName
$x86_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x86_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x86_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x86_nuget_package, $x86_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-x86-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$x86_static_runtime_nuget_package = $nupkgs[0].FullName
$x86_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x86_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x86_static_runtime_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x86_static_runtime_nuget_package, $x86_static_runtime_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-arm64 -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$arm64_nuget_package = $nupkgs[0].FullName
$arm64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$arm64_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm64_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_nuget_package, $arm64_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-arm64-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$arm64_static_runtime_nuget_package = $nupkgs[0].FullName
$arm64_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$arm64_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm64_static_runtime_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_static_runtime_nuget_package, $arm64_static_runtime_nupkg_unzipped_directory)
$x64_static_runtime_path_old = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-x64', '_native')
$x64_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x64', '_native', 'static')
$x86_runtime_path_old = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$x86_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$x86_static_runtime_path_old = [System.IO.Path]::Combine($x86_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$x86_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native', 'static')
$arm64_runtime_path_old = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_static_runtime_path_old = [System.IO.Path]::Combine($arm64_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native', 'static')
$uap_build_path_old = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory, 'build', 'native')
$uap_build_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'build', 'uap10.0')
New-Item -Path $x64_static_runtime_path_new -ItemType Directory
New-Item -Path $x86_runtime_path_new -ItemType Directory
New-Item -Path $x86_static_runtime_path_new -ItemType Directory
New-Item -Path $arm64_runtime_path_new -ItemType Directory
New-Item -Path $arm64_static_runtime_path_new -ItemType Directory
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.dll')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.lib')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.dll')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.lib')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'onnxruntime.dll'))
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'onnxruntime.lib'))
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'microsoft.ai.machinelearning.dll'))
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'microsoft.ai.machinelearning.lib'))
Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'onnxruntime.dll'))
Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'onnxruntime.lib'))
Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'microsoft.ai.machinelearning.dll'))
Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'microsoft.ai.machinelearning.lib'))
Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'onnxruntime.dll'))
Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'onnxruntime.lib'))
Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'microsoft.ai.machinelearning.dll'))
Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'microsoft.ai.machinelearning.lib'))
Copy-Item -Recurse $uap_build_path_old $uap_build_path_new
$merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'merged')
if (!(Test-Path $merged_nuget_path)) {
New-Item -Path $merged_nuget_path -ItemType Directory
}
$merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $x64_nuget_package_name)
Start-Process -FilePath "7z" -ArgumentList "-tzip a -r $merged_nuget ." -WorkingDirectory $x64_nupkg_unzipped_directory -NoNewWindow -Wait
workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64
- task: PowerShell@2
displayName: 'Bundle Symbols NuGet'
inputs:
targetType: 'inline'
script: |
Add-Type -AssemblyName "System.IO.Compression.FileSystem"
$nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse)
$x64_nuget_package_name = $nupkgs[0].Name
$x64_nuget_package = $nupkgs[0].FullName
$x64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x64_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($x64_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x64_nuget_package, $x64_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-x86 -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse)
$x86_nuget_package = $nupkgs[0].FullName
$x86_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x86_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($x86_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x86_nuget_package, $x86_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-arm64 -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse)
$arm64_nuget_package = $nupkgs[0].FullName
$arm64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$arm64_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($arm64_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_nuget_package, $arm64_nupkg_unzipped_directory)
$x86_runtime_path_old = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$x86_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$arm64_runtime_path_old = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
New-Item -Path $x86_runtime_path_new -ItemType Directory
New-Item -Path $arm64_runtime_path_new -ItemType Directory
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.pdb')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.pdb')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm64_runtime_path_new
$merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'merged')
if (!(Test-Path $merged_nuget_path)) {
New-Item -Path $merged_nuget_path -ItemType Directory
}
$merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $x64_nuget_package_name)
Start-Process -FilePath "7z" -ArgumentList "-tzip a -r $merged_nuget ." -WorkingDirectory $x64_nupkg_unzipped_directory -NoNewWindow -Wait
$merged_nuget_without_pdb = [System.IO.Path]::ChangeExtension($merged_nuget, '.nupkg')
# Now we combine the DLLs and PDBs together, put them back in a folder under $(Build.SourcesDirectory)
# We won't upload the unzipped folder. We will just feed it to BinSkim.
7z x -o$(Build.SourcesDirectory)\unzipped $merged_nuget
7z -y x -o$(Build.SourcesDirectory)\unzipped $merged_nuget_without_pdb
workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64
- script: |
dir $(Build.SourcesDirectory)\unzipped\runtimes\win-x64\_native
- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5
displayName: "Sign Nuget package"
inputs:
ConnectedServiceName: 'OnnxrunTimeCodeSign_20240611'
AppRegistrationClientId: '53d54d02-978d-4305-8572-583cf6711c4f'
AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47'
AuthAKVName: 'buildkeyvault'
AuthCertName: '53d54d02-SSL-AutoRotate'
AuthSignCertName: '53d54d02-978d-4305-8572-583cf6711c4f'
FolderPath: $(Build.ArtifactStagingDirectory)
Pattern: '*.nupkg'
SessionTimeout: 90
ServiceEndpointUrl: 'https://api.esrp.microsoft.com/api/v2'
MaxConcurrency: 25
signConfigType: inlineSignParams
inlineOperation: |
[
{
"keyCode": "CP-401405",
"operationSetCode": "NuGetSign",
"parameters": [ ],
"toolName": "sign",
"toolVersion": "6.2.9304.0"
},
{
"keyCode": "CP-401405",
"operationSetCode": "NuGetVerify",
"parameters": [ ],
"toolName": "sign",
"toolVersion": "6.2.9304.0"
}
]
- job: NuGet_Publishing
pool:
type: windows
variables:
ob_outputDirectory: '$(Build.BinariesDirectory)'
ob_sdl_binskim_enabled: false
ob_pipelineartifacts_enabled: false
${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}:
ob_nugetPublishing_enabled: true
dependsOn:
- NuGet_Packaging
condition: succeeded()
steps:
- task: DownloadPipelineArtifact@2
displayName: 'Download Pipeline Artifact'
inputs:
buildType: 'current'
artifactName: 'drop_Windows_Build_NuGet_Packaging'
itemPattern: 'merged/**'
targetPath: '$(Build.BinariesDirectory)'
- powershell: |
Rename-Item -Path merged packages
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Rename nuget folder'