saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-06-27 03:11:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
onnxruntime/tools/ci_build/github/azure-pipelines/nuget-windows-ai.yml
Kyle 74adfc2099
Nuget Windows AI Pipeline, Disable SDL Submodules. (#22711) 
### Description
<!-- Describe your changes. -->
Set SDL's git submodule to false. 


### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
* Previous job's SDL logs:It has 'git submodule sync' command, which
means 'git submodule sync synchronizes all submodules while git
submodule sync'

* After set sdl git submodules to false, the logs don't have 'git
submodule sync' command.
2024-11-04 08:39:28 -08:00

317 lines
20 KiB
YAML
Raw Blame History

trigger: none
variables:
DisableDockerDetector: true
resources:
repositories:
- repository: 1esPipelines
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
extends:
template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
parameters:
pool:
name: onnxruntime-Win-CPU-2022
os: windows
sdl:
git:
submodules: false
tsa:
enabled: true
codeSignValidation:
enabled: true
break: true
policheck:
enabled: true
exclusionsFile: '$(Build.SourcesDirectory)\tools\ci_build\policheck_exclusions.xml'
stages:
- stage: Windows_Build
jobs:
- template: tools/ci_build/github/azure-pipelines/templates/windowsai-steps.yml@self
parameters:
BuildArch: x64
- template: tools/ci_build/github/azure-pipelines/templates/windowsai-steps.yml@self
parameters:
BuildArch: x86
- template: tools/ci_build/github/azure-pipelines/templates/windowsai-steps.yml@self
parameters:
BuildArch: arm64
- template: tools/ci_build/github/azure-pipelines/templates/windowsai-steps.yml@self
parameters:
BuildArch: x64
Runtime: static
- template: tools/ci_build/github/azure-pipelines/templates/windowsai-steps.yml@self
parameters:
BuildArch: x86
Runtime: static
- template: tools/ci_build/github/azure-pipelines/templates/windowsai-steps.yml@self
parameters:
BuildArch: arm64
Runtime: static
- job: NuGet_Packaging
dependsOn:
- Windows_Packaging_x64_dynamic
- Windows_Packaging_x86_dynamic
- Windows_Packaging_arm64_dynamic
- Windows_Packaging_x64_static
- Windows_Packaging_x86_static
- Windows_Packaging_arm64_static
condition: succeeded()
templateContext:
inputs:
- input: pipelineArtifact
artifactName: drop_Windows_Build_Windows_Packaging_x64_dynamic
targetPath: $(Build.BinariesDirectory)/nuget-artifact-x64
- input: pipelineArtifact
artifactName: drop_Windows_Build_Windows_Packaging_x86_dynamic
targetPath: $(Build.BinariesDirectory)/nuget-artifact-x86
- input: pipelineArtifact
artifactName: drop_Windows_Build_Windows_Packaging_arm64_dynamic
targetPath: $(Build.BinariesDirectory)/nuget-artifact-arm64
- input: pipelineArtifact
artifactName: drop_Windows_Build_Windows_Packaging_x64_static
targetPath: $(Build.BinariesDirectory)/nuget-artifact-x64-static-runtime
- input: pipelineArtifact
artifactName: drop_Windows_Build_Windows_Packaging_x86_static
targetPath: $(Build.BinariesDirectory)/nuget-artifact-x86-static-runtime
- input: pipelineArtifact
artifactName: drop_Windows_Build_Windows_Packaging_arm64_static
targetPath: $(Build.BinariesDirectory)/nuget-artifact-arm64-static-runtime
outputs:
- output: pipelineArtifact
path: '$(Build.ArtifactStagingDirectory)/merged'
artifact: drop_Windows_Build_NuGet_Packaging
steps:
- task: PowerShell@2
displayName: 'Bundle NuGet and other binaries'
inputs:
targetType: 'inline'
script: |
Add-Type -AssemblyName "System.IO.Compression.FileSystem"
$nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$x64_nuget_package_name = $nupkgs[0].Name
$x64_nuget_package = $nupkgs[0].FullName
$x64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x64_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x64_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x64_nuget_package, $x64_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-x64-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$x64_static_runtime_nuget_package = $nupkgs[0].FullName
$x64_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x64_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x64_static_runtime_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x64_static_runtime_nuget_package, $x64_static_runtime_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-x86 -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$x86_nuget_package = $nupkgs[0].FullName
$x86_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x86_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x86_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x86_nuget_package, $x86_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-x86-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$x86_static_runtime_nuget_package = $nupkgs[0].FullName
$x86_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x86_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x86_static_runtime_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x86_static_runtime_nuget_package, $x86_static_runtime_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-arm64 -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$arm64_nuget_package = $nupkgs[0].FullName
$arm64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$arm64_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm64_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_nuget_package, $arm64_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-arm64-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse)
$arm64_static_runtime_nuget_package = $nupkgs[0].FullName
$arm64_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$arm64_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm64_static_runtime_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_static_runtime_nuget_package, $arm64_static_runtime_nupkg_unzipped_directory)
$x64_static_runtime_path_old = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-x64', '_native')
$x64_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x64', '_native', 'static')
$x86_runtime_path_old = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$x86_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$x86_static_runtime_path_old = [System.IO.Path]::Combine($x86_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$x86_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native', 'static')
$arm64_runtime_path_old = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_static_runtime_path_old = [System.IO.Path]::Combine($arm64_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native', 'static')
$uap_build_path_old = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory, 'build', 'native')
$uap_build_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'build', 'uap10.0')
New-Item -Path $x64_static_runtime_path_new -ItemType Directory
New-Item -Path $x86_runtime_path_new -ItemType Directory
New-Item -Path $x86_static_runtime_path_new -ItemType Directory
New-Item -Path $arm64_runtime_path_new -ItemType Directory
New-Item -Path $arm64_static_runtime_path_new -ItemType Directory
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.dll')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.lib')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.dll')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.lib')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'onnxruntime.dll'))
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'onnxruntime.lib'))
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'microsoft.ai.machinelearning.dll'))
Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'microsoft.ai.machinelearning.lib'))
Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'onnxruntime.dll'))
Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'onnxruntime.lib'))
Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'microsoft.ai.machinelearning.dll'))
Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'microsoft.ai.machinelearning.lib'))
Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'onnxruntime.dll'))
Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'onnxruntime.lib'))
Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'microsoft.ai.machinelearning.dll'))
Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'microsoft.ai.machinelearning.lib'))
Copy-Item -Recurse $uap_build_path_old $uap_build_path_new
$merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'merged')
if (!(Test-Path $merged_nuget_path)) {
New-Item -Path $merged_nuget_path -ItemType Directory
}
$merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $x64_nuget_package_name)
Start-Process -FilePath "7z" -ArgumentList "-tzip a -r $merged_nuget ." -WorkingDirectory $x64_nupkg_unzipped_directory -NoNewWindow -Wait
workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64
- task: PowerShell@2
displayName: 'Bundle Symbols NuGet'
inputs:
targetType: 'inline'
script: |
Add-Type -AssemblyName "System.IO.Compression.FileSystem"
$nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse)
$x64_nuget_package_name = $nupkgs[0].Name
$x64_nuget_package = $nupkgs[0].FullName
$x64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x64_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($x64_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x64_nuget_package, $x64_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-x86 -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse)
$x86_nuget_package = $nupkgs[0].FullName
$x86_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$x86_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($x86_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($x86_nuget_package, $x86_nupkg_unzipped_directory)
$nupkgs = (Get-ChildItem ..\nuget-artifact-arm64 -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse)
$arm64_nuget_package = $nupkgs[0].FullName
$arm64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName
$arm64_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($arm64_nuget_package))
[System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_nuget_package, $arm64_nupkg_unzipped_directory)
$x86_runtime_path_old = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$x86_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native')
$arm64_runtime_path_old = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
$arm64_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native')
New-Item -Path $x86_runtime_path_new -ItemType Directory
New-Item -Path $arm64_runtime_path_new -ItemType Directory
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.pdb')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $x86_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.pdb')) $arm64_runtime_path_new
Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm64_runtime_path_new
$merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'merged')
if (!(Test-Path $merged_nuget_path)) {
New-Item -Path $merged_nuget_path -ItemType Directory
}
$merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $x64_nuget_package_name)
Start-Process -FilePath "7z" -ArgumentList "-tzip a -r $merged_nuget ." -WorkingDirectory $x64_nupkg_unzipped_directory -NoNewWindow -Wait
$merged_nuget_without_pdb = [System.IO.Path]::ChangeExtension($merged_nuget, '.nupkg')
# Now we combine the DLLs and PDBs together, put them back in a folder under $(Build.SourcesDirectory)
# We won't upload the unzipped folder. We will just feed it to BinSkim.
7z x -o$(Build.SourcesDirectory)\unzipped $merged_nuget
7z -y x -o$(Build.SourcesDirectory)\unzipped $merged_nuget_without_pdb
workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64
- script: |
dir $(Build.SourcesDirectory)\unzipped\runtimes\win-x64\_native
- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5
displayName: "Sign Nuget package"
inputs:
ConnectedServiceName: 'OnnxrunTimeCodeSign_20240611'
AppRegistrationClientId: '53d54d02-978d-4305-8572-583cf6711c4f'
AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47'
AuthAKVName: 'buildkeyvault'
AuthCertName: '53d54d02-SSL-AutoRotate'
AuthSignCertName: '53d54d02-978d-4305-8572-583cf6711c4f'
FolderPath: $(Build.ArtifactStagingDirectory)
Pattern: '*.nupkg'
SessionTimeout: 90
ServiceEndpointUrl: 'https://api.esrp.microsoft.com/api/v2'
MaxConcurrency: 25
signConfigType: inlineSignParams
inlineOperation: |
[
{
"keyCode": "CP-401405",
"operationSetCode": "NuGetSign",
"parameters": [ ],
"toolName": "sign",
"toolVersion": "6.2.9304.0"
},
{
"keyCode": "CP-401405",
"operationSetCode": "NuGetVerify",
"parameters": [ ],
"toolName": "sign",
"toolVersion": "6.2.9304.0"
}
]
- job: NuGet_Publishing
dependsOn:
- NuGet_Packaging
condition: succeeded()
templateContext:
inputs:
- input: pipelineArtifact
artifactName: drop_Windows_Build_NuGet_Packaging
targetPath: $(Build.BinariesDirectory)/merged
outputs:
- output: nuget
# condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) # Optional condition
useDotNetTask: false # The default is false to use the NuGetCommand task. Set to true to use the DotNetCoreCLI task to publish packages.
packagesToPush: '$(Build.BinariesDirectory)/packages/*.nupkg;!$(Build.BinariesDirectory)/packages/*.symbols.nupkg'
packageParentPath: '$(Build.BinariesDirectory)/'
publishVstsFeed: PublicPackages/ORT-Nightly # Required when pushing to internal feed.
nuGetFeedType: internal # Change to external when publishing to external feed
allowPackageConflicts: true # Optional. NuGetCommand task only.
publishPackageMetadata: true # Optional
steps:
- powershell: |
Rename-Item -Path merged packages
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Rename nuget folder'
Reference in a new issue View git blame Copy permalink