diff --git a/.config/tsaoptions.json b/.config/tsaoptions.json new file mode 100644 index 0000000000..b2b3180fef --- /dev/null +++ b/.config/tsaoptions.json @@ -0,0 +1,8 @@ +{ + "notificationAliases": ["chasun@microsoft.com"], + "areaPath": "Vienna\\ONNX Runtime\\Shared Core", + "codebaseName": "onnxruntime_master", + "instanceUrl": "https://msdata.visualstudio.com/", + "projectName": "Vienna", + "ignoreBranchName": true +} \ No newline at end of file diff --git a/.pipelines/OneBranch.Nuget-WindowsAI-Pipeline.Official.yml b/.pipelines/OneBranch.Nuget-WindowsAI-Pipeline.Official.yml new file mode 100644 index 0000000000..5a6500161e --- /dev/null +++ b/.pipelines/OneBranch.Nuget-WindowsAI-Pipeline.Official.yml @@ -0,0 +1,348 @@ +parameters: +- name: UploadSymbols + displayName: Upload Symbols to Microsoft symbol server? + type: boolean + default: false + +trigger: none + +variables: + CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] + DEBIAN_FRONTEND: noninteractive + +resources: + repositories: + - repository: templates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: v2/OneBranch.Official.CrossPlat.yml@templates + parameters: + git: + submodules: false + globalSdl: # https://aka.ms/obpipelines/sdl + # tsa: + # enabled: true + # credscan: + # suppressionsFile: $(Build.SourcesDirectory)\.config\CredScanSuppressions.json + prefast: + enabled: false + cg: + failOnAlert: false + policheck: + break: true # always break the build on policheck issues. You can disable it by setting to 'false' + exclusionsFile: '$(Build.SourcesDirectory)\tools\ci_build\policheck_exclusions.xml' + # suppression: + # suppressionFile: $(Build.SourcesDirectory)\.gdn\global.gdnsuppress + + stages: + - stage: Windows_Build + jobs: + - template: .pipelines/windowsai-steps.yml@self + parameters: + BuildArch: x64 + + - template: .pipelines/windowsai-steps.yml@self + parameters: + BuildArch: x86 + PythonPackageName: pythonx86 + + - template: .pipelines/windowsai-steps.yml@self + parameters: + BuildArch: arm + + - template: .pipelines/windowsai-steps.yml@self + parameters: + BuildArch: arm64 + + - template: .pipelines/windowsai-steps.yml@self + parameters: + BuildArch: x64 + Runtime: static + + - template: .pipelines/windowsai-steps.yml@self + parameters: + BuildArch: x86 + PythonPackageName: pythonx86 + Runtime: static + + - template: .pipelines/windowsai-steps.yml@self + parameters: + BuildArch: arm + Runtime: static + + - template: .pipelines/windowsai-steps.yml@self + parameters: + BuildArch: arm64 + Runtime: static + + + - job: NuGet_Packaging + pool: + type: windows + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + ob_sdl_binskim_break: false + ob_symbolsPublishing_enabled: ${{ parameters.UploadSymbols }} + ob_symbolsPublishing_symbolsFolder: $(Build.SourcesDirectory)/unzipped + dependsOn: + - Windows_Packaging_x64_dynamic + - Windows_Packaging_x86_dynamic + - Windows_Packaging_arm_dynamic + - Windows_Packaging_arm64_dynamic + - Windows_Packaging_x64_static + - Windows_Packaging_x86_static + - Windows_Packaging_arm_static + - Windows_Packaging_arm64_static + condition: succeeded() + steps: + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML x64' + inputs: + artifactName: 'drop_Windows_Build_Windows_Packaging_x64_dynamic' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x64' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML x86' + inputs: + artifactName: 'drop_Windows_Build_Windows_Packaging_x86_dynamic' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x86' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML arm64' + inputs: + artifactName: 'drop_Windows_Build_Windows_Packaging_arm64_dynamic' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML arm' + inputs: + artifactName: 'drop_Windows_Build_Windows_Packaging_arm_dynamic' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML x64 StaticRuntime' + inputs: + artifactName: 'drop_Windows_Build_Windows_Packaging_x64_static' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x64-static-runtime' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML x86 StaticRuntime' + inputs: + artifactName: 'drop_Windows_Build_Windows_Packaging_x86_static' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x86-static-runtime' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML arm64 StaticRuntime' + inputs: + artifactName: 'drop_Windows_Build_Windows_Packaging_arm64_static' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64-static-runtime' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet DirectML arm StaticRuntime' + inputs: + artifactName: 'drop_Windows_Build_Windows_Packaging_arm_static' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm-static-runtime' + + - task: PowerShell@2 + displayName: 'Bundle NuGet and other binaries' + inputs: + targetType: 'inline' + script: | + Add-Type -AssemblyName "System.IO.Compression.FileSystem" + + $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $x64_nuget_package_name = $nupkgs[0].Name + $x64_nuget_package = $nupkgs[0].FullName + $x64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $x64_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x64_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($x64_nuget_package, $x64_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-x64-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $x64_static_runtime_nuget_package = $nupkgs[0].FullName + $x64_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $x64_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x64_static_runtime_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($x64_static_runtime_nuget_package, $x64_static_runtime_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-x86 -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $x86_nuget_package = $nupkgs[0].FullName + $x86_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $x86_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x86_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($x86_nuget_package, $x86_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-x86-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $x86_static_runtime_nuget_package = $nupkgs[0].FullName + $x86_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $x86_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($x86_static_runtime_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($x86_static_runtime_nuget_package, $x86_static_runtime_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-arm64 -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $arm64_nuget_package = $nupkgs[0].FullName + $arm64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $arm64_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm64_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_nuget_package, $arm64_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-arm64-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $arm64_static_runtime_nuget_package = $nupkgs[0].FullName + $arm64_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $arm64_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm64_static_runtime_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_static_runtime_nuget_package, $arm64_static_runtime_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-arm -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $arm_nuget_package = $nupkgs[0].FullName + $arm_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $arm_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($arm_nuget_package, $arm_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-arm-static-runtime -Filter Microsoft.AI.MachineLearning*.nupkg -Recurse) + $arm_static_runtime_nuget_package = $nupkgs[0].FullName + $arm_static_runtime_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $arm_static_runtime_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm_static_runtime_nupkg_unzipped_directory_root, 'binaries', [System.IO.Path]::GetFileNameWithoutExtension($arm_static_runtime_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($arm_static_runtime_nuget_package, $arm_static_runtime_nupkg_unzipped_directory) + + $x64_static_runtime_path_old = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-x64', '_native') + $x64_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x64', '_native', 'static') + $x86_runtime_path_old = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native') + $x86_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native') + $x86_static_runtime_path_old = [System.IO.Path]::Combine($x86_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native') + $x86_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native', 'static') + $arm64_runtime_path_old = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native') + $arm64_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native') + $arm64_static_runtime_path_old = [System.IO.Path]::Combine($arm64_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native') + $arm64_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native', 'static') + $arm_runtime_path_old = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native') + $arm_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native') + $arm_static_runtime_path_old = [System.IO.Path]::Combine($arm_static_runtime_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native') + $arm_static_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native', 'static') + $uap_build_path_old = [System.IO.Path]::Combine($x64_static_runtime_nupkg_unzipped_directory, 'build', 'native') + $uap_build_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'build', 'uap10.0') + + New-Item -Path $x64_static_runtime_path_new -ItemType Directory + New-Item -Path $x86_runtime_path_new -ItemType Directory + New-Item -Path $x86_static_runtime_path_new -ItemType Directory + New-Item -Path $arm64_runtime_path_new -ItemType Directory + New-Item -Path $arm64_static_runtime_path_new -ItemType Directory + New-Item -Path $arm_runtime_path_new -ItemType Directory + New-Item -Path $arm_static_runtime_path_new -ItemType Directory + + Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.dll')) $x86_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.lib')) $x86_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $x86_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $x86_runtime_path_new + + Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.dll')) $arm64_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.lib')) $arm64_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $arm64_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $arm64_runtime_path_new + + Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.dll')) $arm_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.lib')) $arm_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.dll')) $arm_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.lib')) $arm_runtime_path_new + + Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'onnxruntime.dll')) + Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'onnxruntime.lib')) + Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'microsoft.ai.machinelearning.dll')) + Copy-Item ([System.IO.Path]::Combine($x64_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($x64_static_runtime_path_new, 'microsoft.ai.machinelearning.lib')) + + Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'onnxruntime.dll')) + Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'onnxruntime.lib')) + Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'microsoft.ai.machinelearning.dll')) + Copy-Item ([System.IO.Path]::Combine($x86_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($x86_static_runtime_path_new, 'microsoft.ai.machinelearning.lib')) + + Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'onnxruntime.dll')) + Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'onnxruntime.lib')) + Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'microsoft.ai.machinelearning.dll')) + Copy-Item ([System.IO.Path]::Combine($arm64_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($arm64_static_runtime_path_new, 'microsoft.ai.machinelearning.lib')) + + Copy-Item ([System.IO.Path]::Combine($arm_static_runtime_path_old, 'onnxruntime.dll')) ([System.IO.Path]::Combine($arm_static_runtime_path_new, 'onnxruntime.dll')) + Copy-Item ([System.IO.Path]::Combine($arm_static_runtime_path_old, 'onnxruntime.lib')) ([System.IO.Path]::Combine($arm_static_runtime_path_new, 'onnxruntime.lib')) + Copy-Item ([System.IO.Path]::Combine($arm_static_runtime_path_old, 'microsoft.ai.machinelearning.dll')) ([System.IO.Path]::Combine($arm_static_runtime_path_new, 'microsoft.ai.machinelearning.dll')) + Copy-Item ([System.IO.Path]::Combine($arm_static_runtime_path_old, 'microsoft.ai.machinelearning.lib')) ([System.IO.Path]::Combine($arm_static_runtime_path_new, 'microsoft.ai.machinelearning.lib')) + + Copy-Item -Recurse $uap_build_path_old $uap_build_path_new + + $merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'merged') + if (!(Test-Path $merged_nuget_path)) { + New-Item -Path $merged_nuget_path -ItemType Directory + } + + $merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $x64_nuget_package_name) + Start-Process -FilePath "7z" -ArgumentList "-tzip a -r $merged_nuget ." -WorkingDirectory $x64_nupkg_unzipped_directory -NoNewWindow -Wait + + workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64 + + - task: PowerShell@2 + displayName: 'Bundle Symbols NuGet' + inputs: + targetType: 'inline' + script: | + Add-Type -AssemblyName "System.IO.Compression.FileSystem" + + $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse) + $x64_nuget_package_name = $nupkgs[0].Name + $x64_nuget_package = $nupkgs[0].FullName + $x64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $x64_nupkg_unzipped_directory = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($x64_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($x64_nuget_package, $x64_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-x86 -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse) + $x86_nuget_package = $nupkgs[0].FullName + $x86_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $x86_nupkg_unzipped_directory = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($x86_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($x86_nuget_package, $x86_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-arm64 -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse) + $arm64_nuget_package = $nupkgs[0].FullName + $arm64_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $arm64_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($arm64_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($arm64_nuget_package, $arm64_nupkg_unzipped_directory) + + $nupkgs = (Get-ChildItem ..\nuget-artifact-arm -Filter Microsoft.AI.MachineLearning*.snupkg -Recurse) + $arm_nuget_package = $nupkgs[0].FullName + $arm_nupkg_unzipped_directory_root = $nupkgs[0].Directory.FullName + $arm_nupkg_unzipped_directory = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory_root, 'symbols', [System.IO.Path]::GetFileNameWithoutExtension($arm_nuget_package)) + [System.IO.Compression.ZipFile]::ExtractToDirectory($arm_nuget_package, $arm_nupkg_unzipped_directory) + + $x86_runtime_path_old = [System.IO.Path]::Combine($x86_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native') + $x86_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-x86', '_native') + $arm64_runtime_path_old = [System.IO.Path]::Combine($arm64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native') + $arm64_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm64', '_native') + $arm_runtime_path_old = [System.IO.Path]::Combine($arm_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native') + $arm_runtime_path_new = [System.IO.Path]::Combine($x64_nupkg_unzipped_directory, 'runtimes', 'win-arm', '_native') + + New-Item -Path $x86_runtime_path_new -ItemType Directory + New-Item -Path $arm64_runtime_path_new -ItemType Directory + New-Item -Path $arm_runtime_path_new -ItemType Directory + + Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'onnxruntime.pdb')) $x86_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($x86_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $x86_runtime_path_new + + Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.pdb')) $arm64_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm64_runtime_path_new + + Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.pdb')) $arm_runtime_path_new + Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm_runtime_path_new + + $merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'merged') + if (!(Test-Path $merged_nuget_path)) { + New-Item -Path $merged_nuget_path -ItemType Directory + } + + $merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $x64_nuget_package_name) + + Start-Process -FilePath "7z" -ArgumentList "-tzip a -r $merged_nuget ." -WorkingDirectory $x64_nupkg_unzipped_directory -NoNewWindow -Wait + + $merged_nuget_without_pdb = [System.IO.Path]::ChangeExtension($merged_nuget, '.nupkg') + + # Now we combine the DLLs and PDBs together, put them back in a folder under $(Build.SourcesDirectory) + # We won't upload the unzipped folder. We will just feed it to BinSkim. + 7z x -o$(Build.SourcesDirectory)\unzipped $merged_nuget + 7z -y x -o$(Build.SourcesDirectory)\unzipped $merged_nuget_without_pdb + + workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64 + + - script: | + dir $(Build.SourcesDirectory)\unzipped\runtimes\win-x64\_native \ No newline at end of file diff --git a/.pipelines/nuget_config/x64/packages.config b/.pipelines/nuget_config/x64/packages.config new file mode 100644 index 0000000000..73538d4665 --- /dev/null +++ b/.pipelines/nuget_config/x64/packages.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/.pipelines/nuget_config/x86/packages.config b/.pipelines/nuget_config/x86/packages.config new file mode 100644 index 0000000000..030c67af70 --- /dev/null +++ b/.pipelines/nuget_config/x86/packages.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/.pipelines/windowsai-steps.yml b/.pipelines/windowsai-steps.yml new file mode 100644 index 0000000000..bed9c9a2f4 --- /dev/null +++ b/.pipelines/windowsai-steps.yml @@ -0,0 +1,176 @@ +parameters: +- name: BuildArch + displayName: BuildArch + type: string + default: 'x64' + +- name: Runtime + displayName: MSVC Runtime, should be 'dynamic' or 'static'. + type: string + default: 'dynamic' + +- name: PythonPackageName + displayName: PythonPackageName on nuget.org to use + type: string + default: 'python' + +jobs: +- job: Windows_Packaging_${{ parameters.BuildArch }}_${{ parameters.Runtime }} + pool: + type: windows + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + ob_sdl_binskim_break: true + ob_sdl_binskim_scanOutputDirectoryOnly: true + steps: + - template: ../tools/ci_build/github/azure-pipelines/templates/telemetry-steps.yml@self + + - task: NuGetCommand@2 + displayName: 'NuGet restore' + inputs: + feedsToUse: config + nugetConfigPath: NuGet.config + restoreDirectory: '$(Build.BinariesDirectory)' + ${{ if eq(parameters.BuildArch, 'x64') }}: + restoreSolution: $(Build.SourcesDirectory)\.pipelines\nuget_config\x64\packages.config + ${{ if eq(parameters.BuildArch, 'x86') }}: + restoreSolution: $(Build.SourcesDirectory)\.pipelines\nuget_config\x86\packages.config + ${{ if eq(parameters.BuildArch, 'arm') }}: + restoreSolution: $(Build.SourcesDirectory)\.pipelines\nuget_config\x64\packages.config + ${{ if eq(parameters.BuildArch, 'arm64') }}: + restoreSolution: $(Build.SourcesDirectory)\.pipelines\nuget_config\x64\packages.config + + - script: | + @echo off + set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" + for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do ( + set vslatest="%%i" + if exist "%%i\Common7\Tools\vsdevcmd.bat" ( + set vsdevcmd="%%i\Common7\Tools\vsdevcmd.bat" + ) + ) + + @echo vslatest %vslatest% + @echo vsdevcmd %vsdevcmd% + + @echo ##vso[task.setvariable variable=vslatest]%vslatest% + @echo ##vso[task.setvariable variable=vsdevcmd]%vsdevcmd% -arch=${{ parameters.BuildArch }} + displayName: 'locate vsdevcmd via vswhere' + + - powershell: | + Write-Host "##vso[task.setvariable variable=BuildFlags]" + Write-Host "##vso[task.setvariable variable=ArtifactName]Microsoft.AI.MachineLearning.${{ parameters.BuildArch }}" + displayName: Initialize build flags + + - powershell: | + Write-Host "##vso[task.setvariable variable=BuildFlags]$(BuildFlags) --${{ parameters.BuildArch }}" + displayName: Add cross compilation flags for ARM + condition: and(ne('${{ parameters.BuildArch }}', 'x64'), ne('${{ parameters.BuildArch }}', 'x86')) + + - powershell: | + Write-Host "##vso[task.setvariable variable=BuildFlags]$(BuildFlags) --enable_msvc_static_runtime" + Write-Host "##vso[task.setvariable variable=ArtifactName]$(ArtifactName).StaticRuntime" + displayName: Add static runtime flags + condition: eq('${{ parameters.Runtime }}', 'static') + + # must call vsdevcmd first to add cmake to PATH + - script: | + curl -O -L https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-x86_64.zip + 7z x cmake-3.22.2-windows-x86_64.zip + set PYTHONHOME=$(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.7.9\tools + set PYTHONPATH=$(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.7.9\tools + $(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.7.9\tools\python.exe "$(Build.SourcesDirectory)\tools\ci_build\build.py" --build_dir $(Build.BinariesDirectory) --build_shared_lib --enable_onnx_tests --ms_experimental --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" --update --config RelWithDebInfo --enable_lto --use_telemetry --disable_rtti --enable_wcos $(BuildFlags) --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.19041.0 --cmake_path $(Build.BinariesDirectory)\cmake-3.22.2-windows-x86_64\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake-3.22.2-windows-x86_64\bin\ctest.exe + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Generate cmake config' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + ${{ if ne(parameters.BuildArch, 'x86') }}: + platform: ${{ parameters.BuildArch }} + ${{ if eq(parameters.BuildArch, 'x86') }}: + platform: 'Win32' + configuration: RelWithDebInfo + msbuildArchitecture: ${{ parameters.BuildArch }} + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + + - ${{ if eq(parameters.Runtime, 'dynamic') }}: + - script: | + xcopy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\winml_test_api.exe $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\winml_test_scenario.exe $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.SourcesDirectory)\winml\test\api\models\*.onnx $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.SourcesDirectory)\winml\test\scenario\cppwinrt\*.onnx $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.SourcesDirectory)\winml\test\scenario\models\*.onnx $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.SourcesDirectory)\winml\test\common\testdata\squeezenet\* $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.SourcesDirectory)\winml\test\collateral\models\*.onnx $(Build.ArtifactStagingDirectory)\test_artifact\ + xcopy $(Build.SourcesDirectory)\winml\test\collateral\models\ModelSubdirectory $(Build.ArtifactStagingDirectory)\test_artifact\ModelSubdirectory\ /i + copy $(Build.SourcesDirectory)\winml\test\collateral\images\*.png $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.SourcesDirectory)\winml\test\collateral\images\*.jpg $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.SourcesDirectory)\onnxruntime\test\testdata\sequence_length.onnx $(Build.ArtifactStagingDirectory)\test_artifact\ + copy $(Build.SourcesDirectory)\onnxruntime\test\testdata\sequence_construct.onnx $(Build.ArtifactStagingDirectory)\test_artifact\ + displayName: 'Copy WinML test collateral to artifact directory' + + + - ${{ if eq(parameters.BuildArch, 'x64') }}: + - script: | + call $(vsdevcmd) + msbuild Microsoft.AI.MachineLearning.Interop.csproj /p:Configuration=RelWithDebInfo /p:Platform="Any CPU" /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory) -restore + workingDirectory: '$(Build.SourcesDirectory)\csharp\src\Microsoft.AI.MachineLearning.Interop' + displayName: 'Build Microsoft.AI.MachineLearning.Interop.dll' + + - task: onebranch.pipeline.signing@1 + inputs: + command: 'sign' + signing_profile: 'external_distribution' + files_to_sign: '**/*.exe;**/*.dll' + search_root: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + displayName: 'Sign runtime DLLs' + + - ${{ if eq(parameters.BuildArch, 'x64') }}: + - script: | + call $(vsdevcmd) + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreateWindowsAIPackage /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory) /p:OnnxRuntimeSourceDirectory=$(Build.SourcesDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.snupkg $(Build.ArtifactStagingDirectory) + workingDirectory: '$(Build.SourcesDirectory)\csharp' + displayName: 'Create NuGet Package' + + - ${{ if eq(parameters.BuildArch, 'x86') }}: + - script: | + call $(vsdevcmd) + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreateWindowsAIPackage /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory) /p:OnnxRuntimeSourceDirectory=$(Build.SourcesDirectory) /p:TargetArchitecture=x86 + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.snupkg $(Build.ArtifactStagingDirectory) + workingDirectory: '$(Build.SourcesDirectory)\csharp' + displayName: 'Create NuGet Package' + + - ${{ if eq(parameters.BuildArch, 'arm64') }}: + - script: | + call $(vsdevcmd) + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreateWindowsAIPackage /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory) /p:OnnxRuntimeSourceDirectory=$(Build.SourcesDirectory) /p:TargetArchitecture=arm64 /p:ProtocDirectory=$(Build.BinariesDirectory)\host_protoc\Release + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.snupkg $(Build.ArtifactStagingDirectory) + workingDirectory: '$(Build.SourcesDirectory)\csharp' + displayName: 'Create NuGet Package' + + - ${{ if eq(parameters.BuildArch, 'arm') }}: + - script: | + call $(vsdevcmd) + msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreateWindowsAIPackage /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory) /p:OnnxRuntimeSourceDirectory=$(Build.SourcesDirectory) /p:TargetArchitecture=arm /p:ProtocDirectory=$(Build.BinariesDirectory)\host_protoc\Release + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory) + copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.snupkg $(Build.ArtifactStagingDirectory) + workingDirectory: '$(Build.SourcesDirectory)\csharp' + displayName: 'Create NuGet Package' + + - task: onebranch.pipeline.signing@1 + inputs: + command: 'sign' + signing_profile: 'external_distribution' + files_to_sign: '**/*.exe;**/*.dll' + search_root: '$(Build.ArtifactStagingDirectory)\test_artifact' + displayName: 'Sign test_artifact' diff --git a/tools/ci_build/policheck_exclusions.xml b/tools/ci_build/policheck_exclusions.xml new file mode 100644 index 0000000000..9888245b48 --- /dev/null +++ b/tools/ci_build/policheck_exclusions.xml @@ -0,0 +1,4 @@ + + + LABELMAP.CS|OPERATORKERNELS.MD|BABEL.CONFIG.JS|METRO.CONFIG.JS|DMLOPERATORACTIVATION.CPP|DATA_OPS.CC|ONNX_CONVERTER.CC|ONNXOPS.PY|CPYTHON-PUBKEYS.TXT +