From 0fbbb6a43e43fcbcfa176736d3601c65de92acd8 Mon Sep 17 00:00:00 2001 From: Sheil Kumar Date: Thu, 6 Apr 2023 08:51:07 -0700 Subject: [PATCH] WindowsAI build failing due to deprecated .NET5 SDK missing in build image (#15383) WindowsAI build failing due to deprecated .NET5 SDK missing in build image .NET5 was deprecated last year, and recently the build machine images have been updated to not include this SDK. Unblock failing builds by force insalling .NET5 SDK as part of the build pipeline. --- ...anch.Nuget-WindowsAI-Pipeline.Official.yml | 48 ++++---- .pipelines/windowsai-steps.yml | 4 + ...icrosoft.AI.MachineLearning.Interop.csproj | 8 +- .../nuget/templates/windowsai.yml | 116 +++++++++--------- 4 files changed, 90 insertions(+), 86 deletions(-) diff --git a/.pipelines/OneBranch.Nuget-WindowsAI-Pipeline.Official.yml b/.pipelines/OneBranch.Nuget-WindowsAI-Pipeline.Official.yml index ddca6ff805..fa2b475fe9 100644 --- a/.pipelines/OneBranch.Nuget-WindowsAI-Pipeline.Official.yml +++ b/.pipelines/OneBranch.Nuget-WindowsAI-Pipeline.Official.yml @@ -12,7 +12,7 @@ variables: ComponentDetection.Timeout: 1200 resources: - repositories: + repositories: - repository: templates type: git name: OneBranch.Pipelines/GovernedTemplates @@ -22,7 +22,7 @@ extends: template: v2/OneBranch.Official.CrossPlat.yml@templates parameters: nugetPublishing: - feeds: + feeds: - name: PublicPackages/ORT-Nightly files_to_publish: '*.nupkg;!*.symbols.nupkg' continueOnConflict: true @@ -30,7 +30,7 @@ extends: submodules: false globalSdl: # https://aka.ms/obpipelines/sdl tsa: - enabled: true + enabled: true prefast: enabled: false cg: @@ -47,42 +47,42 @@ extends: - template: .pipelines/windowsai-steps.yml@self parameters: BuildArch: x64 - + - template: .pipelines/windowsai-steps.yml@self parameters: BuildArch: x86 PythonPackageName: pythonx86 - + - template: .pipelines/windowsai-steps.yml@self parameters: BuildArch: arm - + - template: .pipelines/windowsai-steps.yml@self parameters: BuildArch: arm64 - + - template: .pipelines/windowsai-steps.yml@self parameters: BuildArch: x64 Runtime: static - + - template: .pipelines/windowsai-steps.yml@self parameters: BuildArch: x86 PythonPackageName: pythonx86 Runtime: static - + - template: .pipelines/windowsai-steps.yml@self parameters: BuildArch: arm Runtime: static - + - template: .pipelines/windowsai-steps.yml@self parameters: BuildArch: arm64 Runtime: static - - + + - job: NuGet_Packaging pool: type: windows @@ -113,7 +113,7 @@ extends: inputs: artifactName: 'drop_Windows_Build_Windows_Packaging_x86_dynamic' targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x86' - + - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - NuGet DirectML arm64' inputs: @@ -275,7 +275,7 @@ extends: $merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $x64_nuget_package_name) Start-Process -FilePath "7z" -ArgumentList "-tzip a -r $merged_nuget ." -WorkingDirectory $x64_nupkg_unzipped_directory -NoNewWindow -Wait - + workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64 - task: PowerShell@2 @@ -326,7 +326,7 @@ extends: Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.pdb')) $arm64_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm64_runtime_path_new - + Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.pdb')) $arm_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm_runtime_path_new @@ -343,14 +343,14 @@ extends: # Now we combine the DLLs and PDBs together, put them back in a folder under $(Build.SourcesDirectory) # We won't upload the unzipped folder. We will just feed it to BinSkim. - 7z x -o$(Build.SourcesDirectory)\unzipped $merged_nuget + 7z x -o$(Build.SourcesDirectory)\unzipped $merged_nuget 7z -y x -o$(Build.SourcesDirectory)\unzipped $merged_nuget_without_pdb - + workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64 - + - script: | dir $(Build.SourcesDirectory)\unzipped\runtimes\win-x64\_native - + - job: NuGet_Publishing pool: type: windows @@ -359,7 +359,7 @@ extends: ob_sdl_binskim_enabled: false ob_pipelineartifacts_enabled: false ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: - ob_nugetPublishing_enabled: true + ob_nugetPublishing_enabled: true dependsOn: - NuGet_Packaging condition: succeeded() @@ -367,13 +367,13 @@ extends: - task: DownloadPipelineArtifact@2 displayName: 'Download Pipeline Artifact' inputs: - buildType: 'current' + buildType: 'current' artifactName: 'drop_Windows_Build_NuGet_Packaging' itemPattern: 'merged/**' targetPath: '$(Build.BinariesDirectory)' - + - powershell: | Rename-Item -Path merged packages - + workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Rename nuget folder' \ No newline at end of file + displayName: 'Rename nuget folder' diff --git a/.pipelines/windowsai-steps.yml b/.pipelines/windowsai-steps.yml index bc5b7d2f4c..0b736da427 100644 --- a/.pipelines/windowsai-steps.yml +++ b/.pipelines/windowsai-steps.yml @@ -24,6 +24,10 @@ jobs: ob_sdl_binskim_break: true ob_sdl_binskim_scanOutputDirectoryOnly: true steps: + - task: UseDotNet@2 + inputs: + version: '5.x' + - template: ../tools/ci_build/github/azure-pipelines/templates/telemetry-steps.yml@self - task: NuGetCommand@2 diff --git a/csharp/src/Microsoft.AI.MachineLearning.Interop/Microsoft.AI.MachineLearning.Interop.csproj b/csharp/src/Microsoft.AI.MachineLearning.Interop/Microsoft.AI.MachineLearning.Interop.csproj index 2fcae569d3..5e727372d0 100644 --- a/csharp/src/Microsoft.AI.MachineLearning.Interop/Microsoft.AI.MachineLearning.Interop.csproj +++ b/csharp/src/Microsoft.AI.MachineLearning.Interop/Microsoft.AI.MachineLearning.Interop.csproj @@ -4,10 +4,10 @@ Microsoft.AI.MachineLearning.Interop net5.0-windows10.0.17763.0 true - + Any CPU Debug - + ..\..\..\build\Windows $(OnnxRuntimeBuildDirectory)\$(Configuration)\$(Configuration) $(BuildOutputDir)\Microsoft.AI.MachineLearning.Interop @@ -40,10 +40,10 @@ - + - \ No newline at end of file + diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml index 3693c37ccd..736b91cb4a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml @@ -22,7 +22,7 @@ jobs: - template: ../../templates/windowsai-nuget-build.yml parameters: BuildArch: 'x86' - + - job: WindowsAI_DirectML_ARM64 timeoutInMinutes: 120 workspace: @@ -125,7 +125,7 @@ jobs: inputs: artifactName: 'Microsoft.AI.MachineLearning.x86' targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x86' - + - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - NuGet DirectML arm64' inputs: @@ -297,7 +297,7 @@ jobs: Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile $zip_tool Start-Process -FilePath $zip_tool -ArgumentList "-r $merged_nuget ." -WorkingDirectory $x64_nupkg_unzipped_directory -NoNewWindow -Wait - + Copy-Item -Path $merged_nuget -Destination $merged_nuget_as_zip workingDirectory: $(Build.BinariesDirectory)\nuget-artifact-x64 @@ -349,7 +349,7 @@ jobs: Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'onnxruntime.pdb')) $arm64_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm64_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm64_runtime_path_new - + Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'onnxruntime.pdb')) $arm_runtime_path_new Copy-Item ([System.IO.Path]::Combine($arm_runtime_path_old, 'microsoft.ai.machinelearning.pdb')) $arm_runtime_path_new @@ -365,7 +365,7 @@ jobs: if (!(Test-Path $zip_tool_directory)) { New-Item -Path $zip_tool_directory -ItemType Directory } - + $zip_tool = [System.IO.Path]::Combine($zip_tool_directory, 'zip.exe') Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile $zip_tool @@ -441,31 +441,31 @@ jobs: msbuild /p:Platform=x64 Microsoft.AI.MachineLearning.Tests.vcxproj workingDirectory: $(Build.SourcesDirectory)\csharp\test\Microsoft.AI.MachineLearning.Tests - - task: PowerShell@2 - displayName: 'NuGet Tests: Fix Nuget Package references (.NET 5.0)' - inputs: - targetType: 'inline' - script: | - Add-Type -AssemblyName "System.IO.Compression.FileSystem" - $src_root_dir = $Env:BUILD_SOURCESDIRECTORY; - $artifacts_staging_dir = $Env:BUILD_ARTIFACTSTAGINGDIRECTORY; - $merged_nuget_path = [System.IO.Path]::Combine($artifacts_staging_dir, 'merged') - $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*) - $merged_nuget_package = $nupkgs[0] - $merged_nuget_package_name = $merged_nuget_package.Name - $matched_name = ($merged_nuget_package_name -match "Microsoft.AI.MachineLearning.(?.*).nupkg") - $package_version = $matches['version'] - $src_dir = [System.IO.Path]::Combine($src_root_dir, 'csharp', 'test', 'Microsoft.AI.MachineLearning.Tests.DotNet5_0') - $input_csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.DotNet5_0.csproj.pp') - $csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.DotNet5_0.csproj') - $input_csproj_content = Get-Content -Path $input_csproj - $csproj_content = $input_csproj_content -replace '\[PackageVersion\]', $package_version - Set-Content -Path $csproj -Value $csproj_content - $input_nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config.pp') - $nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config') - $input_nuget_config_content = Get-Content -Path $input_nuget_config - $nuget_config_content = $input_nuget_config_content -replace '\[BuildPackageSource\]', $merged_nuget_path - Set-Content -Path $nuget_config -Value $nuget_config_content + - task: PowerShell@2 + displayName: 'NuGet Tests: Fix Nuget Package references (.NET 5.0)' + inputs: + targetType: 'inline' + script: | + Add-Type -AssemblyName "System.IO.Compression.FileSystem" + $src_root_dir = $Env:BUILD_SOURCESDIRECTORY; + $artifacts_staging_dir = $Env:BUILD_ARTIFACTSTAGINGDIRECTORY; + $merged_nuget_path = [System.IO.Path]::Combine($artifacts_staging_dir, 'merged') + $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*) + $merged_nuget_package = $nupkgs[0] + $merged_nuget_package_name = $merged_nuget_package.Name + $matched_name = ($merged_nuget_package_name -match "Microsoft.AI.MachineLearning.(?.*).nupkg") + $package_version = $matches['version'] + $src_dir = [System.IO.Path]::Combine($src_root_dir, 'csharp', 'test', 'Microsoft.AI.MachineLearning.Tests.DotNet5_0') + $input_csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.DotNet5_0.csproj.pp') + $csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.DotNet5_0.csproj') + $input_csproj_content = Get-Content -Path $input_csproj + $csproj_content = $input_csproj_content -replace '\[PackageVersion\]', $package_version + Set-Content -Path $csproj -Value $csproj_content + $input_nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config.pp') + $nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config') + $input_nuget_config_content = Get-Content -Path $input_nuget_config + $nuget_config_content = $input_nuget_config_content -replace '\[BuildPackageSource\]', $merged_nuget_path + Set-Content -Path $nuget_config -Value $nuget_config_content workingDirectory: $(Build.ArtifactStagingDirectory)\merged - task: PowerShell@2 @@ -480,31 +480,31 @@ jobs: workingDirectory: $(Build.SourcesDirectory)\csharp\test\Microsoft.AI.MachineLearning.Tests.DotNet5_0 - - task: PowerShell@2 - displayName: 'NuGet Tests: Fix Nuget Package references (C# UWP)' - inputs: - targetType: 'inline' - script: | - Add-Type -AssemblyName "System.IO.Compression.FileSystem" - $src_root_dir = $Env:BUILD_SOURCESDIRECTORY; - $artifacts_staging_dir = $Env:BUILD_ARTIFACTSTAGINGDIRECTORY; - $merged_nuget_path = [System.IO.Path]::Combine($artifacts_staging_dir, 'merged') - $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*) - $merged_nuget_package = $nupkgs[0] - $merged_nuget_package_name = $merged_nuget_package.Name - $matched_name = ($merged_nuget_package_name -match "Microsoft.AI.MachineLearning.(?.*).nupkg") - $package_version = $matches['version'] - $src_dir = [System.IO.Path]::Combine($src_root_dir, 'csharp', 'test', 'Microsoft.AI.MachineLearning.Tests.Uwp') - $input_csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.Uwp.csproj.pp') - $csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.Uwp.csproj') - $input_csproj_content = Get-Content -Path $input_csproj - $csproj_content = $input_csproj_content -replace '\[PackageVersion\]', $package_version - Set-Content -Path $csproj -Value $csproj_content - $input_nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config.pp') - $nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config') - $input_nuget_config_content = Get-Content -Path $input_nuget_config - $nuget_config_content = $input_nuget_config_content -replace '\[BuildPackageSource\]', $merged_nuget_path - Set-Content -Path $nuget_config -Value $nuget_config_content + - task: PowerShell@2 + displayName: 'NuGet Tests: Fix Nuget Package references (C# UWP)' + inputs: + targetType: 'inline' + script: | + Add-Type -AssemblyName "System.IO.Compression.FileSystem" + $src_root_dir = $Env:BUILD_SOURCESDIRECTORY; + $artifacts_staging_dir = $Env:BUILD_ARTIFACTSTAGINGDIRECTORY; + $merged_nuget_path = [System.IO.Path]::Combine($artifacts_staging_dir, 'merged') + $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*) + $merged_nuget_package = $nupkgs[0] + $merged_nuget_package_name = $merged_nuget_package.Name + $matched_name = ($merged_nuget_package_name -match "Microsoft.AI.MachineLearning.(?.*).nupkg") + $package_version = $matches['version'] + $src_dir = [System.IO.Path]::Combine($src_root_dir, 'csharp', 'test', 'Microsoft.AI.MachineLearning.Tests.Uwp') + $input_csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.Uwp.csproj.pp') + $csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.Uwp.csproj') + $input_csproj_content = Get-Content -Path $input_csproj + $csproj_content = $input_csproj_content -replace '\[PackageVersion\]', $package_version + Set-Content -Path $csproj -Value $csproj_content + $input_nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config.pp') + $nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config') + $input_nuget_config_content = Get-Content -Path $input_nuget_config + $nuget_config_content = $input_nuget_config_content -replace '\[BuildPackageSource\]', $merged_nuget_path + Set-Content -Path $nuget_config -Value $nuget_config_content workingDirectory: $(Build.ArtifactStagingDirectory)\merged - task: PowerShell@2 @@ -536,17 +536,17 @@ jobs: pushd .\Microsoft.AI.MachineLearning.Tests\x64\Debug .\Microsoft.AI.MachineLearning.Tests.exe popd - + Write-Host "Run Microsoft.AI.MachineLearning CSharp Tests (DotNet5_0)" pushd .\Microsoft.AI.MachineLearning.Tests.DotNet5_0\bin\Debug\net5.0-windows10.0.17763.0 .\Microsoft.AI.MachineLearning.Tests.DotNet5_0.exe popd - + Write-Host "Run Microsoft.AI.MachineLearning CSharp Tests (AnyCpu)" pushd .\Microsoft.AI.MachineLearning.Tests.DotNet5_0\bin\x64\Debug\net5.0-windows10.0.17763.0 .\Microsoft.AI.MachineLearning.Tests.DotNet5_0.exe popd - + Write-Host "Done!" workingDirectory: $(Build.SourcesDirectory)\csharp\test