From e0c9cdaa73e2de6da4d344c6366744589751982c Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 23 Jan 2020 20:02:18 -0800 Subject: [PATCH] Fix the nuget pipelines (#2901) --- cmake/CMakeLists.txt | 2 +- .../nuget/templates/cpu-mklml.yml | 1 - .../templates/cpu-nocontribops-arm64.yml | 10 +- .../azure-pipelines/nuget/templates/cpu.yml | 2 - .../nuget/templates/test_win.yml | 20 +++- .../templates/win-download-test-data.yml | 8 -- .../azure-pipelines/templates/win-ci-2019.yml | 7 +- .../azure-pipelines/templates/win-ci-arm.yml | 97 +++++++++++-------- 8 files changed, 83 insertions(+), 64 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index bcbf43d3b8..09527d778c 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -280,6 +280,7 @@ set_target_properties(libprotobuf PROPERTIES FOLDER "External/Protobuf") set_target_properties(libprotobuf-lite PROPERTIES FOLDER "External/Protobuf") if(TARGET libprotoc) set_target_properties(libprotoc PROPERTIES FOLDER "External/Protobuf") + add_executable(protobuf::protoc ALIAS protoc) endif() if(TARGET protoc) set_target_properties(protoc PROPERTIES FOLDER "External/Protobuf") @@ -289,7 +290,6 @@ if (onnxruntime_USE_FULL_PROTOBUF) else() add_library(protobuf::libprotobuf ALIAS libprotobuf-lite) endif() -add_executable(protobuf::protoc ALIAS protoc) if(UNIX AND onnxruntime_ENABLE_LTO) #https://github.com/protocolbuffers/protobuf/issues/5923 diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml index 41abf2c511..b887626686 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml @@ -13,7 +13,6 @@ jobs: JobName: 'Windows_CI_Dev' BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_mklml --build_shared_lib --enable_onnx_tests --cmake_generator "Visual Studio 16 2019"' BuildArch: 'x64' - msbuildArchitecture: 'amd64' EnvSetupScript: 'setup_env.bat' sln_platform: 'x64' DoDebugBuild: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml index 5576255ee7..1ca2a43f80 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml @@ -12,9 +12,8 @@ jobs: AgentPool : 'Win-CPU-2019' ArtifactName: 'drop-nuget' JobName: 'Windows_CI_Dev' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019" --disable_contrib_ops --enable_msvc_static_runtime' - BuildArch: 'x64' - msbuildArchitecture: 'amd64' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --cmake_generator "Visual Studio 16 2019" --disable_contrib_ops --enable_msvc_static_runtime' + BuildArch: 'x64' EnvSetupScript: 'setup_env.bat' sln_platform: 'x64' DoDebugBuild: 'false' @@ -32,9 +31,8 @@ jobs: AgentPool : 'Win-CPU-2019' ArtifactName: 'drop-win-x86-zip' JobName: 'Windows_CI_Dev_x86' - BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --x86 --use_telemetry --cmake_generator "Visual Studio 16 2019" --disable_contrib_ops --enable_msvc_static_runtime' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --x86 --cmake_generator "Visual Studio 16 2019" --disable_contrib_ops --enable_msvc_static_runtime' BuildArch: 'x86' - msbuildArchitecture: 'x86' EnvSetupScript: 'setup_env_x86.bat' sln_platform: 'Win32' DoDebugBuild: 'false' @@ -53,7 +51,7 @@ jobs: parameters: AgentPool : 'Win-CPU-2019' JobName: 'Windows_Arm64_Dev' - BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --disable_contrib_ops --enable_msvc_static_runtime --build_shared_lib --arm64' + BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_generator "Visual Studio 16 2019" --disable_contrib_ops --enable_msvc_static_runtime --arm64' DoDebugBuild: 'false' DoNugetPack : 'true' DoCompliance: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 2c9da8e8f8..a5cd84a3c1 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -16,7 +16,6 @@ jobs: JobName: 'Windows_CI_Dev' BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019"' BuildArch: 'x64' - msbuildArchitecture: 'amd64' EnvSetupScript: 'setup_env.bat' sln_platform: 'x64' DoDebugBuild: 'false' @@ -36,7 +35,6 @@ jobs: JobName: 'Windows_CI_Dev_x86' BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --use_featurizers --enable_onnx_tests --x86 --use_telemetry --cmake_generator "Visual Studio 16 2019"' BuildArch: 'x86' - msbuildArchitecture: 'x86' EnvSetupScript: 'setup_env_x86.bat' sln_platform: 'Win32' DoDebugBuild: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml index bf2897e904..93b11c2a19 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml @@ -15,7 +15,25 @@ jobs: value: '$(Build.BinariesDirectory)' steps: - - template: set-test-data-variables-step.yml + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: x64 + + - task: NuGetToolInstaller@0 + displayName: Use Nuget 4.9 + inputs: + versionSpec: 4.9.4 + + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - template: set-test-data-variables-step.yml - template: win-download-test-data.yml - task: BatchScript@1 diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/win-download-test-data.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/win-download-test-data.yml index f5d5790c56..dbaf52ef28 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/win-download-test-data.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/win-download-test-data.yml @@ -5,14 +5,6 @@ steps: inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py' arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) - pythonInterpreter: '$(Build.BinariesDirectory)\packages\python\python.exe' workingDirectory: $(Build.BinariesDirectory) env: Test_Data_Download_Key: $(ort-internal-test-data-sas) - -- task: CmdLine@1 - continueOnError: true - displayName: 'Run OpenCPPCoverage installer' - inputs: - filename: '$(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe' - arguments: '/SP- /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /DIR="$(Build.BinariesDirectory)\OpenCppCoverage"' \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml index dd101c6fbe..272e05af7f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml @@ -9,7 +9,6 @@ parameters: DoEsrp: 'false' DoTestCoverage: 'false' BuildArch: 'x64' # Optional. Options: x86, x64 - msbuildArchitecture: 'amd64' # Options: x86, amd64 sln_platform: 'x64' # Options: Win32, x64 EnvSetupScript: 'setup_env.bat' CudaVersion: '' @@ -153,11 +152,11 @@ jobs: # Nuget packaging if needed - ${{ if eq(parameters['DoNugetPack'], 'true') }}: - task: BatchScript@1 - displayName: 'Setup VS2017 env vars' + displayName: 'Setup VS2019 env vars' inputs: filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat' - arguments: ${{ parameters.msbuildArchitecture }} - modifyEnvironment: true + arguments: ${{ parameters.BuildArch }} + modifyEnvironment: true # Esrp signing - template: esrp_dll.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-arm.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-arm.yml index e69615a703..d6d614c0cc 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci-arm.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-arm.yml @@ -1,6 +1,6 @@ parameters: - AgentPool : 'Win-CPU' DoDebugBuild: 'true' + DoCompliance: 'false' BuildCommand: '' JobName: 'Windows_CI_Dev' DoNugetPack: 'false' @@ -8,6 +8,8 @@ parameters: ArtifactName: 'drop-nuget' DoEsrp: 'false' BuildArch: 'x64' + DoTestCoverage: 'false' + sln_platform: 'Arm64' # Options: Win32, x64,Arm64 SetVcvars: 'false' MsbuildArguments: '/m' EnvSetupScript: 'setup_env.bat' @@ -28,57 +30,70 @@ jobs: steps: - template: set-test-data-variables-step.yml - - template: windows-build-tools-setup-steps.yml - parameters: - EnvSetupScript: ${{ parameters.EnvSetupScript }} - buildArch: ${{ parameters.BuildArch }} - setVcvars: ${{ parameters.SetVcvars }} + - powershell: | + if($env:TELEMETRYGUID) + { + $length = $env:TELEMETRYGUID.length + $fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \ + TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")" + New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force + Write-Output "Enabling TELEMETRY" + } + displayName: 'Create TraceLoggingConfigPrivate.h For WinML Telemetry' + env: + TELEMETRYGUID: $(TELEMETRYGUID) - - task: CmdLine@1 - displayName: 'Download test data and generate cmake config' + - task: BatchScript@1 + displayName: 'setup env' inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: '$(BuildCommand) --update --config Debug RelWithDebInfo' - workingDirectory: "$(Build.BinariesDirectory)" + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' - # Build Debug Mode - - ${{ if eq(parameters['DoDebugBuild'], 'true') }}: - - task: VSBuild@1 - displayName: 'Build Debug' - inputs: - solution: '$(Build.BinariesDirectory)\Debug\onnxruntime.sln' - platform: 'arm64' - configuration: 'Debug' - msbuildArguments: ${{ parameters.MsbuildArguments }} - msbuildArchitecture: 'x64' - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\Debug' - createLogFile: true + - task: BatchScript@1 + displayName: 'Setup VS2019 env vars' + inputs: + filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat' + arguments: amd64_arm + modifyEnvironment: true + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: x64 + + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '$(BuildCommand) --update --config RelWithDebInfo' + workingDirectory: '$(Build.BinariesDirectory)' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: ${{ parameters.sln_platform }} + configuration: RelWithDebInfo + msbuildArchitecture: ${{ parameters.BuildArch }} + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true # Build RelWithDebInfo -- this variable required to build C# - script: | @echo ##vso[task.setvariable variable=Configuration]RelWithDebInfo - - task: VSBuild@1 - displayName: 'Build RelWithDebInfo' - inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: 'arm64' - configuration: 'RelWithDebInfo' - msbuildArguments: ${{ parameters.MsbuildArguments }} - msbuildArchitecture: 'x64' - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' # Nuget packaging if needed - ${{ if eq(parameters['DoNugetPack'], 'true') }}: - - task: BatchScript@1 - displayName: 'Setup VS2017 env vars' - inputs: - filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat' - arguments: 'x64' - modifyEnvironment: true - # Esrp signing - template: esrp_dll.yml parameters: