Windows CI changes(#2650)

This commit is contained in:
Changming Sun 2019-12-13 12:23:49 -08:00 committed by GitHub
parent 18bdde34dc
commit a46a28b7d8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 546 additions and 120 deletions

View file

@ -100,6 +100,13 @@ if(NOT WIN32)
message(WARNING "Instrument is only supported on Windows now")
set(onnxruntime_ENABLE_INSTRUMENT OFF)
endif()
else()
check_cxx_compiler_flag(/d2FH4- HAS_D2FH4)
if (HAS_D2FH4)
message("Enabling /d2FH4-")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /d2FH4-")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /d2FH4-")
endif()
endif()
if(onnxruntime_USE_OPENMP)

View file

@ -122,10 +122,7 @@ jobs:
- template: templates/clean-agent-build-directory-step.yml
- job: Windows_py_Wheels
workspace:
clean: all
pool: Win-CPU
timeoutInMinutes: 120
pool: 'Win-CPU-2019'
strategy:
matrix:
Python35:
@ -135,38 +132,68 @@ jobs:
Python37:
python.version: '3.7'
variables:
buildDirectory: '$(Build.SourcesDirectory)\build'
steps:
- task: CondaEnvironment@1
inputs:
createCustomEnvironment: true
environmentName: 'py$(python.version)'
packageSpecs: 'python=$(python.version)'
cleanEnvironment: true
OrtPackageId: 'Microsoft.ML.OnnxRuntime'
MsbuildArguments: '-maxcpucount'
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
EnvSetupScript: setup_env.bat
buildArch: x64
setVcvars: true
BuildConfig: 'Release'
timeoutInMinutes: 60
workspace:
clean: all
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: $(python.version)
addToPath: true
architecture: 'x64'
- template: templates/set-test-data-variables-step.yml
- task: BatchScript@1
displayName: 'Run build script'
inputs:
filename: 'build.bat'
arguments: ' --build_dir $(buildDirectory) --config Release --use_openmp --build_wheel'
workingFolder: "$(Build.SourcesDirectory)"
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(buildDirectory)'
Contents: '**\dist\*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- task: PythonScript@0
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py'
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
workingDirectory: $(Build.BinariesDirectory)
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
displayName: 'Component Detection'
- task: PythonScript@0
displayName: 'BUILD'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --build_wheel --use_automl --enable_onnx_tests --parallel'
workingDirectory: '$(Build.BinariesDirectory)'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
Contents: '**\dist\*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- template: templates/clean-agent-build-directory-step.yml
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
displayName: 'Component Detection'
condition: succeeded()
- template: templates/clean-agent-build-directory-step.yml
- job: Windows_py_GPU_Wheels
workspace:
@ -185,13 +212,11 @@ jobs:
Python37:
python.version: '3.7'
steps:
- task: CondaEnvironment@1
inputs:
createCustomEnvironment: true
environmentName: 'py$(python.version)'
packageSpecs: 'python=$(python.version)'
cleanEnvironment: true
- task: UsePythonVersion@0
inputs:
versionSpec: $(python.version)
addToPath: true
architecture: 'x64'
- task: BatchScript@1
displayName: 'Setup VS2017 env vars'
inputs:

View file

@ -15,6 +15,6 @@ schedules:
jobs:
- template: templates/cpu.yml
parameters:
AgentPool : $(AgentPoolWin)
AgentPool : 'Win-CPU-2019'
DoEsrp: 'true'
DoCompliance: 'true'

View file

@ -1,11 +1,10 @@
# Defined as pipeline variables
# variables:
# AgentPoolWin : 'Win-CPU'
# AgentPoolLinux : 'Linux-CPU'
# AgentPoolMacOS : 'macOS-10.13'
jobs:
- template: templates/cpu.yml
parameters:
AgentPool : $(AgentPoolWin)
AgentPool : 'Win-CPU-2019'
DoEsrp: 'false'

View file

@ -1,6 +1,5 @@
# Defined as pipeline variables
# variables:
# AgentPoolWin : 'Win-CPU'
# AgentPoolLinux : 'Linux-CPU'
# AgentPoolMacOS : 'macOS-10.13'
@ -9,11 +8,11 @@ parameters:
DoCompliance: 'false'
jobs:
- template: ../../templates/win-ci.yml
- template: ../../templates/win-ci-2019.yml
parameters:
AgentPool : $(AgentPoolWin)
AgentPool : 'Win-CPU-2019'
JobName: 'Windows_CI_Dev'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --use_telemetry'
BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019"'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: ${{ parameters.DoCompliance }}
@ -24,11 +23,11 @@ jobs:
mkdir $(Build.ArtifactStagingDirectory)\testdata
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
- template: ../../templates/win-x86-ci.yml
- template: ../../templates/win-x86-ci-2019.yml
parameters:
AgentPool : $(AgentPoolWin)
AgentPool : 'Win-CPU-2019'
JobName: 'Windows_CI_Dev_x86'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --x86 --use_telemetry'
BuildCommand: '--build_dir $(Build.BinariesDirectory) --skip_submodule_sync --use_openmp --build_shared_lib --enable_onnx_tests --x86 --use_telemetry --cmake_generator "Visual Studio 16 2019"'
DoDebugBuild: 'false'
DoNugetPack : 'true'
DoCompliance: 'false'
@ -103,7 +102,7 @@ jobs:
- job: NuGet_Packaging
workspace:
clean: all
pool: $(AgentPoolWin)
pool: 'Win-CPU-2019'
dependsOn:
- Windows_CI_Dev
- Windows_CI_Dev_x86

View file

@ -1,10 +1,4 @@
steps:
- task: SDLNativeRules@2
displayName: 'Run the PREfast SDL Native Rules for MSBuild'
inputs:
userProvideBuildInfo: 'auto'
continueOnError: true
- task: CredScan@2
displayName: 'Run CredScan'
inputs:
@ -17,7 +11,19 @@ steps:
arguments: 'analyze $(Build.BinariesDirectory)\RelWithDebInfo\*.dll --recurse --verbose'
continueOnError: true
- task: PublishSecurityAnalysisLogs@2
- task: securedevelopmentteam.vss-secure-development-tools.build-task-prefast.SDLNativeRules@2
displayName: 'Run the PREfast SDL Native Rules for MSBuild'
continueOnError: true
- task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1
displayName: 'Create Security Analysis Report'
inputs:
BinSkim: true
BinSkimBreakOn: WarningAbove
CredScan: true
SDLNativeRules: true
- task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
displayName: 'Publish Security Analysis Logs'
continueOnError: true

View file

@ -0,0 +1,188 @@
parameters:
DoDebugBuild: 'true'
DoCompliance: 'false'
BuildCommand: ''
JobName: 'Windows_CI_Dev'
DoNugetPack: 'false'
NuPackScript : ''
ArtifactName: 'drop-nuget'
DoEsrp: 'false'
DoTestCoverage: 'false'
BuildArch: 'x64'
SetVcvars: 'false'
msbuildArchitecture: 'amd64'
EnvSetupScript: 'setup_env.bat'
CudaVersion: ''
AgentPool: 'Win-CPU'
AgentDemands: []
OrtPackageId: Microsoft.ML.OnnxRuntime
jobs:
- job: ${{ parameters.JobName }}
timeoutInMinutes: 120
workspace:
clean: all
pool:
name: ${{ parameters.AgentPool }}
demands: ${{ parameters.AgentDemands }}
variables:
buildDirectory: '$(Build.BinariesDirectory)'
BuildCommand: ${{ parameters.BuildCommand }}
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
DotNetExe: 'dotnet.exe'
CUDA_VERSION: ${{ parameters.CudaVersion }}
steps:
- powershell: |
if($env:TELEMETRYGUID)
{
$length = $env:TELEMETRYGUID.length
$fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \
TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")"
New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force
}
displayName: 'Create TraceLoggingConfigPrivate.h For WinML Telemetry'
- task: UsePythonVersion@0
inputs:
versionSpec: '3.7'
addToPath: true
architecture: ${{ parameters.BuildArch }}
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- task: PythonScript@0
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '$(BuildCommand) --update --config RelWithDebInfo'
workingDirectory: '$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build'
inputs:
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
platform: 'x64'
configuration: RelWithDebInfo
msbuildArchitecture: ${{ parameters.BuildArch }}
maximumCpuCount: true
logProjectEvents: true
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
createLogFile: true
clean: true
# Build RelWithDebInfo -- this variable required to build C#
- script: |
@echo ##vso[task.setvariable variable=Configuration]RelWithDebInfo
- template: set-test-data-variables-step.yml
- task: NuGetToolInstaller@0
displayName: Use Nuget 4.9
inputs:
versionSpec: 4.9.4
- task: PythonScript@0
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py'
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
workingDirectory: $(Build.BinariesDirectory)
- task: DotNetCoreCLI@2
displayName: 'Restore nuget packages'
inputs:
command: restore
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: 'RelWithDebInfo'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
displayName: 'Build C#'
inputs:
command: build
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: 'RelWithDebInfo'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
displayName: 'Test C#'
inputs:
command: test
projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj'
configuration: 'RelWithDebInfo'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- script: |
mklink /D /J $(Build.BinariesDirectory)\RelWithDebInfo\models $(Build.BinariesDirectory)\models
DIR dist\ /S /B > wheel_filename_file
set /p WHEEL_FILENAME=<wheel_filename_file
del wheel_filename_file
python.exe -m pip install -q --upgrade %WHEEL_FILENAME%
set PATH=%PATH%;$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo
@echo %PATH%
python $(Build.SourcesDirectory)\tools\ci_build\build.py $(BuildCommand) --test --config RelWithDebInfo
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
displayName: 'Run tests'
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
# Nuget packaging if needed
- ${{ if eq(parameters['DoNugetPack'], 'true') }}:
- task: BatchScript@1
displayName: 'Setup VS2017 env vars'
inputs:
filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat'
arguments: ${{ parameters.msbuildArchitecture }}
modifyEnvironment: true
# Esrp signing
- template: esrp_dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: ${{ parameters.DoEsrp }}
- template: esrp_dll.yml
parameters:
FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
DisplayName: 'ESRP - Sign C# dlls'
DoEsrp: ${{ parameters.DoEsrp }}
- script: |
${{ parameters.NuPackScript }}
workingDirectory: '$(Build.SourcesDirectory)\csharp'
displayName: 'Create NuGet Package'
- task: PublishPipelineArtifact@0
displayName: 'Publish Pipeline Artifact: drop-nuget'
inputs:
artifactName: ${{ parameters.ArtifactName }}
targetPath: '$(Build.ArtifactStagingDirectory)'
# Compliance tasks require logs from Debug Build
- ${{ if eq(parameters['DoCompliance'], 'true') }}:
- template: compliance.yml
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
displayName: 'Component Detection'
condition: succeeded()
- template: clean-agent-build-directory-step.yml

View file

@ -0,0 +1,190 @@
parameters:
DoDebugBuild: 'true'
DoCompliance: 'false'
BuildCommand: ''
JobName: 'Windows_CI_Dev_x86'
DoNugetPack: 'false'
NuPackScript : ''
ArtifactName: 'drop-win-x86-zip'
DoEsrp: 'false'
DoTestCoverage: 'false'
BuildArch: 'x86'
SetVcvars: 'false'
msbuildArchitecture: 'x86'
EnvSetupScript: 'setup_env_x86.bat'
CudaVersion: ''
AgentPool: 'Win-CPU'
AgentDemands: []
OrtPackageId: Microsoft.ML.OnnxRuntime
jobs:
- job: ${{ parameters.JobName }}
timeoutInMinutes: 120
workspace:
clean: all
pool:
name: ${{ parameters.AgentPool }}
demands: ${{ parameters.AgentDemands }}
variables:
buildDirectory: '$(Build.BinariesDirectory)'
BuildCommand: ${{ parameters.BuildCommand }}
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
DotNetExe: 'dotnet.exe'
CUDA_VERSION: ${{ parameters.CudaVersion }}
PlatformTarget: x86
RuntimeIdentifier: win-x86
steps:
- powershell: |
if($env:TELEMETRYGUID)
{
$length = $env:TELEMETRYGUID.length
$fileContent = "#define ENABLE_TELEMETRY`n#define TraceLoggingOptionMicrosoftTelemetry() \
TraceLoggingOptionGroup("+$env:TELEMETRYGUID.substring(1, $length-2)+")"
New-Item -Path "$(Build.SourcesDirectory)\include\onnxruntime\core\platform\windows\TraceLoggingConfigPrivate.h" -ItemType "file" -Value "$fileContent" -Force
}
displayName: 'Create TraceLoggingConfigPrivate.h For WinML Telemetry'
- task: UsePythonVersion@0
inputs:
versionSpec: '3.7'
addToPath: true
architecture: ${{ parameters.BuildArch }}
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- task: PythonScript@0
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '$(BuildCommand) --update --config RelWithDebInfo'
workingDirectory: '$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build'
inputs:
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
platform: 'Win32'
configuration: RelWithDebInfo
msbuildArchitecture: ${{ parameters.BuildArch }}
maximumCpuCount: true
logProjectEvents: true
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
createLogFile: true
clean: true
# Build RelWithDebInfo -- this variable required to build C#
- script: |
@echo ##vso[task.setvariable variable=Configuration]RelWithDebInfo
- template: set-test-data-variables-step.yml
- task: NuGetToolInstaller@0
displayName: Use Nuget 4.9
inputs:
versionSpec: 4.9.4
- task: PythonScript@0
displayName: 'Download test data'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\download_test_data.py'
arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory)
workingDirectory: $(Build.BinariesDirectory)
- task: DotNetCoreCLI@2
displayName: 'Restore nuget packages'
inputs:
command: restore
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: 'RelWithDebInfo'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
displayName: 'Build C#'
inputs:
command: build
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: 'RelWithDebInfo'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
displayName: 'Test C#'
inputs:
command: test
projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj'
configuration: 'RelWithDebInfo'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- script: |
mklink /D /J $(Build.BinariesDirectory)\RelWithDebInfo\models $(Build.BinariesDirectory)\models
DIR dist\ /S /B > wheel_filename_file
set /p WHEEL_FILENAME=<wheel_filename_file
del wheel_filename_file
python.exe -m pip install -q --upgrade %WHEEL_FILENAME%
set PATH=%PATH%;$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo
@echo %PATH%
python $(Build.SourcesDirectory)\tools\ci_build\build.py $(BuildCommand) --test --config RelWithDebInfo
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
displayName: 'Run tests'
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
# Nuget packaging if needed
- ${{ if eq(parameters['DoNugetPack'], 'true') }}:
- task: BatchScript@1
displayName: 'Setup VS2017 env vars'
inputs:
filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat'
arguments: ${{ parameters.msbuildArchitecture }}
modifyEnvironment: true
# Esrp signing
- template: esrp_dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: ${{ parameters.DoEsrp }}
- template: esrp_dll.yml
parameters:
FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
DisplayName: 'ESRP - Sign C# dlls'
DoEsrp: ${{ parameters.DoEsrp }}
- script: |
${{ parameters.NuPackScript }}
workingDirectory: '$(Build.SourcesDirectory)\csharp'
displayName: 'Create NuGet Package'
- task: PublishPipelineArtifact@0
displayName: 'Publish Pipeline Artifact: drop-nuget'
inputs:
artifactName: ${{ parameters.ArtifactName }}
targetPath: '$(Build.ArtifactStagingDirectory)'
# Compliance tasks require logs from Debug Build
- ${{ if eq(parameters['DoCompliance'], 'true') }}:
- template: compliance.yml
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
displayName: 'Component Detection'
condition: succeeded()
- template: clean-agent-build-directory-step.yml

View file

@ -26,6 +26,18 @@ jobs:
addToPath: true
architecture: $(buildArch)
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- task: PythonScript@0
displayName: 'Generate cmake config'
inputs:
@ -58,27 +70,9 @@ jobs:
- task: NuGetToolInstaller@0
displayName: Use Nuget 4.9
inputs:
versionSpec: 4.9.4
versionSpec: 4.9.4
- task: PowerShell@2
displayName: 'Download AzCopy (used for download test data script)'
inputs:
targetType: 'inline'
script: |
Invoke-WebRequest -OutFile $(Build.BinariesDirectory)\azcopy.exe https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- task: PythonScript@0
displayName: 'Download test data'
@ -96,15 +90,24 @@ jobs:
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId)'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- ${{ if eq(variables['BuildConfig'], 'RelWithDebInfo') }}:
- task: DotNetCoreCLI@2
displayName: 'Test C#'
inputs:
command: test
projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj'
configuration: '$(BuildConfig)'
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
displayName: 'Build C#'
inputs:
command: build
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: '$(BuildConfig)'
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
displayName: 'Test C#'
condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo'))
inputs:
command: test
projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj'
configuration: '$(BuildConfig)'
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- script: |
mklink /D /J $(Build.BinariesDirectory)\$(BuildConfig)\models $(Build.BinariesDirectory)\models

View file

@ -5,7 +5,7 @@ jobs:
AgentDemands: 'Has19H1WinSDK'
DoDebugBuild: 'true'
DoCompliance: 'false'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_dnnl --use_dml --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --gen_doc'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --enable_pybind --use_dnnl --use_dml --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --gen_doc'
JobName: 'Windows_CI_GPU_Dev'
DoNugetPack: 'false'
NuPackScript : ''

View file

@ -4,7 +4,7 @@ jobs:
AgentPool : 'Win-GPU-CUDA10'
DoDebugBuild: 'true'
DoCompliance: 'false'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_dnnl --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --gen_doc'
BuildCommand: '$(Build.SourcesDirectory)\tools\ci_build\build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --enable_pybind --use_dnnl --use_tensorrt --tensorrt_home="C:\local\TensorRT-6.0.1.5" --build_shared_lib --build_csharp --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --gen_doc'
JobName: 'Windows_CI_GPU_Dev'
DoNugetPack: 'false'
NuPackScript : ''

View file

@ -26,6 +26,18 @@ jobs:
addToPath: true
architecture: $(buildArch)
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- task: PythonScript@0
displayName: 'Generate cmake config'
inputs:
@ -58,27 +70,9 @@ jobs:
- task: NuGetToolInstaller@0
displayName: Use Nuget 4.9
inputs:
versionSpec: 4.9.4
versionSpec: 4.9.4
- task: PowerShell@2
displayName: 'Download AzCopy (used for download test data script)'
inputs:
targetType: 'inline'
script: |
Invoke-WebRequest -OutFile $(Build.BinariesDirectory)\azcopy.exe https://onnxruntimetestdata.blob.core.windows.net/models/azcopy.exe
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- script: |
python -m pip install -q pyopenssl setuptools wheel numpy
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- task: PythonScript@0
displayName: 'Download test data'
@ -96,6 +90,15 @@ jobs:
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId)'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
displayName: 'Build C#'
inputs:
command: build
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: '$(BuildConfig)'
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId)'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
displayName: 'Test C#'
condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo'))

View file

@ -69,7 +69,7 @@ def get_endpoint_by_region(current_endpoint, azure_location):
def get_region_based_url(url, azure_location):
current_endpoint = urlsplit(url).netloc
endpoint_by_region = get_endpoint_by_region(current_endpoint, args.azure_region)
endpoint_by_region = get_endpoint_by_region(current_endpoint, azure_location)
url = url.replace(current_endpoint, endpoint_by_region)
print("url changed to %s" % url)
return url
@ -84,9 +84,13 @@ def download_and_unzip(build_dir, url, dest_folder, use_token = True):
url_with_token = url
# Download data using AZCopy tool
# Our linux CI build machine has azcopy in /usr/bin but the version is too old
azcopy_exe = 'azcopy.exe' if sys.platform.startswith("win") and shutil.which('azcopy') else os.path.join(build_dir,'azcopy')
try:
subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', '--recursive', url_with_token, build_dir],check=True)
except:
subprocess.run([azcopy_exe,'cp', '--log-level','ERROR', '--recursive', url_with_token, build_dir],check=True)
except Exception as e:
print(e)
print(azcopy_exe)
raise Exception("Downloading data failed. Source: " + url + " Destination: " + build_dir)
os.makedirs(dest_folder,exist_ok=True)
@ -107,22 +111,23 @@ def download_and_unzip(build_dir, url, dest_folder, use_token = True):
def download_additional_data(build_dir, azure_region):
additional_data_url = 'https://onnxruntimetestdata.blob.core.windows.net/models/'
url = get_region_based_url(args.test_data_url, azure_region)
cmake_url = urljoin(additional_data_url, 'cmake-3.15.1-win64-x64.zip')
print("Starting download for cmake : " + cmake_url)
download_and_unzip(build_dir, cmake_url, 'cmake_temp', False)
dest_dir = os.path.join(build_dir,'cmake')
if os.path.exists(dest_dir):
if not shutil.which('cmake'):
cmake_url = urljoin(additional_data_url, 'cmake-3.15.1-win64-x64.zip')
print("Starting download for cmake : " + cmake_url)
download_and_unzip(build_dir, cmake_url, 'cmake_temp', False)
dest_dir = os.path.join(build_dir,'cmake')
if os.path.exists(dest_dir):
print('deleting %s' % dest_dir)
shutil.rmtree(dest_dir)
shutil.move(os.path.join(build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir)
shutil.move(os.path.join(build_dir,'cmake_temp','cmake-3.15.1-win64-x64'),dest_dir)
# Download OpenCPPCoverageSetup.exe
opencpp_url = urljoin(additional_data_url, 'OpenCppCoverageSetup-x64-0.9.7.0.exe')
print("Starting download for opencppcoverage " + opencpp_url)
dest_folder = os.path.join(build_dir, 'installer','opencppcoverage')
os.makedirs(dest_folder,exist_ok=True)
subprocess.run([os.path.join(build_dir,'azcopy'),'cp', '--log-level','ERROR', opencpp_url, os.path.join(dest_folder,'installer.exe')],check=True)
azcopy_exe = 'azcopy.exe' if shutil.which('azcopy') else os.path.join(build_dir,'azcopy')
subprocess.run([azcopy_exe,'cp', '--log-level','ERROR', opencpp_url, os.path.join(dest_folder,'installer.exe')],check=True)
args = parse_arguments()
models_folder = 'models'
@ -137,8 +142,9 @@ if args.edge_device:
raise Exception(local_file_name + " does not exist on edge device. Downloading test data step failed.")
else:
all_downloads_done = False
azure_region = get_azure_region()
azure_region = args.azure_region
if not azure_region:
azure_region = get_azure_region()
try:
# Download test data
url = get_region_based_url(args.test_data_url, azure_region)

View file

@ -1 +1 @@
set PATH=%BUILD_BINARIESDIRECTORY%\packages\python;%BUILD_BINARIESDIRECTORY%\packages\python\DLLs;%BUILD_BINARIESDIRECTORY%\packages\python\Library\bin;%BUILD_BINARIESDIRECTORY%\packages\python\script;%PATH%
set PATH=C:\azcopy;%BUILD_BINARIESDIRECTORY%\packages\python;%BUILD_BINARIESDIRECTORY%\packages\python\DLLs;%BUILD_BINARIESDIRECTORY%\packages\python\Library\bin;%BUILD_BINARIESDIRECTORY%\packages\python\script;%PATH%

View file

@ -1 +1 @@
set PATH=C:\Program Files (x86)\dotnet;%BUILD_BINARIESDIRECTORY%\packages\python;%BUILD_BINARIESDIRECTORY%\packages\python\DLLs;%BUILD_BINARIESDIRECTORY%\packages\python\Library\bin;%BUILD_BINARIESDIRECTORY%\packages\python\script;%PATH%;C:\local\systools
set PATH=C:\azcopy;C:\Program Files (x86)\dotnet;%BUILD_BINARIESDIRECTORY%\packages\python;%BUILD_BINARIESDIRECTORY%\packages\python\DLLs;%BUILD_BINARIESDIRECTORY%\packages\python\Library\bin;%BUILD_BINARIESDIRECTORY%\packages\python\script;%PATH%;C:\local\systools