Enable CI for training.

This commit is contained in:
Edward Chen 2020-03-11 14:41:32 -07:00
parent e542cfd0e0
commit 80dd62a240
8 changed files with 71 additions and 258 deletions

View file

@ -188,7 +188,7 @@ int main(int argc, char* args[]) {
std::vector<string> feeds{"X", "labels"};
auto trainingData = std::make_shared<DataSet>(feeds);
auto testData = std::make_shared<DataSet>(feeds);
std::string mnist_data_path(params.train_data_dir.begin(), params.train_data_dir.end());
std::string mnist_data_path = ToMBString(params.train_data_dir);
PrepareMNISTData(mnist_data_path, IMAGE_DIMS, LABEL_DIMS, *trainingData, *testData, params.mpi_context.world_rank /* shard_to_load */, device_count /* total_shards */);
if (testData->NumSamples() == 0) {

View file

@ -3,7 +3,7 @@ trigger: none
jobs:
- template: templates/linux-ci.yml
parameters:
AgentPool : 'AI-Linux-CPU'
AgentPool : 'Linux-CPU'
JobName: 'Onnxruntime_Linux_CPU_Training'
SubmoduleCheckoutMode: 'recursive'
BuildCommand: >

View file

@ -3,7 +3,7 @@ trigger: none
jobs:
- template: templates/linux-ci.yml
parameters:
AgentPool : 'AI-Linux-GPU'
AgentPool : 'Linux-GPU-CUDA10'
JobName: 'Onnxruntime_Linux_GPU_Training'
SubmoduleCheckoutMode: 'recursive'
# TODO --skip_onnx_tests is used to disable onnx_test_runner test.

View file

@ -1,16 +0,0 @@
trigger: none
jobs:
- template: templates/linux-ci.yml
parameters:
AgentPool : 'AI-Linux-GPU'
JobName: 'Onnxruntime_Linux_GPU_Inference'
SubmoduleCheckoutMode: 'recursive'
BuildCommand: >
tools/ci_build/github/linux/run_dockerbuild.sh
-o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory)
-x "
"
DoNugetPack: 'false'
ArtifactName: 'drop-linux'
TimeoutInMinutes: 90

View file

@ -3,8 +3,7 @@ trigger: none
jobs:
- template: templates/mac-ci.yml
parameters:
AgentPool : 'Hosted macOS High Sierra'
JobName: 'Onnxruntime_Mac_CPU_Training'
JobName: 'Mac_CPU_Training'
SubmoduleCheckoutMode: 'recursive'
BuildCommand: >
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py

View file

@ -1,102 +1,18 @@
trigger: none
jobs:
- job: Onnxruntime_Win_CPU_Training
timeoutInMinutes: 160
strategy:
maxParallel: 2
matrix:
Debug:
buildConfig: Debug
Release:
buildConfig: Release
variables:
- name: buildDirectory # currently some csproj files depend on this variable - would be nice if they didn't
value: $(Build.BinariesDirectory)
- name: BUILD_PY_COMMON_OPTIONS
value: >-
--build_dir $(Build.BinariesDirectory) --config $(buildConfig)
--cmake_path $(Build.BinariesDirectory)\packages\cmake\bin\cmake.exe
--ctest_path $(Build.BinariesDirectory)\packages\cmake\bin\ctest.exe
--build_shared_lib
--use_dnnl
--use_openmp
--enable_training
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/set-test-data-variables-step.yml
- task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0
displayName: 'Download python installer from Artifact Services Drop'
inputs:
dropServiceURI: 'https://aiinfra.artifacts.visualstudio.com/DefaultCollection'
buildNumber: miniconda3/4.4.10/x64
destinationPath: '$(Build.BinariesDirectory)\installer\python'
- task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0
displayName: 'Download cmake'
inputs:
dropServiceURI: 'https://aiinfra.artifacts.visualstudio.com/DefaultCollection'
buildNumber: cmake/x64/3.13.4
destinationPath: '$(Build.BinariesDirectory)\packages\cmake'
timeoutInMinutes: 10
- task: CmdLine@1
displayName: 'Run python installer'
inputs:
filename: '$(Build.BinariesDirectory)\installer\python\installer.exe'
arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
timeoutInMinutes: 10
- task: BatchScript@1
displayName: 'Setup VS2017 env vars'
inputs:
filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat'
modifyEnvironment: true
- task: CmdLine@1
displayName: 'Generate build files'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: >-
$(Build.SourcesDirectory)\tools\ci_build\build.py
$(BUILD_PY_COMMON_OPTIONS)
--update --skip_submodule_sync
- task: CmdLine@1
displayName: 'Build'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: >-
$(Build.SourcesDirectory)\tools\ci_build\build.py
$(BUILD_PY_COMMON_OPTIONS)
--build --parallel
- task: CmdLine@1
displayName: 'Test'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: >-
$(Build.SourcesDirectory)\tools\ci_build\build.py
$(BUILD_PY_COMMON_OPTIONS)
--test
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
displayName: 'Component Detection'
- template: templates/clean-agent-build-directory-step.yml
- template: templates/win-ci-2019.yml
parameters:
AgentPool : 'Win-CPU-2019'
JobName: 'Win_CPU_Training'
BuildCommand: >-
--build_dir $(Build.BinariesDirectory)
--build_shared_lib
--cmake_generator "Visual Studio 16 2019"
--enable_training
--skip_submodule_sync
--use_openmp
BuildArch: 'x64'
EnvSetupScript: 'setup_env.bat'
sln_platform: 'x64'
BuildConfigurations: ['Debug', 'RelWithDebInfo']

View file

@ -1,121 +1,23 @@
trigger: none
jobs:
- job: Onnxruntime_Win_GPU_Training
timeoutInMinutes: 180
strategy:
maxParallel: 2
matrix:
# TODO: Win-GPU-Debug build is failing due to warning 4996.
# Re-enable this when merge back to public master
# Debug:
# buildConfig: Debug
Release:
buildConfig: Release
variables:
- name: CUDA_VERSION
value: '10.0'
- name: CUDA_PATH
value: 'C:/local/cuda_10.0.130_win10'
- name: CUDNN_PATH
value: 'C:/local/cudnn-10.0-windows10-x64-v7.3.1.20/cuda'
- name: BUILD_PY_COMMON_OPTIONS
value: >-
--build_dir $(Build.BinariesDirectory) --config $(buildConfig)
--cmake_path $(Build.BinariesDirectory)\packages\cmake\bin\cmake.exe
--ctest_path $(Build.BinariesDirectory)\packages\cmake\bin\ctest.exe
--build_shared_lib
--use_cuda --cudnn_home $(CUDNN_PATH) --cuda_home $(CUDA_PATH)
--use_dnnl
--use_mklml
--cmake_extra_defines CUDA_TOOLKIT_ROOT_DIR=$(CUDA_PATH)
--enable_training
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/set-test-data-variables-step.yml
- task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0
displayName: 'Download python installer from Artifact Services Drop'
inputs:
dropServiceURI: 'https://aiinfra.artifacts.visualstudio.com/DefaultCollection'
buildNumber: miniconda3/4.4.10/x64
destinationPath: '$(Build.BinariesDirectory)\installer\python'
- task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0
displayName: 'Download cmake'
inputs:
dropServiceURI: 'https://aiinfra.artifacts.visualstudio.com/DefaultCollection'
buildNumber: cmake/x64/3.13.4
destinationPath: '$(Build.BinariesDirectory)\packages\cmake'
- task: CmdLine@1
displayName: 'Run python installer'
inputs:
filename: '$(Build.BinariesDirectory)\installer\python\installer.exe'
arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
- task: PowerShell@1
displayName: 'Set CUDA path'
inputs:
scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1'
arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-10.0.130-win10 -CudaVersion $(CUDA_VERSION)'
- task: BatchScript@1
displayName: 'Setup VS2017 env vars'
inputs:
filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat'
arguments: 'amd64 -vcvars_ver=14.11'
modifyEnvironment: true
- task: CmdLine@1
displayName: 'Generate build files'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: >-
$(Build.SourcesDirectory)\tools\ci_build\build.py
$(BUILD_PY_COMMON_OPTIONS)
--update --skip_submodule_sync
- task: CmdLine@1
displayName: 'Build'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: >-
$(Build.SourcesDirectory)\tools\ci_build\build.py
$(BUILD_PY_COMMON_OPTIONS)
--build --parallel
- task: CmdLine@1
displayName: 'Test'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: >-
$(Build.SourcesDirectory)\tools\ci_build\build.py
$(BUILD_PY_COMMON_OPTIONS)
--test
- task: PowerShell@1
displayName: 'Clean up CUDA props files'
inputs:
scriptName: 'tools/ci_build/github/windows/clean_up_cuda_prop_files.ps1'
arguments: '-CudaVersion $(CUDA_VERSION)'
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
displayName: 'Component Detection'
- template: templates/clean-agent-build-directory-step.yml
- template: templates/win-ci-2019.yml
parameters:
AgentPool : 'Win-GPU-2019'
JobName: 'Win_GPU_Training'
BuildCommand: >-
--build_dir $(Build.BinariesDirectory)
--build_shared_lib
--cmake_generator "Visual Studio 16 2019"
--enable_training
--skip_submodule_sync
--use_cuda
--cuda_version 10.1
--cuda_home "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1"
--cudnn_home "C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda"
BuildArch: 'x64'
EnvSetupScript: 'setup_env_cuda.bat'
sln_platform: 'x64'
CudaVersion: '10.1'
OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu'
BuildConfigurations: ['Debug', 'RelWithDebInfo']

View file

@ -1,5 +1,5 @@
parameters:
DoDebugBuild: 'true'
DoDebugBuild: 'true' # Unused. Use BuildConfigurations instead.
DoCompliance: 'false'
BuildCommand: ''
JobName: 'Windows_CI_Dev'
@ -15,10 +15,17 @@ parameters:
AgentPool: 'Win-CPU-2019'
AgentDemands: []
OrtPackageId: Microsoft.ML.OnnxRuntime
BuildConfigurations: ['RelWithDebInfo'] # Options: Debug, RelWithDebInfo
RunTests : 'true'
jobs:
- job: ${{ parameters.JobName }}
timeoutInMinutes: 120
strategy:
maxParallel: 2
matrix:
${{ each BuildConfiguration in parameters.BuildConfigurations }}:
${{ BuildConfiguration }}:
BuildConfig: ${{ BuildConfiguration }}
workspace:
clean: all
pool:
@ -33,6 +40,10 @@ jobs:
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
steps:
- checkout: self
clean: true
submodules: recursive
- powershell: |
if($env:TELEMETRYGUID)
{
@ -68,7 +79,7 @@ jobs:
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: '$(BuildCommand) --update --config RelWithDebInfo --enable_lto'
arguments: '$(BuildCommand) --update --config $(BuildConfig) --enable_lto'
workingDirectory: '$(Build.BinariesDirectory)'
- ${{ if notIn(parameters['sln_platform'], 'Win32', 'x64') }}:
@ -79,18 +90,19 @@ jobs:
- task: VSBuild@1
displayName: 'Build'
inputs:
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln'
platform: ${{ parameters.sln_platform }}
configuration: RelWithDebInfo
configuration: $(BuildConfig)
msbuildArchitecture: ${{ parameters.BuildArch }}
maximumCpuCount: true
logProjectEvents: true
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
createLogFile: true
# Build RelWithDebInfo -- this variable required to build C#
# The Configuration variable is required to build C#
- script: |
@echo ##vso[task.setvariable variable=Configuration]RelWithDebInfo
@echo ##vso[task.setvariable variable=Configuration]$(BuildConfig)
displayName: 'Set Configuration variable'
- template: set-test-data-variables-step.yml
@ -104,8 +116,8 @@ jobs:
inputs:
command: restore
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: 'RelWithDebInfo'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
configuration: '$(BuildConfig)'
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- task: DotNetCoreCLI@2
@ -113,8 +125,8 @@ jobs:
inputs:
command: build
projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: 'RelWithDebInfo'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
configuration: '$(BuildConfig)'
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- ${{ if in(parameters['sln_platform'], 'Win32', 'x64') }}:
@ -130,21 +142,21 @@ jobs:
inputs:
command: test
projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj'
configuration: 'RelWithDebInfo'
arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
configuration: '$(BuildConfig)'
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}'
workingDirectory: '$(Build.SourcesDirectory)\csharp'
- ${{ if eq(parameters.RunTests, true) }}:
- script: |
mklink /D /J $(Build.BinariesDirectory)\RelWithDebInfo\models $(Build.BinariesDirectory)\models
mklink /D /J $(Build.BinariesDirectory)\$(BuildConfig)\models $(Build.BinariesDirectory)\models
DIR dist\ /S /B > wheel_filename_file
set /p WHEEL_FILENAME=<wheel_filename_file
del wheel_filename_file
python.exe -m pip install -q --upgrade %WHEEL_FILENAME%
set PATH=%PATH%;$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo
set PATH=%PATH%;$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)
@echo %PATH%
python $(Build.SourcesDirectory)\tools\ci_build\build.py $(BuildCommand) --test --config RelWithDebInfo --enable_lto
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
python $(Build.SourcesDirectory)\tools\ci_build\build.py $(BuildCommand) --test --config $(BuildConfig) --enable_lto
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
displayName: 'Run tests'
- ${{ if eq(parameters.RunTests, true) }}:
@ -157,7 +169,7 @@ jobs:
condition: succeededOrFailed()
# Nuget packaging if needed
- ${{ if eq(parameters['DoNugetPack'], 'true') }}:
- ${{ if and(eq(parameters['DoNugetPack'], 'true'), eq('$(BuildConfig)', 'RelWithDebInfo')) }}:
- task: BatchScript@1
displayName: 'Setup VS2019 env vars'
inputs:
@ -167,13 +179,13 @@ jobs:
# Esrp signing
- template: esrp_dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo'
FolderPath: '$(Build.BinariesDirectory)\$(BuildConfig)'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: ${{ parameters.DoEsrp }}
- template: esrp_dll.yml
parameters:
FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\$(BuildConfig)'
DisplayName: 'ESRP - Sign C# dlls'
DoEsrp: ${{ parameters.DoEsrp }}