Split build pipeline

This commit is contained in:
Changming Sun 2019-01-15 10:29:00 -08:00
parent 7e3923b9b3
commit 7977871740
5 changed files with 234 additions and 64 deletions

View file

@ -320,8 +320,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home
if is_windows():
cmake_args += cmake_extra_args
if args.use_cuda:
os.environ["PATH"] += os.pathsep + os.path.join(cudnn_home, 'bin')
for config in configs:
config_build_dir = get_config_build_dir(build_dir, config)
@ -398,8 +396,7 @@ def setup_cuda_vars(args):
cuda_bin_path = os.path.join(cuda_home, 'bin')
os.environ["CUDA_BIN_PATH"] = cuda_bin_path
os.environ["PATH"] += os.pathsep + cuda_bin_path
os.environ["PATH"] += os.pathsep + cuda_bin_path + os.pathsep + os.path.join(cudnn_home, 'bin')
# Add version specific CUDA_PATH_Vx_y value as the Visual Studio build files require that
version_file = os.path.join(cuda_home, 'version.txt')
if not os.path.exists(version_file):
@ -530,30 +527,21 @@ def main():
# if using cuda, setup cuda paths and env vars
cuda_home, cudnn_home = setup_cuda_vars(args)
# directory from ONNX submodule with ONNX test data
onnx_test_data_dir = '/data/onnx'
if is_windows() or not os.path.exists(onnx_test_data_dir):
onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data")
os.makedirs(build_dir, exist_ok=True)
log.info("Build started")
cmake_extra_args = []
if(is_windows()):
if (args.x86):
cmake_extra_args = ['-A','Win32','-G', 'Visual Studio 15 2017']
else:
toolset = 'host=x64'
if (args.msvc_toolset):
toolset += ',version=' + args.msvc_toolset
cmake_extra_args = ['-A','x64','-T', toolset, '-G', 'Visual Studio 15 2017']
#Add python to PATH. Please remove this after https://github.com/onnx/onnx/issues/1080 is fixed ()
os.environ["PATH"] = os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"]
os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(sys.executable)
if (args.update):
cmake_extra_args = []
if(is_windows()):
if (args.x86):
cmake_extra_args = ['-A','Win32','-G', 'Visual Studio 15 2017']
else:
toolset = 'host=x64'
if (args.msvc_toolset):
toolset += ',version=' + args.msvc_toolset
cmake_extra_args = ['-A','x64','-T', toolset, '-G', 'Visual Studio 15 2017']
if is_ubuntu_1604():
install_ubuntu_deps(args)
install_python_deps()
@ -578,18 +566,22 @@ def main():
if (args.test):
run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, args.enable_pybind, args.use_tvm)
# run the onnx model tests if requested explicitly.
if (args.enable_onnx_tests):
# directory from ONNX submodule with ONNX test data
onnx_test_data_dir = '/data/onnx'
if is_windows() or not os.path.exists(onnx_test_data_dir):
onnx_test_data_dir = os.path.join(source_dir, "cmake", "external", "onnx", "onnx", "backend", "test", "data")
if args.use_cuda:
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', False)
else:
run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, True)
if args.use_mkldnn:
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'mkldnn', True)
# run the onnx model tests if requested explicitly.
if (args.enable_onnx_tests):
if args.use_cuda:
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'cuda', False)
else:
run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, True)
if args.use_mkldnn:
run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'mkldnn', True)
if args.build_wheel:
build_python_wheel(source_dir, build_dir, configs, args.use_cuda)
if args.build:
if args.build_wheel:
build_python_wheel(source_dir, build_dir, configs, args.use_cuda)
log.info("Build complete")

View file

@ -1,19 +1,115 @@
jobs:
- job: Windows_CI_Dev
pool: Win-CPU
variables:
buildDirectory: '$(Build.BinariesDirectory)'
steps:
- template: templates/set-test-data-variables-step.yml
- task: NuGetCommand@2
displayName: 'NuGet restore'
inputs:
restoreSolution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
feedsToUse: config
nugetConfigPath: '$(Build.SourcesDirectory)\csharp\Nuget.CSharp.config'
restoreDirectory: '$(Build.SourcesDirectory)\csharp'
- task: UniversalPackages@0
displayName: 'Download python'
inputs:
command: download
vstsFeed: '$(System.TeamProject)'
vstsFeedPackage: 'miniconda3_win64'
vstsPackageVersion: '4.5.11'
downloadDirectory: '$(Build.BinariesDirectory)\python'
- task: CmdLine@1
displayName: 'Run python installer'
inputs:
filename: '$(Build.BinariesDirectory)\python\installer.exe'
arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
timeoutInMinutes: 10
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- task: CmdLine@1
displayName: 'Install conda modules'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe'
arguments: 'install -q --insecure -y pyopenssl setuptools wheel numpy'
timeoutInMinutes: 10
- task: CmdLine@1
displayName: 'Download cmake'
inputs:
filename: python
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
- task: BatchScript@1
- task: CmdLine@1
displayName: 'Download test data and generate cmake config'
inputs:
filename: build.bat
arguments: ' --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --config Debug Release --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --build_csharp --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum)'
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update'
workingDirectory: "$(Build.BinariesDirectory)"
- task: VSBuild@1
displayName: 'Build Debug'
inputs:
solution: '$(Build.BinariesDirectory)\Debug\onnxruntime.sln'
platform: 'x64'
configuration: 'Debug'
msbuildArgs: '/m'
msbuildArchitecture: 'x64'
logProjectEvents: true
workingFolder: '$(Build.BinariesDirectory)\Debug'
- task: BatchScript@1
displayName: 'Test Debug'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
workingFolder: '$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build C# Debug'
inputs:
solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
platform: 'any cpu'
configuration: 'Debug'
restoreNugetPackages: false
msbuildArchitecture: 'x64'
workingFolder: '$(Build.SourcesDirectory)\csharp'
msbuildArgs: '/m /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build Release'
inputs:
solution: '$(Build.BinariesDirectory)\Release\onnxruntime.sln'
platform: 'x64'
configuration: 'Release'
msbuildArgs: '/m'
msbuildArchitecture: 'x64'
logProjectEvents: true
workingFolder: '$(Build.BinariesDirectory)\Release'
- task: BatchScript@1
displayName: 'Test Release'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
workingFolder: "$(Build.BinariesDirectory)"
- task: VSBuild@1
displayName: 'Build c# Release'
inputs:
solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
platform: 'any cpu'
configuration: 'Release'
msbuildArchitecture: 'x64'
restoreNugetPackages: false
workingFolder: '$(Build.SourcesDirectory)\csharp'
msbuildArgs: '/m /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory)'
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- template: templates/clean-agent-build-directory-step.yml

View file

@ -1,23 +1,51 @@
jobs:
- job: Windows_CI_GPU_Dev
pool: Win-GPU-CUDA10
variables:
buildDirectory: '$(Build.BinariesDirectory)'
CUDA_VERSION: '10.0'
steps:
- template: templates/set-test-data-variables-step.yml
- task: NuGetCommand@2
displayName: 'NuGet restore'
inputs:
restoreSolution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
feedsToUse: config
nugetConfigPath: '$(Build.SourcesDirectory)\csharp\Nuget.CSharp.config'
restoreDirectory: '$(Build.SourcesDirectory)\csharp'
- task: UniversalPackages@0
displayName: 'Download python'
inputs:
command: download
vstsFeed: '$(System.TeamProject)'
vstsFeedPackage: 'miniconda3_win64'
vstsPackageVersion: '4.5.11'
downloadDirectory: '$(Build.BinariesDirectory)\python'
- task: CmdLine@1
displayName: 'Run python installer'
inputs:
filename: '$(Build.BinariesDirectory)\python\installer.exe'
arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python'
timeoutInMinutes: 10
- task: BatchScript@1
displayName: 'setup env'
inputs:
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env.bat'
modifyEnvironment: true
workingFolder: '$(Build.BinariesDirectory)'
- task: CmdLine@1
displayName: 'Install conda modules'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe'
arguments: 'install -q --insecure -y pyopenssl setuptools wheel numpy'
timeoutInMinutes: 10
- task: CmdLine@1
displayName: 'Download cmake'
inputs:
filename: python
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
- task: PowerShell@1
displayName: 'Set CUDA path'
inputs:
scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1'
arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-10.0.130-win10 -CudaVersion $(CUDA_VERSION)'
- task: BatchScript@1
displayName: 'Setup VS2017 env vars'
inputs:
@ -25,16 +53,73 @@ jobs:
arguments: 'amd64 -vcvars_ver=14.11'
modifyEnvironment: true
- task: CmdLine@1
displayName: 'Download test data and generate cmake config'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --update --msvc_toolset=14.11'
workingDirectory: "$(Build.BinariesDirectory)"
- task: VSBuild@1
displayName: 'Build Debug'
inputs:
solution: '$(Build.BinariesDirectory)\Debug\onnxruntime.sln'
platform: 'x64'
configuration: 'Debug'
msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\'
msbuildArchitecture: 'x64'
logProjectEvents: true
workingFolder: '$(Build.BinariesDirectory)\Debug'
- task: BatchScript@1
displayName: 'Test Debug'
inputs:
filename: build.bat
arguments: ' --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --config Debug Release --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --use_mkldnn --build_shared_lib --build_csharp --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"'
workingFolder: "$(Build.SourcesDirectory)"
- task: PowerShell@1
displayName: 'Clean up CUDA props files'
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
workingFolder: '$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build C# Debug'
inputs:
scriptName: 'tools/ci_build/github/windows/clean_up_cuda_prop_files.ps1'
arguments: '-CudaVersion $(CUDA_VERSION)'
solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
configuration: 'Debug'
platform: 'any cpu'
restoreNugetPackages: false
msbuildArchitecture: 'x64'
workingFolder: '$(Build.SourcesDirectory)\csharp'
msbuildArgs: '/m /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory)'
- task: VSBuild@1
displayName: 'Build Release'
inputs:
solution: '$(Build.BinariesDirectory)\Release\onnxruntime.sln'
platform: 'x64'
configuration: 'Release'
msbuildArgs: '/m /p:CudaToolkitDir=C:\local\cuda_10.0.130_win10\'
msbuildArchitecture: 'x64'
logProjectEvents: true
workingFolder: '$(Build.BinariesDirectory)\Release'
- task: BatchScript@1
displayName: 'Test Release'
inputs:
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Release --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_pybind --use_openmp --build_shared_lib --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda" --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --test'
workingFolder: "$(Build.BinariesDirectory)"
- task: VSBuild@1
displayName: 'Build c# Release'
inputs:
solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
platform: 'any cpu'
configuration: 'Release'
msbuildArchitecture: 'x64'
restoreNugetPackages: false
workingFolder: '$(Build.SourcesDirectory)\csharp'
msbuildArgs: '/m /p:OnnxRuntimeBuildDirectory=$(Build.BinariesDirectory)'
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**\*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- template: templates/clean-agent-build-directory-step.yml

View file

@ -58,14 +58,10 @@ def download_test_data(models_dir, src_url, expected_md5):
if os.path.exists(models_dir):
print('deleting %s' % models_dir)
shutil.rmtree(models_dir)
if shutil.which('unzip'):
subprocess.run(['unzip','-qd', models_dir, local_zip_file], check=True)
elif shutil.which('7za'):
subprocess.run(['7za','x', local_zip_file, '-y', '-o' + models_dir], check=True)
if is_windows():
subprocess.run(['powershell', '-Command', 'Expand-Archive -LiteralPath "%s" -DestinationPath "%s" -Force' % (local_zip_file, models_dir)], check=True)
else:
#TODO: use python for unzip
print("No unzip tool for use")
return False
subprocess.run(['unzip','-qd', models_dir, local_zip_file], check=True)
return True

View file

@ -0,0 +1 @@
set PATH=%BUILD_BINARIESDIRECTORY%\packages\python;%BUILD_BINARIESDIRECTORY%\packages\python\DLLs;%BUILD_BINARIESDIRECTORY%\packages\python\Library\bin;%BUILD_BINARIESDIRECTORY%\packages\python\script;%PATH%