diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 1ee9d29932..0fec12c18b 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -49,6 +49,9 @@ endif() onnxruntime_add_shared_library_module(onnxruntime_pybind11_state ${onnxruntime_pybind_srcs}) if(MSVC) target_compile_options(onnxruntime_pybind11_state PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") + if(onnxruntime_ENABLE_TRAINING) + target_compile_options(onnxruntime_pybind11_state PRIVATE "/bigobj") + endif() endif() if(HAS_CAST_FUNCTION_TYPE) target_compile_options(onnxruntime_pybind11_state PRIVATE "-Wno-cast-function-type") diff --git a/tools/ci_build/github/azure-pipelines/orttraining-win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-win-ci-pipeline.yml deleted file mode 100644 index 4e747846be..0000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-win-ci-pipeline.yml +++ /dev/null @@ -1,25 +0,0 @@ -trigger: none - -jobs: -- template: templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-CPU-2019' - JobName: 'Win_CPU_Training' - # TODO fix test failures and remove --skip_onnx_tests - BuildCommand: >- - --build_dir $(Build.BinariesDirectory) - --build_shared_lib - --cmake_generator "Visual Studio 16 2019" - --enable_onnx_tests - --enable_training - --skip_onnx_tests - --skip_submodule_sync - --use_openmp - BuildArch: 'x64' - EnvSetupScript: 'setup_env.bat' - sln_platform: 'x64' - BuildConfigurations: ['Debug', 'RelWithDebInfo'] - EnableLto: false - # Enable unreleased onnx opsets in CI builds - # This facilitates testing the implementation for the new opsets - AllowReleasedOpsetOnly: '0' diff --git a/tools/ci_build/github/azure-pipelines/orttraining-win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-win-gpu-ci-pipeline.yml deleted file mode 100644 index 5f2ef84b17..0000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-win-gpu-ci-pipeline.yml +++ /dev/null @@ -1,31 +0,0 @@ -trigger: none - -jobs: -- template: templates/win-ci-2019.yml - parameters: - AgentPool : 'Win-GPU-2019' - JobName: 'Win_GPU_Training' - # TODO fix test failures and remove --skip_onnx_tests - BuildCommand: >- - --build_dir $(Build.BinariesDirectory) - --build_shared_lib - --cmake_generator "Visual Studio 16 2019" - --enable_onnx_tests - --enable_training - --skip_onnx_tests - --skip_submodule_sync - --use_cuda - --cuda_version 10.2 - --cuda_home "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2" - --cudnn_home "C:\local\cudnn-10.2-windows10-x64-v8.0.3.33\cuda" - --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52" - BuildArch: 'x64' - EnvSetupScript: 'setup_env_cuda.bat' - sln_platform: 'x64' - CudaVersion: '10.2' - OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu' - BuildConfigurations: ['RelWithDebInfo'] - # Enable unreleased onnx opsets in CI builds - # This facilitates testing the implementation for the new opsets - AllowReleasedOpsetOnly: '0' - DoCompliance: 'true' diff --git a/tools/ci_build/github/azure-pipelines/templates/win-cpu-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-cpu-ci.yml index cfc25951f6..0a704ddf4f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-cpu-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-cpu-ci.yml @@ -24,11 +24,20 @@ parameters: type: boolean default: false +- name: EnablePython + type: boolean + default: true + - name: RunOnnxRuntimeTests displayName: Run Tests? type: boolean default: true +- name: RunStaticCodeAnalysis + displayName: Run Static Code Analysis + type: boolean + default: true + jobs: - job: build_${{ parameters.job_name_suffix }} variables: @@ -83,15 +92,16 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' - - powershell: | - $Env:USE_MSVC_STATIC_RUNTIME=1 - $Env:ONNX_ML=1 - $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{ parameters.buildArch }}-windows-static" - python setup.py bdist_wheel - python -m pip uninstall -y onnx -qq - Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} - workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' - displayName: 'Install ONNX' + - ${{ if eq(parameters.RunOnnxRuntimeTests, true) }}: + - powershell: | + $Env:USE_MSVC_STATIC_RUNTIME=1 + $Env:ONNX_ML=1 + $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{ parameters.buildArch }}-windows-static" + python setup.py bdist_wheel + python -m pip uninstall -y onnx -qq + Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} + workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' + displayName: 'Install ONNX' - task: NuGetToolInstaller@0 displayName: Use Nuget 5.7.0 @@ -111,7 +121,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) ${{ parameters.UseOmp }} --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_winml --build_shared_lib --enable_onnx_tests --enable_wcos ${{ parameters.additionalBuildFlags }}' + arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) ${{ parameters.UseOmp }} --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }}' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -127,12 +137,13 @@ jobs: workingFolder: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}' createLogFile: true - - task: PythonScript@0 - displayName: 'Build wheel' - inputs: - scriptPath: '$(Build.SourcesDirectory)\setup.py' - arguments: 'bdist_wheel' - workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}' + - ${{ if eq(parameters.EnablePython, true) }}: + - task: PythonScript@0 + displayName: 'Build wheel' + inputs: + scriptPath: '$(Build.SourcesDirectory)\setup.py' + arguments: 'bdist_wheel' + workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}' - task: NuGetCommand@2 @@ -161,16 +172,22 @@ jobs: arguments: '--configuration ${{ parameters.BuildConfig }} -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) --blame' workingDirectory: '$(Build.SourcesDirectory)\csharp' - - powershell: | - Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} - python $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) ${{ parameters.UseOmp }} --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --use_winml --build_shared_lib --enable_onnx_tests --enable_wcos ${{ parameters.additionalBuildFlags }} - - workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}' - condition: and(succeeded(), eq('${{ parameters.RunOnnxRuntimeTests}}', true)) - displayName: 'Run tests' + - ${{ if eq(parameters.EnablePython, true) }}: + - powershell: | + Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} + + workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}' + displayName: 'Install onnxruntime wheel' + + - ${{ if eq(parameters.RunOnnxRuntimeTests, true) }}: + - powershell: | + python $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) ${{ parameters.UseOmp }} --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }} + + workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}' + displayName: 'Run tests' - - ${{ if eq(parameters.BuildConfig, 'RelWithDebInfo') }}: + - ${{ if eq(parameters.RunStaticCodeAnalysis, true) }}: - task: DeleteFiles@1 displayName: 'Delete binaries files from $(Build.BinariesDirectory)\RelWithDebInfo' inputs: @@ -184,7 +201,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' workingDirectory: '$(Build.BinariesDirectory)' @@ -202,13 +219,14 @@ jobs: createLogFile: true - - task: PublishTestResults@2 - displayName: 'Publish unit test results' - inputs: - testResultsFiles: '**/*.results.xml' - searchFolder: '$(Build.BinariesDirectory)/${{ parameters.BuildConfig }}' - testRunTitle: 'Unit Test Run' - condition: succeededOrFailed() + - ${{ if eq(parameters.RunOnnxRuntimeTests, true) }}: + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)/${{ parameters.BuildConfig }}' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() - template: component-governance-component-detection-steps.yml parameters : diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index e871496749..d4995bd406 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -4,39 +4,104 @@ parameters: type: boolean default: true -jobs: -- template: templates/win-cpu-ci.yml - parameters: - BuildConfig: 'Debug' - UseOmp: '--use_openmp' - EnvSetupScript: setup_env.bat - buildArch: x64 - additionalBuildFlags: --use_dnnl --build_java --build_nodejs - msbuildPlatform: x64 - isX86: false - job_name_suffix: x64_debug - RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} +stages: +- stage: x64_debug + dependsOn: [] + jobs: + - template: templates/win-cpu-ci.yml + parameters: + BuildConfig: 'Debug' + UseOmp: '--use_openmp' + EnvSetupScript: setup_env.bat + buildArch: x64 + additionalBuildFlags: --use_dnnl --build_java --build_nodejs --build_wheel + msbuildPlatform: x64 + isX86: false + job_name_suffix: x64_debug + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + RunStaticCodeAnalysis: false -- template: templates/win-cpu-ci.yml - parameters: - BuildConfig: 'RelWithDebInfo' - UseOmp: '' - EnvSetupScript: setup_env.bat - buildArch: x64 - additionalBuildFlags: --use_dnnl --build_java --build_nodejs - msbuildPlatform: x64 - isX86: false - job_name_suffix: x64_release - RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} +- stage: x64_release + dependsOn: [] + jobs: + - template: templates/win-cpu-ci.yml + parameters: + BuildConfig: 'RelWithDebInfo' + UseOmp: '' + EnvSetupScript: setup_env.bat + buildArch: x64 + # Compare to our Nuget packaging pipeline, this job has "--build_wheel --use_dnnl" but doesn't have "--enable_lto --disable_rtti --use_telemetry --enable_wcos" + # Python bindings use typeid so I can't disable RTTI here. So we may need to split it to two pipelines + additionalBuildFlags: --build_wheel --use_dnnl --build_java --build_nodejs + msbuildPlatform: x64 + isX86: false + job_name_suffix: x64_release + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + RunStaticCodeAnalysis: true -- template: templates/win-cpu-ci.yml - parameters: - BuildConfig: 'RelWithDebInfo' - UseOmp: '' - EnvSetupScript: setup_env_x86.bat - buildArch: x86 - additionalBuildFlags: - msbuildPlatform: Win32 - isX86: true - job_name_suffix: x86_release - RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} \ No newline at end of file +- stage: x64_release_winml + dependsOn: [] + jobs: + - template: templates/win-cpu-ci.yml + parameters: + BuildConfig: 'RelWithDebInfo' + UseOmp: '' + EnvSetupScript: setup_env.bat + buildArch: x64 + additionalBuildFlags: --use_winml --enable_wcos --disable_rtti + msbuildPlatform: x64 + isX86: false + job_name_suffix: x64_release_winml + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + # WinML has many warnings + RunStaticCodeAnalysis: false + EnablePython: false + +- stage: x86_release + dependsOn: [] + jobs: + - template: templates/win-cpu-ci.yml + parameters: + BuildConfig: 'RelWithDebInfo' + UseOmp: '' + EnvSetupScript: setup_env_x86.bat + buildArch: x86 + additionalBuildFlags: --build_wheel + msbuildPlatform: Win32 + isX86: true + job_name_suffix: x86_release + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + RunStaticCodeAnalysis: true + +- stage: training_x64_debug + dependsOn: [] + jobs: + - template: templates/win-cpu-ci.yml + parameters: + BuildConfig: 'Debug' + UseOmp: '' + EnvSetupScript: setup_env.bat + buildArch: x64 + additionalBuildFlags: --enable_training --build_wheel + msbuildPlatform: x64 + isX86: false + job_name_suffix: training_x64_debug + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + RunStaticCodeAnalysis: false + +- stage: training_x64_release + dependsOn: [] + jobs: + - template: templates/win-cpu-ci.yml + parameters: + BuildConfig: 'RelWithDebInfo' + UseOmp: '' + EnvSetupScript: setup_env.bat + buildArch: x64 + additionalBuildFlags: --enable_training --build_wheel + msbuildPlatform: x64 + isX86: false + job_name_suffix: training_x64_release + RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} + # Liqun is fixing them + RunStaticCodeAnalysis: false diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index 2e67f5f0ca..b435b74d96 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -4,193 +4,227 @@ parameters: type: boolean default: true -jobs: -- job: 'build' - pool: 'Win-GPU-2019' - strategy: - matrix: - cuda: - additionalBuildFlags: --build_java --build_nodejs --use_cuda --cuda_version=11.4 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=52 --gen_doc validate - EnvSetupScript: setup_env_cuda_11.bat - ORT_EP_NAME: CUDA - dml: - additionalBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos --use_winml - EnvSetupScript: setup_env.bat - ORT_EP_NAME: DML - variables: - OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu' - MsbuildArguments: '-maxcpucount' - TESTONGPU: 'ON' - OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - setVcvars: true - BuildConfig: 'RelWithDebInfo' - UseOmp: '' - buildArch: x64 - msbuildPlatform: x64 - isX86: false - ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' - DocUpdateNeeded: false - timeoutInMinutes: 180 - workspace: - clean: all - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: $(buildArch) +stages: +- stage: inference + dependsOn: [] + jobs: + - job: 'build' + pool: 'Win-GPU-2019' + strategy: + matrix: + cuda: + additionalBuildFlags: --build_java --build_nodejs --use_cuda --cuda_version=11.4 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=52 --gen_doc validate + EnvSetupScript: setup_env_cuda_11.bat + ORT_EP_NAME: CUDA + dml: + additionalBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos --use_winml + EnvSetupScript: setup_env.bat + ORT_EP_NAME: DML + variables: + OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu' + MsbuildArguments: '-maxcpucount' + TESTONGPU: 'ON' + OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + setVcvars: true + BuildConfig: 'RelWithDebInfo' + UseOmp: '' + buildArch: x64 + msbuildPlatform: x64 + isX86: false + ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' + DocUpdateNeeded: false + timeoutInMinutes: 180 + workspace: + clean: all + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: $(buildArch) - - task: NodeTool@0 - inputs: - versionSpec: '12.x' - force32bit: $(isX86) + - task: NodeTool@0 + inputs: + versionSpec: '12.x' + force32bit: $(isX86) - - task: JavaToolInstaller@0 - #Our build machine doesn't have java x86 - condition: and(succeeded(), eq(variables['buildArch'], 'x64')) - inputs: - versionSpec: '11' - jdkArchitectureOption: $(buildArch) - jdkSourceOption: 'PreInstalled' + - task: JavaToolInstaller@0 + #Our build machine doesn't have java x86 + condition: and(succeeded(), eq(variables['buildArch'], 'x64')) + inputs: + versionSpec: '11' + jdkArchitectureOption: $(buildArch) + jdkSourceOption: 'PreInstalled' - - task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' - - script: | - set ORT_DOXY_SRC=$(Build.SourcesDirectory) - set ORT_DOXY_OUT=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig) - mkdir %ORT_DOXY_SRC% - mkdir %ORT_DOXY_OUT% - "C:\Program Files\doxygen\bin\doxygen.exe" $(Build.SourcesDirectory)\tools\ci_build\github\Doxyfile_csharp.cfg + - script: | + set ORT_DOXY_SRC=$(Build.SourcesDirectory) + set ORT_DOXY_OUT=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig) + mkdir %ORT_DOXY_SRC% + mkdir %ORT_DOXY_OUT% + "C:\Program Files\doxygen\bin\doxygen.exe" $(Build.SourcesDirectory)\tools\ci_build\github\Doxyfile_csharp.cfg - workingDirectory: '$(Build.SourcesDirectory)' - displayName: 'API Documentation Check and generate' + workingDirectory: '$(Build.SourcesDirectory)' + displayName: 'API Documentation Check and generate' - - script: | - python -m pip install -q setuptools wheel numpy - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' + - script: | + python -m pip install -q setuptools wheel numpy + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' - - powershell: | - $Env:USE_MSVC_STATIC_RUNTIME=1 - $Env:ONNX_ML=1 - $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static" - python setup.py bdist_wheel - python -m pip uninstall -y onnx -qq - Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} - workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' - displayName: 'Install ONNX' + - powershell: | + $Env:USE_MSVC_STATIC_RUNTIME=1 + $Env:ONNX_ML=1 + $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static" + python setup.py bdist_wheel + python -m pip uninstall -y onnx -qq + Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} + workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' + displayName: 'Install ONNX' - - task: NuGetToolInstaller@0 - displayName: Use Nuget 5.7.0 - inputs: - versionSpec: 5.7.0 + - task: NuGetToolInstaller@0 + displayName: Use Nuget 5.7.0 + inputs: + versionSpec: 5.7.0 - - task: NuGetCommand@2 - displayName: 'NuGet restore' - inputs: - command: 'restore' - feedsToUse: 'config' - restoreSolution: '$(Build.SourcesDirectory)\packages.config' - nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config' - restoreDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)' + - task: NuGetCommand@2 + displayName: 'NuGet restore' + inputs: + command: 'restore' + feedsToUse: 'config' + restoreSolution: '$(Build.SourcesDirectory)\packages.config' + nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config' + restoreDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)' - - task: PythonScript@0 - displayName: 'Generate cmake config' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests $(additionalBuildFlags)' - workingDirectory: '$(Build.BinariesDirectory)' + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests $(additionalBuildFlags)' + workingDirectory: '$(Build.BinariesDirectory)' - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln' - platform: $(msbuildPlatform) - configuration: $(BuildConfig) - msbuildArgs: $(MsbuildArguments) - msbuildArchitecture: $(buildArch) - maximumCpuCount: true - logProjectEvents: false - workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' - createLogFile: true + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln' + platform: $(msbuildPlatform) + configuration: $(BuildConfig) + msbuildArgs: $(MsbuildArguments) + msbuildArchitecture: $(buildArch) + maximumCpuCount: true + logProjectEvents: false + workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' + createLogFile: true - - task: PythonScript@0 - displayName: 'Build wheel' - inputs: - scriptPath: '$(Build.SourcesDirectory)\setup.py' - arguments: 'bdist_wheel' - workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' + - task: PythonScript@0 + displayName: 'Build wheel' + inputs: + scriptPath: '$(Build.SourcesDirectory)\setup.py' + arguments: 'bdist_wheel' + workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' - - task: NuGetCommand@2 - displayName: Restore NuGet Packages - inputs: - command: "restore" - restoreSolution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + - task: NuGetCommand@2 + displayName: Restore NuGet Packages + inputs: + command: "restore" + restoreSolution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - - task: MSBuild@1 - displayName: 'Build C#' - inputs: - solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - platform: 'Any CPU' - configuration: '$(BuildConfig)' - msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_$(ORT_EP_NAME)' - workingDirectory: '$(Build.SourcesDirectory)\csharp' + - task: MSBuild@1 + displayName: 'Build C#' + inputs: + solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' + platform: 'Any CPU' + configuration: '$(BuildConfig)' + msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_$(ORT_EP_NAME)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' - - task: DotNetCoreCLI@2 - displayName: 'Test C#' - condition: and(and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')),eq('${{ parameters.RunOnnxRuntimeTests}}', true)) - inputs: - command: test - projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj' - configuration: '$(BuildConfig)' - arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_$(ORT_EP_NAME)' - workingDirectory: '$(Build.SourcesDirectory)\csharp' + - task: DotNetCoreCLI@2 + displayName: 'Test C#' + condition: and(and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')),eq('${{ parameters.RunOnnxRuntimeTests}}', true)) + inputs: + command: test + projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj' + configuration: '$(BuildConfig)' + arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_$(ORT_EP_NAME)' + workingDirectory: '$(Build.SourcesDirectory)\csharp' - - powershell: | - Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} - python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests $(additionalBuildFlags) - workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' - condition: and(succeeded(), eq('${{ parameters.RunOnnxRuntimeTests}}', true)) - displayName: 'Run tests' + - powershell: | + Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} + python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests $(additionalBuildFlags) + workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' + condition: and(succeeded(), eq('${{ parameters.RunOnnxRuntimeTests}}', true)) + displayName: 'Run tests' - # if the validation from --gen_doc failed it sets a variable so we can publish the latest version of the docs - # as an artifact, allowing a developer to download this and replace the current version instead of having to build - # and generate the docs locally themselves. handle each of the two md files separately - simpler than copying - # them to another location and publishing from there in a single task. - - task: PublishBuildArtifacts@1 - condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true')) - inputs: - pathtoPublish: '$(Build.SourcesDirectory)/docs/OperatorKernels.md' - artifactName: 'OperatorKernels.md' + # if the validation from --gen_doc failed it sets a variable so we can publish the latest version of the docs + # as an artifact, allowing a developer to download this and replace the current version instead of having to build + # and generate the docs locally themselves. handle each of the two md files separately - simpler than copying + # them to another location and publishing from there in a single task. + - task: PublishBuildArtifacts@1 + condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true')) + inputs: + pathtoPublish: '$(Build.SourcesDirectory)/docs/OperatorKernels.md' + artifactName: 'OperatorKernels.md' - - task: PublishBuildArtifacts@1 - condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true')) - inputs: - pathtoPublish: '$(Build.SourcesDirectory)/docs/ContribOperators.md' - artifactName: 'ContribOperators.md' + - task: PublishBuildArtifacts@1 + condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true')) + inputs: + pathtoPublish: '$(Build.SourcesDirectory)/docs/ContribOperators.md' + artifactName: 'ContribOperators.md' - - task: PublishSecurityAnalysisLogs@3 - displayName: 'Publish Security Analysis Logs' - condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')) - inputs: - ArtifactName: CodeAnalysisLogs + - task: PublishSecurityAnalysisLogs@3 + displayName: 'Publish Security Analysis Logs' + condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')) + inputs: + ArtifactName: CodeAnalysisLogs - - task: PublishTestResults@2 - displayName: 'Publish unit test results' - inputs: - testResultsFiles: '**/*.results.xml' - searchFolder: '$(Build.BinariesDirectory)' - testRunTitle: 'Unit Test Run' - condition: succeededOrFailed() + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + +- stage: training + dependsOn: [] + jobs: + - template: templates/win-ci-2019.yml + parameters: + AgentPool : 'Win-GPU-2019' + JobName: 'Win_GPU_Training' + # TODO fix test failures and remove --skip_onnx_tests + BuildCommand: >- + --build_dir $(Build.BinariesDirectory) + --build_shared_lib + --cmake_generator "Visual Studio 16 2019" + --enable_onnx_tests + --enable_training + --skip_onnx_tests + --skip_submodule_sync + --use_cuda + --cuda_version 10.2 + --cuda_home "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2" + --cudnn_home "C:\local\cudnn-10.2-windows10-x64-v8.0.3.33\cuda" + --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52" + BuildArch: 'x64' + EnvSetupScript: 'setup_env_cuda.bat' + sln_platform: 'x64' + CudaVersion: '10.2' + OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu' + BuildConfigurations: ['RelWithDebInfo'] + # Enable unreleased onnx opsets in CI builds + # This facilitates testing the implementation for the new opsets + AllowReleasedOpsetOnly: '0' + DoCompliance: 'true' \ No newline at end of file