diff --git a/orttraining/orttraining/models/mnist/main.cc b/orttraining/orttraining/models/mnist/main.cc index 185a229726..0f1f4a6397 100644 --- a/orttraining/orttraining/models/mnist/main.cc +++ b/orttraining/orttraining/models/mnist/main.cc @@ -188,7 +188,7 @@ int main(int argc, char* args[]) { std::vector feeds{"X", "labels"}; auto trainingData = std::make_shared(feeds); auto testData = std::make_shared(feeds); - std::string mnist_data_path(params.train_data_dir.begin(), params.train_data_dir.end()); + std::string mnist_data_path = ToMBString(params.train_data_dir); PrepareMNISTData(mnist_data_path, IMAGE_DIMS, LABEL_DIMS, *trainingData, *testData, params.mpi_context.world_rank /* shard_to_load */, device_count /* total_shards */); if (testData->NumSamples() == 0) { diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index bdff6f41a0..b1ce5a97f3 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -3,7 +3,7 @@ trigger: none jobs: - template: templates/linux-ci.yml parameters: - AgentPool : 'AI-Linux-CPU' + AgentPool : 'Linux-CPU' JobName: 'Onnxruntime_Linux_CPU_Training' SubmoduleCheckoutMode: 'recursive' BuildCommand: > diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml index cb0e3b9c90..88ee297fd4 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml @@ -3,7 +3,7 @@ trigger: none jobs: - template: templates/linux-ci.yml parameters: - AgentPool : 'AI-Linux-GPU' + AgentPool : 'Linux-GPU-CUDA10' JobName: 'Onnxruntime_Linux_GPU_Training' SubmoduleCheckoutMode: 'recursive' # TODO --skip_onnx_tests is used to disable onnx_test_runner test. diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-inference-only-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-inference-only-ci-pipeline.yml deleted file mode 100644 index 35f161974a..0000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-inference-only-ci-pipeline.yml +++ /dev/null @@ -1,16 +0,0 @@ -trigger: none - -jobs: -- template: templates/linux-ci.yml - parameters: - AgentPool : 'AI-Linux-GPU' - JobName: 'Onnxruntime_Linux_GPU_Inference' - SubmoduleCheckoutMode: 'recursive' - BuildCommand: > - tools/ci_build/github/linux/run_dockerbuild.sh - -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) - -x " - " - DoNugetPack: 'false' - ArtifactName: 'drop-linux' - TimeoutInMinutes: 90 diff --git a/tools/ci_build/github/azure-pipelines/orttraining-mac-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-mac-ci-pipeline.yml index 2801dd7862..1c8483faf5 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-mac-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-mac-ci-pipeline.yml @@ -3,8 +3,7 @@ trigger: none jobs: - template: templates/mac-ci.yml parameters: - AgentPool : 'Hosted macOS High Sierra' - JobName: 'Onnxruntime_Mac_CPU_Training' + JobName: 'Mac_CPU_Training' SubmoduleCheckoutMode: 'recursive' BuildCommand: > python3 $(Build.SourcesDirectory)/tools/ci_build/build.py diff --git a/tools/ci_build/github/azure-pipelines/orttraining-win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-win-ci-pipeline.yml index beb19f0c7d..023c24b5b5 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-win-ci-pipeline.yml @@ -1,102 +1,18 @@ trigger: none jobs: -- job: Onnxruntime_Win_CPU_Training - - timeoutInMinutes: 160 - - strategy: - maxParallel: 2 - matrix: - Debug: - buildConfig: Debug - Release: - buildConfig: Release - - variables: - - name: buildDirectory # currently some csproj files depend on this variable - would be nice if they didn't - value: $(Build.BinariesDirectory) - - name: BUILD_PY_COMMON_OPTIONS - value: >- - --build_dir $(Build.BinariesDirectory) --config $(buildConfig) - --cmake_path $(Build.BinariesDirectory)\packages\cmake\bin\cmake.exe - --ctest_path $(Build.BinariesDirectory)\packages\cmake\bin\ctest.exe - --build_shared_lib - --use_dnnl - --use_openmp - --enable_training - - steps: - - checkout: self - clean: true - submodules: recursive - - - template: templates/set-test-data-variables-step.yml - - - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0 - displayName: 'Download python installer from Artifact Services Drop' - inputs: - dropServiceURI: 'https://aiinfra.artifacts.visualstudio.com/DefaultCollection' - buildNumber: miniconda3/4.4.10/x64 - destinationPath: '$(Build.BinariesDirectory)\installer\python' - - - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0 - displayName: 'Download cmake' - inputs: - dropServiceURI: 'https://aiinfra.artifacts.visualstudio.com/DefaultCollection' - buildNumber: cmake/x64/3.13.4 - destinationPath: '$(Build.BinariesDirectory)\packages\cmake' - timeoutInMinutes: 10 - - - task: CmdLine@1 - displayName: 'Run python installer' - inputs: - filename: '$(Build.BinariesDirectory)\installer\python\installer.exe' - arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' - timeoutInMinutes: 10 - - - task: BatchScript@1 - displayName: 'Setup VS2017 env vars' - inputs: - filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat' - modifyEnvironment: true - - - task: CmdLine@1 - displayName: 'Generate build files' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: >- - $(Build.SourcesDirectory)\tools\ci_build\build.py - $(BUILD_PY_COMMON_OPTIONS) - --update --skip_submodule_sync - - - task: CmdLine@1 - displayName: 'Build' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: >- - $(Build.SourcesDirectory)\tools\ci_build\build.py - $(BUILD_PY_COMMON_OPTIONS) - --build --parallel - - - task: CmdLine@1 - displayName: 'Test' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: >- - $(Build.SourcesDirectory)\tools\ci_build\build.py - $(BUILD_PY_COMMON_OPTIONS) - --test - - - task: PublishTestResults@2 - displayName: 'Publish unit test results' - inputs: - testResultsFiles: '**\*.results.xml' - searchFolder: '$(Build.BinariesDirectory)' - testRunTitle: 'Unit Test Run' - condition: succeededOrFailed() - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - - - template: templates/clean-agent-build-directory-step.yml +- template: templates/win-ci-2019.yml + parameters: + AgentPool : 'Win-CPU-2019' + JobName: 'Win_CPU_Training' + BuildCommand: >- + --build_dir $(Build.BinariesDirectory) + --build_shared_lib + --cmake_generator "Visual Studio 16 2019" + --enable_training + --skip_submodule_sync + --use_openmp + BuildArch: 'x64' + EnvSetupScript: 'setup_env.bat' + sln_platform: 'x64' + BuildConfigurations: ['Debug', 'RelWithDebInfo'] diff --git a/tools/ci_build/github/azure-pipelines/orttraining-win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-win-gpu-ci-pipeline.yml index 5ba38b4ed1..478230ecb4 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-win-gpu-ci-pipeline.yml @@ -1,121 +1,23 @@ trigger: none jobs: -- job: Onnxruntime_Win_GPU_Training - - timeoutInMinutes: 180 - - strategy: - maxParallel: 2 - matrix: - # TODO: Win-GPU-Debug build is failing due to warning 4996. - # Re-enable this when merge back to public master - # Debug: - # buildConfig: Debug - Release: - buildConfig: Release - - variables: - - name: CUDA_VERSION - value: '10.0' - - name: CUDA_PATH - value: 'C:/local/cuda_10.0.130_win10' - - name: CUDNN_PATH - value: 'C:/local/cudnn-10.0-windows10-x64-v7.3.1.20/cuda' - - name: BUILD_PY_COMMON_OPTIONS - value: >- - --build_dir $(Build.BinariesDirectory) --config $(buildConfig) - --cmake_path $(Build.BinariesDirectory)\packages\cmake\bin\cmake.exe - --ctest_path $(Build.BinariesDirectory)\packages\cmake\bin\ctest.exe - --build_shared_lib - --use_cuda --cudnn_home $(CUDNN_PATH) --cuda_home $(CUDA_PATH) - --use_dnnl - --use_mklml - --cmake_extra_defines CUDA_TOOLKIT_ROOT_DIR=$(CUDA_PATH) - --enable_training - - steps: - - checkout: self - clean: true - submodules: recursive - - - template: templates/set-test-data-variables-step.yml - - - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0 - displayName: 'Download python installer from Artifact Services Drop' - inputs: - dropServiceURI: 'https://aiinfra.artifacts.visualstudio.com/DefaultCollection' - buildNumber: miniconda3/4.4.10/x64 - destinationPath: '$(Build.BinariesDirectory)\installer\python' - - - task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@0 - displayName: 'Download cmake' - inputs: - dropServiceURI: 'https://aiinfra.artifacts.visualstudio.com/DefaultCollection' - buildNumber: cmake/x64/3.13.4 - destinationPath: '$(Build.BinariesDirectory)\packages\cmake' - - - task: CmdLine@1 - displayName: 'Run python installer' - inputs: - filename: '$(Build.BinariesDirectory)\installer\python\installer.exe' - arguments: '/S /NoRegistry=1 /AddToPath=0 /RegisterPython=0 /D=$(Build.BinariesDirectory)\packages\python' - - - task: PowerShell@1 - displayName: 'Set CUDA path' - inputs: - scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1' - arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-10.0.130-win10 -CudaVersion $(CUDA_VERSION)' - - - task: BatchScript@1 - displayName: 'Setup VS2017 env vars' - inputs: - filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat' - arguments: 'amd64 -vcvars_ver=14.11' - modifyEnvironment: true - - - task: CmdLine@1 - displayName: 'Generate build files' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: >- - $(Build.SourcesDirectory)\tools\ci_build\build.py - $(BUILD_PY_COMMON_OPTIONS) - --update --skip_submodule_sync - - - task: CmdLine@1 - displayName: 'Build' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: >- - $(Build.SourcesDirectory)\tools\ci_build\build.py - $(BUILD_PY_COMMON_OPTIONS) - --build --parallel - - - task: CmdLine@1 - displayName: 'Test' - inputs: - filename: '$(Build.BinariesDirectory)\packages\python\python.exe' - arguments: >- - $(Build.SourcesDirectory)\tools\ci_build\build.py - $(BUILD_PY_COMMON_OPTIONS) - --test - - - task: PowerShell@1 - displayName: 'Clean up CUDA props files' - inputs: - scriptName: 'tools/ci_build/github/windows/clean_up_cuda_prop_files.ps1' - arguments: '-CudaVersion $(CUDA_VERSION)' - - - task: PublishTestResults@2 - displayName: 'Publish unit test results' - inputs: - testResultsFiles: '**\*.results.xml' - searchFolder: '$(Build.BinariesDirectory)' - testRunTitle: 'Unit Test Run' - condition: succeededOrFailed() - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - - - template: templates/clean-agent-build-directory-step.yml +- template: templates/win-ci-2019.yml + parameters: + AgentPool : 'Win-GPU-2019' + JobName: 'Win_GPU_Training' + BuildCommand: >- + --build_dir $(Build.BinariesDirectory) + --build_shared_lib + --cmake_generator "Visual Studio 16 2019" + --enable_training + --skip_submodule_sync + --use_cuda + --cuda_version 10.1 + --cuda_home "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" + --cudnn_home "C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda" + BuildArch: 'x64' + EnvSetupScript: 'setup_env_cuda.bat' + sln_platform: 'x64' + CudaVersion: '10.1' + OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu' + BuildConfigurations: ['Debug', 'RelWithDebInfo'] diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml index cb3f88361c..3955836f44 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-2019.yml @@ -1,5 +1,5 @@ parameters: - DoDebugBuild: 'true' + DoDebugBuild: 'true' # Unused. Use BuildConfigurations instead. DoCompliance: 'false' BuildCommand: '' JobName: 'Windows_CI_Dev' @@ -15,10 +15,17 @@ parameters: AgentPool: 'Win-CPU-2019' AgentDemands: [] OrtPackageId: Microsoft.ML.OnnxRuntime + BuildConfigurations: ['RelWithDebInfo'] # Options: Debug, RelWithDebInfo RunTests : 'true' jobs: - job: ${{ parameters.JobName }} timeoutInMinutes: 120 + strategy: + maxParallel: 2 + matrix: + ${{ each BuildConfiguration in parameters.BuildConfigurations }}: + ${{ BuildConfiguration }}: + BuildConfig: ${{ BuildConfiguration }} workspace: clean: all pool: @@ -33,6 +40,10 @@ jobs: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true steps: + - checkout: self + clean: true + submodules: recursive + - powershell: | if($env:TELEMETRYGUID) { @@ -68,7 +79,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '$(BuildCommand) --update --config RelWithDebInfo --enable_lto' + arguments: '$(BuildCommand) --update --config $(BuildConfig) --enable_lto' workingDirectory: '$(Build.BinariesDirectory)' - ${{ if notIn(parameters['sln_platform'], 'Win32', 'x64') }}: @@ -79,18 +90,19 @@ jobs: - task: VSBuild@1 displayName: 'Build' inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln' platform: ${{ parameters.sln_platform }} - configuration: RelWithDebInfo + configuration: $(BuildConfig) msbuildArchitecture: ${{ parameters.BuildArch }} maximumCpuCount: true logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' createLogFile: true - # Build RelWithDebInfo -- this variable required to build C# + # The Configuration variable is required to build C# - script: | - @echo ##vso[task.setvariable variable=Configuration]RelWithDebInfo + @echo ##vso[task.setvariable variable=Configuration]$(BuildConfig) + displayName: 'Set Configuration variable' - template: set-test-data-variables-step.yml @@ -104,8 +116,8 @@ jobs: inputs: command: restore projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - configuration: 'RelWithDebInfo' - arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=${{ parameters.OrtPackageId }}' + configuration: '$(BuildConfig)' + arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OrtPackageId=${{ parameters.OrtPackageId }}' workingDirectory: '$(Build.SourcesDirectory)\csharp' - task: DotNetCoreCLI@2 @@ -113,8 +125,8 @@ jobs: inputs: command: build projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' - configuration: 'RelWithDebInfo' - arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}' + configuration: '$(BuildConfig)' + arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}' workingDirectory: '$(Build.SourcesDirectory)\csharp' - ${{ if in(parameters['sln_platform'], 'Win32', 'x64') }}: @@ -130,21 +142,21 @@ jobs: inputs: command: test projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests\Microsoft.ML.OnnxRuntime.Tests.csproj' - configuration: 'RelWithDebInfo' - arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}' + configuration: '$(BuildConfig)' + arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=${{ parameters.OrtPackageId }}' workingDirectory: '$(Build.SourcesDirectory)\csharp' - ${{ if eq(parameters.RunTests, true) }}: - script: | - mklink /D /J $(Build.BinariesDirectory)\RelWithDebInfo\models $(Build.BinariesDirectory)\models + mklink /D /J $(Build.BinariesDirectory)\$(BuildConfig)\models $(Build.BinariesDirectory)\models DIR dist\ /S /B > wheel_filename_file set /p WHEEL_FILENAME=