From ae8df4db8f3422e4461375dc76f37c2875d7d301 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 3 Jun 2024 14:08:45 -0700 Subject: [PATCH] Split java's gradle build and test (#20817) ### Description This PR to allow `./gradlew cmakeCheck` failed on Windows_Packaging_(CUDA|TensorRT) Job. This way, it will still generate all nessary jar and pom file need for later stage to consume while `./gradlew cmakeCheck`will be also run again in the Windows_Packaging_(CUDA|TensorRT)_Testing stage. ### Motivation and Context Reduce the time of All java packaging stages by 30+ min. --- .../stages/java-cuda-packaging-stage.yml | 6 +-- .../templates/make_java_win_binaries.yml | 15 +++++-- .../azure-pipelines/templates/win-ci.yml | 39 +++++++++---------- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml index 5f355478f2..1d5b810dfe 100644 --- a/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml @@ -10,10 +10,8 @@ stages: - stage: Jar_Packaging_GPU dependsOn: - Linux_C_API_Packaging_GPU -# Because Java Jar is published only after Windows Packaging GPU Testing stage we need to depend on the Testing stages -# TODO: change Windows_Packaging_*_Testing to Windows_Packaging_* once we finish PRODUCT BACKLOG ITEM 34666 - - Windows_Packaging_CUDA_Testing - - Windows_Packaging_TensorRT_Testing + - Windows_Packaging_CUDA + - Windows_Packaging_TensorRT - Download_Java_Tools jobs: - job: Jar_Packaging_GPU diff --git a/tools/ci_build/github/azure-pipelines/templates/make_java_win_binaries.yml b/tools/ci_build/github/azure-pipelines/templates/make_java_win_binaries.yml index 756a7a4834..9fa9f5e4a4 100644 --- a/tools/ci_build/github/azure-pipelines/templates/make_java_win_binaries.yml +++ b/tools/ci_build/github/azure-pipelines/templates/make_java_win_binaries.yml @@ -3,16 +3,25 @@ parameters: type: string - name: java_artifact_id type: string + - name: buildOnly + type: boolean steps: + - task: CmdLine@2 + displayName: 'Gradle cmakeCheck' + continueOnError: ${{ parameters.buildOnly }} + inputs: + script: | + @echo on + call gradlew.bat cmakeCheck -DcmakeBuildDir=$(Build.BinariesDirectory)\RelWithDebInfo + workingDirectory: $(Build.SourcesDirectory)\java + failOnStderr: ${{ not(parameters.buildOnly) }} + - task: CmdLine@2 displayName: 'Add symbols and notices to Java' inputs: script: | @echo on - cd $(Build.SourcesDirectory)\java - call $(Build.SourcesDirectory)\java\gradlew.bat cmakeCheck -DcmakeBuildDir=$(Build.BinariesDirectory)\RelWithDebInfo - if %errorlevel% neq 0 exit /b %errorlevel% cd $(Build.BinariesDirectory)\RelWithDebInfo set NATIVE_FOLDER=$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}\stage\ai\onnxruntime\native\win-x64 mkdir %NATIVE_FOLDER% diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 35c23e311e..c726054d8e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -191,19 +191,24 @@ stages: createLogFile: true # For CPU job, tests are run in the same machine as building - - ${{ if contains(parameters.ort_build_pool_name, 'CPU') }}: - - ${{ if eq(parameters.buildJava, 'true') }}: - - template: make_java_win_binaries.yml - parameters: - msbuildPlatform: ${{ parameters.msbuildPlatform }} - java_artifact_id: ${{ parameters.java_artifact_id }} + - ${{ if eq(parameters.buildJava, 'true') }}: + - template: make_java_win_binaries.yml + parameters: + msbuildPlatform: ${{ parameters.msbuildPlatform }} + java_artifact_id: ${{ parameters.java_artifact_id }} + ${{ if contains(parameters.ort_build_pool_name, 'CPU') }}: + buildOnly: false + # When it is a GPU build, we only assemble the java binaries, testing will be done in the later stage with GPU machine + ${{ else }}: + buildOnly: true - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), eq('${{ parameters.buildJava}}', true)) - displayName: 'Publish Java temp binaries' - inputs: - pathtoPublish: '$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}' - artifactName: 'drop-onnxruntime-java-win-${{ parameters.packageName }}${{parameters.artifact_name_suffix}}' + - task: PublishBuildArtifacts@1 + displayName: 'Publish Java temp binaries' + inputs: + pathtoPublish: '$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}' + artifactName: 'drop-onnxruntime-java-win-${{ parameters.packageName }}${{parameters.artifact_name_suffix}}' + # All GPU builds will be tested in the next stage with GPU machine + - ${{ if contains(parameters.ort_build_pool_name, 'CPU') }}: - task: PythonScript@0 displayName: 'test' condition: and(succeeded(), eq('${{ parameters.runTests}}', true)) @@ -386,16 +391,10 @@ stages: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --enable_onnx_tests $(TelemetryOption) ' workingDirectory: '$(Build.BinariesDirectory)' - +# Previous stage only assembles the java binaries, testing will be done in this stage with GPU machine - ${{ if eq(parameters.buildJava, 'true') }}: - template: make_java_win_binaries.yml parameters: msbuildPlatform: ${{ parameters.msbuildPlatform }} java_artifact_id: ${{ parameters.java_artifact_id }} - - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), eq('${{ parameters.buildJava}}', true)) - displayName: 'Publish Java temp binaries' - inputs: - pathtoPublish: '$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}' - artifactName: 'drop-onnxruntime-java-win-${{ parameters.packageName }}${{parameters.artifact_name_suffix}}' + buildOnly: false \ No newline at end of file