From afca0d15ee20a13895ec36ade2c9c9582e818353 Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Mon, 1 Jun 2020 18:18:57 -0700 Subject: [PATCH] Create Java publishing pipeline (#3944) Create CPU and GPu Java publishing pipelines. Final jars are tested on all platforms. However, signing and publishing to maven are manual steps. --- cmake/CMakeLists.txt | 4 +- cmake/onnxruntime_unittests.cmake | 13 +- .../java/ai/onnxruntime/InferenceTest.java | 4 +- .../java-api-packaging-pipelines-gpu.yml | 243 +++++++++++++ .../java-api-packaging-pipelines.yml | 336 ++++++++++++++++++ ...ifacts-package-and-publish-steps-posix.yml | 29 ++ ...acts-package-and-publish-steps-windows.yml | 56 +++ .../templates/java-test-final-jar-step.yml | 80 +++++ .../github/linux/docker/Dockerfile.centos6 | 4 +- .../linux/docker/Dockerfile.centos6_gpu | 4 +- .../linux/docker/scripts/install_centos.sh | 1 + .../linux/docker/scripts/install_deps.sh | 6 +- .../github/linux/java_copy_strip_binary.sh | 56 +++ .../github/linux/java_linux_final_test.sh | 36 ++ 14 files changed, 860 insertions(+), 12 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml create mode 100644 tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml create mode 100644 tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml create mode 100644 tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-windows.yml create mode 100644 tools/ci_build/github/azure-pipelines/templates/java-test-final-jar-step.yml create mode 100755 tools/ci_build/github/linux/java_copy_strip_binary.sh create mode 100755 tools/ci_build/github/linux/java_linux_final_test.sh diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 33bf09ed62..c253eed572 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -15,12 +15,14 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET "10.12" CACHE STRING "Minimum OS X deployment ve # Project project(onnxruntime C CXX) +# Needed for Java +set (CMAKE_C_STANDARD 99) include(CheckCXXCompilerFlag) include(CheckLanguage) # CentOS compiler is old but it does allow certain C++14 features -# such as lambda captures and they are convinient +# such as lambda captures and they are convenient # On the other hand it does not allow some others. # So we cant' regulate simply with the standard. set(CMAKE_CXX_STANDARD 14) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index f4f19985e0..19a947057f 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -830,12 +830,15 @@ set_property(TARGET custom_op_library APPEND_STRING PROPERTY LINK_FLAGS ${ONNXRU if (onnxruntime_BUILD_JAVA) message(STATUS "Running Java tests") + # native-test is added to resources so custom_op_lib can be loaded + # and we want to symlink it there + set(JAVA_NATIVE_TEST_DIR ${JAVA_OUTPUT_DIR}/native-test) + file(MAKE_DIRECTORY ${JAVA_NATIVE_TEST_DIR}) + # delegate to gradle's test runner if(WIN32) - # If we're on windows, symlink the custom op test library somewhere we can see it - set(JAVA_NATIVE_TEST_DIR ${JAVA_OUTPUT_DIR}/native-test) - file(MAKE_DIRECTORY ${JAVA_NATIVE_TEST_DIR}) - add_custom_command(TARGET custom_op_library POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${JAVA_NATIVE_TEST_DIR}/$) + add_custom_command(TARGET custom_op_library POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ + ${JAVA_NATIVE_TEST_DIR}/$) # On windows ctest requires a test to be an .exe(.com) file # So there are two options 1) Install Chocolatey and its gradle package # That package would install gradle.exe shim to its bin so ctest could run gradle.exe @@ -847,6 +850,8 @@ if (onnxruntime_BUILD_JAVA) -DREPO_ROOT=${REPO_ROOT} -P ${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime_java_unittests.cmake) else() + add_custom_command(TARGET custom_op_library POST_BUILD COMMAND ${CMAKE_COMMAND} -E create_symlink $ + ${JAVA_NATIVE_TEST_DIR}/$) if (onnxruntime_USE_CUDA) add_test(NAME onnxruntime4j_test COMMAND ${GRADLE_EXECUTABLE} cmakeCheck -DcmakeBuildDir=${CMAKE_CURRENT_BINARY_DIR} -DUSE_CUDA=1 WORKING_DIRECTORY ${REPO_ROOT}/java) diff --git a/java/src/test/java/ai/onnxruntime/InferenceTest.java b/java/src/test/java/ai/onnxruntime/InferenceTest.java index 99b2fd3c7c..bea793cddc 100644 --- a/java/src/test/java/ai/onnxruntime/InferenceTest.java +++ b/java/src/test/java/ai/onnxruntime/InferenceTest.java @@ -907,9 +907,9 @@ public class InferenceTest { // So we look it up as a classpath resource and resolve it to a real path customLibraryName = getResourcePath("/custom_op_library.dll").toString(); } else if (osName.contains("mac")) { - customLibraryName = "libcustom_op_library.dylib"; + customLibraryName = getResourcePath("/libcustom_op_library.dylib").toString(); } else if (osName.contains("linux")) { - customLibraryName = "./libcustom_op_library.so"; + customLibraryName = getResourcePath("/libcustom_op_library.so").toString(); } else { fail("Unknown os/platform '" + osName + "'"); } diff --git a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml new file mode 100644 index 0000000000..d6b28a71df --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml @@ -0,0 +1,243 @@ +jobs: +- job: Linux_Java_API_Build_GPU_x64 + workspace: + clean: all + timeoutInMinutes: 60 + pool: 'Linux-GPU-CUDA10' + steps: + - template: templates/set-version-number-variables-step.yml + - task: CmdLine@2 + inputs: + script: | + sudo docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . + workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker + - task: CmdLine@2 + inputs: + script: | + sudo docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 + workingDirectory: $(Build.SourcesDirectory) + - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: 'linux-x64' + buildConfig: 'Release' + artifactName: 'onnxruntime-java-linux-x64-gpu' + version: '$(OnnxRuntimeVersion)' + libraryName: 'libonnxruntime.so' + nativeLibraryName: 'libonnxruntime4j_jni.so' + + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + - template: templates/clean-agent-build-directory-step.yml + +- job: Windows_Java_API_Build_GPU_x64 + workspace: + clean: all + pool: 'Win-GPU-2019' + timeoutInMinutes: 120 + variables: + EnvSetupScript: setup_env_cuda.bat + buildArch: x64 + msbuildArch: amd64 + msbuildPlatform: x64 + buildparameter: --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda" + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: $(buildArch) + + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy scipy + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + + - powershell: | + $Env:USE_MSVC_STATIC_RUNTIME=1 + $Env:ONNX_ML=1 + $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static" + python setup.py bdist_wheel + Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} + workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' + displayName: 'Install ONNX' + + - template: templates/set-test-data-variables-step.yml + - template: templates/set-version-number-variables-step.yml + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_lto --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' + workingDirectory: '$(Build.BinariesDirectory)' + + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: $(msbuildPlatform) + configuration: RelWithDebInfo + msbuildArchitecture: $(buildArch) + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + + - task: PythonScript@0 + displayName: 'test' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' + workingDirectory: '$(Build.BinariesDirectory)' + + - template: templates/java-api-artifacts-package-and-publish-steps-windows.yml + parameters: + buildConfig: RelWithDebInfo + artifactName: 'onnxruntime-java-win-$(buildArch)-gpu' + version: '$(OnnxRuntimeVersion)' + commitId: $(OnnxRuntimeGitCommitHash) + + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + - template: templates/clean-agent-build-directory-step.yml + +- job: Jar_Packaging + workspace: + clean: all + pool: 'Win-CPU-2019' + dependsOn: + - Windows_Java_API_Build_GPU_x64 + - Linux_Java_API_Build_GPU_x64 + condition: succeeded() + steps: + - template: templates/set-version-number-variables-step.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Pipeline Artifact - Win x64' + inputs: + buildType: 'current' + artifactName: 'drop-onnxruntime-java-win-x64-gpu' + targetPath: '$(Build.BinariesDirectory)\java-artifact' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Pipeline Artifact - Linux x64' + inputs: + buildType: 'current' + artifactName: 'drop-onnxruntime-java-linux-x64-gpu' + targetPath: '$(Build.BinariesDirectory)\java-artifact' + + - task: ExtractFiles@1 + inputs: + archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu.zip' + destinationFolder: '$(Build.BinariesDirectory)\java-artifact' + cleanDestinationFolder: false + + - task: ExtractFiles@1 + inputs: + archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64-gpu.zip' + destinationFolder: '$(Build.BinariesDirectory)\java-artifact' + cleanDestinationFolder: false + + - task: CmdLine@2 + inputs: + script: | + pushd onnxruntime-java-linux-x64-gpu + jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu\testing.jar libcustom_op_library.so + del /F /Q libcustom_op_library.so + jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu\onnxruntime-$(OnnxRuntimeVersion).jar . + popd + pushd onnxruntime-java-win-x64-gpu + ren onnxruntime-$(OnnxRuntimeVersion).jar onnxruntime-gpu-$(OnnxRuntimeVersion).jar + ren onnxruntime-$(OnnxRuntimeVersion)-javadoc.jar onnxruntime-gpu-$(OnnxRuntimeVersion)-javadoc.jar + ren onnxruntime-$(OnnxRuntimeVersion)-sources.jar onnxruntime-gpu-$(OnnxRuntimeVersion)-sources.jar + ren onnxruntime-$(OnnxRuntimeVersion).pom onnxruntime-gpu-$(OnnxRuntimeVersion).pom + popd + workingDirectory: '$(Build.BinariesDirectory)\java-artifact' + displayName: 'Create final GPU Jar' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu' + artifact: 'onnxruntime-java-gpu' + publishLocation: 'pipeline' + + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + +- job: Final_Jar_Testing_Windows + workspace: + clean: all + pool: 'Win-GPU-2019' + timeoutInMinutes: 60 + dependsOn: + Jar_Packaging + steps: + - template: templates/set-version-number-variables-step.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: 'onnxruntime-java-gpu' + targetPath: '$(Build.BinariesDirectory)\final-jar' + + - task: BatchScript@1 + displayName: 'Setup CUDA Env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_cuda.bat' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - task: CmdLine@2 + inputs: + script: | + mkdir test + pushd test + jar xf $(Build.BinariesDirectory)\final-jar\testing.jar + popd + powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar" + powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar" + java -DUSE_CUDA=1 -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime-gpu-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner + workingDirectory: '$(Build.BinariesDirectory)\final-jar' + +- job: Final_Jar_Testing_Linux + workspace: + clean: all + pool: 'Linux-GPU-CUDA10' + timeoutInMinutes: 60 + dependsOn: + Jar_Packaging + steps: + - template: templates/set-version-number-variables-step.yml + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: 'onnxruntime-java-gpu' + targetPath: '$(Build.BinariesDirectory)/final-jar' + + - task: CmdLine@2 + inputs: + script: | + sudo docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . + workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker + + - task: CmdLine@2 + inputs: + script: | + sudo docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build + workingDirectory: $(Build.BinariesDirectory)/final-jar diff --git a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml new file mode 100644 index 0000000000..3cdc47ae3c --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml @@ -0,0 +1,336 @@ +jobs: +- job: Linux_Java_API_Build_CPU_x64 + workspace: + clean: all + timeoutInMinutes: 60 + pool: 'Linux-CPU' + steps: + - template: templates/set-version-number-variables-step.yml + - template: templates/linux-set-variables-and-download.yml + + - task: CmdLine@2 + inputs: + script: | + sudo docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . + workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker + displayName: 'Build CentoOS6 docker image' + - task: CmdLine@2 + inputs: + script: | + sudo docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + workingDirectory: $(Build.SourcesDirectory) + displayName: 'Run build and test' + + - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: 'linux-x64' + buildConfig: 'Release' + artifactName: 'onnxruntime-java-linux-x64' + version: '$(OnnxRuntimeVersion)' + libraryName: 'libonnxruntime.so' + nativeLibraryName: 'libonnxruntime4j_jni.so' + + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + - template: templates/clean-agent-build-directory-step.yml + +- job: MacOS_Java_API_Build_CPU_x64 + workspace: + clean: all + pool: + vmImage: 'macOS-10.14' + steps: + - template: templates/set-version-number-variables-step.yml + + - script: | + export JAVA_HOME=/Library/Java/JavaVirtualMachines/zulu-11.jdk/Contents/Home + java --version + javac --version + sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' + sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_java --build_shared_lib --use_openmp --config RelWithDebInfo + + displayName: 'Build and Test MacOS' + - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: 'osx-x64' + buildConfig: 'RelWithDebInfo' + artifactName: 'onnxruntime-java-osx-x64' + version: '$(OnnxRuntimeVersion)' + libraryName: 'libonnxruntime.dylib' + nativeLibraryName: 'libonnxruntime4j_jni.dylib' + + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + - template: templates/clean-agent-build-directory-step.yml + +- job: Windows_Java_API_Build_CPU_x64 + workspace: + clean: all + pool: 'Win-CPU-2019' + timeoutInMinutes: 160 + strategy: + maxParallel: 2 + matrix: +# x86: + #EnvSetupScript: setup_env_x86.bat + #buildArch: x86 + #msbuildArch: x86 + #msbuildPlatform: Win32 + #buildparameter: --x86 + x64: + EnvSetupScript: setup_env.bat + buildArch: x64 + msbuildArch: amd64 + msbuildPlatform: x64 + buildparameter: + + steps: + - template: templates/enable-telemetry.yml + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: $(buildArch) + + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + + - script: | + python -m pip install -q pyopenssl setuptools wheel numpy scipy + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + + - powershell: | + $Env:USE_MSVC_STATIC_RUNTIME=1 + $Env:ONNX_ML=1 + $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static" + python setup.py bdist_wheel + Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} + workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' + displayName: 'Install ONNX' + + - template: templates/set-test-data-variables-step.yml + - template: templates/set-version-number-variables-step.yml + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_wcos --build_java --update --cmake_generator "Visual Studio 16 2019" --enable_lto --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' + workingDirectory: '$(Build.BinariesDirectory)' + + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: $(msbuildPlatform) + configuration: RelWithDebInfo + msbuildArchitecture: $(buildArch) + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + + + - task: PythonScript@0 + displayName: 'test' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --build_shared_lib --enable_wcos --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' + workingDirectory: '$(Build.BinariesDirectory)' + + - template: templates/java-api-artifacts-package-and-publish-steps-windows.yml + parameters: + buildConfig: RelWithDebInfo + artifactName: 'onnxruntime-java-win-$(buildArch)' + version: '$(OnnxRuntimeVersion)' + commitId: $(OnnxRuntimeGitCommitHash) + + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + - template: templates/clean-agent-build-directory-step.yml + +- job: Jar_Packaging + workspace: + clean: all + pool: 'Win-CPU-2019' + dependsOn: + - Windows_Java_API_Build_CPU_x64 + - Linux_Java_API_Build_CPU_x64 + - MacOS_Java_API_Build_CPU_x64 + condition: succeeded() + steps: + - template: templates/set-version-number-variables-step.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Pipeline Artifact - Win x64' + inputs: + buildType: 'current' + artifactName: 'drop-onnxruntime-java-win-x64' + targetPath: '$(Build.BinariesDirectory)\java-artifact' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Pipeline Artifact - Linux x64' + inputs: + buildType: 'current' + artifactName: 'drop-onnxruntime-java-linux-x64' + targetPath: '$(Build.BinariesDirectory)\java-artifact' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Pipeline Artifact - MacOS x64' + inputs: + buildType: 'current' + artifactName: 'drop-onnxruntime-java-osx-x64' + targetPath: '$(Build.BinariesDirectory)\java-artifact' + + - task: ExtractFiles@1 + inputs: + archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64.zip' + destinationFolder: '$(Build.BinariesDirectory)\java-artifact' + cleanDestinationFolder: false + + - task: ExtractFiles@1 + inputs: + archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64.zip' + destinationFolder: '$(Build.BinariesDirectory)\java-artifact' + cleanDestinationFolder: false + + - task: ExtractFiles@1 + inputs: + archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x64.zip' + destinationFolder: '$(Build.BinariesDirectory)\java-artifact' + cleanDestinationFolder: false + + - task: CmdLine@2 + inputs: + script: | + pushd onnxruntime-java-linux-x64 + jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\testing.jar libcustom_op_library.so + del /F /Q libcustom_op_library.so + jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\onnxruntime-$(OnnxRuntimeVersion).jar . + popd + pushd onnxruntime-java-osx-x64 + jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\testing.jar libcustom_op_library.dylib + del /F /Q libcustom_op_library.dylib + jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\onnxruntime-$(OnnxRuntimeVersion).jar . + popd + workingDirectory: '$(Build.BinariesDirectory)\java-artifact' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' + artifact: 'onnxruntime-java' + publishLocation: 'pipeline' + + - template: templates/component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + +- job: Final_Jar_Testing_Windows + workspace: + clean: all + pool: 'Win-CPU-2019' + timeoutInMinutes: 60 + dependsOn: + Jar_Packaging + steps: + - template: templates/set-version-number-variables-step.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: 'onnxruntime-java' + targetPath: '$(Build.BinariesDirectory)\final-jar' + + - task: CmdLine@2 + inputs: + script: | + mkdir test + pushd test + jar xf $(Build.BinariesDirectory)\final-jar\testing.jar + popd + powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar" + powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar" + java -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner + workingDirectory: '$(Build.BinariesDirectory)\final-jar' + +- job: Final_Jar_Testing_Linux + workspace: + clean: all + pool: 'Linux-CPU' + timeoutInMinutes: 60 + dependsOn: + Jar_Packaging + steps: + - template: templates/set-version-number-variables-step.yml + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: 'onnxruntime-java' + targetPath: '$(Build.BinariesDirectory)/final-jar' + + - task: CmdLine@2 + inputs: + script: | + sudo apt-get install -y openjdk-8-jdk + sudo apt autoremove + PATH=/usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin:${PATH} + echo "Java Version" + java --version + mkdir test + pushd test + jar xf $(Build.BinariesDirectory)/final-jar/testing.jar + popd + wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ + wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ + LD_LIBRARY_PATH=./test:${LD_LIBRARY_PATH} + java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner + workingDirectory: '$(Build.BinariesDirectory)/final-jar' + +- job: Final_Jar_Testing_MacOs + workspace: + clean: all + pool: + vmImage: 'macOS-10.14' + timeoutInMinutes: 60 + dependsOn: + Jar_Packaging + steps: + - template: templates/set-version-number-variables-step.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: 'onnxruntime-java' + targetPath: '$(Build.BinariesDirectory)/final-jar' + + - task: CmdLine@2 + inputs: + script: | + echo "Java Version" + java --version + mkdir test + pushd test + jar xf $(Build.BinariesDirectory)/final-jar/testing.jar + popd + wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ + wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ + DYLD_LIBRARY_PATH=./test:${DYLD_LIBRARY_PATH} + java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner + workingDirectory: '$(Build.BinariesDirectory)/final-jar' + diff --git a/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml b/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml new file mode 100644 index 0000000000..ed020758fa --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-posix.yml @@ -0,0 +1,29 @@ +# sets up common build tools for the windows build machines before build + +parameters: + arch: 'linux-x64' + buildConfig: 'RelWithDebInfo' + artifactName: 'onnxruntime-java-linux-x64' + libraryName: 'libonnxruntime.so' + nativeLibraryName: 'libonnxruntime4j_jni.so' + version: '' +steps: + - task: ShellScript@2 + displayName: 'Copy build artifacts for zipping' + inputs: + scriptPath: 'tools/ci_build/github/linux/java_copy_strip_binary.sh' + args: '-r $(Build.BinariesDirectory) -c ${{parameters.buildConfig}} -a ${{parameters.artifactName}} -l ${{parameters.libraryName}} -n ${{parameters.nativeLibraryName}} -v ${{parameters.version}} -h ${{parameters.arch}}' + workingDirectory: '$(Build.BinariesDirectory)/${{parameters.buildConfig}}' + + - task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(Build.BinariesDirectory)/${{parameters.artifactName}}' + includeRootFolder: true + archiveType: 'zip' # Options: zip, 7z, tar, wim + archiveFile: '$(Build.ArtifactStagingDirectory)/${{parameters.artifactName}}.zip' + replaceExistingArchive: true + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)/${{parameters.artifactName}}.zip' + artifactName: 'drop-${{parameters.artifactName}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-windows.yml b/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-windows.yml new file mode 100644 index 0000000000..2911722528 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/java-api-artifacts-package-and-publish-steps-windows.yml @@ -0,0 +1,56 @@ +# sets up common build tools for the windows build machines before build + +parameters: + buildConfig: 'RelWithDebInfo' + artifactName: 'onnxruntime-java-win-x64' + version: '' + comitId: '' +steps: + - task: CmdLine@2 + inputs: + script: | + echo "Java version" + java --version + set NATIVE_FOLDER=$(Build.BinariesDirectory)\${{parameters.artifactName}}\stage\ai\onnxruntime\native\win-x64 + mkdir %NATIVE_FOLDER% + echo "Directories created" + copy .\java\build\libs\*.jar $(Build.BinariesDirectory)\${{parameters.artifactName}} + pushd $(Build.BinariesDirectory)\${{parameters.artifactName}} + jar xf onnxruntime-${{parameters.version}}.jar META-INF\maven\ai.onnxruntime\onnxruntime\pom.xml + move META-INF\maven\ai.onnxruntime\onnxruntime\pom.xml onnxruntime-${{parameters.version}}.pom + rd /s /q META-INF + popd + copy .\${{parameters.buildConfig}}\onnxruntime.pdb %NATIVE_FOLDER% + copy .\${{parameters.buildConfig}}\onnxruntime4j_jni.pdb %NATIVE_FOLDER% + copy $(Build.SourcesDirectory)\docs\Privacy.md $(Build.BinariesDirectory)\${{parameters.artifactName}}\stage\Privacy.md + copy $(Build.SourcesDirectory)\ThirdPartyNotices.txt $(Build.BinariesDirectory)\${{parameters.artifactName}}\stage\ThirdPartyNotices.txt + @echo ${{parameters.commitId}} > $(Build.BinariesDirectory)\${{parameters.artifactName}}\stage\GIT_COMMIT_ID + pushd $(Build.BinariesDirectory)\${{parameters.artifactName}}\stage + jar uf $(Build.BinariesDirectory)\${{parameters.artifactName}}\onnxruntime-${{parameters.version}}.jar ai\onnxruntime\native\win-x64\onnxruntime.pdb + jar uf $(Build.BinariesDirectory)\${{parameters.artifactName}}\onnxruntime-${{parameters.version}}.jar ai\onnxruntime\native\win-x64\onnxruntime4j_jni.pdb + jar uf $(Build.BinariesDirectory)\${{parameters.artifactName}}\onnxruntime-${{parameters.version}}.jar Privacy.md ThirdPartyNotices.txt GIT_COMMIT_ID + popd + pushd $(Build.SourcesDirectory)\java\build\classes\java\test + jar cvf $(Build.BinariesDirectory)\${{parameters.artifactName}}\testing.jar . + popd + pushd $(Build.SourcesDirectory)\java\build\resources\test + rd /s /q ai\onnxruntime\native + jar uvf $(Build.BinariesDirectory)\${{parameters.artifactName}}\testing.jar . + popd + rd /s /q $(Build.BinariesDirectory)\${{parameters.artifactName}}\stage + dir /s /b $(Build.BinariesDirectory)\${{parameters.artifactName}} + workingDirectory: '$(Build.BinariesDirectory)\${{parameters.buildConfig}}' + displayName: 'Add symbols and notices' + + - task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(Build.BinariesDirectory)\${{parameters.artifactName}}' + includeRootFolder: true + archiveType: 'zip' # Options: zip, 7z, tar, wim + archiveFile: '$(Build.ArtifactStagingDirectory)\${{parameters.artifactName}}.zip' + replaceExistingArchive: true + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)\${{parameters.artifactName}}.zip' + artifactName: 'drop-${{parameters.artifactName}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/java-test-final-jar-step.yml b/tools/ci_build/github/azure-pipelines/templates/java-test-final-jar-step.yml new file mode 100644 index 0000000000..17be34a1c6 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/java-test-final-jar-step.yml @@ -0,0 +1,80 @@ +parameters: + WindowsPool: 'Win-CPU-2019' + LinuxPool: 'Linux-CPU' + version: '' + RunMacOs: 'false' + artifactName: '' + +- job: Test_Final_Jar_Linux + timeoutInMinutes: 60 + pool: '${{parameters.LinuxPool}}' + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: '${{parameters.artifactName}}' + targetPath: '$(Build.BinariesDirectory)/final-jar' + + - task: CmdLine@2 + inputs: + script: | + sudo apt-get install -y openjdk-8-jdk + sudo apt autoremove + PATH=/usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin:${PATH} + mkdir test + pushd test + jar xf $(Build.BinariesDirectory)/final-jar/testing.jar + popd + wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ + wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ + java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .;./test;./protobuf-java-3.9.2.jar;./onnxruntime-${{parameters.version}}.jar --scan-class-path --fail-if-no-tests --disable-banner + workingDirectory: '$(Build.BinariesDirectory)/final-jar' + +- job: Test_Final_Jar_MacOs + timeoutInMinutes: 60 + pool: + vmImage: 'macOS-10.14' + condition: eq(parameters['RunMacOs'], 'true') + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: '${{parameters.artifactName}}' + targetPath: '$(Build.BinariesDirectory)/final-jar' + + - task: CmdLine@2 + inputs: + script: | + mkdir test + pushd test + jar xf $(Build.BinariesDirectory)/final-jar/testing.jar + popd + wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ + wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ + java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .;./test;./protobuf-java-3.9.2.jar;./onnxruntime-${{parameters.version}}.jar --scan-class-path --fail-if-no-tests --disable-banner + workingDirectory: '$(Build.BinariesDirectory)/final-jar' + +- job: Test_Final_Jar_Windows + timeoutInMinutes: 60 + pool: '${{parameters.WindowsPool}}' + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Jar' + inputs: + buildType: 'current' + artifactName: '${{parameters.artifactName}}' + targetPath: '$(Build.BinariesDirectory)\final-jar' + + - task: CmdLine@2 + inputs: + script: | + mkdir test + pushd test + jar xf $(Build.BinariesDirectory)\final-jar\testing.jar + popd + powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar" + powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar" + java -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime-${{parameters.version}}.jar --scan-class-path --fail-if-no-tests --disable-banner + workingDirectory: '$(Build.BinariesDirectory)\final-jar' diff --git a/tools/ci_build/github/linux/docker/Dockerfile.centos6 b/tools/ci_build/github/linux/docker/Dockerfile.centos6 index 3a13e5e9f3..a52cf31928 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.centos6 +++ b/tools/ci_build/github/linux/docker/Dockerfile.centos6 @@ -2,8 +2,10 @@ FROM mcr.microsoft.com/dotnet-buildtools/prereqs:centos-6-50f0d02-20190918213956 ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh -ENV PATH /opt/rh/devtoolset-2/root/usr/bin:${PATH} +ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk.x86_64 +ENV PATH /opt/rh/devtoolset-2/root/usr/bin:${JAVA_HOME}/bin:${PATH} RUN /tmp/scripts/install_deps.sh -p 3.6 && rm -rf /tmp/scripts +ENV PATH ${PATH}:/usr/local/gradle/bin ARG BUILD_UID=1000 ARG BUILD_USER=onnxruntimedev diff --git a/tools/ci_build/github/linux/docker/Dockerfile.centos6_gpu b/tools/ci_build/github/linux/docker/Dockerfile.centos6_gpu index c5005b2f22..adcbaaf747 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.centos6_gpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.centos6_gpu @@ -5,8 +5,10 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh -ENV PATH /opt/rh/devtoolset-2/root/usr/bin:${PATH} +ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk.x86_64 +ENV PATH /opt/rh/devtoolset-2/root/usr/bin:${JAVA_HOME}/bin:${PATH} RUN /tmp/scripts/install_deps.sh -p 3.6 && rm -rf /tmp/scripts +ENV PATH ${PATH}:/usr/local/gradle/bin #Below are copied from https://gitlab.com/nvidia/container-images/cuda/tree/master/dist/centos6 diff --git a/tools/ci_build/github/linux/docker/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/scripts/install_centos.sh index 34858b79e8..f7c33e294c 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_centos.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_centos.sh @@ -31,6 +31,7 @@ else fi fi +yum install -y java-1.8.0-openjdk-devel #If the /opt/python folder exists, we assume this is the manylinux docker image if [ "$os_major_version" != "6" ] && [ ! -d "/opt/python/cp35-cp35m" ] diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh index 0133804556..4122f5b064 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_deps.sh @@ -97,10 +97,10 @@ else popd fi -GetFile https://downloads.gradle-dn.com/distributions/gradle-6.2-bin.zip /tmp/src/gradle-6.2-bin.zip +GetFile https://downloads.gradle-dn.com/distributions/gradle-6.3-bin.zip /tmp/src/gradle-6.3-bin.zip cd /tmp/src -unzip gradle-6.2-bin.zip -mv /tmp/src/gradle-6.2 /usr/local/gradle +unzip gradle-6.3-bin.zip +mv /tmp/src/gradle-6.3 /usr/local/gradle if ! [ -x "$(command -v protoc)" ]; then source ${0/%install_deps\.sh/install_protobuf\.sh} diff --git a/tools/ci_build/github/linux/java_copy_strip_binary.sh b/tools/ci_build/github/linux/java_copy_strip_binary.sh new file mode 100755 index 0000000000..568a11bf96 --- /dev/null +++ b/tools/ci_build/github/linux/java_copy_strip_binary.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -e -o -x + +while getopts r:a:l:n:c:h:v: parameter_Option +do case "${parameter_Option}" +in +r) BINARY_DIR=${OPTARG};; +a) ARTIFACT_NAME=${OPTARG};; +c) BUILD_CONFIG=${OPTARG};; +l) LIB_NAME=${OPTARG};; +n) NATIVE_LIB_NAME=${OPTARG};; +h) ARCH=${OPTARG};; +v) VERSION_NUMBER=${OPTARG};; +esac +done + +EXIT_CODE=1 + +uname -a + +echo "Version: $VERSION_NUMBER" +NATIVE_FOLDER=ai/onnxruntime/native/$ARCH + +mkdir -p $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER + +echo "Directories created" + +echo "Copy debug symbols in a separate file and strip the original binary." + +if [[ $LIB_NAME == *.dylib ]] +then + # ORT LIB + dsymutil $BINARY_DIR/$BUILD_CONFIG/$LIB_NAME -o $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/$LIB_NAME.dSYM + cp $BINARY_DIR/$BUILD_CONFIG/$LIB_NAME $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/libonnxruntime.dylib + strip -S $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/libonnxruntime.dylib + # JNI Lib + dsymutil $BINARY_DIR/$BUILD_CONFIG/$NATIVE_LIB_NAME -o $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/$NATIVE_LIB_NAME.dSYM + cp $BINARY_DIR/$BUILD_CONFIG/$NATIVE_LIB_NAME $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/libonnxruntime4j_jni.dylib + strip -S $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/libonnxruntime4j_jni.dylib + # Add custom lib for testing. This should be added to testing.jar + cp $BINARY_DIR/$BUILD_CONFIG/libcustom_op_library.dylib $BINARY_DIR/$ARTIFACT_NAME +elif [[ $LIB_NAME == *.so ]] +then + cp $BINARY_DIR/$BUILD_CONFIG/$LIB_NAME $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/libonnxruntime.so + cp $BINARY_DIR/$BUILD_CONFIG/$NATIVE_LIB_NAME $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER/libonnxruntime4j_jni.so + # Add custom lib + cp $BINARY_DIR/$BUILD_CONFIG/libcustom_op_library.so $BINARY_DIR/$ARTIFACT_NAME +fi + +find $BINARY_DIR/$ARTIFACT_NAME -ls +rm -fr $BINARY_DIR/$ARTIFACT_NAME/jar + +EXIT_CODE=$? + +set -e +exit $EXIT_CODE diff --git a/tools/ci_build/github/linux/java_linux_final_test.sh b/tools/ci_build/github/linux/java_linux_final_test.sh new file mode 100755 index 0000000000..c51394c1d3 --- /dev/null +++ b/tools/ci_build/github/linux/java_linux_final_test.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# This is for testing GPU final jar on Linux +set -e -o -x + +while getopts r:v: parameter_Option +do case "${parameter_Option}" +in +r) BINARY_DIR=${OPTARG};; +v) VERSION_NUMBER=${OPTARG};; +esac +done + +EXIT_CODE=1 + +uname -a + +cd $BINARY_DIR/final-jar + +mkdir test + +echo "Directories created" +echo "Library path:" $LD_LIBRARY_PATH + +pushd test +jar xf $BINARY_DIR/final-jar/testing.jar +popd + +wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ +wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ +java -DUSE_CUDA=1 -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-gpu-${VERSION_NUMBER}.jar --scan-class-path --fail-if-no-tests --disable-banner + +EXIT_CODE=$? + +set -e +exit $EXIT_CODE