mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-19 21:32:23 +00:00
afca0d15ee
Create CPU and GPu Java publishing pipelines. Final jars are tested on all platforms. However, signing and publishing to maven are manual steps.
243 lines
10 KiB
YAML
243 lines
10 KiB
YAML
jobs:
|
|
- job: Linux_Java_API_Build_GPU_x64
|
|
workspace:
|
|
clean: all
|
|
timeoutInMinutes: 60
|
|
pool: 'Linux-GPU-CUDA10'
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
sudo docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu .
|
|
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
sudo docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
- template: templates/java-api-artifacts-package-and-publish-steps-posix.yml
|
|
parameters:
|
|
arch: 'linux-x64'
|
|
buildConfig: 'Release'
|
|
artifactName: 'onnxruntime-java-linux-x64-gpu'
|
|
version: '$(OnnxRuntimeVersion)'
|
|
libraryName: 'libonnxruntime.so'
|
|
nativeLibraryName: 'libonnxruntime4j_jni.so'
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|
|
|
|
- job: Windows_Java_API_Build_GPU_x64
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-GPU-2019'
|
|
timeoutInMinutes: 120
|
|
variables:
|
|
EnvSetupScript: setup_env_cuda.bat
|
|
buildArch: x64
|
|
msbuildArch: amd64
|
|
msbuildPlatform: x64
|
|
buildparameter: --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda"
|
|
steps:
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: '3.7'
|
|
addToPath: true
|
|
architecture: $(buildArch)
|
|
|
|
- task: BatchScript@1
|
|
displayName: 'setup env'
|
|
inputs:
|
|
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
|
|
modifyEnvironment: true
|
|
workingFolder: '$(Build.BinariesDirectory)'
|
|
|
|
- script: |
|
|
python -m pip install -q pyopenssl setuptools wheel numpy scipy
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
displayName: 'Install python modules'
|
|
|
|
- powershell: |
|
|
$Env:USE_MSVC_STATIC_RUNTIME=1
|
|
$Env:ONNX_ML=1
|
|
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
|
|
python setup.py bdist_wheel
|
|
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
|
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
|
|
displayName: 'Install ONNX'
|
|
|
|
- template: templates/set-test-data-variables-step.yml
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Generate cmake config'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_lto --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
|
|
- task: VSBuild@1
|
|
displayName: 'Build'
|
|
inputs:
|
|
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
|
|
platform: $(msbuildPlatform)
|
|
configuration: RelWithDebInfo
|
|
msbuildArchitecture: $(buildArch)
|
|
maximumCpuCount: true
|
|
logProjectEvents: true
|
|
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
|
|
createLogFile: true
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'test'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
- template: templates/java-api-artifacts-package-and-publish-steps-windows.yml
|
|
parameters:
|
|
buildConfig: RelWithDebInfo
|
|
artifactName: 'onnxruntime-java-win-$(buildArch)-gpu'
|
|
version: '$(OnnxRuntimeVersion)'
|
|
commitId: $(OnnxRuntimeGitCommitHash)
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|
|
|
|
- job: Jar_Packaging
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-CPU-2019'
|
|
dependsOn:
|
|
- Windows_Java_API_Build_GPU_x64
|
|
- Linux_Java_API_Build_GPU_x64
|
|
condition: succeeded()
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Pipeline Artifact - Win x64'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'drop-onnxruntime-java-win-x64-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Pipeline Artifact - Linux x64'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'drop-onnxruntime-java-linux-x64-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact'
|
|
|
|
- task: ExtractFiles@1
|
|
inputs:
|
|
archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu.zip'
|
|
destinationFolder: '$(Build.BinariesDirectory)\java-artifact'
|
|
cleanDestinationFolder: false
|
|
|
|
- task: ExtractFiles@1
|
|
inputs:
|
|
archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64-gpu.zip'
|
|
destinationFolder: '$(Build.BinariesDirectory)\java-artifact'
|
|
cleanDestinationFolder: false
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
pushd onnxruntime-java-linux-x64-gpu
|
|
jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu\testing.jar libcustom_op_library.so
|
|
del /F /Q libcustom_op_library.so
|
|
jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu\onnxruntime-$(OnnxRuntimeVersion).jar .
|
|
popd
|
|
pushd onnxruntime-java-win-x64-gpu
|
|
ren onnxruntime-$(OnnxRuntimeVersion).jar onnxruntime-gpu-$(OnnxRuntimeVersion).jar
|
|
ren onnxruntime-$(OnnxRuntimeVersion)-javadoc.jar onnxruntime-gpu-$(OnnxRuntimeVersion)-javadoc.jar
|
|
ren onnxruntime-$(OnnxRuntimeVersion)-sources.jar onnxruntime-gpu-$(OnnxRuntimeVersion)-sources.jar
|
|
ren onnxruntime-$(OnnxRuntimeVersion).pom onnxruntime-gpu-$(OnnxRuntimeVersion).pom
|
|
popd
|
|
workingDirectory: '$(Build.BinariesDirectory)\java-artifact'
|
|
displayName: 'Create final GPU Jar'
|
|
|
|
- task: PublishPipelineArtifact@1
|
|
inputs:
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu'
|
|
artifact: 'onnxruntime-java-gpu'
|
|
publishLocation: 'pipeline'
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
|
|
- job: Final_Jar_Testing_Windows
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-GPU-2019'
|
|
timeoutInMinutes: 60
|
|
dependsOn:
|
|
Jar_Packaging
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Final Jar'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'onnxruntime-java-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)\final-jar'
|
|
|
|
- task: BatchScript@1
|
|
displayName: 'Setup CUDA Env'
|
|
inputs:
|
|
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_cuda.bat'
|
|
modifyEnvironment: true
|
|
workingFolder: '$(Build.BinariesDirectory)'
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
mkdir test
|
|
pushd test
|
|
jar xf $(Build.BinariesDirectory)\final-jar\testing.jar
|
|
popd
|
|
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar"
|
|
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar"
|
|
java -DUSE_CUDA=1 -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime-gpu-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
|
workingDirectory: '$(Build.BinariesDirectory)\final-jar'
|
|
|
|
- job: Final_Jar_Testing_Linux
|
|
workspace:
|
|
clean: all
|
|
pool: 'Linux-GPU-CUDA10'
|
|
timeoutInMinutes: 60
|
|
dependsOn:
|
|
Jar_Packaging
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Final Jar'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'onnxruntime-java-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)/final-jar'
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
sudo docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu .
|
|
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
sudo docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build
|
|
workingDirectory: $(Build.BinariesDirectory)/final-jar
|