mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-14 20:48:00 +00:00
af0750ba1b
Modify gradle build so artifactID has _gpu for GPU builds. Pass USE_CUDA flag on CUDA build Adjust publishing pipelines to extract POM from a correct path. Co-Authored-By: @Craigacp
244 lines
10 KiB
YAML
244 lines
10 KiB
YAML
jobs:
|
|
- job: Linux_Java_API_Build_GPU_x64
|
|
workspace:
|
|
clean: all
|
|
timeoutInMinutes: 60
|
|
pool: 'Linux-GPU-CUDA10'
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
sudo docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu .
|
|
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
sudo docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
- template: templates/java-api-artifacts-package-and-publish-steps-posix.yml
|
|
parameters:
|
|
arch: 'linux-x64'
|
|
buildConfig: 'Release'
|
|
artifactName: 'onnxruntime-java-linux-x64-gpu'
|
|
version: '$(OnnxRuntimeVersion)'
|
|
libraryName: 'libonnxruntime.so'
|
|
nativeLibraryName: 'libonnxruntime4j_jni.so'
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|
|
|
|
- job: Windows_Java_API_Build_GPU_x64
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-GPU-2019'
|
|
timeoutInMinutes: 120
|
|
variables:
|
|
EnvSetupScript: setup_env_cuda.bat
|
|
buildArch: x64
|
|
msbuildArch: amd64
|
|
msbuildPlatform: x64
|
|
buildparameter: --use_cuda --cuda_version=10.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" --cudnn_home="C:\local\cudnn-10.1-windows10-x64-v7.6.5.32\cuda"
|
|
steps:
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: '3.7'
|
|
addToPath: true
|
|
architecture: $(buildArch)
|
|
|
|
- task: BatchScript@1
|
|
displayName: 'setup env'
|
|
inputs:
|
|
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
|
|
modifyEnvironment: true
|
|
workingFolder: '$(Build.BinariesDirectory)'
|
|
|
|
- script: |
|
|
python -m pip install -q pyopenssl setuptools wheel numpy scipy
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
displayName: 'Install python modules'
|
|
|
|
- powershell: |
|
|
$Env:USE_MSVC_STATIC_RUNTIME=1
|
|
$Env:ONNX_ML=1
|
|
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
|
|
python setup.py bdist_wheel
|
|
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
|
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
|
|
displayName: 'Install ONNX'
|
|
|
|
- template: templates/set-test-data-variables-step.yml
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Generate cmake config'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_lto --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
|
|
- task: VSBuild@1
|
|
displayName: 'Build'
|
|
inputs:
|
|
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
|
|
platform: $(msbuildPlatform)
|
|
configuration: RelWithDebInfo
|
|
msbuildArchitecture: $(buildArch)
|
|
maximumCpuCount: true
|
|
logProjectEvents: true
|
|
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
|
|
createLogFile: true
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'test'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
- template: templates/java-api-artifacts-package-and-publish-steps-windows.yml
|
|
parameters:
|
|
buildConfig: RelWithDebInfo
|
|
artifactName: 'onnxruntime-java-win-$(buildArch)-gpu'
|
|
version: '$(OnnxRuntimeVersion)'
|
|
commitId: $(OnnxRuntimeGitCommitHash)
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|
|
|
|
- job: Jar_Packaging
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-CPU-2019'
|
|
dependsOn:
|
|
- Windows_Java_API_Build_GPU_x64
|
|
- Linux_Java_API_Build_GPU_x64
|
|
condition: succeeded()
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Pipeline Artifact - Win x64'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'drop-onnxruntime-java-win-x64-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Pipeline Artifact - Linux x64'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'drop-onnxruntime-java-linux-x64-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact'
|
|
|
|
- task: ExtractFiles@1
|
|
inputs:
|
|
archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu.zip'
|
|
destinationFolder: '$(Build.BinariesDirectory)\java-artifact'
|
|
cleanDestinationFolder: false
|
|
|
|
- task: ExtractFiles@1
|
|
inputs:
|
|
archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64-gpu.zip'
|
|
destinationFolder: '$(Build.BinariesDirectory)\java-artifact'
|
|
cleanDestinationFolder: false
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
@echo on
|
|
pushd onnxruntime-java-linux-x64-gpu
|
|
jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu\testing.jar libcustom_op_library.so
|
|
del /F /Q libcustom_op_library.so
|
|
jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu\onnxruntime-$(OnnxRuntimeVersion).jar .
|
|
popd
|
|
pushd onnxruntime-java-win-x64-gpu
|
|
ren onnxruntime-$(OnnxRuntimeVersion).jar onnxruntime_gpu-$(OnnxRuntimeVersion).jar
|
|
ren onnxruntime-$(OnnxRuntimeVersion)-javadoc.jar onnxruntime_gpu-$(OnnxRuntimeVersion)-javadoc.jar
|
|
ren onnxruntime-$(OnnxRuntimeVersion)-sources.jar onnxruntime_gpu-$(OnnxRuntimeVersion)-sources.jar
|
|
ren onnxruntime-$(OnnxRuntimeVersion).pom onnxruntime_gpu_$(OnnxRuntimeVersion).pom
|
|
popd
|
|
workingDirectory: '$(Build.BinariesDirectory)\java-artifact'
|
|
displayName: 'Create final GPU Jar'
|
|
|
|
- task: PublishPipelineArtifact@1
|
|
inputs:
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu'
|
|
artifact: 'onnxruntime-java-gpu'
|
|
publishLocation: 'pipeline'
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
|
|
- job: Final_Jar_Testing_Windows
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-GPU-2019'
|
|
timeoutInMinutes: 60
|
|
dependsOn:
|
|
Jar_Packaging
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Final Jar'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'onnxruntime-java-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)\final-jar'
|
|
|
|
- task: BatchScript@1
|
|
displayName: 'Setup CUDA Env'
|
|
inputs:
|
|
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_cuda.bat'
|
|
modifyEnvironment: true
|
|
workingFolder: '$(Build.BinariesDirectory)'
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
mkdir test
|
|
pushd test
|
|
jar xf $(Build.BinariesDirectory)\final-jar\testing.jar
|
|
popd
|
|
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar"
|
|
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar"
|
|
java -DUSE_CUDA=1 -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime_gpu-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
|
workingDirectory: '$(Build.BinariesDirectory)\final-jar'
|
|
|
|
- job: Final_Jar_Testing_Linux
|
|
workspace:
|
|
clean: all
|
|
pool: 'Linux-GPU-CUDA10'
|
|
timeoutInMinutes: 60
|
|
dependsOn:
|
|
Jar_Packaging
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Final Jar'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'onnxruntime-java-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)/final-jar'
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
sudo docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu .
|
|
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
sudo docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build
|
|
workingDirectory: $(Build.BinariesDirectory)/final-jar
|