mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-14 20:48:00 +00:00
269 lines
11 KiB
YAML
269 lines
11 KiB
YAML
schedules:
|
|
- cron: "0 8 * * *"
|
|
displayName: Daily Build
|
|
branches:
|
|
include:
|
|
- master
|
|
always: true
|
|
|
|
jobs:
|
|
- job: Linux_Java_API_Build_GPU_x64
|
|
workspace:
|
|
clean: all
|
|
timeoutInMinutes: 120
|
|
pool: 'Linux-GPU-CUDA10'
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
- task: Docker@2
|
|
displayName: login
|
|
inputs:
|
|
containerRegistry: onnxruntimeregistry
|
|
command: login
|
|
addPipelineData: false
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch35 /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
- task: Docker@2
|
|
displayName: logout
|
|
inputs:
|
|
containerRegistry: onnxruntimeregistry
|
|
command: logout
|
|
- template: templates/java-api-artifacts-package-and-publish-steps-posix.yml
|
|
parameters:
|
|
arch: 'linux-x64'
|
|
buildConfig: 'Release'
|
|
artifactName: 'onnxruntime-java-linux-x64-gpu'
|
|
version: '$(OnnxRuntimeVersion)'
|
|
libraryName: 'libonnxruntime.so'
|
|
nativeLibraryName: 'libonnxruntime4j_jni.so'
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|
|
|
|
- job: Windows_Java_API_Build_GPU_x64
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-GPU-2019'
|
|
timeoutInMinutes: 120
|
|
variables:
|
|
EnvSetupScript: setup_env_cuda.bat
|
|
buildArch: x64
|
|
msbuildArch: amd64
|
|
msbuildPlatform: x64
|
|
buildparameter: --use_cuda --cuda_version=10.2 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2" --cudnn_home="C:\local\cudnn-10.2-windows10-x64-v8.0.3.33\cuda"
|
|
steps:
|
|
- template: templates/telemetry-steps.yml
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: '3.7'
|
|
addToPath: true
|
|
architecture: $(buildArch)
|
|
|
|
- task: BatchScript@1
|
|
displayName: 'setup env'
|
|
inputs:
|
|
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
|
|
modifyEnvironment: true
|
|
workingFolder: '$(Build.BinariesDirectory)'
|
|
|
|
- script: |
|
|
python -m pip install -q pyopenssl setuptools wheel numpy
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
displayName: 'Install python modules'
|
|
|
|
- powershell: |
|
|
$Env:USE_MSVC_STATIC_RUNTIME=1
|
|
$Env:ONNX_ML=1
|
|
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
|
|
python setup.py bdist_wheel
|
|
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
|
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
|
|
displayName: 'Install ONNX'
|
|
|
|
- template: templates/set-test-data-variables-step.yml
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Generate cmake config'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --use_telemetry --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --build_shared_lib --enable_onnx_tests $(buildparameter)'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
|
|
- task: VSBuild@1
|
|
displayName: 'Build'
|
|
inputs:
|
|
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
|
|
platform: $(msbuildPlatform)
|
|
configuration: RelWithDebInfo
|
|
msbuildArchitecture: $(buildArch)
|
|
maximumCpuCount: true
|
|
logProjectEvents: true
|
|
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
|
|
createLogFile: true
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'test'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --build_shared_lib --enable_onnx_tests $(buildparameter)'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
- template: templates/java-api-artifacts-package-and-publish-steps-windows.yml
|
|
parameters:
|
|
buildConfig: RelWithDebInfo
|
|
artifactName: 'onnxruntime-java-win-$(buildArch)-gpu'
|
|
version: '$(OnnxRuntimeVersion)'
|
|
commitId: $(OnnxRuntimeGitCommitHash)
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|
|
|
|
- job: Jar_Packaging
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-CPU-2019'
|
|
dependsOn:
|
|
- Windows_Java_API_Build_GPU_x64
|
|
- Linux_Java_API_Build_GPU_x64
|
|
condition: succeeded()
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Pipeline Artifact - Win x64'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'drop-onnxruntime-java-win-x64-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Pipeline Artifact - Linux x64'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'drop-onnxruntime-java-linux-x64-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact'
|
|
|
|
- task: ExtractFiles@1
|
|
inputs:
|
|
archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu.zip'
|
|
destinationFolder: '$(Build.BinariesDirectory)\java-artifact'
|
|
cleanDestinationFolder: false
|
|
|
|
- task: ExtractFiles@1
|
|
inputs:
|
|
archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64-gpu.zip'
|
|
destinationFolder: '$(Build.BinariesDirectory)\java-artifact'
|
|
cleanDestinationFolder: false
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
@echo on
|
|
pushd onnxruntime-java-linux-x64-gpu
|
|
jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu\testing.jar libcustom_op_library.so
|
|
del /F /Q libcustom_op_library.so
|
|
jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu\onnxruntime-$(OnnxRuntimeVersion).jar .
|
|
popd
|
|
pushd onnxruntime-java-win-x64-gpu
|
|
ren onnxruntime-$(OnnxRuntimeVersion).jar onnxruntime_gpu-$(OnnxRuntimeVersion).jar
|
|
ren onnxruntime-$(OnnxRuntimeVersion)-javadoc.jar onnxruntime_gpu-$(OnnxRuntimeVersion)-javadoc.jar
|
|
ren onnxruntime-$(OnnxRuntimeVersion)-sources.jar onnxruntime_gpu-$(OnnxRuntimeVersion)-sources.jar
|
|
ren onnxruntime-$(OnnxRuntimeVersion).pom onnxruntime_gpu-$(OnnxRuntimeVersion).pom
|
|
popd
|
|
workingDirectory: '$(Build.BinariesDirectory)\java-artifact'
|
|
displayName: 'Create final GPU Jar'
|
|
|
|
- task: PublishPipelineArtifact@1
|
|
inputs:
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64-gpu'
|
|
artifact: 'onnxruntime-java-gpu'
|
|
publishLocation: 'pipeline'
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
|
|
- job: Final_Jar_Testing_Windows
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-GPU-2019'
|
|
timeoutInMinutes: 60
|
|
dependsOn:
|
|
Jar_Packaging
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Final Jar'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'onnxruntime-java-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)\final-jar'
|
|
|
|
- task: BatchScript@1
|
|
displayName: 'Setup CUDA Env'
|
|
inputs:
|
|
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_cuda.bat'
|
|
modifyEnvironment: true
|
|
workingFolder: '$(Build.BinariesDirectory)'
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
mkdir test
|
|
pushd test
|
|
jar xf $(Build.BinariesDirectory)\final-jar\testing.jar
|
|
popd
|
|
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar"
|
|
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar"
|
|
java -DUSE_CUDA=1 -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime_gpu-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
|
workingDirectory: '$(Build.BinariesDirectory)\final-jar'
|
|
|
|
- job: Final_Jar_Testing_Linux
|
|
workspace:
|
|
clean: all
|
|
pool: 'Linux-GPU-CUDA10'
|
|
timeoutInMinutes: 60
|
|
dependsOn:
|
|
Jar_Packaging
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Final Jar'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'onnxruntime-java-gpu'
|
|
targetPath: '$(Build.BinariesDirectory)/final-jar'
|
|
|
|
- task: Docker@2
|
|
displayName: login
|
|
inputs:
|
|
containerRegistry: onnxruntimeregistry
|
|
command: login
|
|
addPipelineData: false
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch35 /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build
|
|
workingDirectory: $(Build.BinariesDirectory)/final-jar
|
|
|
|
- task: Docker@2
|
|
displayName: logout
|
|
inputs:
|
|
containerRegistry: onnxruntimeregistry
|
|
command: logout
|
|
addPipelineData: false
|