mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-18 21:21:17 +00:00
5eec4f66ed
1. Publish the image ACR, instead of building it every time for every PR 2. Make USE_MKLML and USE_OPENMP be able to co-exist. Currently both of them are enabled in our Linux CI build but indeed only one of them is taking effect. 3. Split nuphar and DNNL to separated pipelines. 4. Fix two warnings in onnxruntime/core/optimizer/matmul_scale_fusion.cc and onnxruntime/test/tvm/tvm_basic_test.cc. 5. Update the manylinux2010_x86_64 image to the latest.
349 lines
14 KiB
YAML
349 lines
14 KiB
YAML
schedules:
|
|
- cron: "0 8 * * *"
|
|
displayName: Daily Build
|
|
branches:
|
|
include:
|
|
- master
|
|
always: true
|
|
|
|
jobs:
|
|
- job: Linux_Java_API_Build_CPU_x64
|
|
workspace:
|
|
clean: all
|
|
timeoutInMinutes: 60
|
|
pool: 'Linux-CPU'
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
- template: templates/linux-set-variables-and-download.yml
|
|
|
|
- task: Docker@2
|
|
displayName: login
|
|
inputs:
|
|
containerRegistry: onnxruntimeregistry
|
|
command: login
|
|
addPipelineData: false
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
displayName: 'Run build and test'
|
|
- task: Docker@2
|
|
displayName: logout
|
|
inputs:
|
|
containerRegistry: onnxruntimeregistry
|
|
command: logout
|
|
addPipelineData: false
|
|
- template: templates/java-api-artifacts-package-and-publish-steps-posix.yml
|
|
parameters:
|
|
arch: 'linux-x64'
|
|
buildConfig: 'Release'
|
|
artifactName: 'onnxruntime-java-linux-x64'
|
|
version: '$(OnnxRuntimeVersion)'
|
|
libraryName: 'libonnxruntime.so'
|
|
nativeLibraryName: 'libonnxruntime4j_jni.so'
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|
|
|
|
- job: MacOS_Java_API_Build_CPU_x64
|
|
workspace:
|
|
clean: all
|
|
pool:
|
|
vmImage: 'macOS-10.14'
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- script: |
|
|
export JAVA_HOME=$(java_home -v 11)
|
|
sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
|
|
sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer
|
|
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_java --build_shared_lib --use_openmp --config RelWithDebInfo
|
|
|
|
displayName: 'Build and Test MacOS'
|
|
- template: templates/java-api-artifacts-package-and-publish-steps-posix.yml
|
|
parameters:
|
|
arch: 'osx-x64'
|
|
buildConfig: 'RelWithDebInfo'
|
|
artifactName: 'onnxruntime-java-osx-x64'
|
|
version: '$(OnnxRuntimeVersion)'
|
|
libraryName: 'libonnxruntime.dylib'
|
|
nativeLibraryName: 'libonnxruntime4j_jni.dylib'
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|
|
|
|
- job: Windows_Java_API_Build_CPU_x64
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-CPU-2019'
|
|
timeoutInMinutes: 160
|
|
strategy:
|
|
maxParallel: 2
|
|
matrix:
|
|
# x86:
|
|
#EnvSetupScript: setup_env_x86.bat
|
|
#buildArch: x86
|
|
#msbuildArch: x86
|
|
#msbuildPlatform: Win32
|
|
#buildparameter: --x86
|
|
x64:
|
|
EnvSetupScript: setup_env.bat
|
|
buildArch: x64
|
|
msbuildArch: amd64
|
|
msbuildPlatform: x64
|
|
buildparameter:
|
|
|
|
steps:
|
|
- template: templates/telemetry-steps.yml
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: '3.7'
|
|
addToPath: true
|
|
architecture: $(buildArch)
|
|
|
|
- task: BatchScript@1
|
|
displayName: 'setup env'
|
|
inputs:
|
|
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
|
|
modifyEnvironment: true
|
|
workingFolder: '$(Build.BinariesDirectory)'
|
|
|
|
- script: |
|
|
python -m pip install -q pyopenssl setuptools wheel numpy
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
displayName: 'Install python modules'
|
|
|
|
- powershell: |
|
|
$Env:USE_MSVC_STATIC_RUNTIME=1
|
|
$Env:ONNX_ML=1
|
|
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
|
|
python setup.py bdist_wheel
|
|
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
|
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
|
|
displayName: 'Install ONNX'
|
|
|
|
- template: templates/set-test-data-variables-step.yml
|
|
- template: templates/set-version-number-variables-step.yml
|
|
- task: PythonScript@0
|
|
displayName: 'Generate cmake config'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_wcos --build_java --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
|
|
- task: VSBuild@1
|
|
displayName: 'Build'
|
|
inputs:
|
|
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
|
|
platform: $(msbuildPlatform)
|
|
configuration: RelWithDebInfo
|
|
msbuildArchitecture: $(buildArch)
|
|
maximumCpuCount: true
|
|
logProjectEvents: true
|
|
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
|
|
createLogFile: true
|
|
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'test'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --build_shared_lib --enable_wcos --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
- template: templates/java-api-artifacts-package-and-publish-steps-windows.yml
|
|
parameters:
|
|
buildConfig: RelWithDebInfo
|
|
artifactName: 'onnxruntime-java-win-$(buildArch)'
|
|
version: '$(OnnxRuntimeVersion)'
|
|
commitId: $(OnnxRuntimeGitCommitHash)
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- template: templates/clean-agent-build-directory-step.yml
|
|
|
|
- job: Jar_Packaging
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-CPU-2019'
|
|
dependsOn:
|
|
- Windows_Java_API_Build_CPU_x64
|
|
- Linux_Java_API_Build_CPU_x64
|
|
- MacOS_Java_API_Build_CPU_x64
|
|
condition: succeeded()
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Pipeline Artifact - Win x64'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'drop-onnxruntime-java-win-x64'
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Pipeline Artifact - Linux x64'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'drop-onnxruntime-java-linux-x64'
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact'
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Pipeline Artifact - MacOS x64'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'drop-onnxruntime-java-osx-x64'
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact'
|
|
|
|
- task: ExtractFiles@1
|
|
inputs:
|
|
archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64.zip'
|
|
destinationFolder: '$(Build.BinariesDirectory)\java-artifact'
|
|
cleanDestinationFolder: false
|
|
|
|
- task: ExtractFiles@1
|
|
inputs:
|
|
archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64.zip'
|
|
destinationFolder: '$(Build.BinariesDirectory)\java-artifact'
|
|
cleanDestinationFolder: false
|
|
|
|
- task: ExtractFiles@1
|
|
inputs:
|
|
archiveFilePatterns: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x64.zip'
|
|
destinationFolder: '$(Build.BinariesDirectory)\java-artifact'
|
|
cleanDestinationFolder: false
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
@echo on
|
|
pushd onnxruntime-java-linux-x64
|
|
jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\testing.jar libcustom_op_library.so
|
|
del /F /Q libcustom_op_library.so
|
|
jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\onnxruntime-$(OnnxRuntimeVersion).jar .
|
|
popd
|
|
pushd onnxruntime-java-osx-x64
|
|
jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\testing.jar libcustom_op_library.dylib
|
|
del /F /Q libcustom_op_library.dylib
|
|
jar uf $(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64\onnxruntime-$(OnnxRuntimeVersion).jar .
|
|
popd
|
|
workingDirectory: '$(Build.BinariesDirectory)\java-artifact'
|
|
|
|
- task: PublishPipelineArtifact@1
|
|
inputs:
|
|
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64'
|
|
artifact: 'onnxruntime-java'
|
|
publishLocation: 'pipeline'
|
|
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
|
|
- job: Final_Jar_Testing_Windows
|
|
workspace:
|
|
clean: all
|
|
pool: 'Win-CPU-2019'
|
|
timeoutInMinutes: 60
|
|
dependsOn:
|
|
Jar_Packaging
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Final Jar'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'onnxruntime-java'
|
|
targetPath: '$(Build.BinariesDirectory)\final-jar'
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
mkdir test
|
|
pushd test
|
|
jar xf $(Build.BinariesDirectory)\final-jar\testing.jar
|
|
popd
|
|
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar"
|
|
powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar"
|
|
java -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
|
workingDirectory: '$(Build.BinariesDirectory)\final-jar'
|
|
|
|
- job: Final_Jar_Testing_Linux
|
|
workspace:
|
|
clean: all
|
|
pool: 'Linux-CPU'
|
|
timeoutInMinutes: 60
|
|
dependsOn:
|
|
Jar_Packaging
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Final Jar'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'onnxruntime-java'
|
|
targetPath: '$(Build.BinariesDirectory)/final-jar'
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
sudo apt-get install -y openjdk-8-jdk
|
|
sudo apt autoremove
|
|
PATH=/usr/lib/jvm/jre-1.8.0-openjdk.x86_64/bin:${PATH}
|
|
echo "Java Version"
|
|
java --version
|
|
mkdir test
|
|
pushd test
|
|
jar xf $(Build.BinariesDirectory)/final-jar/testing.jar
|
|
popd
|
|
wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./
|
|
wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./
|
|
LD_LIBRARY_PATH=./test:${LD_LIBRARY_PATH}
|
|
java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
|
workingDirectory: '$(Build.BinariesDirectory)/final-jar'
|
|
|
|
- job: Final_Jar_Testing_MacOs
|
|
workspace:
|
|
clean: all
|
|
pool:
|
|
vmImage: 'macOS-10.14'
|
|
timeoutInMinutes: 60
|
|
dependsOn:
|
|
Jar_Packaging
|
|
steps:
|
|
- template: templates/set-version-number-variables-step.yml
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
displayName: 'Download Final Jar'
|
|
inputs:
|
|
buildType: 'current'
|
|
artifactName: 'onnxruntime-java'
|
|
targetPath: '$(Build.BinariesDirectory)/final-jar'
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
echo "Java Version"
|
|
java --version
|
|
mkdir test
|
|
pushd test
|
|
jar xf $(Build.BinariesDirectory)/final-jar/testing.jar
|
|
popd
|
|
wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./
|
|
wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./
|
|
DYLD_LIBRARY_PATH=./test:${DYLD_LIBRARY_PATH}
|
|
java -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner
|
|
workingDirectory: '$(Build.BinariesDirectory)/final-jar'
|
|
|