mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-06-02 23:39:58 +00:00
0d1da41ca8
### Description Improve docker commands to make docker image layer caching works. It can make docker building faster and more stable. So far, A100 pool's system disk is too small to use docker cache. We won't use pipeline cache for docker image and remove some legacy code. ### Motivation and Context There are often an exception of ``` 64.58 + curl https://nodejs.org/dist/v18.17.1/node-v18.17.1-linux-x64.tar.gz -sSL --retry 5 --retry-delay 30 --create-dirs -o /tmp/src/node-v18.17.1-linux-x64.tar.gz --fail 286.4 curl: (92) HTTP/2 stream 0 was not closed cleanly: INTERNAL_ERROR (err 2) ``` Because Onnxruntime pipeline have been sending too many requests to download Nodejs in docker building. Which is the major reason of pipeline failing now In fact, docker image layer caching never works. We can always see the scrips are still running ``` #9 [3/5] RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts #9 0.234 /bin/sh: warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8) #9 0.235 /bin/sh: warning: setlocale: LC_ALL: cannot change locale (en_US.UTF-8) #9 0.235 /tmp/scripts/install_centos.sh: line 1: !/bin/bash: No such file or directory #9 0.235 ++ '[' '!' -f /etc/yum.repos.d/microsoft-prod.repo ']' #9 0.236 +++ tr -dc 0-9. #9 0.236 +++ cut -d . -f1 #9 0.238 ++ os_major_version=8 .... #9 60.41 + curl https://nodejs.org/dist/v18.17.1/node-v18.17.1-linux-x64.tar.gz -sSL --retry 5 --retry-delay 30 --create-dirs -o /tmp/src/node-v18.17.1-linux-x64.tar.gz --fail #9 60.59 + return 0 ... ``` This PR is improving the docker command to make image layer caching work. Thus, CI won't send so many redundant request of downloading NodeJS. ``` #9 [2/5] ADD scripts /tmp/scripts #9 CACHED #10 [3/5] RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts #10 CACHED #11 [4/5] RUN adduser --uid 1000 onnxruntimedev #11 CACHED #12 [5/5] WORKDIR /home/onnxruntimedev #12 CACHED ``` ###Reference https://docs.docker.com/build/drivers/ --------- Co-authored-by: Yi Zhang <your@email.com>
113 lines
4.5 KiB
YAML
113 lines
4.5 KiB
YAML
# This file contains the ADO job that build libonnxruntime.so on Linux
|
|
parameters:
|
|
- name: AdditionalBuildFlags
|
|
displayName: Additional build flags for build.py
|
|
type: string
|
|
default: ''
|
|
|
|
- name: BaseImage
|
|
type: string
|
|
|
|
- name: OnnxruntimeArch
|
|
type: string
|
|
|
|
- name: OnnxruntimeNodejsBindingArch
|
|
type: string
|
|
values:
|
|
- arm64
|
|
- x64
|
|
|
|
- name: PoolName
|
|
type: string
|
|
default: 'onnxruntime-Ubuntu2204-AMD-CPU'
|
|
|
|
- name: ArtifactNamePrefix
|
|
type: string
|
|
default: "onnxruntime"
|
|
|
|
- name: PackageJava
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: PackageNodejs
|
|
type: boolean
|
|
default: true
|
|
|
|
jobs:
|
|
- job: Linux_C_API_Packaging_CPU_${{parameters.OnnxruntimeArch}}
|
|
|
|
workspace:
|
|
clean: all
|
|
timeoutInMinutes: 210
|
|
pool: ${{parameters.PoolName}}
|
|
steps:
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
- template: set-version-number-variables-step.yml
|
|
- ${{ if eq(parameters.OnnxruntimeArch, 'x64') }}:
|
|
- template: get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
|
|
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{parameters.BaseImage}}"
|
|
Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging
|
|
|
|
- ${{ if eq(parameters.OnnxruntimeArch, 'aarch64') }}:
|
|
- template: get-docker-image-steps.yml
|
|
parameters:
|
|
Dockerfile: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile
|
|
Context: tools/ci_build/github/linux/docker/inference/aarch64/default/cpu
|
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{parameters.BaseImage}}"
|
|
Repository: onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging
|
|
UpdateDepsTxt: false
|
|
|
|
- task: CmdLine@2
|
|
inputs:
|
|
script: |
|
|
mkdir -p $HOME/.onnx
|
|
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
|
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3.9 \
|
|
/onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \
|
|
--skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed"
|
|
workingDirectory: $(Build.SourcesDirectory)
|
|
displayName: 'Build'
|
|
|
|
- script: |
|
|
set -e -x
|
|
mkdir $(Build.ArtifactStagingDirectory)/testdata
|
|
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata
|
|
ls -al $(Build.ArtifactStagingDirectory)
|
|
displayName: 'Create Artifacts'
|
|
|
|
- ${{ if eq(parameters.PackageJava, 'true') }}:
|
|
- template: java-api-artifacts-package-and-publish-steps-posix.yml
|
|
parameters:
|
|
arch: 'linux-${{parameters.OnnxruntimeArch}}'
|
|
buildConfig: 'Release'
|
|
artifactName: 'onnxruntime-java-linux-${{parameters.OnnxruntimeArch}}'
|
|
version: '$(OnnxRuntimeVersion)'
|
|
libraryName: 'libonnxruntime.so'
|
|
nativeLibraryName: 'libonnxruntime4j_jni.so'
|
|
|
|
- template: c-api-artifacts-package-and-publish-steps-posix.yml
|
|
parameters:
|
|
buildConfig: 'Release'
|
|
artifactName: '${{parameters.ArtifactNamePrefix}}-linux-${{parameters.OnnxruntimeArch}}-$(OnnxRuntimeVersion)'
|
|
artifactNameNoVersionString: '${{parameters.ArtifactNamePrefix}}-linux-${{parameters.OnnxruntimeArch}}'
|
|
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
|
|
|
|
- ${{ if eq(parameters.PackageNodejs, 'true') }}:
|
|
- template: nodejs-artifacts-package-and-publish-steps-posix.yml
|
|
parameters:
|
|
arch: '${{parameters.OnnxruntimeNodejsBindingArch}}'
|
|
os: 'linux'
|
|
artifactName: 'drop-onnxruntime-nodejs-linux-${{parameters.OnnxruntimeArch}}'
|
|
|
|
- ${{ if not(eq(parameters.OnnxruntimeNodejsBindingArch, 'arm64')) }}:
|
|
- template: component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|