mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-22 19:23:30 +00:00
Remove git and python packages from the docker images used by Zip-Nuget-Java-Nodejs Packaging Pipeline (#11651)
This commit is contained in:
parent
3c1dd9514d
commit
d5e34acb82
17 changed files with 246 additions and 145 deletions
|
|
@ -675,7 +675,9 @@ def get_config_build_dir(build_dir, config):
|
||||||
return os.path.join(build_dir, config)
|
return os.path.join(build_dir, config)
|
||||||
|
|
||||||
|
|
||||||
def run_subprocess(args, cwd=None, capture_stdout=False, dll_path=None, shell=False, env={}, python_path=None):
|
def run_subprocess(
|
||||||
|
args, cwd=None, capture_stdout=False, dll_path=None, shell=False, env={}, python_path=None, cuda_home=None
|
||||||
|
):
|
||||||
if isinstance(args, str):
|
if isinstance(args, str):
|
||||||
raise ValueError("args should be a sequence of strings, not a string")
|
raise ValueError("args should be a sequence of strings, not a string")
|
||||||
|
|
||||||
|
|
@ -691,6 +693,10 @@ def run_subprocess(args, cwd=None, capture_stdout=False, dll_path=None, shell=Fa
|
||||||
my_env["LD_LIBRARY_PATH"] += os.pathsep + dll_path
|
my_env["LD_LIBRARY_PATH"] += os.pathsep + dll_path
|
||||||
else:
|
else:
|
||||||
my_env["LD_LIBRARY_PATH"] = dll_path
|
my_env["LD_LIBRARY_PATH"] = dll_path
|
||||||
|
# Add nvcc's folder to PATH env so that our cmake file can find nvcc
|
||||||
|
if cuda_home:
|
||||||
|
my_env["PATH"] = os.path.join(cuda_home, "bin") + os.pathsep + my_env["PATH"]
|
||||||
|
|
||||||
if python_path:
|
if python_path:
|
||||||
if "PYTHONPATH" in my_env:
|
if "PYTHONPATH" in my_env:
|
||||||
my_env["PYTHONPATH"] += os.pathsep + python_path
|
my_env["PYTHONPATH"] += os.pathsep + python_path
|
||||||
|
|
@ -907,7 +913,8 @@ def generate_build_tree(
|
||||||
add_default_definition(cmake_extra_defines, "onnxruntime_DEV_MODE", use_dev_mode(args))
|
add_default_definition(cmake_extra_defines, "onnxruntime_DEV_MODE", use_dev_mode(args))
|
||||||
if args.use_cuda:
|
if args.use_cuda:
|
||||||
add_default_definition(cmake_extra_defines, "onnxruntime_USE_CUDA", "ON")
|
add_default_definition(cmake_extra_defines, "onnxruntime_USE_CUDA", "ON")
|
||||||
add_default_definition(cmake_extra_defines, "onnxruntime_CUDA_VERSION", args.cuda_version)
|
if args.cuda_version:
|
||||||
|
add_default_definition(cmake_extra_defines, "onnxruntime_CUDA_VERSION", args.cuda_version)
|
||||||
# TODO: this variable is not really needed
|
# TODO: this variable is not really needed
|
||||||
add_default_definition(cmake_extra_defines, "onnxruntime_CUDA_HOME", cuda_home)
|
add_default_definition(cmake_extra_defines, "onnxruntime_CUDA_HOME", cuda_home)
|
||||||
add_default_definition(cmake_extra_defines, "onnxruntime_CUDNN_HOME", cudnn_home)
|
add_default_definition(cmake_extra_defines, "onnxruntime_CUDNN_HOME", cudnn_home)
|
||||||
|
|
@ -1233,6 +1240,7 @@ def generate_build_tree(
|
||||||
"-DCMAKE_BUILD_TYPE={}".format(config),
|
"-DCMAKE_BUILD_TYPE={}".format(config),
|
||||||
],
|
],
|
||||||
cwd=config_build_dir,
|
cwd=config_build_dir,
|
||||||
|
cuda_home=cuda_home,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
0
tools/ci_build/github/apple/build_host_protoc.sh
Normal file → Executable file
0
tools/ci_build/github/apple/build_host_protoc.sh
Normal file → Executable file
|
|
@ -49,19 +49,14 @@ jobs:
|
||||||
- template: templates/set-version-number-variables-step.yml
|
- template: templates/set-version-number-variables-step.yml
|
||||||
- template: templates/get-docker-image-steps.yml
|
- template: templates/get-docker-image-steps.yml
|
||||||
parameters:
|
parameters:
|
||||||
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11
|
Dockerfile: tools/ci_build/github/linux/docker/inference/x64/default/cpu/Dockerfile
|
||||||
Context: tools/ci_build/github/linux/docker
|
Context: tools/ci_build/github/linux/docker/inference/x64/default/cpu
|
||||||
DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg BASEIMAGE=nvidia/cuda:11.4.0-cudnn8-devel-centos7 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )"
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=nvidia/cuda:11.4.3-cudnn8-devel-centos7"
|
||||||
Repository: onnxruntimecuda11build
|
Repository: onnxruntimecuda11centosbuild
|
||||||
- task: CmdLine@2
|
|
||||||
inputs:
|
- script: $(Build.SourcesDirectory)/tools/ci_build/github/linux/build_cuda_c_api_package.sh
|
||||||
script: |
|
workingDirectory: $(Build.SourcesDirectory)
|
||||||
mkdir -p $HOME/.onnx
|
displayName: 'Build and Test'
|
||||||
docker run --gpus all -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
|
|
||||||
--volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \
|
|
||||||
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
|
|
||||||
--skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-10/root/usr/bin/cc 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80'
|
|
||||||
workingDirectory: $(Build.SourcesDirectory)
|
|
||||||
|
|
||||||
- template: templates/java-api-artifacts-package-and-publish-steps-posix.yml
|
- template: templates/java-api-artifacts-package-and-publish-steps-posix.yml
|
||||||
parameters:
|
parameters:
|
||||||
|
|
|
||||||
|
|
@ -44,80 +44,23 @@ parameters:
|
||||||
default: 'default'
|
default: 'default'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
- job: Linux_C_API_Packaging_CPU_x64
|
- template: c-api-linux-cpu.yml
|
||||||
workspace:
|
parameters:
|
||||||
clean: all
|
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }}
|
||||||
timeoutInMinutes: 210
|
BaseImage: 'centos:7'
|
||||||
strategy:
|
OnnxruntimeArch: 'x64'
|
||||||
matrix:
|
OnnxruntimeCFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
|
||||||
ARCH_x86_64:
|
OnnxruntimeCXXFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
|
||||||
BASE_IMAGE: centos:7
|
OnnxruntimeNodejsBindingArch: 'x64'
|
||||||
ONNXRUNTIME_ARCH: x64
|
|
||||||
ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
|
|
||||||
ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
|
|
||||||
NODEJS_BINDING_ARCH: x64
|
|
||||||
ARCH_AARCH64:
|
|
||||||
BASE_IMAGE: arm64v8/centos:7
|
|
||||||
ONNXRUNTIME_ARCH: aarch64
|
|
||||||
ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
|
|
||||||
ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
|
|
||||||
NODEJS_BINDING_ARCH: arm64
|
|
||||||
pool: 'Linux-CPU'
|
|
||||||
steps:
|
|
||||||
- template: set-version-number-variables-step.yml
|
|
||||||
- template: get-docker-image-steps.yml
|
|
||||||
parameters:
|
|
||||||
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
|
|
||||||
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu
|
|
||||||
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=$(BASE_IMAGE)"
|
|
||||||
Repository: onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH)
|
|
||||||
|
|
||||||
- task: CmdLine@2
|
- template: c-api-linux-cpu.yml
|
||||||
inputs:
|
parameters:
|
||||||
script: |
|
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }}
|
||||||
mkdir -p $HOME/.onnx
|
BaseImage: 'arm64v8/centos:7'
|
||||||
docker run --rm -e CFLAGS="$(ONNXRUNTIME_CFLAGS)" -e CXXFLAGS="$(ONNXRUNTIME_CXXFLAGS)" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
|
OnnxruntimeArch: 'aarch64'
|
||||||
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH) /bin/bash -c "python3 \
|
OnnxruntimeCFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
|
||||||
/onnxruntime_src/tools/ci_build/build.py --build_java --build_nodejs --build_dir /build --config Release \
|
OnnxruntimeCXXFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
|
||||||
--skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-$(ONNXRUNTIME_ARCH)"
|
OnnxruntimeNodejsBindingArch: 'arm64'
|
||||||
workingDirectory: $(Build.SourcesDirectory)
|
|
||||||
|
|
||||||
- script: |
|
|
||||||
set -e -x
|
|
||||||
mkdir $(Build.ArtifactStagingDirectory)/testdata
|
|
||||||
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata
|
|
||||||
ls -al $(Build.ArtifactStagingDirectory)
|
|
||||||
displayName: 'Create Artifacts'
|
|
||||||
|
|
||||||
- template: java-api-artifacts-package-and-publish-steps-posix.yml
|
|
||||||
parameters:
|
|
||||||
arch: 'linux-$(ONNXRUNTIME_ARCH)'
|
|
||||||
buildConfig: 'Release'
|
|
||||||
artifactName: 'onnxruntime-java-linux-$(ONNXRUNTIME_ARCH)'
|
|
||||||
version: '$(OnnxRuntimeVersion)'
|
|
||||||
libraryName: 'libonnxruntime.so'
|
|
||||||
nativeLibraryName: 'libonnxruntime4j_jni.so'
|
|
||||||
|
|
||||||
- template: c-api-artifacts-package-and-publish-steps-posix.yml
|
|
||||||
parameters:
|
|
||||||
buildConfig: 'Release'
|
|
||||||
artifactName: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)-$(OnnxRuntimeVersion)'
|
|
||||||
artifactNameNoVersionString: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)'
|
|
||||||
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
|
|
||||||
|
|
||||||
- template: nodejs-artifacts-package-and-publish-steps-posix.yml
|
|
||||||
parameters:
|
|
||||||
arch: '$(NODEJS_BINDING_ARCH)'
|
|
||||||
os: 'linux'
|
|
||||||
artifactName: 'drop-onnxruntime-nodejs-linux-$(ONNXRUNTIME_ARCH)'
|
|
||||||
|
|
||||||
- template: component-governance-component-detection-steps.yml
|
|
||||||
parameters :
|
|
||||||
condition : 'succeeded'
|
|
||||||
|
|
||||||
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
||||||
displayName: 'Clean Agent Directories'
|
|
||||||
condition: always()
|
|
||||||
|
|
||||||
- job: MacOS_C_API_Packaging_CPU_x64
|
- job: MacOS_C_API_Packaging_CPU_x64
|
||||||
workspace:
|
workspace:
|
||||||
|
|
@ -304,6 +247,7 @@ jobs:
|
||||||
pool: 'Win-CPU-2021'
|
pool: 'Win-CPU-2021'
|
||||||
dependsOn:
|
dependsOn:
|
||||||
- Linux_C_API_Packaging_CPU_x64
|
- Linux_C_API_Packaging_CPU_x64
|
||||||
|
- Linux_C_API_Packaging_CPU_aarch64
|
||||||
- MacOS_C_API_Packaging_CPU_x64
|
- MacOS_C_API_Packaging_CPU_x64
|
||||||
- Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }}
|
- Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }}
|
||||||
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
|
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
|
||||||
|
|
@ -387,6 +331,7 @@ jobs:
|
||||||
breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
|
breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
|
||||||
dependsOn:
|
dependsOn:
|
||||||
- Linux_C_API_Packaging_CPU_x64
|
- Linux_C_API_Packaging_CPU_x64
|
||||||
|
- Linux_C_API_Packaging_CPU_aarch64
|
||||||
- MacOS_C_API_Packaging_CPU_x64
|
- MacOS_C_API_Packaging_CPU_x64
|
||||||
- Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }}
|
- Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }}
|
||||||
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
|
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
|
||||||
|
|
@ -591,6 +536,7 @@ jobs:
|
||||||
breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
|
breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
|
||||||
dependsOn:
|
dependsOn:
|
||||||
- Linux_C_API_Packaging_CPU_x64
|
- Linux_C_API_Packaging_CPU_x64
|
||||||
|
- Linux_C_API_Packaging_CPU_aarch64
|
||||||
- MacOS_C_API_Packaging_CPU_x64
|
- MacOS_C_API_Packaging_CPU_x64
|
||||||
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
|
- Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }}
|
||||||
- Windows_Packaging_CPU_arm64_${{ parameters.BuildVariant }}
|
- Windows_Packaging_CPU_arm64_${{ parameters.BuildVariant }}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,84 @@
|
||||||
|
# This file contains the ADO job that build libonnxruntime.so on Linux
|
||||||
|
parameters:
|
||||||
|
- name: AdditionalBuildFlags
|
||||||
|
displayName: Additional build flags for build.py
|
||||||
|
type: string
|
||||||
|
default: ''
|
||||||
|
|
||||||
|
- name: BaseImage
|
||||||
|
type: string
|
||||||
|
|
||||||
|
- name: OnnxruntimeArch
|
||||||
|
type: string
|
||||||
|
|
||||||
|
- name: OnnxruntimeCFlags
|
||||||
|
type: string
|
||||||
|
|
||||||
|
- name: OnnxruntimeCXXFlags
|
||||||
|
type: string
|
||||||
|
|
||||||
|
- name: OnnxruntimeNodejsBindingArch
|
||||||
|
type: string
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
- job: Linux_C_API_Packaging_CPU_${{parameters.OnnxruntimeArch}}
|
||||||
|
workspace:
|
||||||
|
clean: all
|
||||||
|
timeoutInMinutes: 210
|
||||||
|
pool: 'Linux-CPU'
|
||||||
|
steps:
|
||||||
|
- template: set-version-number-variables-step.yml
|
||||||
|
- template: get-docker-image-steps.yml
|
||||||
|
parameters:
|
||||||
|
Dockerfile: tools/ci_build/github/linux/docker/inference/${{parameters.OnnxruntimeArch}}/default/cpu/Dockerfile
|
||||||
|
Context: tools/ci_build/github/linux/docker/inference/${{parameters.OnnxruntimeArch}}/default/cpu
|
||||||
|
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{parameters.BaseImage}}"
|
||||||
|
Repository: onnxruntimecpubuildcentos7${{parameters.OnnxruntimeArch}}
|
||||||
|
|
||||||
|
- task: CmdLine@2
|
||||||
|
inputs:
|
||||||
|
script: |
|
||||||
|
mkdir -p $HOME/.onnx
|
||||||
|
docker run --rm -e CFLAGS="${{parameters.OnnxruntimeCFlags}}" -e CXXFLAGS="${{parameters.OnnxruntimeCXXFlags}}" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
|
||||||
|
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos7${{parameters.OnnxruntimeArch}} /bin/bash -c "python3 \
|
||||||
|
/onnxruntime_src/tools/ci_build/build.py --build_java --build_nodejs --build_dir /build --config Release \
|
||||||
|
--skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-${{parameters.OnnxruntimeArch}}"
|
||||||
|
workingDirectory: $(Build.SourcesDirectory)
|
||||||
|
displayName: 'Build'
|
||||||
|
|
||||||
|
- script: |
|
||||||
|
set -e -x
|
||||||
|
mkdir $(Build.ArtifactStagingDirectory)/testdata
|
||||||
|
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata
|
||||||
|
ls -al $(Build.ArtifactStagingDirectory)
|
||||||
|
displayName: 'Create Artifacts'
|
||||||
|
|
||||||
|
- template: java-api-artifacts-package-and-publish-steps-posix.yml
|
||||||
|
parameters:
|
||||||
|
arch: 'linux-${{parameters.OnnxruntimeArch}}'
|
||||||
|
buildConfig: 'Release'
|
||||||
|
artifactName: 'onnxruntime-java-linux-${{parameters.OnnxruntimeArch}}'
|
||||||
|
version: '$(OnnxRuntimeVersion)'
|
||||||
|
libraryName: 'libonnxruntime.so'
|
||||||
|
nativeLibraryName: 'libonnxruntime4j_jni.so'
|
||||||
|
|
||||||
|
- template: c-api-artifacts-package-and-publish-steps-posix.yml
|
||||||
|
parameters:
|
||||||
|
buildConfig: 'Release'
|
||||||
|
artifactName: 'onnxruntime-linux-${{parameters.OnnxruntimeArch}}-$(OnnxRuntimeVersion)'
|
||||||
|
artifactNameNoVersionString: 'onnxruntime-linux-${{parameters.OnnxruntimeArch}}'
|
||||||
|
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
|
||||||
|
|
||||||
|
- template: nodejs-artifacts-package-and-publish-steps-posix.yml
|
||||||
|
parameters:
|
||||||
|
arch: '${{parameters.OnnxruntimeNodejsBindingArch}}'
|
||||||
|
os: 'linux'
|
||||||
|
artifactName: 'drop-onnxruntime-nodejs-linux-${{parameters.OnnxruntimeArch}}'
|
||||||
|
|
||||||
|
- template: component-governance-component-detection-steps.yml
|
||||||
|
parameters :
|
||||||
|
condition : 'succeeded'
|
||||||
|
|
||||||
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
||||||
|
displayName: 'Clean Agent Directories'
|
||||||
|
condition: always()
|
||||||
10
tools/ci_build/github/linux/build_cuda_c_api_package.sh
Executable file
10
tools/ci_build/github/linux/build_cuda_c_api_package.sh
Executable file
|
|
@ -0,0 +1,10 @@
|
||||||
|
#!/bin/bash
|
||||||
|
export CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all"
|
||||||
|
export CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all"
|
||||||
|
docker run --gpus all -e CFLAGS -e CXXFLAGS -e NVIDIA_VISIBLE_DEVICES=all --rm --volume \
|
||||||
|
$BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \
|
||||||
|
--volume /data/models:/build/models:ro --volume /data/onnx:/data/onnx:ro -e NIGHTLY_BUILD onnxruntimecuda11centosbuild \
|
||||||
|
python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
|
||||||
|
--skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION \
|
||||||
|
--cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr/local/cuda-$CUDA_VERSION \
|
||||||
|
--cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80'
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -e -x
|
||||||
|
|
||||||
|
os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)
|
||||||
|
|
||||||
|
echo "installing for CentOS version : $os_major_version"
|
||||||
|
yum install -y centos-release-scl-rh
|
||||||
|
yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel java-11-openjdk-devel graphviz devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran python3 python3-pip
|
||||||
|
|
||||||
|
pip3 install --upgrade pip
|
||||||
|
localedef -i en_US -f UTF-8 en_US.UTF-8
|
||||||
|
|
@ -34,27 +34,13 @@ function GetFile {
|
||||||
|
|
||||||
return $?
|
return $?
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)
|
|
||||||
|
|
||||||
SYS_LONG_BIT=$(getconf LONG_BIT)
|
|
||||||
mkdir -p /tmp/src
|
mkdir -p /tmp/src
|
||||||
GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.)
|
|
||||||
|
|
||||||
DISTRIBUTOR=$(lsb_release -i -s)
|
|
||||||
|
|
||||||
if [[ "$DISTRIBUTOR" = "CentOS" && $SYS_LONG_BIT = "64" ]]; then
|
|
||||||
LIBDIR="lib64"
|
|
||||||
else
|
|
||||||
LIBDIR="lib"
|
|
||||||
fi
|
|
||||||
|
|
||||||
cd /tmp/src
|
cd /tmp/src
|
||||||
|
|
||||||
echo "Installing cmake"
|
echo "Installing cmake"
|
||||||
GetFile https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-`uname -m`.tar.gz /tmp/src/cmake-3.21.1-linux-`uname -m`.tar.gz
|
GetFile https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-`uname -m`.tar.gz /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz
|
||||||
tar -zxf /tmp/src/cmake-3.21.1-linux-`uname -m`.tar.gz --strip=1 -C /usr
|
tar -zxf /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz --strip=1 -C /usr
|
||||||
|
|
||||||
echo "Installing Ninja"
|
echo "Installing Ninja"
|
||||||
GetFile https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz /tmp/src/ninja-linux.tar.gz
|
GetFile https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz /tmp/src/ninja-linux.tar.gz
|
||||||
|
|
@ -81,20 +67,5 @@ GetFile https://downloads.gradle-dn.com/distributions/gradle-6.3-bin.zip /tmp/sr
|
||||||
unzip /tmp/src/gradle-6.3-bin.zip
|
unzip /tmp/src/gradle-6.3-bin.zip
|
||||||
mv /tmp/src/gradle-6.3 /usr/local/gradle
|
mv /tmp/src/gradle-6.3 /usr/local/gradle
|
||||||
|
|
||||||
if ! [ -x "$(command -v protoc)" ]; then
|
|
||||||
GetFile https://github.com/protocolbuffers/protobuf/archive/v3.18.1.tar.gz /tmp/src/v3.18.1.tar.gz
|
|
||||||
tar -xf /tmp/src/v3.18.1.tar.gz -C /tmp/src
|
|
||||||
cd /tmp/src/protobuf-3.18.1
|
|
||||||
cmake ./cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo -DCMAKE_INSTALL_LIBDIR=lib64
|
|
||||||
make -j$(getconf _NPROCESSORS_ONLN)
|
|
||||||
make install
|
|
||||||
fi
|
|
||||||
|
|
||||||
export ONNX_ML=1
|
|
||||||
export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF"
|
|
||||||
|
|
||||||
pip3 install -r ${0/%install_deps\.sh/requirements\.txt}
|
|
||||||
|
|
||||||
|
|
||||||
cd /
|
cd /
|
||||||
rm -rf /tmp/src
|
rm -rf /tmp/src
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||||
|
# Licensed under the MIT License.
|
||||||
|
|
||||||
|
# This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline
|
||||||
|
ARG BASEIMAGE=centos:7
|
||||||
|
FROM $BASEIMAGE
|
||||||
|
|
||||||
|
ENV PATH /usr/local/gradle/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||||
|
ENV LANG=en_US.utf8
|
||||||
|
ENV LC_ALL=en_US.utf8
|
||||||
|
|
||||||
|
ADD scripts /tmp/scripts
|
||||||
|
RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
|
||||||
|
|
||||||
|
ARG BUILD_UID=1001
|
||||||
|
ARG BUILD_USER=onnxruntimedev
|
||||||
|
RUN adduser --uid $BUILD_UID $BUILD_USER
|
||||||
|
WORKDIR /home/$BUILD_USER
|
||||||
|
USER $BUILD_USER
|
||||||
|
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -e -x
|
||||||
|
|
||||||
|
os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)
|
||||||
|
|
||||||
|
echo "installing for CentOS version : $os_major_version"
|
||||||
|
yum install -y centos-release-scl-rh
|
||||||
|
yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel java-11-openjdk-devel graphviz devtoolset-11-binutils devtoolset-11-gcc devtoolset-11-gcc-c++ devtoolset-11-gcc-gfortran python3 python3-pip
|
||||||
|
|
||||||
|
pip3 install --upgrade pip
|
||||||
|
localedef -i en_US -f UTF-8 en_US.UTF-8
|
||||||
|
|
@ -0,0 +1,71 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -e -x
|
||||||
|
|
||||||
|
# Download a file from internet
|
||||||
|
function GetFile {
|
||||||
|
local uri=$1
|
||||||
|
local path=$2
|
||||||
|
local force=${3:-false}
|
||||||
|
local download_retries=${4:-5}
|
||||||
|
local retry_wait_time_seconds=${5:-30}
|
||||||
|
|
||||||
|
if [[ -f $path ]]; then
|
||||||
|
if [[ $force = false ]]; then
|
||||||
|
echo "File '$path' already exists. Skipping download"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
rm -rf $path
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -f $uri ]]; then
|
||||||
|
echo "'$uri' is a file path, copying file to '$path'"
|
||||||
|
cp $uri $path
|
||||||
|
return $?
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Downloading $uri"
|
||||||
|
# Use aria2c if available, otherwise use curl
|
||||||
|
if command -v aria2c > /dev/null; then
|
||||||
|
aria2c -q -d $(dirname $path) -o $(basename $path) "$uri"
|
||||||
|
else
|
||||||
|
curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail
|
||||||
|
fi
|
||||||
|
|
||||||
|
return $?
|
||||||
|
}
|
||||||
|
mkdir -p /tmp/src
|
||||||
|
|
||||||
|
cd /tmp/src
|
||||||
|
|
||||||
|
echo "Installing cmake"
|
||||||
|
GetFile https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-`uname -m`.tar.gz /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz
|
||||||
|
tar -zxf /tmp/src/cmake-3.23.2-linux-`uname -m`.tar.gz --strip=1 -C /usr
|
||||||
|
|
||||||
|
echo "Installing Ninja"
|
||||||
|
GetFile https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz /tmp/src/ninja-linux.tar.gz
|
||||||
|
tar -zxf ninja-linux.tar.gz
|
||||||
|
cd ninja-1.10.0
|
||||||
|
cmake -Bbuild-cmake -H.
|
||||||
|
cmake --build build-cmake
|
||||||
|
mv ./build-cmake/ninja /usr/bin
|
||||||
|
|
||||||
|
echo "Installing Node.js"
|
||||||
|
CPU_ARCH=`uname -m`
|
||||||
|
if [[ "$CPU_ARCH" = "x86_64" ]]; then
|
||||||
|
NODEJS_ARCH=x64
|
||||||
|
elif [[ "$CPU_ARCH" = "aarch64" ]]; then
|
||||||
|
NODEJS_ARCH=arm64
|
||||||
|
else
|
||||||
|
NODEJS_ARCH=$CPU_ARCH
|
||||||
|
fi
|
||||||
|
GetFile https://nodejs.org/dist/v16.14.2/node-v16.14.2-linux-${NODEJS_ARCH}.tar.gz /tmp/src/node-v16.14.2-linux-${NODEJS_ARCH}.tar.gz
|
||||||
|
tar --strip 1 -xf /tmp/src/node-v16.14.2-linux-${NODEJS_ARCH}.tar.gz -C /usr
|
||||||
|
|
||||||
|
cd /tmp/src
|
||||||
|
GetFile https://downloads.gradle-dn.com/distributions/gradle-6.3-bin.zip /tmp/src/gradle-6.3-bin.zip
|
||||||
|
unzip /tmp/src/gradle-6.3-bin.zip
|
||||||
|
mv /tmp/src/gradle-6.3 /usr/local/gradle
|
||||||
|
|
||||||
|
cd /
|
||||||
|
rm -rf /tmp/src
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -e -x
|
|
||||||
|
|
||||||
os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)
|
|
||||||
CPU_ARCH=`uname -m`
|
|
||||||
PACKAGES_TO_INSTALL="which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel java-11-openjdk-devel graphviz devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran python3 python3-devel python3-pip"
|
|
||||||
if ! rpm -q --quiet epel-release ; then
|
|
||||||
if [[ "$CPU_ARCH" = "x86_64" ]]; then
|
|
||||||
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm https://repo.ius.io/ius-release-el$os_major_version.rpm
|
|
||||||
PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL git224"
|
|
||||||
else
|
|
||||||
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm
|
|
||||||
PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL git"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "installing for os major version : $os_major_version"
|
|
||||||
yum install -y centos-release-scl-rh
|
|
||||||
yum install -y $PACKAGES_TO_INSTALL
|
|
||||||
|
|
||||||
pip3 install --upgrade pip
|
|
||||||
localedef -i en_US -f UTF-8 en_US.UTF-8
|
|
||||||
|
|
@ -1,4 +0,0 @@
|
||||||
numpy
|
|
||||||
git+http://github.com/onnx/onnx.git@be76ca7148396176784ba8733133b9fb1186ea0d#egg=onnx
|
|
||||||
protobuf
|
|
||||||
flatbuffers
|
|
||||||
0
tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh
Normal file → Executable file
0
tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh
Normal file → Executable file
0
tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh
Normal file → Executable file
0
tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh
Normal file → Executable file
0
tools/ci_build/github/pai/pai_get_thread.sh
Normal file → Executable file
0
tools/ci_build/github/pai/pai_get_thread.sh
Normal file → Executable file
Loading…
Reference in a new issue