saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update training packaging pipeline's docker files (#20853)

Browse source 
### Description
Similar to #20786 . The last PR was able to update all pipelines and all
docker files. This is a follow-up to that PR.

### Motivation and Context
1. To extract the common part as a reusable build infra among different
ONNX Runtime projects.
2. Avoid hitting docker hub's limit: 429 Too Many Requests - Server
message: toomanyrequests: You have reached your pull rate limit. You may
increase the limit by authenticating and upgrading:
https://www.docker.com/increase-rate-limit
This commit is contained in:
Changming Sun 2024-05-30 23:48:42 -07:00 committed by GitHub
parent 00589f578d
commit 67bc9438d7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
42 changed files with 229 additions and 958 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml
View file

@ -40,12 +40,6 @@ parameters:
resources:
repositories:
- repository: manylinux
type: Github
endpoint: Microsoft
name: pypa/manylinux
ref: 5eda9aded5462201e6310105728d33016e637ea7
- repository: LLaMa2Onnx
type: Github
endpoint: Microsoft
@ -55,7 +49,7 @@ resources:
variables:
- template: templates/common-variables.yml
- name: docker_base_image
value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240530.3
- name: linux_trt_version
value: 10.0.1.6-1.cuda11.8
- name: Repository
@ -113,7 +107,7 @@ stages:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \
docker run -e --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \

7
tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
View file

@ -93,11 +93,6 @@ variables:
value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8
- name: linux_trt_version
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: 10.0.1.6-1.cuda11.8
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: 10.0.1.6-1.cuda12.4
- name: win_trt_version
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: 11.8
@ -184,7 +179,6 @@ stages:
DoCompliance: ${{ parameters.DoCompliance }}
CudaVersion: ${{ parameters.CudaVersion }}
docker_base_image: ${{ variables.docker_base_image }}
linux_trt_version: ${{ variables.linux_trt_version }}
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
win_trt_home: ${{ variables.win_trt_home }}
@ -228,6 +222,7 @@ stages:
--build-arg PREPEND_PATH=/opt/rh/gcc-toolset-12/root/usr/bin:
--build-arg LD_LIBRARY_PATH_ARG=/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
Repository: onnxruntimetrainingrocmbuild-rocm$(RocmVersion)
CheckOutManyLinux: true
- template: templates/set-version-number-variables-step.yml

6
tools/ci_build/github/azure-pipelines/cuda-packaging-pipeline.yml
View file

@ -66,11 +66,6 @@ variables:
value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8
- name: linux_trt_version
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: 10.0.1.6-1.cuda11.8
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: 10.0.1.6-1.cuda12.4
- name: win_trt_home
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: $(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-11.8
@ -120,7 +115,6 @@ stages:
DoCompliance: ${{ parameters.DoCompliance }}
CudaVersion: ${{ parameters.CudaVersion }}
docker_base_image: ${{ variables.docker_base_image }}
linux_trt_version: ${{ variables.linux_trt_version }}
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
win_trt_home: ${{ variables.win_trt_home }}

8
tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml
View file

@ -1,3 +1,4 @@
##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
trigger:
branches:
@ -27,13 +28,6 @@ pr:
- 'onnxruntime/core/providers/js'
#### end trigger ####
resources:
repositories:
- repository: manylinux # The name used to reference this repository in the checkout step
type: Github
endpoint: Microsoft
name: pypa/manylinux
ref: 5eda9aded5462201e6310105728d33016e637ea7
stages:
- stage: x64
dependsOn: []

8
tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml
View file

@ -46,14 +46,6 @@ pr:
- BUILD.md
- 'js/web'
- 'onnxruntime/core/providers/js'
resources:
repositories:
- repository: manylinux
type: Github
endpoint: Microsoft
name: pypa/manylinux
ref: 5eda9aded5462201e6310105728d33016e637ea7
jobs:
- job: Linux_CPU_Minimal_Build_E2E
timeoutInMinutes: 120

8
tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml
View file

@ -27,14 +27,6 @@ pr:
- 'onnxruntime/core/providers/js'
#### end trigger ####
resources:
repositories:
- repository: manylinux
type: Github
endpoint: Microsoft
name: pypa/manylinux
ref: 5eda9aded5462201e6310105728d33016e637ea7
jobs:
- job: Linux_py_Wheels
timeoutInMinutes: 180

67
tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml
View file

@ -45,26 +45,12 @@ parameters:
type: string
default: '0'
resources:
repositories:
- repository: manylinux
type: Github
endpoint: Microsoft
name: pypa/manylinux
ref: 5eda9aded5462201e6310105728d33016e637ea7
variables:
- name: docker_base_image
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240530.3
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8
- name: linux_trt_version
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: 10.0.1.6-1.cuda11.8
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: 10.0.1.6-1.cuda12.4
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_2_x64_ubi8_gcc12:20240530.3
- name: Repository
${{ if eq(parameters.CudaVersion, '11.8') }}:
@ -97,12 +83,7 @@ stages:
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "
--network=host
--build-arg BASEIMAGE=$(docker_base_image)
--build-arg TRT_VERSION=$(linux_trt_version)
--build-arg BUILD_UID=$( id -u )
"
DockerBuildArgs: "--build-arg BASEIMAGE=$(docker_base_image) --build-arg BUILD_UID=$( id -u )"
Repository: $(Repository)
- task: Cache@2
@ -123,7 +104,7 @@ stages:
- script: |
set -e -x
mkdir -p $HOME/.onnx
docker run -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \
docker run --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
@ -133,39 +114,12 @@ stages:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
-e CCACHE_DIR=/cache \
$(Repository) \
/bin/bash -c "
set -ex; \
env; \
ccache -s; \
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Release --update --build \
--skip_submodule_sync \
--build_shared_lib \
--parallel --use_binskim_compliant_compile_flags \
--build_wheel \
--enable_onnx_tests --use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda-${{parameters.CudaVersion}} --cudnn_home=/usr/local/cuda-${{parameters.CudaVersion}} \
--enable_cuda_profiling --enable_cuda_nhwc_ops \
--enable_pybind --build_java \
--use_cache \
--cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 \
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=ON \
--cmake_extra_defines onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS=ON; \
ccache -sv; \
ccache -z"
-e CCACHE_DIR=/cache -w /onnxruntime_src \
$(Repository) tools/ci_build/github/linux/build_cuda_ci.sh
workingDirectory: $(Build.SourcesDirectory)
displayName: Build Onnxruntime
- task: CmdLine@2
inputs:
script: |
rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11
rm -f $(Build.BinariesDirectory)/Release/models
find $(Build.BinariesDirectory)/Release/_deps -mindepth 1 ! -regex '^$(Build.BinariesDirectory)/Release/_deps/onnx-src\(/.*\)?' -delete
cd $(Build.BinariesDirectory)/Release
find -executable -type f > $(Build.BinariesDirectory)/Release/perms.txt
- script: $(Build.SourcesDirectory)/tools/ci_build/github/linux/delete_unused_files_before_upload.sh
- task: PublishPipelineArtifact@0
displayName: 'Publish Pipeline Artifact'
@ -203,12 +157,7 @@ stages:
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "
--network=host
--build-arg BASEIMAGE=$(docker_base_image)
--build-arg TRT_VERSION=$(linux_trt_version)
--build-arg BUILD_UID=$( id -u )
"
DockerBuildArgs: "--build-arg BASEIMAGE=$(docker_base_image) --build-arg BUILD_UID=$( id -u )"
Repository: $(Repository)
- task: CmdLine@2

42
tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml
View file

@ -34,29 +34,19 @@ parameters:
values:
- 11.8
- 12.2
- 12.4
resources:
repositories:
- repository: manylinux
type: Github
endpoint: Microsoft
name: pypa/manylinux
ref: 5eda9aded5462201e6310105728d33016e637ea7
variables:
- name: docker_base_image
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240530.3
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8
${{ if eq(parameters.CudaVersion, '12.4') }}:
value: nvidia/cuda:12.4.1-cudnn-devel-ubi8
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_2_x64_ubi8_gcc12:20240530.3
- name: linux_trt_version
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: 10.0.1.6-1.cuda11.8
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: 10.0.1.6-1.cuda12.4
${{ if eq(parameters.CudaVersion, '12.4') }}:
value: 10.0.1.6-1.cuda12.4
jobs:
- job: Linux_Build
timeoutInMinutes: 180
@ -99,7 +89,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \
docker run --gpus all --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
@ -109,26 +99,8 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
-e CCACHE_DIR=/cache \
onnxruntimetensorrt86gpubuild \
/bin/bash -c "
set -ex; \
ccache -s; \
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Release \
--skip_submodule_sync \
--build_shared_lib \
--parallel --use_binskim_compliant_compile_flags \
--build_wheel \
--enable_onnx_tests \
--use_cuda --cuda_home=/usr/local/cuda-${{ parameters.CudaVersion }} --cudnn_home=/usr/lib64/ \
--enable_pybind --build_java \
--use_tensorrt --tensorrt_home /usr \
--cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 \
--use_cache; \
ccache -sv; \
ccache -z"
-e CCACHE_DIR=/cache -w /onnxruntime_src \
onnxruntimetensorrt86gpubuild tools/ci_build/github/linux/build_tensorrt_ci.sh
workingDirectory: $(Build.SourcesDirectory)
- template: templates/explicitly-defined-final-tasks.yml

1
tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml
View file

@ -62,6 +62,7 @@ jobs:
--build-arg PREPEND_PATH=/opt/rh/gcc-toolset-12/root/usr/bin:
--build-arg LD_LIBRARY_PATH_ARG=/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
Repository: onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-manylinux-build
CheckOutManyLinux: true
- task: Cache@2
inputs:

106
tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml
View file

@ -1,103 +1,23 @@
parameters:
- name: cmake_build_type
type: string
displayName: 'Linux packages cmake build type. Linux Only.'
default: 'Release'
values:
- Debug
- Release
- RelWithDebInfo
- MinSizeRel
trigger: none
resources:
repositories:
- repository: manylinux
type: Github
endpoint: Microsoft
name: pypa/manylinux
ref: 5eda9aded5462201e6310105728d33016e637ea7
stages:
- stage: Python_Packaging_Linux_Training_CPU
jobs:
- job: Linux_Training_CPU_Wheels
timeoutInMinutes: 180
workspace:
clean: all
pool: onnxruntime-Ubuntu2204-AMD-CPU
strategy:
matrix:
Python38:
PythonVersion: '3.8'
Python39:
PythonVersion: '3.9'
Python310:
PythonVersion: '3.10'
Python311:
PythonVersion: '3.11'
Python312:
PythonVersion: '3.12'
steps:
- checkout: self
clean: true
submodules: recursive
- template: templates/set-python-manylinux-variables-step.yml
- template: templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: >-
--build-arg PYTHON_VERSION=$(PythonVersion)
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tu
--build-arg BUILD_UID=$(id -u)
Repository: onnxruntimetrainingcpubuild_$(PythonVersion)
- task: CmdLine@2
displayName: 'build onnxruntime'
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
-e ORT_DISABLE_PYTHON_PACKAGE_LOCAL_VERSION \
-e DEFAULT_TRAINING_PACKAGE_DEVICE \
onnxruntimetrainingcpubuild_$(PythonVersion) \
$(PythonManylinuxDir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
--skip_submodule_sync \
--build_shared_lib \
--parallel --use_binskim_compliant_compile_flags \
--build_wheel \
--enable_onnx_tests \
--enable_pybind --enable_training
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)'
Contents: 'Release/dist/*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel and documentation'
inputs:
ArtifactName: onnxruntime_training_cpu
- template: templates/component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- template: templates/clean-agent-build-directory-step.yml
- template: templates/py-packaging-stage.yml
parameters:
build_py_parameters: --enable_training
cmake_build_type: ${{ parameters.cmake_build_type }}
enable_linux_gpu: false
enable_linux_cpu: false
enable_linux_cpu: true
enable_windows_cpu: true
enable_windows_gpu: false
enable_mac_cpu: true

8
tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda.yml
View file

@ -1,13 +1,5 @@
trigger: none
resources:
repositories:
- repository: manylinux
type: Github
endpoint: Microsoft
name: pypa/manylinux
ref: 5eda9aded5462201e6310105728d33016e637ea7
parameters:
- name: SpecificArtifact
displayName: Use Specific Artifact

8
tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda12.yml
View file

@ -1,13 +1,5 @@
trigger: none
resources:
repositories:
- repository: manylinux
type: Github
endpoint: Microsoft
name: pypa/manylinux
ref: 5eda9aded5462201e6310105728d33016e637ea7
stages:
- template: templates/py-packaging-training-cuda-stage.yml
parameters:

2
tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml
View file

@ -54,7 +54,7 @@ stages:
machine_pool: 'Onnxruntime-Linux-GPU'
python_wheel_suffix: '_gpu'
timeout: 480
docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240523.1
docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240530.3
trt_version: '10.0.1.6-1.cuda11.8'
cuda_version: '11.8'

4
tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml
View file

@ -141,9 +141,9 @@ stages:
value: false
- name: docker_base_image
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240530.3
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_2_x64_ubi8_gcc12:20240530.3
timeoutInMinutes: 60
steps:

4
tools/ci_build/github/azure-pipelines/stages/nuget-combine-cuda-stage.yml
View file

@ -14,9 +14,6 @@ parameters:
- name: docker_base_image
type: string
- name: linux_trt_version
type: string
- name: win_trt_home
type: string
@ -46,7 +43,6 @@ stages:
parameters:
CudaVersion: ${{ parameters.CudaVersion }}
docker_base_image: ${{ parameters.docker_base_image }}
linux_trt_version: ${{ parameters.linux_trt_version }}
buildJava: ${{ parameters.buildJava }}
buildNodejs: ${{ parameters.buildNodejs }}

45
tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml
View file

@ -4,8 +4,6 @@ parameters:
default: '11.8'
- name: docker_base_image
type: string
- name: linux_trt_version
type: string
- name: buildJava
type: boolean
- name: buildNodejs
@ -102,10 +100,9 @@ stages:
submodules: recursive
- template: ../templates/get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
Context: tools/ci_build/github/linux/docker
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/gpu/Dockerfile
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/gpu
DockerBuildArgs: "
--network=host
--build-arg BASEIMAGE=${{ variables.docker_base_image }}
--build-arg TRT_VERSION=${{ variables.linux_trt_version }}
--build-arg BUILD_UID=$( id -u )
@ -162,38 +159,34 @@ stages:
value: '11'
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: '12'
- name: linux_trt_version
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: 10.0.1.6-1.cuda11.8
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: 10.0.1.6-1.cuda12.4
- name: docker_base_image
${{ if eq(parameters.CudaVersion, '11.8') }}:
value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
${{ if eq(parameters.CudaVersion, '12.2') }}:
value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8
steps:
- checkout: self # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
submodules: false
- checkout: onnxruntime-inference-examples # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime-inference-examples
submodules: false
- checkout: manylinux # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/manylinux
submodules: false
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- script: |
set -e -x
cd $(Build.SourcesDirectory)
mv manylinux onnxruntime
ls
- template: ../templates/with-container-registry-steps.yml
- template: ../templates/get-docker-image-steps.yml
parameters:
Steps:
- script: |
tools/ci_build/get_docker_image.py \
--dockerfile tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda \
--context tools/ci_build/github/linux/docker \
--docker-build-args "--network=host --build-arg BASEIMAGE=${{ parameters.docker_base_image }} --build-arg TRT_VERSION=${{ parameters.linux_trt_version }} --build-arg BUILD_UID=$( id -u )" \
--container-registry onnxruntimebuildcache \
--multiple_repos \
--repository onnxruntimecuda${{ variables.CUDA_VERSION_MAJOR }}xtrt86build
displayName: "Get onnxruntimecuda${{ variables.CUDA_VERSION_MAJOR }}xtrt86build image for tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda"
workingDirectory: $(Build.SourcesDirectory)/onnxruntime
ContainerRegistry: onnxruntimebuildcache
ScriptName: $(Build.SourcesDirectory)/onnxruntime/tools/ci_build/get_docker_image.py
Dockerfile: $(Build.SourcesDirectory)/onnxruntime/tools/ci_build/github/linux/docker/inference/x86_64/default/gpu/Dockerfile
Context: $(Build.SourcesDirectory)/onnxruntime/tools/ci_build/github/linux/docker/inference/x86_64/default/gpu
DockerBuildArgs: "--build-arg BASEIMAGE=${{ variables.docker_base_image }} --build-arg TRT_VERSION=${{ variables.linux_trt_version }} --build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecuda${{ variables.CUDA_VERSION_MAJOR }}xtrt86build
UpdateDepsTxt: false
- template: ../templates/set-version-number-variables-step.yml
parameters:

4
tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml
View file

@ -78,8 +78,8 @@ stages:
cmake_build_type: ${{ parameters.cmake_build_type }}
cuda_version: ${{ parameters.cuda_version }}
${{ if eq(parameters.cuda_version, '11.8') }}:
docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240523.1
docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240530.3
trt_version: 10.0.1.6-1.cuda11.8
${{ if eq(parameters.cuda_version, '12.2') }}:
docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_2_x64_ubi8_gcc12:20240523.1
docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_2_x64_ubi8_gcc12:20240530.3
trt_version: 10.0.1.6-1.cuda12.4

2
tools/ci_build/github/azure-pipelines/templates/android-binary-size-check-stage.yml
View file

@ -38,7 +38,7 @@ stages:
submodules: none
- template: use-android-ndk.yml
#TODO: use a different docker file since this job doesn't need to rely on manylinux
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu

1
tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml
View file

@ -64,6 +64,7 @@ jobs:
mkdir -p $(artifacts_directory)
workingDirectory: $(Build.BinariesDirectory)
#TODO: use a different docker file since this job doesn't need to rely on manylinux
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu

5
tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml
View file

@ -16,6 +16,9 @@ parameters:
- name: UsePipelineCache
type: boolean
default: false
- name: CheckOutManyLinux
type: boolean
default: false
- name: ScriptName
type: string
default: "tools/ci_build/get_docker_image.py"
@ -31,7 +34,7 @@ steps:
- ${{ if eq(parameters.UpdateDepsTxt, true)}}:
- template: download-deps.yml
- ${{ if contains(parameters.Dockerfile, 'manylinux') }}:
- ${{ if eq(parameters.CheckOutManyLinux, true) }}:
- checkout: manylinux
- script: |
set -e -x

2
tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
View file

@ -472,7 +472,7 @@ stages:
parameters:
arch: 'x86_64'
machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240523.1
docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240530.3
extra_build_arg: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
trt_version: '10.0.1.6-1.cuda11.8'

13
tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml
View file

@ -66,11 +66,7 @@ stages:
--build-arg OPSET_VERSION=${{ parameters.opset_version }}
--build-arg PYTHON_VERSION=${{ parameters.python_version }}
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tu
--build-arg BUILD_UID=$(id -u)
--network=host --build-arg POLICY=manylinux_2_28 --build-arg PLATFORM=x86_64
--build-arg DEVTOOLSET_ROOTPATH=/usr
--build-arg PREPEND_PATH=/usr/local/cuda/bin:
--build-arg LD_LIBRARY_PATH_ARG=/usr/local/lib64
--build-arg BUILD_UID=$(id -u)
Repository: $(Repository)
- task: CmdLine@2
@ -79,7 +75,7 @@ stages:
script: |
set -e -x
mkdir -p $HOME/.onnx
docker run --rm -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \
docker run --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
@ -184,10 +180,7 @@ stages:
--build-arg PYTHON_VERSION=${{ parameters.python_version }}
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tu
--build-arg BUILD_UID=$(id -u)
--network=host --build-arg POLICY=manylinux_2_28 --build-arg PLATFORM=x86_64
--build-arg DEVTOOLSET_ROOTPATH=/usr
--build-arg PREPEND_PATH=/usr/local/cuda/bin:
--build-arg LD_LIBRARY_PATH_ARG=/usr/local/lib64
--network=host
Repository: $(Repository)
- task: CmdLine@2

1
tools/ci_build/github/azure-pipelines/templates/rocm.yml
View file

@ -47,6 +47,7 @@ jobs:
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm
Context: tools/ci_build/github/linux/docker
CheckOutManyLinux: true
DockerBuildArgs: >-
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
--build-arg BUILD_UID=$(id -u)

46
tools/ci_build/github/linux/build_cuda_ci.sh Executable file
View file

@ -0,0 +1,46 @@
#!/bin/bash
set -ex
#Every cuda container has this $CUDA_VERSION env var set.
SHORT_CUDA_VERSION=$(echo $CUDA_VERSION | sed 's/\([[:digit:]]\+\.[[:digit:]]\+\)\.[[:digit:]]\+/\1/')
BUILD_ARGS=('--config' 'Release' '--update' '--build'
'--skip_submodule_sync'
'--build_shared_lib'
'--parallel' '--use_binskim_compliant_compile_flags'
'--build_wheel'
'--enable_onnx_tests'
'--use_cuda'
"--cuda_version=$SHORT_CUDA_VERSION"
"--cuda_home=/usr/local/cuda-$SHORT_CUDA_VERSION"
"--cudnn_home=/usr/local/cuda-$SHORT_CUDA_VERSION"
"--enable_cuda_profiling"
"--enable_cuda_nhwc_ops"
"--enable_pybind"
"--build_java"
"--cmake_extra_defines"
"CMAKE_CUDA_ARCHITECTURES=75"
"onnxruntime_BUILD_UNIT_TESTS=ON"
"onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS=ON")
if [ -x "$(command -v ninja)" ]; then
BUILD_ARGS+=('--cmake_generator' 'Ninja')
fi
if [ -d /build ]; then
BUILD_ARGS+=('--build_dir' '/build')
else
BUILD_ARGS+=('--build_dir' 'build')
fi
if [ -x "$(command -v ccache)" ]; then
ccache -s;
BUILD_ARGS+=("--use_cache")
fi
if [ -f /opt/python/cp38-cp38/bin/python3 ]; then
/opt/python/cp38-cp38/bin/python3 tools/ci_build/build.py "${BUILD_ARGS[@]}"
else
python3 tools/ci_build/build.py "${BUILD_ARGS[@]}"
fi
if [ -x "$(command -v ccache)" ]; then
ccache -sv
ccache -z
fi

6
tools/ci_build/github/linux/build_linux_python_package.sh
View file

@ -23,7 +23,13 @@ c) BUILD_CONFIG=${OPTARG};;
esac
done
BUILD_ARGS=("--build_dir" "/build" "--config" "$BUILD_CONFIG" "--update" "--build" "--skip_submodule_sync" "--parallel" "--use_binskim_compliant_compile_flags" "--build_wheel")
if [[ "$EXTRA_ARG" == *"training"* ]]; then
echo "Skip building unit tests because the container is a manylinux docker"
BUILD_ARGS+=("--cmake_extra_defines" "onnxruntime_BUILD_UNIT_TESTS=OFF")
fi
if [ "$BUILD_CONFIG" != "Debug" ]; then
BUILD_ARGS+=("--enable_lto")

4
tools/ci_build/github/linux/build_tensorrt_c_api_package.sh
View file

@ -1,8 +1,8 @@
#!/bin/bash
set -e -x
mkdir -p $HOME/.onnx
docker run -e CFLAGS -e CXXFLAGS --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \
--volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}xtrt86build \
/opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \
/usr/bin/python3.9 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \
--skip_tests \
--skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_java --build_nodejs --use_tensorrt --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr --tensorrt_home=/usr --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80'

46
tools/ci_build/github/linux/build_tensorrt_ci.sh Executable file
View file

@ -0,0 +1,46 @@
#!/bin/bash
set -ex
#Every cuda container has this $CUDA_VERSION env var set.
SHORT_CUDA_VERSION=$(echo $CUDA_VERSION | sed 's/\([[:digit:]]\+\.[[:digit:]]\+\)\.[[:digit:]]\+/\1/')
#TODO: add --update --build
BUILD_ARGS=('--config' 'Release'
'--skip_submodule_sync'
'--build_shared_lib'
'--parallel' '--use_binskim_compliant_compile_flags'
'--build_wheel'
'--enable_onnx_tests'
'--use_cuda'
"--cuda_version=$SHORT_CUDA_VERSION"
"--cuda_home=/usr/local/cuda-$SHORT_CUDA_VERSION"
"--cudnn_home=/usr/local/cuda-$SHORT_CUDA_VERSION"
"--use_tensorrt" "--tensorrt_home" "/usr"
"--enable_pybind"
"--build_java"
"--cmake_extra_defines"
"CMAKE_CUDA_ARCHITECTURES=75"
"onnxruntime_BUILD_UNIT_TESTS=ON"
"onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS=ON")
if [ -x "$(command -v ninja)" ]; then
BUILD_ARGS+=('--cmake_generator' 'Ninja')
fi
if [ -d /build ]; then
BUILD_ARGS+=('--build_dir' '/build')
else
BUILD_ARGS+=('--build_dir' 'build')
fi
if [ -x "$(command -v ccache)" ]; then
ccache -s;
BUILD_ARGS+=("--use_cache")
fi
if [ -f /opt/python/cp38-cp38/bin/python3 ]; then
/opt/python/cp38-cp38/bin/python3 tools/ci_build/build.py "${BUILD_ARGS[@]}"
else
python3 tools/ci_build/build.py "${BUILD_ARGS[@]}"
fi
if [ -x "$(command -v ccache)" ]; then
ccache -sv
ccache -z
fi

7
tools/ci_build/github/linux/delete_unused_files_before_upload.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
set -e -x
rm -rf $BUILD_BINARIESDIRECTORY/Release/onnxruntime $BUILD_BINARIESDIRECTORY/Release/pybind11
rm -f $BUILD_BINARIESDIRECTORY/Release/models
find $BUILD_BINARIESDIRECTORY/Release/_deps -mindepth 1 ! -regex "^$BUILD_BINARIESDIRECTORY/Release/_deps/onnx-src\(/.*\)?" -delete
cd $BUILD_BINARIESDIRECTORY/Release
find -executable -type f > $BUILD_BINARIESDIRECTORY/Release/perms.txt

2
tools/ci_build/github/linux/docker/Dockerfile.aten_cpu
View file

@ -1,4 +1,4 @@
FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20240523.1
FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20240530.3
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps_aten.sh && rm -rf /tmp/scripts

160
tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu
View file

@ -1,163 +1,5 @@
ARG BASEIMAGE=registry.access.redhat.com/ubi8/ubi
ARG POLICY=manylinux_2_28
ARG PLATFORM=x86_64
ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-12/root
ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64
ARG PREPEND_PATH=/usr/lib/jvm/msopenjdk-11/bin:${DEVTOOLSET_ROOTPATH}/usr/bin:
FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20240530.3
#Build manylinux2014 docker image begin
FROM $BASEIMAGE AS runtime_base
ARG POLICY
ARG PLATFORM
ARG DEVTOOLSET_ROOTPATH
ARG LD_LIBRARY_PATH_ARG
ARG PREPEND_PATH
LABEL maintainer="The ManyLinux project"
ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG}
ENV PATH=${PREPEND_PATH}${PATH}
ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
# first copy the fixup mirrors script, keep the script around
COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
# setup entrypoint, this will wrap commands with `linux32` with i686 images
COPY build_scripts/install-entrypoint.sh \
build_scripts/build_utils.sh \
/build_scripts/
RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts
COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
ENTRYPOINT ["manylinux-entrypoint"]
COPY build_scripts/install-runtime-packages.sh \
build_scripts/build_utils.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/
COPY build_scripts/build_utils.sh /build_scripts/
COPY build_scripts/install-autoconf.sh /build_scripts/
RUN export AUTOCONF_ROOT=autoconf-2.71 && \
export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \
export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \
manylinux-entrypoint /build_scripts/install-autoconf.sh
COPY build_scripts/install-automake.sh /build_scripts/
RUN export AUTOMAKE_ROOT=automake-1.16.5 && \
export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \
export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \
manylinux-entrypoint /build_scripts/install-automake.sh
COPY build_scripts/install-libtool.sh /build_scripts/
RUN export LIBTOOL_ROOT=libtool-2.4.7 && \
export LIBTOOL_HASH=04e96c2404ea70c590c546eba4202a4e12722c640016c12b9b2f1ce3d481e9a8 && \
export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \
manylinux-entrypoint /build_scripts/install-libtool.sh
COPY build_scripts/install-libxcrypt.sh /build_scripts/
RUN export LIBXCRYPT_VERSION=4.4.28 && \
export LIBXCRYPT_HASH=db7e37901969cb1d1e8020cb73a991ef81e48e31ea5b76a101862c806426b457 && \
export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \
export PERL_ROOT=perl-5.34.0 && \
export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \
export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \
manylinux-entrypoint /build_scripts/install-libxcrypt.sh
FROM runtime_base AS build_base
COPY build_scripts/install-build-packages.sh /build_scripts/
RUN manylinux-entrypoint /build_scripts/install-build-packages.sh
FROM build_base AS build_git
COPY build_scripts/build-git.sh /build_scripts/
RUN export GIT_ROOT=git-2.36.2 && \
export GIT_HASH=6dc2cdea5fb23d823ba4871cc23222c1db31dfbb6d6c6ff74c4128700df57c68 && \
export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \
manylinux-entrypoint /build_scripts/build-git.sh
FROM build_base AS build_cpython
COPY build_scripts/build-sqlite3.sh /build_scripts/
RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3390200 && \
export SQLITE_AUTOCONF_HASH=852be8a6183a17ba47cee0bbff7400b7aa5affd283bf3beefc34fcd088a239de && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \
manylinux-entrypoint /build_scripts/build-sqlite3.sh
COPY build_scripts/build-openssl.sh /build_scripts/
RUN export OPENSSL_ROOT=openssl-1.1.1q && \
export OPENSSL_HASH=d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca && \
export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
manylinux-entrypoint /build_scripts/build-openssl.sh
COPY build_scripts/build-cpython.sh /build_scripts/
FROM build_cpython AS build_cpython38
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.13
FROM build_cpython AS build_cpython39
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.13
FROM build_cpython AS build_cpython310
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.5
FROM build_cpython AS build_cpython311
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.11.2
FROM build_cpython AS build_cpython312
COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.1
FROM build_cpython AS all_python
COPY build_scripts/install-pypy.sh \
build_scripts/pypy.sha256 \
build_scripts/finalize-python.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.9
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.9
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
COPY --from=build_cpython310 /opt/_internal /opt/_internal/
COPY --from=build_cpython311 /opt/_internal /opt/_internal/
COPY --from=build_cpython312 /opt/_internal /opt/_internal/
RUN manylinux-entrypoint /build_scripts/finalize-python.sh
FROM runtime_base
COPY --from=build_git /manylinux-rootfs /
COPY --from=build_cpython /manylinux-rootfs /
COPY --from=all_python /opt/_internal /opt/_internal/
COPY build_scripts/finalize.sh \
build_scripts/python-tag-abi-tag.py \
build_scripts/requirements3.8.txt \
build_scripts/requirements3.9.txt \
build_scripts/requirements3.10.txt \
build_scripts/requirements3.11.txt \
build_scripts/requirements3.12.txt \
build_scripts/requirements-base-tools.txt \
/build_scripts/
COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
CMD ["/bin/bash"]
#Build manylinux2014 docker image end
ENV PATH ${DEVTOOLSET_ROOTPATH}/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV JAVA_HOME=/usr/lib/jvm/msopenjdk-11
ADD scripts /tmp/scripts

155
tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
View file

@ -2,163 +2,10 @@
# Please overwrite BASEIMAGE, TRT_VERSION and other arguments with
# --docker-build-args ' --build-arg BASEIMAGE=other_base_image --build-arg TRT_VERSION=other_trt_version etc...'
# for other cuda version and TRT version
ARG POLICY=manylinux_2_28
ARG PLATFORM=x86_64
ARG BASEIMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubi8
ARG DEVTOOLSET_ROOTPATH=/usr
ARG LD_LIBRARY_PATH_ARG=/usr/local/lib64
ARG PREPEND_PATH=/usr/local/cuda/bin
ARG TRT_VERSION=10.0.1.6-1.cuda11.8
#Build manylinux docker image begin
FROM $BASEIMAGE AS runtime_base
ARG POLICY
ARG PLATFORM
ARG DEVTOOLSET_ROOTPATH
ARG LD_LIBRARY_PATH_ARG
ARG PREPEND_PATH
FROM $BASEIMAGE
ARG TRT_VERSION
LABEL maintainer="The ManyLinux project"
ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG}
ENV PATH=${PREPEND_PATH}${PATH}
ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
# first copy the fixup mirrors script, keep the script around
COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
# setup entrypoint, this will wrap commands with `linux32` with i686 images
COPY build_scripts/install-entrypoint.sh \
build_scripts/build_utils.sh \
/build_scripts/
RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts
COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
ENTRYPOINT ["manylinux-entrypoint"]
COPY build_scripts/install-runtime-packages.sh \
build_scripts/build_utils.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/
COPY build_scripts/build_utils.sh /build_scripts/
COPY build_scripts/install-autoconf.sh /build_scripts/
RUN export AUTOCONF_ROOT=autoconf-2.71 && \
export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \
export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \
manylinux-entrypoint /build_scripts/install-autoconf.sh
COPY build_scripts/install-automake.sh /build_scripts/
RUN export AUTOMAKE_ROOT=automake-1.16.5 && \
export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \
export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \
manylinux-entrypoint /build_scripts/install-automake.sh
COPY build_scripts/install-libtool.sh /build_scripts/
RUN export LIBTOOL_ROOT=libtool-2.4.7 && \
export LIBTOOL_HASH=04e96c2404ea70c590c546eba4202a4e12722c640016c12b9b2f1ce3d481e9a8 && \
export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \
manylinux-entrypoint /build_scripts/install-libtool.sh
COPY build_scripts/install-libxcrypt.sh /build_scripts/
RUN export LIBXCRYPT_VERSION=4.4.28 && \
export LIBXCRYPT_HASH=db7e37901969cb1d1e8020cb73a991ef81e48e31ea5b76a101862c806426b457 && \
export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \
export PERL_ROOT=perl-5.34.0 && \
export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \
export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \
manylinux-entrypoint /build_scripts/install-libxcrypt.sh
FROM runtime_base AS build_base
COPY build_scripts/install-build-packages.sh /build_scripts/
RUN manylinux-entrypoint /build_scripts/install-build-packages.sh
FROM build_base AS build_git
COPY build_scripts/build-git.sh /build_scripts/
RUN export GIT_ROOT=git-2.36.2 && \
export GIT_HASH=6dc2cdea5fb23d823ba4871cc23222c1db31dfbb6d6c6ff74c4128700df57c68 && \
export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \
manylinux-entrypoint /build_scripts/build-git.sh
FROM build_base AS build_cpython
COPY build_scripts/build-sqlite3.sh /build_scripts/
RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3390200 && \
export SQLITE_AUTOCONF_HASH=852be8a6183a17ba47cee0bbff7400b7aa5affd283bf3beefc34fcd088a239de && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \
manylinux-entrypoint /build_scripts/build-sqlite3.sh
COPY build_scripts/build-openssl.sh /build_scripts/
RUN export OPENSSL_ROOT=openssl-1.1.1q && \
export OPENSSL_HASH=d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca && \
export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
manylinux-entrypoint /build_scripts/build-openssl.sh
COPY build_scripts/build-cpython.sh /build_scripts/
FROM build_cpython AS build_cpython38
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.13
FROM build_cpython AS build_cpython39
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.13
FROM build_cpython AS build_cpython310
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.5
FROM build_cpython AS build_cpython311
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.11.2
FROM build_cpython AS build_cpython312
COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.1
FROM build_cpython AS all_python
COPY build_scripts/install-pypy.sh \
build_scripts/pypy.sha256 \
build_scripts/finalize-python.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.9
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.9
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
COPY --from=build_cpython310 /opt/_internal /opt/_internal/
COPY --from=build_cpython311 /opt/_internal /opt/_internal/
COPY --from=build_cpython312 /opt/_internal /opt/_internal/
RUN manylinux-entrypoint /build_scripts/finalize-python.sh
FROM runtime_base
COPY --from=build_git /manylinux-rootfs /
COPY --from=build_cpython /manylinux-rootfs /
COPY --from=all_python /opt/_internal /opt/_internal/
COPY build_scripts/finalize.sh \
build_scripts/python-tag-abi-tag.py \
build_scripts/requirements3.8.txt \
build_scripts/requirements3.9.txt \
build_scripts/requirements3.10.txt \
build_scripts/requirements3.11.txt \
build_scripts/requirements3.12.txt \
build_scripts/requirements-base-tools.txt \
/build_scripts/
COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
CMD ["/bin/bash"]
#Build manylinux docker image end
#Install TensorRT only if TRT_VERSION is not empty
RUN if [ -n "$TRT_VERSION" ]; then \

162
tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda11_8
View file

@ -1,164 +1,4 @@
ARG BASEIMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubi8
ARG POLICY=manylinux2014
ARG PLATFORM=x86_64
ARG DEVTOOLSET_ROOTPATH=
ARG LD_LIBRARY_PATH_ARG=
ARG PREPEND_PATH=
#We need both CUDA and manylinux. But the CUDA Toolkit End User License Agreement says NVIDIA CUDA Driver Libraries(libcuda.so, libnvidia-ptxjitcompiler.so) are only distributable in applications that meet this criteria:
#1. The application was developed starting from a NVIDIA CUDA container obtained from Docker Hub or the NVIDIA GPU Cloud, and
#2. The resulting application is packaged as a Docker container and distributed to users on Docker Hub or the NVIDIA GPU Cloud only.
#So we use CUDA as the base image then add manylinux on top of it.
#Build manylinux2014 docker image begin
FROM $BASEIMAGE AS runtime_base
ARG POLICY
ARG PLATFORM
ARG DEVTOOLSET_ROOTPATH
ARG LD_LIBRARY_PATH_ARG
ARG PREPEND_PATH
LABEL maintainer="The ManyLinux project"
ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG}
ENV PATH=${PREPEND_PATH}${PATH}
ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
# first copy the fixup mirrors script, keep the script around
COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
# setup entrypoint, this will wrap commands with `linux32` with i686 images
COPY build_scripts/install-entrypoint.sh \
build_scripts/build_utils.sh \
/build_scripts/
RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts
COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
ENTRYPOINT ["manylinux-entrypoint"]
COPY build_scripts/install-runtime-packages.sh \
build_scripts/build_utils.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/
COPY build_scripts/build_utils.sh /build_scripts/
COPY build_scripts/install-autoconf.sh /build_scripts/
RUN export AUTOCONF_ROOT=autoconf-2.71 && \
export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \
export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \
manylinux-entrypoint /build_scripts/install-autoconf.sh
COPY build_scripts/install-automake.sh /build_scripts/
RUN export AUTOMAKE_ROOT=automake-1.16.5 && \
export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \
export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \
manylinux-entrypoint /build_scripts/install-automake.sh
COPY build_scripts/install-libtool.sh /build_scripts/
RUN export LIBTOOL_ROOT=libtool-2.4.7 && \
export LIBTOOL_HASH=04e96c2404ea70c590c546eba4202a4e12722c640016c12b9b2f1ce3d481e9a8 && \
export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \
manylinux-entrypoint /build_scripts/install-libtool.sh
COPY build_scripts/install-libxcrypt.sh /build_scripts/
RUN export LIBXCRYPT_VERSION=4.4.28 && \
export LIBXCRYPT_HASH=db7e37901969cb1d1e8020cb73a991ef81e48e31ea5b76a101862c806426b457 && \
export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \
export PERL_ROOT=perl-5.34.0 && \
export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \
export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \
manylinux-entrypoint /build_scripts/install-libxcrypt.sh
FROM runtime_base AS build_base
COPY build_scripts/install-build-packages.sh /build_scripts/
RUN manylinux-entrypoint /build_scripts/install-build-packages.sh
FROM build_base AS build_git
COPY build_scripts/build-git.sh /build_scripts/
RUN export GIT_ROOT=git-2.36.2 && \
export GIT_HASH=6dc2cdea5fb23d823ba4871cc23222c1db31dfbb6d6c6ff74c4128700df57c68 && \
export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \
manylinux-entrypoint /build_scripts/build-git.sh
FROM build_base AS build_cpython
COPY build_scripts/build-sqlite3.sh /build_scripts/
RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3390200 && \
export SQLITE_AUTOCONF_HASH=852be8a6183a17ba47cee0bbff7400b7aa5affd283bf3beefc34fcd088a239de && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \
manylinux-entrypoint /build_scripts/build-sqlite3.sh
COPY build_scripts/build-openssl.sh /build_scripts/
RUN export OPENSSL_ROOT=openssl-1.1.1q && \
export OPENSSL_HASH=d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca && \
export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
manylinux-entrypoint /build_scripts/build-openssl.sh
COPY build_scripts/build-cpython.sh /build_scripts/
FROM build_cpython AS build_cpython38
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.13
FROM build_cpython AS build_cpython39
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.13
FROM build_cpython AS build_cpython310
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.5
FROM build_cpython AS build_cpython311
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.11.2
FROM build_cpython AS build_cpython312
COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.1
FROM build_cpython AS all_python
COPY build_scripts/install-pypy.sh \
build_scripts/pypy.sha256 \
build_scripts/finalize-python.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.9
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.9
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
COPY --from=build_cpython310 /opt/_internal /opt/_internal/
COPY --from=build_cpython311 /opt/_internal /opt/_internal/
COPY --from=build_cpython312 /opt/_internal /opt/_internal/
RUN manylinux-entrypoint /build_scripts/finalize-python.sh
FROM runtime_base
COPY --from=build_git /manylinux-rootfs /
COPY --from=build_cpython /manylinux-rootfs /
COPY --from=all_python /opt/_internal /opt/_internal/
COPY build_scripts/finalize.sh \
build_scripts/python-tag-abi-tag.py \
build_scripts/requirements3.8.txt \
build_scripts/requirements3.9.txt \
build_scripts/requirements3.10.txt \
build_scripts/requirements3.11.txt \
build_scripts/requirements3.12.txt \
build_scripts/requirements-base-tools.txt \
/build_scripts/
COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
CMD ["/bin/bash"]
#Build manylinux2014 docker image end
FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240530.3
ARG PYTHON_VERSION=3.9
ARG TORCH_VERSION=2.0.0
ARG OPSET_VERSION=17

161
tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda12_2
View file

@ -1,164 +1,5 @@
ARG BASEIMAGE=nvidia/cuda:12.2.2-cudnn8-devel-ubi8
ARG POLICY=manylinux2014
ARG PLATFORM=x86_64
ARG DEVTOOLSET_ROOTPATH=
ARG LD_LIBRARY_PATH_ARG=
ARG PREPEND_PATH=
FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_2_x64_ubi8_gcc12:20240530.3
#We need both CUDA and manylinux. But the CUDA Toolkit End User License Agreement says NVIDIA CUDA Driver Libraries(libcuda.so, libnvidia-ptxjitcompiler.so) are only distributable in applications that meet this criteria:
#1. The application was developed starting from a NVIDIA CUDA container obtained from Docker Hub or the NVIDIA GPU Cloud, and
#2. The resulting application is packaged as a Docker container and distributed to users on Docker Hub or the NVIDIA GPU Cloud only.
#So we use CUDA as the base image then add manylinux on top of it.
#Build manylinux2014 docker image begin
FROM $BASEIMAGE AS runtime_base
ARG POLICY
ARG PLATFORM
ARG DEVTOOLSET_ROOTPATH
ARG LD_LIBRARY_PATH_ARG
ARG PREPEND_PATH
LABEL maintainer="The ManyLinux project"
ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG}
ENV PATH=${PREPEND_PATH}${PATH}
ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
# first copy the fixup mirrors script, keep the script around
COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
# setup entrypoint, this will wrap commands with `linux32` with i686 images
COPY build_scripts/install-entrypoint.sh \
build_scripts/build_utils.sh \
/build_scripts/
RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts
COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
ENTRYPOINT ["manylinux-entrypoint"]
COPY build_scripts/install-runtime-packages.sh \
build_scripts/build_utils.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/
COPY build_scripts/build_utils.sh /build_scripts/
COPY build_scripts/install-autoconf.sh /build_scripts/
RUN export AUTOCONF_ROOT=autoconf-2.71 && \
export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \
export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \
manylinux-entrypoint /build_scripts/install-autoconf.sh
COPY build_scripts/install-automake.sh /build_scripts/
RUN export AUTOMAKE_ROOT=automake-1.16.5 && \
export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \
export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \
manylinux-entrypoint /build_scripts/install-automake.sh
COPY build_scripts/install-libtool.sh /build_scripts/
RUN export LIBTOOL_ROOT=libtool-2.4.7 && \
export LIBTOOL_HASH=04e96c2404ea70c590c546eba4202a4e12722c640016c12b9b2f1ce3d481e9a8 && \
export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \
manylinux-entrypoint /build_scripts/install-libtool.sh
COPY build_scripts/install-libxcrypt.sh /build_scripts/
RUN export LIBXCRYPT_VERSION=4.4.28 && \
export LIBXCRYPT_HASH=db7e37901969cb1d1e8020cb73a991ef81e48e31ea5b76a101862c806426b457 && \
export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \
export PERL_ROOT=perl-5.34.0 && \
export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \
export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \
manylinux-entrypoint /build_scripts/install-libxcrypt.sh
FROM runtime_base AS build_base
COPY build_scripts/install-build-packages.sh /build_scripts/
RUN manylinux-entrypoint /build_scripts/install-build-packages.sh
FROM build_base AS build_git
COPY build_scripts/build-git.sh /build_scripts/
RUN export GIT_ROOT=git-2.36.2 && \
export GIT_HASH=6dc2cdea5fb23d823ba4871cc23222c1db31dfbb6d6c6ff74c4128700df57c68 && \
export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \
manylinux-entrypoint /build_scripts/build-git.sh
FROM build_base AS build_cpython
COPY build_scripts/build-sqlite3.sh /build_scripts/
RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3390200 && \
export SQLITE_AUTOCONF_HASH=852be8a6183a17ba47cee0bbff7400b7aa5affd283bf3beefc34fcd088a239de && \
export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \
manylinux-entrypoint /build_scripts/build-sqlite3.sh
COPY build_scripts/build-openssl.sh /build_scripts/
RUN export OPENSSL_ROOT=openssl-1.1.1q && \
export OPENSSL_HASH=d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca && \
export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
manylinux-entrypoint /build_scripts/build-openssl.sh
COPY build_scripts/build-cpython.sh /build_scripts/
FROM build_cpython AS build_cpython38
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.13
FROM build_cpython AS build_cpython39
COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.13
FROM build_cpython AS build_cpython310
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.5
FROM build_cpython AS build_cpython311
COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.11.2
FROM build_cpython AS build_cpython312
COPY build_scripts/cpython-pubkey-312-313.txt /build_scripts/cpython-pubkeys.txt
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.12.1
FROM build_cpython AS all_python
COPY build_scripts/install-pypy.sh \
build_scripts/pypy.sha256 \
build_scripts/finalize-python.sh \
/build_scripts/
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.9
RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.9
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
COPY --from=build_cpython310 /opt/_internal /opt/_internal/
COPY --from=build_cpython311 /opt/_internal /opt/_internal/
COPY --from=build_cpython312 /opt/_internal /opt/_internal/
RUN manylinux-entrypoint /build_scripts/finalize-python.sh
FROM runtime_base
COPY --from=build_git /manylinux-rootfs /
COPY --from=build_cpython /manylinux-rootfs /
COPY --from=all_python /opt/_internal /opt/_internal/
COPY build_scripts/finalize.sh \
build_scripts/python-tag-abi-tag.py \
build_scripts/requirements3.8.txt \
build_scripts/requirements3.9.txt \
build_scripts/requirements3.10.txt \
build_scripts/requirements3.11.txt \
build_scripts/requirements3.12.txt \
build_scripts/requirements-base-tools.txt \
/build_scripts/
COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
CMD ["/bin/bash"]
#Build manylinux2014 docker image end
ARG PYTHON_VERSION=3.9
ARG TORCH_VERSION=2.1.0
ARG OPSET_VERSION=17

11
tools/ci_build/github/linux/docker/Dockerfile.package_ubi8_cuda_tensorrt10_0
View file

@ -9,20 +9,11 @@ ARG BASEIMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubi8
ARG TRT_VERSION=10.0.1.6-1.cuda11.8
FROM $BASEIMAGE AS base
ARG TRT_VERSION
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/src/tensorrt/bin:${PATH}
ENV PATH /opt/python/cp38-cp38/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/src/tensorrt/bin:${PATH}
RUN dnf install -y bash wget &&\
dnf clean dbcache
# Install python3
RUN dnf install -y \
python3.8 \
python38-pip \
python38-wheel &&\
cd /usr/local/bin &&\
ln -s /usr/bin/python3 python3.8 &&\
ln -s /usr/bin/pip3 pip3.8;
RUN pip3 install --upgrade pip
RUN pip3 install setuptools>=68.2.2

2
tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/Dockerfile
View file

@ -1,4 +1,4 @@
FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_aarch64_ubi8_gcc12:20240523.1
FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_aarch64_ubi8_gcc12:20240530.3
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts

10
tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh
View file

@ -1,9 +1,9 @@
!/bin/bash
set -e -x
os_major_version=$(tr -dc '0-9.' < /etc/redhat-release |cut -d \. -f1)
echo "installing for CentOS version : $os_major_version"
rpm -Uvh https://packages.microsoft.com/config/centos/$os_major_version/packages-microsoft-prod.rpm
if [ ! -f /etc/yum.repos.d/microsoft-prod.repo ]; then
os_major_version=$(tr -dc '0-9.' < /etc/redhat-release |cut -d \. -f1)
echo "installing for CentOS version : $os_major_version"
rpm -Uvh https://packages.microsoft.com/config/centos/$os_major_version/packages-microsoft-prod.rpm
fi
dnf install -y python39-devel glibc-langpack-\* glibc-locale-source which redhat-lsb-core expat-devel tar unzip zlib-devel make bzip2 bzip2-devel msopenjdk-11 graphviz gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran gcc-toolset-12-libasan-devel libasan.x86_64
locale

27
tools/ci_build/github/linux/docker/inference/x86_64/default/gpu/Dockerfile
View file

@ -4,6 +4,33 @@
# This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline
ARG BASEIMAGE=nvidia/cuda:11.8.0-cudnn8-devel-ubi8
FROM $BASEIMAGE
ARG TRT_VERSION
#Install TensorRT only if TRT_VERSION is not empty
RUN if [ -n "$TRT_VERSION" ]; then \
echo "TRT_VERSION is $TRT_VERSION" && \
dnf -y install \
libnvinfer10-${TRT_VERSION} \
libnvinfer-headers-devel-${TRT_VERSION} \
libnvinfer-devel-${TRT_VERSION} \
libnvinfer-lean10-${TRT_VERSION} \
libnvonnxparsers10-${TRT_VERSION} \
libnvonnxparsers-devel-${TRT_VERSION} \
libnvinfer-dispatch10-${TRT_VERSION} \
libnvinfer-plugin10-${TRT_VERSION} \
libnvinfer-vc-plugin10-${TRT_VERSION} \
libnvinfer-bin-${TRT_VERSION} \
libnvinfer-plugin10-${TRT_VERSION} \
libnvinfer-plugin-devel-${TRT_VERSION} \
libnvinfer-vc-plugin-devel-${TRT_VERSION} \
libnvinfer-lean-devel-${TRT_VERSION} \
libnvinfer-dispatch-devel-${TRT_VERSION} \
libnvinfer-headers-plugin-devel-${TRT_VERSION} && \
dnf clean dbcache ; \
else \
echo "TRT_VERSION is none skipping Tensor RT Installation" ; \
fi
ENV PATH /usr/lib/jvm/msopenjdk-11/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV LANG=en_US.UTF-8
ENV LC_ALL=en_US.UTF-8

10
tools/ci_build/github/linux/docker/inference/x86_64/default/gpu/scripts/install_centos.sh
View file

@ -1,9 +1,9 @@
#!/bin/bash
set -e -x
os_major_version=$(tr -dc '0-9.' < /etc/redhat-release |cut -d \. -f1)
echo "installing for CentOS version : $os_major_version"
rpm -Uvh https://packages.microsoft.com/config/centos/$os_major_version/packages-microsoft-prod.rpm
if [ ! -f /etc/yum.repos.d/microsoft-prod.repo ]; then
os_major_version=$(tr -dc '0-9.' < /etc/redhat-release |cut -d \. -f1)
echo "installing for CentOS version : $os_major_version"
rpm -Uvh https://packages.microsoft.com/config/centos/$os_major_version/packages-microsoft-prod.rpm
fi
dnf install -y python39-devel glibc-langpack-\* glibc-locale-source which redhat-lsb-core expat-devel tar unzip zlib-devel make bzip2 bzip2-devel msopenjdk-11
locale

2
tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile
View file

@ -1,4 +1,4 @@
FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20240523.1
FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20240530.3
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts

8
tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh
View file

@ -7,12 +7,10 @@ echo "installing for os major version : $os_major_version"
if [ "$os_major_version" -gt 7 ]; then
PACKAGE_MANAGER="dnf"
$PACKAGE_MANAGER install -y which redhat-lsb-core expat-devel tar unzip zlib-devel make bzip2 bzip2-devel perl-IPC-Cmd openssl-devel wget
else
PACKAGE_MANAGER="yum"
$PACKAGE_MANAGER install -y which redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind bzip2 bzip2-devel perl-IPC-Cmd openssl-devel wget
fi
rpm -Uvh https://packages.microsoft.com/config/centos/$os_major_version/packages-microsoft-prod.rpm
if [ ! -f /etc/yum.repos.d/microsoft-prod.repo ]; then
rpm -Uvh https://packages.microsoft.com/config/centos/$os_major_version/packages-microsoft-prod.rpm
fi
# Install Java
# Install automatic documentation generation dependencies
$PACKAGE_MANAGER install -y msopenjdk-11 graphviz

2
tools/ci_build/github/linux/run_python_dockerbuild.sh
View file

@ -30,6 +30,8 @@ docker run --rm \
-w /onnxruntime_src \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
-e ORT_DISABLE_PYTHON_PACKAGE_LOCAL_VERSION \
-e DEFAULT_TRAINING_PACKAGE_DEVICE \
$ADDITIONAL_DOCKER_PARAMETER \
$DOCKER_IMAGE tools/ci_build/github/linux/build_linux_python_package.sh $DOCKER_SCRIPT_OPTIONS

4
tools/ci_build/github/linux/run_python_tests.sh
View file

@ -26,6 +26,10 @@ echo "Package name:$PYTHON_PACKAGE_NAME"
BUILD_ARGS="--build_dir /build --config $BUILD_CONFIG --test --skip_submodule_sync --parallel --enable_lto --build_wheel "
if [[ "$PYTHON_PACKAGE_NAME" == *"training"* ]]; then
BUILD_ARGS="$BUILD_ARGS --enable_training"
fi
ARCH=$(uname -m)
if [ $ARCH == "x86_64" ]; then