saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update C API pipeline to use CentOS 6 (#2198)

Browse source
This commit is contained in:
Changming Sun 2019-10-19 22:25:42 -07:00 committed by GitHub
parent 0dd781fd57
commit cff7879d89
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 56 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmake/CMakeLists.txt
View file

@ -89,7 +89,9 @@ set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
#nsync tests failed on Mac Build
set(NSYNC_ENABLE_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE)
set(ONNX_ML 1)
if(NOT onnxruntime_ENABLE_PYTHON)
set(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS OFF)
endif()
if(onnxruntime_USE_OPENMP)
find_package(OpenMP)
if (OPENMP_FOUND)

44
tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml
View file

@ -1,29 +1,42 @@
jobs:
- job: Linux_C_API_Packaging_CPU_x64
pool: Linux-CPU
timeoutInMinutes: 60
pool:
vmImage: 'ubuntu-latest'
steps:
- template: templates/linux-set-variables-and-download.yml
- template: templates/set-version-number-variables-step.yml
- script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory)'
displayName: 'Build and Test Linux on Docker'
- task: CmdLine@2
inputs:
script: |
docker build -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl
workingDirectory: $(Build.SourcesDirectory)
- template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
buildConfig: 'Release'
artifactName: 'onnxruntime-linux-x64-$(OnnxRuntimeVersion)'
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
commitId: $(OnnxRuntimeGitCommitHash)
- template: templates/clean-agent-build-directory-step.yml
- job: Linux_C_API_Packaging_GPU_x64
pool: Linux-GPU-CUDA10
timeoutInMinutes: 60
pool: 'Linux-GPU-CUDA10'
steps:
- template: templates/linux-set-variables-and-download.yml
- template: templates/set-version-number-variables-step.yml
- script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory)'
displayName: 'Build and Test Linux on Docker'
- task: CmdLine@2
inputs:
script: |
docker build -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu .
workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_automl --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0
workingDirectory: $(Build.SourcesDirectory)
- template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
buildConfig: 'Release'
@ -42,6 +55,7 @@ jobs:
- template: templates/set-version-number-variables-step.yml
- script: |
sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config RelWithDebInfo --enable_onnx_tests
displayName: 'Build and Test MacOS'
@ -140,7 +154,7 @@ jobs:
displayName: 'Build and Test OnnxRuntime'
inputs:
script: |
$(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --use_openmp --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10_trt6015dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"
$(Build.BinariesDirectory)\packages\python\python.exe $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(buildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --enable_onnx_tests --use_cuda --cuda_version=10.0 --cuda_home="C:\local\cuda_10.0.130_win10_trt6015dll" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"
workingDirectory: '$(Build.BinariesDirectory)'
- template: templates/c-api-artifacts-package-and-publish-steps-windows.yml
@ -149,4 +163,4 @@ jobs:
artifactName: 'onnxruntime-win-$(buildArch)-gpu-$(OnnxRuntimeVersion)'
commitId: $(OnnxRuntimeGitCommitHash)
- template: templates/clean-agent-build-directory-step.yml
- template: templates/clean-agent-build-directory-step.yml

12
tools/ci_build/github/linux/docker/Dockerfile.centos6 Normal file
View file

@ -0,0 +1,12 @@
FROM mcr.microsoft.com/dotnet-buildtools/prereqs:centos-6-50f0d02-20190918213956
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh
ENV PATH /opt/rh/devtoolset-2/root/usr/bin:${PATH}
RUN /tmp/scripts/install_deps.sh -p 3.6 && rm -rf /tmp/scripts
ARG BUILD_UID=1000
ARG BUILD_USER=onnxruntimedev
RUN adduser --uid $BUILD_UID $BUILD_USER
WORKDIR /home/$BUILD_USER
USER $BUILD_USER

16
tools/ci_build/github/linux/docker/Dockerfile.centos6_gpu
View file

@ -1,23 +1,20 @@
# FROM mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-50f0d02-20190918214028
FROM centos:6
FROM mcr.microsoft.com/dotnet-buildtools/prereqs:centos-6-50f0d02-20190918213956
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ARG PYTHON_VERSION
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:/bin:/usr/bin:/usr/local/bin:/opt/rh/devtoolset-7/root/usr/bin:{PATH}
RUN /tmp/scripts/install_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh
ENV PATH /opt/rh/devtoolset-2/root/usr/bin:${PATH}
RUN /tmp/scripts/install_deps.sh -p 3.6 && rm -rf /tmp/scripts
#Below are copied from https://gitlab.com/nvidia/container-images/cuda/tree/master/dist/centos6
RUN NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel6/x86_64/7fa2af80.pub | sed '/^Version/d' > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel6/x86_64/7fa2af80.pub | sed '/^Version/d' > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
echo "$NVIDIA_GPGKEY_SUM /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA" | sha256sum -c -
COPY cuda_manylinux2010.repo /etc/yum.repos.d/cuda.repo
COPY cuda_manylinux2010.repo /etc/yum.repos.d/cuda.repo
ENV CUDA_VERSION 10.0.130
@ -67,4 +64,3 @@ ARG BUILD_USER=onnxruntimedev
RUN /usr/sbin/adduser --uid $BUILD_UID $BUILD_USER
WORKDIR /home/$BUILD_USER
USER $BUILD_USER

15
tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu
View file

@ -20,21 +20,6 @@ RUN _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) && \
ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so && \
ln -s /usr/local/cudnn{-$_CUDNN_VERSION,}
# Build and Install LLVM
ARG LLVM_VERSION=6.0.1
RUN cd /tmp && \
wget --no-verbose http://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz && \
xz -d llvm-$LLVM_VERSION.src.tar.xz && \
tar xvf llvm-$LLVM_VERSION.src.tar && \
cd llvm-$LLVM_VERSION.src && \
mkdir -p build && \
cd build && \
cmake .. -DCMAKE_BUILD_TYPE=Release && \
cmake --build . -- -j$(nproc) && \
cmake -DCMAKE_INSTALL_PREFIX=/usr/local/llvm-$LLVM_VERSION -DBUILD_TYPE=Release -P cmake_install.cmake && \
cd /tmp && \
rm -rf llvm*
ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
ARG BUILD_USER=onnxruntimedev

20
tools/ci_build/github/linux/docker/scripts/install_centos.sh
View file

@ -11,29 +11,21 @@ fi
echo "installing for os major version : $os_major_version"
if [ "$os_major_version" == "5" ]; then
#Be careful, don't pull gcc into the base system, because we already have one in /opt/rh/devtoolset-2/root/usr/bin
yum install -y redhat-lsb expat-devel libcurl-devel tar unzip curl zlib-devel make python2-devel icu rsync bzip2 git bzip2-devel
yum install -y redhat-lsb expat-devel libcurl-devel tar unzip curl zlib-devel make icu rsync bzip2 git bzip2-devel
elif [ "$os_major_version" == "6" ] && [ ! -d "/opt/python/cp35-cp35m" ]; then
yum install -y centos-release-scl
yum repolist
#The base image we are using already contains devtoolset-2
yum install -y redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel
yum upgrade -y
yum install -y \
ccache \
devtoolset-7-binutils \
devtoolset-7-gcc \
devtoolset-7-gcc-c++ \
devtoolset-7-gcc-gfortran
# The way to get python 3.6.8
#Install python 3.6
yum install -y https://centos6.iuscommunity.org/ius-release.rpm
yum --enablerepo=ius install -y python36u python36u-devel python36u-pip python36u-numpy python36u-setuptools python36u-wheel protobuf
yum --enablerepo=ius install -y python36u python36u-devel python36u-pip python36u-numpy python36u-setuptools python36u-wheel
/usr/bin/python3.6 -m pip install --upgrade pip
else
yum install -y redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make python2-devel libunwind icu aria2 rsync bzip2 git bzip2-devel
yum install -y redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel
fi
#If the /opt/python folder exists, we assume this is the manylinux docker image
if [ "$os_major_version" != "6" ] && [ ! -d "/opt/python/cp35-cp35m" ]
then
yum install -y ccache gcc gcc-c++ python3 python3-devel python3-pip python3-numpy python3-setuptools python3-wheel
yum install -y ccache gcc gcc-c++ python3 python3-devel python3-pip
fi