Delete docker/scripts/install_deps_x86.sh and enable onnx tests for x86 (#1191)

This commit is contained in:
Changming Sun 2019-06-08 16:17:18 -07:00 committed by GitHub
parent 6b586bc041
commit be36385a8c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 73 additions and 188 deletions

View file

@ -1,6 +1,5 @@
ARG OS_VERSION=27
FROM fedora:${OS_VERSION}
ENV PATH="/opt/cmake/bin:${PATH}"
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_fedora.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts

View file

@ -4,7 +4,7 @@ FROM fedora:${OS_VERSION}
ENV NVIDIA_VISIBLE_DEVICES=all \
NVIDIA_DRIVER_CAPABILITIES=compute,utility \
NVIDIA_REQUIRE_CUDA="cuda>=9.2" \
PATH="/usr/lib64/ccache:/opt/cmake/bin:${PATH}"
PATH="/usr/lib64/ccache:${PATH}"
ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_fedora_gpu.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts

View file

@ -4,7 +4,6 @@ FROM ubuntu:${OS_VERSION}
ARG PYTHON_VERSION=3.5
ADD scripts /tmp/scripts
ENV PATH="/opt/cmake/bin:${PATH}"
RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
WORKDIR /root
@ -13,6 +12,6 @@ ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
ARG BUILD_UID=1000
ARG BUILD_USER=onnxruntimedev
WORKDIR /home/$BUILD_USER
RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID
WORKDIR /home/$BUILD_USER
USER $BUILD_USER

View file

@ -2,10 +2,8 @@ ARG OS_VERSION=16.04
FROM ubuntu:${OS_VERSION}
ARG PYTHON_VERSION=3.5
ADD scripts /tmp/scripts
ENV PATH="/opt/cmake/bin:${PATH}"
RUN /tmp/scripts/install_ubuntu_for_android.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps_android.sh && rm -rf /tmp/scripts
RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps_android.sh && /tmp/scripts/install_protobuf.sh && rm -rf /tmp/scripts
WORKDIR /root
@ -13,6 +11,6 @@ ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
ARG BUILD_UID=1000
ARG BUILD_USER=onnxruntimedev
WORKDIR /home/$BUILD_USER
RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID
WORKDIR /home/$BUILD_USER
USER $BUILD_USER

View file

@ -7,7 +7,6 @@ FROM nvidia/cuda@sha256:362e4e25aa46a18dfa834360140e91b61cdb0a3a2796c8e09dadb268
ARG PYTHON_VERSION=3.5
ADD scripts /tmp/scripts
ENV PATH="/opt/cmake/bin:${PATH}"
RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
WORKDIR /root

View file

@ -7,7 +7,6 @@ FROM nvcr.io/nvidia/tensorrt:19.02-py3
ARG PYTHON_VERSION=3.5
ADD scripts /tmp/scripts
ENV PATH="/opt/cmake/bin:${PATH}"
RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
WORKDIR /root

View file

@ -4,15 +4,14 @@ FROM i386/ubuntu:${OS_VERSION}
ARG PYTHON_VERSION=3.5
ADD scripts /tmp/scripts
ENV PATH="/opt/cmake/bin:${PATH}"
RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps_x86.sh && rm -rf /tmp/scripts
RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
WORKDIR /root
ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
ARG BUILD_UID=1000
ARG BUILD_USER=onnxruntimedev
WORKDIR /home/$BUILD_USER
RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID
WORKDIR /home/$BUILD_USER
USER $BUILD_USER

View file

@ -1,27 +1,25 @@
#!/bin/bash
set -e
#install protobuf
mkdir -p /tmp/src
mkdir -p /opt/cmake
aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2-Linux-x86_64.tar.gz
tar -xf /tmp/src/cmake-3.13.2-Linux-x86_64.tar.gz --strip 1 -C /opt/cmake
aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz
tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src
cd /tmp/src/protobuf-3.6.1
if [ -f /etc/redhat-release ] ; then
PB_LIBDIR=lib64
else
PB_LIBDIR=lib
SYS_LONG_BIT=$(getconf LONG_BIT)
echo "Installing azcopy"
if [ $SYS_LONG_BIT = "64" ]; then
mkdir -p /tmp/azcopy
aria2c -q -d /tmp/azcopy -o azcopy.tar.gz https://aka.ms/downloadazcopy-v10-linux
tar --strip 1 -xf /tmp/azcopy/azcopy.tar.gz -C /tmp/azcopy
cp /tmp/azcopy/azcopy /usr/bin
fi
for build_type in 'Debug' 'Relwithdebinfo'; do
pushd .
mkdir build_$build_type
cd build_$build_type
/opt/cmake/bin/cmake ../cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=$PB_LIBDIR -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$build_type
make -j`nproc`
make install
popd
done
mkdir -p /tmp/src
aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2.tar.gz
tar -xf /tmp/src/cmake-3.13.2.tar.gz -C /tmp/src
cd /tmp/src/cmake-3.13.2
./configure --prefix=/usr --parallel=`nproc` --system-curl --system-zlib --system-expat
make -j`nproc`
make install
#install onnx
export ONNX_ML=1
INSTALLED_PYTHON_VERSION=$(python3 -c 'import sys; version=sys.version_info[:2]; print("{0}.{1}".format(*version));')
if [ "$INSTALLED_PYTHON_VERSION" = "3.7" ];then
@ -30,13 +28,16 @@ fi
if [ "$INSTALLED_PYTHON_VERSION" = "3.4" ];then
echo "Python 3.5 and above is needed for running onnx tests!" 1>&2
else
source /tmp/scripts/install_onnx.sh
source /tmp/scripts/install_onnx.sh $INSTALLED_PYTHON_VERSION
fi
#The last onnx version will be kept
cd /
rm -rf /tmp/src
rm -rf /usr/include/google
rm -rf /usr/lib/libproto*
DISTRIBUTOR=$(lsb_release -i -s)
if [ "$DISTRIBUTOR" = "Ubuntu" ]; then
apt-get -y remove libprotobuf-dev protobuf-compiler
else
dnf remove -y protobuf-devel protobuf-compiler
fi

View file

@ -1,41 +1,19 @@
#!/bin/bash
set -e
mkdir -p /tmp/src
aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2.tar.gz
tar -xf /tmp/src/cmake-3.13.2.tar.gz -C /tmp/src
cd /tmp/src/cmake-3.13.2
./configure --prefix=/usr --parallel=`nproc` --system-curl --system-zlib --system-expat
make -j`nproc`
make install
#download Android NDK r19c
aria2c -q -d /tmp https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip
unzip -oq /tmp/android-ndk-r19c-linux-x86_64.zip -d /tmp/android-ndk && mv /tmp/android-ndk/* /android-ndk
#install ninja
aria2c -q -d /tmp https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
unzip -oq /tmp/ninja-linux.zip -d /usr/bin
rm -f /tmp/ninja-linux.zip
#install protobuf
mkdir -p /tmp/src
mkdir -p /opt/cmake
aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2-Linux-x86_64.tar.gz
tar -xf /tmp/src/cmake-3.13.2-Linux-x86_64.tar.gz --strip 1 -C /opt/cmake
aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz
tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src
cd /tmp/src/protobuf-3.6.1
if [ -f /etc/redhat-release ] ; then
PB_LIBDIR=lib64
else
PB_LIBDIR=lib
fi
for build_type in 'Debug' 'Relwithdebinfo'; do
pushd .
mkdir build_$build_type
cd build_$build_type
/opt/cmake/bin/cmake -G Ninja ../cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=$PB_LIBDIR -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$build_type
ninja
ninja install
popd
done
export ONNX_ML=1
#The last onnx version will be kept
cd /
rm -rf /tmp/src
rm -rf /usr/include/google
rm -rf /usr/lib/libproto*
apt-get -y remove libprotobuf-dev protobuf-compiler

View file

@ -1,38 +0,0 @@
#!/bin/bash
set -e
aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2.tar.gz
tar -xf /tmp/src/cmake-3.13.2.tar.gz -C /tmp/src
cd /tmp/src/cmake-3.13.2
./configure
make
make install
aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz
tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src
cd /tmp/src/protobuf-3.6.1
if [ -f /etc/redhat-release ] ; then
PB_LIBDIR=lib64
else
PB_LIBDIR=lib
fi
for build_type in 'Debug' 'Relwithdebinfo'; do
pushd .
mkdir build_$build_type
cd build_$build_type
cmake ../cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=$PB_LIBDIR -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$build_type
make -j`nproc`
make install
popd
done
export ONNX_ML=1
INSTALLED_PYTHON_VERSION=$(python3 -c 'import sys; version=sys.version_info[:2]; print("{0}.{1}".format(*version));')
if [ "$INSTALLED_PYTHON_VERSION" = "3.7" ];then
pip3 install --upgrade setuptools
else
source /tmp/scripts/install_onnx.sh x86
fi
#The last onnx version will be kept
rm -rf /tmp/src

View file

@ -1,3 +1,3 @@
#!/bin/bash
set -e
dnf install -y --best --allowerasing rpmdevtools tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2
dnf install -y --best --allowerasing redhat-lsb-core expat-devel libcurl-devel protobuf-compiler protobuf-devel protobuf rpmdevtools tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2

View file

@ -1,11 +1,6 @@
#!/bin/bash
set -e
dnf install -y --best --allowerasing compat-openssl10 lttng-ust libcurl openssl-libs krb5-libs libicu tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2
mkdir -p /tmp/azcopy
aria2c -q -d /tmp/azcopy -o azcopy.tar.gz https://aka.ms/downloadazcopy-v10-linux
tar -xf /tmp/azcopy/azcopy.tar.gz --strip 1 -C /tmp/azcopy
/bin/cp /tmp/azcopy/azcopy /usr/bin/azcopy
rm -rf /tmp/azcopy
dnf install -y --best --allowerasing redhat-lsb-core expat-devel libcurl-devel protobuf-compiler protobuf-devel protobuf compat-openssl10 lttng-ust libcurl openssl-libs krb5-libs libicu tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2
#install cuda 9.2 and cudnn 7
aria2c -q -d /tmp 'http://developer.download.nvidia.com/compute/cuda/repos/fedora27/x86_64/cuda-repo-fedora27-10.0.130-1.x86_64.rpm'
dnf install -y /tmp/cuda-repo-fedora27-10.0.130-1.x86_64.rpm

View file

@ -1,4 +1,7 @@
#!/bin/bash
set -e
PYTHON_VER=$1
#Install ONNX
#5af210ca8a1c73aa6bae8754c9346ec54d0a756e is v1.2.3
#bae6333e149a59a3faa9c4d9c44974373dcf5256 is v1.3.0
@ -16,21 +19,15 @@ for onnx_version in ${!version2tag[@]}; do
echo "first pass";
else
echo "deleting old onnx-${lastest_onnx_version}";
pip3 uninstall -y onnx
/usr/bin/python${PYTHON_VER} -m pip uninstall -y onnx
fi
lastest_onnx_version=$onnx_version
aria2c -q -d /tmp/src https://github.com/onnx/onnx/archive/$onnx_version.tar.gz
tar -xf /tmp/src/onnx-$onnx_version.tar.gz -C /tmp/src
cd /tmp/src/onnx-$onnx_version
git clone https://github.com/pybind/pybind11.git third_party/pybind11
python3 setup.py bdist_wheel
if [ $1 = "x86" ]; then
pip3 install -q onnx
else
pip3 install -q dist/*
fi
/usr/bin/python${PYTHON_VER} -m pip install .
mkdir -p /data/onnx/${version2tag[$onnx_version]}
backend-test-tools generate-data -o /data/onnx/${version2tag[$onnx_version]}
echo $onnx_version":"${version2tag[$onnx_version]} >> /data/onnx/version2tag
done

View file

@ -0,0 +1,8 @@
#!/bin/bash
set -e
aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz
tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src
cd /tmp/src/protobuf-3.6.1
cmake ./cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo
make -j`nproc`
make install

View file

@ -18,14 +18,14 @@ apt-get update && apt-get install -y --no-install-recommends \
autotools-dev \
automake \
build-essential \
git apt-transport-https \
git apt-transport-https apt-utils \
ca-certificates \
pkg-config \
wget \
zlib1g \
zlib1g-dev \
libssl-dev \
curl \
curl libcurl4-openssl-dev \
autoconf \
sudo \
gfortran \
@ -43,13 +43,14 @@ apt-get update && apt-get install -y --no-install-recommends \
bzip2 \
unzip \
zip \
rsync libunwind8 libpng16-dev \
python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest
rsync libunwind8 libpng16-dev libexpat1-dev \
python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \
libprotobuf-dev libprotobuf9v5 protobuf-compiler
locale-gen en_US.UTF-8
update-locale LANG=en_US.UTF-8
#Install dotnet-sdk
echo "Installing dotnet-sdk"
if [ $SYS_LONG_BIT = "64" ]; then
OS_VER=`lsb_release -r -s`
mkdir -p /tmp/dotnet
@ -60,24 +61,19 @@ if [ $SYS_LONG_BIT = "64" ]; then
rm -rf /tmp/dotnet
fi
if [ $PYTHON_VER!="3.5" ]; then
if [ $PYTHON_VER != "3.5" ]; then
apt-get install -y --no-install-recommends \
python${PYTHON_VER} \
python${PYTHON_VER}-dev
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VER} 1
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.5 2
update-alternatives --set python3 /usr/bin/python${PYTHON_VER}
#TODO: the old one(/usr/bin/pip3) should be uninstalled first. Because the one will be
#put at /usr/local/. Then there will be two pips.
/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3
fi
/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3
/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall numpy==1.15.0
/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall requests==2.21.0
rm -rf /var/lib/apt/lists/*
#Install azcopy
if [ $SYS_LONG_BIT = "64" ]; then
mkdir -p /tmp/azcopy
aria2c -q -d /tmp/azcopy -o azcopy.tar.gz https://aka.ms/downloadazcopy-v10-linux
tar --strip 1 -xf /tmp/azcopy/azcopy.tar.gz -C /tmp/azcopy
cp /tmp/azcopy/azcopy /usr/bin
fi

View file

@ -1,45 +0,0 @@
#!/bin/bash
set -e
while getopts p: parameter_Option
do case "${parameter_Option}"
in
p) PYTHON_VER=${OPTARG};;
esac
done
PYTHON_VER=${PYTHON_VER:=3.5}
DEBIAN_FRONTEND=noninteractive
apt-get update && apt-get install -y software-properties-common
add-apt-repository ppa:deadsnakes/ppa
apt-get update && apt-get install -y --no-install-recommends \
autotools-dev \
build-essential \
git apt-transport-https \
ca-certificates \
pkg-config \
wget \
zlib1g \
zlib1g-dev \
libssl-dev \
curl \
autoconf \
sudo \
gfortran \
language-pack-en \
libopenblas-dev \
liblttng-ust0 \
libcurl3 \
libssl1.0.0 \
libkrb5-3 \
libicu55 \
aria2 \
bzip2 \
unzip \
zip \
rsync libunwind8 libpng16-dev
locale-gen en_US.UTF-8
update-locale LANG=en_US.UTF-8
rm -rf /var/lib/apt/lists/*

View file

@ -17,10 +17,10 @@ done
if [ $BUILD_OS = "android" ]; then
pushd /onnxruntime_src
mkdir build-android && cd build-android
/opt/cmake/bin/cmake -DCMAKE_TOOLCHAIN_FILE=/android-ndk/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DONNX_CUSTOM_PROTOC_EXECUTABLE=/usr/bin/protoc ../cmake
/opt/cmake/bin/cmake --build . -- -j$(nproc)
cmake -DCMAKE_TOOLCHAIN_FILE=/android-ndk/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DONNX_CUSTOM_PROTOC_EXECUTABLE=/usr/bin/protoc ../cmake
make -j$(nproc)
else
COMMON_BUILD_ARGS="--skip_submodule_sync --enable_onnx_tests --parallel --build_shared_lib --use_openmp"
COMMON_BUILD_ARGS="--skip_submodule_sync --enable_onnx_tests --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest"
if [ $BUILD_DEVICE = "gpu" ]; then
_CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2)
python3 $SCRIPT_DIR/../../build.py --build_dir /build \

View file

@ -25,7 +25,7 @@ esac
done
EXIT_CODE=1
PYTHON_VER=${PYTHON_VER:=3.5}
echo "bo=$BUILD_OS bd=$BUILD_DEVICE bdir=$BUILD_DIR pv=$PYTHON_VER bex=$BUILD_EXTR_PAR"
cd $SCRIPT_DIR/docker