From be36385a8c5796ecedc7a59f1760b5004e69e819 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 8 Jun 2019 16:17:18 -0700 Subject: [PATCH] Delete docker/scripts/install_deps_x86.sh and enable onnx tests for x86 (#1191) --- .../github/linux/docker/Dockerfile.fedora | 1 - .../github/linux/docker/Dockerfile.fedora_gpu | 2 +- .../github/linux/docker/Dockerfile.ubuntu | 3 +- .../docker/Dockerfile.ubuntu_for_android | 6 +-- .../github/linux/docker/Dockerfile.ubuntu_gpu | 1 - .../linux/docker/Dockerfile.ubuntu_tensorrt | 1 - .../github/linux/docker/Dockerfile.ubuntu_x86 | 5 +- .../linux/docker/scripts/install_deps.sh | 53 ++++++++++--------- .../docker/scripts/install_deps_android.sh | 44 ++++----------- .../linux/docker/scripts/install_deps_x86.sh | 38 ------------- .../linux/docker/scripts/install_fedora.sh | 2 +- .../docker/scripts/install_fedora_gpu.sh | 7 +-- .../linux/docker/scripts/install_onnx.sh | 13 ++--- .../linux/docker/scripts/install_protobuf.sh | 8 +++ .../linux/docker/scripts/install_ubuntu.sh | 24 ++++----- .../scripts/install_ubuntu_for_android.sh | 45 ---------------- tools/ci_build/github/linux/run_build.sh | 6 +-- .../ci_build/github/linux/run_dockerbuild.sh | 2 +- 18 files changed, 73 insertions(+), 188 deletions(-) delete mode 100755 tools/ci_build/github/linux/docker/scripts/install_deps_x86.sh create mode 100755 tools/ci_build/github/linux/docker/scripts/install_protobuf.sh delete mode 100755 tools/ci_build/github/linux/docker/scripts/install_ubuntu_for_android.sh diff --git a/tools/ci_build/github/linux/docker/Dockerfile.fedora b/tools/ci_build/github/linux/docker/Dockerfile.fedora index 30205cbbaa..6ef24e3dc1 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.fedora +++ b/tools/ci_build/github/linux/docker/Dockerfile.fedora @@ -1,6 +1,5 @@ ARG OS_VERSION=27 FROM fedora:${OS_VERSION} -ENV PATH="/opt/cmake/bin:${PATH}" ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_fedora.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts diff --git a/tools/ci_build/github/linux/docker/Dockerfile.fedora_gpu b/tools/ci_build/github/linux/docker/Dockerfile.fedora_gpu index 4523847260..552f70213f 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.fedora_gpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.fedora_gpu @@ -4,7 +4,7 @@ FROM fedora:${OS_VERSION} ENV NVIDIA_VISIBLE_DEVICES=all \ NVIDIA_DRIVER_CAPABILITIES=compute,utility \ NVIDIA_REQUIRE_CUDA="cuda>=9.2" \ - PATH="/usr/lib64/ccache:/opt/cmake/bin:${PATH}" + PATH="/usr/lib64/ccache:${PATH}" ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_fedora_gpu.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu index c4e201106a..bdff95e154 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu @@ -4,7 +4,6 @@ FROM ubuntu:${OS_VERSION} ARG PYTHON_VERSION=3.5 ADD scripts /tmp/scripts -ENV PATH="/opt/cmake/bin:${PATH}" RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts WORKDIR /root @@ -13,6 +12,6 @@ ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH ARG BUILD_UID=1000 ARG BUILD_USER=onnxruntimedev -WORKDIR /home/$BUILD_USER RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID +WORKDIR /home/$BUILD_USER USER $BUILD_USER diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_for_android b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_for_android index 74301e1e75..c7e427b773 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_for_android +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_for_android @@ -2,10 +2,8 @@ ARG OS_VERSION=16.04 FROM ubuntu:${OS_VERSION} ARG PYTHON_VERSION=3.5 - ADD scripts /tmp/scripts -ENV PATH="/opt/cmake/bin:${PATH}" -RUN /tmp/scripts/install_ubuntu_for_android.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps_android.sh && rm -rf /tmp/scripts +RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps_android.sh && /tmp/scripts/install_protobuf.sh && rm -rf /tmp/scripts WORKDIR /root @@ -13,6 +11,6 @@ ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH ARG BUILD_UID=1000 ARG BUILD_USER=onnxruntimedev -WORKDIR /home/$BUILD_USER RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID +WORKDIR /home/$BUILD_USER USER $BUILD_USER diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu index e93847e9f0..df38f70a90 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu @@ -7,7 +7,6 @@ FROM nvidia/cuda@sha256:362e4e25aa46a18dfa834360140e91b61cdb0a3a2796c8e09dadb268 ARG PYTHON_VERSION=3.5 ADD scripts /tmp/scripts -ENV PATH="/opt/cmake/bin:${PATH}" RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts WORKDIR /root diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt index 678b976886..ba17a56f29 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt @@ -7,7 +7,6 @@ FROM nvcr.io/nvidia/tensorrt:19.02-py3 ARG PYTHON_VERSION=3.5 ADD scripts /tmp/scripts -ENV PATH="/opt/cmake/bin:${PATH}" RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts WORKDIR /root diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_x86 b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_x86 index 828f8e7505..235cb8bd88 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_x86 +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_x86 @@ -4,15 +4,14 @@ FROM i386/ubuntu:${OS_VERSION} ARG PYTHON_VERSION=3.5 ADD scripts /tmp/scripts -ENV PATH="/opt/cmake/bin:${PATH}" -RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps_x86.sh && rm -rf /tmp/scripts +RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts WORKDIR /root ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH ARG BUILD_UID=1000 ARG BUILD_USER=onnxruntimedev -WORKDIR /home/$BUILD_USER RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID +WORKDIR /home/$BUILD_USER USER $BUILD_USER diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh index 06a54d74e8..6b72121172 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_deps.sh @@ -1,27 +1,25 @@ #!/bin/bash set -e -#install protobuf -mkdir -p /tmp/src -mkdir -p /opt/cmake -aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2-Linux-x86_64.tar.gz -tar -xf /tmp/src/cmake-3.13.2-Linux-x86_64.tar.gz --strip 1 -C /opt/cmake -aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz -tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src -cd /tmp/src/protobuf-3.6.1 -if [ -f /etc/redhat-release ] ; then - PB_LIBDIR=lib64 -else - PB_LIBDIR=lib + +SYS_LONG_BIT=$(getconf LONG_BIT) + +echo "Installing azcopy" +if [ $SYS_LONG_BIT = "64" ]; then + mkdir -p /tmp/azcopy + aria2c -q -d /tmp/azcopy -o azcopy.tar.gz https://aka.ms/downloadazcopy-v10-linux + tar --strip 1 -xf /tmp/azcopy/azcopy.tar.gz -C /tmp/azcopy + cp /tmp/azcopy/azcopy /usr/bin fi -for build_type in 'Debug' 'Relwithdebinfo'; do - pushd . - mkdir build_$build_type - cd build_$build_type - /opt/cmake/bin/cmake ../cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=$PB_LIBDIR -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$build_type - make -j`nproc` - make install - popd -done + +mkdir -p /tmp/src +aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2.tar.gz +tar -xf /tmp/src/cmake-3.13.2.tar.gz -C /tmp/src +cd /tmp/src/cmake-3.13.2 +./configure --prefix=/usr --parallel=`nproc` --system-curl --system-zlib --system-expat +make -j`nproc` +make install + +#install onnx export ONNX_ML=1 INSTALLED_PYTHON_VERSION=$(python3 -c 'import sys; version=sys.version_info[:2]; print("{0}.{1}".format(*version));') if [ "$INSTALLED_PYTHON_VERSION" = "3.7" ];then @@ -30,13 +28,16 @@ fi if [ "$INSTALLED_PYTHON_VERSION" = "3.4" ];then echo "Python 3.5 and above is needed for running onnx tests!" 1>&2 else - source /tmp/scripts/install_onnx.sh + source /tmp/scripts/install_onnx.sh $INSTALLED_PYTHON_VERSION fi #The last onnx version will be kept - +cd / rm -rf /tmp/src -rm -rf /usr/include/google -rm -rf /usr/lib/libproto* - +DISTRIBUTOR=$(lsb_release -i -s) +if [ "$DISTRIBUTOR" = "Ubuntu" ]; then + apt-get -y remove libprotobuf-dev protobuf-compiler +else + dnf remove -y protobuf-devel protobuf-compiler +fi diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps_android.sh b/tools/ci_build/github/linux/docker/scripts/install_deps_android.sh index feefca233b..771b2ae4f2 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_deps_android.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_deps_android.sh @@ -1,41 +1,19 @@ #!/bin/bash set -e +mkdir -p /tmp/src +aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2.tar.gz +tar -xf /tmp/src/cmake-3.13.2.tar.gz -C /tmp/src +cd /tmp/src/cmake-3.13.2 +./configure --prefix=/usr --parallel=`nproc` --system-curl --system-zlib --system-expat +make -j`nproc` +make install + #download Android NDK r19c aria2c -q -d /tmp https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip unzip -oq /tmp/android-ndk-r19c-linux-x86_64.zip -d /tmp/android-ndk && mv /tmp/android-ndk/* /android-ndk -#install ninja -aria2c -q -d /tmp https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip -unzip -oq /tmp/ninja-linux.zip -d /usr/bin -rm -f /tmp/ninja-linux.zip -#install protobuf -mkdir -p /tmp/src -mkdir -p /opt/cmake -aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2-Linux-x86_64.tar.gz -tar -xf /tmp/src/cmake-3.13.2-Linux-x86_64.tar.gz --strip 1 -C /opt/cmake -aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz -tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src -cd /tmp/src/protobuf-3.6.1 -if [ -f /etc/redhat-release ] ; then - PB_LIBDIR=lib64 -else - PB_LIBDIR=lib -fi -for build_type in 'Debug' 'Relwithdebinfo'; do - pushd . - mkdir build_$build_type - cd build_$build_type - /opt/cmake/bin/cmake -G Ninja ../cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=$PB_LIBDIR -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$build_type - ninja - ninja install - popd -done -export ONNX_ML=1 - -#The last onnx version will be kept - +cd / rm -rf /tmp/src -rm -rf /usr/include/google -rm -rf /usr/lib/libproto* - + +apt-get -y remove libprotobuf-dev protobuf-compiler diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps_x86.sh b/tools/ci_build/github/linux/docker/scripts/install_deps_x86.sh deleted file mode 100755 index 7f27798900..0000000000 --- a/tools/ci_build/github/linux/docker/scripts/install_deps_x86.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -set -e -aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2.tar.gz -tar -xf /tmp/src/cmake-3.13.2.tar.gz -C /tmp/src -cd /tmp/src/cmake-3.13.2 -./configure -make -make install -aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz -tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src -cd /tmp/src/protobuf-3.6.1 -if [ -f /etc/redhat-release ] ; then - PB_LIBDIR=lib64 -else - PB_LIBDIR=lib -fi -for build_type in 'Debug' 'Relwithdebinfo'; do - pushd . - mkdir build_$build_type - cd build_$build_type - cmake ../cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=$PB_LIBDIR -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$build_type - make -j`nproc` - make install - popd -done -export ONNX_ML=1 -INSTALLED_PYTHON_VERSION=$(python3 -c 'import sys; version=sys.version_info[:2]; print("{0}.{1}".format(*version));') -if [ "$INSTALLED_PYTHON_VERSION" = "3.7" ];then - pip3 install --upgrade setuptools -else - source /tmp/scripts/install_onnx.sh x86 -fi - -#The last onnx version will be kept - -rm -rf /tmp/src - - diff --git a/tools/ci_build/github/linux/docker/scripts/install_fedora.sh b/tools/ci_build/github/linux/docker/scripts/install_fedora.sh index c39edb2dc0..30b64d7c41 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_fedora.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_fedora.sh @@ -1,3 +1,3 @@ #!/bin/bash set -e -dnf install -y --best --allowerasing rpmdevtools tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2 +dnf install -y --best --allowerasing redhat-lsb-core expat-devel libcurl-devel protobuf-compiler protobuf-devel protobuf rpmdevtools tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2 diff --git a/tools/ci_build/github/linux/docker/scripts/install_fedora_gpu.sh b/tools/ci_build/github/linux/docker/scripts/install_fedora_gpu.sh index 6ec7704406..80cc3b4a73 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_fedora_gpu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_fedora_gpu.sh @@ -1,11 +1,6 @@ #!/bin/bash set -e -dnf install -y --best --allowerasing compat-openssl10 lttng-ust libcurl openssl-libs krb5-libs libicu tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2 -mkdir -p /tmp/azcopy -aria2c -q -d /tmp/azcopy -o azcopy.tar.gz https://aka.ms/downloadazcopy-v10-linux -tar -xf /tmp/azcopy/azcopy.tar.gz --strip 1 -C /tmp/azcopy -/bin/cp /tmp/azcopy/azcopy /usr/bin/azcopy -rm -rf /tmp/azcopy +dnf install -y --best --allowerasing redhat-lsb-core expat-devel libcurl-devel protobuf-compiler protobuf-devel protobuf compat-openssl10 lttng-ust libcurl openssl-libs krb5-libs libicu tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2 #install cuda 9.2 and cudnn 7 aria2c -q -d /tmp 'http://developer.download.nvidia.com/compute/cuda/repos/fedora27/x86_64/cuda-repo-fedora27-10.0.130-1.x86_64.rpm' dnf install -y /tmp/cuda-repo-fedora27-10.0.130-1.x86_64.rpm diff --git a/tools/ci_build/github/linux/docker/scripts/install_onnx.sh b/tools/ci_build/github/linux/docker/scripts/install_onnx.sh index a767ace27a..3aa773739f 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_onnx.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_onnx.sh @@ -1,4 +1,7 @@ #!/bin/bash +set -e +PYTHON_VER=$1 + #Install ONNX #5af210ca8a1c73aa6bae8754c9346ec54d0a756e is v1.2.3 #bae6333e149a59a3faa9c4d9c44974373dcf5256 is v1.3.0 @@ -16,21 +19,15 @@ for onnx_version in ${!version2tag[@]}; do echo "first pass"; else echo "deleting old onnx-${lastest_onnx_version}"; - pip3 uninstall -y onnx + /usr/bin/python${PYTHON_VER} -m pip uninstall -y onnx fi lastest_onnx_version=$onnx_version aria2c -q -d /tmp/src https://github.com/onnx/onnx/archive/$onnx_version.tar.gz tar -xf /tmp/src/onnx-$onnx_version.tar.gz -C /tmp/src cd /tmp/src/onnx-$onnx_version git clone https://github.com/pybind/pybind11.git third_party/pybind11 - python3 setup.py bdist_wheel - if [ $1 = "x86" ]; then - pip3 install -q onnx - else - pip3 install -q dist/* - fi + /usr/bin/python${PYTHON_VER} -m pip install . mkdir -p /data/onnx/${version2tag[$onnx_version]} backend-test-tools generate-data -o /data/onnx/${version2tag[$onnx_version]} echo $onnx_version":"${version2tag[$onnx_version]} >> /data/onnx/version2tag done - diff --git a/tools/ci_build/github/linux/docker/scripts/install_protobuf.sh b/tools/ci_build/github/linux/docker/scripts/install_protobuf.sh new file mode 100755 index 0000000000..622918b0aa --- /dev/null +++ b/tools/ci_build/github/linux/docker/scripts/install_protobuf.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e +aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz +tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src +cd /tmp/src/protobuf-3.6.1 +cmake ./cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo +make -j`nproc` +make install diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh index b15c67a1c6..acd3bbcea9 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -18,14 +18,14 @@ apt-get update && apt-get install -y --no-install-recommends \ autotools-dev \ automake \ build-essential \ - git apt-transport-https \ + git apt-transport-https apt-utils \ ca-certificates \ pkg-config \ wget \ zlib1g \ zlib1g-dev \ libssl-dev \ - curl \ + curl libcurl4-openssl-dev \ autoconf \ sudo \ gfortran \ @@ -43,13 +43,14 @@ apt-get update && apt-get install -y --no-install-recommends \ bzip2 \ unzip \ zip \ - rsync libunwind8 libpng16-dev \ - python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest + rsync libunwind8 libpng16-dev libexpat1-dev \ + python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \ + libprotobuf-dev libprotobuf9v5 protobuf-compiler locale-gen en_US.UTF-8 update-locale LANG=en_US.UTF-8 -#Install dotnet-sdk +echo "Installing dotnet-sdk" if [ $SYS_LONG_BIT = "64" ]; then OS_VER=`lsb_release -r -s` mkdir -p /tmp/dotnet @@ -60,24 +61,19 @@ if [ $SYS_LONG_BIT = "64" ]; then rm -rf /tmp/dotnet fi -if [ $PYTHON_VER!="3.5" ]; then +if [ $PYTHON_VER != "3.5" ]; then apt-get install -y --no-install-recommends \ python${PYTHON_VER} \ python${PYTHON_VER}-dev update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VER} 1 update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.5 2 update-alternatives --set python3 /usr/bin/python${PYTHON_VER} + #TODO: the old one(/usr/bin/pip3) should be uninstalled first. Because the one will be + #put at /usr/local/. Then there will be two pips. + /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3 fi -/usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall numpy==1.15.0 /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall requests==2.21.0 rm -rf /var/lib/apt/lists/* -#Install azcopy -if [ $SYS_LONG_BIT = "64" ]; then - mkdir -p /tmp/azcopy - aria2c -q -d /tmp/azcopy -o azcopy.tar.gz https://aka.ms/downloadazcopy-v10-linux - tar --strip 1 -xf /tmp/azcopy/azcopy.tar.gz -C /tmp/azcopy - cp /tmp/azcopy/azcopy /usr/bin -fi diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu_for_android.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu_for_android.sh deleted file mode 100755 index 95499794d3..0000000000 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu_for_android.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -e -while getopts p: parameter_Option -do case "${parameter_Option}" -in -p) PYTHON_VER=${OPTARG};; -esac -done - -PYTHON_VER=${PYTHON_VER:=3.5} -DEBIAN_FRONTEND=noninteractive - -apt-get update && apt-get install -y software-properties-common -add-apt-repository ppa:deadsnakes/ppa -apt-get update && apt-get install -y --no-install-recommends \ - autotools-dev \ - build-essential \ - git apt-transport-https \ - ca-certificates \ - pkg-config \ - wget \ - zlib1g \ - zlib1g-dev \ - libssl-dev \ - curl \ - autoconf \ - sudo \ - gfortran \ - language-pack-en \ - libopenblas-dev \ - liblttng-ust0 \ - libcurl3 \ - libssl1.0.0 \ - libkrb5-3 \ - libicu55 \ - aria2 \ - bzip2 \ - unzip \ - zip \ - rsync libunwind8 libpng16-dev - -locale-gen en_US.UTF-8 -update-locale LANG=en_US.UTF-8 - -rm -rf /var/lib/apt/lists/* diff --git a/tools/ci_build/github/linux/run_build.sh b/tools/ci_build/github/linux/run_build.sh index d784ed265a..87ab8c5b87 100755 --- a/tools/ci_build/github/linux/run_build.sh +++ b/tools/ci_build/github/linux/run_build.sh @@ -17,10 +17,10 @@ done if [ $BUILD_OS = "android" ]; then pushd /onnxruntime_src mkdir build-android && cd build-android - /opt/cmake/bin/cmake -DCMAKE_TOOLCHAIN_FILE=/android-ndk/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DONNX_CUSTOM_PROTOC_EXECUTABLE=/usr/bin/protoc ../cmake - /opt/cmake/bin/cmake --build . -- -j$(nproc) + cmake -DCMAKE_TOOLCHAIN_FILE=/android-ndk/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DONNX_CUSTOM_PROTOC_EXECUTABLE=/usr/bin/protoc ../cmake + make -j$(nproc) else - COMMON_BUILD_ARGS="--skip_submodule_sync --enable_onnx_tests --parallel --build_shared_lib --use_openmp" + COMMON_BUILD_ARGS="--skip_submodule_sync --enable_onnx_tests --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest" if [ $BUILD_DEVICE = "gpu" ]; then _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) python3 $SCRIPT_DIR/../../build.py --build_dir /build \ diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index 2bd5e06c16..011b1865b7 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -25,7 +25,7 @@ esac done EXIT_CODE=1 - +PYTHON_VER=${PYTHON_VER:=3.5} echo "bo=$BUILD_OS bd=$BUILD_DEVICE bdir=$BUILD_DIR pv=$PYTHON_VER bex=$BUILD_EXTR_PAR" cd $SCRIPT_DIR/docker